"
-TN_VERSE_NOTES_ENCLOSING_DIV_FMT_STR: str = "
{}
"
-TQ_HEADING_AND_QUESTIONS_FMT_STR: str = (
- "
{}
\n
{}
"
-)
CHAPTER_HEADER_FMT_STR: str = '
Chapter {}
'
@@ -67,7 +64,8 @@ def adjust_commentary_headings(
def chapter_intro(
tn_book: Optional[TNBook],
chapter_num: int,
- hr: str = "
",
+ use_section_visual_separator: bool,
+ hr: str = settings.HR,
) -> str:
"""Get the chapter intro."""
content = []
@@ -77,7 +75,8 @@ def chapter_intro(
and tn_book.chapters[chapter_num].intro_html
):
content.append(tn_book.chapters[chapter_num].intro_html)
- content.append(hr)
+ if use_section_visual_separator:
+ content.append(hr)
return "".join(content)
@@ -87,39 +86,48 @@ def has_footnotes(html_content: str) -> bool:
def bc_book_intro(
bc_book: Optional[BCBook],
- hr: str = "
",
+ use_section_visual_separator: bool,
+ hr: str = settings.HR,
) -> str:
- content = ""
+ content = []
if bc_book and bc_book.book_intro:
- content = f"{bc_book.book_intro}{hr}"
- return content
+ content.append(bc_book.book_intro)
+ if use_section_visual_separator:
+ content.append(hr)
+ return "".join(content)
def tn_book_intro(
tn_book: Optional[TNBook],
- hr: str = "
",
+ use_section_visual_separator: bool,
+ hr: str = settings.HR,
show_tn_book_intro: bool = settings.SHOW_TN_BOOK_INTRO,
) -> str:
- content = ""
+ content = []
if show_tn_book_intro and tn_book and tn_book.book_intro:
- content = f"{tn_book.book_intro}{hr}"
- return content
+ content.append(tn_book.book_intro)
+ if use_section_visual_separator:
+ content.append(hr)
+ return "".join(content)
def chapter_commentary(
bc_book: Optional[BCBook],
chapter_num: int,
- hr: str = "
",
+ use_section_visual_separator: bool,
+ hr: str = settings.HR,
) -> str:
"""Get the chapter commentary."""
- content = ""
+ content = []
if (
bc_book
and chapter_num in bc_book.chapters
and bc_book.chapters[chapter_num].commentary
):
- content = f"{bc_book.chapters[chapter_num].commentary}{hr}"
- return content
+ content.append(bc_book.chapters[chapter_num].commentary)
+ if use_section_visual_separator:
+ content.append(hr)
+ return "".join(content)
def usfm_language_direction_html(
@@ -169,45 +177,62 @@ def rg_language_direction_html(
def tn_chapter_verses(
tn_book: Optional[TNBook],
chapter_num: int,
- fmt_str: str = TN_VERSE_NOTES_ENCLOSING_DIV_FMT_STR,
- hr: str = "
",
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tn_notes: bool,
+ hr: str = settings.HR,
) -> str:
"""
Return the HTML for verses that are in the chapter with
chapter_num.
"""
+ tn_verse_notes_enclosing_div_fmt_str: str = (
+ "
{}
"
+ if use_two_column_layout_for_tn_notes
+ else "
{}
"
+ )
content = []
if tn_book and chapter_num in tn_book.chapters:
tn_verses = tn_book.chapters[chapter_num].verses
- content.append(fmt_str.format("".join(tn_verses.values())))
- content.append(hr)
+ content.append(
+ tn_verse_notes_enclosing_div_fmt_str.format("".join(tn_verses.values()))
+ )
+ if use_section_visual_separator:
+ content.append(hr)
return "".join(content)
def tq_chapter_verses(
tq_book: Optional[TQBook],
chapter_num: int,
- fmt_str: str = TQ_HEADING_AND_QUESTIONS_FMT_STR,
- hr: str = "
",
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tq_notes: bool,
+ hr: str = settings.HR,
) -> str:
"""Return the HTML for verses in chapter_num."""
+ tq_verse_notes_enclosing_div_fmt_str: str = (
+ "
{}
"
+ if use_two_column_layout_for_tq_notes
+ else "
{}
"
+ )
content = []
if tq_book and chapter_num in tq_book.chapters:
tq_verses = tq_book.chapters[chapter_num].verses
content.append(
- fmt_str.format(
- tq_book.resource_type_name,
+ tq_verse_notes_enclosing_div_fmt_str.format(
+ # tq_book.resource_type_name,
"".join(tq_verses.values()),
)
)
- content.append(hr)
+ if use_section_visual_separator:
+ content.append(hr)
return "".join(content)
def rg_chapter_verses(
rg_book: Optional[RGBook],
chapter_num: int,
- hr: str = "
",
+ use_section_visual_separator: bool,
+ hr: str = settings.HR,
) -> str:
"""
Return the HTML for verses that are in the chapter with
@@ -217,7 +242,8 @@ def rg_chapter_verses(
if rg_book and chapter_num in rg_book.chapters:
rg_verses = render_chapter(rg_book.chapters[chapter_num])
content.append(rg_verses)
- content.append(hr)
+ if use_section_visual_separator:
+ content.append(hr)
return "".join(content)
diff --git a/backend/doc/domain/assembly_strategies_docx/assembly_strategies_book_then_lang_by_chapter.py b/backend/doc/domain/assembly_strategies_docx/assembly_strategies_book_then_lang_by_chapter.py
index d5b0d0269..8ea36c09f 100644
--- a/backend/doc/domain/assembly_strategies_docx/assembly_strategies_book_then_lang_by_chapter.py
+++ b/backend/doc/domain/assembly_strategies_docx/assembly_strategies_book_then_lang_by_chapter.py
@@ -10,17 +10,12 @@
tn_chapter_verses,
tq_chapter_verses,
)
-from doc.domain.assembly_strategies_docx.assembly_strategy_utils import (
- add_one_column_section,
- add_page_break,
- add_two_column_section,
- create_docx_subdoc,
-)
-from doc.domain.bible_books import BOOK_CHAPTERS, BOOK_NAMES
+from doc.domain.bible_books import BOOK_CHAPTERS, BOOK_ID_MAP, BOOK_NAMES
from doc.domain.model import (
AssemblyLayoutEnum,
BCBook,
ChunkSizeEnum,
+ DocumentPart,
LangDirEnum,
TNBook,
TQBook,
@@ -28,12 +23,9 @@
USFMBook,
)
from doc.reviewers_guide.model import RGBook
-from docx import Document # type: ignore
-from docxcompose.composer import Composer # type: ignore
-logger = settings.logger(__name__)
-BOOK_NAME_FMT_STR: str = "
{}
"
+logger = settings.logger(__name__)
def assemble_content_by_book_then_lang(
@@ -45,15 +37,18 @@ def assemble_content_by_book_then_lang(
rg_books: Sequence[RGBook],
assembly_layout_kind: AssemblyLayoutEnum,
chunk_size: ChunkSizeEnum,
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tn_notes: bool,
+ use_two_column_layout_for_tq_notes: bool,
book_names: Mapping[str, str] = BOOK_NAMES,
-) -> Composer:
+ book_id_map: dict[str, int] = BOOK_ID_MAP,
+) -> list[DocumentPart]:
"""
- Assemble by book then by language in alphabetic order before
- delegating more atomic ordering/interleaving to an assembly
- sub-strategy.
+ Assemble by book in canonical bible book order then by language in
+ alphabetic order before delegating more atomic ordering/interleaving
+ to an assembly sub-strategy.
"""
- # Sort the books in canonical order so that groupby does what we want.
- book_id_map = dict((id, pos) for pos, id in enumerate(BOOK_NAMES.keys()))
+ document_parts: list[DocumentPart] = []
most_book_codes = max(
[
[usfm_book.book_code for usfm_book in usfm_books],
@@ -88,50 +83,64 @@ def assemble_content_by_book_then_lang(
rg_book for rg_book in rg_books if rg_book.book_code == book_code
]
if selected_usfm_books:
- composer = assemble_usfm_by_chapter(
- usfm_books,
- tn_books,
- tq_books,
- tw_books,
- bc_books,
- rg_books,
+ document_parts.extend(
+ assemble_usfm_by_chapter(
+ selected_usfm_books,
+ selected_tn_books,
+ selected_tq_books,
+ selected_tw_books,
+ selected_bc_books,
+ selected_rg_books,
+ use_section_visual_separator,
+ use_two_column_layout_for_tn_notes,
+ use_two_column_layout_for_tq_notes,
+ )
)
- return composer
elif not selected_usfm_books and selected_tn_books:
- composer = assemble_tn_by_chapter(
- usfm_books,
- tn_books,
- tq_books,
- tw_books,
- bc_books,
- rg_books,
+ document_parts.extend(
+ assemble_tn_by_chapter(
+ selected_usfm_books,
+ selected_tn_books,
+ selected_tq_books,
+ selected_tw_books,
+ selected_bc_books,
+ selected_rg_books,
+ use_section_visual_separator,
+ use_two_column_layout_for_tn_notes,
+ use_two_column_layout_for_tq_notes,
+ )
)
- return composer
elif not selected_usfm_books and not selected_tn_books and selected_tq_books:
- composer = assemble_tq_by_chapter(
- usfm_books,
- tn_books,
- tq_books,
- tw_books,
- bc_books,
- rg_books,
+ document_parts.extend(
+ assemble_tq_by_chapter(
+ selected_usfm_books,
+ selected_tn_books,
+ selected_tq_books,
+ selected_tw_books,
+ selected_bc_books,
+ selected_rg_books,
+ use_section_visual_separator,
+ use_two_column_layout_for_tq_notes,
+ )
)
- return composer
elif (
not selected_usfm_books
and not selected_tn_books
and not selected_tq_books
and (selected_tw_books or selected_bc_books or selected_rg_books)
):
- composer = assemble_tw_by_chapter(
- usfm_books,
- tn_books,
- tq_books,
- tw_books,
- bc_books,
- rg_books,
+ document_parts.extend(
+ assemble_tw_by_chapter(
+ selected_usfm_books,
+ selected_tn_books,
+ selected_tq_books,
+ selected_tw_books,
+ selected_bc_books,
+ selected_rg_books,
+ use_section_visual_separator,
+ )
)
- return composer
+ return document_parts
def assemble_usfm_by_chapter(
@@ -141,103 +150,94 @@ def assemble_usfm_by_chapter(
tw_books: Sequence[TWBook],
bc_books: Sequence[BCBook],
rg_books: Sequence[RGBook],
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tn_notes: bool,
+ use_two_column_layout_for_tq_notes: bool,
book_chapters: Mapping[str, int] = BOOK_CHAPTERS,
show_tn_book_intro: bool = settings.SHOW_TN_BOOK_INTRO,
- fmt_str: str = BOOK_NAME_FMT_STR,
-) -> Composer:
+ fmt_str: str = settings.BOOK_NAME_FMT_STR,
+) -> list[DocumentPart]:
"""
Construct the Docx wherein at least one USFM resource exists, one column
layout.
"""
- def sort_key(resource: USFMBook) -> str:
- return resource.lang_code
- def tn_sort_key(resource: TNBook) -> str:
- return resource.lang_code
- def tq_sort_key(resource: TQBook) -> str:
- return resource.lang_code
- def bc_sort_key(resource: BCBook) -> str:
- return resource.lang_code
- def rg_sort_key(resource: RGBook) -> str:
- return resource.lang_code
- usfm_books = sorted(usfm_books, key=sort_key)
- tn_books = sorted(tn_books, key=tn_sort_key)
- tq_books = sorted(tq_books, key=tq_sort_key)
- bc_books = sorted(bc_books, key=bc_sort_key)
- rg_books = sorted(rg_books, key=rg_sort_key)
- doc = Document()
- composer = Composer(doc)
+ document_parts: list[DocumentPart] = []
if show_tn_book_intro:
for tn_book in tn_books:
if tn_book.book_intro:
book_intro_ = tn_book.book_intro
book_intro_adj = adjust_book_intro_headings(book_intro_)
- subdoc = create_docx_subdoc(
- book_intro_adj,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=book_intro_adj,
+ is_rtl=tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for bc_book in bc_books:
- # Add the commentary book intro
- subdoc = create_docx_subdoc(bc_book.book_intro, bc_book.lang_code)
- composer.append(subdoc)
+ document_parts.append(
+ DocumentPart(
+ content=bc_book.book_intro,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
book_codes = {usfm_book.book_code for usfm_book in usfm_books}
for book_code in book_codes:
num_chapters = book_chapters[book_code]
for chapter_num in range(1, num_chapters + 1):
- add_one_column_section(doc)
- # Add chapter intro for each language
for tn_book in [
tn_book for tn_book in tn_books if tn_book.book_code == book_code
]:
if chapter_num in tn_book.chapters:
- subdoc = create_docx_subdoc(
- chapter_intro(tn_book, chapter_num),
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=chapter_intro(
+ tn_book, chapter_num, use_section_visual_separator
+ ),
+ is_rtl=tn_book
+ and tn_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for bc_book in [
bc_book for bc_book in bc_books if bc_book.book_code == book_code
]:
if chapter_num in bc_book.chapters:
- # Add the chapter commentary.
- subdoc = create_docx_subdoc(
- chapter_commentary(bc_book, chapter_num),
- bc_book.lang_code,
+ document_parts.append(
+ DocumentPart(
+ content=chapter_commentary(
+ bc_book, chapter_num, use_section_visual_separator
+ ),
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for usfm_book in [
usfm_book
for usfm_book in usfm_books
if usfm_book.book_code == book_code
]:
- add_one_column_section(doc)
- # Add the book title, e.g., 1 Peter
- subdoc = create_docx_subdoc(
- fmt_str.format(usfm_book.national_book_name),
- usfm_book.lang_code,
+ document_parts.append(
+ DocumentPart(
+ content=fmt_str.format(usfm_book.national_book_name),
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
if chapter_num in usfm_book.chapters:
- # Add the interleaved USFM chapters
- add_one_column_section(doc)
- # fmt: off
- is_rtl = usfm_book and usfm_book.lang_direction == LangDirEnum.RTL
- # fmt: on
- subdoc = create_docx_subdoc(
- usfm_book.chapters[chapter_num].content,
- usfm_book.lang_code,
- is_rtl,
+ document_parts.append(
+ DocumentPart(
+ content=usfm_book.chapters[chapter_num].content,
+ is_rtl=usfm_book
+ and usfm_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- # Add the interleaved tn notes
tn_verses = None
for tn_book in [
tn_book
@@ -245,45 +245,81 @@ def rg_sort_key(resource: RGBook) -> str:
if tn_book.book_code == usfm_book.book_code
]:
if chapter_num in tn_book.chapters:
- tn_verses = tn_chapter_verses(tn_book, chapter_num)
+ tn_verses = tn_chapter_verses(
+ tn_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tn_notes,
+ )
if tn_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tn_verses,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tn_verses,
+ is_rtl=tn_book
+ and tn_book.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=use_two_column_layout_for_tn_notes,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- # Add the interleaved tq questions
for tq_book in [
tq_book
for tq_book in tq_books
if tq_book.book_code == usfm_book.book_code
]:
if chapter_num in tq_book.chapters:
- tq_verses = tq_chapter_verses(tq_book, chapter_num)
- # Add TQ verse content, if any
+ tq_verses = tq_chapter_verses(
+ tq_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tq_notes,
+ )
if tq_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tq_verses,
- tq_book.lang_code,
- tq_book and tq_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tq_verses,
+ is_rtl=tq_book
+ and tq_book.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=use_two_column_layout_for_tq_notes,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for rg_book in [
rg_book
for rg_book in rg_books
if rg_book.book_code == usfm_book.book_code
]:
if chapter_num in rg_book.chapters:
- subdoc = create_docx_subdoc(
- rg_chapter_verses(rg_book, chapter_num),
- rg_book.lang_code,
+ document_parts.append(
+ DocumentPart(
+ content=rg_chapter_verses(
+ rg_book, chapter_num, use_section_visual_separator
+ ),
+ use_section_visual_separator=use_section_visual_separator,
+ ),
)
- composer.append(subdoc)
- add_page_break(doc)
- return composer
+ document_parts.append(
+ DocumentPart(
+ content="",
+ add_hr_p=False,
+ add_page_break=True,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ return document_parts
def assemble_tn_by_chapter(
@@ -293,125 +329,157 @@ def assemble_tn_by_chapter(
tw_books: Sequence[TWBook],
bc_books: Sequence[BCBook],
rg_books: Sequence[RGBook],
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tn_notes: bool,
+ use_two_column_layout_for_tq_notes: bool,
book_chapters: Mapping[str, int] = BOOK_CHAPTERS,
show_tn_book_intro: bool = settings.SHOW_TN_BOOK_INTRO,
-) -> Composer:
+) -> list[DocumentPart]:
"""
Construct the HTML for a 'by chapter' strategy wherein at least
tn_book_content_units exists.
"""
- def tn_sort_key(resource: TNBook) -> str:
- return resource.lang_code
- def tq_sort_key(resource: TQBook) -> str:
- return resource.lang_code
- def bc_sort_key(resource: BCBook) -> str:
- return resource.lang_code
- def rg_sort_key(resource: RGBook) -> str:
- return resource.lang_code
- tn_books = sorted(tn_books, key=tn_sort_key)
- tq_books = sorted(tq_books, key=tq_sort_key)
- bc_books = sorted(bc_books, key=bc_sort_key)
- rg_books = sorted(rg_books, key=rg_sort_key)
- doc = Document()
- composer = Composer(doc)
- add_one_column_section(doc)
+ document_parts: list[DocumentPart] = []
if show_tn_book_intro:
# Add book intros for each tn_book
for tn_book in tn_books:
if tn_book.book_intro:
book_intro_ = tn_book.book_intro
book_intro_adj = adjust_book_intro_headings(book_intro_)
- subdoc = create_docx_subdoc(
- book_intro_adj,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=book_intro_adj,
+ is_rtl=tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for bc_book in bc_books:
- subdoc = create_docx_subdoc(
- bc_book_intro(bc_book),
- bc_book.lang_code,
+ document_parts.append(
+ DocumentPart(
+ content=bc_book_intro(bc_book, use_section_visual_separator),
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
book_codes = {tn_book.book_code for tn_book in tn_books}
for book_code in book_codes:
num_chapters = book_chapters[book_code]
for chapter_num in range(1, num_chapters + 1):
- add_one_column_section(doc)
for tn_book in [
tn_book for tn_book in tn_books if tn_book.book_code == book_code
]:
one_column_html = []
- # one_column_html.append("Chapter {}".format(chapter_num))
if chapter_num in tn_book.chapters:
- # Add the translation notes chapter intro.
- one_column_html.append(chapter_intro(tn_book, chapter_num))
+ one_column_html.append(
+ chapter_intro(
+ tn_book, chapter_num, use_section_visual_separator
+ )
+ )
one_column_html_ = "".join(one_column_html)
if one_column_html_:
- subdoc = create_docx_subdoc(
- one_column_html_,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=one_column_html_,
+ is_rtl=tn_book
+ and tn_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for bc_book in [
bc_book for bc_book in bc_books if bc_book.book_code == book_code
]:
if chapter_num in bc_book.chapters:
- # Add the chapter commentary.
- subdoc = create_docx_subdoc(
- chapter_commentary(bc_book, chapter_num),
- bc_book.lang_code,
+ document_parts.append(
+ DocumentPart(
+ content=chapter_commentary(
+ bc_book, chapter_num, use_section_visual_separator
+ ),
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- # Add the interleaved tn notes
for tn_book in [
tn_book for tn_book in tn_books if tn_book.book_code == book_code
]:
if chapter_num in tn_book.chapters:
- tn_verses = tn_chapter_verses(tn_book, chapter_num)
+ tn_verses = tn_chapter_verses(
+ tn_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tn_notes,
+ )
if tn_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tn_verses,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tn_verses,
+ is_rtl=tn_book
+ and tn_book.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=use_two_column_layout_for_tn_notes,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- # Add the interleaved tq questions
for tq_book in [
tq_book for tq_book in tq_books if tq_book.book_code == book_code
]:
- tq_verses = tq_chapter_verses(tq_book, chapter_num)
- # Add TQ verse content, if any
+ tq_verses = tq_chapter_verses(
+ tq_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tq_notes,
+ )
if tq_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tq_verses,
- tq_book.lang_code,
- tq_book and tq_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tq_verses,
+ is_rtl=tq_book
+ and tq_book.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=use_two_column_layout_for_tq_notes,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for rg_book in [
rg_book for rg_book in rg_books if rg_book.book_code == book_code
]:
- rg_verses = rg_chapter_verses(rg_book, chapter_num)
+ rg_verses = rg_chapter_verses(
+ rg_book, chapter_num, use_section_visual_separator
+ )
if rg_verses:
- add_one_column_section(doc)
- # add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- rg_verses,
- rg_book.lang_code,
- rg_book and rg_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=rg_verses,
+ is_rtl=rg_book
+ and rg_book.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=True,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- add_page_break(doc)
- return composer
+ document_parts.append(
+ DocumentPart(
+ content="",
+ add_hr_p=False,
+ add_page_break=True,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ return document_parts
def assemble_tq_by_chapter(
@@ -421,27 +489,19 @@ def assemble_tq_by_chapter(
tw_books: Sequence[TWBook],
bc_books: Sequence[BCBook],
rg_books: Sequence[RGBook],
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tq_notes: bool,
book_chapters: Mapping[str, int] = BOOK_CHAPTERS,
-) -> Composer:
+) -> list[DocumentPart]:
"""
Construct the HTML for a 'by chapter' strategy wherein at least
tq_book_content_units exists.
"""
- def tq_sort_key(resource: TQBook) -> str:
- return resource.lang_code
- def bc_sort_key(resource: BCBook) -> str:
- return resource.lang_code
- def rg_sort_key(resource: RGBook) -> str:
- return resource.lang_code
- tq_books = sorted(tq_books, key=tq_sort_key)
- bc_books = sorted(bc_books, key=bc_sort_key)
- rg_books = sorted(rg_books, key=rg_sort_key)
- doc = Document()
- composer = Composer(doc)
+ document_parts: list[DocumentPart] = []
book_codes = {tq_book.book_code for tq_book in tq_books}
for book_code in book_codes:
num_chapters = book_chapters[book_code]
@@ -451,42 +511,61 @@ def rg_sort_key(resource: RGBook) -> str:
for bc_book in [
bc_book for bc_book in bc_books if bc_book.book_code == book_code
]:
- one_column_html.append(chapter_commentary(bc_book, chapter_num))
+ one_column_html.append(
+ chapter_commentary(
+ bc_book, chapter_num, use_section_visual_separator
+ )
+ )
if one_column_html:
- add_one_column_section(doc)
- subdoc = create_docx_subdoc("".join(one_column_html), bc_book.lang_code)
- composer.append(subdoc)
- # Add the interleaved tq questions
+ document_parts.append(DocumentPart(content="".join(one_column_html)))
for tq_book in [
tq_book
for tq_book in tq_books
if tq_book.book_code == tq_book.book_code
]:
- tq_verses = tq_chapter_verses(tq_book, chapter_num)
+ tq_verses = tq_chapter_verses(
+ tq_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tq_notes,
+ )
if tq_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tq_verses,
- tq_book.lang_code,
- tq_book and tq_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tq_verses,
+ is_rtl=tq_book
+ and tq_book.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=use_two_column_layout_for_tq_notes,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for rg_book in [
rg_book
for rg_book in rg_books
if rg_book.book_code == rg_book.book_code
]:
- rg_verses = rg_chapter_verses(rg_book, chapter_num)
+ rg_verses = rg_chapter_verses(
+ rg_book, chapter_num, use_section_visual_separator
+ )
if rg_verses:
- add_one_column_section(doc)
- subdoc = create_docx_subdoc(
- rg_verses,
- rg_book.lang_code,
- rg_book and rg_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=rg_verses,
+ is_rtl=rg_book
+ and rg_book.lang_direction == LangDirEnum.RTL,
+ )
)
- composer.append(subdoc)
- add_page_break(doc)
- return composer
+ document_parts.append(
+ DocumentPart(content="", add_hr_p=False, add_page_break=True)
+ )
+ return document_parts
# This function could be a little confusing for newcomers. TW lives at
@@ -503,24 +582,33 @@ def assemble_tw_by_chapter(
tw_books: Sequence[TWBook],
bc_books: Sequence[BCBook],
rg_books: Sequence[RGBook],
-) -> Composer:
+ use_section_visual_separator: bool,
+) -> list[DocumentPart]:
"""Construct the HTML for BC and TW."""
- doc = Document()
- composer = Composer(doc)
+ document_parts: list[DocumentPart] = []
- def bc_sort_key(resource: BCBook) -> str:
- return resource.lang_code
- def rg_sort_key(resource: RGBook) -> str:
- return resource.lang_code
- bc_books = sorted(bc_books, key=bc_sort_key)
- rg_books = sorted(rg_books, key=rg_sort_key)
for bc_book in bc_books:
- subdoc = create_docx_subdoc(bc_book.book_intro, bc_book.lang_code)
- composer.append(subdoc)
+ document_parts.append(
+ DocumentPart(
+ content=bc_book.book_intro,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
for chapter in bc_book.chapters.values():
- subdoc = create_docx_subdoc(chapter.commentary, bc_book.lang_code)
- composer.append(subdoc)
- add_page_break(doc)
- return composer
+ document_parts.append(
+ DocumentPart(
+ content=chapter.commentary,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ add_hr_p=False,
+ add_page_break=True,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ return document_parts
diff --git a/backend/doc/domain/assembly_strategies_docx/assembly_strategies_lang_then_book_by_chapter.py b/backend/doc/domain/assembly_strategies_docx/assembly_strategies_lang_then_book_by_chapter.py
index 992821193..54ac74b4c 100755
--- a/backend/doc/domain/assembly_strategies_docx/assembly_strategies_lang_then_book_by_chapter.py
+++ b/backend/doc/domain/assembly_strategies_docx/assembly_strategies_lang_then_book_by_chapter.py
@@ -2,26 +2,19 @@
from doc.config import settings
from doc.domain.assembly_strategies.assembly_strategy_utils import (
- chapter_commentary,
chapter_heading,
+ chapter_commentary,
chapter_intro,
rg_chapter_verses,
tn_chapter_verses,
tq_chapter_verses,
)
-from doc.domain.assembly_strategies_docx.assembly_strategy_utils import (
- add_hr,
- create_docx_subdoc,
- add_one_column_section,
- add_two_column_section,
- add_page_break,
-)
-
-from doc.domain.bible_books import BOOK_NAMES
+from doc.domain.bible_books import BOOK_ID_MAP, BOOK_NAMES
from doc.domain.model import (
AssemblyLayoutEnum,
BCBook,
ChunkSizeEnum,
+ DocumentPart,
LangDirEnum,
TNBook,
TQBook,
@@ -29,12 +22,9 @@
USFMBook,
)
from doc.reviewers_guide.model import RGBook
-from docx import Document # type: ignore
-from docxcompose.composer import Composer # type: ignore
-logger = settings.logger(__name__)
-BOOK_NAME_FMT_STR: str = "
{}
"
+logger = settings.logger(__name__)
def assemble_content_by_lang_then_book(
@@ -46,16 +36,19 @@ def assemble_content_by_lang_then_book(
rg_books: Sequence[RGBook],
assembly_layout_kind: AssemblyLayoutEnum,
chunk_size: ChunkSizeEnum,
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tn_notes: bool,
+ use_two_column_layout_for_tq_notes: bool,
book_names: Mapping[str, str] = BOOK_NAMES,
-) -> Composer:
+ book_id_map: dict[str, int] = BOOK_ID_MAP,
+) -> list[DocumentPart]:
"""
Group content by language and then by book and then pass content
and a couple other parameters, assembly_layout_kind and
chunk_size, to interleaving strategy to do the actual
interleaving.
"""
- composers: list[Composer] = []
- book_id_map = dict((id, pos) for pos, id in enumerate(BOOK_NAMES.keys()))
+ document_parts: list[DocumentPart] = []
all_lang_codes = (
{usfm_book.lang_code for usfm_book in usfm_books}
.union(tn_book.lang_code for tn_book in tn_books)
@@ -75,7 +68,6 @@ def assemble_content_by_lang_then_book(
.union(rg_book.book_code for rg_book in rg_books)
)
most_book_codes = list(all_book_codes)
- # Cache book_id_map lookup
book_codes_sorted = sorted(
most_book_codes, key=lambda book_code: book_id_map[book_code]
)
@@ -127,7 +119,7 @@ def assemble_content_by_lang_then_book(
]
rg_book = selected_rg_books[0] if selected_rg_books else None
if usfm_book is not None:
- composers.append(
+ document_parts.extend(
assemble_usfm_by_book(
usfm_book,
tn_book,
@@ -136,10 +128,13 @@ def assemble_content_by_lang_then_book(
usfm_book2,
bc_book,
rg_book,
+ use_section_visual_separator,
+ use_two_column_layout_for_tn_notes,
+ use_two_column_layout_for_tq_notes,
)
)
elif usfm_book is None and tn_book is not None:
- composers.append(
+ document_parts.extend(
assemble_tn_by_book(
usfm_book,
tn_book,
@@ -148,10 +143,13 @@ def assemble_content_by_lang_then_book(
usfm_book2,
bc_book,
rg_book,
+ use_section_visual_separator,
+ use_two_column_layout_for_tn_notes,
+ use_two_column_layout_for_tq_notes,
)
)
elif usfm_book is None and tn_book is None and tq_book is not None:
- composers.append(
+ document_parts.extend(
assemble_tq_by_book(
usfm_book,
tn_book,
@@ -160,6 +158,8 @@ def assemble_content_by_lang_then_book(
usfm_book2,
bc_book,
rg_book,
+ use_section_visual_separator,
+ use_two_column_layout_for_tq_notes,
)
)
elif (
@@ -168,7 +168,7 @@ def assemble_content_by_lang_then_book(
and tq_book is None
and (tw_book is not None or bc_book is not None or rg_book is not None)
):
- composers.append(
+ document_parts.extend(
assemble_tw_by_book(
usfm_book,
tn_book,
@@ -177,12 +177,10 @@ def assemble_content_by_lang_then_book(
usfm_book2,
bc_book,
rg_book,
+ use_section_visual_separator,
)
)
- first_composer = composers[0]
- for composer in composers[1:]:
- first_composer.append(composer.doc)
- return first_composer
+ return document_parts
def assemble_usfm_by_book(
@@ -193,124 +191,164 @@ def assemble_usfm_by_book(
usfm_book2: Optional[USFMBook],
bc_book: Optional[BCBook],
rg_book: Optional[RGBook],
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tn_notes: bool,
+ use_two_column_layout_for_tq_notes: bool,
show_tn_book_intro: bool = settings.SHOW_TN_BOOK_INTRO,
- fmt_str: str = BOOK_NAME_FMT_STR,
-) -> Composer:
+ fmt_str: str = settings.BOOK_NAME_FMT_STR,
+) -> list[DocumentPart]:
"""
Construct the HTML for a 'by book' strategy wherein at least
usfm_book_content_unit exists.
"""
- doc = Document()
- composer = Composer(doc)
+ document_parts: list[DocumentPart] = []
if show_tn_book_intro and tn_book and tn_book.book_intro:
- subdoc = create_docx_subdoc(
- tn_book.book_intro,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tn_book.book_intro,
+ is_rtl=tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
if bc_book:
if bc_book.book_intro:
- subdoc = create_docx_subdoc(
- bc_book.book_intro,
- bc_book.lang_code,
+ document_parts.append(
+ DocumentPart(
+ content=bc_book.book_intro,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
if usfm_book:
- # fmt: off
is_rtl = usfm_book and usfm_book.lang_direction == LangDirEnum.RTL
- # fmt: on
# Add book name
- subdoc = create_docx_subdoc(
- fmt_str.format(usfm_book.national_book_name),
- usfm_book.lang_code,
- is_rtl,
- False,
+ document_parts.append(
+ DocumentPart(
+ content=fmt_str.format(usfm_book.national_book_name),
+ is_rtl=is_rtl,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
for (
chapter_num,
chapter,
) in usfm_book.chapters.items():
- add_one_column_section(doc)
tn_verses: str = ""
tq_verses: str = ""
rg_verses: str = ""
chapter_intro_ = ""
chapter_commentary_ = ""
- if tn_book:
- chapter_intro_ = chapter_intro(tn_book, chapter_num)
- tn_verses = tn_chapter_verses(tn_book, chapter_num)
- if bc_book:
- chapter_commentary_ = chapter_commentary(bc_book, chapter_num)
- if tq_book:
- tq_verses = tq_chapter_verses(tq_book, chapter_num)
- if rg_book:
- rg_verses = rg_chapter_verses(rg_book, chapter_num)
- subdoc = create_docx_subdoc(
- chapter.content,
- usfm_book.lang_code,
- is_rtl,
+ chapter_intro_ = chapter_intro(
+ tn_book, chapter_num, use_section_visual_separator
+ )
+ tn_verses = tn_chapter_verses(
+ tn_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tn_notes,
+ )
+ chapter_commentary_ = chapter_commentary(
+ bc_book, chapter_num, use_section_visual_separator
+ )
+ tq_verses = tq_chapter_verses(
+ tq_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tq_notes,
+ )
+ rg_verses = rg_chapter_verses(
+ rg_book, chapter_num, use_section_visual_separator
+ )
+ document_parts.append(
+ DocumentPart(
+ content=chapter.content,
+ is_rtl=is_rtl,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
if chapter_intro_:
- subdoc = create_docx_subdoc(chapter_intro_, usfm_book.lang_code, is_rtl)
- composer.append(subdoc)
+ document_parts.append(
+ DocumentPart(
+ content=chapter_intro_,
+ is_rtl=is_rtl,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
if chapter_commentary_:
- subdoc = create_docx_subdoc(
- chapter_commentary_, usfm_book.lang_code, is_rtl, False
+ document_parts.append(
+ DocumentPart(
+ content=chapter_commentary_,
+ is_rtl=is_rtl,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
if tn_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tn_verses,
- usfm_book.lang_code,
- is_rtl,
- False,
+ document_parts.append(
+ DocumentPart(
+ content=tn_verses,
+ is_rtl=is_rtl,
+ add_hr_p=False,
+ contained_in_two_column_section=use_two_column_layout_for_tn_notes,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- add_one_column_section(doc)
- p = doc.add_paragraph()
- add_hr(p)
if tq_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tq_verses,
- usfm_book.lang_code,
- is_rtl,
- False,
+ document_parts.append(
+ DocumentPart(
+ content=tq_verses,
+ is_rtl=is_rtl,
+ add_hr_p=False,
+ contained_in_two_column_section=use_two_column_layout_for_tq_notes,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- add_one_column_section(doc)
- p = doc.add_paragraph()
- add_hr(p)
if rg_verses:
- # add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- rg_verses,
- usfm_book.lang_code,
- is_rtl,
- False,
+ document_parts.append(
+ DocumentPart(
+ content=rg_verses,
+ is_rtl=is_rtl,
+ add_hr_p=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- add_one_column_section(doc)
# TODO Get feedback on whether we should allow a user to select a primary _and_
# a secondary USFM resource. If we want to limit the user to only one USFM per
# document then we would want to control that in the UI and maybe also at the API
# level. The API level control would be implemented in the DocumentRequest
# validation.
if usfm_book2:
- add_one_column_section(doc)
# Here we add the whole chapter's worth of verses for the secondary usfm
- subdoc = create_docx_subdoc(
- usfm_book2.chapters[chapter_num].content,
- usfm_book.lang_code,
- usfm_book2 and usfm_book2.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=usfm_book2.chapters[chapter_num].content,
+ is_rtl=usfm_book2
+ and usfm_book2.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=False,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ add_hr_p=False,
+ add_page_break=True,
+ use_section_visual_separator=use_section_visual_separator,
)
- composer.append(subdoc)
- add_page_break(doc)
- return composer
+ )
+ return document_parts
def assemble_tn_by_book(
@@ -321,86 +359,110 @@ def assemble_tn_by_book(
usfm_book2: Optional[USFMBook],
bc_book: Optional[BCBook],
rg_book: Optional[RGBook],
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tn_notes: bool,
+ use_two_column_layout_for_tq_notes: bool,
show_tn_book_intro: bool = settings.SHOW_TN_BOOK_INTRO,
-) -> Composer:
+) -> list[DocumentPart]:
"""
Construct the HTML for a 'by book' strategy wherein at least
tn_book exists.
"""
- doc = Document()
- composer = Composer(doc)
+ document_parts: list[DocumentPart] = []
if tn_book:
if show_tn_book_intro and tn_book.book_intro:
- subdoc = create_docx_subdoc(
- tn_book.book_intro,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tn_book.book_intro,
+ is_rtl=tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
if bc_book and bc_book.book_intro:
- subdoc = create_docx_subdoc(
- bc_book.book_intro,
- tn_book.lang_code,
- )
- composer.append(subdoc)
+ document_parts.append(DocumentPart(content=bc_book.book_intro))
for chapter_num in tn_book.chapters:
- add_one_column_section(doc)
one_column_html = []
one_column_html.append(chapter_heading(chapter_num))
- one_column_html.append(chapter_intro(tn_book, chapter_num))
+ one_column_html.append(
+ chapter_intro(tn_book, chapter_num, use_section_visual_separator)
+ )
one_column_html_ = "".join(one_column_html)
if one_column_html_:
- subdoc = create_docx_subdoc(
- one_column_html_,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=one_column_html_,
+ is_rtl=tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
if bc_book:
- subdoc = create_docx_subdoc(
- chapter_commentary(bc_book, chapter_num),
- bc_book.lang_code,
+ document_parts.append(
+ DocumentPart(
+ content=chapter_commentary(
+ bc_book, chapter_num, use_section_visual_separator
+ ),
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- tn_verses = tn_chapter_verses(tn_book, chapter_num)
+ tn_verses = tn_chapter_verses(
+ tn_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tn_notes,
+ )
if tn_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tn_verses,
- tn_book.lang_code,
- tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tn_verses,
+ is_rtl=tn_book and tn_book.lang_direction == LangDirEnum.RTL,
+ add_hr_p=False,
+ contained_in_two_column_section=use_two_column_layout_for_tn_notes,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- add_one_column_section(doc)
- p = doc.add_paragraph()
- add_hr(p)
- tq_verses = tq_chapter_verses(tq_book, chapter_num)
+ document_parts.append(DocumentPart(content=""))
+ tq_verses = tq_chapter_verses(
+ tq_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tq_notes,
+ )
if tq_book and tq_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tq_verses,
- tq_book.lang_code,
- tq_book and tq_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tq_verses,
+ is_rtl=tq_book and tq_book.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=use_two_column_layout_for_tq_notes,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- add_one_column_section(doc)
- p = doc.add_paragraph()
- add_hr(p)
- rg_verses = rg_chapter_verses(rg_book, chapter_num)
+ document_parts.append(DocumentPart(content=""))
+ rg_verses = rg_chapter_verses(
+ rg_book, chapter_num, use_section_visual_separator
+ )
if rg_book and rg_verses:
- # add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- rg_verses,
- rg_book.lang_code,
- rg_book and rg_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=rg_verses,
+ is_rtl=rg_book and rg_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- # add_one_column_section(doc)
- p = doc.add_paragraph()
- add_hr(p)
- add_page_break(doc)
-
- return composer
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ add_hr_p=False,
+ add_page_break=True,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ return document_parts
def assemble_tq_by_book(
@@ -411,48 +473,67 @@ def assemble_tq_by_book(
usfm_book2: Optional[USFMBook],
bc_book: Optional[BCBook],
rg_book: Optional[RGBook],
-) -> Composer:
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tq_notes: bool,
+) -> list[DocumentPart]:
"""
Construct the HTML for a 'by book' strategy wherein at least
tq_book exists.
"""
- doc = Document()
- composer = Composer(doc)
+ document_parts: list[DocumentPart] = []
if tq_book:
for chapter_num in tq_book.chapters:
- add_one_column_section(doc)
if bc_book:
- subdoc = create_docx_subdoc(
- chapter_commentary(bc_book, chapter_num),
- bc_book.lang_code,
+ document_parts.append(
+ DocumentPart(
+ content=chapter_commentary(
+ bc_book, chapter_num, use_section_visual_separator
+ ),
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content=chapter_heading(chapter_num),
+ is_rtl=tq_book and tq_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
)
- composer.append(subdoc)
- subdoc = create_docx_subdoc(
- chapter_heading(chapter_num),
- tq_book.lang_code,
- tq_book and tq_book.lang_direction == LangDirEnum.RTL,
)
- composer.append(subdoc)
- tq_verses = tq_chapter_verses(tq_book, chapter_num)
+ tq_verses = tq_chapter_verses(
+ tq_book,
+ chapter_num,
+ use_section_visual_separator,
+ use_two_column_layout_for_tq_notes,
+ )
if tq_verses:
- add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- tq_verses,
- tq_book.lang_code,
- tq_book and tq_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=tq_verses,
+ is_rtl=tq_book and tq_book.lang_direction == LangDirEnum.RTL,
+ contained_in_two_column_section=use_two_column_layout_for_tq_notes,
+ use_section_visual_separator=use_section_visual_separator,
+ )
)
- composer.append(subdoc)
- rg_verses = rg_chapter_verses(rg_book, chapter_num)
+ rg_verses = rg_chapter_verses(
+ rg_book, chapter_num, use_section_visual_separator
+ )
if rg_book and rg_verses:
- # add_two_column_section(doc)
- subdoc = create_docx_subdoc(
- rg_verses,
- rg_book.lang_code,
- rg_book and rg_book.lang_direction == LangDirEnum.RTL,
+ document_parts.append(
+ DocumentPart(
+ content=rg_verses,
+ is_rtl=rg_book and rg_book.lang_direction == LangDirEnum.RTL,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ add_hr_p=False,
+ add_page_break=True,
+ use_section_visual_separator=use_section_visual_separator,
)
- composer.append(subdoc)
- add_page_break(doc)
- return composer
+ )
+ return document_parts
def assemble_tw_by_book(
@@ -463,18 +544,28 @@ def assemble_tw_by_book(
usfm_book2: Optional[USFMBook],
bc_book: Optional[BCBook],
rg_book: Optional[RGBook],
-) -> Composer:
+ use_section_visual_separator: bool,
+) -> list[DocumentPart]:
"""
TW is handled outside this module, that is why no
code for TW is explicitly included here.
"""
- doc = Document()
- composer = Composer(doc)
+ document_parts: list[DocumentPart] = []
if bc_book:
- subdoc = create_docx_subdoc(bc_book.book_intro, bc_book.lang_code)
- composer.append(subdoc)
+ document_parts.append(DocumentPart(content=bc_book.book_intro))
for chapter in bc_book.chapters.values():
- subdoc = create_docx_subdoc(chapter.commentary, bc_book.lang_code)
- composer.append(subdoc)
- add_page_break(doc)
- return composer
+ document_parts.append(
+ DocumentPart(
+ content=chapter.commentary,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ add_hr_p=False,
+ add_page_break=True,
+ use_section_visual_separator=use_section_visual_separator,
+ )
+ )
+ return document_parts
diff --git a/backend/doc/domain/assembly_strategies_docx/assembly_strategy_utils.py b/backend/doc/domain/assembly_strategies_docx/assembly_strategy_utils.py
index 913f70e12..4fa0498af 100644
--- a/backend/doc/domain/assembly_strategies_docx/assembly_strategy_utils.py
+++ b/backend/doc/domain/assembly_strategies_docx/assembly_strategy_utils.py
@@ -2,14 +2,19 @@
Utility functions used by assembly_strategies.
"""
+from typing import Optional
+
from doc.config import settings
+from doc.domain.model import BCBook, TNBook, TQBook
+from doc.reviewers_guide.model import RGBook
+from doc.reviewers_guide.render_to_html import render_chapter
from docx import Document # type: ignore
from docx.enum.section import WD_SECTION # type: ignore
from docx.enum.text import WD_BREAK # type: ignore
from docx.oxml.ns import qn # type: ignore
from docx.oxml.shared import OxmlElement # type: ignore
from docx.text.paragraph import Paragraph # type: ignore
-from htmldocx import HtmlToDocx # type: ignore
+
logger = settings.logger(__name__)
@@ -66,6 +71,118 @@
]
+def tn_book_intro(
+ tn_book: Optional[TNBook],
+ show_tn_book_intro: bool = settings.SHOW_TN_BOOK_INTRO,
+) -> str:
+ content = ""
+ if show_tn_book_intro and tn_book and tn_book.book_intro:
+ content = tn_book.book_intro
+ return content
+
+
+def bc_book_intro(
+ bc_book: Optional[BCBook],
+) -> str:
+ content = ""
+ if bc_book and bc_book.book_intro:
+ content = bc_book.book_intro
+ return content
+
+
+def chapter_commentary(
+ bc_book: Optional[BCBook],
+ chapter_num: int,
+) -> str:
+ """Get the chapter commentary."""
+ content = ""
+ if (
+ bc_book
+ and chapter_num in bc_book.chapters
+ and bc_book.chapters[chapter_num].commentary
+ ):
+ content = bc_book.chapters[chapter_num].commentary
+ return content
+
+
+def chapter_intro(
+ tn_book: Optional[TNBook],
+ chapter_num: int,
+) -> str:
+ """Get the chapter intro."""
+ content = []
+ if (
+ tn_book
+ and chapter_num in tn_book.chapters
+ and tn_book.chapters[chapter_num].intro_html
+ ):
+ content.append(tn_book.chapters[chapter_num].intro_html)
+ return "".join(content)
+
+
+def tn_chapter_verses(
+ tn_book: Optional[TNBook],
+ chapter_num: int,
+ use_two_column_layout_for_tn_notes: bool,
+ # fmt_str: str = TN_VERSE_NOTES_ENCLOSING_DIV_FMT_STR,
+) -> str:
+ """
+ Return the HTML for verses that are in the chapter with
+ chapter_num.
+ """
+ tn_verse_notes_enclosing_div_fmt_str: str = (
+ "
{}
"
+ if use_two_column_layout_for_tn_notes
+ else "
{}
"
+ )
+ content = []
+ if tn_book and chapter_num in tn_book.chapters:
+ tn_verses = tn_book.chapters[chapter_num].verses
+ content.append(
+ tn_verse_notes_enclosing_div_fmt_str.format("".join(tn_verses.values()))
+ )
+ return "".join(content)
+
+
+def tq_chapter_verses(
+ tq_book: Optional[TQBook],
+ chapter_num: int,
+ use_two_column_layout_for_tq_notes: bool,
+ # fmt_str: str = TQ_HEADING_AND_QUESTIONS_FMT_STR,
+) -> str:
+ """Return the HTML for verses in chapter_num."""
+ tq_verse_notes_enclosing_div_fmt_str: str = (
+ "
{}
"
+ if use_two_column_layout_for_tq_notes
+ else "
{}
"
+ )
+ content = []
+ if tq_book and chapter_num in tq_book.chapters:
+ tq_verses = tq_book.chapters[chapter_num].verses
+ content.append(
+ tq_verse_notes_enclosing_div_fmt_str.format(
+ tq_book.resource_type_name,
+ "".join(tq_verses.values()),
+ )
+ )
+ return "".join(content)
+
+
+def rg_chapter_verses(
+ rg_book: Optional[RGBook],
+ chapter_num: int,
+) -> str:
+ """
+ Return the HTML for verses that are in the chapter with
+ chapter_num.
+ """
+ content = []
+ if rg_book and chapter_num in rg_book.chapters:
+ rg_verses = render_chapter(rg_book.chapters[chapter_num])
+ content.append(rg_verses)
+ return "".join(content)
+
+
def add_hr(paragraph: Paragraph) -> None:
"""Add a horizontal line at the end of the given paragraph."""
p = paragraph._p # p is the
XML element
@@ -109,26 +226,15 @@ def add_hr(paragraph: Paragraph) -> None:
pBdr.append(bottom)
-def create_docx_subdoc(
- content: str,
+def set_docx_language(
+ doc: Document,
lang_code: str,
- is_rtl: bool = False,
- add_hr_p: bool = True,
oxml_language_list_lowercase: list[str] = OXML_LANGUAGE_LIST_LOWERCASE,
oxml_language_list_lowercase_split: list[str] = OXML_LANGUAGE_LIST_LOWERCASE_SPLIT,
) -> Document:
- """
- Create and return a Document instance from the content parameter.
- """
- html_to_docx = HtmlToDocx()
- subdoc = html_to_docx.parse_html_string(content)
- if is_rtl:
- # Setting each run to be RTL language direction
- for p in subdoc.paragraphs:
- for run in p.runs:
- run.font.rtl = True
- if subdoc.paragraphs:
- p = subdoc.paragraphs[-1]
+ """Set the Language for spell check"""
+ if doc.paragraphs:
+ p = doc.paragraphs[-1]
# Set the language for this paragraph for the sake of the Word
# spellchecker.
p_run = p.add_run()
@@ -175,10 +281,6 @@ def create_docx_subdoc(
p_run_lang.set(qn("w:eastAsia"), "en-US")
p_run_lang.set(qn("w:bidi"), "en-US")
p_rpr.append(p_run_lang)
- # Add a horizontal ruler at the end of the paragraph if requested.
- if add_hr_p:
- add_hr(p)
- return subdoc
def add_one_column_section(doc: Document) -> None:
diff --git a/backend/doc/domain/bible_books.py b/backend/doc/domain/bible_books.py
index c336020d9..e1f71709a 100644
--- a/backend/doc/domain/bible_books.py
+++ b/backend/doc/domain/bible_books.py
@@ -74,6 +74,10 @@
"rev": "Revelation",
}
+
+# Establish a map by which books can be sorted in bible book order
+BOOK_ID_MAP = dict((id, pos) for pos, id in enumerate(BOOK_NAMES.keys()))
+
BOOK_NUMBERS: Mapping[str, str] = {
"gen": "01",
"exo": "02",
diff --git a/backend/doc/domain/document_generator.py b/backend/doc/domain/document_generator.py
index 8066d7608..679dd9775 100755
--- a/backend/doc/domain/document_generator.py
+++ b/backend/doc/domain/document_generator.py
@@ -7,7 +7,7 @@
import time
from datetime import datetime
from os.path import exists, join
-from typing import Any, Optional, Sequence, cast
+from typing import Optional, Sequence, cast
from celery import current_task
from doc.config import settings
@@ -20,12 +20,15 @@
)
from doc.domain.assembly_strategies_docx import (
assembly_strategies_book_then_lang_by_chapter as book_then_lang,
-)
-from doc.domain.assembly_strategies_docx import (
assembly_strategies_lang_then_book_by_chapter as lang_then_book,
)
-from doc.domain.assembly_strategies_docx.assembly_strategy_utils import add_hr
-from doc.domain.bible_books import BOOK_NAMES
+from doc.domain.assembly_strategies_docx.assembly_strategy_utils import (
+ add_hr,
+ add_one_column_section,
+ add_page_break,
+ add_two_column_section,
+)
+from doc.domain.bible_books import BOOK_ID_MAP, BOOK_NAMES
from doc.domain.email_utils import send_email_with_attachment, should_send_email
from doc.domain.model import (
AssemblyLayoutEnum,
@@ -33,6 +36,7 @@
Attachment,
BCBook,
ChunkSizeEnum,
+ DocumentPart,
DocumentRequest,
DocumentRequestSourceEnum,
ResourceLookupDto,
@@ -58,15 +62,100 @@
filter_unique_by_lang_code,
translation_words_section,
)
+from docx import Document # type: ignore
from docx.enum.section import WD_SECTION # type: ignore
from docxcompose.composer import Composer # type: ignore
from docxtpl import DocxTemplate # type: ignore
from htmldocx import HtmlToDocx # type: ignore
-from pydantic import Json
+
logger = settings.logger(__name__)
+def initialize_document_request_and_key(
+ document_request_json: str,
+) -> tuple[DocumentRequest, str]:
+ document_request = DocumentRequest.parse_raw(document_request_json)
+ logger.info("document_request: %s", document_request)
+ document_request.assembly_layout_kind = select_assembly_layout_kind(
+ document_request
+ )
+ # Generate the document request key that identifies this and
+ # identical document requests.
+ document_request_key_ = document_request_key(
+ document_request.resource_requests,
+ document_request.assembly_strategy_kind,
+ document_request.assembly_layout_kind,
+ document_request.chunk_size,
+ document_request.limit_words,
+ document_request.use_chapter_labels,
+ document_request.use_section_visual_separator,
+ document_request.use_two_column_layout_for_tn_notes,
+ document_request.use_two_column_layout_for_tq_notes,
+ )
+ return document_request, document_request_key_
+
+
+def locate_acquire_and_build_resource_objects(
+ document_request: DocumentRequest,
+) -> tuple[
+ Sequence[ResourceLookupDto],
+ Sequence[USFMBook],
+ Sequence[TNBook],
+ Sequence[TQBook],
+ Sequence[TWBook],
+ Sequence[BCBook],
+ Sequence[RGBook],
+]:
+ current_task.update_state(state="Locating assets")
+ resource_lookup_dtos = []
+ for resource_request in document_request.resource_requests:
+ resource_lookup_dto = resource_lookup.resource_lookup_dto(
+ resource_request.lang_code,
+ resource_request.resource_type,
+ resource_request.book_code,
+ )
+ if resource_lookup_dto:
+ resource_lookup_dtos.append(resource_lookup_dto)
+ found_resource_lookup_dtos = [
+ resource_lookup_dto
+ for resource_lookup_dto in resource_lookup_dtos
+ if resource_lookup_dto.url is not None
+ ]
+ current_task.update_state(state="Provisioning asset files")
+ t0 = time.time()
+ resource_dirs = [
+ resource_lookup.prepare_resource_filepath(dto)
+ for dto in found_resource_lookup_dtos
+ ]
+ for resource_dir, dto in zip(resource_dirs, found_resource_lookup_dtos):
+ resource_lookup.provision_asset_files(dto.url, resource_dir)
+ t1 = time.time()
+ logger.info(
+ "Time to provision asset files (acquire and write to disk): %s", t1 - t0
+ )
+ current_task.update_state(state="Parsing asset files")
+ t0 = time.time()
+ usfm_books, tn_books, tq_books, tw_books, bc_books, rg_books = parsing.books(
+ found_resource_lookup_dtos,
+ resource_dirs,
+ document_request.resource_requests,
+ document_request.layout_for_print,
+ document_request.use_chapter_labels,
+ )
+ t1 = time.time()
+ logger.info("Time to parse all resource content: %s", t1 - t0)
+ return (
+ found_resource_lookup_dtos,
+ usfm_books,
+ tn_books,
+ tq_books,
+ tw_books,
+ bc_books,
+ rg_books,
+ )
+
+
# @worker.app.task(
# autoretry_for=(Exception,),
# retry_backoff=True,
@@ -74,7 +163,8 @@
# )
@worker.app.task
def generate_document(
- document_request_json: str, output_dir: str = settings.DOCUMENT_OUTPUT_DIR
+ document_request_json: str,
+ output_dir: str = settings.DOCUMENT_OUTPUT_DIR,
) -> str:
"""
This is the main entry point for this module for non-docx generation.
@@ -82,75 +172,23 @@ def generate_document(
>>> document_request_json = '{"email_address":null,"assembly_strategy_kind":"lbo","assembly_layout_kind":"1c","layout_for_print":false,"resource_requests":[{"lang_code":"es-419","resource_type":"ulb","book_code":"mat"}],"generate_pdf":true,"generate_epub":false,"generate_docx":false,"chunk_size":"chapter","limit_words":false,"include_tn_book_intros":false,"document_request_source":"ui"}'
>>> document_generator.generate_document(document_request_json)
"""
- logger.info("document_request_json: %s", document_request_json)
current_task.update_state(state="Receiving request")
- document_request = DocumentRequest.parse_raw(document_request_json)
- document_request.assembly_layout_kind = select_assembly_layout_kind(
- document_request
- )
- # Generate the document request key that identifies this and
- # identical document requests.
- document_request_key_ = document_request_key(
- document_request.resource_requests,
- document_request.assembly_strategy_kind,
- document_request.assembly_layout_kind,
- document_request.chunk_size,
- document_request.limit_words,
- document_request.use_chapter_labels,
+ document_request, document_request_key_ = initialize_document_request_and_key(
+ document_request_json
)
html_filepath_ = html_filepath(document_request_key_)
pdf_filepath_ = pdf_filepath(document_request_key_)
epub_filepath_ = epub_filepath(document_request_key_)
if file_needs_update(html_filepath_):
- # Update the state of the worker process. This is used by the
- # UI to report status.
- current_task.update_state(state="Locating assets")
- # HTML didn't exist in cache so go ahead and start by getting the
- # resource lookup DTOs for each resource request in the document
- # request.
- resource_lookup_dtos = []
- for resource_request in document_request.resource_requests:
- resource_lookup_dto = resource_lookup.resource_lookup_dto(
- resource_request.lang_code,
- resource_request.resource_type,
- resource_request.book_code,
- )
- if resource_lookup_dto:
- resource_lookup_dtos.append(resource_lookup_dto)
- # Determine which resource URLs were actually found.
- found_resource_lookup_dtos = [
- resource_lookup_dto
- for resource_lookup_dto in resource_lookup_dtos
- if resource_lookup_dto.url is not None
- ]
- # if not found_resource_lookup_dtos:
- # raise exceptions.ResourceAssetFileNotFoundError(
- # message="No supported resource assets were found"
- # )
- current_task.update_state(state="Provisioning asset files")
- t0 = time.time()
- resource_dirs = [
- resource_lookup.prepare_resource_filepath(dto)
- for dto in found_resource_lookup_dtos
- ]
- for resource_dir, dto in zip(resource_dirs, found_resource_lookup_dtos):
- resource_lookup.provision_asset_files(dto.url, resource_dir)
- t1 = time.time()
- logger.info(
- "Time to provision asset files (acquire and write to disk): %s", t1 - t0
- )
- current_task.update_state(state="Parsing asset files")
- # Initialize found resources from their provisioned assets.
- t0 = time.time()
- usfm_books, tn_books, tq_books, tw_books, bc_books, rg_books = parsing.books(
+ (
found_resource_lookup_dtos,
- resource_dirs,
- document_request.resource_requests,
- document_request.layout_for_print,
- document_request.use_chapter_labels,
- )
- t1 = time.time()
- logger.info("Time to parse all resource content: %s", t1 - t0)
+ usfm_books,
+ tn_books,
+ tq_books,
+ tw_books,
+ bc_books,
+ rg_books,
+ ) = locate_acquire_and_build_resource_objects(document_request)
current_task.update_state(state="Assembling content")
content = assemble_content(
document_request_key_,
@@ -163,19 +201,25 @@ def generate_document(
rg_books,
found_resource_lookup_dtos,
)
+ content_str = "".join(content)
if usfm_books:
- content = check_content_for_issues(content)
- content = create_title_page_and_wrap_in_template(
- content, document_request, found_resource_lookup_dtos, usfm_books
+ content_str = check_content_for_issues(content_str)
+ content_str = create_title_page_and_wrap_in_template(
+ content_str, document_request, found_resource_lookup_dtos, usfm_books
)
- write_html_content_to_file(content, html_filepath_)
+ write_html_content_to_file(content_str, html_filepath_)
else:
logger.info("Cache hit for %s", html_filepath_)
# Immediately return pre-built PDF if the document has previously been
# generated and is fresh enough.
if document_request.generate_pdf and file_needs_update(pdf_filepath_):
current_task.update_state(state="Converting to PDF")
- convert_html_to_pdf(html_filepath_, pdf_filepath_, document_request_key_)
+ convert_html_to_pdf(
+ html_filepath_,
+ pdf_filepath_,
+ document_request_key_,
+ document_request.use_prince,
+ )
if should_send_email(document_request.email_address):
attachments = [
Attachment(filepath=pdf_filepath_, mime_type=("application", "pdf"))
@@ -206,80 +250,30 @@ def generate_document(
@worker.app.task
def generate_docx_document(
- document_request_json: Json[Any],
+ document_request_json: str,
output_dir: str = settings.DOCUMENT_OUTPUT_DIR,
-) -> Json[str]:
+) -> str:
"""
This is the alternative entry point for Docx document creation only.
"""
- document_request = DocumentRequest.parse_raw(document_request_json)
- logger.info(
- "document_request: %s",
- document_request,
- )
- document_request.assembly_layout_kind = select_assembly_layout_kind(
- document_request
- )
- # Generate the document request key that identifies this and
- # identical document requests.
- document_request_key_ = document_request_key(
- document_request.resource_requests,
- document_request.assembly_strategy_kind,
- document_request.assembly_layout_kind,
- document_request.chunk_size,
- document_request.limit_words,
- document_request.use_chapter_labels,
+ current_task.update_state(state="Receiving request")
+ document_request, document_request_key_ = initialize_document_request_and_key(
+ document_request_json
)
html_filepath_ = html_filepath(document_request_key_)
docx_filepath_ = docx_filepath(document_request_key_)
if document_request.generate_docx and file_needs_update(docx_filepath_):
- # Update the state of the worker process. This is used by the
- # UI to report status.
- current_task.update_state(state="Locating assets")
- # Docx didn't exist in cache so go ahead and start by getting the
- # resource lookup DTOs for each resource request in the document
- # request.
- resource_lookup_dtos = []
- for resource_request in document_request.resource_requests:
- resource_lookup_dto = resource_lookup.resource_lookup_dto(
- resource_request.lang_code,
- resource_request.resource_type,
- resource_request.book_code,
- )
- if resource_lookup_dto:
- resource_lookup_dtos.append(resource_lookup_dto)
- # Determine which resource URLs were actually found.
- found_resource_lookup_dtos = [
- resource_lookup_dto
- for resource_lookup_dto in resource_lookup_dtos
- if resource_lookup_dto.url is not None
- ]
- current_task.update_state(state="Provisioning asset files")
- t0 = time.time()
- resource_dirs = [
- resource_lookup.prepare_resource_filepath(dto)
- for dto in found_resource_lookup_dtos
- ]
- for resource_dir, dto in zip(resource_dirs, found_resource_lookup_dtos):
- resource_lookup.provision_asset_files(dto.url, resource_dir)
- t1 = time.time()
- logger.info(
- "Time to provision asset files (acquire and write to disk): %s", t1 - t0
- )
- current_task.update_state(state="Parsing asset files")
- # Initialize found resources from their provisioned assets.
- t0 = time.time()
- usfm_books, tn_books, tq_books, tw_books, bc_books, rg_books = parsing.books(
+ (
found_resource_lookup_dtos,
- resource_dirs,
- document_request.resource_requests,
- document_request.layout_for_print,
- document_request.use_chapter_labels,
- )
- t1 = time.time()
- logger.info("Time to parse all resource content: %s", t1 - t0)
+ usfm_books,
+ tn_books,
+ tq_books,
+ tw_books,
+ bc_books,
+ rg_books,
+ ) = locate_acquire_and_build_resource_objects(document_request)
current_task.update_state(state="Assembling content")
- composer = assemble_docx_content(
+ document_parts = assemble_docx_content(
document_request_key_,
document_request,
usfm_books,
@@ -289,13 +283,6 @@ def generate_docx_document(
bc_books,
rg_books,
)
- # TODO At this point, like in generate_document, we should check the
- # underlying HTML content to see if it contains verses and display a
- # message in the document to the end user if it does not (so that they
- # get some indication of why the scripture is missing).
- #
- # Construct sensical phrases to display for title1 and title2 on first
- # page of Word document.
title1, title2 = get_languages_title_page_strings(
found_resource_lookup_dtos, usfm_books
)
@@ -303,8 +290,9 @@ def generate_docx_document(
convert_html_to_docx(
html_filepath_,
docx_filepath_,
- composer,
+ document_parts,
document_request.layout_for_print,
+ document_request.use_section_visual_separator,
title1,
title2,
)
@@ -336,6 +324,9 @@ def document_request_key(
chunk_size: ChunkSizeEnum,
limit_words: bool,
use_chapter_labels: bool,
+ use_section_visual_separator: bool,
+ use_two_column_layout_for_tn_notes: bool,
+ use_two_column_layout_for_tq_notes: bool,
max_filename_len: int = 240,
underscore: str = "_",
hyphen: str = "-",
@@ -368,13 +359,13 @@ def document_request_key(
]
)
if any(contains_tw(resource_request) for resource_request in resource_requests):
- document_request_key = f'{resource_request_keys}_{assembly_strategy_kind.value}_{assembly_layout_kind.value}_{chunk_size.value}_{"clt" if use_chapter_labels else "clf"}_{"lwt" if limit_words else "lwf"}'
+ document_request_key = f'{resource_request_keys}_{assembly_strategy_kind.value}_{assembly_layout_kind.value}_{chunk_size.value}_{"clt" if use_chapter_labels else "clf"}_{"lwt" if limit_words else "lwf"}_{"sst" if use_section_visual_separator else "ssf"}_{"2ctn" if use_two_column_layout_for_tn_notes else "1ctn"}_{"2ctq" if use_two_column_layout_for_tq_notes else "1ctq"}'
else:
- document_request_key = f"{resource_request_keys}_{assembly_strategy_kind.value}_{assembly_layout_kind.value}_{chunk_size.value}_{"clt" if use_chapter_labels else "clf"}"
+ document_request_key = f'{resource_request_keys}_{assembly_strategy_kind.value}_{assembly_layout_kind.value}_{chunk_size.value}_{"clt" if use_chapter_labels else "clf"}_{"sst" if use_section_visual_separator else "ssf"}_{"2ctn" if use_two_column_layout_for_tn_notes else "1ctn"}_{"2ctq" if use_two_column_layout_for_tq_notes else "1ctq"}'
if len(document_request_key) >= max_filename_len:
- # Likely the generated filename was too long for the OS where this is
- # running. In that case, use the current time as a document_request_key
- # value as doing so results in an acceptably short length.
+ # The generated filename could be too long for the OS where this is
+ # running. Therefore, use the current time as a document_request_key
+ # so that filename is not too long.
timestamp_components = str(time.time()).split(".")
return f"{timestamp_components[0]}_{timestamp_components[1]}"
else:
@@ -421,7 +412,6 @@ def document_html_header(
if generate_docx:
template = env.get_template("html/header_no_css_enclosing.html")
return template.render()
-
if assembly_layout_kind and assembly_layout_kind in [
AssemblyLayoutEnum.ONE_COLUMN_COMPACT,
AssemblyLayoutEnum.TWO_COLUMN_SCRIPTURE_LEFT_SCRIPTURE_RIGHT_COMPACT,
@@ -444,18 +434,20 @@ def assemble_content(
bc_books: Sequence[BCBook],
rg_books: Sequence[RGBook],
found_resource_lookup_dtos: Sequence[ResourceLookupDto],
-) -> str:
+ hr: str = "
",
+) -> list[str]:
"""
Assemble and return the content from all requested resources according to the
assembly_strategy requested.
+
"""
t0 = time.time()
- content = ""
+ content = []
if (
document_request.assembly_strategy_kind
== AssemblyStrategyEnum.LANGUAGE_BOOK_ORDER
):
- content = "".join(
+ content.extend(
assemble_content_by_lang_then_book(
usfm_books,
tn_books,
@@ -464,13 +456,16 @@ def assemble_content(
bc_books,
rg_books,
cast(AssemblyLayoutEnum, document_request.assembly_layout_kind),
+ document_request.use_section_visual_separator,
+ document_request.use_two_column_layout_for_tn_notes,
+ document_request.use_two_column_layout_for_tq_notes,
)
)
elif (
document_request.assembly_strategy_kind
== AssemblyStrategyEnum.BOOK_LANGUAGE_ORDER
):
- content = "".join(
+ content.extend(
assemble_content_by_book_then_lang(
usfm_books,
tn_books,
@@ -479,17 +474,27 @@ def assemble_content(
bc_books,
rg_books,
cast(AssemblyLayoutEnum, document_request.assembly_layout_kind),
+ document_request.use_section_visual_separator,
+ document_request.use_two_column_layout_for_tn_notes,
+ document_request.use_two_column_layout_for_tq_notes,
)
)
t1 = time.time()
logger.info("Time for interleaving document: %s", t1 - t0)
t0 = time.time()
# Add the translation words definition section for each language requested.
- unique_lang_codes = set()
- for tw_book in tw_books:
- if tw_book.lang_code not in unique_lang_codes:
- unique_lang_codes.add(tw_book.lang_code)
- content = f"{content}{translation_words_section(tw_book, usfm_books, document_request.limit_words, document_request.resource_requests)}
"
+ unique_tw_books = filter_unique_by_lang_code(tw_books)
+ for tw_book in unique_tw_books:
+ content.extend(
+ translation_words_section(
+ tw_book,
+ usfm_books,
+ document_request.limit_words,
+ document_request.resource_requests,
+ )
+ )
+ if document_request.use_section_visual_separator:
+ content.append(hr)
t1 = time.time()
logger.info("Time for add TW content to document: %s", t1 - t0)
return content
@@ -504,7 +509,8 @@ def create_title_page_and_wrap_in_template(
title1, title2 = get_languages_title_page_strings(
found_resource_lookup_dtos, usfm_books
)
- title3 = "Formatted for Translators"
+ logger.debug("title1: %s, title2: %s", title1, title2)
+ title3 = ""
header = document_html_header(
document_request.assembly_layout_kind,
document_request.generate_docx,
@@ -525,18 +531,18 @@ def assemble_docx_content(
tw_books: Sequence[TWBook],
bc_books: Sequence[BCBook],
rg_books: Sequence[RGBook],
-) -> Composer:
+) -> list[DocumentPart]:
"""
Assemble and return the content from all requested resources according to the
assembly_strategy requested.
"""
t0 = time.time()
- composer = None
+ document_parts: list[DocumentPart] = []
if (
document_request.assembly_strategy_kind
== AssemblyStrategyEnum.LANGUAGE_BOOK_ORDER
):
- composer = lang_then_book.assemble_content_by_lang_then_book(
+ document_parts = lang_then_book.assemble_content_by_lang_then_book(
usfm_books,
tn_books,
tq_books,
@@ -545,12 +551,15 @@ def assemble_docx_content(
rg_books,
cast(AssemblyLayoutEnum, document_request.assembly_layout_kind),
document_request.chunk_size,
+ document_request.use_section_visual_separator,
+ document_request.use_two_column_layout_for_tn_notes,
+ document_request.use_two_column_layout_for_tq_notes,
)
elif (
document_request.assembly_strategy_kind
== AssemblyStrategyEnum.BOOK_LANGUAGE_ORDER
):
- composer = book_then_lang.assemble_content_by_book_then_lang(
+ document_parts = book_then_lang.assemble_content_by_book_then_lang(
usfm_books,
tn_books,
tq_books,
@@ -559,42 +568,47 @@ def assemble_docx_content(
rg_books,
cast(AssemblyLayoutEnum, document_request.assembly_layout_kind),
document_request.chunk_size,
+ document_request.use_section_visual_separator,
+ document_request.use_two_column_layout_for_tn_notes,
+ document_request.use_two_column_layout_for_tq_notes,
)
t1 = time.time()
logger.info("Time for interleaving document: %s", t1 - t0)
- tw_subdocs = []
if tw_books:
- html_to_docx = HtmlToDocx()
t0 = time.time()
# Add the translation words definition section for each language requested.
unique_tw_books = filter_unique_by_lang_code(tw_books)
for tw_book in unique_tw_books:
- tw_subdoc = html_to_docx.parse_html_string(
- translation_words_section(
- tw_book,
- usfm_books,
- document_request.limit_words,
- document_request.resource_requests,
+ document_parts.append(
+ DocumentPart(
+ content=translation_words_section(
+ tw_book,
+ usfm_books,
+ document_request.limit_words,
+ document_request.resource_requests,
+ ),
+ use_section_visual_separator=document_request.use_section_visual_separator,
+ )
+ )
+ document_parts.append(
+ DocumentPart(
+ content="",
+ use_section_visual_separator=document_request.use_section_visual_separator,
)
)
- if tw_subdoc.paragraphs:
- p = tw_subdoc.paragraphs[-1]
- add_hr(p)
- tw_subdocs.append(tw_subdoc)
t1 = time.time()
logger.info("Time for adding TW content to document: %s", t1 - t0)
- # Now add any TW subdocs to the composer
- if composer:
- for tw_subdoc_ in tw_subdocs:
- composer.append(tw_subdoc_)
- return composer
+ return document_parts
# HTML to PDF converters:
-# princexml ($$$$) (fastest); also available through docraptor api ($$) but slow,
+# princexml ($$$$ or non-commercial with watermark) (fastest); we use
+# with non-commercial license (watermark on 1st page of pdf). Handles
+# layout flawlessly and is dramatically faster than weasyprint and
+# all other solutions.
+# weasyprint (does a nice job, we also use this),
# wkhtmltopdf via pdfkit (can't handle column-count directive so can't use due to
# multi-column layouts requirement),
-# weasyprint (does a nice job, we use this),
# pagedjs-cli (does a really nice job, but is really slow - uses puppeteer underneath),
# electron-pdf (similar speed to wkhtmltopdf) which uses chrome underneath the hood,
# gotenburg which uses chrome under the hood and provides a nice api in Docker (untested),
@@ -604,6 +618,9 @@ def convert_html_to_pdf(
html_filepath: str,
pdf_filepath: str,
document_request_key: str,
+ use_prince: bool,
+ default_converter: str = "weasyprint",
+ alternative_converter: str = "prince",
) -> None:
"""
Generate PDF from HTML and copy it to output directory.
@@ -611,17 +628,15 @@ def convert_html_to_pdf(
assert exists(html_filepath)
logger.info("Generating PDF %s...", pdf_filepath)
t0 = time.time()
- # command = [
- # "ebook-convert",
- # html_filepath,
- # pdf_filepath,
- # "--disable-font-rescaling",
- # ]
- command = [
- "weasyprint",
- html_filepath,
- pdf_filepath,
- ]
+ if use_prince:
+ command = [
+ alternative_converter,
+ html_filepath,
+ "-o",
+ pdf_filepath,
+ ]
+ else:
+ command = [default_converter, html_filepath, pdf_filepath]
logger.info("Generate PDF command: %s", " ".join(command))
subprocess.run(
command,
@@ -656,26 +671,57 @@ def convert_html_to_epub(
logger.info("Time for converting HTML to ePub: %s", t1 - t0)
+def compose_docx_document(
+ document_parts: list[DocumentPart], use_section_visual_separator: bool
+) -> Document:
+ doc = Document()
+ html_to_docx = HtmlToDocx()
+ t0 = time.time()
+ for part in document_parts:
+ if part.contained_in_two_column_section:
+ add_two_column_section(doc)
+ try:
+ html_to_docx.add_html_to_document(part.content, doc)
+ except ValueError as e:
+ logger.exception(e)
+ else:
+ add_one_column_section(doc)
+ try:
+ html_to_docx.add_html_to_document(part.content, doc)
+ except ValueError as e:
+ logger.exception(e)
+ # Set the language for spellcheck
+ # set_docx_language(doc, lang_code)
+ if use_section_visual_separator and part.add_hr_p:
+ add_hr(doc.paragraphs[-1])
+ if part.add_page_break:
+ add_page_break(doc)
+ t1 = time.time()
+ logger.info("Time for converting HTML to Docx: %s", t1 - t0)
+ return doc
+
+
def convert_html_to_docx(
html_filepath: str,
docx_filepath: str,
- composer: Composer,
+ document_parts: list[DocumentPart],
layout_for_print: bool,
+ use_section_visual_separator: bool,
title1: str = "title1",
title2: str = "title2",
- title3: str = "Formatted for Translators",
+ title3: str = "",
docx_template_path: str = settings.DOCX_TEMPLATE_PATH,
docx_compact_template_path: str = settings.DOCX_COMPACT_TEMPLATE_PATH,
) -> None:
- """Generate Docx and copy it to output directory."""
+ """Generate Docx and write it to output directory."""
t0 = time.time()
# Get data for front page of Docx template.
title1 = title1
title2 = title2
title3 = title3
- # fmt: off
- template_path = docx_compact_template_path if layout_for_print else docx_template_path
- # fmt: on
+ template_path = (
+ docx_compact_template_path if layout_for_print else docx_template_path
+ )
doc = DocxTemplate(template_path)
toc_path = generate_docx_toc(docx_filepath)
toc = doc.new_subdoc(toc_path)
@@ -690,8 +736,7 @@ def convert_html_to_docx(
new_section = doc.add_section(WD_SECTION.CONTINUOUS)
new_section.start_type
master = Composer(doc)
- # Add the main (non-front-matter) content.
- master.append(composer.doc)
+ master.append(compose_docx_document(document_parts, use_section_visual_separator))
master.save(docx_filepath)
t1 = time.time()
logger.info("Time for converting HTML to Docx: %s", t1 - t0)
@@ -769,7 +814,6 @@ def write_html_content_to_file(
Write HTML content to file.
"""
logger.info("About to write HTML to %s", output_filename)
- # Write the HTML file to disk.
write_file(
output_filename,
content,
@@ -806,61 +850,40 @@ def check_content_for_issues(
def get_languages_title_page_strings(
resource_lookup_dtos: Sequence[ResourceLookupDto],
usfm_books: Sequence[USFMBook],
+ book_names: dict[str, str] = BOOK_NAMES,
+ book_id_map: dict[str, int] = BOOK_ID_MAP,
) -> tuple[str, str]:
- lang_codes = list(
- {resource_lookup_dto.lang_code for resource_lookup_dto in resource_lookup_dtos}
- )
- lang0_book_names = set()
- lang0_resource_type_names = set()
- lang1_book_names = set()
- lang1_resource_type_names = set()
- lang0_title, lang1_title = "", ""
- if usfm_books:
- lang0_books = [
- usfm_book
- for usfm_book in usfm_books
- if usfm_book.lang_code == lang_codes[0]
- ]
- for book in lang0_books:
- if book.national_book_name:
- lang0_book_names.add(book.national_book_name)
- lang0_resource_type_names.add(book.resource_type_name)
- if lang0_books:
- lang0_title = f"{lang0_books[0].lang_name}: {', '.join(sorted(lang0_resource_type_names))} for {', '.join(sorted(lang0_book_names))}"
- else:
- language0_resource_lookup_dtos = [
- resource_lookup_dto
- for resource_lookup_dto in resource_lookup_dtos
- if resource_lookup_dto.lang_code == lang_codes[0]
- ]
- for dto in language0_resource_lookup_dtos:
- lang0_book_names.add(BOOK_NAMES[dto.book_code])
- lang0_resource_type_names.add(dto.resource_type_name)
- if language0_resource_lookup_dtos:
- lang0_title = f"{language0_resource_lookup_dtos[0].lang_name}: {', '.join(sorted(lang0_resource_type_names))} for {', '.join(sorted(lang0_book_names))}"
- if len(lang_codes) > 1:
- lang1_books = [
- usfm_book
- for usfm_book in usfm_books
- if usfm_book.lang_code == lang_codes[1]
- ]
- for book in lang1_books:
- if book.national_book_name:
- lang1_book_names.add(book.national_book_name)
- lang1_resource_type_names.add(book.resource_type_name)
- if lang1_books:
- lang1_title = f"{lang1_books[0].lang_name}: {', '.join(sorted(lang1_resource_type_names))} for {', '.join(sorted(lang1_book_names))}"
- else:
- language1_resource_lookup_dtos = [
- resource_lookup_dto
- for resource_lookup_dto in resource_lookup_dtos
- if resource_lookup_dto.lang_code == lang_codes[1]
+ """
+ Construct sensical phrases to display for title1 and title2 for
+ first page of Word document.
+ """
+ lang_codes = list(dict.fromkeys(dto.lang_code for dto in resource_lookup_dtos))
+
+ def get_language_details(lang_code: str) -> str:
+ book_names_ = []
+ resource_type_names = []
+ dtos = [dto for dto in resource_lookup_dtos if dto.lang_code == lang_code]
+ for dto in dtos:
+ usfm_books_ = [
+ usfm_book
+ for usfm_book in usfm_books
+ if usfm_book.book_code == dto.book_code
+ and usfm_book.lang_code == lang_code
]
- for dto in language1_resource_lookup_dtos:
- lang1_book_names.add(BOOK_NAMES[dto.book_code])
- lang1_resource_type_names.add(dto.resource_type_name)
- if language1_resource_lookup_dtos:
- lang1_title = f"{language1_resource_lookup_dtos[0].lang_name}: {', '.join(sorted(lang1_resource_type_names))} for {', '.join(sorted(lang1_book_names))}"
+ if usfm_books_:
+ book_name = usfm_books_[0].national_book_name
+ else:
+ book_name = book_names[dto.book_code]
+ if book_name not in book_names_:
+ book_names_.append(book_name)
+ if dto.resource_type_name not in resource_type_names:
+ resource_type_names.append(dto.resource_type_name)
+ if dtos:
+ return f"{dtos[0].lang_name} ({dtos[0].localized_lang_name}): {', '.join(resource_type_names)} for {', '.join(book_names_)}"
+ return ""
+
+ lang0_title = get_language_details(lang_codes[0]) if lang_codes else ""
+ lang1_title = get_language_details(lang_codes[1]) if len(lang_codes) > 1 else ""
return lang0_title, lang1_title
diff --git a/backend/doc/domain/model.py b/backend/doc/domain/model.py
index c0adbce0d..99688e3a9 100644
--- a/backend/doc/domain/model.py
+++ b/backend/doc/domain/model.py
@@ -6,12 +6,12 @@
"""
from enum import Enum
-from typing import Any, NamedTuple, Optional, Sequence, TypedDict, final
+from typing import NamedTuple, Optional, Sequence, TypedDict, final
from doc.config import settings
from doc.domain.bible_books import BOOK_NAMES
from doc.utils.number_utils import is_even
-from pydantic import BaseModel, EmailStr
+from pydantic import BaseModel, EmailStr, HttpUrl
from pydantic.functional_validators import model_validator
# These type aliases give us more self-documenting code, but of course
@@ -201,6 +201,26 @@ class DocumentRequest(BaseModel):
# is True, then the chapter label will be localized to the language(s)
# requested.
use_chapter_labels: bool = False
+ # We have two PDF processors that we can use: Weasyprint and Princexml.
+ # Weasyprint is an open source project and produces decent PDFs, but is
+ # slow and doesn't always obey the HTML and CSS formatting exactly,
+ # e.g., sometimes text is flowed around chapter numbers as it should be
+ # and sometimes it isn't (when CSS dictates it). It is a bug in
+ # weasyprint. Princexml is a top of class closed source solution.
+ # Princexml is the fastest HTML to PDF converter known and its fidelity
+ # to CSS is best in class. We are able to use Princexml under a
+ # non-commercial license since it leaves a Princexml logo on the first
+ # page of the generated PDF. The logo isn't too obtrusive. One may
+ # choose to use Princexml rather than Weasyprint by setting use_prince
+ # to True.
+ use_prince: bool = False
+ # Some languages, e.g., Khmer, don't layout well in 2 column
+ use_two_column_layout_for_tn_notes: bool = False
+ # Some languages, e.g., Khmer, don't layout well in 2 column
+ use_two_column_layout_for_tq_notes: bool = False
+
+ # Indicate whether to show visual separator between sections, e.g., hr element
+ use_section_visual_separator: bool = False
# Indicate whether TN book intros should be included. Currently,
# the content team does not want them included.
include_tn_book_intros: bool = False
@@ -212,7 +232,7 @@ class DocumentRequest(BaseModel):
document_request_source: DocumentRequestSourceEnum = DocumentRequestSourceEnum.TEST
@model_validator(mode="after")
- def ensure_valid_document_request(self) -> Any:
+ def ensure_valid_document_request(self) -> "DocumentRequest":
"""
See ValueError messages below for the rules we are enforcing.
"""
@@ -342,11 +362,12 @@ class ResourceLookupDto(NamedTuple):
lang_code: str
lang_name: str
+ localized_lang_name: str
resource_type: str
resource_type_name: str
book_code: str
lang_direction: LangDirEnum
- url: Optional[str]
+ url: Optional[HttpUrl]
@final
@@ -484,6 +505,7 @@ class USFMBook(BaseModel):
lang_code: str
lang_name: str
+ localized_lang_name: str
book_code: str
national_book_name: str
resource_type_name: str
@@ -520,3 +542,36 @@ class Data(TypedDict):
class JsonManifestData(TypedDict):
project: JsonManifestBook
+
+
+# Model the source data returned in fetch_source_data from the data API:
+
+
+class Language(BaseModel):
+ english_name: str
+ ietf_code: str
+ national_name: str
+ direction: LangDirEnum
+
+
+class Content(BaseModel):
+ resource_type: str
+ language: Language
+
+
+class RepoEntry(BaseModel):
+ repo_url: HttpUrl
+ content: Content
+
+
+class SourceData(BaseModel):
+ git_repo: list[RepoEntry]
+
+
+class DocumentPart(BaseModel):
+ content: str
+ is_rtl: bool = False
+ add_hr_p: bool = True
+ contained_in_two_column_section: bool = False
+ add_page_break: bool = False
+ use_section_visual_separator: bool = False
diff --git a/backend/doc/domain/parsing.py b/backend/doc/domain/parsing.py
index eef0dcd51..ef32ecbb6 100644
--- a/backend/doc/domain/parsing.py
+++ b/backend/doc/domain/parsing.py
@@ -3,21 +3,20 @@
"""
import re
-import subprocess
import time
from glob import glob
-from os import DirEntry, getenv, scandir, walk
+from os import DirEntry, scandir, walk
from os.path import exists, join, split
from pathlib import Path
-from typing import Mapping, Optional, Sequence
+from typing import Mapping, Optional, Sequence, cast
import mistune
+import requests
from doc.config import settings
from doc.domain.assembly_strategies.assembly_strategy_utils import (
adjust_commentary_headings,
)
-from doc.domain.bible_books import BOOK_NAMES
-from doc.domain.exceptions import MissingChapterMarkerError
+from doc.domain.bible_books import BOOK_ID_MAP, BOOK_NAMES
from doc.domain.model import (
BC_RESOURCE_TYPE,
EN_TN_CONDENSED_RESOURCE_TYPE,
@@ -59,6 +58,13 @@
translation_words_dict,
tw_resource_dir,
)
+from doc.utils.url_utils import (
+ get_last_segment,
+ get_book_names_from_title_file,
+ book_codes_and_names_from_manifest,
+)
+from pydantic import HttpUrl
+
logger = settings.logger(__name__)
@@ -69,7 +75,6 @@
# fmt: on
-# CHAPTER_LABEL_REGEX = r"\\cl\s+.*"
CHAPTER_LABEL_REGEX = re.compile(r"\\cl\s+[^\n]+")
CHAPTER_LABEL_REGEX2 = re.compile(r"\\cl\s+(.+)")
CHAPTER_REGEX = re.compile(r"\\c\s+\d+")
@@ -142,36 +147,28 @@ def print_directory_contents(directory: str) -> None:
def convert_usfm_chapter_to_html(
content: str,
- resource_filepath_sans_suffix: str,
+ input_file: str,
+ output_file: str,
+ api_url: str = "http://usfmparserapi:80/api/converter/convert",
) -> None:
"""
- Invoke the dotnet USFM parser to parse the USFM file, if it exists,
- and render it into HTML and store on disk.
+ Invoke the dotnet USFM parser through an HTTP POST request to parse the USFM file,
+ and render it into HTML and store it on disk.
"""
- content_file = write_usfm_content_to_file(content, resource_filepath_sans_suffix)
- logger.info("About to convert USFM to HTML")
- dll_path = "/app/USFMParserDriver/bin/Release/net8.0/USFMParserDriver.dll"
- if not exists(f"{getenv('DOTNET_ROOT')}/dotnet"):
- logger.info("dotnet cli not found!")
- raise Exception("dotnet cli not found")
- if not exists(dll_path):
- logger.info("dotnet parser executable not found!")
- # print_directory_contents("/app/USFMParserDriver")
- raise Exception("dotnet parser executable not found!")
- if not exists(content_file):
- logger.info("dotnet parser expects %s to exist, but it does not!", content_file)
- command = [
- f"{getenv('DOTNET_ROOT')}/dotnet",
- dll_path,
- f"/app/{content_file}",
- f"/app/{resource_filepath_sans_suffix}.html",
- ]
- logger.info("dotnet command: %s", " ".join(command))
- subprocess.run(
- command,
- check=True,
- text=True,
- )
+ logger.info("About to convert USFM to HTML via HTTP POST")
+ payload = {
+ "InputFile": input_file,
+ "OutputFile": output_file,
+ }
+ try:
+ response = requests.post(api_url, json=payload)
+ # response.raise_for_status() # Raise an error for 4xx/5xx responses
+ if response.json():
+ logger.info("Conversion successful: %s", output_file)
+ else:
+ logger.error("Conversion failed with an unknown error.")
+ except requests.exceptions.RequestException as e:
+ logger.error("HTTP request failed: %s", e)
def usfm_asset_file(
@@ -210,32 +207,22 @@ def usfm_asset_file(
def usfm_chapter_html(
content: str,
- resource_lookup_dto: ResourceLookupDto,
+ input_file: str,
+ output_file: str,
chapter_num: int,
- working_dir: str = settings.WORKING_DIR,
) -> Optional[str]:
- resource_filepath_sans_suffix = "_".join(
- [
- resource_lookup_dto.lang_code,
- resource_lookup_dto.resource_type,
- resource_lookup_dto.book_code,
- str(chapter_num),
- ]
- )
- resource_filepath_sans_suffix = f"{working_dir}/{resource_filepath_sans_suffix}"
- html_content_filepath = f"{resource_filepath_sans_suffix}.html"
t0 = time.time()
- convert_usfm_chapter_to_html(content, resource_filepath_sans_suffix)
+ with open(input_file, "w") as f:
+ f.write(content)
+ convert_usfm_chapter_to_html(content, input_file, output_file)
t1 = time.time()
logger.info(
- "Time to convert USFM to HTML for %s-%s-%s: %s",
- resource_lookup_dto.lang_code,
- resource_lookup_dto.resource_type,
- resource_lookup_dto.book_code,
+ "Time to convert USFM to HTML for %s: %s",
+ output_file,
t1 - t0,
)
- if exists(html_content_filepath):
- html_content = read_file(html_content_filepath)
+ if exists(output_file):
+ html_content = read_file(output_file)
return html_content
return None
@@ -253,11 +240,11 @@ def split_usfm_by_chapters(
resource_type: str,
book_code: str,
usfm_text: str,
+ check_usfm: bool = settings.CHECK_USFM,
chapter_regex: re.Pattern[str] = CHAPTER_REGEX,
resources_with_usfm_defects: Sequence[
tuple[str, str, str]
] = RESOURCES_WITH_USFM_DEFECTS,
- check_usfm: bool = settings.CHECK_USFM,
check_all_books_for_language: bool = settings.CHECK_ALL_BOOKS_FOR_LANGUAGE,
) -> tuple[str, list[str], list[str]]:
r"""
@@ -407,16 +394,19 @@ def maybe_localized_book_name(frontmatter: str) -> str:
Steps 5 and 6 happen outside this function.
"""
- # logger.debug("frontmatter: %s", frontmatter)
frontmatter_data = extract_usfm_frontmatter(frontmatter)
localized_book_name = (
frontmatter_data.get("h")
or frontmatter_data.get("mt")
+ or frontmatter_data.get("mt1")
or frontmatter_data.get("toc1")
or frontmatter_data.get("toc2")
or ""
)
- localized_book_name = normalize_localized_book_name(localized_book_name)
+ logger.debug("localized_book_name: %s", localized_book_name)
+ if localized_book_name:
+ localized_book_name = normalize_localized_book_name(localized_book_name)
+ logger.debug("normalized localized_book_name: %s", localized_book_name)
return localized_book_name
@@ -449,6 +439,8 @@ def usfm_book_content(
resource_dir: str,
use_chapter_labels: bool,
book_names: Mapping[str, str] = BOOK_NAMES,
+ working_dir: str = settings.WORKING_DIR,
+ use_localized_book_name: bool = settings.USE_LOCALIZED_BOOK_NAME,
) -> USFMBook:
"""
First produce HTML content from USFM content and then break the
@@ -470,17 +462,31 @@ def usfm_book_content(
resource_lookup_dto.book_code,
content,
)
- localized_book_name = maybe_localized_book_name(frontmatter)
+ localized_book_name = ""
+ if use_localized_book_name:
+ localized_book_name = get_localized_book_name(
+ frontmatter, resource_dir, resource_lookup_dto
+ )
for chapter_marker, chapter_usfm in zip(chapter_markers, chapters_usfm):
- # chapter_usfm = chapter_marker + "\n" + chapter_usfm
chapter_num = get_chapter_num(chapter_usfm)
if chapter_num == -1:
chapter_num = chapter_label_numeric_part(chapter_usfm)
if use_chapter_labels:
chapter_usfm = ensure_chapter_label(chapter_usfm, chapter_num)
chapter_usfm = ensure_chapter_marker(chapter_usfm, chapter_num)
+ resource_filename_sans_suffix = "_".join(
+ [
+ resource_lookup_dto.lang_code,
+ resource_lookup_dto.resource_type,
+ resource_lookup_dto.book_code,
+ str(chapter_num),
+ ]
+ )
+ resource_filepath_sans_suffix = join(working_dir, resource_filename_sans_suffix)
+ input_file = f"{resource_filepath_sans_suffix}.usfm"
+ output_file = f"{resource_filepath_sans_suffix}.html"
chapter_html_content = usfm_chapter_html(
- chapter_usfm, resource_lookup_dto, chapter_num
+ chapter_usfm, input_file, output_file, chapter_num
)
cleaned_chapter_html_content = remove_null_bytes_and_control_characters(
chapter_html_content
@@ -494,6 +500,7 @@ def usfm_book_content(
return USFMBook(
lang_code=resource_lookup_dto.lang_code,
lang_name=resource_lookup_dto.lang_name,
+ localized_lang_name=resource_lookup_dto.localized_lang_name,
book_code=resource_lookup_dto.book_code,
national_book_name=(
localized_book_name
@@ -506,6 +513,40 @@ def usfm_book_content(
)
+def get_localized_book_name(
+ frontmatter: str,
+ resource_dir: str,
+ resource_lookup_dto: ResourceLookupDto,
+ usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
+) -> str:
+ localized_book_name = maybe_localized_book_name(frontmatter)
+ if not localized_book_name:
+ book_codes_and_names_from_manifest_ = book_codes_and_names_from_manifest(
+ resource_dir
+ )
+ localized_book_name = book_codes_and_names_from_manifest_.get(
+ resource_lookup_dto.book_code, ""
+ )
+ if not localized_book_name:
+ last_segment = get_last_segment(
+ # We know that url is not null because of how we got here
+ cast(HttpUrl, resource_lookup_dto.url),
+ resource_lookup_dto.lang_code,
+ )
+ repo_components = last_segment.split("_")
+ if (
+ len(repo_components) > 2
+ and resource_lookup_dto.resource_type in usfm_resource_types
+ ):
+ book_names_from_title_file = get_book_names_from_title_file(
+ resource_dir, resource_lookup_dto.lang_code, repo_components
+ )
+ localized_book_name = book_names_from_title_file.get(
+ resource_lookup_dto.book_code, ""
+ )
+ return localized_book_name
+
+
def load_manifest(file_path: str) -> str:
with open(file_path, "r") as file:
return file.read()
@@ -666,6 +707,19 @@ def tn_book_content(
)
+def clean_numeric_string(s: str) -> str:
+ """
+ Removes all non-numeric characters from the input string.
+
+ Args:
+ s (str): Input string that may contain non-numeric characters.
+
+ Returns:
+ str: String containing only numeric characters.
+ """
+ return "".join(c for c in s if c.isdigit())
+
+
def tq_chapter_verses(
resource_dir: str,
lang_code: str,
@@ -685,6 +739,9 @@ def tq_chapter_verses(
verses_html: dict[VerseRef, str] = {}
for filepath in verse_paths:
verse_ref = Path(filepath).stem
+ # There was a case of a verse file being named '18.txt, so we handle
+ # such cases since we need to cast to int below:
+ verse_ref = clean_numeric_string(verse_ref)
verse_md_content = read_file(filepath)
verse_md_content = markdown_transformer.transform_ta_and_tn_links(
verse_md_content,
@@ -909,6 +966,8 @@ def books(
bc_resource_type: str = BC_RESOURCE_TYPE,
rg_resource_type: str = RG_RESOURCE_TYPE,
docx_file_path: str = "en_rg_nt_survey.docx",
+ en_rg_dir: str = settings.EN_RG_DIR,
+ book_id_map: dict[str, int] = BOOK_ID_MAP,
) -> tuple[
Sequence[USFMBook],
Sequence[TNBook],
@@ -957,8 +1016,8 @@ def books(
)
bc_books.append(bc_book)
elif resource_lookup_dto.resource_type == rg_resource_type:
- path = join(resource_dir, docx_file_path)
- # logger.debug("About to get_rg_books from: %s", path)
+ path = join(en_rg_dir, docx_file_path)
+ logger.debug("About to get_rg_books from: %s", path)
rg_books = get_rg_books(
path,
resource_lookup_dto.lang_code,
@@ -981,7 +1040,6 @@ def ensure_paragraph_before_verses(
usfm_verse_one_file_regex: str = r"^01\..*",
chapter_marker_not_on_own_line_regex: str = r"^\\c [0-9]+ .*|\n",
chapter_marker_not_on_own_line_with_match_groups: str = r"(^\\c [0-9]+) (.*|\n)",
- # chapter_marker_not_on_own_line_repair_regex: str = r"\1\n\\p\n\2\n",
chapter_marker_not_on_own_line_repair_regex: str = r"\1\n\n\2\n",
) -> str:
r"""
@@ -1041,34 +1099,69 @@ def assemble_chapter_usfm(
use_chapter_labels: bool,
) -> list[str]:
chapter_usfm_content = []
- try:
- chapter_num = int(str(chapter_dir.name))
- except ValueError:
- logger.info(
- "%s is not a valid chapter number, assigning -1 as chapter number",
- str(chapter_dir.name),
- )
- chapter_num = -1 # use this as a sentinal
+ chapter_num = get_chapter_number(chapter_dir.name)
chapter_usfm_content.append("\n" + rf"\c {chapter_num}" + "\n")
if use_chapter_labels:
chapter_word_file = join(chapter_dir.path, "title.txt")
- try:
- with open(chapter_word_file, "r") as fin:
- chapter_word = fin.read()
- chapter_word = chapter_word.strip()
- chapter_word = chapter_label_sans_numeric_part(chapter_word)
- chapter_label = "\n" + rf"\cl {chapter_word} {chapter_num}" + "\n"
- chapter_usfm_content.append(chapter_label)
- except FileNotFoundError:
- pass # No file containing chapter label
- # In ensure_chapter_label an English chapter label will be
- # inserted if a chapter label is missing and
- # use_chapter_labels is True
+ chapter_word = read_chapter_label(chapter_word_file)
+ if chapter_word is not None:
+ chapter_label = "\n" + rf"\cl {chapter_word} {chapter_num}" + "\n"
+ chapter_usfm_content.append(chapter_label)
logger.info(
"Adding a USFM chapter marker for chapter: %s",
chapter_num,
)
- chapter_verse_files = sorted(
+ chapter_verse_chunk_files = get_chapter_verse_chunk_files(chapter_dir)
+ for usfm_file in chapter_verse_chunk_files:
+ verse_content = read_verse_file(usfm_file)
+ cleaned_verse_content = clean_verse_content(verse_content)
+ verse_content = ensure_paragraph_before_verses(usfm_file, cleaned_verse_content)
+ chapter_usfm_content.append(cleaned_verse_content)
+ chapter_usfm_content.append(
+ " \n"
+ ) # Make sure a space before next chunk, e.g., auh, mat, ch 9, v 14
+ return chapter_usfm_content
+
+
+def get_chapter_number(chapter_dir_name: str) -> int:
+ try:
+ return int(chapter_dir_name)
+ except ValueError:
+ logger.info(
+ "%s is not a valid chapter number, assigning -1 as chapter number",
+ chapter_dir_name,
+ )
+ return -1 # Sentinel value
+
+
+def read_chapter_label(chapter_word_file: str) -> Optional[str]:
+ try:
+ with open(chapter_word_file, "r") as fin:
+ chapter_word = fin.read().strip()
+ return chapter_label_sans_numeric_part(chapter_word)
+ except FileNotFoundError:
+ return None
+
+
+def read_verse_file(usfm_file: str) -> str:
+ with open(usfm_file, "r") as fin:
+ return fin.read()
+
+
+def clean_verse_content(verse_content: str) -> str:
+ """
+ Some languages put a chapter marker in front of verse 1 in the
+ verse file which covers a verse span which includes verse 1. Since we
+ ensure chapter markers ourselves when assembling multiple verse files
+ into a chapter this ends up creating a duplicate chapter marker.
+ We deal with that here.
+ """
+ cleaned_verse_content = re.sub(r"^\\c\s+\d+", "", verse_content)
+ return cleaned_verse_content
+
+
+def get_chapter_verse_chunk_files(chapter_dir: DirEntry[str]) -> Sequence[str]:
+ return sorted(
[
file.path
for file in scandir(chapter_dir)
@@ -1078,19 +1171,6 @@ def assemble_chapter_usfm(
and (file.name.endswith(".usfm") or file.name.endswith(".txt"))
]
)
- for usfm_file in chapter_verse_files:
- with open(usfm_file, "r") as fin:
- # logger.debug("usfm_file: %s", usfm_file)
- verse_content = fin.read()
- # Some languages put a chapter marker in front of verse 1 in the verse
- # file which covers a verse span which includes verse 1 . Since we
- # ensure chapter markers ourselves when assembling multiple verse files
- # into a chapter this ends up creating a duplicate chapter marker.
- verse_content = re.sub(r"^\\c\s+\d+", "", verse_content)
- verse_content = ensure_paragraph_before_verses(usfm_file, verse_content)
- chapter_usfm_content.append(verse_content)
- chapter_usfm_content.append("\n")
- return chapter_usfm_content
def combine_usfm_files(
diff --git a/backend/doc/domain/resource_lookup.py b/backend/doc/domain/resource_lookup.py
index 6c33eb924..20c42a462 100644
--- a/backend/doc/domain/resource_lookup.py
+++ b/backend/doc/domain/resource_lookup.py
@@ -4,30 +4,28 @@
assets.
"""
-import json
import re
import shutil
import subprocess
-from functools import lru_cache
-from glob import glob
-from os import scandir
-from os.path import basename, exists, isdir, join
+from datetime import datetime, timedelta
+from os import scandir, stat
+from os.path import exists, isdir, join
from pathlib import Path
-from typing import Any, Mapping, Optional, Sequence
-from urllib.parse import urlparse
+from typing import Mapping, Optional, Sequence
import requests
-import yaml
+from cachetools import TTLCache, cached
from doc.config import settings
-from doc.domain import parsing, worker
-from doc.domain.bible_books import BOOK_CHAPTERS, BOOK_NAMES
+from doc.domain import worker, parsing
+from doc.domain.bible_books import BOOK_CHAPTERS, BOOK_ID_MAP, BOOK_NAMES
from doc.domain.model import (
NON_USFM_RESOURCE_TYPES,
- Data,
- JsonManifestBook,
- JsonManifestData,
+ Content,
LangDirEnum,
+ Language,
+ RepoEntry,
ResourceLookupDto,
+ SourceData,
)
from doc.reviewers_guide.model import BibleReference
from doc.reviewers_guide.parser import (
@@ -35,17 +33,26 @@
get_rg_books,
parse_bible_reference,
)
-from doc.utils.file_utils import file_needs_update, make_dir, read_file
-from doc.utils.list_utils import unique_tuples
+from doc.utils.file_utils import (
+ delete_tree,
+ file_needs_update,
+ read_file,
+)
+from doc.utils.list_utils import unique_tuples, unique_book_codes
from doc.utils.text_utils import normalize_localized_book_name
+from doc.utils.url_utils import (
+ get_last_segment,
+ get_book_names_from_title_file,
+ book_codes_and_names_from_manifest,
+)
from fastapi import HTTPException, status
-from pydantic import HttpUrl
+from pydantic import HttpUrl, ValidationError
+
logger = settings.logger(__name__)
-SOURCE_DATA_JSON_FILENAME = "resources.json"
+fetch_source_data_cache: TTLCache[str, SourceData] = TTLCache(maxsize=1, ttl=180)
-SOURCE_GATEWAY_LANGUAGES_FILENAME = "gateway_languages.json"
# This can be expanded to include any additional types (if
# there are any) that we want to be available to users. These are all
@@ -57,8 +64,8 @@
"cuv": "新标点和合本",
"f10": "French Louis Segond 1910 Bible",
"nav": "New Arabic Version (Ketab El Hayat)",
- "reg": "Bible",
- "rg": "NT Survey Reviewer's Guide",
+ "reg": "Regular",
+ "rg": "NT Survey Reviewers' Guide",
"tn": "Translation Notes",
"tn-condensed": "Condensed Translation Notes",
"tq": "Translation Questions",
@@ -69,7 +76,7 @@
"ulb": "Unlocked Literal Bible",
}
-# NOTE This is only used to see if a lang_code is in the collection
+# This is only used to see if a lang_code is in the collection
# otherwise it is a heart language. Eventually the graphql data api may
# provide gateway/heart boolean value.
GATEWAY_LANGUAGES: Sequence[str] = [
@@ -171,13 +178,13 @@
"zlm",
]
-# The book name in the tuple key is what
-# resource_lookup.get_book_codes_for_lang is returning for lang_code in
-# the tuple key and the associated value is what we would prefer to
-# use.
BOOK_NAME_CORRECTION_TABLE: dict[tuple[str, str], str] = {
- ("pt-br", "1 Corintios"): "1 Coríntios",
("es-419", "I juan"): "1 Juan",
+ ("fr", "Ephésiens"): "Éphésiens",
+ ("pt-br", "1 Corintios"): "1 Coríntios",
+ ("sw", "Matendo ya mitume"): "Matendo ya Mitume",
+ ("sw", "Luke"): "Luka",
+ ("sw", "Waraka wa yakobo"): "Yakobo",
}
# List of languages which do not have USFM available for any books. We use this
@@ -186,54 +193,27 @@
# selecting a language which might have non-USFM resources available but
# not USFM so that when their resulting doc is generated no scripture is
# present. It makes it seem like a bug in STET and is bad UX.
-LANG_CODE_WITH_NO_USFM_FILTER_LIST: list[str] = ["ru"]
-
+LANG_CODES_WITH_NO_USFM: list[str] = ["ru"]
-@lru_cache(maxsize=2)
-def fetch_source_data(
- json_file_name: str = SOURCE_DATA_JSON_FILENAME,
- assets_dir: str = settings.RESOURCE_ASSETS_DIR,
-) -> Any:
- """
- Obtain the source data, by downloading it from json_file_url, and
- then reifying it into its JSON object form.
-
- >>> from doc.domain import resource_lookup
- >>> ();result = resource_lookup.fetch_source_data();() # doctest: +ELLIPSIS
- (...)
- >>> result["git_repo"][0]
- {'repo_url': 'https://content.bibletranslationtools.org/bahasatech.indotengah/adn_1jn_text_reg', 'content': {'resource_type': 'reg', 'language': {'english_name': 'Adang', 'ietf_code': 'adn', 'national_name': 'Adang', 'direction': 'ltr'}}}
- """
- json_file_path = join(assets_dir, json_file_name)
- data = None
- if file_needs_update(json_file_path):
- logger.info("About to download %s...", json_file_name)
- try:
- data = download_data(json_file_path)
- # logger.debug("data: %s", data)
- except Exception:
- logger.exception("Caught exception: ")
- else:
- if exists(json_file_path):
- logger.info("json_file_path, %s exists", json_file_path)
- content = read_file(json_file_path)
- # logger.debug("json data: %s", content)
- data = json.loads(content)
- return data
+# For cloudflare
+USER_AGENT_STR: str = "wa-doc"
+X_REQUESTED_WITH_VALUE: str = "WA-Tool-Doc"
-def download_data(
- jsonfile_path: str,
+@cached(fetch_source_data_cache)
+def fetch_source_data(
data_api_url: HttpUrl = settings.DATA_API_URL,
-) -> Any:
+ user_agent_str: str = USER_AGENT_STR,
+ x_requested_with_value: str = X_REQUESTED_WITH_VALUE,
+) -> Optional[SourceData]:
"""
- Downloads data from a GraphQL API and saves it to a JSON file.
+ Downloads data from a GraphQL API.
>>> from doc.domain import resource_lookup
- >>> ();result = resource_lookup.download_data("assets_download/resources.json");() # doctest: +ELLIPSIS
+ >>> ();result = resource_lookup.fetch_source_data();() # doctest: +ELLIPSIS
(...)
- >>> result["git_repo"][0]
- {'repo_url': 'https://content.bibletranslationtools.org/bahasatech.indotengah/adn_1jn_text_reg', 'content': {'resource_type': 'reg', 'language': {'english_name': 'Adang', 'ietf_code': 'adn', 'national_name': 'Adang', 'direction': 'ltr'}}}
+ >>> result.git_repo[0]
+ RepoEntry(repo_url=HttpUrl('https://content.bibletranslationtools.org/ebenezer-sako/ahm_eph_text_reg'), content=Content(resource_type='reg', language=Language(english_name='Aizi, Mobumrin', ietf_code='ahm', national_name='Mobumrin Aizi', direction=)))
"""
graphql_query = """
query MyQuery {
@@ -254,142 +234,41 @@ def download_data(
}
"""
query_json = {"query": graphql_query}
-
- def _read_file(file_path: str) -> str:
- with open(file_path, "r") as file:
- return file.read()
-
- def _write_json_to_file(file_path: str, data: Any) -> None:
- with open(file_path, "w") as file:
- json.dump(data, file, indent=4)
-
- def fetch_cached_data() -> Any:
- logger.info("About to fetch cached data API results from %s", jsonfile_path)
- content = _read_file(jsonfile_path)
- return json.loads(content)
-
+ headers = {"User-Agent": user_agent_str, "X-Requested-With": x_requested_with_value}
try:
- response = requests.post(str(data_api_url), json=query_json)
+ response = requests.post(str(data_api_url), json=query_json, headers=headers)
if response.status_code == 200:
data_payload = response.json().get("data", {})
if "git_repo" in data_payload:
- logger.info("Writing json data to: %s", jsonfile_path)
- _write_json_to_file(jsonfile_path, data_payload)
- return data_payload
+ # return SourceData.model_validate(data_payload)
+ valid_repos = [
+ repo
+ for repo in data_payload["git_repo"]
+ if repo.get("content", {}).get("resource_type") is not None
+ ]
+ return SourceData.model_validate({"git_repo": valid_repos})
else:
- logger.info("Invalid payload structure, using cached data.")
- return fetch_cached_data()
+ logger.info("Invalid payload structure, no data.")
+ return SourceData(git_repo=[])
else:
logger.info(
"Failed to get data from data API, graphql API might be down..."
)
- return fetch_cached_data()
+ return SourceData(git_repo=[])
except requests.RequestException as e:
logger.exception("Request failed: %s", e)
logger.info("Failed to get data from data API, API might be down...")
- return fetch_cached_data()
-
-
-def fetch_gateway_languages(
- jsonfile_path: str,
- data_api_url: HttpUrl = settings.DATA_API_URL,
-) -> Any:
- """
- >>> from doc.domain import resource_lookup
- >>> ();result = resource_lookup.fetch_gateway_languages("assets_download/gateway_languages.json");() # doctest: +ELLIPSIS
- (...)
- >>> result["language"][0]
- {'gateway_languages': [{'gateway_language': {'ietf_code': 'es-419', 'national_name': 'Español Latin America', 'english_name': 'Latin American Spanish'}}]}
- """
- graphql_query = """
-query MyQuery {
- language {
- gateway_languages {
- gateway_language {
- ietf_code
- national_name
- english_name
- }
- }
- }
-}
- """
- payload = {"query": graphql_query}
- try:
- response = requests.post(str(data_api_url), json=payload)
- if response.status_code == 200:
- data = response.json()
- logger.info("Writing json data to: %s", jsonfile_path)
- # Filter out empty gateway_languages entries
- with open(jsonfile_path, "w") as fp:
- fp.write(str(json.dumps(data["data"])))
- filtered_data = {
- "language": [
- entry
- for entry in data["data"]["language"]
- if entry["gateway_languages"]
- ]
- }
- return filtered_data
- else:
- logger.info(
- "Failed to get data from data API, graphql API might be down..."
- )
- response.raise_for_status()
- except Exception as e:
- return {"error": str(e)}
-
+ return SourceData(git_repo=[])
+ except ValidationError as e:
+ logger.exception(
+ "Request failed due to invalid data returned from data API: %s", e
+ )
+ logger.info(
+ "Some of the data returned by data API is invalid, check logs for details"
+ )
+ return SourceData(git_repo=[])
-def get_gateway_languages(
- json_file_name: str = SOURCE_GATEWAY_LANGUAGES_FILENAME,
- working_dir: str = settings.RESOURCE_ASSETS_DIR,
- use_hardcoded_gateway_language_values: bool = True,
- hardcoded_gateway_languages: Sequence[str] = GATEWAY_LANGUAGES,
-) -> Any:
- """
- Obtain the source data, by downloading it from json_file_url, and
- then reifying it into its JSON object form unless
- use_hardcoded_gateway_language_values is True in which case just
- return the hardcoded list of gateway languages.
- >>> from doc.domain import resource_lookup
- >>> ();result = resource_lookup.get_gateway_languages();() # doctest: +ELLIPSIS
- (...)
- >>> result[0]
- 'abs'
- """
- gateway_languages_collection = []
- if use_hardcoded_gateway_language_values:
- return hardcoded_gateway_languages
- json_file_path = join(working_dir, json_file_name)
- data = None
- if file_needs_update(json_file_path):
- logger.info("About to download %s...", json_file_name)
- try:
- data = fetch_gateway_languages(json_file_path)
- # logger.debug("data: %s", data)
- except Exception:
- logger.exception("Caught exception: ")
- else:
- if exists(json_file_path):
- logger.info("json_file_path, %s, does exist", json_file_path)
- content = read_file(json_file_path)
- # logger.debug("json data: %s", content)
- data = json.loads(content)
- gateway_languages = data["language"] if data and "language" in data else []
- if gateway_languages:
- for gateway_language in gateway_languages:
- languages = gateway_language["gateway_languages"]
- for language in languages:
- language_ = language["gateway_language"]
- ietf_code = language_["ietf_code"]
- if ietf_code not in gateway_languages_collection:
- gateway_languages_collection.append(ietf_code)
- # logger.debug("gateway_languages: %s", gateway_languages_collection)
- return gateway_languages_collection
-
-
-@lru_cache(maxsize=100)
def lang_codes_and_names(
# lang_code_filter_list: Sequence[str] = settings.LANG_CODE_FILTER_LIST,
gateway_languages: Sequence[str] = GATEWAY_LANGUAGES,
@@ -399,77 +278,64 @@ def lang_codes_and_names(
>>> ();result = resource_lookup.lang_codes_and_names();() # doctest: +ELLIPSIS
(...)
>>> result[0]
- ('cdi', ': Chodri (: Chaudhari)', False)
+ ('abz', 'Abui', False)
>>> heart_lang_codes = [lang_code_and_name[0] for lang_code_and_name in resource_lookup.lang_codes_and_names() if not lang_code_and_name[2]]
- >>> for heart_lang_code in heart_lang_codes:
- ... resource_lookup.book_codes_for_lang(heart_lang_code)
- ...
+ >>> sorted(heart_lang_codes)[0]
+ 'aao'
"""
- gateway_languages_ = get_gateway_languages()
- if not gateway_languages_:
- gateway_languages_ = gateway_languages
data = fetch_source_data()
values = []
- if data and "git_repo" not in data:
- raise Exception("Data API is down!")
+ if data is None or not data.git_repo:
+ logger.info("Data API is down or no git_repo found!")
+ return []
try:
- repos_info = data["git_repo"]
- for repo_info in repos_info:
- language_info = repo_info["content"]
- language = language_info["language"]
- ietf_code = language["ietf_code"]
- english_name = (
- language["english_name"] if "english_name" in language else ""
- )
- localized_name = language["national_name"]
- is_gateway = ietf_code in gateway_languages_
- # if ietf_code not in lang_code_filter_list:
+ # if ietf_code not in lang_code_filter_list:
+ for repo_info in data.git_repo:
+ language_info = repo_info.content
+ language = language_info.language
+ ietf_code = language.ietf_code
+ english_name = language.english_name if language.english_name else ""
+ localized_name = language.national_name
+ is_gateway = ietf_code in gateway_languages
if english_name in localized_name:
values.append((ietf_code, localized_name, is_gateway))
else:
values.append(
(ietf_code, f"{localized_name} ({english_name})", is_gateway)
)
- except:
+ except Exception:
logger.exception("Failed due to the following exception.")
unique_values = unique_tuples(values)
return sorted(unique_values, key=lambda value: value[1])
-@lru_cache(maxsize=100)
def lang_codes_and_names_having_usfm(
- lang_code_filter_list: Sequence[str] = LANG_CODE_WITH_NO_USFM_FILTER_LIST,
+ lang_code_filter_list: Sequence[str] = LANG_CODES_WITH_NO_USFM,
gateway_languages: Sequence[str] = GATEWAY_LANGUAGES,
) -> Sequence[tuple[str, str, bool]]:
"""
>>> from doc.domain import resource_lookup
- >>> ();result = resource_lookup.lang_codes_and_names();() # doctest: +ELLIPSIS
+ >>> ();result = resource_lookup.lang_codes_and_names_having_usfm();() # doctest: +ELLIPSIS
(...)
>>> result[0]
- ('cdi', ': Chodri (: Chaudhari)', False)
- >>> heart_lang_codes = [lang_code_and_name[0] for lang_code_and_name in resource_lookup.lang_codes_and_names() if not lang_code_and_name[2]]
- >>> for heart_lang_code in heart_lang_codes:
- ... resource_lookup.book_codes_for_lang(heart_lang_code)
- ...
+ ('abz', 'Abui', False)
+ >>> heart_lang_codes = [lang_code_and_name[0] for lang_code_and_name in resource_lookup.lang_codes_and_names_having_usfm() if not lang_code_and_name[2]]
+ >>> sorted(heart_lang_codes)[0]
+ 'aao'
"""
- gateway_languages_ = get_gateway_languages()
- if not gateway_languages_:
- gateway_languages_ = gateway_languages
data = fetch_source_data()
values = []
- if data and "git_repo" not in data:
- raise Exception("Data API is down!")
+ if data is None or not data.git_repo:
+ logger.info("Data API is down or no git_repo found!")
+ return []
try:
- repos_info = data["git_repo"]
- for repo_info in repos_info:
- language_info = repo_info["content"]
- language = language_info["language"]
- ietf_code = language["ietf_code"]
- english_name = (
- language["english_name"] if "english_name" in language else ""
- )
- localized_name = language["national_name"]
- is_gateway = ietf_code in gateway_languages_
+ for repo_info in data.git_repo:
+ language_info = repo_info.content
+ language = language_info.language
+ ietf_code = language.ietf_code
+ english_name = language.english_name if language.english_name else ""
+ localized_name = language.national_name
+ is_gateway = ietf_code in gateway_languages
if ietf_code not in lang_code_filter_list:
if english_name in localized_name:
values.append((ietf_code, localized_name, is_gateway))
@@ -477,74 +343,62 @@ def lang_codes_and_names_having_usfm(
values.append(
(ietf_code, f"{localized_name} ({english_name})", is_gateway)
)
- except:
+ except Exception:
logger.exception("Failed due to the following exception.")
unique_values = unique_tuples(values)
return sorted(unique_values, key=lambda value: value[1])
-@lru_cache(maxsize=100)
-@worker.app.task
-def resource_types(
+def repos_to_clone(
lang_code: str,
- book_codes_str: str,
+ augmented_repos_info: list[RepoEntry],
resource_assets_dir: str = settings.RESOURCE_ASSETS_DIR,
+ dcs_mirror_git_username: str = "DCS-Mirror",
+ resource_type_codes_and_names: Sequence[str] = list(
+ RESOURCE_TYPE_CODES_AND_NAMES.keys()
+ ),
+) -> list[tuple[HttpUrl, str, str]]:
+ repo_clone_list: list[tuple[HttpUrl, str, str]] = []
+ try:
+ for repo_info in augmented_repos_info:
+ content = repo_info.content
+ resource_type = content.resource_type
+ if (
+ content.language.ietf_code == lang_code
+ and resource_type in resource_type_codes_and_names
+ ):
+ url = repo_info.repo_url
+ last_segment = get_last_segment(url, lang_code)
+ resource_filepath = f"{resource_assets_dir}/{last_segment}"
+ repo_components = last_segment.split("_")
+ if dcs_mirror_git_username in str(url):
+ repo_components = update_repo_components(repo_components)
+ if not any(item[0] == url for item in repo_clone_list):
+ repo_clone_list.append(
+ (
+ url,
+ resource_filepath,
+ resource_type,
+ )
+ )
+ except Exception:
+ logger.exception("Error during repos_to_clone")
+ finally:
+ return repo_clone_list
+
+
+def get_resource_types(
+ repo_clone_list: Sequence[tuple[HttpUrl, str, str]],
+ book_codes: Sequence[str],
bc_book_asset_pattern: str = r"^\d{2,}-[0-9a-z]{3}$",
- resource_type_codes_and_names: Mapping[str, str] = RESOURCE_TYPE_CODES_AND_NAMES,
usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
- book_names: dict[str, str] = BOOK_NAMES,
docx_file_path: str = "en_rg_nt_survey.docx",
-) -> Sequence[tuple[str, str]]:
- """
- >>> from doc.domain import resource_lookup
- >>> lang_code = "pt-br"
- >>> books = resource_lookup.book_codes_for_lang(lang_code)
- >>> ();result = resource_lookup.resource_types(lang_code, "".join([book[0] for book in books]));() # doctest: +ELLIPSIS
- (...)
- >>> result
- [('blv', 'Portuguese Bíblia Livre'), ('tw', 'Translation Words'), ('ulb', 'Unlocked Literal Bible')]
- Fetches and processes available resource types for the given language and book codes.
- """
- book_codes = book_codes_str.split(",")
- if book_codes and book_codes[0] == "all":
- book_codes = list(book_names.keys())
- data = fetch_source_data()
+ en_rg: str = settings.EN_RG_DIR,
+ resource_type_codes_and_names: Mapping[str, str] = RESOURCE_TYPE_CODES_AND_NAMES,
+) -> list[tuple[str, str]]:
resource_types = []
- repo_clone_list = [] # Collect URLs and file paths for batch cloning
- try:
- repos_info = data["git_repo"]
- augmented_repos_info = add_data_not_supplied_by_data_api(repos_info)
- for repo_info in augmented_repos_info:
- content = repo_info["content"]
- language_info = content["language"]
- if language_info["ietf_code"] == lang_code:
- resource_type = content["resource_type"]
- if resource_type in resource_type_codes_and_names:
- url = repo_info["repo_url"]
- last_segment = get_last_segment(url, lang_code)
- resource_filepath = f"{resource_assets_dir}/{last_segment}"
- repo_clone_list.append((url, resource_filepath))
- repo_clone_list = list(set(repo_clone_list))
- # Separate repos that need to be cloned from en_rg
- repos_to_clone = [
- (url, path) for url, path in repo_clone_list if "en_rg" not in path
- ]
- # Perform batch cloning only on filtered list
- batch_clone_git_repos(repos_to_clone)
- # Process cloned repositories
- for url, resource_filepath in repo_clone_list:
- resource_type = next(
- (
- repo_info["content"]["resource_type"]
- for repo_info in augmented_repos_info
- if repo_info["repo_url"] == url
- ),
- None,
- )
- # logger.debug("resource_type: %s, url: %s", resource_type, url)
- if not resource_type:
- continue
- # Determine book assets
+ for url, resource_filepath, resource_type in repo_clone_list:
+ if resource_type:
book_assets = []
if resource_type in ["tq", "tn", "tn-condensed"]:
book_assets = [
@@ -567,7 +421,7 @@ def resource_types(
book_assets = parsing.find_usfm_files(resource_filepath)
elif resource_type == "rg":
between_texts, bible_reference_strs = find_bible_references(
- f"{resource_filepath}/{docx_file_path}"
+ join(en_rg, docx_file_path)
)
bible_references = [
parse_bible_reference(bible_reference)
@@ -581,7 +435,6 @@ def resource_types(
book_assets = [
book_code for book_code in book_codes if book_code in book_codes_
]
- # Check if at least one selected book exists in the repo
if book_assets or resource_type == "tw":
resource_types.append(
(
@@ -589,49 +442,165 @@ def resource_types(
resource_type_codes_and_names[resource_type],
)
)
- except:
- pass
- unique_values = unique_tuples(resource_types)
+ return resource_types
+
+
+@worker.app.task
+def resource_types(
+ lang_code: str,
+ book_codes_str: str,
+ resource_assets_dir: str = settings.RESOURCE_ASSETS_DIR,
+ book_names: Mapping[str, str] = BOOK_NAMES,
+ download_assets: bool = settings.DOWNLOAD_ASSETS,
+) -> Sequence[tuple[str, str]]:
+ """
+ Fetches and processes available resource types for the given language and book codes.
+
+ >>> from doc.domain import resource_lookup
+ >>> lang_code = "pt-br"
+ >>> books = resource_lookup.book_codes_for_lang(lang_code)
+ >>> ();result = resource_lookup.resource_types(lang_code, "".join([book[0] for book in books]));() # doctest: +ELLIPSIS
+ (...)
+ >>> result
+ [('blv', 'Portuguese Bíblia Livre'), ('tw', 'Translation Words'), ('ulb', 'Unlocked Literal Bible')]
+ """
+ book_codes = book_codes_str.split(",")
+ resource_types_: list[tuple[str, str]] = []
+ if book_codes and book_codes[0] == "all":
+ book_codes = list(book_names.keys())
+ data = fetch_source_data()
+ repo_clone_list: list[tuple[HttpUrl, str, str]] = []
+ if data is None or not data.git_repo:
+ logger.info("Data API is down or no git_repo found!")
+ return []
+ try:
+ repos_info = data.git_repo
+ augmented_repos_info = add_data_not_supplied_by_data_api(repos_info)
+ repo_clone_list = repos_to_clone(lang_code, augmented_repos_info)
+ repos_to_clone_ = [
+ (url, path)
+ for url, path, resource_type_ in repo_clone_list
+ if "rg" != resource_type_
+ ]
+ if download_assets:
+ batch_download_repos(repos_to_clone_)
+ else:
+ batch_clone_git_repos(repos_to_clone_)
+ resource_types_ = get_resource_types(repo_clone_list, book_codes)
+ except Exception:
+ logger.exception("Failed due to the following exception.")
+ unique_values = unique_tuples(resource_types_)
return sorted(unique_values, key=lambda value: value[1])
+# We found that there are fewer downloads available than clonable repos,
+# so we don't currently have the system configured to use this.
+def batch_download_repos(
+ repos: list[tuple[HttpUrl, str]],
+ asset_caching_enabled: bool = settings.ASSET_CACHING_ENABLED,
+ asset_caching_period: int = settings.ASSET_CACHING_PERIOD,
+ base_url: HttpUrl = HttpUrl("https://content.bibletranslationtools.org/"),
+ base_url_replacement: HttpUrl = HttpUrl(
+ "https://content.bibletranslationtools.org/api/v1/repos/"
+ ),
+ resource_assets_dir: str = settings.RESOURCE_ASSETS_DIR,
+ user_agent_str: str = USER_AGENT_STR,
+ x_requested_with_value: str = X_REQUESTED_WITH_VALUE,
+) -> None:
+ """Batch download repos and then batch unzip repos."""
+ download_commands = []
+ zip_file_paths = []
+ for url, resource_filepath in repos:
+ zip_file_path = Path(f"{resource_filepath}.zip")
+ if isdir(resource_filepath):
+ # Instead of checking the directory which was created from
+ # a zip with timestamps current when the zip file was created (which
+ # would likely be quite old), we instead check the zip file itself.
+ if file_needs_update(zip_file_path):
+ logger.info(
+ f"Removing stale, incomplete, or corrupt repository: {resource_filepath} and zip file: {zip_file_path}"
+ )
+ if zip_file_path.is_file():
+ zip_file_path.unlink()
+ delete_tree(resource_filepath)
+ else:
+ logger.info(f"Skipping download: {resource_filepath} already exists.")
+ continue
+ zip_url_base = re.sub(str(base_url), str(base_url_replacement), str(url))
+ zip_url = f"{zip_url_base}/archive/master.zip"
+ download_commands.append(
+ f"curl -A {user_agent_str} -H 'X-Requested-With: {x_requested_with_value}' -X 'GET' {zip_url} -H 'accept: application/json' --output {zip_file_path} --parallel"
+ )
+ zip_file_paths.append(zip_file_path)
+ download_command = " && ".join(download_commands)
+ logger.info(f"Downloading repos with command: {download_command}")
+ try:
+ subprocess.check_call(download_command, shell=True)
+ unzip_commands = [
+ f"unzip -q -o {zip_file_path} -d {resource_assets_dir}"
+ for zip_file_path in zip_file_paths
+ ]
+ unzip_command = " && ".join(unzip_commands)
+ logger.info(f"Unzipping downloaded files with command: {unzip_command}")
+ subprocess.check_call(unzip_command, shell=True)
+ except subprocess.CalledProcessError:
+ logger.error("Batch download or unzip failed!")
+ raise HTTPException(
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ detail="Download of repo master.zip failed",
+ )
+
+
def batch_clone_git_repos(
- repos: list[tuple[str, str]],
+ repos: list[tuple[HttpUrl, str]],
asset_caching_enabled: bool = settings.ASSET_CACHING_ENABLED,
+ asset_caching_period: int = settings.ASSET_CACHING_PERIOD,
+ user_agent_str: str = USER_AGENT_STR,
+ x_requested_with_value: str = X_REQUESTED_WITH_VALUE,
) -> None:
"""
Clones multiple git repositories in a single batch operation.
- - If a repository already exists and is fully cloned, it is skipped.
- - If a repository exists but is a partial clone (corrupt or missing key files), it is removed first.
- - The 'en_rg' directory is preserved and never deleted or cloned.
- - If asset_caching_enabled is False, repositories are always deleted and re-cloned (except 'en_rg').
+ - If a repository already exists, is fully cloned, and not stale (with respect to cache period), it is skipped.
+ Conversely, if a repository is fully cloned, but stale then it is removed before (re)cloning.
+ - If a repository exists but is a partial clone (corrupt or missing key files), it is removed before (re)cloning.
+ - If asset_caching_enabled is False, repositories are always removed and re-cloned.
"""
clone_commands = []
for url, resource_filepath in repos:
- if isdir(resource_filepath):
- if basename(resource_filepath) == "en_rg":
- logger.info(f"Preserving special directory: {resource_filepath}")
- continue
- git_dir = join(resource_filepath, ".git")
- if asset_caching_enabled:
- if isdir(git_dir):
- if all(
+ if asset_caching_enabled:
+ try:
+ git_dir = join(resource_filepath, ".git")
+ stat_ = stat(git_dir)
+ mod_time = datetime.fromtimestamp(stat_.st_mtime)
+ expiry = timedelta(minutes=asset_caching_period)
+ if (
+ all(
exists(join(git_dir, filename))
for filename in ["config", "HEAD", "objects"]
- ) and any(scandir(resource_filepath)):
- logger.info(
- f"Skipping clone: {resource_filepath} already exists and is a full repo."
- )
- continue # ✅ Fully cloned, reuse
- logger.warning(
- f"Removing incomplete or corrupt repository: {resource_filepath}"
- )
- else:
- logger.info(
- f"Asset caching disabled: forcibly removing {resource_filepath}"
- )
+ )
+ and any(scandir(resource_filepath))
+ and datetime.now() - mod_time <= expiry
+ ):
+ logger.info(
+ f"Skipping clone: {resource_filepath} already exists, is a valid git repo, and is not stale."
+ )
+ continue # ✅ Fully cloned and not stale, reuse
+ except FileNotFoundError:
+ logger.warning(f"Git directory, {git_dir}, not found")
+ logger.info(
+ f"Removing stale, incomplete, or corrupt repository: {resource_filepath}"
+ )
+ else:
+ logger.info(
+ f"Asset caching disabled: forcibly removing {resource_filepath}"
+ )
+ if isdir(resource_filepath):
shutil.rmtree(resource_filepath)
- clone_command = f"git clone --depth=1 '{url}' '{resource_filepath}' || true"
+ clone_command = (
+ f"git -c http.userAgent='{user_agent_str}' "
+ f"-c http.extraHeader='X-Requested-With:{x_requested_with_value}' "
+ f"clone --depth=1 --single-branch '{url}' '{resource_filepath}' || true"
+ )
clone_commands.append(clone_command)
if clone_commands:
full_command = " && ".join(clone_commands)
@@ -642,7 +611,6 @@ def batch_clone_git_repos(
# Used by some tests
-@lru_cache(maxsize=100)
def usfm_resource_types_and_book_tuples(
lang_code: str,
book_codes_str: str,
@@ -660,58 +628,60 @@ def usfm_resource_types_and_book_tuples(
[('reg', '1co'), ('reg', '1jn'), ('reg', '1pe'), ('reg', '1th'), ('reg', '1ti'), ('reg', '2co'), ('reg', '2jn'), ('reg', '2pe'), ('reg', '2th'), ('reg', '2ti'), ('reg', '3jn'), ('reg', 'act'), ('reg', 'col'), ('reg', 'eph'), ('reg', 'gal'), ('reg', 'heb'), ('reg', 'jas'), ('reg', 'jhn'), ('reg', 'jud'), ('reg', 'luk'), ('reg', 'mat'), ('reg', 'mrk'), ('reg', 'phm'), ('reg', 'php'), ('reg', 'rev'), ('reg', 'rom'), ('reg', 'tit')]
"""
book_codes = book_codes_str.split(",")
- data = fetch_source_data()
+ data: SourceData | None = fetch_source_data()
resource_type_and_book_tuples = set()
- try:
- repos_info = data["git_repo"]
- augmented_repos_info = add_data_not_supplied_by_data_api(repos_info)
- for repo_info in augmented_repos_info:
- content = repo_info["content"]
- language_info = content["language"]
- if language_info["ietf_code"] == lang_code:
- resource_type = content["resource_type"]
- if resource_type in usfm_resource_types:
- url = repo_info["repo_url"]
- for book_code in book_codes:
- dto = ResourceLookupDto(
- lang_code=lang_code,
- lang_name="",
- resource_type=resource_type,
- resource_type_name="",
- url=url,
- lang_direction=LangDirEnum.LTR,
- book_code=book_code,
- )
- # logger.debug("dto: %s", dto)
- resource_filepath = prepare_resource_filepath(dto)
- if file_needs_update(resource_filepath):
- provision_asset_files(dto.url, resource_filepath)
- content_file = parsing.usfm_asset_file(
- dto,
- resource_filepath,
- False,
- )
- # logger.debug("content_file: %s", content_file)
- if content_file:
- resource_type_and_book_tuples.add(
- (resource_type, book_code)
- )
- except:
- pass
+ if data is None:
+ return []
+ repos_info = data.git_repo
+ augmented_repos_info = add_data_not_supplied_by_data_api(repos_info)
+ for repo_info in augmented_repos_info:
+ content = repo_info.content
+ language_info = content.language
+ if language_info.ietf_code == lang_code:
+ resource_type = content.resource_type
+ if resource_type in usfm_resource_types:
+ url = repo_info.repo_url
+ for book_code in book_codes:
+ dto = ResourceLookupDto(
+ lang_code=lang_code,
+ lang_name=language_info.english_name,
+ localized_lang_name=language_info.national_name,
+ resource_type=resource_type,
+ resource_type_name="",
+ url=url,
+ lang_direction=LangDirEnum(language_info.direction),
+ book_code=book_code,
+ )
+ resource_filepath = prepare_resource_filepath(dto)
+ if file_needs_update(resource_filepath):
+ provision_asset_files(dto.url, resource_filepath)
+ content_file = parsing.usfm_asset_file(
+ dto, resource_filepath, False
+ )
+ if content_file:
+ resource_type_and_book_tuples.add((resource_type, book_code))
return sorted(resource_type_and_book_tuples, key=lambda value: value[0])
def shared_book_codes(lang0_code: str, lang1_code: str) -> Sequence[tuple[str, str]]:
"""
- Given two language codes, return the intersection of resource
+ Given two language codes, return the intersection of book
codes between the two languages.
>>> from doc.domain import resource_lookup
>>> # Hack to ignore logging output: https://stackoverflow.com/a/33400983/3034580
+ >>> ();data = resource_lookup.book_codes_for_lang("pt-br");() # doctest: +ELLIPSIS
+ (...)
+ >>> list(data)
+ [('gen', 'Gênesis'), ('exo', 'Êxodo'), ('lev', 'Levítico'), ('num', 'Números'), ('deu', 'Deuteronômio'), ('jos', 'Josué'), ('jdg', 'Juízes'), ('rut', 'Rute'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reis'), ('2ki', '2 Reis'), ('1ch', '1 Crônicas'), ('2ch', '2 Crônicas'), ('ezr', 'Esdras'), ('neh', 'Neemias'), ('est', 'Ester'), ('job', 'Jó'), ('psa', 'Salmos'), ('pro', 'Provérbios'), ('ecc', 'Eclesiastes'), ('sng', 'Cantares'), ('isa', 'Isaías'), ('jer', 'Jeremias'), ('lam', 'Lamentações'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseias'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Obadias'), ('jon', 'Jonas'), ('mic', 'Miqueias'), ('nam', 'Naum'), ('hab', 'Habacuque'), ('zep', 'Sofonias'), ('hag', 'Ageu'), ('zec', 'Zacarias'), ('mal', 'Malaquias'), ('mat', 'Mateus'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'João'), ('act', 'Atos'), ('rom', 'Romanos'), ('1co', '1 Coríntios'), ('2co', '2 Coríntios'), ('gal', 'Gálatas'), ('eph', 'Efésios'), ('php', 'Filipenses'), ('col', 'Colossenses'), ('1th', '1 Tessalonicenses'), ('2th', '2 Tessalonicenses'), ('1ti', '1 Timóteo'), ('2ti', '2 Timóteo'), ('tit', 'Tito'), ('phm', 'Filemom'), ('heb', 'Hebreus'), ('jas', 'Tiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 João'), ('2jn', '2 João'), ('3jn', '3 João'), ('jud', 'Judas'), ('rev', 'Apocalipse')]
+ >>> ();data = resource_lookup.book_codes_for_lang("es-419");() # doctest: +ELLIPSIS
+ (...)
+ >>> list(data)
+ [('gen', 'Génesis'), ('exo', 'Éxodo'), ('lev', 'Levítico'), ('num', 'Números'), ('deu', 'Deuteronomio'), ('jos', 'Josué'), ('jdg', 'Jueces'), ('rut', 'Ruth'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reyes'), ('2ki', '2 Reyes'), ('1ch', '1 Crónicas'), ('2ch', '2 Crónicas'), ('ezr', 'Esdras'), ('neh', 'Nehemías'), ('est', 'Ester'), ('job', 'Job'), ('psa', 'Salmos'), ('pro', 'Proverbios'), ('ecc', 'Eclesiastés'), ('sng', 'Cántico de Salomón'), ('isa', 'Isaías'), ('jer', 'Jeremías'), ('lam', 'Lamentaciones'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseas'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Abdías'), ('jon', 'Jonás'), ('mic', 'Miqueas'), ('nam', 'Nahúm'), ('hab', 'Habacuc'), ('zep', 'Sofonías'), ('hag', 'Hageo'), ('zec', 'Zacarías'), ('mal', 'Malaquías'), ('mat', 'Mateo'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'Juan'), ('act', 'Hechos'), ('rom', 'Romanos'), ('1co', '1 Corintios'), ('2co', '2 Corintios'), ('gal', 'Gálatas'), ('eph', 'Efesios'), ('php', 'Filipenses'), ('col', 'Colosenses'), ('1th', '1 Tesalonicenses'), ('2th', '2 Tesalonicenses'), ('1ti', '1 Timoteo'), ('2ti', '2 Timoteo'), ('tit', 'Tito'), ('phm', 'Filemón'), ('heb', 'Hebreos'), ('jas', 'Santiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 Juan'), ('2jn', '2 Juan'), ('3jn', '3 Juan'), ('jud', 'Judas'), ('rev', 'Apocalipsis')]
>>> ();data = resource_lookup.shared_book_codes("pt-br", "es-419");() # doctest: +ELLIPSIS
(...)
>>> list(data)
- [('gen', 'Gênesis'), ('exo', 'Êxodo'), ('lev', 'Levíticos'), ('num', 'Números'), ('deu', 'Deuteronômio'), ('jos', 'Josué'), ('jdg', 'Juízes'), ('rut', 'Rute'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reis'), ('2ki', '2 Reis'), ('1ch', '1 Crônicas'), ('2ch', '2 Crônicas'), ('ezr', 'Esdras'), ('neh', 'Neemias'), ('est', 'Ester'), ('job', 'Jó'), ('psa', 'Salmos'), ('pro', 'Provérbios'), ('ecc', 'Eclesiastes'), ('sng', 'Cantares de salomão'), ('isa', 'Isaías'), ('jer', 'Jeremias'), ('lam', 'Lamentações'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseias'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Obadias'), ('jon', 'Jonas'), ('mic', 'Miqueias'), ('nam', 'Naum'), ('hab', 'Habacuque'), ('zep', 'Sofonias'), ('hag', 'Ageu'), ('zec', 'Zacarias'), ('mal', 'Malaquias'), ('mat', 'Mateus'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'João'), ('act', 'Atos'), ('rom', 'Romanos'), ('1co', '1 Corintios'), ('2co', '2 Coríntios'), ('gal', 'Gálatas'), ('eph', 'Efésios'), ('php', 'Filipenses'), ('col', 'Colossenses'), ('1th', '1 Tessalonicenses'), ('2th', '2 Tessalonicenses'), ('1ti', '1 Timóteo'), ('2ti', '2 Timóteo'), ('tit', 'Tito'), ('phm', 'Filemom'), ('heb', 'Hebreus'), ('jas', 'Tiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 João'), ('2jn', '2 João'), ('3jn', '3 João'), ('jud', 'Judas'), ('rev', 'Apocalipse')]
+ [('gen', 'Gênesis'), ('exo', 'Êxodo'), ('lev', 'Levítico'), ('num', 'Números'), ('deu', 'Deuteronômio'), ('jos', 'Josué'), ('jdg', 'Juízes'), ('rut', 'Rute'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reis'), ('2ki', '2 Reis'), ('1ch', '1 Crônicas'), ('2ch', '2 Crônicas'), ('ezr', 'Esdras'), ('neh', 'Neemias'), ('est', 'Ester'), ('job', 'Jó'), ('psa', 'Salmos'), ('pro', 'Provérbios'), ('ecc', 'Eclesiastes'), ('sng', 'Cantares'), ('isa', 'Isaías'), ('jer', 'Jeremias'), ('lam', 'Lamentações'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseias'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Obadias'), ('jon', 'Jonas'), ('mic', 'Miqueias'), ('nam', 'Naum'), ('hab', 'Habacuque'), ('zep', 'Sofonias'), ('hag', 'Ageu'), ('zec', 'Zacarias'), ('mal', 'Malaquias'), ('mat', 'Mateus'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'João'), ('act', 'Atos'), ('rom', 'Romanos'), ('1co', '1 Coríntios'), ('2co', '2 Coríntios'), ('gal', 'Gálatas'), ('eph', 'Efésios'), ('php', 'Filipenses'), ('col', 'Colossenses'), ('1th', '1 Tessalonicenses'), ('2th', '2 Tessalonicenses'), ('1ti', '1 Timóteo'), ('2ti', '2 Timóteo'), ('tit', 'Tito'), ('phm', 'Filemom'), ('heb', 'Hebreus'), ('jas', 'Tiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 João'), ('2jn', '2 João'), ('3jn', '3 João'), ('jud', 'Judas'), ('rev', 'Apocalipse')]
"""
lang0_book_codes = book_codes_for_lang(lang0_code)
@@ -722,215 +692,6 @@ def shared_book_codes(lang0_code: str, lang1_code: str) -> Sequence[tuple[str, s
]
-def get_last_segment(url: str, lang_code: str) -> str:
- """
- Handle special cases where git repo URL does not follow the expected pattern.
- Ideally these repos URLs would have their last segment renamed
- properly, e.g.,
- 'https://content.bibletranslationtools.org/faustin_azaza/faustin_azaza'
- renamed to
- 'https://content.bibletranslationtools.org/faustin_azaza/zmq_mrk_text_reg',
- but since we don't have control over that, we handle these anomalies
- here.
- """
- parsed_url = urlparse(url)
- path_segments = parsed_url.path.split("/")
- last_segment = path_segments[-1]
- if lang_code == "zmq" and last_segment == "faustin_azaza":
- last_segment = "zmq_mrk_text_reg"
- elif lang_code == "my" and last_segment == "my_juds":
- last_segment = "my_ulb"
- elif lang_code == "fa" and last_segment == "fa_opv":
- last_segment = "fa_ulb"
- # This next one is just a special case for the NT Survery Reviewer's Guide
- elif lang_code == "en" and last_segment[-4:] == "docx":
- last_segment = "en_rg"
- elif last_segment.startswith(
- "parfait-ayanou_"
- ): # aba and abu languages (and maybe others)
- last_segment = re.sub("parfait-ayanou_", "", last_segment)
- elif last_segment.startswith("faustin-azaza_"):
- last_segment = re.sub("faustin-azaza_", "", last_segment)
- elif last_segment.startswith("azz_athan_"):
- last_segment = re.sub("azz_athan_", "", last_segment)
- elif last_segment.startswith("burje_duro_"):
- last_segment = re.sub("burje_duro_", "", last_segment)
- elif last_segment.startswith("Dawit-Dessie_"):
- last_segment = re.sub("Dawit-Dessie_", "", last_segment)
- elif last_segment.startswith("jdwood_"):
- last_segment = re.sub("jdwood_", "", last_segment)
- elif last_segment.startswith("otlaadisa_"):
- last_segment = re.sub("otlaadisa_", "", last_segment)
- elif last_segment.startswith("romantts2_"):
- last_segment = re.sub("romantts2_", "", last_segment)
- elif lang_code == "ndh" and last_segment.startswith("chindali_"):
- last_segment = re.sub("chindali_", "", last_segment)
- elif lang_code == "knx-x-bajanya" and last_segment.startswith("lawadinusah_"):
- last_segment = re.sub("lawadinusah_", "", last_segment)
- elif lang_code == "knx-x-bajanya" and last_segment.startswith("bajanya_knx"):
- last_segment = re.sub(r"^bajanya_", "", last_segment)
- elif lang_code == "scg-x-dayakkatarak" and last_segment.startswith("yustius_"):
- last_segment = re.sub(r"^yustius_", "", last_segment)
- elif lang_code == "iba-x-ketungau" and last_segment.startswith("dayakketungau_"):
- last_segment = re.sub(r"^dayakketungau_", "", last_segment)
- elif lang_code == "iba-x-ketungau" and last_segment.startswith("Lawadinusah_"):
- last_segment = re.sub(r"^Lawadinusah_", "", last_segment)
- elif last_segment.startswith("Lawadinusah_"): # lang_code == "xdy-x-mentebah" and
- last_segment = re.sub(r"^Lawadinusah_", "", last_segment)
- elif lang_code == "xdy-x-mentebah" and last_segment.startswith("dayakfdkj_"):
- last_segment = re.sub(r"^dayakfdkj_", "", last_segment)
- elif lang_code == "sdm-x-pangkalsuka" and last_segment.startswith("dayaksuka_"):
- last_segment = re.sub(r"^dayaksuka_", "", last_segment)
- elif lang_code == "xdy-x-dayakpunti" and last_segment.startswith("anselmus_"):
- last_segment = re.sub(r"^anselmus_", "", last_segment)
- elif lang_code == "xdy-x-senduruhan" and last_segment.startswith(
- "dayaksenduruhan_"
- ):
- last_segment = re.sub(r"^dayaksenduruhan_", "", last_segment)
- elif last_segment.startswith("dijim1_"):
- last_segment = re.sub(r"^dijim1_", "", last_segment)
- elif last_segment.startswith("nbtt_"):
- last_segment = re.sub(r"^nbtt_", "", last_segment)
- elif last_segment.startswith("ezekieldabere_"):
- last_segment = re.sub(r"^ezekieldabere_", "", last_segment)
- elif last_segment.startswith("krispy_"):
- last_segment = re.sub(r"^krispy_", "", last_segment)
- elif last_segment.startswith("jks222111_"):
- last_segment = re.sub(r"^jks222111_", "", last_segment)
- elif last_segment.startswith("mushohe-25nb_63.kum_"):
- last_segment = re.sub(r"^mushohe-25nb_63.kum_", "", last_segment)
- elif last_segment.startswith("botsw01_"):
- last_segment = re.sub(r"^botsw01_", "", last_segment)
- elif last_segment.startswith("gravy_"):
- last_segment = re.sub(r"^gravy_", "", last_segment)
- elif last_segment.startswith("tom-88pn_0003.machinga_"):
- last_segment = re.sub(r"^tom-88pn_0003.machinga_", "", last_segment)
- elif last_segment.startswith("jonathan_"):
- last_segment = re.sub(r"^jonathan_", "", last_segment)
- elif last_segment.startswith("lversaw_"):
- last_segment = re.sub(r"^lversaw_", "", last_segment)
- elif last_segment.startswith("alexandre_brazil_"):
- last_segment = re.sub(r"^alexandre_brazil_", "", last_segment)
- elif last_segment.startswith("jathapu_"):
- last_segment = re.sub(r"^jathapu_", "", last_segment)
- elif last_segment.startswith("translator09_"):
- last_segment = re.sub(r"^translator09_", "", last_segment)
- elif last_segment.startswith("ngamo_"):
- last_segment = re.sub(r"^ngamo_", "", last_segment)
- elif last_segment.startswith("danjuma_alfred_h_"):
- last_segment = re.sub(r"^danjuma_alfred_h_", "", last_segment)
- elif last_segment.startswith("ngamo1_"):
- last_segment = re.sub(r"^ngamo1_", "", last_segment)
- elif last_segment.startswith("bayan_"):
- last_segment = re.sub(r"^bayan_", "", last_segment)
- elif last_segment.startswith("oratab01_"):
- last_segment = re.sub(r"^oratab01_", "", last_segment)
- elif last_segment.startswith("Jordan_"):
- last_segment = re.sub(r"^Jordan_", "", last_segment)
- elif last_segment.startswith("mvccbtt_"):
- last_segment = re.sub(r"^mvccbtt_", "", last_segment)
- elif last_segment.startswith("sambadanum_"):
- last_segment = re.sub(r"^sambadanum_", "", last_segment)
- elif last_segment.startswith("shyarpa_"):
- last_segment = re.sub(r"^shyarpa_", "", last_segment)
- elif last_segment.startswith("mitikiwostky_"):
- last_segment = re.sub(r"^mitikiwostky_", "", last_segment)
- elif last_segment.startswith("michael_"):
- last_segment = re.sub(r"^michael_", "", last_segment)
- elif last_segment.startswith("timothydanjuma_"):
- last_segment = re.sub(r"^timothydanjuma_", "", last_segment)
- elif last_segment.startswith("ukum1_"):
- last_segment = re.sub(r"^ukum1_", "", last_segment)
- elif last_segment.startswith("vere3_"):
- last_segment = re.sub(r"^vere3_", "", last_segment)
- elif last_segment.startswith("yukuben1_"):
- last_segment = re.sub(r"^yukuben1_", "", last_segment)
- elif last_segment.startswith("tersitzewde_"):
- last_segment = re.sub(r"^tersitzewde_", "", last_segment)
- elif last_segment.startswith("moufida_"):
- last_segment = re.sub(r"^moufida_", "", last_segment)
- elif last_segment.startswith("billburns58_"):
- last_segment = re.sub(r"^billburns58_", "", last_segment)
-
- # Incomplete database of repo related issues still yet to be resolved:
- #
- # FIXME Cloning into 'assets_download/bji_1pe_text_reg'...
- # Username for 'https://content.bibletranslationtools.org':
- # Password for 'https://content.bibletranslationtools.org':
- # There appears to be an issue with authentication being required for bji_1pe
- # I emailed Craig about it today, 4/8/25
- #
- # FIXME Cloning into 'assets_download/igw-x-sale_tit_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/Bayan/igw-x-sale_tit_text_reg/': The requested URL returned error: 500
-
- # FIXME fatal: destination path 'assets_download/shr-x-hwindja_2co_text_reg' already exists and is not an empty directory.
-
- # FIXME Cloning into 'assets_download/isn_mat_text_reg'...
- # error: RPC failed; HTTP 500 curl 22 The requested URL returned error: 500
- # fatal: expected 'packfile'
- # Cloning into 'assets_download/isn_rom_text_reg'...
- # error: RPC failed; HTTP 500 curl 22 The requested URL returned error: 500
-
- # FIXME fatal: unable to access 'https://content.bibletranslationtools.org/Elton_cv/kea_ezr_text_ulb/': The requested URL returned error: 500
- # Cloning into 'assets_download/kea_2co_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/Elton_cv/kea_2co_text_reg/': The requested URL returned error: 500
- # Cloning into 'assets_download/kea_luk_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/Elton_cv/kea_luk_text_reg/': The requested URL returned error: 500
- # Cloning into 'assets_download/kea_gal_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/Elton_cv/kea_gal_text_reg/': The requested URL returned error: 500
- # Cloning into 'assets_download/kea_2pe_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/Elton_cv/kea_2pe_text_reg/': The requested URL returned error: 500
-
- # FIXME Cloning into 'assets_download/jka_1pe_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/bahasatech.indotengah/jka_1pe_text_reg/': The requested URL returned error: 500
-
- # FIXME Cloning into 'assets_download/kdp_3jn_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/DCS-Mirror/nbtt_kdp_3jn_text_reg/': The requested URL returned error: 500
-
- # FIXME Cloning into 'assets_download/kdp_act_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/DCS-Mirror/nbtt_kdp_act_text_reg/': The requested URL returned error: 500
- # Cloning into 'assets_download/kdp_tit_text_reg'...
- # error: RPC failed; HTTP 500 curl 22 The requested URL returned error: 500
- # Cloning into 'assets_download/kdp_2ti_text_reg'...
- # error: RPC failed; HTTP 500 curl 22 The requested URL returned error: 500
- # Cloning into 'assets_download/kdp_php_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/DCS-Mirror/nbtt_kdp_php_text_reg/': The requested URL returned error: 500
-
- # FIXME Cloning into 'assets_download/gqa-x-kabinda_phm_text_ulb'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/nbtt/gqa-x-kabinda_phm_text_ulb/': The requested URL returned error: 500
- # Cloning into 'assets_download/gqa-x-kabinda_2co_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/nbtt/gqa-x-kabinda_2co_text_reg/': The requested URL returned error: 500
- # Cloning into 'assets_download/gqa-x-kabinda_gal_text_ulb'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/nbtt/gqa-x-kabinda_gal_text_ulb/': The requested URL returned error: 500
- # Cloning into 'assets_download/gqa-x-kabinda_tit_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/nbtt/gqa-x-kabinda_tit_text_reg/': The requested URL returned error: 500
- # Cloning into 'assets_download/gqa-x-kabinda_jud_text_ulb'...
- # error: RPC failed; HTTP 500 curl 22 The requested URL returned error: 500
-
- # FIXME Cloning into 'assets_download/kdy_mrk_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/Keijerkeider.Indotimur/kdy_mrk_text_reg/': The requested URL returned error: 500
- # Cloning into 'assets_download/kdy_2th_text_reg'...
- # error: RPC failed; HTTP 500 curl 22 The requested URL returned error: 500
- # Cloning into 'assets_download/kdy_3jn_text_reg'...
- # error: RPC failed; HTTP 500 curl 22 The requested URL returned error: 500
- # Cloning into 'assets_download/kdy_col_text_reg'...
- # fatal: unable to access 'https://content.bibletranslationtools.org/Keijerkeider.Indotimur/kdy_col_text_reg/': The requested URL returned error: 500
-
- # FIXME Cloning into 'assets_download/kkq-x-kikubere_jhn_text_reg'...
- # Username for 'https://content.bibletranslationtools.org':
- # Password for 'https://content.bibletranslationtools.org':
-
- # FIXME Cloning into 'assets_download/mgv_1jn_text_ulb'...
- # Username for 'https://content.bibletranslationtools.org':
- # Password for 'https://content.bibletranslationtools.org':
-
- # FIXME Cloning into 'assets_download/mgv_1th_text_ulb'...
- # Username for 'https://content.bibletranslationtools.org':
- # Password for 'https://content.bibletranslationtools.org':
-
- return last_segment
-
-
def update_repo_components(
repo_components: list[str],
usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
@@ -961,81 +722,71 @@ def update_repo_components(
return repo_components
-def add_data_not_supplied_by_data_api(repos_info: Any) -> Any:
+def add_data_not_supplied_by_data_api(repos_info: list[RepoEntry]) -> list[RepoEntry]:
"""
DOC needs to support some resources which are not supplied by the
data API so we augment the data returned from the data API to include
- them here.
+ them here. If a (language code, resource type) pair already exists in
+ repos_info, do not add it again.
"""
- # The data API only provides id_tn repo for id, we have to
- # add the other repos for id that are available for DOC's use.
- id_ayt = {
- "repo_url": "https://content.bibletranslationtools.org/WA-Catalog/id_ayt",
- "content": {
- "resource_type": "ayt",
- "language": {
- "english_name": "Indonesian",
- "ietf_code": "id",
- "national_name": "Bahasa Indonesian",
- "direction": "ltr",
- },
- },
- }
- repos_info.append(id_ayt)
- id_tq = {
- "repo_url": "https://content.bibletranslationtools.org/WA-Catalog/id_tq",
- "content": {
- "resource_type": "tq",
- "language": {
- "english_name": "Indonesian",
- "ietf_code": "id",
- "national_name": "Bahasa Indonesian",
- "direction": "ltr",
- },
- },
- }
- repos_info.append(id_tq)
- id_tw = {
- "repo_url": "https://content.bibletranslationtools.org/WA-Catalog/id_tw",
- "content": {
- "resource_type": "tw",
- "language": {
- "english_name": "Indonesian",
- "ietf_code": "id",
- "national_name": "Bahasa Indonesian",
- "direction": "ltr",
- },
- },
- }
- repos_info.append(id_tw)
- # data API does not provide tn_condensed for en, but DOC needs to support it
- en_tn_condensed = {
- "repo_url": "https://content.bibletranslationtools.org/WycliffeAssociates/en_tn_condensed",
- "content": {
- "resource_type": "tn-condensed",
- "language": {
- "english_name": "English",
- "ietf_code": "en",
- "national_name": "English",
- "direction": "ltr",
- },
- },
- }
- repos_info.append(en_tn_condensed)
- # data API does not provide rg for en, but DOC needs to support it
- en_rg = {
- "repo_url": "https://github.com/WycliffeAssociates/TS-biel-files/blob/master/training/en/Refinement%20and%20Publication/Reviewers'%20Guide/NT%20Survey%20RG%20Files/NT%20Survey%20Reviewers'%20Guide.docx",
- "content": {
- "resource_type": "rg",
- "language": {
- "english_name": "English",
- "ietf_code": "en",
- "national_name": "English",
- "direction": "ltr",
- },
- },
+
+ def make_entry(url: HttpUrl, resource_type: str, lang: Language) -> RepoEntry:
+ return RepoEntry(
+ repo_url=url, content=Content(resource_type=resource_type, language=lang)
+ )
+
+ id_lang = Language(
+ english_name="Indonesian",
+ ietf_code="id",
+ national_name="Bahasa Indonesian",
+ direction=LangDirEnum.LTR,
+ )
+ en_lang = Language(
+ english_name="English",
+ ietf_code="en",
+ national_name="English",
+ direction=LangDirEnum.LTR,
+ )
+ extra_entries = [
+ make_entry(
+ HttpUrl("https://content.bibletranslationtools.org/WA-Catalog/id_ayt"),
+ "ayt",
+ id_lang,
+ ),
+ make_entry(
+ HttpUrl("https://content.bibletranslationtools.org/WA-Catalog/id_tq"),
+ "tq",
+ id_lang,
+ ),
+ make_entry(
+ HttpUrl("https://content.bibletranslationtools.org/WA-Catalog/id_tw"),
+ "tw",
+ id_lang,
+ ),
+ make_entry(
+ HttpUrl(
+ "https://content.bibletranslationtools.org/WycliffeAssociates/en_tn_condensed"
+ ),
+ "tn-condensed",
+ en_lang,
+ ),
+ make_entry(
+ HttpUrl(
+ "https://github.com/WycliffeAssociates/TS-biel-files/blob/master/training/en/Refinement%20and%20Publication/Reviewers'%20Guide/NT%20Survey%20RG%20Files/NT%20Survey%20Reviewers'%20Guide.docx"
+ ),
+ "rg",
+ en_lang,
+ ),
+ ]
+ existing_pairs = {
+ (entry.content.language.ietf_code, entry.content.resource_type)
+ for entry in repos_info
}
- repos_info.append(en_rg)
+ for entry in extra_entries:
+ key = (entry.content.language.ietf_code, entry.content.resource_type)
+ if key not in existing_pairs:
+ repos_info.append(entry)
+ existing_pairs.add(key)
return repos_info
@@ -1055,183 +806,237 @@ def maybe_correct_book_name(
def get_book_codes_for_lang(
lang_code: str,
- resource_assets_dir: str,
- book_names: Mapping[str, str],
- dcs_mirror_git_username: str,
- usfm_resource_types: Sequence[str],
- use_localized_book_name: bool,
usfm_only: bool = False,
+ download_assets: bool = settings.DOWNLOAD_ASSETS,
) -> Sequence[tuple[str, str]]:
data = fetch_source_data()
- book_codes_and_names_localized: list[tuple[str, str]] = []
- book_codes_and_names = []
- book_codes_and_names2: list[tuple[str, str]] = []
- repo_clone_list = []
+ if data is None:
+ return []
+ repo_clone_list: list[tuple[HttpUrl, str, str]] = []
+ book_codes_and_names: list[tuple[str, str]] = []
try:
- repos_info = data["git_repo"]
+ repos_info = data.git_repo
augmented_repos_info = add_data_not_supplied_by_data_api(repos_info)
- for repo_info in augmented_repos_info:
- content = repo_info["content"]
- language_info = content["language"]
- url = repo_info["repo_url"]
- if language_info["ietf_code"] == lang_code:
- last_segment = get_last_segment(url, lang_code)
- repo_components = last_segment.split("_")
- if dcs_mirror_git_username in url:
- repo_components = update_repo_components(repo_components)
- if any(
- usfm_resource_type in url
- for usfm_resource_type in usfm_resource_types
- ):
- resource_filepath = f"{resource_assets_dir}/{last_segment}"
- repo_clone_list.append((url, resource_filepath))
- repo_clone_list = list(set(repo_clone_list))
- repos_to_clone = [
- (url, path) for url, path in repo_clone_list if "en_rg" not in path
+ repo_clone_list = repos_to_clone(lang_code, augmented_repos_info)
+ repos_to_clone_ = [
+ (url, path)
+ for url, path, resource_type in repo_clone_list
+ if "en_rg" not in path
]
- # Perform batch cloning
- batch_clone_git_repos(repos_to_clone)
- # Process cloned repositories
- for url, resource_filepath in repo_clone_list:
- repo_info = next(
- (repo for repo in augmented_repos_info if repo["repo_url"] == url),
- None,
+ if download_assets:
+ batch_download_repos(repos_to_clone_)
+ else:
+ batch_clone_git_repos(repos_to_clone_)
+ book_codes_and_names = get_book_codes_for_lang_(
+ repo_clone_list,
+ lang_code,
+ usfm_only,
+ )
+ except Exception:
+ logger.exception("Error during get_book_codes_for_lang")
+ return book_codes_and_names
+
+
+def get_book_codes_for_lang_(
+ repo_clone_list: list[tuple[HttpUrl, str, str]],
+ lang_code: str,
+ usfm_only: bool,
+ book_names: Mapping[str, str] = BOOK_NAMES,
+ usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
+ use_localized_book_name: bool = settings.USE_LOCALIZED_BOOK_NAME,
+ book_id_map: dict[str, int] = BOOK_ID_MAP,
+) -> list[tuple[str, str]]:
+ book_codes_and_names_localized: list[tuple[str, str]] = []
+ book_codes_and_names: list[tuple[str, str]] = []
+ for url, resource_filepath, resource_type in repo_clone_list:
+ last_segment = get_last_segment(url, lang_code)
+ repo_components = last_segment.split("_")
+ if (
+ use_localized_book_name
+ and len(repo_components) == 2
+ and resource_type in usfm_resource_types
+ ):
+ book_codes_and_names_localized_from_metadata = (
+ get_book_names_from_usfm_metadata(
+ resource_filepath,
+ lang_code,
+ resource_type,
+ )
)
- if not repo_info:
- continue
- last_segment = get_last_segment(url, lang_code)
- logger.debug("last_segment: %s", last_segment)
- repo_components = last_segment.split("_")
- if len(repo_components) == 2 and repo_components[-1] in usfm_resource_types:
- book_codes_and_names_localized = []
- usfm_files = parsing.find_usfm_files(resource_filepath)
- for usfm_file in usfm_files:
- usfm_file_components = Path(usfm_file).stem.lower().split("-")
- book_code = usfm_file_components[1]
- resource_type = repo_components[1]
- content = read_file(usfm_file) if usfm_file else ""
- frontmatter, _, _ = parsing.split_usfm_by_chapters(
- lang_code, resource_type, book_code, content
- )
- localized_book_name = parsing.maybe_localized_book_name(frontmatter)
- localized_book_name = maybe_correct_book_name(
- lang_code, localized_book_name
- )
+ book_codes_and_names_localized_from_manifest = (
+ book_codes_and_names_from_manifest(resource_filepath)
+ )
+ logger.debug(
+ "book_codes_and_names_localized_from_metadata: %s",
+ book_codes_and_names_localized_from_metadata,
+ )
+ logger.debug(
+ "book_codes_and_names_localized_from_manifest: %s",
+ book_codes_and_names_localized_from_manifest,
+ )
+ for code, name in book_codes_and_names_localized_from_metadata.items():
+ manifest_name = book_codes_and_names_localized_from_manifest.get(
+ code, ""
+ )
+ if not name and manifest_name:
book_codes_and_names_localized.append(
- (book_code, localized_book_name)
- )
- break
- if (
- use_localized_book_name
- and len(repo_components) > 2
- and repo_components[-1] in usfm_resource_types
- ):
- book_name_file = f"{resource_filepath}/front/title.txt"
- if exists(book_name_file):
- with open(book_name_file, "r") as fin:
- book_name = fin.read()
- localized_book_name_ = normalize_localized_book_name(book_name)
- localized_book_name = maybe_correct_book_name(
- lang_code, localized_book_name_
+ (
+ code,
+ maybe_correct_book_name(
+ lang_code, normalize_localized_book_name(manifest_name)
+ ),
)
- book_code = repo_components[1]
- book_codes_and_names_localized.append(
- (
- book_code,
- localized_book_name,
- )
+ )
+ else:
+ book_codes_and_names_localized.append(
+ (
+ code,
+ maybe_correct_book_name(
+ lang_code, normalize_localized_book_name(name)
+ ),
)
- if not usfm_only:
- if not book_codes_and_names_localized or any(
- name == "" for _, name in book_codes_and_names_localized
- ):
- if len(repo_components) > 2:
- book_code = repo_components[1]
- if book_code in book_names:
- book_codes_and_names.append(
- (book_code, book_names[book_code])
- )
- elif len(repo_components) == 2 and not book_codes_and_names:
- if not book_codes_and_names2:
- if repo_components[-1] in usfm_resource_types:
- usfm_files = parsing.find_usfm_files(resource_filepath)
- for usfm_file in usfm_files:
- book_code = (
- Path(usfm_file).stem.lower().split("-")[1]
- )
- book_codes_and_names2.append(
- (book_code, book_names[book_code])
- )
- if not book_codes_and_names2 and repo_components[-1] in [
- "tn",
- "tq",
- ]:
- subdirs = [
- file
- for file in scandir(resource_filepath)
- if file.is_dir() and file.name in book_names
- ]
- for subdir in subdirs:
- book_codes_and_names2.append(
- (
- subdir.name.lower(),
- book_names[subdir.name.lower()],
- )
- )
- except:
- pass
- unique_values = []
+ )
+ elif (
+ use_localized_book_name
+ and len(repo_components) > 2
+ and resource_type in usfm_resource_types
+ ):
+ book_codes_and_names_localized_from_title_file = (
+ get_book_names_from_title_file(
+ resource_filepath,
+ lang_code,
+ repo_components,
+ )
+ )
+ logger.debug(
+ "book_codes_and_names_localized_from_title_file: %s",
+ book_codes_and_names_localized_from_title_file,
+ )
+ for code, name in book_codes_and_names_localized_from_title_file.items():
+ book_codes_and_names_localized.append(
+ (
+ code,
+ maybe_correct_book_name(
+ lang_code, normalize_localized_book_name(name)
+ ),
+ )
+ )
+ if (
+ not usfm_only
+ or not book_codes_and_names_localized
+ or any(name == "" for _, name in book_codes_and_names_localized)
+ ): # No localized book name sources were found, so use other alternatives for book name lookup
+ book_codes_and_names.extend(
+ get_non_localized_book_names(
+ repo_components,
+ book_names,
+ resource_type,
+ usfm_resource_types,
+ resource_filepath,
+ )
+ )
+ logger.debug("book_codes_and_names: %s", book_codes_and_names)
+ logger.debug("book_codes_and_names_localized: %s", book_codes_and_names_localized)
if not book_codes_and_names_localized or any(
name == "" for _, name in book_codes_and_names_localized
):
- book_codes_and_names.extend(book_codes_and_names2)
- unique_values = unique_tuples(book_codes_and_names)
+ unique_values = unique_book_codes(book_codes_and_names)
else:
- unique_values = unique_tuples(book_codes_and_names_localized)
- book_id_map = {id: pos for pos, id in enumerate(book_names.keys())}
+ unique_values = unique_book_codes(book_codes_and_names_localized)
return sorted(
unique_values, key=lambda book_code_and_name: book_id_map[book_code_and_name[0]]
)
-# TODO Rename to book_codes_and_names_for_lang
-@lru_cache(maxsize=100)
+def get_non_localized_book_names(
+ repo_components: list[str],
+ book_names: Mapping[str, str],
+ resource_type: str,
+ usfm_resource_types: Sequence[str],
+ resource_filepath: str,
+) -> list[tuple[str, str]]:
+ """
+ Get English book names
+ """
+ book_codes_and_names: list[tuple[str, str]] = []
+ if len(repo_components) > 2:
+ # Get book code from repo URL components and then lookup in English book names
+ book_code = repo_components[1]
+ if book_code in book_names:
+ book_codes_and_names.append((book_code, book_names[book_code]))
+ elif len(repo_components) == 2:
+ # if resource_type in usfm_resource_types:
+ # logger.debug("FUBAR") # DEBUG This case happened
+ # # Get book code from USFM file name and then lookup name in English book names
+ # usfm_files = find_usfm_files(resource_filepath)
+ # for usfm_file in usfm_files:
+ # book_code = Path(usfm_file).stem.lower().split("-")[1]
+ # book_codes_and_names.append((book_code, book_names[book_code]))
+ if resource_type in ["tn", "tq"]:
+ # Get book code from TN and TQ repo book sub-directory
+ # names and use to lookup in English book names
+ subdirs = [
+ file
+ for file in scandir(resource_filepath)
+ if file.is_dir() and file.name in book_names
+ ]
+ for subdir in subdirs:
+ book_codes_and_names.append(
+ (
+ subdir.name.lower(),
+ book_names[subdir.name.lower()],
+ )
+ )
+ return book_codes_and_names
+
+
+def get_book_names_from_usfm_metadata(
+ resource_filepath: str,
+ lang_code: str,
+ resource_type: str,
+) -> dict[str, str]:
+ """
+ Book names obtained from USFM frontmatter/metadata may or may not
+ be localized, it depends on the translation work done for language
+ lang_code.
+ """
+ book_codes_and_names_localized: dict[str, str] = {}
+ usfm_files = parsing.find_usfm_files(resource_filepath)
+ for usfm_file in usfm_files:
+ usfm_file_components = Path(usfm_file).stem.lower().split("-")
+ book_code = usfm_file_components[1]
+ usfm = read_file(usfm_file) if usfm_file else ""
+ frontmatter, _, _ = parsing.split_usfm_by_chapters(
+ lang_code, resource_type, book_code, usfm
+ )
+ localized_book_name = parsing.maybe_localized_book_name(frontmatter)
+ # localized_book_name = maybe_correct_book_name(lang_code, localized_book_name)
+ book_codes_and_names_localized[book_code] = localized_book_name
+ logger.debug("book_codes_and_names_localized: %s", book_codes_and_names_localized)
+ return book_codes_and_names_localized
+
+
@worker.app.task
def book_codes_for_lang(
lang_code: str,
- resource_assets_dir: str = settings.RESOURCE_ASSETS_DIR,
- book_names: Mapping[str, str] = BOOK_NAMES,
- dcs_mirror_git_username: str = "DCS-Mirror",
- usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
- use_localized_book_name: bool = settings.USE_LOCALIZED_BOOK_NAME,
) -> Sequence[tuple[str, str]]:
"""
- >>> from doc.domain import resource_lookup
- >>> ();result = resource_lookup.book_codes_for_lang("pt-br");() # doctest: +ELLIPSIS
- (...)
- >>> result[0]
- ('gen', 'Gênesis')
+ >>> from doc.domain.resource_lookup import book_codes_for_lang
+ >>> book_codes_for_lang("zh") # zh doesn't have USFM resource available, get books from non-USFM resources
+ [('gen', 'Genesis'), ('exo', 'Exodus'), ('lev', 'Leviticus'), ('num', 'Numbers'), ('deu', 'Deuteronomy'), ('jos', 'Joshua'), ('jdg', 'Judges'), ('rut', 'Ruth'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Kings'), ('2ki', '2 Kings'), ('1ch', '1 Chronicles'), ('2ch', '2 Chronicles'), ('ezr', 'Ezra'), ('neh', 'Nehemiah'), ('est', 'Esther'), ('job', 'Job'), ('psa', 'Psalms'), ('pro', 'Proverbs'), ('ecc', 'Ecclesiastes'), ('sng', 'Song of Solomon'), ('isa', 'Isaiah'), ('jer', 'Jeremiah'), ('lam', 'Lamentations'), ('ezk', 'Ezekiel'), ('dan', 'Daniel'), ('hos', 'Hosea'), ('jol', 'Joel'), ('amo', 'Amos'), ('oba', 'Obadiah'), ('jon', 'Jonah'), ('mic', 'Micah'), ('nam', 'Nahum'), ('hab', 'Habakkuk'), ('zep', 'Zephaniah'), ('hag', 'Haggai'), ('zec', 'Zechariah'), ('mal', 'Malachi'), ('mat', 'Matthew'), ('mrk', 'Mark'), ('luk', 'Luke'), ('jhn', 'John'), ('act', 'Acts'), ('rom', 'Romans'), ('1co', '1 Corinthians'), ('2co', '2 Corinthians'), ('gal', 'Galatians'), ('eph', 'Ephesians'), ('php', 'Philippians'), ('col', 'Colossians'), ('1th', '1 Thessalonians'), ('2th', '2 Thessalonians'), ('1ti', '1 Timothy'), ('2ti', '2 Timothy'), ('tit', 'Titus'), ('phm', 'Philemon'), ('heb', 'Hebrews'), ('jas', 'James'), ('1pe', '1 Peter'), ('2pe', '2 Peter'), ('1jn', '1 John'), ('2jn', '2 John'), ('3jn', '3 John'), ('jud', 'Jude'), ('rev', 'Revelation')]
+ >>> book_codes_for_lang("pt-br") # pt-br has, for example, two book names for lev
+ [('gen', 'Gênesis'), ('exo', 'Êxodo'), ('lev', 'Levítico'), ('num', 'Números'), ('deu', 'Deuteronômio'), ('jos', 'Josué'), ('jdg', 'Juízes'), ('rut', 'Rute'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reis'), ('2ki', '2 Reis'), ('1ch', '1 Crônicas'), ('2ch', '2 Crônicas'), ('ezr', 'Esdras'), ('neh', 'Neemias'), ('est', 'Ester'), ('job', 'Jó'), ('psa', 'Salmos'), ('pro', 'Provérbios'), ('ecc', 'Eclesiastes'), ('sng', 'Cantares'), ('isa', 'Isaías'), ('jer', 'Jeremias'), ('lam', 'Lamentações'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseias'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Obadias'), ('jon', 'Jonas'), ('mic', 'Miqueias'), ('nam', 'Naum'), ('hab', 'Habacuque'), ('zep', 'Sofonias'), ('hag', 'Ageu'), ('zec', 'Zacarias'), ('mal', 'Malaquias'), ('mat', 'Mateus'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'João'), ('act', 'Atos'), ('rom', 'Romanos'), ('1co', '1 Coríntios'), ('2co', '2 Coríntios'), ('gal', 'Gálatas'), ('eph', 'Efésios'), ('php', 'Filipenses'), ('col', 'Colossenses'), ('1th', '1 Tessalonicenses'), ('2th', '2 Tessalonicenses'), ('1ti', '1 Timóteo'), ('2ti', '2 Timóteo'), ('tit', 'Tito'), ('phm', 'Filemom'), ('heb', 'Hebreus'), ('jas', 'Tiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 João'), ('2jn', '2 João'), ('3jn', '3 João'), ('jud', 'Judas'), ('rev', 'Apocalipse')]
+ >>> book_codes_for_lang("ta")
"""
return get_book_codes_for_lang(
lang_code,
- resource_assets_dir,
- book_names,
- dcs_mirror_git_username,
- usfm_resource_types,
- use_localized_book_name,
usfm_only=False,
)
-@lru_cache(maxsize=100)
@worker.app.task
def book_codes_for_lang_from_usfm_only(
lang_code: str,
- resource_assets_dir: str = settings.RESOURCE_ASSETS_DIR,
- book_names: Mapping[str, str] = BOOK_NAMES,
- dcs_mirror_git_username: str = "DCS-Mirror",
- usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
- use_localized_book_name: bool = settings.USE_LOCALIZED_BOOK_NAME,
) -> Sequence[tuple[str, str]]:
"""
>>> from doc.domain import resource_lookup
@@ -1242,16 +1047,10 @@ def book_codes_for_lang_from_usfm_only(
"""
return get_book_codes_for_lang(
lang_code,
- resource_assets_dir,
- book_names,
- dcs_mirror_git_username,
- usfm_resource_types,
- use_localized_book_name,
usfm_only=True,
)
-@lru_cache(maxsize=100)
def chapters_in_books(
book_chapters: Mapping[str, int] = BOOK_CHAPTERS
) -> dict[str, list[int]]:
@@ -1262,64 +1061,6 @@ def chapters_in_books(
return chapters_in_book
-def load_manifest(file_path: str) -> str:
- with open(file_path, "r") as file:
- return file.read()
-
-
-def book_codes_and_names_from_manifest(
- resource_dir: str,
- manifest_glob_fmt_str: str = "{}/**/manifest.{}",
- manifest_glob_alt_fmt_str: str = "{}/manifest.{}",
-) -> list[tuple[str, str]]:
- """
- Look up the language direction in the manifest file if one is
- available for this resource.
- """
- # Try to find manifest yaml at typical directory
- manifest_candidates = glob(manifest_glob_fmt_str.format(resource_dir, "yaml"))
- if not manifest_candidates:
- # Now try to find manifest yaml at parent directory of typical directory
- manifest_candidates = glob(
- manifest_glob_alt_fmt_str.format(resource_dir, "yaml")
- )
- if not manifest_candidates:
- # Some languages provide their manifest in json format.
- # Try to find manifest json at typical directory
- manifest_candidates = glob(
- manifest_glob_fmt_str.format(resource_dir, "json")
- )
- if not manifest_candidates:
- # Try to find manifest json at parent directory of typical directory
- manifest_candidates = glob(
- manifest_glob_alt_fmt_str.format(resource_dir, "json")
- )
- # logger.debug("manifest_candidates: %s", manifest_candidates)
- if manifest_candidates:
- # logger.debug("len(manifest_candidates): %s", len(manifest_candidates))
- candidate = manifest_candidates[0]
- suffix = str(Path(candidate).suffix)
- book_codes_and_names: list[tuple[str, str]] = []
- # Get localized book names
- manifest_data = load_manifest(candidate)
- # logger.debug("manifest_data: %s", manifest_data)
- if suffix == ".yaml":
- data: Data = yaml.safe_load(manifest_data)
- book_codes_and_names = [
- (book["identifier"], book["title"]) for book in data["projects"]
- ]
- # Heart languages often have .json manifest files
- # per book and not per language.
- elif suffix == ".json":
- json_data: JsonManifestData = json.loads(manifest_data)
- logger.debug("json_data: %s", json_data)
- project: JsonManifestBook = json_data["project"]
- book_codes_and_names = [(project["id"], project["name"])]
- # logger.debug("book_codes_and_names from json: %s", book_codes_and_names)
- return book_codes_and_names
-
-
-@lru_cache(maxsize=100)
def resource_lookup_dto(
lang_code: str,
resource_type: str,
@@ -1333,116 +1074,105 @@ def resource_lookup_dto(
>>> ();data = resource_lookup.resource_lookup_dto("pt-br", "ulb", "mat");() # doctest: +ELLIPSIS
(...)
>>> data
- ResourceLookupDto(lang_code='pt-br', lang_name='Brazilian Portuguese', resource_type='ulb', resource_type_name='Unlocked Literal Bible', book_code='mat', lang_direction='ltr', url='https://content.bibletranslationtools.org/WA-Catalog/pt-br_ulb')
+ ResourceLookupDto(lang_code='pt-br', lang_name='Brazilian Portuguese', localized_lang_name='Português Brasileiro', resource_type='ulb', resource_type_name='Unlocked Literal Bible', book_code='mat', lang_direction=, url=HttpUrl('https://content.bibletranslationtools.org/WA-Catalog/pt-br_ulb'))
"""
- data = fetch_source_data()
- resource_lookup_dto = None
- rg_resource_lookup_dtos = []
- two_component_url_resource_lookup_dtos = []
- more_than_two_component_url_resource_lookup_dtos = []
+ data = fetch_source_data() # Fetch source data
+ if data is None:
+ return None
+ resource_lookup_dto: Optional[ResourceLookupDto] = None
+ rg_resource_lookup_dtos: list[ResourceLookupDto] = []
+ two_component_url_resource_lookup_dtos: list[ResourceLookupDto] = []
+ more_than_two_component_url_resource_lookup_dtos: list[ResourceLookupDto] = []
try:
- repos_info = data["git_repo"]
+ repos_info = data.git_repo
augmented_repos_info = add_data_not_supplied_by_data_api(repos_info)
for repo_info in augmented_repos_info:
- content = repo_info["content"]
- language_info = content["language"]
- resource_type_ = content["resource_type"]
- url = repo_info["repo_url"]
- if language_info["ietf_code"] == lang_code:
+ content = repo_info.content
+ language_info = content.language
+ resource_type_ = content.resource_type
+ url = repo_info.repo_url
+ if language_info.ietf_code == lang_code:
last_segment = get_last_segment(url, lang_code)
if last_segment[-4:] == "docx":
- # logger.debug("docx detected, url: %s", url)
resource_lookup_dto = ResourceLookupDto(
lang_code=lang_code,
- lang_name=language_info["english_name"],
+ lang_name=language_info.english_name,
+ localized_lang_name=language_info.national_name,
resource_type=resource_type,
resource_type_name=resource_type_codes_and_names[resource_type],
book_code=book_code,
- lang_direction=language_info["direction"],
+ lang_direction=language_info.direction,
url=url,
)
rg_resource_lookup_dtos.append(resource_lookup_dto)
else:
repo_components = last_segment.split("_")
repo_components = update_repo_components(repo_components)
- # logger.debug(
- # "url: %s, repo_components: %s, resource_type: %s",
- # url,
- # repo_components,
- # resource_type_,
- # )
if len(repo_components) > 2:
book_code_ = repo_components[1]
if (
- (book_code_ in url or zmq_git_username in url)
+ (book_code_ in str(url) or zmq_git_username in str(url))
and resource_type == resource_type_
and resource_type_ in resource_type_codes_and_names
and book_code_ == book_code
):
resource_lookup_dto = ResourceLookupDto(
lang_code=lang_code,
- lang_name=language_info["english_name"],
+ lang_name=language_info.english_name,
+ localized_lang_name=language_info.national_name,
resource_type=resource_type,
resource_type_name=resource_type_codes_and_names[
resource_type
],
book_code=book_code,
- lang_direction=language_info["direction"],
+ lang_direction=language_info.direction,
url=url,
)
more_than_two_component_url_resource_lookup_dtos.append(
resource_lookup_dto
)
- elif (
- len(repo_components) == 2 and resource_type == resource_type_
- ): # Here we handle cases like es-419_ulb, es-419_tn, en_ulb, etc.
+ elif len(repo_components) == 2 and resource_type == resource_type_:
+ # Handle cases like es-419_ulb, es-419_tn, en_ulb, etc.
resource_lookup_dto = ResourceLookupDto(
lang_code=lang_code,
- lang_name=language_info["english_name"],
+ lang_name=language_info.english_name,
+ localized_lang_name=language_info.national_name,
resource_type=resource_type,
resource_type_name=resource_type_codes_and_names[
resource_type
],
book_code=book_code,
- lang_direction=language_info["direction"],
+ lang_direction=language_info.direction,
url=url,
)
two_component_url_resource_lookup_dtos.append(
resource_lookup_dto
)
- except:
+ except Exception:
logger.info(
"Problem creating ResourceLookupDto instance for %s, %s, %s, likely a data problem",
lang_code,
resource_type,
book_code,
)
- # logger.debug(
- # "two_component_url_resource_lookup_dtos: %s",
- # two_component_url_resource_lookup_dtos,
- # )
- # logger.debug(
- # "more_than_two_component_url_resource_lookup_dtos: %s",
- # more_than_two_component_url_resource_lookup_dtos,
- # )
if rg_resource_lookup_dtos:
resource_lookup_dto = rg_resource_lookup_dtos[0]
elif more_than_two_component_url_resource_lookup_dtos:
resource_lookup_dto = more_than_two_component_url_resource_lookup_dtos[0]
elif two_component_url_resource_lookup_dtos:
resource_lookup_dto = two_component_url_resource_lookup_dtos[0]
- # logger.debug("resource_lookup_dto: %s", resource_lookup_dto)
return resource_lookup_dto
def provision_asset_files(
- url: Optional[str],
+ url: Optional[HttpUrl],
resource_filepath: str,
) -> None:
- if url is not None and url[-4:] != "docx":
- clone_git_repo(url, resource_filepath)
- elif url is not None and url[-4:] == "docx":
- download_rg_file(url, resource_filepath)
+ if url is not None:
+ if str(url)[-4:] != "docx":
+ clone_git_repo(url, resource_filepath)
+ elif str(url)[-4:] == "docx":
+ download_rg_file(url, resource_filepath)
def prepare_resource_filepath(
@@ -1461,7 +1191,7 @@ def prepare_resource_filepath(
def clone_git_repo(
- url: str,
+ url: HttpUrl,
resource_filepath: str,
branch: Optional[str] = None,
) -> None:
@@ -1487,7 +1217,7 @@ def clone_git_repo(
def download_rg_file(
- url: str,
+ url: HttpUrl,
resource_filepath: str,
) -> None:
# TODO Until data API provides reviewer's guide URL that is
@@ -1520,36 +1250,21 @@ def download_rg_file(
# )
-@lru_cache(maxsize=100)
def nt_survey_rg_passages(
lang_code: str = "en",
lang_name: str = "English",
- resource_type_name: str = "NT Survey Reviewer's Guide",
- lang_direction: LangDirEnum = LangDirEnum.LTR,
- assets_dir: str = settings.RESOURCE_ASSETS_DIR,
- resource_dir: str = "en_rg",
docx_file_path: str = "en_rg_nt_survey.docx",
+ resource_type_name: str = "NT Survey Reviewers' Guide",
+ lang_direction: LangDirEnum = LangDirEnum.LTR,
+ resource_dir: str = settings.EN_RG_DIR,
) -> list[BibleReference]:
"""
>>> from doc.domain import resource_lookup
>>> rg_books = resource_lookup.nt_survey_rg_passages()
>>> rg_books[0]
- RGBook(
- lang_code=en,
- lang_name=English,
- book_code=mat,
- resource_type_name=NT Survey Reviewer's Guide,
- chapters={2: RGChapter(content=ParsedText(bible_reference=BibleReference(book_code='mat', book_name='Matthew', chapter=2, verse_ref='1-12'), background='', directive='Read the passage.', part_1=[Part1Item(text='Wise men came from the east to Jerusalem looking for the one who was born king of the Jews.', reference='[2:1-2]'), Part1Item(text='The chief priests and scribes told Herod that the Christ was to be born in Bethlehem according to Scripture.', reference='[2:5-6]'), Part1Item(text='Herod sent the wise men to Bethlehem.', reference='[2:7-8]'), Part1Item(text='Herod ordered the wise men to return, telling them that he also wanted to worship the baby.', reference='[2:8]'), Part1Item(text='The wise men went to Bethlehem, found the baby, gave him gifts, and worshiped him.', reference='[2:11]'), Part1Item(text='They did not return to Herod.', reference='[2:12]')], part_1_directive='Tell in your own words what you just read in this passage.', part_2=[Part2Item(reference='[2:1-2]', question='After Jesus was born in Bethlehem, why did some wise men go to Jerusalem?', answer='They were looking for the one who had been born King of the Jews.'), Part2Item(reference='[2:2]', question='Why were they looking for the King of the Jews?', answer='They said that they saw his star and were coming to worship him.'), Part2Item(reference='[2:3]', question='Why do you think that Herod was troubled when he heard the wise men wanted to find the King of the Jews that had just been born?', answer='(Answer may vary.) Herod was probably worried that a new king might take away his own position as king.'), Part2Item(reference='[2:4]', question='How did Herod find out where the Christ would be born?', answer='He asked the priests and scribes of the Jewish people where Christ would be born.'), Part2Item(reference='[2:5-6]', question='How did the priests and scribes know where Christ would be born?', answer='The scriptures said that a ruler would come from Bethlehem.'), Part2Item(reference='[2:9-10]', question='How did the wise men find the one who was born king of the Jews?', answer='They left Jerusalem, and the star guided them to the house where the child was.'), Part2Item(reference='[2:11]', question='What did the wise men do when they found the child?', answer='They fell down and worshiped him, and they opened their gifts and gave them to him.'), Part2Item(reference='[2:11]', question='Who do you think the child was that they were worshiping?', answer='From the rest of the passage, it is clear that the child was Jesus.'), Part2Item(reference='[2:12]', question='Why didn’t the wise men return to Herod?', answer='They had been warned by God in a dream not to return to Herod.')], part_2_directive='Answer the following questions from the specified verses.', comment_section='')),
- 3: RGChapter(content=ParsedText(bible_reference=BibleReference(book_code='mat', book_name='Matthew', chapter=3, verse_ref='13-17'), background='After Jesus grew up, John the Baptist went into the wilderness and preached that people should stop sinning and be baptized. John also taught that someone more powerful than he would come, and that person would judge people.', directive='Read the passage.', part_1=[Part1Item(text='Jesus came to the Jordan River to be baptized by John.', reference='[3:13]'), Part1Item(text='John tried to stop Jesus, but Jesus said it was right for John to baptize him.', reference='[3:14-15]'), Part1Item(text='After John baptized Jesus, God’s Spirit came down like a dove and rested on Jesus.', reference='[3:16]'), Part1Item(text='A voice from heaven said, “This is my beloved Son. I am very pleased with him.”', reference='[3:17]')], part_1_directive='Tell in your own words what you just read in this passage.', part_2=[Part2Item(reference='[3:14]', question='When Jesus came to be baptized, what did John tell Jesus?', answer='John told Jesus that he needed to be baptized by Jesus.'), Part2Item(reference='[3:15]', question='What did Jesus say was the reason that John should baptize him?', answer='It was right for them to fulfill all righteousness.'), Part2Item(reference='[3:16]', question='What did the Spirit of God do when Jesus came out of the water?', answer='The Spirit of God came down and rested on Jesus.'), Part2Item(reference='[3:17]', question='What did the voice from heaven say after Jesus came out of the water?', answer='The voice said, “This is my beloved Son. I am very pleased with him.”'), Part2Item(reference='[3:17]', question='Whose voice do you think was speaking?', answer='By saying the voice came out of heaven, the passage shows that it was the voice of God the Father.')], part_2_directive='Answer the following questions from the specified verses.', comment_section='')),
- 4: RGChapter(content=ParsedText(bible_reference=BibleReference(book_code='mat', book_name='Matthew', chapter=4, verse_ref='1-11'), background='', directive='Read the passage.', part_1=[Part1Item(text='The Spirit led Jesus into the wilderness.', reference='[4:1]'), Part1Item(text='Jesus fasted, and he was hungry.', reference='[4:2]'), Part1Item(text='The devil tempted Jesus three times.', reference='[4:3-10]'), Part1Item(text='Each time, Jesus quoted scripture to the devil.', reference='[4:4-10]'), Part1Item(text='After the devil left him, angels came and served Jesus.', reference='[4:11]')], part_1_directive='Tell in your own words what you just read in this passage.', part_2=[Part2Item(reference='[4:2]', question='How long did Jesus fast?', answer='Jesus fasted for forty days.'), Part2Item(reference='[4:3]', question='What did the devil first tempt Jesus to do?', answer='The devil tempted Jesus to command the stones to become bread.'), Part2Item(reference='[4:4]', question='How did Jesus respond?', answer='Jesus quoted what was written about not living on just bread but on every word that comes out of the mouth of God.'), Part2Item(reference='[4:4]', question='What do you think it means to live on God’s word?', answer='(Answer may vary.) It may mean to believe God’s word and obey it. It may mean that God’s words are needed by all, just as food is.'), Part2Item(reference='[4:5-6]', question='What did the devil next tempt Jesus to do?', answer='The devil tempted Jesus to throw himself down from the highest part of the temple building.'), Part2Item(reference='[4:7]', question='How did Jesus respond?', answer='Jesus quoted what was written about not testing the Lord God.'), Part2Item(reference='[4:3, 6]', question='When the devil tempted Jesus the first two times, what part of what he said was the same?', answer='Both times the devil said, “If you are the Son of God.”'), Part2Item(reference='[4:3-4]', question='Do you think that Jesus would have been able to turn the stones into bread?', answer='Why do you think that? (Answer may vary.) Probably Jesus would have been able to do that because he was the Son of God.'), Part2Item(reference='[4:8-9]', question='The devil said that he would give Jesus the kingdoms of the world if Jesus would do what?', answer='The devil said that he would give the kingdoms to Jesus if Jesus would fall down and worship him.'), Part2Item(reference='[4:10]', question='How did Jesus respond?', answer='Jesus told Satan to go away, and he quoted what was written about worshiping only the Lord God.'), Part2Item(reference='[4:4, 7, 10]', question='Where do you think the words that Jesus quoted were written?', answer='Here and in other parts of the New Testament, the words “it is written” indicate that they were written in the scriptures—in the part of the Bible now called the Old Testament.')], part_2_directive='Answer the following questions from the specified verses.', comment_section='')),
- 5: RGChapter(content=ParsedText(bible_reference=BibleReference(book_code='mat', book_name='Matthew', chapter=5, verse_ref='1-12'), background='Jesus went around teaching people to repent because the kingdom of God was near. He also told some men to follow him, and he healed many people. Large crowds of people followed Jesus.', directive='Read the passage.', part_1=[Part1Item(text='Jesus went up on a mountain and taught his disciples.', reference='[5:1-2]'), Part1Item(text='Jesus described people who were blessed and then told how they would be blessed.', reference='[5:2-12]'), Part1Item(text='Jesus told the disciples to rejoice when they were persecuted for believing in him.', reference='[5:11-12]')], part_1_directive='Tell in your own words what you just read in this passage.', part_2=[Part2Item(reference='[5:1-2]', question='What did Jesus do when he saw the crowds?', answer='Jesus went up on a mountain and taught his disciples.'), Part2Item(reference='[5:3]', question='Who does the kingdom of heaven belong to?', answer='It belongs to the poor in spirit.'), Part2Item(reference='[5:4]', question='Who will be comforted?', answer='Those who mourn will be comforted.'), Part2Item(reference='[5:5]', question='Who will inherit the earth?', answer='The meek will inherit the earth.'), Part2Item(reference='[5:6]', question='Who will be filled?', answer='Those who hunger and thirst for righteousness will be filled.'), Part2Item(reference='[5:6]', question='What do you think those who hunger and thirst for righteousness will be filled with?', answer='(Answer may vary.) The passage probably means they will be filled with righteousness.'), Part2Item(reference='[5:7]', question='Who will receive mercy?', answer='The merciful will receive mercy.'), Part2Item(reference='[5:8]', question='Who will see God?', answer='The pure in heart will see God.'), Part2Item(reference='[5:9]', question='Who will be called sons of God?', answer='The peacemakers will be called sons of God.'), Part2Item(reference='[5:9]', question='Why do you think the peacemakers will be called sons of God?', answer='(Answer may vary.) The passage might mean that by helping people to have peace with one another, they will reflect the character of God, who enables people to have peace with him.'), Part2Item(reference='[5:10]', question='Who does the kingdom of heaven belong to?', answer="It belongs to those who are persecuted for righteousness' sake.")], part_2_directive='Answer the following questions from the specified verses.', comment_section='')),
- 6: RGChapter(content=ParsedText(bible_reference=BibleReference(book_code='mat', book_name='Matthew', chapter=6, verse_ref='1-15'), background='Jesus continued to teach his disciples on the mountain.', directive='Read the passage.', part_1=[Part1Item(text='Jesus taught people not to do good works in order to be seen by other people.', reference='[6:1-4]'), Part1Item(text='Jesus taught people not to pray in order to be seen by other people.', reference='[6:5-8]'), Part1Item(text='Jesus taught people how to pray.', reference='[6:9-13]'), Part1Item(text='Jesus taught about forgiveness.', reference='[6:14-15]')], part_1_directive='Tell in your own words what you just read in this passage.', part_2=[Part2Item(reference='[6:1]', question='What did Jesus warn them against doing?', answer='He warned them not to do their good deeds to be seen by others.'), Part2Item(reference='[6:1]', question='What did he say the result of doing their good deeds to be seen by others would be?', answer='Their Father in heaven would not reward them.'), Part2Item(reference='[6:1]', question='Who do you think Jesus meant when he spoke of their Father in heaven?', answer='It is evident that he was talking about God the Father.'), Part2Item(reference='[6:2]', question='Why did hypocrites sound a trumpet when they gave to the poor?', answer='Because they wanted other people to glorify them.'), Part2Item(reference='[6:3]', question='When Jesus said not to let their left hand know what their right hand was doing, what do you think he meant?', answer='(Answer may vary.) He may have meant that they should do their good works so secretly that even those closest to them would not know about it. Or maybe it should become so customary or simple to do a good work that they might not even remember doing it.'), Part2Item(reference='[6:4, 6]', question='What did Jesus say about the Father in verses 4 and 6?', answer='The Father sees who gives and prays in secret, and he will reward these people.'), Part2Item(reference='[6:8]', question='What did he say the Father knows in verse 8?', answer='He knows what people need before they ask for it.'), Part2Item(reference='[6:8]', question='What did Jesus tell his disciples to call God when they prayed?', answer='He told them to call him Father.'), Part2Item(reference='[6:9-13]', question='What things did Jesus tell them to pray about?', answer='He said to pray for: the Father’s name to be honored as holy his kingdom to come his will to be done their daily needs forgiveness protection from temptation and evil'), Part2Item(reference='[6:14]', question='What did Jesus say the Father would do if they would forgive other people?', answer='He said the Father would forgive them.')], part_2_directive='Answer the following questions from the specified verses.', comment_section='')),
- 13: RGChapter(content=ParsedText(bible_reference=BibleReference(book_code='mat', book_name='Matthew', chapter=13, verse_ref='44-46'), background='Jesus went and sat down beside the sea. A large crowd gathered around him, and he used parables to teach them about the kingdom of God.', directive='Read the passage.', part_1=[Part1Item(text='Jesus said that the kingdom of heaven is like a treasure hidden in a field.', reference='[13:44]'), Part1Item(text='Jesus also said that the kingdom of heaven is like a merchant looking for valuable pearls.', reference='[13:45-46]')], part_1_directive='Tell in your own words what you just read in this passage.', part_2=[Part2Item(reference='[13:44]', question='How did Jesus describe the kingdom of heaven in verse 44?', answer='It is like a treasure hidden in a field.'), Part2Item(reference='[13:44]', question='What did the man do when he found the treasure hidden in the field?', answer='The man hid the treasure and sold all his possessions and bought the field.'), Part2Item(reference='[13:44]', question='Why do you think the man bought the field?', answer='He probably wanted the treasure that was hidden in the field.'), Part2Item(reference='[13:45]', question='How did Jesus describe the kingdom of heaven in verse 45?', answer='The kingdom of heaven is like a man who was a merchant looking for valuable pearls.'), Part2Item(reference='[13:46]', question='What did the man do when he found a very valuable pearl?', answer='He went and sold everything that he owned and bought it.'), Part2Item(reference='[13:44-46]', question='Why do you think the two men in the stories sold everything they possessed?', answer='It appears that they sold everything so they could get enough money to buy the land and the pearl.')], part_2_directive='Answer the following questions from the specified verses.', comment_section='')),
- 14: RGChapter(content=ParsedText(bible_reference=BibleReference(book_code='mat', book_name='Matthew', chapter=14, verse_ref='13-21'), background='King Herod killed Jesus’ cousin, John the Baptist. John’s disciples went and told Jesus about it. ', directive='Read the passage. Read the passage.', part_1=[Part1Item(text='Jesus went to a place far away from people, but the crowds followed him, and he had compassion on them.', reference='[14:13-14]'), Part1Item(text='Jesus told his disciples to feed the people, but they could only find a small amount of food.', reference='[14:15-18]'), Part1Item(text='Jesus blessed the food and gave it to the disciples, and they gave it to the people.', reference='[14:19]'), Part1Item(text='There was enough for over 5,000 people. After the people ate, there was a lot of food left over.', reference='[14:20-21]'), Part1Item(text='Jesus’ disciples got into a boat and went out on the sea.', reference='[14:22]'), Part1Item(text='At night, Jesus walked on the water to his disciples. He told them not to be afraid.', reference='[14:25-27]'), Part1Item(text='Peter walked toward Jesus on the water, but he began to sink. Jesus saved him.', reference='[14:29-31]'), Part1Item(text='The wind stopped blowing.', reference='[14:32]'), Part1Item(text='The disciples worshiped Jesus.', reference='[14:33]')], part_1_directive='Tell in your own words what you just read in this passage. Tell in your own words what you just read in this passage.', part_2=[Part2Item(reference='[14:13]', question='Where did Jesus go in a boat?', answer='He went to a place far away from people.'), Part2Item(reference='[14:14]', question='What did Jesus do when he saw the large crowd of people?', answer='He had compassion on them, and he healed those who were sick.'), Part2Item(reference='[14:15]', question='Why did the disciples want Jesus to send the crowds away?', answer='Because it was late in the day, the disciples knew the people needed to go to the villages to buy food.'), Part2Item(reference='[14:16]', question='What did Jesus tell his disciples to do?', answer='He told them to give the people something to eat.'), Part2Item(reference='[14:17]', question='How did the disciples reply to Jesus?', answer='They said that they had only five loaves of bread and two fish.'), Part2Item(reference='[14:17]', question='Why do you think the disciples said this to Jesus?', answer='It appears that they were telling Jesus they did not have enough food to give to so many people.'), Part2Item(reference='[14:19]', question='What did Jesus do with the food?', answer='He looked up to heaven, blessed the food, broke the loaves, and gave them to his disciples.'), Part2Item(reference='[14:20]', question='After the people all ate, how much food was left over?', answer='Twelve baskets full of food were left over.'), Part2Item(reference='[14:21]', question='How many people ate the food that Jesus provided?', answer='About 5,000 men, plus women and children, ate the food.'), Part2Item(reference='[14:23]', question='What did Jesus do on the mountain?', answer='He prayed.'), Part2Item(reference='[14:24]', question='What happened while the disciples were in the boat?', answer='The wind blew strongly against them, and the boat was tossed around by the waves.'), Part2Item(reference='[14:25]', question='What unusual thing did Jesus do in verse 25?', answer='Jesus walked to them on the water.'), Part2Item(reference='[14:26]', question='Why do you think that the disciples thought that the person walking to them on the sea was a ghost?', answer='(Answer may vary.) They probably thought this because people have bodies that cannot walk on the sea.'), Part2Item(reference='[14:28]', question='What did Peter ask Jesus to do to show that it really was him walking on the water?', answer='Peter asked Jesus to command him to go to him on the water.'), Part2Item(reference='[14:30]', question='What happened when Peter was walking on the water, and he saw the wind?', answer='Peter became afraid, and he began to sink.'), Part2Item(reference='[14:30]', question='What did Peter do when he began to sink?', answer='He cried out to the Lord to save him.'), Part2Item(reference='[14:31-32]', question='How did Jesus save Peter?', answer='Jesus grabbed Peter and they went into the boat.'), Part2Item(reference='[14:32]', question='What happened when Jesus and Peter went into the boat?', answer='The wind stopped blowing.'), Part2Item(reference='[14:33]', question='How did the disciples respond?', answer='They worshiped Jesus and acknowledged that Jesus is the Son of God.'), Part2Item(reference='[14:25-32]', question='What events in this passage do you think showed the disciples that Jesus was the Son of God?', answer='Jesus walked on the sea, Jesus enabled Peter to walk on the sea, and Jesus made the wind stop blowing.'), Part2Item(reference='[14:34-35]', question='When the boat got to land, what did the men in that place do?', answer='They sent messages to the surrounding area, and people brought sick people to Jesus.'), Part2Item(reference='[14:36]', question='What do you think was the reason that people wanted to touch the edge of Jesus’ garment?', answer='They probably believed (had faith) that just touching Jesus’ garment would heal them.')], part_2_directive='Answer the following questions from the specified verses. Answer the following questions from the specified verses.', comment_section=' '))},
- lang_direction=LangDirEnum.LTR
- )
+ BibleReference(book_code='mat', book_name='Matthew', start_chapter=2, start_chapter_verse_ref='1-12', end_chapter=None, end_chapter_verse_ref=None)
"""
- path = join(assets_dir, resource_dir, docx_file_path)
+ path = join(resource_dir, docx_file_path)
# logger.debug("path: %s exists: %s", path, exists(path))
# TODO Check if resource_dir exists and if it doesn't then submit
# a document request to DOC API to make sure it is cloned.
@@ -1567,9 +1282,7 @@ def nt_survey_rg_passages(
chapter for rg_book in rg_books for chapter in rg_book.chapters.values()
]
bible_references = [
- pt.bible_reference
- for chapter in rg_book_chapters
- for pt in chapter.content # content is now list[ParsedText]
+ pt.bible_reference for chapter in rg_book_chapters for pt in chapter.content
]
# Localize the book names since they are provided in English from en_rg_nt_survey.docx
book_name_map = {
@@ -1585,6 +1298,198 @@ def nt_survey_rg_passages(
return bible_references
+def ot_survey_rg1_passages(
+ lang_code: str = "en",
+ lang_name: str = "English",
+ docx_file_path: str = "en_ot_survey_rg1_gen_deu.docx",
+ resource_type_name: str = "OT Survey Reviewers' Guide (Genesis to Deuteronomy)",
+ lang_direction: LangDirEnum = LangDirEnum.LTR,
+ resource_dir: str = settings.EN_RG_DIR,
+) -> list[BibleReference]:
+ """
+ >>> from doc.domain import resource_lookup
+ >>> rg_books = resource_lookup.ot_survey_rg1_passages()
+ >>> rg_books[0]
+ BibleReference(book_code='gen', book_name='Genesis', start_chapter=2, start_chapter_verse_ref='1-12', end_chapter=None, end_chapter_verse_ref=None)
+ """
+ path = join(resource_dir, docx_file_path)
+ # logger.debug("path: %s exists: %s", path, exists(path))
+ # TODO Check if resource_dir exists and if it doesn't then submit
+ # a document request to DOC API to make sure it is cloned.
+ # Currently we don't have to do this because at startup we copy
+ # English OT Survey RG1 doc into place.
+ # assert exists(path)
+ rg_books = get_rg_books(
+ path,
+ lang_code,
+ lang_name,
+ resource_type_name,
+ lang_direction,
+ )
+ rg_book_chapters = [
+ chapter for rg_book in rg_books for chapter in rg_book.chapters.values()
+ ]
+ bible_references = [
+ pt.bible_reference for chapter in rg_book_chapters for pt in chapter.content
+ ]
+ # Localize the book names since they are provided in English from en_ot_survey_rg1_gen_deu.docx
+ book_name_map = {
+ book_code_and_name[0]: book_code_and_name[1]
+ for book_code_and_name in book_codes_for_lang_from_usfm_only(lang_code)
+ }
+ for bible_reference in bible_references:
+ maybe_localized_book_name = book_name_map.get(
+ bible_reference.book_code, bible_reference.book_name
+ )
+ logger.debug("maybe_localized_book_name: %s", maybe_localized_book_name)
+ bible_reference.book_name = maybe_localized_book_name
+ return bible_references
+
+
+def ot_survey_rg2_passages(
+ lang_code: str = "en",
+ lang_name: str = "English",
+ docx_file_path: str = "en_ot_survey_rg2_jos_est.docx",
+ resource_type_name: str = "OT Survey Reviewers' Guide (Joshua to Esther)",
+ lang_direction: LangDirEnum = LangDirEnum.LTR,
+ resource_dir: str = settings.EN_RG_DIR,
+) -> list[BibleReference]:
+ """
+ >>> from doc.domain import resource_lookup
+ >>> rg_books = resource_lookup.ot_survey_rg2_passages()
+ >>> rg_books[0]
+ BibleReference(book_code='jos', book_name='Joshua', start_chapter=2, start_chapter_verse_ref='1-12', end_chapter=None, end_chapter_verse_ref=None)
+ """
+ path = join(resource_dir, docx_file_path)
+ # logger.debug("path: %s exists: %s", path, exists(path))
+ # TODO Check if resource_dir exists and if it doesn't then submit
+ # a document request to DOC API to make sure it is cloned.
+ # Currently we don't have to do this because at startup we copy
+ # English OT Survey RG2 doc into place.
+ # assert exists(path)
+ rg_books = get_rg_books(
+ path,
+ lang_code,
+ lang_name,
+ resource_type_name,
+ lang_direction,
+ )
+ rg_book_chapters = [
+ chapter for rg_book in rg_books for chapter in rg_book.chapters.values()
+ ]
+ bible_references = [
+ pt.bible_reference for chapter in rg_book_chapters for pt in chapter.content
+ ]
+ # Localize the book names since they are provided in English from en_ot_survey_rg2_jos_est.docx
+ book_name_map = {
+ book_code_and_name[0]: book_code_and_name[1]
+ for book_code_and_name in book_codes_for_lang_from_usfm_only(lang_code)
+ }
+ for bible_reference in bible_references:
+ maybe_localized_book_name = book_name_map.get(
+ bible_reference.book_code, bible_reference.book_name
+ )
+ logger.debug("maybe_localized_book_name: %s", maybe_localized_book_name)
+ bible_reference.book_name = maybe_localized_book_name
+ return bible_references
+
+
+def ot_survey_rg3_passages(
+ lang_code: str = "en",
+ lang_name: str = "English",
+ docx_file_path: str = "en_ot_survey_rg3_job_sng.docx",
+ resource_type_name: str = "OT Survey Reviewers' Guide (Job to Song of Songs)",
+ lang_direction: LangDirEnum = LangDirEnum.LTR,
+ resource_dir: str = settings.EN_RG_DIR,
+) -> list[BibleReference]:
+ """
+ >>> from doc.domain import resource_lookup
+ >>> rg_books = resource_lookup.ot_survey_rg3_passages()
+ >>> rg_books[0]
+ BibleReference(book_code='job', book_name='Job', start_chapter=2, start_chapter_verse_ref='1-12', end_chapter=None, end_chapter_verse_ref=None)
+ """
+ path = join(resource_dir, docx_file_path)
+ # logger.debug("path: %s exists: %s", path, exists(path))
+ # TODO Check if resource_dir exists and if it doesn't then submit
+ # a document request to DOC API to make sure it is cloned.
+ # Currently we don't have to do this because at startup we copy
+ # English OT Survey RG3 doc into place.
+ # assert exists(path)
+ rg_books = get_rg_books(
+ path,
+ lang_code,
+ lang_name,
+ resource_type_name,
+ lang_direction,
+ )
+ rg_book_chapters = [
+ chapter for rg_book in rg_books for chapter in rg_book.chapters.values()
+ ]
+ bible_references = [
+ pt.bible_reference for chapter in rg_book_chapters for pt in chapter.content
+ ]
+ # Localize the book names since they are provided in English from en_ot_survey_rg3_job_sng.docx
+ book_name_map = {
+ book_code_and_name[0]: book_code_and_name[1]
+ for book_code_and_name in book_codes_for_lang_from_usfm_only(lang_code)
+ }
+ for bible_reference in bible_references:
+ maybe_localized_book_name = book_name_map.get(
+ bible_reference.book_code, bible_reference.book_name
+ )
+ logger.debug("maybe_localized_book_name: %s", maybe_localized_book_name)
+ bible_reference.book_name = maybe_localized_book_name
+ return bible_references
+
+
+def ot_survey_rg4_passages(
+ lang_code: str = "en",
+ lang_name: str = "English",
+ docx_file_path: str = "en_ot_survey_rg4_isa_mal.docx",
+ resource_type_name: str = "OT Survey Reviewers' Guide (Isaiah to Malachi)",
+ lang_direction: LangDirEnum = LangDirEnum.LTR,
+ resource_dir: str = settings.EN_RG_DIR,
+) -> list[BibleReference]:
+ """
+ >>> from doc.domain import resource_lookup
+ >>> rg_books = resource_lookup.ot_survey_rg4_passages()
+ >>> rg_books[0]
+ BibleReference(book_code='isa', book_name='Isaiah', start_chapter=2, start_chapter_verse_ref='1-12', end_chapter=None, end_chapter_verse_ref=None)
+ """
+ path = join(resource_dir, docx_file_path)
+ # logger.debug("path: %s exists: %s", path, exists(path))
+ # TODO Check if resource_dir exists and if it doesn't then submit
+ # a document request to DOC API to make sure it is cloned.
+ # Currently we don't have to do this because at startup we copy
+ # English OT Survey RG4 doc into place.
+ # assert exists(path)
+ rg_books = get_rg_books(
+ path,
+ lang_code,
+ lang_name,
+ resource_type_name,
+ lang_direction,
+ )
+ rg_book_chapters = [
+ chapter for rg_book in rg_books for chapter in rg_book.chapters.values()
+ ]
+ bible_references = [
+ pt.bible_reference for chapter in rg_book_chapters for pt in chapter.content
+ ]
+ # Localize the book names since they are provided in English from en_ot_survey_rg4_isa_mal.docx
+ book_name_map = {
+ book_code_and_name[0]: book_code_and_name[1]
+ for book_code_and_name in book_codes_for_lang_from_usfm_only(lang_code)
+ }
+ for bible_reference in bible_references:
+ maybe_localized_book_name = book_name_map.get(
+ bible_reference.book_code, bible_reference.book_name
+ )
+ logger.debug("maybe_localized_book_name: %s", maybe_localized_book_name)
+ bible_reference.book_name = maybe_localized_book_name
+ return bible_references
+
+
if __name__ == "__main__":
# To run the doctests in this module, in the root of the project do:
diff --git a/backend/doc/domain/usfm_error_detection_and_fixes.py b/backend/doc/domain/usfm_error_detection_and_fixes.py
index 2aecbf92c..f4f0ef9ff 100644
--- a/backend/doc/domain/usfm_error_detection_and_fixes.py
+++ b/backend/doc/domain/usfm_error_detection_and_fixes.py
@@ -19,7 +19,6 @@
("ach-SS-acholi", "reg", "gal"),
("adh", "reg", "1th"),
("adn", "reg", "mat"),
- ("aec", "reg", "mat"),
("agd-x-namel", "reg", "2th"),
("ahm", "reg", "php"),
("ahm", "reg", "php"),
@@ -44,18 +43,8 @@
("bi", "reg", "act"),
("bji", "reg", "mat"),
("bji", "reg", "1co"),
- ("bji", "reg", "jud"),
- ("bji", "reg", "1co"),
- ("bji", "reg", "luk"),
- ("bji", "reg", "gal"),
- ("bji", "reg", "col"),
- ("bji", "reg", "2pe"),
("bji", "reg", "luk"),
- ("bji", "reg", "col"),
- ("bji", "reg", "php"),
("bji", "reg", "heb"),
- ("bji", "reg", "1jn"),
- ("bji", "reg", "col"),
("bjz", "reg", "eph"),
("blo", "reg", "rom"),
("blo", "reg", "act"),
@@ -93,7 +82,6 @@
("byn", "reg", "2pe"),
("bzu", "reg", "tit"),
("cbt", "reg", "jos"),
- ("cbt", "reg", "rut"),
("cbt", "reg", "est"),
("ccp", "reg", "mat"),
("ccp", "reg", "gal"),
@@ -106,7 +94,6 @@
("erk-x-epang", "reg", "php"),
("eyo", "reg", "2jn"),
("eyo", "reg", "php"),
- # ("gow", "reg", "3jn"), # 3jn is given as choice, but it is not cloned, so does it exist?
("gux-x-gourmantche", "reg", "deu"),
("gux-x-gourmantche", "reg", "jon"),
("gux-x-gourmantche", "reg", "jos"),
@@ -140,12 +127,12 @@
("kng-x-kilemfu", "reg", "eph"),
("kod", "reg", "2ti"),
("kod", "reg", "phm"),
- ("kqi", "reg", "2th"),
- ("kqi", "reg", "2ti"),
- # ("kqi", "reg", "mrk"), # book not available from data api anymore
+ # ("kqi", "reg", "2th"), # unavailable from data API
+ # ("kqi", "reg", "2ti"), # unavailable from data API
+ # ("kqi", "reg", "mrk"), # book no longer available from data api
("kqi", "reg", "heb"),
- ("kqi", "reg", "1pe"),
- ("kqi", "reg", "tit"),
+ # ("kqi", "reg", "1pe"), # unavailable from data API
+ # ("kqi", "reg", "tit"), # unavailable from data API
("ksm", "reg", "rom"),
("ksm", "reg", "1pe"),
("ksm", "reg", "2ti"),
@@ -164,7 +151,7 @@
("lky", "reg", "2th"),
("lbx-x-capuracu", "reg", "eph"),
("mfq-x-mual", "reg", "1ki"),
- ("mgs", "reg", "php"),
+ # ("mgs", "reg", "php"), # book no longer available from data api
("mgs", "reg", "2th"),
("mhi-x-burolo", "reg", "mat"),
("mhi-x-burolo", "reg", "2jn"),
@@ -173,7 +160,7 @@
("mhi-x-burolo", "reg", "2th"),
("mhy-x-benualima", "reg", "mrk"),
# ("mwe", "reg", "tit"), # book is available as choice, but resource not cloned?
- ("mxo", "reg", "mrk"),
+ # ("mxo", "reg", "mrk"), # unavailable from data API
("nak-x-bileki", "reg", "mat"),
("nak-x-bileki", "reg", "1ti"),
("nak-x-bileki", "reg", "eph"),
@@ -187,7 +174,7 @@
("nfd", "reg", "2th"),
("nfd", "reg", "2ti"),
("nfd", "reg", "heb"),
- ("nhx", "reg", "jos"),
+ # ("nhx", "reg", "jos"), # unavailable from data API
("nnb-x-kishula", "reg", "mrk"),
("not", "reg", "jos"),
("now", "reg", "mic"),
@@ -220,7 +207,6 @@
("pip", "reg", "rom"),
("pip", "reg", "mat"),
("pse-x-riauasli", "reg", "luk"),
- # ("rmn-x-yerliroman", "reg", "mat"), # Feb 20, 2025: This language seems to not be served by data API. More investigation needed.
# ("rmp", "ulb", "jas"), # failed to fix; repo is cloned and source looks good other than duplicate \c markers, but we handle those (BUG?)
("ruc", "reg", "jhn"),
("ruc", "reg", "1ti"),
@@ -253,8 +239,6 @@
("spy-x-pok", "reg", "jud"),
("spy-x-pok", "reg", "3jn"),
("ssc-x-kine", "reg", "2jn"),
- ("ssn-x-sanye", "reg", "col"),
- ("tar-x-ralamuli", "reg", "mrk"),
("tbp-x-airo", "reg", "php"),
("thr", "reg", "tit"),
("ttl-x-totelnamib", "reg", "3jn"),
@@ -262,10 +246,10 @@
("txy", "reg", "2jn"),
("tyn", "reg", "jud"),
("tyn", "reg", "mrk"),
- ("vin", "reg", "2co"),
- ("vin", "reg", "1th"),
- ("vin", "reg", "1ti"),
- ("vin", "reg", "gal"),
+ # ("vin", "reg", "2co"), # no longer available from data API
+ # ("vin", "reg", "1th"),
+ # ("vin", "reg", "1ti"),
+ # ("vin", "reg", "gal"),
("wbj", "reg", "luk"),
("wbj", "reg", "tit"),
("wbj", "reg", "2jn"),
@@ -294,6 +278,7 @@
"fix_missing_space_after_number": r"(\s+|^)(\d+)(\S+)",
"fix_missing_space_before_verse_marker": r"(\S)(\\v\s+\d+)",
"fix_standalone_verse_numbers": r"(? str:
return content
+def fix_standalone_verse_number_and_period(content: str) -> str:
+ # E.g., in Russian (ru) some of the USFM exhibits verse markers of the
+ # form 1. rather than \v 1
+ if match := compiled_patterns["fix_standalone_verse_number_and_period"].search(
+ content
+ ):
+ # Calculate safe slice indices
+ length_of_context = 50
+ num_of_occurrences = 3
+ start_index = max(0, match.start() - length_of_context)
+ end_index = min(len(content), match.end() + length_of_context)
+ context_for_standalone_verse_and_period = content[start_index:end_index]
+ logger.debug(
+ "context_for_standalone_verse_and_period: %s",
+ context_for_standalone_verse_and_period,
+ )
+ # Extract all standalone number and period (likely verse numbers) from
+ # content but skip the first part, 6 characters, of content
+ # which could contain a chapter marker and its value.
+ matches = [int(m.group(1)) for m in re.finditer(r"\b(\d+)\.", content[7:])]
+ logger.debug("standalone verse number and period matches: %s", matches)
+ is_ascending = all(
+ earlier < later for earlier, later in zip(matches, matches[1:])
+ )
+ logger.debug("is_ascending: %s", is_ascending)
+ num_matches = len(matches)
+ if (
+ not re.compile(r"""\\v \d+""").search(
+ context_for_standalone_verse_and_period
+ )
+ and not num_matches >= num_of_occurrences
+ ) or is_ascending: # Check for non-ascending numbers
+ return re.sub(
+ pattern_matchers["fix_standalone_verse_number_and_period"],
+ r"\\v \1 \2",
+ content,
+ )
+ else:
+ logger.info(
+ "Actually, we can't be certain it was a standalone verse number and period after all upon further checking"
+ )
+ return content
+
+
def replace_n_with_v(content: str) -> str:
"""Replace \n used mistakenly as verse markers with \v"""
return re.sub(pattern_matchers["replace_n_with_v"], r"""\\v""", content)
@@ -485,7 +514,7 @@ def fix_usfm(
"""
# logger.debug("Possibly defective USFM content: %s", usfm_content)
corrected_usfm_content: str = usfm_content
- # NOTE This is called in a different place now, leaving commented out fo now.
+ # NOTE This is called in a different place now, leaving commented out for now.
# if compiled_patterns["remove_null_bytes_and_control_characters"].search(
# corrected_usfm_content
# ):
@@ -603,6 +632,25 @@ def fix_usfm(
book_code,
)
corrected_usfm_content = fix_standalone_verse_numbers(corrected_usfm_content)
+ if match := compiled_patterns["fix_standalone_verse_number_and_period"].search(
+ corrected_usfm_content
+ ):
+ logger.debug(
+ "Possible USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, if confirmed, attempt to fix...",
+ "fix_standalone_verse_number_and_period",
+ match.group(),
+ corrected_usfm_content[
+ max(0, match.start() - 5) : min(
+ len(corrected_usfm_content), match.end() + 5
+ )
+ ],
+ lang_code,
+ resource_type,
+ book_code,
+ )
+ corrected_usfm_content = fix_standalone_verse_number_and_period(
+ corrected_usfm_content
+ )
if match := compiled_patterns["replace_n_with_v"].search(corrected_usfm_content):
logger.debug(
"USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
diff --git a/backend/doc/entrypoints/app.py b/backend/doc/entrypoints/app.py
index 92b798101..5da569044 100644
--- a/backend/doc/entrypoints/app.py
+++ b/backend/doc/entrypoints/app.py
@@ -1,30 +1,19 @@
"""This module provides the FastAPI API definition."""
+import shutil
from os import makedirs
from os.path import join, exists
-import shutil
from doc.config import settings
from doc.domain import exceptions
from doc.entrypoints.routes import router as doc_router
-from stet.entrypoints.routes import router as stet_router
-from passages.entrypoints.routes import router as passages_router
from fastapi import FastAPI, Request, status
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
+from passages.entrypoints.routes import router as passages_router
+from stet.entrypoints.routes import router as stet_router
-# Docker container paths
-DOCKER_BASE_DIR = "/app"
-DOCKER_ASSETS_DOWNLOAD_DIR = join(DOCKER_BASE_DIR, settings.RESOURCE_ASSETS_DIR)
-DOCKER_EN_RG_DIR = join(DOCKER_ASSETS_DOWNLOAD_DIR, "en_rg")
-DOCKER_DOCX_FILE_SRC = join(DOCKER_BASE_DIR, "en_rg_nt_survey.docx")
-DOCKER_DOCX_FILE_DEST = join(DOCKER_EN_RG_DIR, "en_rg_nt_survey.docx")
-# Local filesystem paths
-LOCAL_ASSETS_DOWNLOAD_DIR = settings.RESOURCE_ASSETS_DIR
-LOCAL_EN_RG_DIR = join(LOCAL_ASSETS_DOWNLOAD_DIR, "en_rg")
-LOCAL_DOCX_FILE_SRC = "en_rg_nt_survey.docx"
-LOCAL_DOCX_FILE_DEST = join(LOCAL_EN_RG_DIR, "en_rg_nt_survey.docx")
app = FastAPI()
@@ -83,25 +72,78 @@ async def validation_exception_handler(
)
-# Until reviewer's guides can be accessed via data API, create their
-# directory and copy them into place
-@app.on_event("startup")
-async def initialize_assets() -> None:
+DOCKER_BASE_DIR = "/app"
+
+
+SURVEY_FILES = {
+ "nt": "en_rg_nt_survey.docx",
+ "ot_rg1": "en_ot_survey_rg1_gen_deu.docx",
+ "ot_rg2": "en_ot_survey_rg2_jos_est.docx",
+ "ot_rg3": "en_ot_survey_rg3_job_sng.docx",
+ "ot_rg4": "en_ot_survey_rg4_isa_mal.docx",
+}
+
+
+def build_paths(base_dir: str, en_rg_dir: str) -> dict[str, tuple[str, str]]:
+ """
+ Build src/dest pairs for each survey file.
+ Returns a dict where key is 'nt', 'ot_rg1', etc.
+ Each value is (src, dest).
+ """
+ return {
+ key: (
+ join(base_dir, filename),
+ join(en_rg_dir, filename),
+ )
+ for key, filename in SURVEY_FILES.items()
+ }
+
+
+DOCKER_PATHS = build_paths(DOCKER_BASE_DIR, join(DOCKER_BASE_DIR, settings.EN_RG_DIR))
+LOCAL_PATHS = build_paths("", settings.EN_RG_DIR) # src is just filename in local case
+
+
+def initialize_assets(
+ docker_base_dir: str = DOCKER_BASE_DIR,
+ docker_paths: dict[str, tuple[str, str]] = DOCKER_PATHS,
+ survey_files: dict[str, str] = SURVEY_FILES,
+ resource_assets_dir: str = settings.RESOURCE_ASSETS_DIR,
+ local_paths: dict[str, tuple[str, str]] = LOCAL_PATHS,
+) -> None:
"""
Ensures the en_rg directory and the .docx file exist in the assets_download volume.
"""
try:
- if exists(DOCKER_BASE_DIR): # Executing inside Docker container
- makedirs(DOCKER_EN_RG_DIR, exist_ok=True)
- if not exists(DOCKER_DOCX_FILE_DEST):
- shutil.copy(DOCKER_DOCX_FILE_SRC, DOCKER_DOCX_FILE_DEST)
- elif exists(LOCAL_ASSETS_DOWNLOAD_DIR): # Executing outside Docker container
- makedirs(LOCAL_EN_RG_DIR, exist_ok=True)
- if not exists(LOCAL_DOCX_FILE_DEST):
- shutil.copy(LOCAL_DOCX_FILE_SRC, LOCAL_DOCX_FILE_DEST)
- print("Assets initialized successfully.")
+ if exists(docker_base_dir): # inside Docker
+ makedirs(docker_paths["nt"][1].rsplit("/", 1)[0], exist_ok=True)
+ for key, (src, dest) in docker_paths.items():
+ if not exists(dest):
+ shutil.copy(src, dest)
+ if not exists(dest):
+ raise AssertionError(
+ f"{survey_files[key]} not copied into place!"
+ )
+ elif exists(resource_assets_dir): # outside Docker
+ makedirs(local_paths["nt"][1].rsplit("/", 1)[0], exist_ok=True)
+ for src, dest in local_paths.values():
+ if not exists(dest):
+ shutil.copy(src, dest)
+ if not exists(dest):
+ raise AssertionError(
+ f"{survey_files[key]} not copied into place!"
+ )
+ logger.info("Assets initialized successfully.")
except Exception as e:
- print(f"Error initializing assets: {e}")
+ logger.info(f"Error initializing assets: {e}")
+
+
+
+
+# Until reviewer's guides can be accessed via data API, create their
+# directory and copy them into place
+@app.on_event("startup")
+async def startup() -> None:
+ initialize_assets()
app.include_router(doc_router)
diff --git a/backend/doc/entrypoints/routes.py b/backend/doc/entrypoints/routes.py
index a67f82031..b4f0f308e 100644
--- a/backend/doc/entrypoints/routes.py
+++ b/backend/doc/entrypoints/routes.py
@@ -1,16 +1,14 @@
from typing import Sequence
-from fastapi import APIRouter
import celery.states
from celery.result import AsyncResult
from doc.config import settings
from doc.domain import document_generator, model, resource_lookup
from doc.reviewers_guide.model import BibleReference
-
-from fastapi import HTTPException, status
-
+from fastapi import APIRouter, HTTPException, status
from fastapi.responses import JSONResponse
+
router = APIRouter()
logger = settings.logger(__name__)
@@ -146,6 +144,42 @@ async def nt_survey_rg_passages(lang_code: str) -> Sequence[BibleReference]:
return bible_references
+@router.get("/ot_survey_rg1_passages/{lang_code}")
+async def ot_survey_rg1_passages(lang_code: str) -> Sequence[BibleReference]:
+ """
+ Return list of reified OT Survey Reviewer's Guide 1 passages as BibleReference instances.
+ """
+ bible_references = resource_lookup.ot_survey_rg1_passages(lang_code)
+ return bible_references
+
+
+@router.get("/ot_survey_rg2_passages/{lang_code}")
+async def ot_survey_rg2_passages(lang_code: str) -> Sequence[BibleReference]:
+ """
+ Return list of reified OT Survey Reviewer's Guide 2 passages as BibleReference instances.
+ """
+ bible_references = resource_lookup.ot_survey_rg2_passages(lang_code)
+ return bible_references
+
+
+@router.get("/ot_survey_rg3_passages/{lang_code}")
+async def ot_survey_rg3_passages(lang_code: str) -> Sequence[BibleReference]:
+ """
+ Return list of reified OT Survey Reviewer's Guide 3 passages as BibleReference instances.
+ """
+ bible_references = resource_lookup.ot_survey_rg3_passages(lang_code)
+ return bible_references
+
+
+@router.get("/ot_survey_rg4_passages/{lang_code}")
+async def ot_survey_rg4_passages(lang_code: str) -> Sequence[BibleReference]:
+ """
+ Return list of reified OT Survey Reviewer's Guide 4 passages as BibleReference instances.
+ """
+ bible_references = resource_lookup.ot_survey_rg4_passages(lang_code)
+ return bible_references
+
+
# @router.get("/chapters_in_book/{book_code}")
# async def chapters_in_book(book_code: str) -> list[int]:
# return resource_lookup.chapters_in_book(book_code)
diff --git a/backend/doc/markdown_transforms/markdown_transformer.py b/backend/doc/markdown_transforms/markdown_transformer.py
index 22cab2c91..2f7f5e589 100644
--- a/backend/doc/markdown_transforms/markdown_transformer.py
+++ b/backend/doc/markdown_transforms/markdown_transformer.py
@@ -1,14 +1,11 @@
-from os.path import exists, join
import re
+from os.path import exists, join
from re import finditer, search
from typing import Sequence
from doc.config import settings
from doc.domain.bible_books import BOOK_NUMBERS
from doc.domain.model import ResourceRequest
-from doc.markdown_transforms.model import (
- WikiLink,
-)
from doc.markdown_transforms.link_regexes import (
TA_MARKDOWN_HTTPS_LINK_RE,
TA_PREFIXED_MARKDOWN_HTTPS_LINK_RE,
@@ -29,9 +26,13 @@
TW_WIKI_RC_LINK_RE2,
WIKI_LINK_RE,
)
+from doc.markdown_transforms.model import (
+ WikiLink,
+)
from doc.utils.file_utils import read_file
from doc.utils.tw_utils import localized_translation_word
+
logger = settings.logger(__name__)
TRANSLATION_WORD_ANCHOR_LINK_FMT_STR: str = "[{}](#{}-{})"
@@ -559,6 +560,8 @@ def transform_ta_wiki_rc_links(source: str) -> str:
return source
+# TODO zh gen, e.g., 1:20 you end up with things like:（参：）. We
+# should probably remove the whole parenthesized expression.
def transform_ta_star_rc_links(source: str) -> str:
"""
Transform the translation academy rc wikilink into source anchor link
@@ -573,6 +576,8 @@ def transform_ta_star_rc_links(source: str) -> str:
return source
+# TODO zh gen, e.g., 1:20 you end up with things like:（参：）. We
+# should probably remove the whole parenthesized expression.
def transform_ta_markdown_links(source: str) -> str:
"""
Transform the translation academy markdown link into source anchor link
diff --git a/backend/doc/reviewers_guide/__init__.py b/backend/doc/reviewers_guide/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/doc/reviewers_guide/model.py b/backend/doc/reviewers_guide/model.py
index d3fd9a6ad..2f1b48100 100644
--- a/backend/doc/reviewers_guide/model.py
+++ b/backend/doc/reviewers_guide/model.py
@@ -1,7 +1,8 @@
"""Models for reviewer's guide"""
-from typing import Optional, final
from pprint import pformat
+from typing import Optional, final
+
from doc.domain.model import ChapterNum, LangDirEnum
from pydantic import BaseModel
@@ -28,6 +29,37 @@ class BibleReference(BaseModel):
end_chapter: Optional[ChapterNum]
end_chapter_verse_ref: Optional[str]
+ def __hash__(self: "BibleReference") -> int:
+ return hash(
+ (
+ self.book_code,
+ self.book_name,
+ self.start_chapter,
+ self.start_chapter_verse_ref,
+ self.end_chapter,
+ self.end_chapter_verse_ref,
+ )
+ )
+
+ def __eq__(self: "BibleReference", other: object) -> bool:
+ if not isinstance(other, BibleReference):
+ return NotImplemented
+ return (
+ self.book_code,
+ self.book_name,
+ self.start_chapter,
+ self.start_chapter_verse_ref,
+ self.end_chapter,
+ self.end_chapter_verse_ref,
+ ) == (
+ other.book_code,
+ other.book_name,
+ other.start_chapter,
+ other.start_chapter_verse_ref,
+ other.end_chapter,
+ other.end_chapter_verse_ref,
+ )
+
@final
class ParsedText(BaseModel):
diff --git a/backend/doc/reviewers_guide/parser.py b/backend/doc/reviewers_guide/parser.py
index 2aebeb790..def7268bb 100644
--- a/backend/doc/reviewers_guide/parser.py
+++ b/backend/doc/reviewers_guide/parser.py
@@ -1,9 +1,8 @@
-from pprint import pprint
-from collections import defaultdict
import re
+from collections import defaultdict
+from pprint import pprint
from doc.config import settings
-from docx import Document # type: ignore
from doc.domain.bible_books import BOOK_NAMES
from doc.domain.model import ChapterNum, LangDirEnum
from doc.reviewers_guide.model import (
@@ -14,6 +13,7 @@
BibleReference,
ParsedText,
)
+from docx import Document # type: ignore
# Pattern for chapter and verse references after Bible books
@@ -22,74 +22,138 @@
logger = settings.logger(__name__)
-def get_book_code(book_name: str) -> str:
- return next(key for key, name in BOOK_NAMES.items() if name == book_name)
+def get_book_code(book_name: str, book_names: dict[str, str] = BOOK_NAMES) -> str:
+ return next(key for key, name in book_names.items() if name == book_name)
def parse_bible_reference(
- raw_bible_reference: str, book_names: dict[str, str] = BOOK_NAMES
+ raw_bible_reference: str, book_names: list[str] = list(BOOK_NAMES.values())
) -> BibleReference:
bible_reference_components = raw_bible_reference.split()
- if len(bible_reference_components) == 3:
+ if (
+ bible_reference_components
+ and len(bible_reference_components) == 3
+ and (
+ f"{bible_reference_components[0]} {bible_reference_components[1]}"
+ in book_names
+ )
+ and ":" in bible_reference_components[2]
+ and bible_reference_components[2][0].isdigit()
+ ):
book_name = f"{bible_reference_components[0]} {bible_reference_components[1]}"
book_code = get_book_code(book_name)
- temp_components = bible_reference_components[2].split(":")
- if len(temp_components) == 3 and "-" in temp_components[1]:
- start_chapter = temp_components[0]
- temp_components_ = temp_components[1].split("-")
- start_chapter_verse_ref = temp_components_[0]
- end_chapter = temp_components_[1]
- end_chapter_verse_ref = temp_components[2]
- bible_reference = BibleReference(
- book_code=book_code,
- book_name=book_name,
- start_chapter=int(start_chapter),
- start_chapter_verse_ref=start_chapter_verse_ref,
- end_chapter=int(end_chapter),
- end_chapter_verse_ref=end_chapter_verse_ref,
+ chapter_verse_components = bible_reference_components[2].split(":")
+ logger.debug("chapter_verse_components: %s", chapter_verse_components)
+ if len(chapter_verse_components) == 3 and "-" in chapter_verse_components[1]:
+ # e.g., 1 Corinthians 3:4-4:12 -> chapter_verse_components = ["3", "4-4", "12"]
+ bible_reference = get_bible_reference_spanning_chapter_boundary(
+ chapter_verse_components, book_code, book_name
)
- # elif len(temp_components) == 2:
else:
- start_chapter = temp_components[0]
- start_chapter_verse_ref = temp_components[1]
- bible_reference = BibleReference(
- book_code=book_code,
- book_name=book_name,
- start_chapter=int(start_chapter),
- start_chapter_verse_ref=start_chapter_verse_ref,
- end_chapter=None,
- end_chapter_verse_ref=None,
+ # e.g., 1 Corinthians 3:12-14 -> chapter_verse_components = ["3", "12-14"]
+ # or
+ # e.g., 1 Corinthians 3:12 -> chapter_verse_components = ["3", "12"]
+ bible_reference = get_ordinal_bible_reference(
+ chapter_verse_components, book_code, book_name
)
- else: # if len(bible_reference_components) == 2:
+ elif (
+ bible_reference_components
+ and len(bible_reference_components) == 2
+ and bible_reference_components[0] in book_names
+ and ":" in bible_reference_components[1]
+ and bible_reference_components[1][0].isdigit()
+ ):
book_name = bible_reference_components[0]
book_code = get_book_code(book_name)
- temp_components = bible_reference_components[1].split(":")
- if len(temp_components) == 3 and "-" in temp_components[1]:
- start_chapter = temp_components[0]
- temp_components_ = temp_components[1].split("-")
- start_chapter_verse_ref = temp_components_[0]
- end_chapter = temp_components_[1]
- end_chapter_verse_ref = temp_components[2]
- bible_reference = BibleReference(
- book_code=book_code,
- book_name=book_name,
- start_chapter=int(start_chapter),
- start_chapter_verse_ref=start_chapter_verse_ref,
- end_chapter=int(end_chapter),
- end_chapter_verse_ref=end_chapter_verse_ref,
+ chapter_verse_components = bible_reference_components[1].split(":")
+ if len(chapter_verse_components) == 3 and "-" in chapter_verse_components[1]:
+ # e.g., Samuel 4:3-6:10 -> chapter_verse_components = ["4", "3-6", "10"]
+ bible_reference = get_bible_reference_spanning_chapter_boundary(
+ chapter_verse_components, book_code, book_name
)
else:
- start_chapter = temp_components[0]
- start_chapter_verse_ref = temp_components[1]
- bible_reference = BibleReference(
- book_code=book_code,
- book_name=book_name,
- start_chapter=int(start_chapter),
- start_chapter_verse_ref=start_chapter_verse_ref,
- end_chapter=None,
- end_chapter_verse_ref=None,
+ # e.g., Samuel 4:3-6 -> chapter_verse_components = ["4", "3-6"]
+ bible_reference = get_ordinal_bible_reference(
+ chapter_verse_components, book_code, book_name
)
- logger.debug("bible_reference: %s", bible_reference)
+ else:
+ logger.info("Likely parsing an OT rg bible reference")
+ if len(bible_reference_components) >= 3:
+ if bible_reference_components[2][0].isdigit():
+ # Likely an OT reviewer's guide reference which sometimes looks like ['1', 'Samuel', '1:1-2:3', 'foo', 'bar']
+ bible_reference_components = bible_reference_components[0:3]
+ logger.debug(
+ "1. Updated bible_reference_components: %s",
+ bible_reference_components,
+ )
+ book_name = (
+ f"{bible_reference_components[0]} {bible_reference_components[1]}"
+ )
+ book_code = get_book_code(book_name)
+ chapter_verse_components = bible_reference_components[2].split(":")
+ bible_reference = get_bible_reference_spanning_chapter_boundary(
+ chapter_verse_components, book_code, book_name
+ )
+ elif bible_reference_components[1][
+ 0
+ ].isdigit() and not bible_reference_components[2].endswith("continued"):
+ # Likely an OT reviewer's guide reference which sometimes looks like ['Genesis', '1:1-2:3', 'God', 'creates', 'everything']
+ bible_reference_components = bible_reference_components[0:2]
+ logger.debug(
+ "2. Updated bible_reference_components: %s",
+ bible_reference_components,
+ )
+ book_name = bible_reference_components[0]
+ book_code = get_book_code(book_name)
+ chapter_verse_components = bible_reference_components[1].split(":")
+ bible_reference = get_ordinal_bible_reference(
+ chapter_verse_components, book_code, book_name
+ )
+ return bible_reference
+
+
+def get_ordinal_bible_reference(
+ chapter_verse_components: list[str], book_code: str, book_name: str
+) -> BibleReference:
+ start_chapter = chapter_verse_components[0]
+ start_chapter_verse_ref = chapter_verse_components[1]
+ bible_reference = BibleReference(
+ book_code=book_code,
+ book_name=book_name,
+ start_chapter=int(start_chapter),
+ start_chapter_verse_ref=start_chapter_verse_ref,
+ end_chapter=None,
+ end_chapter_verse_ref=None,
+ )
+ return bible_reference
+
+
+def get_bible_reference_spanning_chapter_boundary(
+ chapter_verse_components: list[str], book_code: str, book_name: str
+) -> BibleReference:
+ # e.g., Samuel 4:3-6:10 -> chapter_verse_components = ["4", "3-6", "10"]
+ start_chapter = chapter_verse_components[0]
+ verse_chapter_components = chapter_verse_components[1].split("-")
+ # e.g., verse_chapter_components = ["3", "6"]
+ start_chapter_verse_ref = (
+ verse_chapter_components[0]
+ if len(chapter_verse_components) >= 3
+ else chapter_verse_components[1]
+ )
+ end_chapter = (
+ verse_chapter_components[1] if len(chapter_verse_components) >= 3 else None
+ )
+ end_chapter_verse_ref = (
+ chapter_verse_components[2] if len(chapter_verse_components) >= 3 else None
+ )
+ bible_reference = BibleReference(
+ book_code=book_code,
+ book_name=book_name,
+ start_chapter=int(start_chapter),
+ start_chapter_verse_ref=start_chapter_verse_ref,
+ end_chapter=int(end_chapter) if end_chapter else None,
+ end_chapter_verse_ref=end_chapter_verse_ref,
+ )
return bible_reference
@@ -117,29 +181,59 @@ def find_bible_references(
bible_references: list[str] = []
current_text: list[str] = []
inside_bible_reference: bool = False
- for paragraph in doc.paragraphs:
- # Check if the paragraph is a potential Bible reference
- paragraph_text = paragraph.text.strip()
- words = paragraph_text.split()
- if (
- words
- and len(words) > 1
- and len(words) < 5
- and (words[0] in book_names or (f"{words[0]} {words[1]}" in book_names))
- ):
+ testament_paragraph = doc.paragraphs[2]
+ if "Old Testament" in testament_paragraph.text.strip():
+ logger.info("Parsing Old Testament rg document")
+ for paragraph in doc.paragraphs:
+ # Check if the paragraph is a potential Bible reference
+ paragraph_text = paragraph.text.strip()
+ words = paragraph_text.split()
+ if (
+ words
+ and len(words) > 1
+ and (words[0] in book_names or (f"{words[0]} {words[1]}" in book_names))
+ and (
+ words[1][0].isdigit() or (len(words) > 2 and words[2][0].isdigit())
+ )
+ ):
+ if paragraph_text.endswith(
+ "continued"
+ ): # or "\t" in paragraph_text # \t is in paragraph_text when it is a TOC entry
+ continue
+ if inside_bible_reference:
+ if current_text:
+ between_texts.append(" ".join(current_text))
+ current_text = []
+ bible_references.append(paragraph_text)
+ inside_bible_reference = True
+ continue
+ if inside_bible_reference:
+ current_text.append(paragraph_text)
+ else:
+ logger.info("Parsing New Testament rg document")
+ for paragraph in doc.paragraphs:
+ # Check if the paragraph is a potential Bible reference
+ paragraph_text = paragraph.text.strip()
+ words = paragraph_text.split()
if (
- paragraph_text.endswith("continued") or "\t" in paragraph_text
- ): # \t is in paragraph_text when it is a TOC entry
+ words
+ and len(words) > 1
+ and len(words) < 5
+ and (words[0] in book_names or (f"{words[0]} {words[1]}" in book_names))
+ ):
+ if (
+ paragraph_text.endswith("continued") or "\t" in paragraph_text
+ ): # \t is in paragraph_text when it is a TOC entry
+ continue
+ if inside_bible_reference:
+ if current_text:
+ between_texts.append(" ".join(current_text))
+ current_text = []
+ bible_references.append(paragraph_text)
+ inside_bible_reference = True
continue
if inside_bible_reference:
- if current_text:
- between_texts.append(" ".join(current_text))
- current_text = []
- bible_references.append(paragraph_text)
- inside_bible_reference = True
- continue
- if inside_bible_reference:
- current_text.append(paragraph_text)
+ current_text.append(paragraph_text)
if current_text:
between_texts.append(" ".join(current_text))
return between_texts, bible_references
@@ -174,6 +268,8 @@ def parse_text(text: str, raw_bible_reference: str) -> ParsedText:
sections = re.split(r"(Background:|Part 1|Part 2|Comment Section:)", text)
current_section = None
background = None
+ directive = ""
+ comment_section = None
for section in sections:
section = section.strip()
if section == "Background:":
@@ -297,6 +393,46 @@ def get_rg_books(
rg_books = get_rg_books(
docx_file_path, "en", "English", "Reviewers' Guide NT Survey", LangDirEnum.LTR
)
+ for rg_book in rg_books:
+ pprint(rg_book)
+ docx_file_path = "en_ot_survey_rg1_gen_deu.docx"
+ rg_books = get_rg_books(
+ docx_file_path,
+ "en",
+ "English",
+ "Reviewers' Guide OT Survey RG1",
+ LangDirEnum.LTR,
+ )
+ for rg_book in rg_books:
+ pprint(rg_book)
+ docx_file_path = "en_ot_survey_rg2_jos_est.docx"
+ rg_books = get_rg_books(
+ docx_file_path,
+ "en",
+ "English",
+ "Reviewers' Guide OT Survey RG2",
+ LangDirEnum.LTR,
+ )
+ for rg_book in rg_books:
+ pprint(rg_book)
+ docx_file_path = "en_ot_survey_rg3_job_sng.docx"
+ rg_books = get_rg_books(
+ docx_file_path,
+ "en",
+ "English",
+ "Reviewers' Guide OT Survey RG3",
+ LangDirEnum.LTR,
+ )
+ for rg_book in rg_books:
+ pprint(rg_book)
+ docx_file_path = "en_ot_survey_rg4_isa_mal.docx"
+ rg_books = get_rg_books(
+ docx_file_path,
+ "en",
+ "English",
+ "Reviewers' Guide OT Survey RG4",
+ LangDirEnum.LTR,
+ )
for rg_book in rg_books:
pprint(rg_book)
diff --git a/backend/doc/utils/file_utils.py b/backend/doc/utils/file_utils.py
index 6754b3183..3963afac8 100644
--- a/backend/doc/utils/file_utils.py
+++ b/backend/doc/utils/file_utils.py
@@ -3,19 +3,20 @@
import codecs
import json
import os
-import pathlib
import shutil
import urllib
import zipfile
from contextlib import closing
from datetime import datetime, timedelta
from os.path import join
-from typing import Any, Optional, Union
+from pathlib import Path
+from typing import Any, Optional
from urllib.request import urlopen
import yaml
from doc.config import settings
+
logger = settings.logger(__name__)
# User agent value required by domain host to allow serving
@@ -113,31 +114,22 @@ def write_file(
out_file.write(text_to_write)
-
-
def file_needs_update(
- file_path: Union[str, pathlib.Path],
+ file_path: str | Path,
asset_caching_enabled: bool = settings.ASSET_CACHING_ENABLED,
asset_caching_period: int = settings.ASSET_CACHING_PERIOD,
) -> bool:
- """
- Return True if settings.ASSET_CACHING_ENABLED is False or if
- file_path either does not exist or does exist and has not been
- updated within settings.ASSET_CACHING_PERIOD hours.
- """
if not asset_caching_enabled:
return True
- if not os.path.exists(file_path):
+ try:
+ path = Path(file_path)
+ stat = path.stat()
+ mod_time = datetime.fromtimestamp(stat.st_mtime)
+ expiry = timedelta(minutes=asset_caching_period)
+ return stat.st_size == 0 or (datetime.now() - mod_time > expiry)
+ except FileNotFoundError:
logger.debug("Cache miss for %s", file_path)
return True
- if os.path.exists(file_path) and os.path.getsize(file_path) == 0:
- logger.debug("File exists, but is empty!")
- return True
- file_mod_time: datetime = datetime.fromtimestamp(os.stat(file_path).st_mtime)
- now: datetime = datetime.today()
- max_delay: timedelta = timedelta(minutes=60 * asset_caching_period)
- # Has it been more than settings.ASSET_CACHING_PERIOD hours since last modification time?
- return now - file_mod_time > max_delay
def html_filepath(
diff --git a/backend/doc/utils/list_utils.py b/backend/doc/utils/list_utils.py
index c3c9a02ed..7243aa070 100644
--- a/backend/doc/utils/list_utils.py
+++ b/backend/doc/utils/list_utils.py
@@ -1,5 +1,6 @@
from typing import TypeVar
+
T = TypeVar("T", tuple[str, str], tuple[str, str, bool])
@@ -11,3 +12,17 @@ def unique_tuples(lst: list[T]) -> list[T]:
>>> assert unique_tuples(data_3) == [('x', 'y', True), ('x', 'y', False), ('a', 'b', True)]
"""
return list(dict.fromkeys(lst))
+
+
+def unique_book_codes(lst: list[T]) -> list[T]:
+ """
+ >>> input_list = [("lev", "value1"), ("lev", "value2", True), ("abc", "value3"), ("abc", "value4", False)]
+ >>> result = unique_tuples(input_list)
+ [('lev', 'value1'), ('abc', 'value3')]
+ """
+ seen = {}
+ for item in lst:
+ key = item[0]
+ if key not in seen:
+ seen[key] = item
+ return list(seen.values())
diff --git a/backend/doc/utils/template_env.py b/backend/doc/utils/template_env.py
index f71c8c7f5..b8380c06f 100644
--- a/backend/doc/utils/template_env.py
+++ b/backend/doc/utils/template_env.py
@@ -1,3 +1,4 @@
from jinja2 import Environment, FileSystemLoader
+
env = Environment(loader=FileSystemLoader("backend/templates"))
diff --git a/backend/doc/utils/text_utils.py b/backend/doc/utils/text_utils.py
index 19363d1cd..f0d65699b 100644
--- a/backend/doc/utils/text_utils.py
+++ b/backend/doc/utils/text_utils.py
@@ -1,6 +1,8 @@
import re
+
from doc.config import settings
+
logger = settings.logger(__name__)
@@ -8,22 +10,60 @@ def normalize_spaces(text: str) -> str:
return re.sub(r"\s+", " ", text).strip()
+_ROMAN_TO_INT = {
+ "I": "1",
+ "II": "2",
+ "III": "3",
+}
+
+
def normalize_localized_book_name(localized_book_name: str) -> str:
- # Deal with irregularities in localized book names, e.g., bem: 1timote
- if localized_book_name and localized_book_name[0] in [
- "1",
- "2",
- "3",
- ]: # E.g., bem: 1Timote or 2 Timote
- localized_book_name = (
- localized_book_name[0]
- + " "
- + localized_book_name[1:].strip().lower().capitalize()
- )
- localized_book_name = normalize_spaces(localized_book_name)
- else:
- localized_book_name = localized_book_name.strip().lower().capitalize()
- return localized_book_name
+ """
+ >>> normalize_localized_book_name("1Peter")
+ '1 Peter'
+ >>> normalize_localized_book_name("I peter")
+ '1 Peter'
+ >>> normalize_localized_book_name("III John")
+ '3 John'
+ >>> normalize_localized_book_name("II john")
+ '2 John'
+ >>> normalize_localized_book_name("IIjohn")
+ '2 John'
+ >>> normalize_localized_book_name("john")
+ 'John'
+ >>> normalize_localized_book_name("John")
+ 'John'
+ >>> normalize_localized_book_name("Isaías")
+ 'Isaías'
+ """
+ name = localized_book_name.strip()
+ match = re.match(r"^(1|2|3|i{1,3})", name, re.IGNORECASE)
+ if match:
+ numeral_raw = match.group(1)
+ numeral_upper = numeral_raw.upper()
+ next_char = name[len(numeral_raw) : len(numeral_raw) + 1]
+ # Special case: single "I" must be followed by space or uppercase to count as numeral
+ if numeral_upper == "I" and not (
+ next_char.isspace() or (next_char and next_char.isupper())
+ ):
+ pass # treat as normal word
+ else:
+ if numeral_upper in _ROMAN_TO_INT:
+ number = _ROMAN_TO_INT[numeral_upper]
+ else:
+ number = numeral_upper # already numeric
+ rest = name[len(numeral_raw) :].strip()
+ if rest:
+ # rest = rest[0].upper() + rest[1:]
+ rest = rest.lower().capitalize()
+ name = f"{number} {rest}"
+ else:
+ name = number
+ return normalize_spaces(name)
+ # Default: just capitalize first letter
+ # name = name[0].upper() + name[1:]
+ name = name.lower().capitalize()
+ return normalize_spaces(name)
def chapter_label_sans_numeric_part(s: str) -> str:
@@ -45,3 +85,16 @@ def chapter_label_numeric_part(s: str) -> int:
else:
result = -1 # Sentinel
return result
+
+
+if __name__ == "__main__":
+
+ # To run the doctests in this module, in the root of the project do:
+ # python backend/document/domain/resource_lookup.py
+ # or
+ # python backend/document/domain/resource_lookup.py -v
+ # See https://docs.python.org/3/library/doctest.html
+ # for more details.
+ import doctest
+
+ doctest.testmod()
diff --git a/backend/doc/utils/tw_utils.py b/backend/doc/utils/tw_utils.py
index 0060669cf..b919ea829 100644
--- a/backend/doc/utils/tw_utils.py
+++ b/backend/doc/utils/tw_utils.py
@@ -14,6 +14,7 @@
from doc.domain import parsing, resource_lookup
from doc.domain.model import ResourceRequest, TWBook, TWNameContentPair, USFMBook
+
logger = settings.logger(__name__)
TW = "tw"
@@ -123,9 +124,8 @@ def translation_words_section(
the list of all translation words for this language, book combination.
Limit the translation words to only those that appear in the USFM
resouce chosen if limit_words is True and a USFM resource was also
- chosen.
+ chosen otherwise include all the translation words for the language.
"""
-
content = []
if tw_book.name_content_pairs:
content.append(resource_type_name_fmt_str.format(tw_book.resource_type_name))
@@ -147,9 +147,17 @@ def get_selected_name_content_pairs(
selected_name_content_pairs = []
if usfm_books and limit_words:
selected_name_content_pairs = filter_name_content_pairs(tw_book, usfm_books)
- elif not usfm_books and limit_words:
+ elif (
+ not usfm_books and limit_words
+ ): # This branch is necessarily expensive computationally and in IO
+ t0 = time.time()
usfm_books = fetch_usfm_book_content_units(resource_requests)
selected_name_content_pairs = filter_name_content_pairs(tw_book, usfm_books)
+ t1 = time.time()
+ logger.info(
+ "Time for acquiring and filtering TW content based on books chosen: %s",
+ t1 - t0,
+ )
else:
selected_name_content_pairs = tw_book.name_content_pairs
return selected_name_content_pairs
@@ -159,6 +167,7 @@ def filter_name_content_pairs(
tw_book: TWBook, usfm_books: Optional[Sequence[USFMBook]]
) -> list[TWNameContentPair]:
selected_name_content_pairs = []
+ added_pairs = set()
if usfm_books:
for name_content_pair in tw_book.name_content_pairs:
for usfm_book in usfm_books:
@@ -167,7 +176,9 @@ def filter_name_content_pairs(
re.escape(name_content_pair.localized_word),
chapter.content,
):
- selected_name_content_pairs.append(name_content_pair)
+ if name_content_pair not in added_pairs:
+ selected_name_content_pairs.append(name_content_pair)
+ added_pairs.add(name_content_pair)
break
return selected_name_content_pairs
diff --git a/backend/doc/utils/url_utils.py b/backend/doc/utils/url_utils.py
new file mode 100644
index 000000000..28d0cbb48
--- /dev/null
+++ b/backend/doc/utils/url_utils.py
@@ -0,0 +1,213 @@
+import json
+import re
+from glob import glob
+from os.path import exists, join
+from pathlib import Path
+from urllib.parse import urlparse
+
+import yaml
+from doc.config import settings
+from doc.domain.model import Data, JsonManifestBook, JsonManifestData
+from pydantic import HttpUrl
+
+
+logger = settings.logger(__name__)
+
+# Specific replacements: lang code, last_segment -> replacement last_segment
+REPLACEMENTS_BY_LANG_CODE_AND_LAST_SEGMENT = {
+ ("fa", "fa_opv"): "fa_ulb",
+ ("my", "my_juds"): "my_ulb",
+ ("zmq", "faustin_azaza"): "zmq_mrk_text_reg",
+}
+
+# Prefixes to remove in last repo URL segment regardless of lang code
+PREFIXES_TO_REMOVE = [
+ "Jordan_",
+ "alexandre_brazil_",
+ "azz_athan_",
+ "bayan_",
+ "botsw01_",
+ "danjuma_alfred_h_",
+ "dijim1_",
+ "ezekieldabere_",
+ "gravy_",
+ "jdwood_",
+ "jks222111_",
+ "jonathan_",
+ "krispy_",
+ "lversaw_",
+ "michael_",
+ "mitikiwostky_",
+ "moufida_",
+ "mushohe-25nb_63.kum_",
+ "nbtt_",
+ "ngamo1_",
+ "ngamo_",
+ "oratab01_",
+ "otlaadisa_",
+ "parfait-ayanou_",
+ "romantts2_",
+ "sambadanum_",
+ "timothydanjuma_",
+ "tom-88pn_0003.machinga_",
+ "translator09_",
+ "ukum1_",
+ "vere3_",
+ "yukuben1_",
+]
+
+# lang code -> prefixes to remove
+LANG_SPECIFIC_PREFIXES_TO_REMOVE = {
+ "acq": ["Dawit-Dessie_", "burje_duro_", "tersitzewde_"],
+ "arb": ["burje_duro_"],
+ "byn": ["Dawit-Dessie_", "burje_duro_"],
+ "dz": ["Dzongkha_", "chuck_"],
+ "iba-x-ketungau": ["dayakketungau_", "lawadinusah_", "Lawadinusah_"],
+ "kcn": ["mvccbtt_"],
+ "kmq": ["Dawit-Dessie_"],
+ "knx-x-bajanya": ["bajanya_knx"],
+ "kun": ["Dawit-Dessie_"],
+ "kxh": ["burje_duro_"],
+ "kxv": ["jathapu_"],
+ "ndh": ["chindali_"],
+ "sbx": ["faustin-azaza_"],
+ "scg-x-dayakkatarak": ["yustius_"],
+ "sdm-x-pangkalsuka": ["dayaksuka_"],
+ "svr": ["billburns58_"],
+ "sze": ["Dawit-Dessie_", "burje_duro_"],
+ "xdy-x-dayakpunti": ["anselmus_"],
+ "xdy-x-mentebah": ["dayakfdkj_", "lawadinusah_", "Lawadinusah_"],
+ "xdy-x-senduruhan": ["dayaksenduruhan_", "Lawadinusah_"],
+ "xsr-x-shyarpa": ["shyarpa_"],
+ "xwg": ["Dawit-Dessie_", "burje_duro_"],
+}
+
+
+def normalize_last_segment(
+ lang_code: str,
+ last_segment: str,
+ hardcoded_replacements: dict[
+ tuple[str, str], str
+ ] = REPLACEMENTS_BY_LANG_CODE_AND_LAST_SEGMENT,
+ universal_prefixes: list[str] = PREFIXES_TO_REMOVE,
+ lang_specific_prefixes: dict[str, list[str]] = LANG_SPECIFIC_PREFIXES_TO_REMOVE,
+ en_rg_dir: str = settings.EN_RG_DIR,
+) -> str:
+ """
+ Handle special cases where git repo URL does not follow the expected pattern.
+ Ideally these repos URLs would have their last segment renamed
+ properly, e.g.,
+ 'https://content.bibletranslationtools.org/faustin_azaza/faustin_azaza'
+ renamed to
+ 'https://content.bibletranslationtools.org/faustin_azaza/zmq_mrk_text_reg',
+ but since we don't have control over that, we handle these anomalies
+ here.
+ """
+ if lang_code == "en" and last_segment.endswith(".docx"):
+ return en_rg_dir
+ if (lang_code, last_segment) in hardcoded_replacements:
+ return hardcoded_replacements[(lang_code, last_segment)]
+ for prefix in universal_prefixes:
+ if last_segment.startswith(prefix):
+ return re.sub(f"^{re.escape(prefix)}", "", last_segment)
+ for lang, prefixes in lang_specific_prefixes.items():
+ if lang_code == lang:
+ for prefix in prefixes:
+ if last_segment.startswith(prefix):
+ return re.sub(f"^{re.escape(prefix)}", "", last_segment)
+ return last_segment
+
+
+def get_last_segment(url: HttpUrl, lang_code: str) -> str:
+ """
+ Extract the last segment of the URL path and normalize it
+ according to known anomalies and naming patterns.
+ """
+ parsed_url = urlparse(str(url))
+ path_segments = parsed_url.path.strip("/").split("/")
+ last_segment = path_segments[-1] if path_segments else ""
+ return normalize_last_segment(lang_code, last_segment)
+
+
+def get_book_names_from_title_file(
+ resource_filepath: str,
+ lang_code: str,
+ repo_components: list[str],
+) -> dict[str, str]:
+ """
+ Book names in front/title.txt files may or may not be localized,
+ it depends on the translation work done for lang_code.
+ """
+ book_codes_and_names_localized: dict[str, str] = {}
+ book_name_file = join(resource_filepath, "front", "title.txt")
+ if exists(book_name_file):
+ with open(book_name_file, "r") as fin:
+ book_name = fin.read()
+ logger.debug("book_name: %s", book_name)
+ if book_name:
+ # Moved this code to the caller
+ # localized_book_name_ = normalize_localized_book_name(book_name)
+ # localized_book_name = maybe_correct_book_name(
+ # lang_code, localized_book_name_
+ # )
+ book_code = repo_components[1]
+ book_codes_and_names_localized[book_code] = book_name
+ return book_codes_and_names_localized
+
+
+def load_manifest(file_path: str) -> str:
+ with open(file_path, "r") as file:
+ return file.read()
+
+
+def book_codes_and_names_from_manifest(
+ resource_dir: str,
+ manifest_glob_fmt_str: str = "{}/**/manifest.{}",
+ manifest_glob_alt_fmt_str: str = "{}/manifest.{}",
+) -> dict[str, str]:
+ """
+ Look up the language direction in the manifest file if one is
+ available for this resource.
+ """
+ book_codes_and_names: dict[str, str] = {}
+ # Try to find manifest yaml at typical directory
+ manifest_candidates = glob(manifest_glob_fmt_str.format(resource_dir, "yaml"))
+ if not manifest_candidates:
+ # Now try to find manifest yaml at parent directory of typical directory
+ manifest_candidates = glob(
+ manifest_glob_alt_fmt_str.format(resource_dir, "yaml")
+ )
+ if not manifest_candidates:
+ # Some languages provide their manifest in json format.
+ # Try to find manifest json at typical directory
+ manifest_candidates = glob(
+ manifest_glob_fmt_str.format(resource_dir, "json")
+ )
+ if not manifest_candidates:
+ # Try to find manifest json at parent directory of typical directory
+ manifest_candidates = glob(
+ manifest_glob_alt_fmt_str.format(resource_dir, "json")
+ )
+ # logger.debug("manifest_candidates: %s", manifest_candidates)
+ if manifest_candidates:
+ candidate = manifest_candidates[0]
+ suffix = str(Path(candidate).suffix)
+ # book_codes_and_names: dict[str, str] = {}
+ # Get localized book names
+ manifest_data = load_manifest(candidate)
+ # logger.debug("manifest_data: %s", manifest_data)
+ if suffix == ".yaml":
+ data: Data = yaml.safe_load(manifest_data)
+ book_codes_and_names = {
+ book["identifier"]: book["title"] for book in data["projects"]
+ }
+ logger.debug("book_codes_and_names from yaml: %s", book_codes_and_names)
+ # Heart languages often have .json manifest files
+ # per book and not per language.
+ elif suffix == ".json":
+ json_data: JsonManifestData = json.loads(manifest_data)
+ logger.debug("json_data: %s", json_data)
+ project: JsonManifestBook = json_data["project"]
+ book_codes_and_names = {project["id"]: project["name"]}
+ logger.debug("book_codes_and_names from json: %s", book_codes_and_names)
+ return book_codes_and_names
diff --git a/backend/passages/data/Spiritual_Terms_Evaluation_Exhaustive_Verse_List.txt b/backend/passages/data/Spiritual_Terms_Evaluation_Exhaustive_Verse_List.txt
new file mode 100644
index 000000000..41920d44a
--- /dev/null
+++ b/backend/passages/data/Spiritual_Terms_Evaluation_Exhaustive_Verse_List.txt
@@ -0,0 +1,985 @@
+Abba (5)
+Mark 14:36, Romans 8:15, Galatians 4:6
+
+adoption (5206)
+Romans 8:15, 8:23, 9:4, Galatians 4:5, Ephesians 1:5
+
+angel (32)
+Matthew 1:20, 1:24, 2:13, 2:19, 4:6, 4:11, 11:10, 13:39, 13:41, 13:49, 16:27, 18:10, 22:30,
+24:31, 24:36, 25:31, 25:41, 26:53, 28:2, 28:5 Mark 1:2, 1:13, 8:38, 12:25, 13:27, 13:32 Luke
+1:11, 1:13, 1:18, 1:19, 1:26, 1:30, 1:34, 1:35, 1:38, 2:9, 2:10, 2:13, 2:15, 2:21, 4:10, 7:24,
+7:27, 9:26, 9:52, 12:8, 12:9, 15:10, 16:22, 22:43, 24:23 John 1:51, 12:29, 20:12 Acts 5:19,
+6:15, 7:30, 7:35, 7:38, 7:53, 8:26, 10:3, 10:7, 10:22, 11:13, 12:7, 12:8, 12:9, 12:10, 12:11,
+12:15, 12:23, 23:8, 23:9, 27:23 Romans 8:38 1 Corinthians 4:9, 6:3, 11:10, 13:1 2
+Corinthians 11:14, 12:7 Galatians 1:8, 3:19, 4:14 Colossians 2:18 2 Thessalonians 1:7 1
+Timothy 3:16, 5:21 Hebrews 1:4, 1:5, 1:6, 1:7, 1:13, 2:2, 2:5, 2:7, 2:9, 2:16, 12:22, 13:2
+James 2:25 1 Peter 1:12, 3:22 2 Peter 2:4, 2:11 Jude 1:6 Revelation 1:1, 1:20, 2:1, 2:8,
+2:12, 2:18, 3:1, 3:5, 3:7, 3:14, 5:2, 5:11, 7:1, 7:2, 7:11, 8:2, 8:3, 8:4, 8:5, 8:6, 8:8, 8:10, 8:12,
+8:13, 9:1, 9:11, 9:13, 9:14, 9:15, 10:1, 10:5, 10:7, 10:8, 10:9, 10:10, 11:15, 12:7, 12:9, 14:6,
+14:8, 14:9, 14:10, 14:15, 14:17, 14:18, 14:19, 15:1, 15:6, 15:7, 15:8, 16:1, 16:5, 17:1, 17:7,
+18:1, 18:21, 19:17, 20:1, 21:9, 21:12, 21:17, 22:6, 22:8, 22:16
+
+anger (3709, 3710, 3949)
+Matthew 3:7, 5:22, 18:34, 22:7 Mark 3:5 Luke 3:7, 14:21, 15:28, 21:23 John 3:36 Romans
+1:18, 2:5, 2:8, 3:5, 4:15, 5:9, 9:22, 10:19, 12:19, 13:4, 13:5 Ephesians 2:3, 4:26, 4:31, 5:6,
+6:4 Colossians 3:6, 3:8 1 Thessalonians 1:10, 2:16, 5:9 1 Timothy 2:8 Hebrews 3:11, 4:3
+James 1:19, 1:20 Revelation 6:16, 6:17, 11:18, 12:17, 14:10, 16:19, 19:15
+
+apostle (652)
+Matthew 10:2 Mark 3:14, 6:30 Luke 6:13, 9:10, 11:49, 17:5, 22:14, 24:10 John 13:16 Acts
+1:2, 1:26, 2:37, 2:42, 2:43, 4:33, 4:35, 4:36, 4:37, 5:2, 5:12, 5:18, 5:29, 5:40, 6:6, 8:1, 8:14,
+8:18, 9:27, 11:1, 14:4, 14:14, 15:2, 15:4, 15:6, 15:22, 15:23, 16:4 Romans 1:1, 11:13, 16:7 1
+Corinthians 1:1, 4:9, 9:1, 9:2, 9:5, 12:28, 12:29, 15:7, 15:9 2 Corinthians 1:1, 8:23, 11:5,
+11:13, 12:11, 12:12 Galatians 1:1, 1:17, 1:19 Ephesians 1:1, 2:20, 3:5, 4:11 Philippians
+2:25 Colossians 1:1 1 Thessalonians 2:7 1 Timothy 1:1, 2:7 2 Timothy 1:1, 1:11 Titus
+1:1 Hebrews 3:1 1 Peter 1:1 2 Peter 1:1, 3:2 Jude 1:17 Revelation 2:2, 18:20, 21:14
+
+ashamed (1870, 153)
+Mark 8:38 Luke 9:26, 16:3 Romans 1:16, 6:21 2 Corinthians 10:8 Philippians 1:20 2
+Timothy 1:8, 1:12, 1:16 Hebrews 2:11, 11:16 1 Peter 4:16 1 John 2:28
+
+authority (1849, 2715)
+Matthew 7:29, 8:9, 9:6, 9:8, 10:1, 20:25, 21:23, 21:24, 21:27, 28:18 Mark 1:22, 1:27, 2:10,
+3:15, 6:7, 10:42, 11:28, 11:29, 11:33 Luke 4:6, 4:32, 4:36, 5:24, 7:8, 9:1, 10:19, 12:5, 12:11,
+19:17, 20:2, 20:8, 20:20, 22:53, 23:7 John 1:12, 5:27, 10:18, 17:2, 19:10, 19:11 Acts 1:7, 5:4,
+8:19, 9:14, 26:10, 26:12, 26:18 Romans 9:21, 13:1, 13:2 1 Corinthians 8:9, 9:4, 9:5, 9:6,
+9:12, 9:18, 11:10, 15:24 2 Corinthians 10:8, 13:10 Ephesians 1:21, 2:2, 3:10, 6:12
+Colossians 1:13, 1:16, 2:10, 2:15 2 Thessalonians 3:9 Titus 3:1 Hebrews 13:10 1 Peter
+3:22 Jude 1:25 Revelation 2:26, 6:8, 9:3, 9:10, 9:19, 11:6, 12:10, 13:2, 13:4, 13:5, 13:7,
+13:12, 14:18, 16:9, 17:12, 17:13, 18:1, 20:6, 22:14
+
+baptize (907, 908, 909)
+Matthew 3:6, 3:7, 3:11, 3:13, 3:14, 3:16, 21:25, 28:19 Mark 1:4, 1:5, 1:8, 1:9, 6:14, 6:24, 7:4,
+10:38, 10:39, 11:30, 16:16 Luke 3:3, 3:7, 3:12, 3:16, 3:21, 7:29, 7:30, 11:38, 12:50, 20:4
+John 1:25, 1:26, 1:28, 1:31, 1:33, 3:22, 3:23, 3:26, 4:1, 4:2, 10:40 Acts 1:5, 1:22, 2:38, 2:41,
+8:12, 8:13, 8:16, 8:36, 8:38, 9:18, 10:37, 10:47, 10:48, 11:16, 13:24, 16:15, 16:33, 18:8,
+18:25, 19:3, 19:4, 19:5, 22:16 Romans 6:3, 6:4 1 Corinthians 1:13, 1:14, 1:15, 1:16, 1:17,
+10:2, 12:13, 15:29 Galatians 3:27 Ephesians 4:5 Colossians 2:12 Hebrews 6:2, 9:10 1
+Peter 3:21
+
+believe (4100)
+Matthew 8:13, 9:28, 18:6, 21:22, 21:25, 21:32, 24:23, 24:26, 27:42 Mark 1:15, 5:36, 9:23,
+9:24, 9:42, 11:23, 11:24, 11:31, 13:21, 15:32, 16:14, 16:16, 16:17 Luke 1:20, 1:45, 8:12,
+8:13, 8:50, 16:11, 20:5, 22:67, 24:25 John 1:7, 1:12, 1:50, 2:11, 2:22, 2:23, 2:24, 3:12, 3:15,
+3:16, 3:18, 3:36, 4:21, 4:39, 4:41, 4:42, 4:48, 4:50, 4:53, 5:24, 5:38, 5:44, 5:46, 5:47, 6:29,
+6:30, 6:35, 6:36, 6:40, 6:47, 6:64, 6:69, 7:31, 7:38, 7:39, 7:48, 8:24, 8:30, 8:31, 8:45, 8:46,
+9:18, 9:35, 9:36, 9:38, 10:25, 10:26, 10:37, 10:38, 10:42, 11:15, 11:25, 11:26, 11:27, 11:40,
+11:42, 11:45, 11:48, 12:11, 12:36, 12:37, 12:38, 12:39, 12:42, 12:44, 13:19, 14:1, 14:10,
+14:11, 14:12, 14:29, 16:9, 16:27, 16:30, 16:31, 17:8, 17:20, 17:21, 19:35, 20:8, 20:25, 20:29,
+20:31 Acts 2:44, 4:4, 4:32, 5:14, 8:12, 8:13, 9:26, 9:42, 10:43, 11:17, 11:21, 13:12, 13:39,
+13:41, 13:48, 14:1, 14:23, 15:5, 15:7, 15:11, 16:31, 16:34, 17:12, 17:34, 18:8, 18:27, 19:2,
+19:4, 19:18, 21:20, 21:25, 22:19, 24:14, 26:27, 27:25 Romans 1:16, 3:2, 3:22, 4:3, 4:5, 4:11,
+4:17, 4:18, 4:24, 6:8, 9:33, 10:4, 10:9, 10:10, 10:11, 10:14, 10:16, 13:11, 14:2, 15:13 1
+Corinthians 1:21, 3:5, 9:17, 11:18, 13:7, 14:22, 15:2, 15:11 2 Corinthians 4:13 Galatians
+2:7, 2:16, 3:6, 3:22 Ephesians 1:13, 1:19 Philippians 1:29 1 Thessalonians 1:7, 2:4, 2:10,
+2:13, 4:14 2 Thessalonians 1:10, 2:11, 2:12 1 Timothy 1:11, 1:16, 3:16 2 Timothy 1:12
+
+Titus 1:3, 3:8 Hebrews 4:3, 11:6 James 2:19, 2:23 1 Peter 1:8, 2:6, 2:7 1 John 3:23, 4:1,
+4:16, 5:1, 5:5, 5:10, 5:13 Jude 1:5
+
+beloved (27)
+Matthew 3:17, 12:18, 17:5 Mark 1:11, 9:7, 12:6 Luke 3:22, 20:13 Acts 15:25 Romans 1:7,
+11:28, 12:19, 16:5, 16:8, 16:9, 16:12 1 Corinthians 4:14, 4:17, 10:14, 15:58 2 Corinthians
+7:1, 12:19 Ephesians 5:1, 6:21 Philippians 2:12, 4:1 Colossians 1:7, 4:7, 4:9, 4:14 1
+Thessalonians 2:8 1 Timothy 6:2 2 Timothy 1:2 Philemon 1:1, 1:16 Hebrews 6:9 James
+1:16, 1:19, 2:5 1 Peter 2:11, 4:12 2 Peter 1:17, 3:1, 3:8, 3:14, 3:15, 3:17 1 John 2:7, 3:2,
+3:21, 4:1, 4:7, 4:11 3 John 1:1, 1:2, 1:5, 1:11 Jude 1:3, 1:17, 1:20
+
+blameless (273, 299, 298, 410)
+Luke 1:6 1 Corinthians 1:8 Ephesians 1:4, 5:27 Philippians 2:15, 3:6 Colossians 1:22 1
+Thessalonians 3:13 1 Timothy 3:10 Titus 1:6, 1:7 Hebrews 8:7, 9:14 1 Peter 1:19 2
+Peter 3:14 Jude 1:24 Revelation 14:5, 18:13
+
+blaspheme (987, 988)
+Matthew 9:3, 12:31, 15:19, 26:65, 27:39 Mark 2:7, 3:28, 3:29, 7:22, 14:64, 15:29 Luke
+5:21, 12:10, 22:65, 23:39 John 10:33, 10:36 Acts 13:45, 18:6, 19:37, 26:11 Romans 2:24,
+3:8, 14:16 1 Corinthians 10:30 Ephesians 4:31 Colossians 3:8 1 Timothy 1:20, 6:1, 6:4
+Titus 2:5, 3:2 James 2:7 1 Peter 4:4 2 Peter 2:2, 2:10, 2:12 Jude 1:8, 1:9, 1:10 Revelation
+2:9, 13:1, 13:5, 13:6, 16:9, 16:11, 16:21, 17:3
+
+bless (3107, 3106, 3105, 2129, 2128, 2127, 1757)
+Matthew 5:3, 5:4, 5:5, 5:6, 5:7, 5:8, 5:9, 5:10, 5:11, 11:6, 13:16, 14:19, 16:17, 21:9, 23:39,
+24:46, 25:34, 26:26 Mark 6:41, 8:7, 11:9, 11:10, 14:22, 14:61 Luke 1:42, 1:45, 1:48, 1:64,
+1:68, 2:28, 2:34, 6:20, 6:21, 6:22, 6:28, 7:23, 9:16, 10:23, 11:27, 11:28, 12:37, 12:38, 12:43,
+13:35, 14:14, 14:15, 19:38, 23:29, 24:30, 24:50, 24:53 John 10:20, 12:13, 13:17, 20:29 Acts
+3:25, 3:26, 12:15, 20:35, 26:2, 26:24, 26:25 Romans 1:25, 4:7, 4:8, 9:5, 12:14, 14:22, 15:29,
+16:18 1 Corinthians 4:12, 7:40, 10:16, 14:16, 14:23 2 Corinthians 1:3, 9:5, 9:6, 11:31
+Galatians 3:8, 3:9, 3:14 Ephesians 1:3 1 Timothy 1:11, 6:15 Titus 2:13 Hebrews 6:7, 7:1,
+7:6, 7:7, 11:20, 11:21, 12:17 James 1:12, 1:25, 3:9, 3:10, 5:11 1 Peter 1:3, 3:9, 3:14, 4:14
+Revelation 1:3, 5:12, 5:13, 7:12, 14:13, 16:15, 19:9, 20:6, 22:7, 22:14
+
+blood (129)
+Matthew 16:17, 23:30, 23:35, 26:28, 27:4, 27:6, 27:8, 27:24, 27:25 Mark 5:25, 14:24 Luke
+11:50, 11:51, 13:1, 22:20, 22:44 John 1:13, 6:53, 6:54, 6:55, 6:56, 19:34 Acts 1:19, 2:19,
+
+2:20, 5:28, 15:20, 15:29, 18:6, 20:26, 20:28, 21:25, 22:20 Romans 3:15, 3:25, 5:9 1
+Corinthians 10:16, 11:25, 11:27, 15:50 Galatians 1:16 Ephesians 1:7, 2:13, 6:12
+Colossians 1:20 Hebrews 2:14, 9:7, 9:12, 9:13, 9:14, 9:18, 9:19, 9:20, 9:21, 9:22, 9:25, 10:4,
+10:19, 10:29, 11:28, 12:4, 12:24, 13:11, 13:12, 13:20 1 Peter 1:2, 1:19 1 John 1:7, 5:6, 5:8
+Revelation 1:5, 5:9, 6:10, 6:12, 7:14, 8:7, 8:8, 11:6, 12:11, 14:20, 16:3, 16:4, 16:6, 17:6,
+18:24, 19:2, 19:13
+
+Christ (5547)
+Matthew 1:1, 1:16, 1:17, 1:18, 2:4, 11:2, 16:16, 16:20, 22:42, 23:10, 24:5, 24:23, 26:63,
+26:68, 27:17, 27:22 Mark 1:1, 8:29, 9:41, 12:35, 13:21, 14:61, 15:32 Luke 2:11, 2:26, 3:15,
+9:20, 20:41, 22:67, 23:2, 23:35, 23:39, 24:26, 24:46 John 1:17, 1:20, 1:25, 1:41, 3:28, 4:25,
+7:26, 7:27, 7:31, 7:41, 7:42, 9:22, 10:24, 11:27, 12:34, 17:3, 20:31 Acts 2:31, 2:36, 2:38, 3:6,
+3:18, 3:20, 4:10, 4:26, 5:42, 8:5, 8:12, 9:22, 9:34, 10:36, 10:48, 11:17, 15:26, 16:18, 17:3,
+18:5, 18:28, 24:24, 26:23, 28:31 Romans 1:1, 1:4, 1:6, 1:7, 1:8, 2:16, 3:22, 3:24, 5:1, 5:6, 5:8,
+5:11, 5:15, 5:17, 5:21, 6:3, 6:4, 6:8, 6:9, 6:11, 6:23, 7:4, 7:25, 8:1, 8:2, 8:9, 8:10, 8:11, 8:17,
+8:34, 8:35, 8:39, 9:1, 9:3, 9:5, 10:4, 10:6, 10:7, 10:17, 12:5, 13:14, 14:9, 14:15, 14:18, 15:3,
+15:5, 15:6, 15:7, 15:8, 15:16, 15:17, 15:18, 15:19, 15:20, 15:29, 15:30, 16:3, 16:5, 16:7, 16:9,
+16:10, 16:16, 16:18, 16:25, 16:27 1 Corinthians 1:1, 1:2, 1:3, 1:4, 1:6, 1:7, 1:8, 1:9, 1:10,
+1:12, 1:13, 1:17, 1:23, 1:24, 1:30, 2:2, 2:16, 3:1, 3:11, 3:23, 4:1, 4:10, 4:15, 4:17, 5:7, 6:11,
+6:15, 7:22, 8:6, 8:11, 8:12, 9:12, 9:21, 10:4, 10:9, 10:16, 11:1, 11:3, 12:12, 12:27, 15:3,
+15:12, 15:13, 15:14, 15:15, 15:16, 15:17, 15:18, 15:19, 15:20, 15:22, 15:23, 15:31, 15:57,
+16:24 2 Corinthians 1:1, 1:2, 1:3, 1:5, 1:19, 1:21, 2:10, 2:12, 2:14, 2:15, 2:17, 3:3, 3:4, 3:14,
+4:4, 4:5, 4:6, 5:10, 5:14, 5:16, 5:17, 5:18, 5:19, 5:20, 6:15, 8:9, 8:23, 9:13, 10:1, 10:5, 10:7,
+10:14, 11:2, 11:3, 11:10, 11:13, 11:23, 12:2, 12:9, 12:10, 12:19, 13:3, 13:5, 13:13 Galatians
+1:1, 1:3, 1:6, 1:7, 1:10, 1:12, 1:22, 2:4, 2:16, 2:17, 2:19, 2:20, 2:21, 3:1, 3:13, 3:14, 3:16, 3:22,
+3:24, 3:26, 3:27, 3:28, 3:29, 4:14, 4:19, 5:1, 5:2, 5:4, 5:6, 5:24, 6:2, 6:12, 6:14, 6:18
+Ephesians 1:1, 1:2, 1:3, 1:5, 1:10, 1:12, 1:17, 1:20, 2:5, 2:6, 2:7, 2:10, 2:12, 2:13, 2:20, 3:1,
+3:4, 3:6, 3:8, 3:11, 3:17, 3:19, 3:21, 4:7, 4:12, 4:13, 4:15, 4:20, 4:32, 5:2, 5:5, 5:14, 5:20, 5:21,
+5:23, 5:24, 5:25, 5:29, 5:32, 6:5, 6:6, 6:23, 6:24 Philippians 1:1, 1:2, 1:6, 1:8, 1:10, 1:11,
+1:13, 1:15, 1:17, 1:18, 1:19, 1:20, 1:21, 1:23, 1:26, 1:27, 1:29, 2:1, 2:5, 2:11, 2:16, 2:21, 2:30,
+3:3, 3:7, 3:8, 3:9, 3:12, 3:14, 3:18, 3:20, 4:7, 4:19, 4:21, 4:23 Colossians 1:1, 1:2, 1:3, 1:4,
+1:7, 1:24, 1:27, 1:28, 2:2, 2:5, 2:6, 2:8, 2:11, 2:17, 2:20, 3:1, 3:3, 3:4, 3:11, 3:15, 3:16, 3:24,
+4:3, 4:12 1 Thessalonians 1:1, 1:3, 2:7, 2:14, 3:2, 4:16, 5:9, 5:18, 5:23, 5:28 2
+Thessalonians 1:1, 1:2, 1:12, 2:1, 2:14, 2:16, 3:5, 3:6, 3:12, 3:18 1 Timothy 1:1, 1:2, 1:12,
+1:14, 1:15, 1:16, 2:5, 3:13, 4:6, 5:11, 5:21, 6:3, 6:13, 6:14 2 Timothy 1:1, 1:2, 1:9, 1:10, 1:13,
+2:1, 2:3, 2:8, 2:10, 3:12, 3:15, 4:1 Titus 1:1, 1:4, 2:13, 3:6 Philemon 1:1, 1:3, 1:6, 1:8, 1:9,
+1:20, 1:23, 1:25 Hebrews 3:6, 3:14, 5:5, 6:1, 9:11, 9:14, 9:24, 9:28, 10:10, 11:26, 13:8, 13:21
+James 1:1, 2:1 1 Peter 1:1, 1:2, 1:3, 1:7, 1:11, 1:13, 1:19, 2:5, 2:21, 3:15, 3:16, 3:18, 3:21,
+4:1, 4:11, 4:13, 4:14, 5:1, 5:10, 5:14 2 Peter 1:1, 1:8, 1:11, 1:14, 1:16, 2:20, 3:18 1 John 1:3,
+2:1, 2:22, 3:23, 4:2, 5:1, 5:6, 5:20 2 John 1:3, 1:7, 1:9 Jude 1:1, 1:4, 1:17, 1:21, 1:25
+Revelation 1:1, 1:2, 1:5, 11:15, 12:10, 20:4, 20:6
+
+Christian (5546)
+Acts 11:26, 26:28 1 Peter 4:16
+
+church (1577)
+Matthew 16:18, 18:17 Acts 5:11, 7:38, 8:1, 8:3, 9:31, 11:22, 11:26, 12:1, 12:5, 13:1, 14:23,
+14:27, 15:3, 15:4, 15:22, 15:41, 16:5, 18:22, 19:32, 19:39, 19:40, 20:17, 20:28 Romans
+16:1, 16:4, 16:5, 16:16, 16:23 1 Corinthians 1:2, 4:17, 6:4, 7:17, 10:32, 11:16, 11:18, 11:22,
+12:28, 14:4, 14:5, 14:12, 14:19, 14:23, 14:28, 14:33, 14:34, 14:35, 15:9, 16:1, 16:19 2
+Corinthians 1:1, 8:1, 8:18, 8:19, 8:23, 8:24, 11:8, 11:28, 12:13 Galatians 1:2, 1:13, 1:22
+Ephesians 1:22, 3:10, 3:21, 5:23, 5:24, 5:25, 5:27, 5:29, 5:32 Philippians 3:6, 4:15
+Colossians 1:18, 1:24, 4:15, 4:16 1 Thessalonians 1:1, 2:14 2 Thessalonians 1:1, 1:4 1
+Timothy 3:5, 3:15, 5:16 Philemon 1:2 Hebrews 2:12, 12:23 James 5:14 3 John 1:6, 1:9,
+1:10 Revelation 1:4, 1:11, 1:20, 2:1, 2:7, 2:8, 2:11, 2:12, 2:17, 2:18, 2:23, 2:29, 3:1, 3:6, 3:7,
+3:13, 3:14, 3:22, 22:16
+
+compassion (4697, 3627)
+Matthew 9:36, 14:14, 15:32, 18:27, 20:34 Mark 1:41, 6:34, 8:2, 9:22 Luke 7:13, 10:33,
+15:20 Romans 9:15
+
+condemn, condemnation (2613, 2632, 2631)
+Matthew 12:7, 12:37, 12:41, 12:42, 20:18, 27:3 Mark 10:33, 14:64, 16:16 Luke 6:37,
+11:31, 11:32 John 8:10, 8:11 Romans 2:1, 8:1, 8:3, 8:34, 14:23 1 Corinthians 11:32
+Hebrews 11:7 James 5:6 2 Peter 2:6
+
+confess, confession (1843, 3670, 3671)
+Matthew 3:6, 7:23, 10:32, 11:25, 14:7 Mark 1:5 Luke 10:21, 12:8, 22:6 John 1:20, 9:22,
+12:42 Acts 7:17, 19:18, 23:8, 24:14 Romans 10:9, 10:10, 14:11, 15:9 2 Corinthians 9:13
+Philippians 2:11 1 Timothy 6:12, 6:13 Titus 1:16 Hebrews 3:1, 4:14, 11:13, 13:15 James
+5:16 1 John 1:9, 2:23, 4:2, 4:3, 4:15 Revelation 3:5
+
+cross (4716)
+Matthew 10:38, 16:24, 27:32, 27:40, 27:42 Mark 8:34, 15:21, 15:30, 15:32 Luke 9:23,
+14:27, 23:26 John 19:17, 19:19, 19:25, 19:31 1 Corinthians 1:17, 1:18 Galatians 5:11,
+6:12, 6:14 Ephesians 2:16 Philippians 2:8, 3:18 Colossians 1:20, 2:14 Hebrews 12:2
+
+crucify (4717, 4957, 388)
+Matthew 20:19, 23:34, 26:2, 27:22, 27:23, 27:26, 27:31, 27:35, 27:38, 27:44, 28:5 Mark
+15:13, 15:14, 15:24, 15:25, 15:27, 15:32, 16:6 Luke 23:21, 23:23, 23:33, 24:7, 24:20 John
+19:6, 19:10, 19:15, 19:18, 19:20, 19:23, 19:32, 19:41 Acts 2:36, 4:10 Romans 6:6 1
+Corinthians 1:13, 1:23, 2:2, 2:8 2 Corinthians 13:4 Galatians 2:19, 3:1, 5:24, 6:14
+Hebrews 6:6 Revelation 11:8
+
+death (2288, 2289)
+Matthew 4:16, 10:21, 16:28, 20:18, 26:38, 26:59, 26:66, 27:1 Mark 7:10, 9:1, 10:33, 13:12,
+14:34, 14:55, 14:64 Luke 1:79, 2:26, 9:27, 21:16, 22:33, 23:15, 23:22, 24:20 John 5:24,
+8:51, 8:52, 11:4, 11:13, 12:33, 18:32, 21:19 Acts 2:24, 13:28, 22:4, 23:29, 25:11, 25:25,
+26:31, 28:18 Romans 1:32, 5:10, 5:12, 5:14, 5:17, 5:21, 6:3, 6:4, 6:5, 6:9, 6:21, 6:23, 7:4, 7:5,
+7:10, 7:13, 7:24, 8:2, 8:6, 8:13, 8:36, 8:38 1 Corinthians 3:22, 11:26, 15:21, 15:26, 15:54,
+15:55, 15:56 2 Corinthians 1:9, 1:10, 2:16, 3:7, 4:11, 4:12, 6:9, 7:10, 11:23 Philippians
+1:20, 2:8, 2:27, 2:30, 3:10 Colossians 1:22 2 Timothy 1:10 Hebrews 2:9, 2:14, 2:15, 5:7,
+7:23, 9:15, 9:16, 11:5 James 1:15, 5:20 1 Peter 3:18 1 John 3:14, 5:16, 5:17 Revelation
+1:18, 2:10, 2:11, 2:23, 6:8, 9:6, 12:11, 13:3, 13:12, 18:8, 20:6, 20:13, 20:14, 21:4, 21:8
+
+demon (1140, 1139)
+Matthew 4:24, 7:22, 8:16, 8:28, 8:33, 9:32, 9:33, 9:34, 10:8, 11:18, 12:22, 12:24, 12:27,
+12:28, 15:22, 17:18 Mark 1:32, 1:34, 1:39, 3:15, 3:22, 5:15, 5:16, 5:18, 6:13, 7:26, 7:29,
+7:30, 9:38, 16:9, 16:17 Luke 4:33, 4:35, 4:41, 7:33, 8:2, 8:27, 8:29, 8:30, 8:33, 8:35, 8:36,
+8:38, 9:1, 9:42, 9:49, 10:17, 11:14, 11:15, 11:18, 11:19, 11:20, 13:32 John 7:20, 8:48, 8:49,
+8:52, 10:20, 10:21 Acts 17:18 1 Corinthians 10:20, 10:21 1 Timothy 4:1 James 2:19
+Revelation 9:20, 16:14, 18:2
+
+discipline (3809, 3811)
+Luke 23:16, 23:22 Acts 7:22, 22:3 1 Corinthians 11:32 2 Corinthians 6:9 Ephesians 6:4 1
+Timothy 1:20 2 Timothy 2:25, 3:16 Titus 2:12 Hebrews 12:5, 12:6, 12:7, 12:8, 12:10,
+12:11 Revelation 3:19
+
+dishonor (818, 819)
+Mark 12:4 Luke 20:11 John 8:49 Acts 5:41 Romans 1:24, 1:26, 2:23, 9:21 1 Corinthians
+11:14, 15:43 2 Corinthians 6:8, 11:21 2 Timothy 2:20 James 2:6
+
+divine (2304, 2305)
+Acts 17:29 Romans 1:20 2 Peter 1:3, 1:4
+
+elect (1588, 1589, 1586)
+Matthew 22:14, 24:22, 24:24, 24:31 Mark 13:20, 13:22, 13:27 Luke 6:13, 9:35, 10:42,
+14:7, 23:35 John 6:70, 13:18, 15:16, 15:19 Acts 1:2, 1:24, 6:5, 9:15, 13:17, 15:7, 15:22,
+15:25 Romans 8:33, 9:11, 11:5, 11:7, 11:28, 16:13 1 Corinthians 1:27, 1:28 Ephesians 1:4
+Colossians 3:12 1 Thessalonians 1:4 1 Timothy 5:21 2 Timothy 2:10 Titus 1:1 James
+2:5 1 Peter 1:1, 2:4, 2:6, 2:9 2 Peter 1:10 2 John 1:1, 1:13 Revelation 17:14
+
+endure (5278, 5281, 5297)
+Matthew 10:22, 24:13 Mark 13:13 Luke 2:43, 8:15, 21:19 Acts 17:14 Romans 2:7, 5:3,
+5:4, 8:25, 12:12, 15:4, 15:5 1 Corinthians 10:13, 13:7 2 Corinthians 1:6, 6:4, 12:12
+Colossians 1:11 1 Thessalonians 1:3 2 Thessalonians 1:4, 3:5 1 Timothy 6:11 2
+Timothy 2:10, 2:12, 3:10, 3:11 Titus 2:2 Hebrews 10:32, 10:36, 12:1, 12:2, 12:3, 12:7
+James 1:3, 1:4, 1:12, 5:11 1 Peter 2:19, 2:20 2 Peter 1:6 Revelation 1:9, 2:2, 2:3, 2:19,
+13:10
+
+eternal (166)
+Matthew 18:8, 19:16, 19:29, 25:41, 25:46 Mark 3:29, 10:17, 10:30, 16:20 Luke 10:25,
+16:9, 18:18, 18:30 John 3:15, 3:16, 3:36, 4:14, 4:36, 5:24, 5:39, 6:27, 6:40, 6:47, 6:54, 6:68,
+10:28, 12:25, 12:50, 17:2, 17:3 Acts 13:46, 13:48 Romans 2:7, 5:21, 6:22, 6:23, 16:26 2
+Corinthians 4:17, 4:18, 5:1 Galatians 6:8 2 Thessalonians 1:9, 2:16 1 Timothy 1:16,
+6:12, 6:16 2 Timothy 1:9, 2:10 Titus 1:2, 3:7 Philemon 1:15 Hebrews 5:9, 6:2, 9:12, 9:14,
+9:15, 13:20 1 Peter 5:10 2 Peter 1:11 1 John 1:2, 2:25, 3:15, 5:11, 5:13, 5:20 Jude 1:7, 1:21
+Revelation 14:6
+
+evil (4190, 2554, 2555, 2556)
+Matthew 5:11, 5:37, 5:39, 5:45, 6:13, 6:23, 7:11, 7:17, 7:18, 9:4, 12:34, 12:35, 12:39, 12:45,
+13:19, 13:38, 13:49, 15:19, 16:4, 18:32, 21:41, 22:10, 24:48, 25:26, 27:23 Mark 3:4, 7:21,
+7:23, 15:14 Luke 3:19, 6:9, 6:22, 6:35, 6:45, 7:21, 8:2, 11:13, 11:26, 11:29, 11:34, 19:22,
+23:22 John 3:19, 7:7, 17:15, 18:23 Acts 9:13, 16:28, 17:5, 18:14, 19:12, 19:13, 19:15, 19:16,
+23:9, 25:18, 28:5, 28:21 Romans 1:30, 2:9, 7:19, 12:17, 13:3, 13:4, 13:10, 14:20 1
+Corinthians 5:13, 10:6, 13:5, 15:33 Galatians 1:4 Ephesians 5:16, 6:13, 6:16 Philippians
+3:2 Colossians 1:21, 3:5 1 Thessalonians 5:15, 5:22 2 Thessalonians 3:2 1 Timothy 6:4,
+6:10 2 Timothy 3:13, 4:14, 4:18 Titus 1:12 Hebrews 3:12, 5:14, 10:22 James 1:13, 2:4,
+3:8, 4:16 1 Peter 2:12, 2:14, 3:9, 3:10, 3:11, 3:12, 3:17, 4:15 1 John 3:12 2 John 1:11 3
+John 1:10, 1:11 Revelation 2:2, 16:2
+
+faith (4102)
+Matthew 9:2, 9:22, 9:29, 15:28, 17:20, 21:21, 23:23 Mark 2:5, 4:40, 5:34, 10:52, 11:22
+Luke 5:20, 7:9, 7:50, 8:25, 8:48, 17:5, 17:6, 17:19, 18:8, 18:42, 22:32 Acts 3:16, 6:5, 6:7,
+11:24, 13:8, 14:9, 14:22, 14:27, 15:9, 16:5, 17:31, 20:21, 24:24, 26:18 Romans 1:5, 1:8,
+1:12, 1:17, 3:3, 3:22, 3:25, 3:26, 3:27, 3:28, 3:30, 3:31, 4:5, 4:9, 4:11, 4:12, 4:13, 4:14, 4:16,
+4:19, 4:20, 5:1, 5:2, 9:30, 9:32, 10:6, 10:8, 10:17, 11:20, 12:3, 12:6, 14:1, 14:22, 14:23, 16:26
+1 Corinthians 2:5, 12:9, 13:2, 13:13, 15:14, 15:17, 16:13 2 Corinthians 1:24, 4:13, 5:7, 8:7,
+10:15, 13:5 Galatians 1:23, 2:16, 2:20, 3:2, 3:5, 3:7, 3:8, 3:9, 3:11, 3:12, 3:14, 3:22, 3:23,
+3:24, 3:25, 3:26, 5:5, 5:6, 5:22, 6:10 Ephesians 1:15, 2:8, 3:12, 3:17, 4:5, 4:13, 6:16, 6:23
+Philippians 1:25, 1:27, 2:17, 3:9 Colossians 1:4, 1:23, 2:5, 2:7, 2:12 1 Thessalonians 1:3,
+1:8, 3:2, 3:5, 3:6, 3:7, 3:10, 5:8 2 Thessalonians 1:3, 1:4, 1:11, 2:13, 3:2 1 Timothy 1:2, 1:4,
+1:5, 1:14, 1:19, 2:7, 2:15, 3:9, 3:13, 4:1, 4:6, 4:12, 5:8, 5:12, 6:10, 6:11, 6:12, 6:21 2 Timothy
+1:5, 1:13, 2:18, 2:22, 3:8, 3:10, 3:15, 4:7 Titus 1:1, 1:4, 1:13, 2:2, 2:10, 3:15 Philemon 1:5,
+1:6 Hebrews 4:2, 6:1, 6:12, 10:22, 10:38, 10:39, 11:1, 11:3, 11:4, 11:5, 11:6, 11:7, 11:8,
+11:9, 11:11, 11:13, 11:17, 11:20, 11:21, 11:22, 11:23, 11:24, 11:27, 11:28, 11:29, 11:30,
+11:31, 11:33, 11:39, 12:2, 13:7 James 1:3, 1:6, 2:1, 2:5, 2:14, 2:17, 2:18, 2:20, 2:22, 2:24,
+2:26, 5:15 1 Peter 1:5, 1:7, 1:9, 1:21, 5:9 2 Peter 1:1, 1:5 1 John 5:4 Jude 1:3, 1:20
+Revelation 2:13, 2:19, 13:10, 14:12
+
+faithful (4103)
+Matthew 24:45, 25:21, 25:23 Luke 12:42, 16:10, 16:11, 16:12, 19:17 John 20:27 Acts
+10:45, 13:34, 16:1, 16:15 1 Corinthians 1:9, 4:2, 4:17, 7:25, 10:13 2 Corinthians 1:18, 6:15
+Galatians 3:9 Ephesians 1:1, 6:21 Colossians 1:2, 1:7, 4:7, 4:9 1 Thessalonians 5:24 2
+Thessalonians 3:3 1 Timothy 1:12, 1:15, 3:1, 3:11, 4:3, 4:9, 4:10, 4:12, 5:16, 6:2 2
+Timothy 2:2, 2:11, 2:13 Titus 1:6, 1:9, 3:8 Hebrews 2:17, 3:2, 3:5, 10:23, 11:11 1 Peter
+4:19, 5:12 1 John 1:9 3 John 1:5 Revelation 1:5, 2:10, 3:14, 17:14, 19:11, 21:5, 22:6
+
+father (3962, 3971)
+Matthew 2:22, 3:9, 4:21, 4:22, 5:16, 5:45, 5:48, 6:1, 6:4, 6:6, 6:8, 6:9, 6:14, 6:15, 6:18, 6:26,
+6:32, 7:11, 7:21, 8:21, 10:20, 10:21, 10:29, 10:32, 10:33, 10:35, 10:37, 11:25, 11:26, 11:27,
+12:50, 13:43, 15:4, 15:5, 15:6, 15:13, 16:17, 16:27, 18:10, 18:14, 18:19, 18:35, 19:5, 19:19,
+19:29, 20:23, 21:31, 23:9, 23:30, 23:32, 24:36, 25:34, 26:29, 26:39, 26:42, 26:53, 28:19
+Mark 1:20, 5:40, 7:10, 7:11, 7:12, 8:38, 9:21, 9:24, 10:7, 10:19, 10:29, 11:10, 11:25, 13:12,
+13:32, 14:36, 15:21 Luke 1:17, 1:32, 1:55, 1:59, 1:62, 1:67, 1:72, 1:73, 2:33, 2:48, 3:8, 6:23,
+6:26, 6:36, 8:51, 9:26, 9:42, 9:59, 10:21, 10:22, 11:2, 11:11, 11:13, 11:47, 11:48, 12:30,
+12:32, 12:53, 14:26, 15:12, 15:17, 15:18, 15:20, 15:21, 15:22, 15:27, 15:28, 15:29, 16:24,
+16:27, 16:30, 18:20, 22:29, 22:42, 23:34, 23:46, 24:49 John 1:14, 1:18, 2:16, 3:35, 4:12,
+4:20, 4:21, 4:23, 4:53, 5:17, 5:18, 5:19, 5:20, 5:21, 5:22, 5:23, 5:26, 5:36, 5:37, 5:43, 5:45,
+6:27, 6:31, 6:32, 6:37, 6:40, 6:42, 6:44, 6:45, 6:46, 6:49, 6:57, 6:58, 6:65, 7:22, 8:16, 8:18,
+8:19, 8:27, 8:28, 8:38, 8:39, 8:41, 8:42, 8:44, 8:49, 8:53, 8:54, 8:56, 10:15, 10:17, 10:18,
+10:25, 10:29, 10:30, 10:32, 10:36, 10:37, 10:38, 11:41, 12:26, 12:27, 12:28, 12:49, 12:50,
+
+13:1, 13:3, 14:2, 14:6, 14:7, 14:8, 14:9, 14:10, 14:11, 14:12, 14:13, 14:16, 14:20, 14:21,
+14:23, 14:24, 14:26, 14:28, 14:31, 15:1, 15:8, 15:9, 15:10, 15:15, 15:16, 15:23, 15:24, 15:26,
+16:3, 16:10, 16:15, 16:17, 16:23, 16:25, 16:26, 16:27, 16:28, 16:32, 17:1, 17:5, 17:11, 17:21,
+17:24, 17:25, 18:11, 20:17, 20:21 Acts 1:4, 1:7, 2:33, 3:13, 3:25, 4:25, 5:30, 7:2, 7:4, 7:11,
+7:12, 7:14, 7:15, 7:19, 7:20, 7:32, 7:38, 7:39, 7:44, 7:45, 7:51, 7:52, 13:17, 13:32, 13:36,
+15:10, 16:1, 16:3, 22:1, 22:3, 22:14, 24:14, 26:6, 28:8, 28:17, 28:25 Romans 1:7, 4:11, 4:12,
+4:16, 4:17, 4:18, 6:4, 8:15, 9:5, 9:10, 11:28, 15:6, 15:8 1 Corinthians 1:3, 4:15, 5:1, 8:6,
+10:1, 15:24 2 Corinthians 1:2, 1:3, 6:18, 11:31 Galatians 1:1, 1:3, 1:4, 4:2, 4:6 Ephesians
+1:2, 1:3, 1:17, 2:18, 3:14, 4:6, 5:20, 5:31, 6:2, 6:4, 6:23 Philippians 1:2, 2:11, 2:22, 4:20
+Colossians 1:2, 1:3, 1:12, 3:17, 3:21 1 Thessalonians 1:1, 1:3, 2:11, 3:11, 3:13 2
+Thessalonians 1:1, 1:2, 2:16 1 Timothy 1:2, 5:1 2 Timothy 1:2 Titus 1:4 Philemon 1:3
+Hebrews 1:1, 1:5, 3:9, 7:10, 8:9, 11:23, 12:7, 12:9 James 1:17, 1:27, 2:21, 3:9 1 Peter 1:2,
+1:3, 1:17 2 Peter 1:17, 3:4 1 John 1:2, 1:3, 2:1, 2:13, 2:14, 2:15, 2:16, 2:22, 2:23, 2:24, 3:1,
+4:14 2 John 1:3, 1:4, 1:9 Jude 1:1 Revelation 1:6, 2:28, 3:5, 3:21, 14:1
+
+flesh (4561)
+Matthew 16:17, 19:5, 19:6, 24:22, 26:41 Mark 10:8, 13:20, 14:38 Luke 3:6, 24:39 John
+1:13, 1:14, 3:6, 6:51, 6:52, 6:53, 6:54, 6:55, 6:56, 6:63, 8:15, 17:2 Acts 2:17, 2:26, 2:31
+Romans 1:3, 2:28, 3:20, 4:1, 6:19, 7:5, 7:18, 7:25, 8:3, 8:4, 8:5, 8:6, 8:7, 8:8, 8:9, 8:12, 8:13,
+9:3, 9:5, 9:8, 11:14, 13:14 1 Corinthians 1:26, 1:29, 5:5, 6:16, 7:28, 10:18, 15:39, 15:50 2
+Corinthians 1:17, 4:11, 5:16, 7:1, 7:5, 10:2, 10:3, 11:18, 12:7 Galatians 1:16, 2:16, 2:20,
+3:3, 4:13, 4:14, 4:23, 4:29, 5:13, 5:16, 5:17, 5:19, 5:24, 6:8, 6:12, 6:13 Ephesians 2:3, 2:11,
+2:14, 5:29, 5:31, 6:5, 6:12 Philippians 1:22, 1:24, 3:3, 3:4 Colossians 1:22, 1:24, 2:1, 2:5,
+2:11, 2:13, 2:18, 2:23, 3:22 1 Timothy 3:16 Philemon 1:16 Hebrews 2:14, 5:7, 9:10, 9:13,
+10:20, 12:9 James 5:3 1 Peter 1:24, 3:18, 3:21, 4:1, 4:2, 4:6 2 Peter 2:10, 2:18 1 John 2:16,
+4:2 2 John 1:7 Jude 1:8, 1:23 Revelation 17:16, 19:18, 19:21
+
+foreknow (4267, 4268)
+Acts 2:23, 26:5 Romans 8:29, 11:2 1 Peter 1:2, 1:20 2 Peter 3:17
+
+forgive (863, 5483)
+Matthew 3:15, 4:11, 4:20, 4:22, 5:24, 5:40, 6:12, 6:14, 6:15, 7:4, 8:15, 8:22, 9:2, 9:5, 9:6,
+12:31, 12:32, 13:30, 13:36, 15:14, 18:12, 18:21, 18:27, 18:32, 18:35, 19:14, 19:27, 19:29,
+22:22, 22:25, 23:13, 23:23, 23:38, 24:2, 24:40, 24:41, 26:44, 26:56, 27:49, 27:50 Mark 1:18,
+1:20, 1:31, 1:34, 2:5, 2:7, 2:9, 2:10, 3:28, 4:12, 4:36, 5:19, 5:37, 7:8, 7:12, 7:27, 8:13, 10:14,
+10:28, 10:29, 11:6, 11:16, 11:25, 12:12, 12:19, 12:20, 12:22, 13:2, 13:34, 14:6, 14:50, 15:36,
+15:37 Luke 4:39, 5:11, 5:20, 5:21, 5:23, 5:24, 7:42, 7:43, 7:47, 7:48, 7:49, 8:51, 9:60, 10:30,
+11:4, 12:10, 12:39, 13:8, 13:35, 17:3, 17:4, 17:34, 17:35, 18:16, 18:28, 18:29, 19:44, 21:6,
+23:34 John 4:3, 4:28, 4:52, 8:29, 10:12, 11:44, 11:48, 12:7, 14:18, 14:27, 16:28, 18:8, 20:23
+Acts 3:14, 5:38, 8:22, 14:17, 25:11, 25:16, 27:24 Romans 1:27, 4:7, 8:32 1 Corinthians
+
+2:12, 7:11, 7:12, 7:13 2 Corinthians 2:7, 2:10, 12:13 Galatians 3:18 Ephesians 4:32
+Philippians 1:29, 2:9 Colossians 2:13, 3:13 Philemon 1:22 Hebrews 6:1 James 5:15 1
+John 2:12 Revelation 2:4, 2:20, 11:9
+
+fulfill (4137, 1603)
+Matthew 1:22, 2:15, 2:17, 2:23, 3:15, 4:14, 5:17, 8:17, 12:17, 13:35, 13:48, 23:32, 26:54,
+26:56, 27:9 Mark 1:15, 14:49 Luke 1:20, 2:40, 3:5, 4:21, 9:31, 21:24, 22:16, 24:44 John
+3:29, 7:8, 12:3, 12:38, 13:18, 15:11, 16:6, 16:24, 17:12, 17:13, 18:32, 19:24, 19:36 Acts
+1:16, 2:2, 2:28, 3:18, 5:3, 5:28, 7:23, 7:30, 12:25, 13:25, 13:27, 13:33, 13:52, 14:26, 19:21,
+24:27 Romans 1:29, 8:4, 13:8, 15:13, 15:14 2 Corinthians 7:4, 10:6 Galatians 5:14
+Ephesians 1:23, 3:19, 4:10, 5:18 Philippians 1:11, 2:2, 4:18, 4:19 Colossians 1:9, 1:25,
+2:10, 4:17 2 Thessalonians 1:11 2 Timothy 1:4 James 2:23 1 John 1:4 2 John 1:12
+Revelation 3:2, 6:11
+
+glory, glorify (1391, 1392, 1740)
+Matthew 4:8, 5:16, 6:2, 6:29, 9:8, 15:31, 16:27, 19:28, 24:30, 25:31 Mark 2:12, 8:38, 10:37,
+13:26 Luke 2:9, 2:14, 2:20, 2:32, 4:6, 4:15, 5:25, 5:26, 7:16, 9:26, 9:31, 9:32, 12:27, 13:13,
+14:10, 17:15, 17:18, 18:43, 19:38, 21:27, 23:47, 24:26 John 1:14, 2:11, 5:41, 5:44, 7:18,
+7:39, 8:50, 8:54, 9:24, 11:4, 11:40, 12:16, 12:28, 12:41, 12:43, 13:31, 13:32, 14:13, 15:8,
+16:14, 17:1, 17:4, 17:5, 17:10, 17:22, 17:24, 21:19 Acts 3:13, 4:21, 7:2, 7:55, 11:18, 12:23,
+13:48, 21:20, 22:11 Romans 1:21, 1:23, 2:7, 2:10, 3:7, 3:23, 4:20, 5:2, 6:4, 8:18, 8:21, 8:30,
+9:4, 9:23, 11:13, 11:36, 15:6, 15:7, 15:9, 16:27 1 Corinthians 2:7, 2:8, 6:20, 10:31, 11:7,
+11:15, 12:26, 15:40, 15:41, 15:43 2 Corinthians 1:20, 3:7, 3:9, 3:10, 3:11, 3:18, 4:4, 4:6,
+4:15, 4:17, 6:8, 8:19, 8:23, 9:13 Galatians 1:5, 1:24 Ephesians 1:6, 1:12, 1:14, 1:17, 1:18,
+3:13, 3:16, 3:21 Philippians 1:11, 2:11, 3:19, 3:21, 4:19, 4:20 Colossians 1:11, 1:27, 3:4 1
+Thessalonians 2:6, 2:12, 2:20 2 Thessalonians 1:9, 1:10, 1:12, 2:14, 3:1 1 Timothy 1:11,
+1:17, 3:16 2 Timothy 2:10, 4:18 Titus 2:13 Hebrews 1:3, 2:7, 2:9, 2:10, 3:3, 5:5, 9:5, 13:21
+James 2:1 1 Peter 1:7, 1:8, 1:11, 1:21, 1:24, 2:12, 4:11, 4:14, 4:16, 5:1, 5:4, 5:10 2 Peter 1:3,
+1:17, 2:10, 3:18 Jude 1:8, 1:24, 1:25 Revelation 1:6, 4:9, 4:11, 5:12, 5:13, 7:12, 11:13, 14:7,
+15:4, 15:8, 16:9, 18:1, 18:7, 19:1, 19:7, 21:11, 21:23, 21:24, 21:26
+
+godly, godliness (2150, 2152, 2153, 2317)
+Acts 3:12, 10:2, 10:7 1 Timothy 2:2, 2:10, 3:16, 4:7, 4:8, 6:3, 6:5, 6:6, 6:11 2 Timothy 3:5,
+3:12 Titus 1:1, 2:12 2 Peter 1:3, 1:6, 1:7, 2:9, 3:11
+
+good (18, 16)
+Matthew 5:45, 7:11, 7:17, 7:18, 12:34, 12:35, 19:16, 19:17, 20:15, 22:10, 25:21, 25:23
+Mark 3:4, 10:17, 10:18 Luke 1:53, 6:45, 8:8, 8:15, 11:13, 12:18, 12:19, 16:25, 18:18, 18:19,
+19:17, 23:50 John 1:46, 7:12 Acts 9:36, 11:24, 23:1 Romans 2:7, 2:10, 7:12, 7:13, 7:18,
+
+7:19, 8:28, 9:11, 10:15, 12:2, 13:3, 14:16, 15:2 2 Corinthians 5:10, 9:8 Galatians 6:6, 6:10
+Ephesians 2:10, 4:28, 4:29, 6:8 Philippians 1:6 Colossians 1:10 1 Thessalonians 3:6,
+5:15 2 Thessalonians 2:16, 2:17 1 Timothy 1:5, 1:19, 2:10, 5:10 2 Timothy 2:21, 3:17
+Titus 1:16, 2:5, 2:10, 3:1 Philemon 1:6, 1:14 Hebrews 9:11, 10:1, 13:21 James 1:17, 3:17
+1 Peter 2:18, 3:10, 3:11, 3:13, 3:16, 3:21, 4:19
+
+Good News (Evangelia) (2098, 2097)
+Matthew 4:23, 9:35, 11:5, 24:14, 26:13 Mark 1:1, 1:14, 1:15, 8:35, 10:29, 13:10, 14:9,
+16:15 Luke 1:19, 2:10, 3:18, 4:18, 7:22, 8:1, 9:6, 16:16, 20:1 Acts 5:42, 8:4, 8:12, 8:25, 8:35,
+8:40, 10:36, 11:20, 13:32, 14:7, 14:15, 14:21, 15:7, 15:35, 16:10, 17:18, 20:24 Romans 1:1,
+1:9, 1:15, 1:16, 2:16, 10:15, 10:16, 11:28, 15:16, 15:19, 15:20, 16:25 1 Corinthians 1:17,
+4:15, 9:12, 9:14, 9:16, 9:18, 9:23, 15:1, 15:2 2 Corinthians 4:3, 4:4, 9:13, 10:14, 10:16, 11:4,
+11:7 Galatians 1:6, 1:7, 1:8, 1:9, 1:11, 1:16, 1:23, 2:2, 2:5, 2:7, 2:14, 4:13 Ephesians 1:13,
+2:17, 3:6, 3:8, 6:15, 6:19 Philippians 1:5, 1:7, 1:12, 1:16, 1:27, 2:22, 4:3, 4:15 Colossians
+1:5, 1:23 1 Thessalonians 1:5, 2:2, 2:4, 2:8, 2:9, 3:2, 3:6 2 Thessalonians 1:8, 2:14 1
+Timothy 1:11 2 Timothy 1:8, 1:10, 2:8 Philemon 1:13 Hebrews 4:2, 4:6 1 Peter 1:12,
+1:25, 4:6, 4:17 Revelation 10:7, 14:6
+
+grace (5485)
+Luke 1:30, 2:40, 2:52, 4:22, 6:32, 6:33, 6:34, 17:9 John 1:14, 1:16, 1:17 Acts 2:47, 4:33, 6:8,
+7:10, 7:46, 11:23, 13:43, 14:3, 14:26, 15:11, 15:40, 18:27, 20:24, 20:32, 24:27, 25:3, 25:9
+Romans 1:5, 1:7, 3:24, 4:4, 4:16, 5:2, 5:15, 5:17, 5:20, 5:21, 6:1, 6:14, 6:15, 6:17, 7:25, 11:5,
+11:6, 12:3, 12:6, 15:15, 16:20 1 Corinthians 1:3, 1:4, 3:10, 10:30, 15:10, 15:57, 16:3, 16:23
+2 Corinthians 1:2, 1:12, 1:15, 2:14, 4:15, 6:1, 8:1, 8:4, 8:6, 8:7, 8:9, 8:16, 8:19, 9:8, 9:14,
+9:15, 12:9, 13:13 Galatians 1:3, 1:6, 1:15, 2:9, 2:21, 5:4, 6:18 Ephesians 1:2, 1:6, 1:7, 2:5,
+2:7, 2:8, 3:2, 3:7, 3:8, 4:7, 4:29, 6:24 Philippians 1:2, 1:7, 4:23 Colossians 1:2, 1:6, 3:16,
+4:6, 4:18 1 Thessalonians 1:1, 5:28 2 Thessalonians 1:2, 1:12, 2:16, 3:18 1 Timothy 1:2,
+1:14, 6:21 2 Timothy 1:2, 1:9, 2:1, 4:22 Titus 1:4, 2:11, 3:7, 3:15 Philemon 1:3, 1:25
+Hebrews 2:9, 4:16, 10:29, 12:15, 13:9, 13:25 James 4:6 1 Peter 1:2, 1:10, 1:13, 2:19, 2:20,
+3:7, 4:10, 5:5, 5:10, 5:12 2 Peter 1:2, 3:18 2 John 1:3 Jude 1:4 Revelation 1:4, 22:21
+
+heart (2588)
+Matthew 5:8, 5:28, 6:21, 9:4, 11:29, 12:34, 12:40, 13:15, 13:19, 15:8, 15:18, 15:19, 18:35,
+22:37, 24:48 Mark 2:6, 2:8, 3:5, 6:52, 7:6, 7:19, 7:21, 8:17, 11:23, 12:30, 12:33 Luke 1:17,
+1:51, 1:66, 2:19, 2:35, 2:51, 3:15, 5:22, 6:45, 8:12, 8:15, 9:47, 10:27, 12:34, 12:45, 16:15,
+21:14, 21:34, 24:25, 24:32, 24:38 John 12:40, 13:2, 14:1, 14:27, 16:6, 16:22 Acts 2:26, 2:37,
+2:46, 4:32, 5:3, 5:4, 7:23, 7:39, 7:51, 7:54, 8:21, 8:22, 11:23, 13:22, 14:17, 15:9, 16:14,
+21:13, 28:27 Romans 1:21, 1:24, 2:5, 2:15, 2:29, 5:5, 6:17, 8:27, 9:2, 10:1, 10:6, 10:8, 10:9,
+10:10, 16:18 1 Corinthians 2:9, 4:5, 7:37, 14:25 2 Corinthians 1:22, 2:4, 3:2, 3:15, 4:6,
+6:11, 7:3, 8:16, 9:7 Galatians 4:6 Ephesians 1:18, 3:17, 4:18, 5:19, 6:5, 6:22 Philippians
+
+1:7, 4:7 Colossians 2:2, 3:15, 3:16, 3:22, 4:8 1 Thessalonians 2:4, 2:17, 3:13 2
+Thessalonians 2:17, 3:5 1 Timothy 1:5 2 Timothy 2:22 Hebrews 3:8, 3:10, 3:12, 3:15,
+4:7, 4:12, 8:10, 10:16, 10:22, 13:9 James 1:26, 3:14, 4:8, 5:5, 5:8 1 Peter 1:22, 3:4, 3:15 2
+Peter 1:19, 2:14 1 John 3:19, 3:20, 3:21 Revelation 2:23, 17:17, 18:7
+
+heaven (3772)
+Matthew 3:2, 3:16, 3:17, 4:17, 5:3, 5:10, 5:12, 5:16, 5:18, 5:19, 5:20, 5:34, 5:45, 6:1, 6:9,
+6:10, 6:20, 6:26, 7:11, 7:21, 8:11, 8:20, 10:7, 10:32, 10:33, 11:11, 11:12, 11:23, 11:25, 12:50,
+13:11, 13:24, 13:31, 13:32, 13:33, 13:44, 13:45, 13:47, 13:52, 14:19, 16:1, 16:2, 16:3, 16:17,
+16:19, 18:1, 18:3, 18:4, 18:10, 18:14, 18:18, 18:19, 18:23, 19:12, 19:14, 19:21, 19:23, 20:1,
+21:25, 22:2, 22:30, 23:13, 23:22, 24:29, 24:30, 24:31, 24:35, 24:36, 25:1, 26:64, 28:2, 28:18
+Mark 1:10, 1:11, 4:32, 6:41, 7:34, 8:11, 10:21, 11:25, 11:30, 11:31, 12:25, 13:25, 13:27,
+13:31, 13:32, 14:62, 16:19 Luke 2:15, 3:21, 3:22, 4:25, 6:23, 8:5, 9:16, 9:54, 9:58, 10:15,
+10:18, 10:20, 10:21, 11:13, 11:16, 12:33, 12:56, 13:19, 15:7, 15:18, 15:21, 16:17, 17:24,
+17:29, 18:13, 18:22, 19:38, 20:4, 20:5, 21:11, 21:26, 21:33, 22:43, 24:51 John 1:32, 1:51,
+3:13, 3:27, 3:31, 6:31, 6:32, 6:33, 6:38, 6:41, 6:42, 6:50, 6:51, 6:58, 12:28, 17:1 Acts 1:10,
+1:11, 2:2, 2:5, 2:19, 2:34, 3:21, 4:12, 4:24, 7:42, 7:49, 7:55, 7:56, 9:3, 10:11, 10:12, 10:16,
+11:5, 11:6, 11:9, 11:10, 14:15, 17:24, 22:6 Romans 1:18, 10:6 1 Corinthians 8:5, 15:47 2
+Corinthians 5:1, 12:2 Galatians 1:8 Ephesians 1:10, 3:15, 4:10, 6:9 Philippians 3:20
+Colossians 1:5, 1:16, 1:20, 1:23, 4:1 1 Thessalonians 1:10, 4:16 2 Thessalonians 1:7
+Hebrews 1:10, 4:14, 7:26, 8:1, 9:23, 9:24, 11:12, 12:23, 12:25, 12:26 James 5:12, 5:18 1
+Peter 1:4, 1:12, 3:22 2 Peter 1:18, 3:5, 3:7, 3:10, 3:12, 3:13 Revelation 3:12, 4:1, 4:2, 5:3,
+5:13, 6:13, 6:14, 8:1, 8:10, 9:1, 10:1, 10:4, 10:5, 10:6, 10:8, 11:6, 11:12, 11:13, 11:15, 11:19,
+12:1, 12:3, 12:4, 12:7, 12:8, 12:10, 12:12, 13:6, 13:13, 14:2, 14:7, 14:13, 14:17, 15:1, 15:5,
+16:11, 16:21, 18:1, 18:4, 18:5, 18:20, 19:1, 19:11, 19:14, 20:1, 20:9, 20:11, 21:1, 21:2, 21:10
+
+heir (2818, 4789)
+Matthew 21:38 Mark 12:7 Luke 20:14 Romans 4:13, 4:14, 8:17 Galatians 3:29, 4:1, 4:7
+Ephesians 3:6 Titus 3:7 Hebrews 1:2, 6:17, 11:7, 11:9 James 2:5 1 Peter 3:7
+
+hell (1067)
+Matthew 5:22, 5:29, 5:30, 10:28, 18:9, 23:15, 23:33 Mark 9:43, 9:45, 9:47 Luke 12:5
+James 3:6
+
+holy (40)
+Matthew 1:18, 1:20, 3:11, 4:5, 7:6, 12:32, 24:15, 27:52, 27:53, 28:19 Mark 1:8, 1:24, 3:29,
+6:20, 8:38, 12:36, 13:11 Luke 1:15, 1:35, 1:41, 1:49, 1:67, 1:70, 1:72, 2:25, 2:26, 3:16, 3:22,
+4:1, 4:34, 9:26, 10:21, 11:13, 12:10, 12:12 John 1:33, 6:69, 14:26, 17:11, 20:22 Acts 1:2, 1:5,
+1:8, 1:16, 2:4, 2:33, 2:38, 3:14, 3:21, 4:8, 4:25, 4:27, 4:30, 4:31, 5:3, 5:32, 6:5, 6:13, 7:33,
+
+7:51, 7:55, 8:15, 8:17, 8:19, 9:17, 9:31, 10:22, 10:38, 10:44, 10:45, 10:47, 11:15, 11:16,
+11:24, 13:2, 13:4, 13:9, 13:52, 15:8, 15:28, 16:6, 19:2, 19:6, 20:23, 20:28, 21:11, 21:28,
+28:25 Romans 1:2, 5:5, 7:12, 9:1, 11:16, 12:1, 14:17, 15:13, 15:16, 16:16 1 Corinthians
+1:2, 3:17, 6:19, 7:14, 7:34, 12:3, 14:33, 16:20 2 Corinthians 6:6, 13:12, 13:13 Ephesians
+1:4, 1:13, 2:21, 3:5, 4:30, 5:27 Colossians 1:22, 3:12 1 Thessalonians 1:5, 1:6, 3:13, 4:8,
+5:26 2 Timothy 1:9, 1:14 Titus 3:5 Philemon 1:5 Hebrews 2:4, 3:1, 3:7, 6:4, 9:1, 9:2, 9:8,
+9:12, 9:24, 9:25, 10:15, 10:19, 13:11 1 Peter 1:12, 1:15, 1:16, 2:5, 2:9, 3:5 2 Peter 1:18,
+1:21, 2:21, 3:2, 3:11 Jude 1:14, 1:20 Revelation 4:8, 6:10, 11:2, 14:10, 20:6, 21:2, 21:10,
+22:11, 22:19
+
+Holy Spirit ()
+Matthew 1:18, 1:20, 3:11, 12:32, 28:19 Mark 1:8, 3:29, 12:36, 13:11 Luke 1:15, 1:35, 1:41,
+1:67, 2:25, 2:26, 3:16, 3:22, 4:1, 10:21, 12:10, 12:12 John 1:33, 14:26, 20:22 Acts 1:2, 1:5,
+1:8, 1:16, 2:4, 2:33, 2:38, 4:8, 4:25, 4:31, 5:3, 5:32, 6:5, 7:51, 7:55, 8:15, 8:17, 8:19, 9:17,
+9:31, 10:38, 10:44, 10:45, 10:47, 11:15, 11:16, 11:24, 13:2, 13:4, 13:9, 13:52, 15:8, 15:28,
+16:6, 19:2, 19:6, 20:23, 20:28, 21:11, 28:25 Romans 5:5, 9:1, 14:17, 15:13, 15:16 1
+Corinthians 6:19, 12:3 2 Corinthians 6:6, 13:13 (or 13:14) Ephesians 1:13, 4:30 1
+Thessalonians 1:5, 1:6, 4:8 2 Timothy 1:14 Titus 3:5 Hebrews 2:4, 3:7, 6:4, 9:8, 10:15 1
+Peter 1:12 2 Peter 1:21 (1 John 5:7 in some) Jude 1:20
+
+honor (5091, 5092)
+Matthew 15:4, 15:6, 15:8, 19:19, 27:6, 27:9 Mark 7:6, 7:10, 10:19 Luke 18:20 John 4:44,
+5:23, 8:49, 12:26 Acts 4:34, 5:2, 5:3, 7:16, 19:19, 28:10 Romans 2:7, 2:10, 9:21, 12:10, 13:7
+1 Corinthians 6:20, 7:23, 12:23, 12:24 Ephesians 6:2 Colossians 2:23 1 Thessalonians
+4:4 1 Timothy 1:17, 5:3, 5:17, 6:1, 6:16 2 Timothy 2:20, 2:21 Hebrews 2:7, 2:9, 3:3, 5:4 1
+Peter 1:7, 2:7, 2:17, 3:7 2 Peter 1:17 Revelation 4:9, 4:11, 5:12, 5:13, 7:12, 21:26
+
+hope (1680, 1679)
+Matthew 12:21 Luke 6:34, 23:8, 24:21 John 5:45 Acts 2:26, 16:19, 23:6, 24:15, 24:26,
+26:6, 26:7, 27:20, 28:20 Romans 4:18, 5:2, 5:4, 5:5, 8:20, 8:24, 8:25, 12:12, 15:4, 15:12,
+15:13, 15:24 1 Corinthians 9:10, 13:7, 13:13, 15:19, 16:7 2 Corinthians 1:7, 1:10, 1:13,
+3:12, 5:11, 8:5, 10:15, 13:6 Galatians 5:5 Ephesians 1:18, 2:12, 4:4 Philippians 1:20, 2:19,
+2:23 Colossians 1:5, 1:23, 1:27 1 Thessalonians 1:3, 2:19, 4:13, 5:8 2 Thessalonians 2:16
+1 Timothy 1:1, 3:14, 4:10, 5:5, 6:17 Titus 1:2, 2:13 Philemon 1:22 Hebrews 3:6, 6:11,
+6:18, 7:19, 10:23 1 Peter 1:3, 1:13, 1:21, 3:5, 3:15 1 John 3:3 2 John 1:12 3 John 1:14
+
+inheritance (2817)
+Matthew 21:38 Mark 12:7 Luke 12:13, 20:14 Acts 7:5, 20:32 Galatians 3:18 Ephesians
+1:14, 1:18, 5:5 Colossians 3:24 Hebrews 9:15, 11:8 1 Peter 1:4
+
+Jesus Christ, Christ Jesus ()
+Matthew 1:1, 1:18 Mark 1:1 John 1:17, 17:3 Acts 2:38, 3:6, 4:10, 8:12, 9:34, 10:36, 10:48,
+11:17, 15:26, 16:18, 24:24, 28:31 Romans 1:1, 1:4, 1:6, 1:7, 1:8, 2:16, 3:22, 3:24, 3:26, 4:24,
+5:1, 5:11, 5:15, 5:17, 5:21, 6:3, 6:11, 6:23, 7:25, 8:1, 8:2, 8:11, 8:34, 8:39, 13:14, 15:5, 15:6,
+15:8, 15:16, 15:17, 15:30, 16:3, 16:25, 16:27 1 Corinthians 1:1, 1:2, 1:3, 1:4, 1:7, 1:8, 1:9,
+1:10, 1:30, 2:2, 3:11, 4:15, 6:11, 8:6, 15:31, 15:57, 16:24 2 Corinthians 1:1, 1:2, 1:3, 1:19,
+4:5, 4:6, 8:9, 13:5, 13:13 Galatians 1:1, 1:3, 1:12, 2:4, 2:16, 3:1, 3:14, 3:22, 3:26, 3:28, 4:14,
+5:6, 5:24, 6:14, 6:18 Ephesians 1:1, 1:2, 1:3, 1:5, 1:17, 2:6, 2:7, 2:10, 2:13, 2:20, 3:1, 3:6,
+3:11, 3:21, 4:20, 6:23, 6:24 Philippians 1:1, 1:2, 1:6, 1:8, 1:11, 1:19, 1:26, 2:5, 2:10, 2:11,
+2:21, 3:3, 3:8, 3:14, 3:20, 4:7, 4:19, 4:21, 4:23 Colossians 1:1, 1:3, 1:4, 2:6, 4:12 1
+Thessalonians 1:1, 1:3, 2:14, 5:9, 5:18, 5:23, 5:28 2 Thessalonians 1:1, 1:2, 1:12, 2:1, 2:14,
+2:16, 3:6, 3:12, 3:18 1 Timothy 1:1, 1:2, 1:12, 1:14, 1:15, 1:16, 2:5, 3:13, 4:6, 5:21, 6:3, 6:13,
+6:14 2 Timothy 1:1, 1:2, 1:9, 1:10, 1:13, 2:1, 2:3, 2:8, 2:10, 3:12, 3:15, 4:1 Titus 1:1, 1:4,
+2:13, 3:6 Philemon 1:1, 1:3, 1:9, 1:23, 1:25 Hebrews 10:10, 13:8, 13:21 James 1:1, 2:1 1
+Peter 1:1, 1:2, 1:3, 1:7, 1:13, 2:5, 3:21, 4:11 2 Peter 1:1, 1:8, 1:11, 1:14, 1:16, 2:20, 3:18 1
+John 1:3, 2:1, 3:23, 4:2, 5:6, 5:20 2 John 1:3, 1:7 Revelation 1:1, 1:2, 1:5
+
+joy (5479)
+Matthew 2:10, 13:20, 13:44, 25:21, 25:23, 28:8 Mark 4:16 Luke 2:10, 8:13, 10:17, 15:7,
+15:10, 24:41, 24:52 John 3:29, 15:11, 16:20, 16:21, 16:22, 16:24, 17:13 Acts 8:8, 12:14,
+13:52, 15:3 Romans 14:17, 15:13, 15:32 2 Corinthians 1:24, 2:3, 7:4, 7:13, 8:2 Galatians
+5:22 Philippians 1:4, 1:25, 2:2, 2:29, 4:1 Colossians 1:11 1 Thessalonians 1:6, 2:19, 2:20,
+3:9 2 Timothy 1:4 Philemon 1:7 Hebrews 10:34, 12:2, 12:11, 13:17 James 1:2, 4:9 1
+Peter 1:8 1 John 1:4 2 John 1:12 3 John 1:4
+
+judge (2919)
+Matthew 5:40, 7:1, 7:2, 19:28 Luke 6:37, 7:43, 12:57, 19:22, 22:30 John 3:18, 5:22, 5:30,
+7:24, 7:51, 8:15, 8:16, 8:26, 8:50, 12:47, 12:48, 16:11, 18:31 Acts 3:13, 4:19, 7:7, 13:27,
+13:46, 15:19, 16:4, 16:15, 17:31, 20:16, 21:25, 23:3, 23:6, 24:21, 25:9, 25:10, 25:20, 25:25,
+26:6, 26:8, 27:1 Romans 2:1, 2:3, 2:12, 2:16, 2:27, 3:6, 3:7, 14:3, 14:4, 14:5, 14:10, 14:13,
+14:22 1 Corinthians 2:2, 4:5, 5:3, 5:12, 5:13, 6:1, 6:2, 6:3, 6:6, 7:37, 10:15, 10:29, 11:13,
+11:31, 11:32 2 Corinthians 2:1, 5:14 Colossians 2:16 2 Thessalonians 2:12 2 Timothy
+4:1 Titus 3:12 Hebrews 10:30, 13:4 James 2:12, 4:11, 4:12, 5:9 1 Peter 1:17, 2:23, 4:5, 4:6
+Revelation 11:18, 16:5, 18:8, 18:20, 19:2, 19:11, 20:12, 20:13
+
+justify (1344)
+Matthew 11:19, 12:37 Luke 7:29, 7:35, 10:29, 18:14 Acts 13:39 Romans 2:13, 3:4, 3:20,
+3:24, 3:26, 3:28, 3:30, 4:2, 4:5, 5:1, 5:9, 6:7, 8:30, 8:33 1 Corinthians 6:11 Galatians 2:16,
+2:17, 3:8, 3:11, 3:24, 5:4 1 Timothy 3:16 Titus 3:7 James 2:21, 2:24, 2:25
+
+kingdom (932)
+Matthew 3:2, 4:8, 4:17, 4:23, 5:3, 5:10, 5:19, 5:20, 6:10, 6:33, 7:21, 8:11, 8:12, 9:35, 10:7,
+11:11, 11:12, 12:25, 12:26, 12:28, 13:11, 13:19, 13:24, 13:31, 13:33, 13:38, 13:41, 13:43,
+13:44, 13:45, 13:47, 13:52, 16:19, 16:28, 18:1, 18:3, 18:4, 18:23, 19:12, 19:14, 19:23, 19:24,
+20:1, 20:21, 21:31, 21:43, 22:2, 23:13, 24:7, 24:14, 25:1, 25:34, 26:29 Mark 1:15, 3:24,
+4:11, 4:26, 4:30, 6:23, 9:1, 9:47, 10:14, 10:15, 10:23, 10:24, 10:25, 11:10, 12:34, 13:8, 14:25,
+15:43 Luke 1:33, 4:5, 4:43, 6:20, 7:28, 8:1, 8:10, 9:2, 9:11, 9:27, 9:60, 9:62, 10:9, 10:11,
+11:2, 11:17, 11:18, 11:20, 12:31, 12:32, 13:18, 13:20, 13:28, 13:29, 14:15, 16:16, 17:20,
+17:21, 18:16, 18:17, 18:24, 18:25, 18:29, 19:11, 19:12, 19:15, 21:10, 21:31, 22:16, 22:18,
+22:29, 22:30, 23:42, 23:51 John 3:3, 3:5, 18:36 Acts 1:3, 1:6, 8:12, 14:22, 19:8, 20:25, 28:23,
+28:31 Romans 14:17 1 Corinthians 4:20, 6:9, 6:10, 15:24, 15:50 Galatians 5:21
+Ephesians 5:5 Colossians 1:13, 4:11 1 Thessalonians 2:12 2 Thessalonians 1:5 2
+Timothy 4:1, 4:18 Hebrews 1:8, 11:33, 12:28 James 2:5 2 Peter 1:11 Revelation 1:6, 1:9,
+5:10, 11:15, 12:10, 16:10, 17:12, 17:17, 17:18
+
+lamb (721, 286)
+John 1:29, 1:36, 21:15 Acts 8:32 1 Peter 1:19 Revelation 5:6, 5:8, 5:12, 5:13, 6:1, 6:16, 7:9,
+7:10, 7:14, 7:17, 12:11, 13:8, 13:11, 14:1, 14:4, 14:10, 15:3, 17:14, 19:7, 19:9, 21:9, 21:14,
+21:22, 21:23, 21:27, 22:1, 22:3
+
+law (3551, 3544)
+Matthew 5:17, 5:18, 7:12, 11:13, 12:5, 22:35, 22:36, 22:40, 23:23 Luke 2:22, 2:23, 2:24,
+2:27, 2:39, 7:30, 10:25, 10:26, 11:45, 11:46, 11:52, 14:3, 16:16, 16:17, 24:44 John 1:17,
+1:45, 7:19, 7:23, 7:49, 7:51, 8:5, 8:17, 10:34, 12:34, 15:25, 18:31, 19:7 Acts 6:13, 7:53,
+13:15, 13:38, 15:5, 18:13, 18:15, 21:20, 21:24, 21:28, 22:3, 22:12, 23:3, 23:29, 24:14, 25:8,
+28:23 Romans 2:12, 2:13, 2:14, 2:15, 2:17, 2:18, 2:20, 2:23, 2:25, 2:26, 2:27, 3:19, 3:20,
+3:21, 3:27, 3:28, 3:31, 4:13, 4:14, 4:15, 4:16, 5:13, 5:20, 6:14, 6:15, 7:1, 7:2, 7:3, 7:4, 7:5, 7:6,
+7:7, 7:8, 7:9, 7:12, 7:14, 7:16, 7:21, 7:22, 7:23, 7:25, 8:2, 8:3, 8:4, 8:7, 9:31, 10:4, 10:5, 13:8,
+13:10 1 Corinthians 9:8, 9:9, 9:20, 14:21, 14:34, 15:56 Galatians 2:16, 2:19, 2:21, 3:2, 3:5,
+3:10, 3:11, 3:12, 3:13, 3:17, 3:18, 3:19, 3:21, 3:23, 3:24, 4:4, 4:5, 4:21, 5:3, 5:4, 5:14, 5:18,
+5:23, 6:2, 6:13 Ephesians 2:15 Philippians 3:5, 3:6, 3:9 1 Timothy 1:8, 1:9 Titus 3:9, 3:13
+Hebrews 7:5, 7:12, 7:16, 7:19, 7:28, 8:4, 8:10, 9:19, 9:22, 10:1, 10:8, 10:16, 10:28 James
+1:25, 2:8, 2:9, 2:10, 2:11, 2:12, 4:11
+
+life (2222)
+Matthew 7:14, 18:8, 18:9, 19:16, 19:17, 19:29, 25:46 Mark 9:43, 9:45, 10:17, 10:30 Luke
+10:25, 12:15, 16:25, 18:18, 18:30 John 1:4, 3:15, 3:16, 3:36, 4:14, 4:36, 5:24, 5:26, 5:29,
+5:39, 5:40, 6:27, 6:33, 6:35, 6:40, 6:47, 6:48, 6:51, 6:53, 6:54, 6:63, 6:68, 8:12, 10:10, 10:28,
+11:25, 12:25, 12:50, 14:6, 17:2, 17:3, 20:31 Acts 2:28, 3:15, 5:20, 8:33, 11:18, 13:46, 13:48,
+
+17:25 Romans 2:7, 5:10, 5:17, 5:18, 5:21, 6:4, 6:22, 6:23, 8:2, 8:6, 8:10, 8:38, 11:15 1
+Corinthians 3:22, 15:19 2 Corinthians 2:16, 4:10, 4:11, 4:12, 5:4 Galatians 6:8 Ephesians
+4:18 Philippians 1:20, 2:16, 4:3 Colossians 3:3, 3:4 1 Timothy 1:16, 4:8, 6:12, 6:19 2
+Timothy 1:1, 1:10 Titus 1:2, 3:7 Hebrews 7:3, 7:16 James 1:12, 4:14 1 Peter 3:7, 3:10 2
+Peter 1:3 1 John 1:1, 1:2, 2:25, 3:14, 3:15, 5:11, 5:12, 5:13, 5:16, 5:20 Jude 1:21
+Revelation 2:7, 2:10, 3:5, 7:17, 11:11, 13:8, 17:8, 20:12, 20:15, 21:6, 21:27, 22:1, 22:2,
+22:14, 22:17, 22:19
+
+lion (3023)
+2 Timothy 4:17 Hebrews 11:33 1 Peter 5:8 Revelation 4:7, 5:5, 9:8, 9:17, 10:3, 13:2
+
+Lord (2962)
+Matthew 1:20, 1:22, 1:24, 2:13, 2:15, 2:19, 3:3, 4:7, 4:10, 5:33, 6:24, 7:21, 7:22, 8:2, 8:6, 8:8,
+8:21, 8:25, 9:28, 9:38, 10:24, 10:25, 11:25, 12:8, 13:27, 14:28, 14:30, 15:22, 15:25, 15:27,
+16:22, 17:4, 17:15, 18:21, 18:25, 18:27, 18:31, 18:32, 18:34, 20:8, 20:30, 20:31, 20:33, 21:3,
+21:9, 21:30, 21:40, 21:42, 22:37, 22:43, 22:44, 22:45, 23:39, 24:42, 24:45, 24:46, 24:48,
+24:50, 25:11, 25:18, 25:19, 25:20, 25:21, 25:22, 25:23, 25:24, 25:26, 25:37, 25:44, 26:22,
+27:10, 27:63, 28:2 Mark 1:3, 2:28, 5:19, 7:28, 11:3, 11:9, 12:9, 12:11, 12:29, 12:30, 12:36,
+12:37, 13:20, 13:35, 16:19, 16:20 Luke 1:6, 1:9, 1:11, 1:15, 1:16, 1:17, 1:25, 1:28, 1:32, 1:38,
+1:43, 1:45, 1:46, 1:58, 1:66, 1:68, 1:76, 2:9, 2:11, 2:15, 2:22, 2:23, 2:24, 2:26, 2:39, 3:4, 4:8,
+4:12, 4:18, 4:19, 5:8, 5:12, 5:17, 6:5, 6:46, 7:6, 7:13, 7:19, 9:54, 9:59, 9:61, 10:1, 10:2, 10:17,
+10:21, 10:27, 10:39, 10:40, 10:41, 11:1, 11:39, 12:36, 12:37, 12:41, 12:42, 12:43, 12:45,
+12:46, 12:47, 13:8, 13:15, 13:23, 13:25, 13:35, 14:21, 14:22, 14:23, 16:3, 16:5, 16:8, 16:13,
+17:5, 17:6, 17:37, 18:6, 18:41, 19:8, 19:16, 19:18, 19:20, 19:25, 19:31, 19:33, 19:34, 19:38,
+20:13, 20:15, 20:37, 20:42, 20:44, 22:33, 22:38, 22:49, 22:61, 24:3, 24:34 John 1:23, 4:11,
+4:15, 4:19, 4:49, 5:7, 6:23, 6:34, 6:68, 8:11, 9:36, 9:38, 11:2, 11:3, 11:12, 11:21, 11:27,
+11:32, 11:34, 11:39, 12:13, 12:21, 12:38, 13:6, 13:9, 13:13, 13:14, 13:16, 13:25, 13:36,
+13:37, 14:5, 14:8, 14:22, 15:15, 15:20, 20:2, 20:13, 20:15, 20:18, 20:20, 20:25, 20:28, 21:7,
+21:12, 21:15, 21:16, 21:17, 21:20, 21:21 Acts 1:6, 1:21, 1:24, 2:20, 2:21, 2:25, 2:34, 2:36,
+2:39, 2:47, 3:20, 3:22, 4:26, 4:29, 4:33, 5:9, 5:14, 5:19, 7:31, 7:33, 7:49, 7:59, 7:60, 8:16,
+8:22, 8:24, 8:25, 8:26, 8:39, 9:1, 9:5, 9:10, 9:11, 9:13, 9:15, 9:17, 9:27, 9:28, 9:31, 9:35, 9:42,
+10:4, 10:14, 10:33, 10:36, 11:8, 11:16, 11:17, 11:20, 11:21, 11:23, 11:24, 12:7, 12:11, 12:17,
+12:23, 13:2, 13:10, 13:11, 13:12, 13:44, 13:47, 13:48, 13:49, 14:3, 14:23, 15:11, 15:17,
+15:26, 15:35, 15:36, 15:40, 16:14, 16:15, 16:16, 16:19, 16:30, 16:31, 16:32, 17:24, 18:8,
+18:9, 18:25, 19:5, 19:10, 19:13, 19:17, 19:20, 20:19, 20:21, 20:24, 20:35, 21:13, 21:14, 22:8,
+22:10, 22:19, 23:11, 25:26, 26:15, 28:31 Romans 1:4, 1:7, 4:8, 4:24, 5:1, 5:11, 5:21, 6:23,
+7:25, 8:39, 9:28, 9:29, 10:9, 10:12, 10:13, 10:16, 11:3, 11:34, 12:11, 12:19, 13:14, 14:4, 14:6,
+14:8, 14:11, 14:14, 15:6, 15:11, 15:30, 16:2, 16:8, 16:11, 16:12, 16:13, 16:18, 16:20, 16:22 1
+Corinthians 1:2, 1:3, 1:7, 1:8, 1:9, 1:10, 1:31, 2:8, 2:16, 3:5, 3:20, 4:4, 4:5, 4:17, 4:19, 5:4,
+5:5, 6:11, 6:13, 6:14, 6:17, 7:10, 7:12, 7:17, 7:22, 7:25, 7:32, 7:34, 7:35, 7:39, 8:5, 8:6, 9:1,
+9:2, 9:5, 9:14, 10:21, 10:22, 10:26, 11:11, 11:23, 11:26, 11:27, 11:32, 12:3, 12:5, 14:21,
+14:37, 15:31, 15:57, 15:58, 16:7, 16:10, 16:19, 16:22, 16:23 2 Corinthians 1:2, 1:3, 1:14,
+
+2:12, 3:16, 3:17, 3:18, 4:5, 4:14, 5:6, 5:8, 5:11, 6:17, 6:18, 8:5, 8:9, 8:19, 8:21, 10:8, 10:17,
+10:18, 11:31, 12:1, 12:8, 13:10, 13:13 Galatians 1:3, 1:19, 4:1, 5:10, 6:14, 6:18 Ephesians
+1:2, 1:3, 1:15, 1:17, 2:21, 3:11, 4:1, 4:5, 4:17, 5:8, 5:10, 5:17, 5:19, 5:20, 5:22, 6:1, 6:4, 6:5,
+6:7, 6:8, 6:9, 6:10, 6:21, 6:23, 6:24 Philippians 1:2, 1:14, 2:11, 2:19, 2:24, 2:29, 3:1, 3:8,
+3:20, 4:1, 4:2, 4:4, 4:5, 4:10, 4:23 Colossians 1:3, 1:10, 2:6, 3:13, 3:17, 3:18, 3:20, 3:22, 3:23,
+3:24, 4:1, 4:7, 4:17 1 Thessalonians 1:1, 1:3, 1:6, 1:8, 2:15, 2:19, 3:8, 3:11, 3:12, 3:13, 4:1,
+4:2, 4:6, 4:15, 4:16, 4:17, 5:2, 5:9, 5:12, 5:23, 5:27, 5:28 2 Thessalonians 1:1, 1:2, 1:7, 1:8,
+1:9, 1:12, 2:1, 2:2, 2:8, 2:13, 2:14, 2:16, 3:1, 3:3, 3:4, 3:5, 3:6, 3:12, 3:16, 3:18 1 Timothy 1:2,
+1:12, 1:14, 6:3, 6:14, 6:15 2 Timothy 1:2, 1:8, 1:16, 1:18, 2:7, 2:19, 2:22, 2:24, 3:11, 4:8,
+4:14, 4:17, 4:18, 4:22 Philemon 1:3, 1:5, 1:16, 1:20, 1:25 Hebrews 1:10, 2:3, 7:14, 7:21,
+8:2, 8:8, 8:9, 8:10, 8:11, 10:16, 10:30, 12:5, 12:6, 12:14, 13:6, 13:20 James 1:1, 1:7, 2:1, 3:9,
+4:10, 4:15, 5:4, 5:7, 5:8, 5:10, 5:11, 5:14, 5:15 1 Peter 1:3, 1:25, 2:3, 2:13, 3:6, 3:12, 3:15 2
+Peter 1:2, 1:8, 1:11, 1:14, 1:16, 2:9, 2:11, 2:20, 3:2, 3:8, 3:9, 3:10, 3:15, 3:18 Jude 1:4, 1:9,
+1:14, 1:17, 1:21, 1:25 Revelation 1:8, 4:8, 4:11, 7:14, 11:4, 11:8, 11:15, 11:17, 14:13, 15:3,
+15:4, 16:7, 17:14, 18:8, 19:6, 19:16, 21:22, 22:5, 22:6, 22:20, 22:21
+
+love (26, 27)
+Matthew 3:17, 12:18, 17:5, 24:12 Mark 1:11, 9:7, 12:6 Luke 3:22, 11:42, 20:13 John 5:42,
+13:35, 15:9, 15:10, 15:13, 17:26 Acts 15:25 Romans 1:7, 5:5, 5:8, 8:35, 8:39, 11:28, 12:9,
+12:19, 13:10, 14:15, 15:30, 16:5, 16:8, 16:9, 16:12 1 Corinthians 4:14, 4:17, 4:21, 8:1,
+10:14, 13:1, 13:2, 13:3, 13:4, 13:8, 13:13, 14:1, 15:58, 16:14, 16:24 2 Corinthians 2:4, 2:8,
+5:14, 6:6, 7:1, 8:7, 8:8, 8:24, 12:19, 13:11, 13:13 Galatians 5:6, 5:13, 5:22 Ephesians 1:4,
+1:15, 2:4, 3:17, 3:19, 4:2, 4:15, 4:16, 5:1, 5:2, 6:21, 6:23 Philippians 1:9, 1:16, 2:1, 2:2, 2:12,
+4:1 Colossians 1:4, 1:7, 1:8, 1:13, 2:2, 3:14, 4:7, 4:9, 4:14 1 Thessalonians 1:3, 2:8, 3:6,
+3:12, 5:8, 5:13 2 Thessalonians 1:3, 2:10, 3:5 1 Timothy 1:5, 1:14, 2:15, 4:12, 6:2, 6:11 2
+Timothy 1:2, 1:7, 1:13, 2:22, 3:10 Titus 2:2 Philemon 1:1, 1:5, 1:7, 1:9, 1:16 Hebrews 6:9,
+6:10, 10:24 James 1:16, 1:19, 2:5 1 Peter 2:11, 4:8, 4:12, 5:14 2 Peter 1:7, 1:17, 3:1, 3:8,
+3:14, 3:15, 3:17 1 John 2:5, 2:7, 2:15, 3:1, 3:2, 3:16, 3:17, 3:21, 4:1, 4:7, 4:8, 4:9, 4:10, 4:11,
+4:12, 4:16, 4:17, 4:18, 5:3 2 John 1:3, 1:6 3 John 1:1, 1:2, 1:5, 1:6, 1:11 Jude 1:2, 1:3, 1:12,
+1:17, 1:20, 1:21 Revelation 2:4, 2:19
+
+mediator (3316)
+Galatians 3:19, 3:20 1 Timothy 2:5 Hebrews 8:6, 9:15, 12:24
+
+mercy (1656, 1653)
+Matthew 5:7, 9:13, 9:27, 12:7, 15:22, 17:15, 18:33, 20:30, 20:31, 23:23 Mark 5:19, 10:47,
+10:48 Luke 1:50, 1:54, 1:58, 1:72, 10:37, 16:24, 17:13, 18:38, 18:39 Romans 9:15, 9:16,
+9:18, 9:23, 11:30, 11:31, 11:32, 12:8, 15:9 1 Corinthians 7:25 2 Corinthians 4:1 Galatians
+6:16 Ephesians 2:4 Philippians 2:27 1 Timothy 1:2, 1:13, 1:16 2 Timothy 1:2, 1:16, 1:18
+
+Titus 3:5 Hebrews 4:16 James 2:13, 3:17 1 Peter 1:3, 2:10 2 John 1:3 Jude 1:2, 1:21, 1:22,
+1:23
+
+miracle (1411)
+Matthew 7:22, 11:20, 11:21, 11:23, 13:54, 13:58, 14:2, 22:29, 24:29, 24:30, 25:15, 26:64
+Mark 5:30, 6:2, 6:5, 6:14, 9:1, 9:39, 12:24, 13:25, 13:26, 14:62 Luke 1:17, 1:35, 4:14, 4:36,
+5:17, 6:19, 8:46, 9:1, 10:13, 10:19, 19:37, 21:26, 21:27, 22:69, 24:49 Acts 1:8, 2:22, 3:12,
+4:7, 4:33, 6:8, 8:10, 8:13, 10:38, 19:11 Romans 1:4, 1:16, 1:20, 8:38, 9:17, 15:13, 15:19 1
+Corinthians 1:18, 1:24, 2:4, 2:5, 4:19, 4:20, 5:4, 6:14, 12:10, 12:28, 12:29, 14:11, 15:24,
+15:43, 15:56 2 Corinthians 1:8, 4:7, 6:7, 12:9, 12:12, 13:4 Galatians 3:5 Ephesians 1:19,
+1:21, 3:7, 3:16, 3:20 Philippians 3:10 Colossians 1:11, 1:29 1 Thessalonians 1:5 2
+Thessalonians 1:7, 1:11, 2:9 2 Timothy 1:7, 1:8, 3:5 Hebrews 1:3, 2:4, 6:5, 7:16, 11:11,
+11:34 1 Peter 1:5, 3:22 2 Peter 1:3, 1:16, 2:11 Revelation 1:16, 3:8, 4:11, 5:12, 7:12,
+11:17, 12:10, 13:2, 15:8, 17:13, 18:3, 19:1
+
+nation (1484)
+Matthew 4:15, 6:32, 10:5, 10:18, 12:18, 12:21, 20:19, 20:25, 21:43, 24:7, 24:9, 24:14,
+25:32, 28:19 Mark 10:33, 10:42, 11:17, 13:8, 13:10 Luke 2:32, 7:5, 12:30, 18:32, 21:10,
+21:24, 21:25, 22:25, 23:2, 24:47 John 11:48, 11:50, 11:51, 11:52, 18:35 Acts 2:5, 4:25, 4:27,
+7:7, 7:45, 8:9, 9:15, 10:22, 10:35, 10:45, 11:1, 11:18, 13:19, 13:46, 13:47, 13:48, 14:2, 14:5,
+14:16, 14:27, 15:3, 15:7, 15:12, 15:14, 15:17, 15:19, 15:23, 17:26, 18:6, 21:11, 21:19, 21:21,
+21:25, 22:21, 24:2, 24:10, 24:17, 26:4, 26:17, 26:20, 26:23, 28:19, 28:28 Romans 1:5, 1:13,
+2:14, 2:24, 3:29, 4:17, 4:18, 9:24, 9:30, 10:19, 11:11, 11:12, 11:13, 11:25, 15:9, 15:10, 15:11,
+15:12, 15:16, 15:18, 15:27, 16:4, 16:26 1 Corinthians 1:23, 5:1, 12:2 2 Corinthians 11:26
+Galatians 1:16, 2:2, 2:8, 2:9, 2:12, 2:14, 2:15, 3:8, 3:14 Ephesians 2:11, 3:1, 3:6, 3:8, 4:17
+Colossians 1:27 1 Thessalonians 2:16, 4:5 1 Timothy 2:7, 3:16 2 Timothy 4:17 1 Peter
+2:9, 2:12, 4:3 Revelation 2:26, 5:9, 7:9, 10:11, 11:2, 11:9, 11:18, 12:5, 13:7, 14:6, 14:8, 15:3,
+15:4, 16:19, 17:15, 18:3, 18:23, 19:15, 20:3, 20:8, 21:24, 21:26, 22:2
+
+obey, obedience (5219, 5218)
+Matthew 8:27 Mark 1:27, 4:41 Luke 8:25, 17:6 Acts 6:7, 12:13 Romans 1:5, 5:19, 6:16,
+6:17, 10:16, 15:18, 16:19, 16:26 2 Corinthians 7:15, 10:5, 10:6 Ephesians 6:1, 6:5
+Philippians 2:12 Colossians 3:20, 3:22 2 Thessalonians 1:8, 3:14 Philemon 1:21
+Hebrews 5:8, 5:9, 11:8 1 Peter 1:2, 1:14, 1:22, 3:6
+
+peace (1515)
+Matthew 10:13, 10:34 Mark 5:34 Luke 1:79, 2:14, 2:29, 7:50, 8:48, 10:5, 10:6, 11:21,
+12:51, 14:32, 19:38, 24:36 John 14:27, 16:33, 20:19, 20:21, 20:26 Acts 7:26, 9:31, 10:36,
+12:20, 15:33, 16:36, 24:2 Romans 1:7, 2:10, 3:17, 5:1, 8:6, 14:17, 14:19, 15:13, 15:33, 16:20
+
+1 Corinthians 1:3, 7:15, 14:33, 16:11 2 Corinthians 1:2, 13:11 Galatians 1:3, 5:22, 6:16
+Ephesians 1:2, 2:14, 2:15, 2:17, 4:3, 6:15, 6:23 Philippians 1:2, 4:7, 4:9 Colossians 1:2,
+3:15 1 Thessalonians 1:1, 5:3, 5:23 2 Thessalonians 1:2, 3:16 1 Timothy 1:2 2 Timothy
+1:2, 2:22 Titus 1:4 Philemon 1:3 Hebrews 7:2, 11:31, 12:14, 13:20 James 2:16, 3:18 1
+Peter 1:2, 3:11, 5:14 2 Peter 1:2, 3:14 2 John 1:3 3 John 1:15 Jude 1:2 Revelation 1:4, 6:4
+
+praise (1868, 1867, 134)
+Luke 2:13, 2:20, 16:8, 19:37 Acts 2:47, 3:8, 3:9 Romans 2:29, 13:3, 15:11 1 Corinthians
+4:5, 11:2, 11:17, 11:22 2 Corinthians 8:18 Ephesians 1:6, 1:12, 1:14 Philippians 1:11, 4:8
+1 Peter 1:7, 2:14 Revelation 19:5
+
+pray (4336)
+Matthew 5:44, 6:5, 6:6, 6:7, 6:9, 14:23, 19:13, 24:20, 26:36, 26:39, 26:41, 26:42, 26:44
+Mark 1:35, 6:46, 11:24, 11:25, 12:40, 13:18, 14:32, 14:35, 14:38, 14:39 Luke 1:10, 3:21,
+5:16, 6:12, 6:28, 9:28, 11:1, 11:2, 18:1, 18:10, 18:11, 20:47, 22:40, 22:41, 22:44, 22:46 Acts
+1:24, 6:6, 8:15, 9:11, 9:40, 10:9, 10:30, 11:5, 12:12, 13:3, 14:23, 16:25, 20:36, 21:5, 28:8
+Romans 8:26 1 Corinthians 11:4, 11:5, 11:13, 14:13, 14:14, 14:15 Ephesians 6:18
+Philippians 1:9 Colossians 1:3, 1:9, 4:3 1 Thessalonians 5:17, 5:25 2 Thessalonians
+1:11, 3:1 1 Timothy 2:8 Hebrews 13:18 James 5:13, 5:14, 5:18 Jude 1:20
+
+promise (1860)
+Luke 24:49 Acts 1:4, 2:33, 2:39, 7:17, 13:23, 13:32, 23:21, 26:6 Romans 4:13, 4:14, 4:16,
+4:20, 9:4, 9:8, 9:9, 15:8 2 Corinthians 1:20, 7:1 Galatians 3:14, 3:16, 3:17, 3:18, 3:21, 3:22,
+3:29, 4:23, 4:28 Ephesians 1:13, 2:12, 3:6, 6:2 1 Timothy 4:8 2 Timothy 1:1 Hebrews 4:1,
+6:12, 6:15, 6:17, 7:6, 8:6, 9:15, 11:9, 11:13, 11:17, 11:33, 11:39 2 Peter 3:4, 3:9 1 John 2:25
+
+prophecy, prophesy (4394, 4395)
+Matthew 7:22, 11:13, 13:14, 15:7, 26:68 Mark 7:6, 14:65 Luke 1:67, 22:64 John 11:51
+Acts 2:17, 2:18, 19:6, 21:9 Romans 12:6 1 Corinthians 11:4, 11:5, 12:10, 13:2, 13:8, 13:9,
+14:1, 14:3, 14:4, 14:5, 14:6, 14:22, 14:24, 14:31, 14:39 1 Thessalonians 5:20 1 Timothy
+1:18, 4:14 1 Peter 1:10 2 Peter 1:20, 1:21 Jude 1:14 Revelation 1:3, 10:11, 11:3, 19:10,
+22:7, 22:10, 22:18, 22:19
+
+redeem (3084, 1805)
+Luke 24:21 Galatians 3:13, 4:5 Ephesians 5:16 Colossians 4:5 Titus 2:14 1 Peter 1:18
+
+redemption (3085, 629)
+Luke 1:68, 2:38, 21:28 Romans 3:24, 8:23 1 Corinthians 1:30 Ephesians 1:7, 1:14, 4:30
+Colossians 1:14 Hebrews 9:12, 11:35
+
+repent (3340, 3338)
+Matthew 3:2, 4:17, 11:20, 11:21, 12:41, 21:29, 21:32, 27:3 Mark 1:15, 6:12 Luke 10:13,
+11:32, 13:3, 13:5, 15:7, 15:10, 16:30, 17:3, 17:4 Acts 2:38, 3:19, 8:22, 17:30, 26:20 2
+Corinthians 7:8, 12:21 Hebrews 7:21 Revelation 2:5, 2:16, 2:21, 2:22, 3:3, 3:19, 9:20,
+9:21, 16:9
+
+resurrection (386, 1454)
+Matthew 22:23, 22:28, 22:30, 22:31, 27:53 Mark 12:18, 12:23 Luke 2:34, 14:14, 20:27,
+20:33, 20:35, 20:36 John 5:29, 11:24, 11:25 Acts 1:22, 2:31, 4:2, 4:33, 17:18, 17:32, 23:6,
+23:8, 24:15, 24:21, 26:23 Romans 1:4, 6:5 1 Corinthians 15:12, 15:13, 15:21, 15:42
+Philippians 3:10 2 Timothy 2:18 Hebrews 6:2, 11:35 1 Peter 1:3, 3:21 Revelation 20:5,
+20:6
+
+righteous (1342)
+Matthew 1:19, 5:45, 9:13, 10:41, 13:17, 13:43, 13:49, 20:4, 23:28, 23:29, 23:35, 25:37,
+25:46, 27:19 Mark 2:17, 6:20 Luke 1:6, 1:17, 2:25, 5:32, 12:57, 14:14, 15:7, 18:9, 20:20,
+23:47, 23:50 John 5:30, 17:25 Acts 3:14, 4:19, 10:22, 22:14, 24:15 Romans 1:17, 2:13,
+3:10, 3:26, 5:7, 5:19, 7:12 Galatians 3:11 Ephesians 6:1 Philippians 1:7, 4:8 Colossians
+4:1 2 Thessalonians 1:5, 1:6 1 Timothy 1:9 2 Timothy 4:8 Titus 1:8 Hebrews 10:38,
+11:4, 12:23 1 Peter 3:12, 3:18, 4:18 2 Peter 1:13, 2:7, 2:8 1 John 1:9, 2:1, 2:29, 3:7, 3:12
+Revelation 15:3, 16:5, 16:7, 19:2, 22:11
+
+sacrifice (2378, 2380)
+Matthew 9:13, 12:7, 22:4 Mark 12:33, 14:12 Luke 2:24, 13:1, 15:23, 15:27, 15:30, 22:7
+John 10:10 Acts 7:41, 7:42, 10:13, 11:7, 14:13 Romans 12:1 1 Corinthians 5:7, 10:18,
+10:20 Ephesians 5:2 Philippians 2:17, 4:18 Hebrews 5:1, 7:27, 8:3, 9:9, 9:23, 9:26, 10:1,
+10:5, 10:8, 10:11, 10:12, 10:26, 11:4, 13:15, 13:16 1 Peter 2:5
+
+salvation (4991, 4992)
+Mark 16:20 Luke 1:69, 1:71, 1:77, 2:30, 3:6, 19:9 John 4:22 Acts 4:12, 7:25, 13:26, 16:17,
+27:34, 28:28 Romans 1:16, 10:1, 11:11, 13:11 2 Corinthians 1:6, 6:2 Ephesians 1:13, 6:17
+Philippians 1:19, 1:28, 2:12 1 Thessalonians 5:8, 5:9 2 Timothy 2:10, 3:15 Titus 2:11
+
+Hebrews 1:14, 2:3, 2:10, 5:9, 6:9, 9:28 1 Peter 1:5, 1:9, 1:10, 2:2 2 Peter 3:15 Jude 1:3
+Revelation 7:10, 12:10, 19:1
+
+sanctify, sanctification (37, 38)
+Matthew 6:9, 23:17, 23:19 Luke 11:2 John 10:36, 17:17, 17:19 Acts 20:32, 26:18 Romans
+6:19, 6:22, 15:16 1 Corinthians 1:2, 1:30, 6:11, 7:14 Ephesians 5:26 1 Thessalonians 4:3,
+4:4, 4:7, 5:23 2 Thessalonians 2:13 1 Timothy 2:15, 4:5 2 Timothy 2:21 Hebrews 2:11,
+9:13, 10:10, 10:14, 10:29, 12:14, 13:12 1 Peter 1:2, 3:15 Revelation 22:11
+
+save (4982)
+Matthew 1:21, 8:25, 9:21, 9:22, 10:22, 14:30, 16:25, 19:25, 24:13, 24:22, 27:40, 27:42,
+27:49 Mark 3:4, 5:23, 5:28, 5:34, 6:56, 8:35, 10:26, 10:52, 13:13, 13:20, 15:30, 15:31, 16:16
+Luke 6:9, 7:50, 8:12, 8:36, 8:48, 8:50, 9:24, 13:23, 17:19, 18:26, 18:42, 19:10, 23:35, 23:37,
+23:39 John 5:34, 10:9, 11:12, 12:27 Acts 2:21, 2:40, 2:47, 4:9, 4:12, 11:14, 14:9, 15:1, 15:11,
+16:31, 27:20, 27:31 Romans 5:9, 5:10, 8:24, 9:27, 10:9, 10:13, 11:14, 11:26 1 Corinthians
+1:18, 1:21, 3:15, 5:5, 7:16, 9:22, 10:33, 15:2 2 Corinthians 2:15 Ephesians 2:5, 2:8 1
+Thessalonians 2:16 2 Thessalonians 2:10 1 Timothy 1:15, 2:4, 2:15, 4:16 2 Timothy 1:9,
+4:18 Titus 3:5 Hebrews 5:7, 7:25 James 1:21, 2:14, 4:12, 5:15, 5:20 1 Peter 3:21, 4:18
+Jude 1:5, 1:23
+
+scriptures (1124)
+Matthew 21:42, 22:29, 26:54, 26:56 Mark 12:10, 12:24, 14:49 Luke 4:21, 24:27, 24:32,
+24:45 John 2:22, 5:39, 7:38, 7:42, 10:35, 13:18, 17:12, 19:24, 19:28, 19:36, 19:37, 20:9 Acts
+1:16, 8:32, 8:35, 17:2, 17:11, 18:24, 18:28 Romans 1:2, 4:3, 9:17, 10:11, 11:2, 15:4, 16:26 1
+Corinthians 15:3, 15:4 Galatians 3:8, 3:22, 4:30 1 Timothy 5:18 2 Timothy 3:16 James
+2:8, 2:23, 4:5 2 Peter 1:20, 3:16
+
+shame (152, 2617)
+Luke 13:17, 14:9 Romans 5:5, 9:33, 10:11 1 Corinthians 11:4, 11:5, 11:22 2 Corinthians
+4:2, 7:14, 9:4 Philippians 3:19 Hebrews 12:2 1 Peter 2:6, 3:16 Jude 1:13 Revelation 3:18
+
+sheep (4263)
+Matthew 7:15, 9:36, 10:6, 10:16, 12:11, 12:12, 15:24, 18:12, 25:32, 25:33, 26:31 Mark
+6:34, 14:27 Luke 15:4, 15:6 John 2:14, 2:15, 10:2, 10:3, 10:4, 10:7, 10:8, 10:11, 10:12,
+10:13, 10:15, 10:16, 10:26, 10:27, 21:16, 21:17 Acts 8:32 Romans 8:36 Hebrews 13:20 1
+Peter 2:25 Revelation 18:13
+
+shepherd (4166, 4165)
+Matthew 2:6, 9:36, 25:32, 26:31 Mark 6:34, 14:27 Luke 2:8, 2:15, 2:18, 2:20, 17:7 John
+10:2, 10:11, 10:12, 10:14, 10:16, 21:16 Acts 20:28 1 Corinthians 9:7 Ephesians 4:11
+Hebrews 13:20 1 Peter 2:25, 5:2 Revelation 2:27, 7:17, 12:5, 19:15
+
+sin (266)
+Matthew 1:21, 3:6, 9:2, 9:5, 9:6, 12:31, 26:28 Mark 1:4, 1:5, 2:5, 2:7, 2:9, 2:10 Luke 1:77,
+3:3, 5:20, 5:21, 5:23, 5:24, 7:47, 7:48, 7:49, 11:4, 24:47 John 1:29, 8:21, 8:24, 8:34, 8:46,
+9:34, 9:41, 15:22, 15:24, 16:8, 16:9, 19:11, 20:23 Acts 2:38, 3:19, 5:31, 7:60, 10:43, 13:38,
+22:16, 26:18 Romans 3:9, 3:20, 4:7, 4:8, 5:12, 5:13, 5:20, 5:21, 6:1, 6:2, 6:6, 6:7, 6:10, 6:11,
+6:12, 6:13, 6:14, 6:16, 6:17, 6:18, 6:20, 6:22, 6:23, 7:5, 7:7, 7:8, 7:9, 7:11, 7:13, 7:14, 7:17,
+7:20, 7:23, 7:25, 8:2, 8:3, 8:10, 11:27, 14:23 1 Corinthians 15:3, 15:17, 15:56 2
+Corinthians 5:21, 11:7 Galatians 1:4, 2:17, 3:22 Ephesians 2:1 Colossians 1:14 1
+Thessalonians 2:16 1 Timothy 5:22, 5:24 2 Timothy 3:6 Hebrews 1:3, 2:17, 3:13, 4:15,
+5:1, 5:3, 7:27, 8:12, 9:26, 9:28, 10:2, 10:3, 10:4, 10:11, 10:12, 10:17, 10:18, 10:26, 11:25,
+12:1, 12:4, 13:11 James 1:15, 2:9, 4:17, 5:16, 5:20 1 Peter 2:22, 2:24, 3:18, 4:1, 4:8 2 Peter
+1:9, 2:14 1 John 1:7, 1:8, 1:9, 2:2, 2:12, 3:4, 3:5, 3:8, 4:10, 5:16, 5:17 Revelation 1:5, 18:4,
+18:5
+
+Son of God ()
+Matthew 4:3, 4:6, 8:29, 14:33, 26:63, 27:40, 27:43, 27:54 Mark 1:1, 3:11, 15:39 Luke 1:35,
+4:3, 4:9, 4:41, 8:28, 22:70 John 1:34, 1:49, 3:18, 5:25, 9:35, 10:36, 11:4, 11:27, 19:7, 20:31
+Acts 8:37, 9:20 Romans 1:4 2 Corinthians 1:19 Galatians 2:20 Ephesians 4:13 Hebrews
+4:14, 6:6, 7:3, 10:29 1 John 3:8, 4:15, 5:5, 5:10, 5:12, 5:13 Revelation 2:18
+
+Son of Man ()
+Matthew 8:20, 9:6, 10:23, 11:19, 12:8, 12:32, 12:40, 13:37, 13:41, 16:13, 16:27, 16:28,
+17:9, 17:12, 17:22, 18:11, 19:28, 20:18, 20:28, 24:27, 24:30, 24:37, 24:39, 24:44, 25:13,
+25:31, 26:2, 26:24, 26:45, 26:64 Mark 2:10, 2:28, 8:31, 8:38, 9:9, 9:12, 9:31, 10:33, 10:45,
+13:26, 13:34, 14:21, 14:41, 14:62 Luke 5:24, 6:5, 6:22, 7:34, 9:22, 9:26, 9:44, 9:56, 9:58,
+11:30, 12:8, 12:10, 12:40, 17:22, 17:24, 17:26, 17:30, 18:8, 18:31, 19:10, 21:27, 21:36,
+22:22, 22:48, 22:69, 24:7 John 1:51, 3:13, 3:14, 5:27, 6:27, 6:53, 6:62, 8:28, 12:23, 12:34,
+13:31 Acts 7:56 Hebrews 2:6 Revelation 1:13, 14:14
+
+soul (5590)
+Matthew 2:20, 6:25, 10:28, 10:39, 11:29, 12:18, 16:25, 16:26, 20:28, 22:37, 26:38 Mark
+3:4, 8:35, 8:36, 8:37, 10:45, 12:30, 14:34 Luke 1:46, 2:35, 6:9, 9:24, 10:27, 12:19, 12:20,
+12:22, 12:23, 14:26, 17:33, 21:19 John 10:11, 10:15, 10:17, 12:25, 12:27, 13:37, 13:38,
+
+15:13 Acts 2:27, 2:41, 2:43, 3:23, 4:32, 7:14, 14:2, 14:22, 15:24, 15:26, 20:10, 20:24, 27:10,
+27:22, 27:37 Romans 2:9, 11:3, 13:1, 16:4 1 Corinthians 15:45 2 Corinthians 1:23, 12:15
+Ephesians 6:6 Philippians 1:27, 2:30 Colossians 3:23 1 Thessalonians 2:8, 5:23
+Hebrews 4:12, 6:19, 10:38, 10:39, 12:3, 13:17 James 1:21 1 Peter 1:9, 1:22, 2:11, 2:25,
+3:20, 4:19 2 Peter 2:8, 2:14 1 John 3:16 3 John 1:2 Revelation 6:9, 12:11, 18:13, 20:4
+
+spirit (4151)
+Matthew 1:18, 1:20, 3:11, 3:16, 4:1, 5:3, 8:16, 10:1, 10:20, 12:18, 12:28, 12:31, 12:32,
+12:43, 12:45, 22:43, 26:41, 27:50, 28:19 Mark 1:8, 1:10, 1:12, 1:23, 1:26, 1:27, 2:8, 3:11,
+3:29, 3:30, 5:2, 5:8, 5:13, 6:7, 7:25, 8:12, 9:17, 9:20, 9:25, 12:36, 13:11, 14:38 Luke 1:15,
+1:17, 1:35, 1:41, 1:47, 1:67, 1:80, 2:25, 2:26, 2:27, 3:16, 3:22, 4:1, 4:14, 4:18, 4:33, 4:36,
+6:18, 7:21, 8:2, 8:29, 8:55, 9:39, 9:42, 10:20, 10:21, 11:13, 11:24, 11:26, 12:10, 12:12, 13:11,
+23:46, 24:37, 24:39 John 1:32, 1:33, 3:5, 3:6, 3:8, 3:34, 4:23, 4:24, 6:63, 7:39, 11:33, 13:21,
+14:17, 14:26, 15:26, 16:13, 19:30, 20:22 Acts 1:2, 1:5, 1:8, 1:16, 2:4, 2:17, 2:18, 2:33, 2:38,
+4:8, 4:25, 4:31, 5:3, 5:9, 5:16, 5:32, 6:3, 6:5, 6:10, 7:51, 7:55, 7:59, 8:7, 8:15, 8:17, 8:18, 8:19,
+8:29, 8:39, 9:17, 9:31, 10:19, 10:38, 10:44, 10:45, 10:47, 11:12, 11:15, 11:16, 11:24, 11:28,
+13:2, 13:4, 13:9, 13:52, 15:8, 15:28, 16:6, 16:7, 16:16, 16:18, 17:16, 18:25, 19:2, 19:6, 19:12,
+19:13, 19:15, 19:16, 19:21, 20:22, 20:23, 20:28, 21:4, 21:11, 23:8, 23:9, 28:25 Romans 1:4,
+1:9, 2:29, 5:5, 7:6, 8:2, 8:4, 8:5, 8:6, 8:9, 8:10, 8:11, 8:13, 8:14, 8:15, 8:16, 8:23, 8:26, 8:27,
+9:1, 11:8, 12:11, 14:17, 15:13, 15:16, 15:19, 15:30 1 Corinthians 2:4, 2:10, 2:11, 2:12, 2:13,
+2:14, 3:16, 4:21, 5:3, 5:5, 6:11, 6:17, 6:19, 7:34, 7:40, 12:3, 12:4, 12:7, 12:8, 12:9, 12:10,
+12:11, 12:13, 14:2, 14:12, 14:14, 14:15, 14:16, 14:32, 15:45, 16:18 2 Corinthians 1:22,
+2:13, 3:3, 3:6, 3:8, 3:17, 3:18, 4:13, 5:5, 6:6, 7:1, 7:13, 11:4, 12:18, 13:13 Galatians 3:2, 3:3,
+3:5, 3:14, 4:6, 4:29, 5:5, 5:16, 5:17, 5:18, 5:22, 5:25, 6:1, 6:8, 6:18 Ephesians 1:13, 1:17, 2:2,
+2:18, 2:22, 3:5, 3:16, 4:3, 4:4, 4:23, 4:30, 5:18, 6:17, 6:18 Philippians 1:19, 1:27, 2:1, 3:3,
+4:23 Colossians 1:8, 2:5 1 Thessalonians 1:5, 1:6, 4:8, 5:19, 5:23 2 Thessalonians 2:2,
+2:8, 2:13 1 Timothy 3:16, 4:1 2 Timothy 1:7, 1:14, 4:22 Titus 3:5 Philemon 1:25
+Hebrews 1:7, 1:14, 2:4, 3:7, 4:12, 6:4, 9:8, 9:14, 10:15, 10:29, 12:9, 12:23 James 2:26, 4:5 1
+Peter 1:2, 1:11, 1:12, 3:4, 3:18, 3:19, 4:6, 4:14 2 Peter 1:21 1 John 3:24, 4:1, 4:2, 4:3, 4:6,
+4:13, 5:6, 5:8 Jude 1:19, 1:20 Revelation 1:4, 1:10, 2:7, 2:11, 2:17, 2:29, 3:1, 3:6, 3:13, 3:22,
+4:2, 4:5, 5:6, 11:11, 13:15, 14:13, 16:13, 16:14, 17:3, 18:2, 19:10, 21:10, 22:6, 22:17
+
+tempt (3985)
+Matthew 4:1, 4:3, 16:1, 19:3, 22:18, 22:35 Mark 1:13, 8:11, 10:2, 12:15 Luke 4:2, 11:16
+John 8:6 Acts 5:9, 9:26, 15:10, 16:7, 24:6 1 Corinthians 7:5, 10:9, 10:13 2 Corinthians
+13:5 Galatians 6:1 1 Thessalonians 3:5 Hebrews 2:18, 3:9, 4:15, 11:17 James 1:13, 1:14
+Revelation 2:2, 2:10, 3:10
+
+throne (2362)
+Matthew 5:34, 19:28, 23:22, 25:31 Luke 1:32, 1:52, 22:30 Acts 2:30, 7:49 Colossians 1:16
+Hebrews 1:8, 4:16, 8:1, 12:2 Revelation 1:4, 2:13, 3:21, 4:2, 4:3, 4:4, 4:5, 4:6, 4:9, 4:10, 5:1,
+5:6, 5:7, 5:11, 5:13, 6:16, 7:9, 7:10, 7:11, 7:15, 7:17, 8:3, 11:16, 12:5, 13:2, 14:3, 16:10,
+16:17, 19:4, 19:5, 20:4, 20:11, 20:12, 21:3, 21:5, 22:1, 22:3
+
+tree (when used to mean "cross") ()
+Acts 5:30, 10:39, 13:29 Galatians 3:13 1 Peter 2:24
+
+true, truth (227, 228, 225)
+Matthew 22:16 Mark 5:33, 12:14 Luke 4:25, 16:11, 20:21 John 1:9, 1:14, 1:17, 3:21, 3:33,
+4:18, 4:23, 4:24, 4:37, 5:31, 5:32, 5:33, 6:32, 6:55, 7:18, 7:28, 8:13, 8:14, 8:16, 8:17, 8:26,
+8:32, 8:40, 8:44, 8:45, 8:46, 10:41, 14:6, 14:17, 15:1, 15:26, 16:7, 16:13, 17:3, 17:17, 17:19,
+18:37, 18:38, 19:35, 21:24 Acts 12:9, 26:25 Romans 1:18, 1:25, 2:2, 2:8, 2:20, 3:4, 3:7, 9:1,
+15:8 1 Corinthians 5:8, 13:6 2 Corinthians 4:2, 6:7, 6:8, 7:14, 11:10, 12:6, 13:8 Galatians
+2:5, 2:14, 5:7 Ephesians 1:13, 4:21, 4:24, 4:25, 5:9, 6:14 Philippians 1:18, 4:8 Colossians
+1:5, 1:6 1 Thessalonians 1:9 2 Thessalonians 2:10, 2:12, 2:13 1 Timothy 2:4, 2:7, 3:15,
+4:3, 6:5 2 Timothy 2:15, 2:18, 2:25, 3:7, 3:8, 4:4 Titus 1:1, 1:13, 1:14 Hebrews 8:2, 9:24,
+10:22, 10:26 James 1:18, 3:14, 5:19 1 Peter 1:22, 5:12 2 Peter 1:12, 2:2, 2:22 1 John 1:6,
+1:8, 2:4, 2:8, 2:21, 2:27, 3:18, 3:19, 4:6, 5:6, 5:20 2 John 1:1, 1:2, 1:3, 1:4 3 John 1:1, 1:3, 1:4,
+1:8, 1:12 Revelation 3:14, 6:10, 15:3, 16:7, 19:2, 19:9, 19:11, 21:5, 22:6
+
+unbelief, unbelieving (570, 571)
+Matthew 13:58, 17:17 Mark 6:6, 9:19, 9:24, 16:14 Luke 9:41, 12:46 John 20:27 Acts 26:8
+Romans 3:3, 4:20, 11:20, 11:23 1 Corinthians 6:6, 7:12, 7:13, 7:14, 7:15, 10:27, 14:22,
+14:23, 14:24 2 Corinthians 4:4, 6:14, 6:15 1 Timothy 1:13, 5:8 Titus 1:15 Hebrews 3:12,
+3:19 Revelation 21:8
+
+will (of God) ()
+Matthew 6:10, 7:21, 12:50, 18:14, 26:42 Mark 3:35 Luke 11:2, 22:42 John 4:34, 5:30, 6:38,
+6:39, 6:40, 7:17, 9:31 Acts 13:22, 21:14, 22:14 Romans 1:10, 2:18, 12:2, 15:32 1
+Corinthians 1:1 2 Corinthians 1:1, 8:5 Galatians 1:4 Ephesians 1:1, 1:5, 1:9, 1:11, 5:17,
+6:6 Colossians 1:1, 1:9, 4:12 1 Thessalonians 4:3, 5:18 2 Timothy 1:1 Hebrews 10:7,
+10:9, 10:10, 10:36, 13:21 1 Peter 2:15, 3:17, 4:2, 4:19 1 John 2:17, 5:14 Revelation 4:11
+
+wisdom (4678)
+Matthew 11:19, 12:42, 13:54 Mark 6:2 Luke 2:40, 2:52, 7:35, 11:31, 11:49, 21:15 Acts 6:3,
+6:10, 7:10, 7:22 Romans 11:33 1 Corinthians 1:19, 1:20, 1:21, 1:22, 1:24, 1:30, 2:1, 2:4,
+2:5, 2:6, 2:7, 2:13, 3:19, 12:8 2 Corinthians 1:12 Ephesians 1:8, 1:17, 3:10 Colossians 1:9,
+1:28, 2:3, 2:23, 3:16, 4:5 James 1:5, 3:13, 3:15, 3:17 2 Peter 3:15 Revelation 5:12, 7:12,
+17:9
+
+wise (4680, 5429)
+Matthew 7:24, 10:16, 11:25, 23:34, 24:45, 25:2, 25:4, 25:8, 25:9 Luke 10:21, 12:42, 16:8
+Romans 1:14, 1:22, 11:25, 12:16, 16:19, 16:27 1 Corinthians 1:19, 1:20, 1:25, 1:26, 1:27,
+3:10, 3:18, 3:19, 3:20, 4:10, 6:5, 10:15 2 Corinthians 11:19 Ephesians 5:15 James 3:13
+
+word (3056)
+Matthew 5:32, 5:37, 7:24, 7:26, 7:28, 8:8, 8:16, 10:14, 12:32, 12:36, 12:37, 13:19, 13:20,
+13:21, 13:22, 13:23, 15:6, 15:12, 15:23, 18:23, 19:1, 19:11, 21:24, 22:15, 22:46, 24:35,
+25:19, 26:1, 26:44, 28:15 Mark 1:45, 2:2, 4:14, 4:15, 4:16, 4:17, 4:18, 4:19, 4:20, 4:33, 5:36,
+7:13, 8:32, 8:38, 9:10, 10:22, 10:24, 11:29, 12:13, 13:31, 14:39, 16:20 Luke 1:2, 1:4, 1:20,
+1:29, 3:4, 4:22, 5:1, 5:15, 6:47, 7:7, 7:17, 8:11, 8:12, 8:13, 8:15, 8:21, 9:26, 9:28, 9:44, 10:39,
+11:28, 12:10, 16:2, 20:3, 20:20, 21:33, 23:9, 24:17, 24:19, 24:44 John 1:1, 1:14, 2:22, 4:37,
+4:39, 4:41, 4:50, 5:24, 5:38, 7:36, 7:40, 8:31, 8:37, 8:43, 8:51, 8:52, 8:55, 10:19, 10:35,
+12:38, 12:48, 14:23, 14:24, 15:3, 15:20, 15:25, 17:6, 17:14, 17:17, 17:20, 18:9, 18:32, 19:8,
+19:13, 21:23 Acts 1:1, 2:22, 2:40, 2:41, 4:4, 4:29, 4:31, 5:5, 5:24, 6:2, 6:4, 6:5, 6:7, 7:22, 7:29,
+8:4, 8:14, 8:21, 8:25, 10:36, 10:44, 11:1, 11:19, 11:22, 12:24, 13:5, 13:7, 13:15, 13:26, 13:44,
+13:46, 13:48, 13:49, 14:3, 14:12, 14:25, 15:6, 15:7, 15:15, 15:24, 15:27, 15:32, 15:35, 15:36,
+16:6, 16:32, 16:36, 17:11, 17:13, 18:5, 18:11, 18:15, 19:10, 19:20, 19:38, 19:40, 20:2, 20:7,
+20:24, 20:32, 20:35, 20:38, 22:22 Romans 3:4, 9:6, 9:9, 9:28, 13:9, 14:12, 15:18 1
+Corinthians 1:5, 1:18, 2:1, 2:4, 2:13, 4:19, 4:20, 12:8, 14:19, 14:36, 15:2, 15:54 2
+Corinthians 1:18, 2:17, 4:2, 5:19, 6:7, 8:7, 10:10, 10:11, 11:6 Galatians 5:14, 6:6
+Ephesians 1:13, 4:29, 5:6, 6:19 Philippians 1:14, 2:16, 4:15, 4:17 Colossians 1:5, 1:25,
+2:23, 3:16, 3:17, 4:3, 4:6 1 Thessalonians 1:5, 1:6, 1:8, 2:5, 2:13, 4:15, 4:18 2
+Thessalonians 2:2, 2:15, 2:17, 3:1, 3:14 1 Timothy 1:15, 3:1, 4:5, 4:6, 4:9, 4:12, 5:17, 6:3 2
+Timothy 1:13, 2:9, 2:11, 2:15, 2:17, 4:2, 4:15 Titus 1:3, 1:9, 2:5, 2:8, 3:8 Hebrews 2:2, 4:2,
+4:12, 5:13, 6:1, 7:28, 12:19, 13:7, 13:17, 13:22 James 1:18, 1:21, 1:22, 1:23, 3:2 1 Peter
+1:23, 2:8, 3:1, 4:5 2 Peter 1:19, 2:3, 3:5, 3:7 1 John 1:1, 1:10, 2:5, 2:7, 2:14, 3:18 3 John
+1:10 Revelation 1:2, 1:3, 1:9, 3:8, 6:9, 12:11, 17:17, 19:9, 19:13, 20:4, 21:5, 22:6, 22:7, 22:9,
+22:10, 22:18, 22:19
+
+worship (4352, 4576, 4573)
+Matthew 2:2, 2:8, 2:11, 4:9, 4:10, 8:2, 9:18, 14:33, 15:9, 15:25, 18:26, 20:20, 28:9, 28:17
+Mark 5:6, 7:7, 15:19 Luke 4:8, 24:52 John 4:20, 4:21, 4:22, 4:23, 4:24, 9:38 Acts 7:43, 8:27,
+10:25, 13:43, 13:50, 16:14, 17:4, 17:17, 18:7, 18:13, 19:27, 24:11 Romans 1:25 1
+Corinthians 14:25 Hebrews 1:6, 11:21 Revelation 3:9, 4:10, 5:14, 7:11, 9:20, 11:1, 11:16,
+13:4, 13:8, 14:7, 14:9, 14:11, 15:4, 16:2, 19:4, 19:10, 19:20, 20:4, 22:8, 22:9
+
+worthy (514, 515, 516, 2661)
+Matthew 3:8, 10:10, 10:11, 10:13, 10:37, 10:38, 22:8 Luke 3:8, 7:4, 7:7, 10:7, 12:48, 15:19,
+15:21, 20:35, 23:15 John 1:27 Acts 5:41, 13:25, 15:38, 23:29, 25:11, 25:25, 26:20, 26:31,
+28:22 Romans 1:32, 8:18, 16:2 1 Corinthians 16:4 Ephesians 4:1 Philippians 1:27
+Colossians 1:10 1 Thessalonians 2:12 2 Thessalonians 1:3, 1:5, 1:11 1 Timothy 1:15,
+4:9, 5:17, 5:18, 6:1 Hebrews 3:3, 10:29, 11:38 3 John 1:6 Revelation 3:4, 4:11, 5:2, 5:4,
+5:9, 5:12
diff --git a/backend/passages/domain/document_generator.py b/backend/passages/domain/document_generator.py
index 966d24b40..ae4c4a5b5 100644
--- a/backend/passages/domain/document_generator.py
+++ b/backend/passages/domain/document_generator.py
@@ -11,14 +11,14 @@
from doc.domain.parsing import split_chapter_into_verses, usfm_book_content
from doc.domain.resource_lookup import (
RESOURCE_TYPE_CODES_AND_NAMES,
+ book_codes_for_lang_from_usfm_only,
+ maybe_correct_book_name,
prepare_resource_filepath,
provision_asset_files,
resource_lookup_dto,
resource_types,
)
-from passages.utils.docx_utils import add_footer, add_header
-from passages.domain.model import PassageDto, PassageReferenceDto
-from passages.domain.parser import verse_text_html
+from doc.reviewers_guide.model import BibleReference
from doc.utils.file_utils import docx_filepath, file_needs_update
from docx import Document # type: ignore
from docx.oxml import OxmlElement # type: ignore
@@ -26,6 +26,10 @@
from docx.shared import Inches # type: ignore
from docx.table import _Cell # type: ignore
from htmldocx import HtmlToDocx # type: ignore
+from passages.domain.model import Passage, BibleReference as PassageReference
+from passages.domain.parser import verse_text_html
+from passages.domain.stet_verse_list_parser import BOOK_INDEX, parse_bible_blocks
+from passages.utils.docx_utils import add_footer, add_header
from pydantic import Json
logger = settings.logger(__name__)
@@ -34,7 +38,7 @@
def generate_docx_document(
lang_code: str,
lang_name: str,
- passage_reference_dtos: list[PassageReferenceDto],
+ passage_reference_dtos: list[PassageReference],
document_request_key_: str,
docx_filepath_: str,
working_dir: str = settings.WORKING_DIR,
@@ -118,30 +122,30 @@ def generate_docx_document(
verse_text_html_ = ""
non_book_name_portion_of_reference = ""
if (
- passage_ref_dto.end_chapter_num
- and passage_ref_dto.end_chapter_num > 0
- and passage_ref_dto.end_chapter_verse_reference
+ passage_ref_dto.end_chapter
+ and passage_ref_dto.end_chapter > 0
+ and passage_ref_dto.end_chapter_verse_ref
):
- non_book_name_portion_of_reference = f"{passage_ref_dto.start_chapter_num}:{passage_ref_dto.start_chapter_verse_reference}-{passage_ref_dto.end_chapter_num}:{passage_ref_dto.end_chapter_verse_reference}"
+ non_book_name_portion_of_reference = f"{passage_ref_dto.start_chapter}:{passage_ref_dto.start_chapter_verse_ref}-{passage_ref_dto.end_chapter}:{passage_ref_dto.end_chapter_verse_ref}"
else:
- non_book_name_portion_of_reference = f"{passage_ref_dto.start_chapter_num}:{passage_ref_dto.start_chapter_verse_reference}"
- nationalized_reference = (
+ non_book_name_portion_of_reference = f"{passage_ref_dto.start_chapter}:{passage_ref_dto.start_chapter_verse_ref}"
+ localized_reference = (
f"{selected_usfm_book.national_book_name} {non_book_name_portion_of_reference}"
if selected_usfm_book and non_book_name_portion_of_reference
- else passage_ref_dto.start_chapter_verse_reference
+ else passage_ref_dto.start_chapter_verse_ref
)
- passage_dto = PassageDto(
- passage_reference=nationalized_reference,
+ passage = Passage(
+ bible_reference=localized_reference,
passage_text=verse_text_html_,
)
- passages.append(passage_dto)
+ passages.append(passage)
current_task.update_state(state="Converting to Docx")
generate_docx(passages, docx_filepath_, lang_code, lang_name)
return docx_filepath_
def generate_docx(
- passage_dtos: list[PassageDto],
+ passage_dtos: list[Passage],
docx_filepath: str,
lang_code: str,
lang_name: str,
@@ -169,7 +173,7 @@ def generate_docx(
tcPr.append(tcW)
# Fill left cell
cell_left = table.cell(0, 0)
- html_to_docx.add_html_to_document(passage_dto.passage_reference, cell_left)
+ html_to_docx.add_html_to_document(passage_dto.bible_reference, cell_left)
html_to_docx.add_html_to_document(passage_dto.passage_text, cell_left)
# Fill right cell (empty, just add vertical line)
cell_right = table.cell(0, 1)
@@ -203,7 +207,7 @@ def add_vertical_line(cell: _Cell) -> None:
def document_request_key(
lang_code: str,
- passage_reference_dtos: list[PassageReferenceDto],
+ passage_reference_dtos: list[PassageReference],
max_filename_len: int = 240,
underscore: str = "_",
hyphen: str = "-",
@@ -227,7 +231,7 @@ def document_request_key(
translation_table = str.maketrans(":;,-", "____")
passages_key = underscore.join(
[
- f"{passage_reference.book_code}_{passage_reference.start_chapter_num}_{passage_reference.start_chapter_verse_reference.translate(translation_table)}"
+ f"{passage_reference.book_code}_{passage_reference.start_chapter}_{passage_reference.start_chapter_verse_ref.translate(translation_table)}"
for passage_reference in passage_reference_dtos
]
)
@@ -254,7 +258,7 @@ def generate_passages_docx_document(
) -> Json[str]:
passage_reference_dtos_list = json.loads(passage_reference_dtos_json)
passage_reference_dtos = [
- PassageReferenceDto(**d) for d in passage_reference_dtos_list
+ PassageReference(**d) for d in passage_reference_dtos_list
]
# logger.debug(
# "passed args: lang_code: %s, passage_references: %s, email_adress: %s",
@@ -291,3 +295,73 @@ def generate_passages_docx_document(
else:
logger.debug("Cache hit for %s", docx_filepath_)
return document_request_key_
+
+
+@worker.app.task
+def stet_exhaustive_verse_list(
+ lang_code: str = "en",
+ filepath: str = "backend/passages/data/Spiritual_Terms_Evaluation_Exhaustive_Verse_List.txt",
+) -> Sequence[BibleReference]:
+ """
+ >>> from passages.domain.document_generator import stet_exhaustive_verse_list
+ >>> result = stet_exhaustive_verse_list()
+ >>> result[0]
+ Matthew 10:1
+ """
+ bible_references = []
+ with open(filepath, "r") as fi:
+ text = fi.read()
+ parsed = parse_bible_blocks(text)
+ for sublist in parsed.values():
+ for value in sublist:
+ if len(value.split()) >= 2:
+ bible_references.append(parse_bible_reference(value))
+ # Remove duplicates and sort based on the book index, chapter, and verse
+ unique_bible_references = sorted(
+ set(bible_references),
+ key=lambda ref: (
+ BOOK_INDEX[ref.book_code],
+ ref.start_chapter,
+ ref.start_chapter_verse_ref,
+ ),
+ )
+ # Localize the book names
+ book_name_map = {
+ book_code_and_name[0]: book_code_and_name[1]
+ for book_code_and_name in book_codes_for_lang_from_usfm_only(lang_code)
+ }
+ for bible_reference in unique_bible_references:
+ maybe_localized_book_name = book_name_map.get(
+ bible_reference.book_code, bible_reference.book_name
+ )
+ logger.debug("maybe_localized_book_name: %s", maybe_localized_book_name)
+ localized_book_name = maybe_correct_book_name(
+ lang_code, maybe_localized_book_name
+ )
+ bible_reference.book_name = localized_book_name
+ return unique_bible_references
+
+
+def get_book_code(book_name: str) -> str:
+ return next(code for code, name in BOOK_NAMES.items() if name == book_name)
+
+
+def parse_bible_reference(book_and_reference_raw: str) -> BibleReference:
+ book_name_and_reference = book_and_reference_raw.split()
+ book_name = (
+ " ".join(book_name_and_reference[:-1])
+ if len(book_name_and_reference) > 2
+ else book_name_and_reference[0]
+ )
+ chapter_reference = book_name_and_reference[-1]
+ chapter = int(chapter_reference.split(":")[0])
+ chapter_verse_ref = chapter_reference.split(":")[1]
+ bible_reference = BibleReference(
+ book_code=get_book_code(book_name),
+ book_name=book_name,
+ start_chapter=chapter,
+ start_chapter_verse_ref=chapter_verse_ref,
+ end_chapter=None,
+ end_chapter_verse_ref=None,
+ )
+ return bible_reference
diff --git a/backend/passages/domain/model.py b/backend/passages/domain/model.py
index d344ce6c6..d7af948e5 100644
--- a/backend/passages/domain/model.py
+++ b/backend/passages/domain/model.py
@@ -4,25 +4,25 @@
@final
-class PassageReferenceDto(BaseModel):
+class BibleReference(BaseModel):
lang_code: str
book_code: str
book_name: str
- start_chapter_num: ChapterNum
- start_chapter_verse_reference: str
- end_chapter_num: Optional[ChapterNum]
- end_chapter_verse_reference: Optional[str]
+ start_chapter: ChapterNum
+ start_chapter_verse_ref: str
+ end_chapter: Optional[ChapterNum]
+ end_chapter_verse_ref: Optional[str]
@final
-class PassageDto(NamedTuple):
+class Passage(NamedTuple):
passage_text: str # HTML of passage
- passage_reference: str
+ bible_reference: str
@final
class PassagesDocumentRequest(BaseModel):
lang_code: str
lang_name: str
- passage_references: list[PassageReferenceDto]
+ bible_references: list[BibleReference]
email_address: Optional[EmailStr]
diff --git a/backend/passages/domain/parser.py b/backend/passages/domain/parser.py
index 1d159db5c..aeb327211 100644
--- a/backend/passages/domain/parser.py
+++ b/backend/passages/domain/parser.py
@@ -1,5 +1,5 @@
from typing import Mapping
-from passages.domain.model import PassageReferenceDto
+from passages.domain.model import BibleReference
from doc.domain.parsing import lookup_verse_text
from doc.domain.model import USFMBook
from doc.domain.bible_books import BOOK_CHAPTER_VERSES
@@ -9,26 +9,26 @@
def verse_text_html(
- passage_ref_dto: PassageReferenceDto,
+ bible_reference: BibleReference,
usfm_book: USFMBook,
book_chapter_verses: Mapping[str, Mapping[str, str]] = BOOK_CHAPTER_VERSES,
) -> str:
verse_text = []
if (
- passage_ref_dto.end_chapter_num
- and passage_ref_dto.end_chapter_num > 0
- and passage_ref_dto.end_chapter_verse_reference
+ bible_reference.end_chapter
+ and bible_reference.end_chapter > 0
+ and bible_reference.end_chapter_verse_ref
): # chapter boundary traversal
- start_chapter_lower_verse = int(passage_ref_dto.start_chapter_verse_reference)
+ start_chapter_lower_verse = int(bible_reference.start_chapter_verse_ref)
start_chapter_upper_verse = int(
- book_chapter_verses[passage_ref_dto.book_code][
- str(passage_ref_dto.start_chapter_num)
+ book_chapter_verses[bible_reference.book_code][
+ str(bible_reference.start_chapter)
]
)
for idx in range(start_chapter_lower_verse, start_chapter_upper_verse + 1):
start_chapter_verse_text = lookup_verse_text(
usfm_book,
- passage_ref_dto.start_chapter_num,
+ bible_reference.start_chapter,
str(idx),
)
if start_chapter_verse_text:
@@ -36,11 +36,11 @@ def verse_text_html(
f'{str(idx)}{start_chapter_verse_text}'
)
end_chapter_lower_verse = 1
- end_chapter_upper_verse = int(passage_ref_dto.end_chapter_verse_reference)
+ end_chapter_upper_verse = int(bible_reference.end_chapter_verse_ref)
for idx in range(end_chapter_lower_verse, end_chapter_upper_verse + 1):
end_chapter_verse_text = lookup_verse_text(
usfm_book,
- passage_ref_dto.end_chapter_num,
+ bible_reference.end_chapter,
str(idx),
)
if end_chapter_verse_text:
@@ -48,10 +48,8 @@ def verse_text_html(
f'{str(idx)}{end_chapter_verse_text}'
)
else:
- if "," in passage_ref_dto.start_chapter_verse_reference:
- verse_range_components = (
- passage_ref_dto.start_chapter_verse_reference.split(",")
- )
+ if "," in bible_reference.start_chapter_verse_ref:
+ verse_range_components = bible_reference.start_chapter_verse_ref.split(",")
for verse_ in verse_range_components:
if "-" in verse_:
verse__range_components = verse_.split("-")
@@ -60,7 +58,7 @@ def verse_text_html(
for idx in range(lower_verse_, upper_verse_ + 1):
verse_text__ = lookup_verse_text(
usfm_book,
- passage_ref_dto.start_chapter_num,
+ bible_reference.start_chapter,
str(idx),
)
if verse_text__:
@@ -70,23 +68,21 @@ def verse_text_html(
else:
verse_text__ = lookup_verse_text(
usfm_book,
- passage_ref_dto.start_chapter_num,
+ bible_reference.start_chapter,
verse_,
)
if verse_text__:
verse_text.append(
f'{verse_}{verse_text__}'
)
- elif "-" in passage_ref_dto.start_chapter_verse_reference:
- verse_range_components = (
- passage_ref_dto.start_chapter_verse_reference.split("-")
- )
+ elif "-" in bible_reference.start_chapter_verse_ref:
+ verse_range_components = bible_reference.start_chapter_verse_ref.split("-")
lower_verse = int(verse_range_components[0])
upper_verse = int(verse_range_components[1])
for idx in range(lower_verse, upper_verse + 1):
verse_text_ = lookup_verse_text(
usfm_book,
- passage_ref_dto.start_chapter_num,
+ bible_reference.start_chapter,
str(idx),
)
if verse_text_:
@@ -96,11 +92,11 @@ def verse_text_html(
else:
verse_text___ = lookup_verse_text(
usfm_book,
- passage_ref_dto.start_chapter_num,
- passage_ref_dto.start_chapter_verse_reference.strip(),
+ bible_reference.start_chapter,
+ bible_reference.start_chapter_verse_ref.strip(),
)
if verse_text___:
verse_text.append(
- f'{passage_ref_dto.start_chapter_verse_reference.strip()}{verse_text___}'
+ f'{bible_reference.start_chapter_verse_ref.strip()}{verse_text___}'
)
return "".join(verse_text)
diff --git a/backend/passages/domain/stet_verse_list_parser.py b/backend/passages/domain/stet_verse_list_parser.py
new file mode 100644
index 000000000..22bfe84ab
--- /dev/null
+++ b/backend/passages/domain/stet_verse_list_parser.py
@@ -0,0 +1,53 @@
+import json
+import re
+from doc.domain.bible_books import BOOK_NAMES
+
+
+BOOK_INDEX = dict((id, pos) for pos, id in enumerate(BOOK_NAMES.keys()))
+BOOK_VERSE_PATTERN = re.compile(r"(?:([1-3]?\s?[A-Z][a-z]+)\s)?(\d+:\d+)")
+HEADER_PATTERN = re.compile(r"^(\w+)\s+\(([\d,; ]+)\)")
+
+
+def parse_bible_blocks(text: str) -> dict[str, list[str]]:
+ entries = {}
+ blocks = re.split(r"\n(?=\w+\s+\([\d,; ]+\))", text.strip(), flags=re.MULTILINE)
+ for block in blocks:
+ lines = block.strip().splitlines()
+ if not lines:
+ continue
+ header_match = HEADER_PATTERN.match(lines[0])
+ if not header_match:
+ continue
+ word = header_match.group(1)
+ strongs = header_match.group(2).replace(";", ",").replace(" ", "")
+ key = f"{word} ({strongs})"
+ # Flatten all lines into one paragraph and remove commas
+ reference_text = " ".join(lines[1:]).replace(",", "")
+ # Extract references
+ result = []
+ current_book = None
+ for match in BOOK_VERSE_PATTERN.finditer(reference_text):
+ book, ref = match.groups()
+ if book:
+ current_book = book
+ if current_book:
+ result.append(f"{current_book} {ref}")
+ else:
+ result.append(f"UNKNOWN {ref}")
+
+ # Sort canonically
+ def sort_key(ref: str) -> tuple[int, int, int]:
+ try:
+ book, chap_verse = ref.rsplit(" ", 1)
+ chapter, verse = map(int, chap_verse.split(":"))
+ book_index = BOOK_INDEX.get(book, 999)
+ return (book_index, chapter, verse)
+ except Exception:
+ return (999, 0, 0)
+
+ entries[key] = sorted(result, key=sort_key)
+ return entries
+
+
+def to_json(parsed_data: dict[str, list[str]]) -> str:
+ return json.dumps(parsed_data, indent=2)
diff --git a/backend/passages/entrypoints/routes.py b/backend/passages/entrypoints/routes.py
index d33ffe5a5..a06111eaa 100644
--- a/backend/passages/entrypoints/routes.py
+++ b/backend/passages/entrypoints/routes.py
@@ -1,15 +1,14 @@
import json
-from fastapi import APIRouter
+from typing import Sequence
import celery.states
from celery.result import AsyncResult
from doc.config import settings
-from passages.domain import document_generator
-from passages.domain import model
-
-from fastapi import HTTPException, status
-
+from doc.reviewers_guide.model import BibleReference
+from fastapi import APIRouter, HTTPException, status
from fastapi.responses import JSONResponse
+from passages.domain import document_generator, model
+from passages.domain.document_generator import stet_exhaustive_verse_list
router = APIRouter()
@@ -28,7 +27,7 @@ async def generate_passages_docx_document(
passages_document_request.lang_name,
# Serialize the list of objects to a JSON string
json.dumps(
- passages_document_request.passage_references,
+ passages_document_request.bible_references,
default=lambda obj: obj.model_dump(),
),
passages_document_request.email_address,
@@ -62,3 +61,8 @@ async def task_status(task_id: str) -> JSONResponse:
"state": res.state,
}
)
+
+
+@router.get("/passages/stet_verse_list/{lang_code}")
+async def stet_verse_list(lang_code: str) -> Sequence[BibleReference]:
+ return stet_exhaustive_verse_list(lang_code)
diff --git a/backend/requirements-prod.txt b/backend/requirements-prod.txt
index 3b9a5831c..4c4a80931 100644
--- a/backend/requirements-prod.txt
+++ b/backend/requirements-prod.txt
@@ -1,5 +1,5 @@
#
-# This file is autogenerated by pip-compile with Python 3.12
+# This file is autogenerated by pip-compile with Python 3.13
# by the following command:
#
# pip-compile ./backend/requirements-prod.in
diff --git a/backend/requirements.in b/backend/requirements.in
index 8f667bba9..43b0dcbf2 100644
--- a/backend/requirements.in
+++ b/backend/requirements.in
@@ -5,13 +5,13 @@
# cython # For pydantic: https://pydantic-docs.helpmanual.io/install/
# TODO do we still need aiofiles?
aiofiles
+cachetools
celery
celery-types
docxtpl
fastapi[all]
flower
htmldocx
-html2docx
gunicorn
jinja2
mistune
@@ -30,6 +30,7 @@ termcolor
uvicorn
weasyprint
+types-cachetools
types-PyYAML
types-beautifulsoup4
types-orjson
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 10ba4a199..310e3692d 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -23,6 +23,8 @@ billiard==4.2.1
# via celery
brotli==1.1.0
# via fonttools
+cachetools==6.1.0
+ # via -r ./backend/requirements.in
celery==5.4.0
# via
# -r ./backend/requirements.in
@@ -78,8 +80,6 @@ h11==0.14.0
# via
# httpcore
# uvicorn
-html2docx==1.6.0
- # via -r ./backend/requirements.in
htmldocx==0.0.6
# via -r ./backend/requirements.in
httpcore==1.0.7
@@ -156,7 +156,6 @@ python-docx==1.1.0
# via
# docxcompose
# docxtpl
- # html2docx
# htmldocx
python-dotenv==1.0.1
# via
@@ -198,7 +197,6 @@ termcolor==2.5.0
tinycss2==1.4.0
# via
# cssselect2
- # html2docx
# weasyprint
tinyhtml5==2.0.0
# via weasyprint
@@ -208,6 +206,8 @@ typer==0.13.1
# via fastapi-cli
types-beautifulsoup4==4.12.0.20241020
# via -r ./backend/requirements.in
+types-cachetools==6.1.0.20250717
+ # via -r ./backend/requirements.in
types-html5lib==1.1.11.20241018
# via types-beautifulsoup4
types-orjson==3.6.2
diff --git a/backend/stet/data/stet_fr.docx b/backend/stet/data/stet_fr.docx
new file mode 100644
index 000000000..985562b41
Binary files /dev/null and b/backend/stet/data/stet_fr.docx differ
diff --git a/backend/stet/data/stet_sw.docx b/backend/stet/data/stet_sw.docx
new file mode 100644
index 000000000..5ee9bc2b7
Binary files /dev/null and b/backend/stet/data/stet_sw.docx differ
diff --git a/backend/stet/data/stet_tpi.docx b/backend/stet/data/stet_tpi.docx
new file mode 100644
index 000000000..596aac99d
Binary files /dev/null and b/backend/stet/data/stet_tpi.docx differ
diff --git a/backend/stet/domain/strings.py b/backend/stet/domain/strings.py
index 69eaded6c..28ddbe956 100644
--- a/backend/stet/domain/strings.py
+++ b/backend/stet/domain/strings.py
@@ -1,23 +1,35 @@
TRANSLATED_HEADER_PHRASES_TABLE: dict[str, str] = {
"en": "Spiritual Terms Evaluation Tool (STET)",
"es-419": "Herramienta de Evaluación de Términos Espirituales (STET)",
+ "fr": "Évaluation des Termes Spirituels (STET)",
"pt-br": "Ferramenta de Avaliação de Termos Espirituais (STET)",
+ "sw": "Chombo cha Kutathmini Maneno ya Kiroho (STET)",
+ "tpi": "Tul bilong skelim ol spirit tok bilong buk trenslesen (STET)",
}
TRANSLATED_FOOTER_PHRASES_TABLE: dict[str, str] = {
"en": "Generated on",
"es-419": "Generado el",
+ "fr": "Généré le",
"pt-br": "Gerado em",
+ "sw": "Imetolewa tarehe",
+ "tpi": "Wok i bin kamap long",
}
LOCALIZED_DATE_FORMAT_STRINGS: dict[str, str] = {
"en": "%m/%d/%Y %H:%M:%S",
"es-419": "%d/%m/%Y %H:%M:%S",
+ "fr": "%d/%m/%Y %H:%M:%S",
"pt-br": "%d/%m/%Y %H:%M:%S",
+ "sw": "%d/%m/%Y %H:%M:%S",
+ "tpi": "%d/%m/%Y %H:%M:%S",
}
TRANSLATED_TABLE_COLUMN_HEADERS = {
"en": ("Source Reference", "Target Reference", "Status", "OK"),
"es-419": ("Fuente", "Idioma Materna", "Estado", "OK"),
+ "fr": ("Référence de source", "Référence Cible", "Statut", "OK"),
"pt-br": ("Referência de Origem", "Referência de Destino", "Status", "OK"),
+ "sw": ("Marejeo Chanzo", "Marejeo Lengwa", "Hali", "OK"),
+ "tpi": ("Narapela baibel ves", "Tokples ves", "Sek", "OK"),
}
diff --git a/backend/stet/entrypoints/routes.py b/backend/stet/entrypoints/routes.py
index 9b686d333..32f02f14a 100644
--- a/backend/stet/entrypoints/routes.py
+++ b/backend/stet/entrypoints/routes.py
@@ -1,16 +1,15 @@
from os import scandir
from typing import Sequence
-from fastapi import APIRouter
import celery.states
from celery.result import AsyncResult
from doc.config import settings
-from stet.domain import document_generator, model
from doc.domain import resource_lookup
-
-from fastapi import HTTPException, status
-
+from docx import Document # type: ignore
+from fastapi import APIRouter, Request, HTTPException, status
from fastapi.responses import JSONResponse
+from stet.domain import document_generator, model
+
router = APIRouter()
@@ -19,11 +18,14 @@
@router.get("/stet/source_languages")
async def source_lang_codes_and_names(
+ request: Request,
stet_dir: str = settings.STET_DIR,
) -> Sequence[tuple[str, str, bool]]:
"""
Return list of all available language code, name tuples for which Translation Services has provided a source document.
"""
+ is_production = request.headers.get("x-is-production") == "true"
+ logger.debug("is_production: %s", is_production)
# Scan what source docs are available and make sure to filter
# source language candidates to only those languages.
ietf_codes = [
@@ -34,11 +36,23 @@ async def source_lang_codes_and_names(
and entry.name.endswith(".docx")
]
# logger.debug("source ietf_codes: %s", ietf_codes)
- languages = [
- lang_code_and_name
- for lang_code_and_name in resource_lookup.lang_codes_and_names_having_usfm()
- if lang_code_and_name[0] in ietf_codes
- ]
+ ietf_codes_for_docs_with_fourth_column = []
+ for ietf_code in ietf_codes:
+ doc = Document(f"{stet_dir}/stet_{ietf_code}.docx")
+ for table in doc.tables:
+ for row in table.rows:
+ # If 4th column exists, get bolded words from it
+ if len(row.cells) > 3 and row.cells[3].text:
+ if ietf_code not in ietf_codes_for_docs_with_fourth_column:
+ ietf_codes_for_docs_with_fourth_column.append(ietf_code)
+ languages = []
+ for lang_code_and_name in resource_lookup.lang_codes_and_names_having_usfm():
+ if is_production:
+ if lang_code_and_name[0] in ietf_codes_for_docs_with_fourth_column:
+ languages.append(lang_code_and_name)
+ else:
+ if lang_code_and_name[0] in ietf_codes:
+ languages.append(lang_code_and_name)
# logger.debug("source languages: %s", languages)
return languages
diff --git a/backend/templates/html/bible_reference.html b/backend/templates/html/bible_reference.html
index f0516aabb..19d12ac19 100644
--- a/backend/templates/html/bible_reference.html
+++ b/backend/templates/html/bible_reference.html
@@ -1,4 +1,4 @@
-{% if start_chapter and start_chapter > 0 and start_chapter_verse_ref %}
+{% if start_chapter and start_chapter > 0 and start_chapter_verse_ref and end_chapter and end_chapter > 0 and end_chapter_verse_ref %}
{{book_name}} {{start_chapter}}:{{start_chapter_verse_ref}}-{{end_chapter}}:{{end_chapter_verse_ref}}
{% else %}
{{book_name}} {{start_chapter}}:{{start_chapter_verse_ref}}
diff --git a/backend/templates/html/header_enclosing.html b/backend/templates/html/header_enclosing.html
index 105377a6f..c427a9e10 100644
--- a/backend/templates/html/header_enclosing.html
+++ b/backend/templates/html/header_enclosing.html
@@ -1,22 +1,39 @@
-
+
Bible
diff --git a/frontend/src/routes/books/+page.svelte b/frontend/src/routes/books/+page.svelte
index 2a5d34834..f6df0da59 100644
--- a/frontend/src/routes/books/+page.svelte
+++ b/frontend/src/routes/books/+page.svelte
@@ -1,4 +1,5 @@
-
diff --git a/frontend/src/routes/passages/language/+page.svelte b/frontend/src/routes/passages/language/+page.svelte
index 9a2be7a20..88227c7a9 100644
--- a/frontend/src/routes/passages/language/+page.svelte
+++ b/frontend/src/routes/passages/language/+page.svelte
@@ -12,30 +12,7 @@
import { getCode, getName } from '$lib/passages/utils'
let showGatewayLanguages = true
- // Track if the user manually changed the tab:
- let userInteracted = false
- const selectGatewayTab = () => {
- userInteracted = true
- showGatewayLanguages = true
- }
- const selectHeartTab = () => {
- userInteracted = true
- showGatewayLanguages = false
- }
- // If user has previously chosen (during this session, i.e., prior
- // to browser reload) any heart languages and no gateway languages then default to
- // showing the heart languages, otherwise the default stands of
- // showing the gateway languages.
- $: {
- if (
- !userInteracted &&
- $langCodeAndNameStore &&
- heartCodesAndNames.includes($langCodeAndNameStore)
- ) {
- showGatewayLanguages = false
- }
- }
// For use by Mobile UI
let showFilterMenu = false
let showWizardBasketModal = false
@@ -45,12 +22,11 @@
langCodesAndNamesUrl: string = PUBLIC_LANG_CODES_NAMES_URL
): Promise> {
const response = await fetch(`${apiRootUrl}${langCodesAndNamesUrl}`)
- const langCodeNameAndTypes: Array<[string, string, boolean]> = await response.json()
if (!response.ok) {
console.log(`Error: ${response.statusText}`)
throw new Error(response.statusText)
}
- return langCodeNameAndTypes
+ return await response.json()
}
// Resolve promise for data
@@ -58,23 +34,22 @@
let gatewayCodesAndNames: Array = []
let heartCodesAndNames: Array = []
- onMount(() => {
- getLangCodesNames()
- .then((langCodeNameAndTypes_) => {
- // Save result for later use
- langCodeNameAndTypes = langCodeNameAndTypes_
- gatewayCodesAndNames = langCodeNameAndTypes_
- .filter((element: [string, string, boolean]) => {
- return element[2]
- })
- .map((tuple: [string, string, boolean]) => `${tuple[0]}, ${tuple[1]}`)
- heartCodesAndNames = langCodeNameAndTypes_
- .filter((element: [string, string, boolean]) => {
- return !element[2]
- })
- .map((tuple) => `${tuple[0]}, ${tuple[1]}`)
- })
- .catch((err) => console.log(err))
+ async function loadLangCodeNameAndTypes() {
+ try {
+ langCodeNameAndTypes = await getLangCodesNames()
+ gatewayCodesAndNames = langCodeNameAndTypes
+ .filter(([, , isGateway]) => isGateway)
+ .map(([code, name]) => `${code}, ${name}`)
+ heartCodesAndNames = langCodeNameAndTypes
+ .filter(([, , isGateway]) => !isGateway)
+ .map(([code, name]) => `${code}, ${name}`)
+ } catch (error) {
+ console.error(error)
+ }
+ }
+
+ onMount(async () => {
+ await loadLangCodeNameAndTypes()
})
// Set $langCountStore
@@ -93,8 +68,10 @@
let filteredGatewayCodeAndNames: Array = []
$: {
if (gatewayCodesAndNames) {
- filteredGatewayCodeAndNames = gatewayCodesAndNames.filter((item: string) =>
- getName(item.toLowerCase()).includes(gatewaySearchTerm.toLowerCase())
+ filteredGatewayCodeAndNames = gatewayCodesAndNames.filter(
+ (item: string) =>
+ getName(item.toLowerCase()).includes(gatewaySearchTerm.toLowerCase()) ||
+ getCode(item.toLowerCase()).includes(gatewaySearchTerm.toLowerCase())
)
}
}
@@ -104,13 +81,15 @@
let filteredHeartCodeAndNames: Array = []
$: {
if (heartCodesAndNames) {
- filteredHeartCodeAndNames = heartCodesAndNames.filter((item: string) =>
- getName(item.toLowerCase()).includes(heartSearchTerm.toLowerCase())
+ filteredHeartCodeAndNames = heartCodesAndNames.filter(
+ (item: string) =>
+ getName(item.toLowerCase()).includes(heartSearchTerm.toLowerCase()) ||
+ getCode(item.toLowerCase()).includes(heartSearchTerm.toLowerCase())
)
}
}
- let windowWidth: number = typeof window !== "undefined" ? window.innerWidth : 0
+ let windowWidth: number = typeof window !== 'undefined' ? window.innerWidth : 0
$: console.log(`windowWidth: ${windowWidth}`)
let TAILWIND_SM_MIN_WIDTH: number = PUBLIC_TAILWIND_SM_MIN_WIDTH as unknown as number
@@ -128,13 +107,10 @@
{#if (gatewayCodesAndNames && gatewayCodesAndNames.length > 0) || (heartCodesAndNames && heartCodesAndNames.length > 0)}
{#if windowWidth < TAILWIND_SM_MIN_WIDTH}
@@ -171,52 +147,3 @@