Passages ot rg and conditional stet passages

.env

@@ -32,10 +32,10 @@ SEND_EMAIL = false
 # The port to pass to gunicorn via ./backend/gunicorn.conf.py
 PORT=5005
 # Control caching of resource assets to save on network traffic
-ASSET_CACHING_ENABLED=false
+ASSET_CACHING_ENABLED=true
 # Caching window of time in which cloned or downloaded resource asset
-# files on disk are considered fresh rather than reacqiring them. In hours.
-ASSET_CACHING_PERIOD=168
+# files on disk are considered fresh rather than reacqiring them. In minutes.
+ASSET_CACHING_PERIOD=30
 
 # Control whether GitPython package does git cloning or the git cli in
 # a subpocess. git cli is more robust to errors and faster.
@@ -65,6 +65,11 @@ USE_LOCALIZED_BOOK_NAME=true
 # in parsing.RESOURCES_WITH_USFM_DEFECTS.
 CHECK_ALL_BOOKS_FOR_LANGUAGE=true
 
+# When true the system will acquire git repos via download of its
+# master.zip file. When false it will clone the repo. Both approaches
+# are optimized maximally by taking advantage of curl and git options.
+DOWNLOAD_ASSETS=false
+
 # * http://localhost:3000 covers requests originating from the case
 # where 'npm run dev' is invoked to run vite (to run svelte js frontend)
 # outside Docker. This results in vite's development mode which runs on

.github/workflows/docker-publish.yml

@@ -35,16 +35,31 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+  Build-USFMParser-Image:
+    name: Build USFMParser image
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: FranzDiebold/github-env-vars-action@v2
+      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-buildx-action@v2
+      - name: Build USFMParser image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./dotnet/USFMParserApi
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
   Test-API-Image:
     name: Test API image
     runs-on: ubuntu-24.04
-    needs: Build-API-Image
+    needs: [Build-API-Image, Build-USFMParser-Image]
     steps:
       - uses: actions/checkout@v3
       - uses: FranzDiebold/github-env-vars-action@v2
       - uses: docker/setup-qemu-action@v2
       - uses: docker/setup-buildx-action@v2
-      - name: Build API image with buildx
+      - name: Build local API image with buildx
         uses: docker/build-push-action@v3
         with:
           context: .
@@ -53,35 +68,54 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Build local USFMParser image with buildx
+        uses: docker/build-push-action@v3
+        with:
+          context: ./dotnet/USFMParserApi
+          tags: wycliffeassociates/doc-usfmparser:local
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Test API image with docker-compose
         run: |
           docker compose -f docker-compose.yml -f docker-compose.api-test.yml up --exit-code-from test-runner
 
   Test-Frontend-Image:
     name: Test Frontend image
     runs-on: ubuntu-24.04
-    needs: [Build-API-Image, Build-Frontend-Image]
+    needs: [Build-API-Image, Build-Frontend-Image, Build-USFMParser-Image]
     steps:
       - uses: actions/checkout@v3
       - uses: FranzDiebold/github-env-vars-action@v2
       - uses: docker/setup-qemu-action@v2
       - uses: docker/setup-buildx-action@v2
-      - name: Build api image
+      - name: Build local api image
         uses: docker/build-push-action@v3
         with:
           context: .
           tags: wycliffeassociates/doc:local
           load: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
-      - name: Build frontend image
+      - name: Build local frontend image
         uses: docker/build-push-action@v3
         with:
           context: ./frontend
           tags: wycliffeassociates/doc-ui:local
           load: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+      - name: Build local USFMParser image with buildx
+        uses: docker/build-push-action@v3
+        with:
+          context: ./dotnet/USFMParserApi
+          tags: wycliffeassociates/doc-usfmparser:local
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Build frontend-tests image
         uses: docker/build-push-action@v3
         with:
@@ -120,6 +154,11 @@ jobs:
           echo "DOC_UI_TAG_SHA=wycliffeassociates/doc-ui:$GITHUB_SHA" >> $GITHUB_ENV && \
           echo "DOC_UI_TAG_BRANCH=wycliffeassociates/doc-ui:$CI_REF_NAME_SLUG" >> $GITHUB_ENV && \
           echo "DOC_UI_TAG_LATEST=wycliffeassociates/doc-ui:latest" >> $GITHUB_ENV
+      - name: Set DOC USFM parser tags
+        run: |
+          echo "DOC_PARSER_TAG_SHA=wycliffeassociates/doc-usfmparser:$GITHUB_SHA" >> $GITHUB_ENV && \
+          echo "DOC_PARSER_TAG_BRANCH=wycliffeassociates/doc-usfmparser:$CI_REF_NAME_SLUG" >> $GITHUB_ENV && \
+          echo "DOC_PARSER_TAG_LATEST=wycliffeassociates/doc-usfmparser:latest" >> $GITHUB_ENV
 
       - name: Set version inside UI
         run: |
@@ -152,6 +191,17 @@ jobs:
             PUBLIC_DOC_BUILD_TIMESTAMP_ARG=${{ env.PUBLIC_DOC_BUILD_TIMESTAMP }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+      - name: Build and conditional push Parser image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./dotnet/USFMParserApi
+          push: true
+          tags: |
+            ${{ env.DOC_PARSER_TAG_SHA }}
+            ${{ env.DOC_PARSER_TAG_BRANCH }}
+            ${{ env.DOC_PARSER_TAG_LATEST }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
   Deploy-develop:
     name: Develop deploy on successful develop build
     needs: [Push-Images]

.gitignore

@@ -18,3 +18,6 @@ tests_output.log
 /logs/
 /build/
 /DOC.egg-info/
+/assets_download/
+/dotnet/USFMParserApi/bin/
+/dotnet/USFMParserApi/obj/

Dockerfile

@@ -25,10 +25,45 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libdeflate0 \
     # For weasyprint
     pango1.0-tools \
-    # For stet
-    # pandoc \
     # For fc-cache
-    fontconfig
+    fontconfig \
+    # For princexml
+    fonts-khmeros \
+    fonts-lklug-sinhala \
+    fonts-tlwg-garuda-otf \
+    fonts-lohit-orya \
+    fonts-lohit-mlym \
+    fonts-lohit-knda \
+    fonts-lohit-telu \
+    fonts-lohit-taml \
+    fonts-lohit-gujr \
+    fonts-lohit-guru \
+    fonts-lohit-beng-bengali \
+    fonts-lohit-deva \
+    fonts-noto \
+    # fonts-noto-core \
+    # fonts-noto-unhinted \
+    fonts-noto-cjk \
+    fonts-dzongkha \
+    fonts-tibetan-machine \
+    fonts-baekmuk \
+    fonts-ipafont-mincho \
+    fonts-arphic-uming \
+    fonts-opensymbol \
+    fonts-liberation2 \
+    libaom3 \
+    libavif15 \
+    libgif7 \
+    libjpeg62-turbo \
+    liblcms2-2 \
+    libtiff6 \
+    libwebp7 \
+    libwebpdemux2 \
+ && fc-cache -f -v
+
+# Download and install the PrinceXML .deb file
+RUN wget https://www.princexml.com/download/prince_16.1-1_debian12_amd64.deb \
+    && dpkg -i prince_16.1-1_debian12_amd64.deb || apt-get install -fy
 
 # Get and install needed fonts.
 RUN cd /tmp \
@@ -50,37 +85,23 @@ RUN ebook-convert --version
 
 WORKDIR /app
 
-RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \
-    && chmod +x ./dotnet-install.sh
-
-# Create a directory for .NET SDK
-RUN mkdir -p /home/appuser/.dotnet
-
-# Install .NET SDK to the created directory
-RUN ./dotnet-install.sh --channel 8.0 --install-dir /usr/share/dotnet
-
-COPY dotnet ./
-
-# Set environment variables for .NET
-ENV DOTNET_ROOT=/usr/share/dotnet
-ENV PATH=$PATH:$DOTNET_ROOT:$DOTNET_ROOT/tools
-ENV DOTNET_CLI_TELEMETRY_OPTOUT=1
-
-# Install dependencies and build the .NET project
-RUN cd USFMParserDriver && \
-    ${DOTNET_ROOT}/dotnet restore && \
-    ${DOTNET_ROOT}/dotnet build --configuration Release
 
 # Make the output directory where resource asset files are cloned.
 RUN mkdir -p assets_download
+# Make the input directory where en_rg_nt_survey.docx is stored.
+RUN mkdir -p en_rg
 # Make the directory where intermediate document parts are saved.
 RUN mkdir -p working_temp
 # Make the output directory where generated HTML and PDFs are placed.
 RUN mkdir -p document_output
 # Make the directory where stet source documents are stored
 RUN mkdir -p stet
+# Make the directory where passage source documents are stored
+RUN mkdir -p passages
+
 
 COPY backend/stet/data/stet_*.docx stet/
+COPY backend/passages/data/Spiritual_Terms_Evaluation_Exhaustive_Verse_List.txt passages/
 
 COPY pyproject.toml .
 COPY ./backend/requirements.txt .
@@ -101,17 +122,19 @@ COPY ./tests ./tests
 COPY .env .
 COPY template.docx .
 COPY template_compact.docx .
+
 # Next two lines are useful when the data (graphql) API are down so
 # that we can still test
 # COPY resources.json assets_download/resources.json
 # RUN touch assets_download/resources.json
 
-# We copy this into its final place using a FastAPI initialization hook. We
-# can't do it in Dockerfile because of the volumes definition that we
-# need which overshadows /app/assets_download directory. It is not yet
-# available through the data API or at a reasonably sized clonable
-# github repo.
+# We copy these files into their final place using a FastAPI
+# initialization hook. We can't do it in Dockerfile because of the
+# volumes definition that we need which overshadows /app/assets_download
+# directory and these files are not yet available through the data API
+# or at a reasonably sized clonable github repo.
 COPY en_rg_nt_survey.docx .
+COPY en_ot_survey_rg* .
 
 # Make sure Python can find the code to run
 ENV PYTHONPATH=/app/backend:/app/tests
@@ -123,13 +146,10 @@ RUN mypy --strict --install-types --non-interactive backend/passages/**/*.py
 RUN mypy --strict --install-types --non-interactive tests/**/*.py
 
 # Change ownership of app specific directories to the non-root user
-RUN chown -R appuser:appgroup /app /home/appuser/calibre-bin /usr/share/dotnet
+RUN chown -R appuser:appgroup /app /home/appuser/calibre-bin
 
 # Switch to the non-root user
 USER appuser
 
 # Expose necessary ports (if any)
 EXPOSE 8000
-
-# Command to run the application
-CMD ["python", "backend/main.py"]

Makefile

@@ -32,34 +32,39 @@ install-cython: checkvenv
 .PHONY: build
 build: checkvenv down clean-mypyc-artifacts install-cython local-install-deps-dev local-install-deps-prod
 	export IMAGE_TAG=local && \
+	docker build --progress=plain -t wycliffeassociates/doc-usfmparser:$${IMAGE_TAG} ./dotnet/USFMParserApi && \
 	docker build --progress=plain -t wycliffeassociates/doc:$${IMAGE_TAG} . && \
 	docker build --progress=plain -t wycliffeassociates/doc-ui:$${IMAGE_TAG} ./frontend && \
         docker build --progress=plain -t wycliffeassociates/doc-ui-tests:$${IMAGE_TAG} -f ./frontend/testsDockerfile ./frontend
 
 .PHONY: build-no-pip-update
 build-no-pip-update: checkvenv down clean-mypyc-artifacts
 	export IMAGE_TAG=local && \
+	docker build --progress=plain -t wycliffeassociates/doc-usfmparser:$${IMAGE_TAG} ./dotnet/USFMParserApi && \
 	docker build --progress=plain -t wycliffeassociates/doc:$${IMAGE_TAG} . && \
 	docker build --progress=plain -t wycliffeassociates/doc-ui:$${IMAGE_TAG} ./frontend && \
         docker build --progress=plain -t wycliffeassociates/doc-ui-tests:$${IMAGE_TAG} -f ./frontend/testsDockerfile ./frontend
 
 .PHONY: build-no-pip-update-run-tests
 build-no-pip-update-run-tests: checkvenv down clean-mypyc-artifacts
 	export IMAGE_TAG=local && \
+	docker build --progress=plain -t wycliffeassociates/doc-usfmparser:$${IMAGE_TAG} ./dotnet/USFMParserApi && \
 	docker build --build-arg RUN_TESTS=true --progress=plain -t wycliffeassociates/doc:$${IMAGE_TAG} . && \
 	docker build --progress=plain -t wycliffeassociates/doc-ui:$${IMAGE_TAG} ./frontend && \
         docker build --progress=plain -t wycliffeassociates/doc-ui-tests:$${IMAGE_TAG} -f ./frontend/testsDockerfile ./frontend
 
 .PHONY: build-no-cache
 build-no-cache: checkvenv down clean-mypyc-artifacts local-install-deps-prod
 	export IMAGE_TAG=local && \
+	docker build --progress=plain --no-cache --pull -t wycliffeassociates/doc-usfmparser:$${IMAGE_TAG} ./dotnet/USFMParserApi && \
 	docker build --progress=plain --no-cache --pull -t wycliffeassociates/doc:$${IMAGE_TAG} . && \
 	docker build --progress=plain --no-cache --pull -t wycliffeassociates/doc-ui:$${IMAGE_TAG} ./frontend && \
         docker build --progress=plain --no-cache --pull -t wycliffeassociates/doc-ui-tests:$${IMAGE_TAG} -f ./frontend/testsDockerfile ./frontend
 
 .PHONY: build-no-cache-no-pip-update
 build-no-cache-no-pip-update: checkvenv down clean-mypyc-artifacts
 	export IMAGE_TAG=local && \
+	docker build --progress=plain --no-cache --pull -t wycliffeassociates/doc-usfmparser:$${IMAGE_TAG} ./dotnet/USFMParserApi && \
 	docker build --progress=plain --no-cache --pull -t wycliffeassociates/doc:$${IMAGE_TAG} . && \
 	docker build --progress=plain --no-cache --pull -t wycliffeassociates/doc-ui:$${IMAGE_TAG} ./frontend && \
         docker build --progress=plain --no-cache --pull -t wycliffeassociates/doc-ui-tests:$${IMAGE_TAG} -f ./frontend/testsDockerfile ./frontend

backend/doc/config.py

@@ -18,7 +18,6 @@ class Settings(BaseSettings):
     (which have higher priority).
     """
 
-    # GITHUB_API_TOKEN: str = "FOO"  # This might be used in a later version
     DATA_API_URL: HttpUrl
 
     USFM_RESOURCE_TYPES: Sequence[str] = [
@@ -42,6 +41,12 @@ class Settings(BaseSettings):
     USE_LOCALIZED_BOOK_NAME: bool
     CHECK_ALL_BOOKS_FOR_LANGUAGE: bool
 
+    BOOK_NAME_FMT_STR: str = "<h2 class='book-name' style='text-align: center;'>{}</h2>"
+    END_OF_CHAPTER_HTML: str = '<div class="end-of-chapter"></div>'
+    HR: str = "<hr/>"
+
+    DOWNLOAD_ASSETS: bool  # If true then download assets, else clone assets
+
     def logger(self, name: str) -> logging.Logger:
         """
         Return a Logger for scope named by name, e.g., module, that can be
@@ -66,6 +71,7 @@ def api_test_url(self) -> str:
 
     # Location where resource assets will be cloned.
     RESOURCE_ASSETS_DIR: str = "assets_download"
+    EN_RG_DIR: str = "en_rg"
 
     # Location where intermediate generated document parts are saved.
     WORKING_DIR: str = "working_temp"
@@ -88,7 +94,7 @@ def api_test_url(self) -> str:
     # Caching window of time in which asset
     # files on disk are considered fresh rather than re-acquiring (in
     # the case of resource asset files) or re-generating them (in the
-    # case of the final PDF). In hours.
+    # case of the final PDF). In minutes.
     ASSET_CACHING_PERIOD: int
 
     EMAIL_SEND_SUBJECT: str