Merge convert-document into ingest-file (#395)

* Merge convert-document into ingest-file

Fixes #122

* Re-raise exception during conversion

* Add retry logic

* Document the unit for timeout

Author: stchris

.github/workflows/build.yml

@@ -32,11 +32,6 @@ jobs:
           docker push ghcr.io/alephdata/ingest-file:${GITHUB_SHA}
           docker tag ghcr.io/alephdata/ingest-file ghcr.io/alephdata/ingest-file:cache
           docker push ghcr.io/alephdata/ingest-file:cache
-
-          docker tag ghcr.io/alephdata/convert-document ghcr.io/alephdata/convert-document:${GITHUB_SHA}
-          docker push ghcr.io/alephdata/convert-document:${GITHUB_SHA}
-          docker tag ghcr.io/alephdata/convert-document ghcr.io/alephdata/convert-document:cache
-          docker push ghcr.io/alephdata/convert-document:cache
       - name: Push docker images for tags
         if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
         run: |
@@ -45,7 +40,3 @@ jobs:
           docker tag ghcr.io/alephdata/ingest-file ghcr.io/alephdata/ingest-file:${TAG};
           docker push ghcr.io/alephdata/ingest-file:${TAG};
           docker push ghcr.io/alephdata/ingest-file;
-
-          docker tag ghcr.io/alephdata/convert-document ghcr.io/alephdata/convert-document:${TAG};
-          docker push ghcr.io/alephdata/convert-document:${TAG};
-          docker push ghcr.io/alephdata/convert-document;

.github/workflows/daily.yml

@@ -20,4 +20,3 @@ jobs:
         run: |
           echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
           docker push ghcr.io/alephdata/ingest-file:cache
-          docker push ghcr.io/alephdata/convert-document:cache

Dockerfile

@@ -96,6 +96,12 @@ RUN apt-get -qq -y update \
     tesseract-ocr-aze \
     tesseract-ocr-bel \
     tesseract-ocr-uzb \
+    ### pdf convert: libreoffice + a bunch of fonts 
+    libreoffice fonts-opensymbol hyphen-fr hyphen-de \
+    hyphen-en-us hyphen-it hyphen-ru fonts-dejavu fonts-dejavu-core fonts-dejavu-extra \
+    fonts-droid-fallback fonts-dustin fonts-f500 fonts-fanwood fonts-freefont-ttf \
+    fonts-liberation fonts-lmodern fonts-lyx fonts-sil-gentium fonts-texgyre \
+    fonts-tlwg-purisa \
     ###
     && apt-get -qq -y autoremove \
     && apt-get clean \
@@ -146,8 +152,7 @@ RUN chown -R app:app /ingestors
 ENV ARCHIVE_TYPE=file \
     ARCHIVE_PATH=/data \
     FTM_STORE_URI=postgresql://aleph:aleph@postgres/aleph \
-    REDIS_URL=redis://redis:6379/0 \
-    INGESTORS_CONVERT_DOCUMENT_URL=http://convert-document:3000/convert
+    REDIS_URL=redis://redis:6379/0
 
 # USER app
 CMD ingestors process

Makefile

@@ -1,5 +1,4 @@
 INGEST=ghcr.io/alephdata/ingest-file
-CONVERT=ghcr.io/alephdata/convert-document
 COMPOSE=docker-compose
 DOCKER=$(COMPOSE) run --rm ingest-file
 
@@ -12,30 +11,26 @@ build:
 
 pull-cache:
 	-docker pull -q $(INGEST):cache
-	-docker pull -q $(CONVERT):cache
 
 cached-build: pull-cache
 	docker build --cache-from $(INGEST):cache -t $(INGEST) .
-	docker build --cache-from $(CONVERT):cache -t $(CONVERT) convert
 
 fresh-cache:
 	# re-generate cache images on a daily basis to avoid using
 	# stale docker containers from upstream.
 	docker build --pull --no-cache -t $(INGEST):cache .
-	docker build --pull --no-cache -t $(CONVERT):cache convert
 
 services:
 	$(COMPOSE) up -d --remove-orphans postgres redis
 
 shell: services
-	$(COMPOSE) up -d --remove-orphans convert-document
 	$(DOCKER) /bin/bash
 
 test: services
 	$(DOCKER) pytest --cov=ingestors --cov-report html --cov-report term
 
 restart: build
-	$(COMPOSE) up --force-recreate --no-deps --detach convert-document ingest-file
+	$(COMPOSE) up --force-recreate --no-deps --detach ingest-file
 
 tail:
 	$(COMPOSE) logs -f