diff --git a/api/connectors/blob_connector.py b/api/connectors/blob_connector.py index 47005a1..15e4c9d 100644 --- a/api/connectors/blob_connector.py +++ b/api/connectors/blob_connector.py @@ -96,7 +96,7 @@ async def list_blobs(config: Dict[str, Any], full_sync: bool = False) -> List[Di elif extensions: allowed_extensions = set(extensions) else: - allowed_extensions = {".pdf", ".txt", ".json", ".md", ".csv"} + allowed_extensions = {".pdf", ".txt", ".json", ".md", ".csv", ".docx", ".doc"} prefix = config.get("prefix", "") @@ -150,7 +150,7 @@ async def list_blobs_paginated( elif extensions: allowed_extensions = set(extensions) else: - allowed_extensions = {".pdf", ".txt", ".json", ".md", ".csv"} + allowed_extensions = {".pdf", ".txt", ".json", ".md", ".csv", ".docx", ".doc"} prefix = config.get("prefix", "") diff --git a/docgrok/pipeline-worker/Dockerfile b/docgrok/pipeline-worker/Dockerfile index d2f05ac..1b7f647 100644 --- a/docgrok/pipeline-worker/Dockerfile +++ b/docgrok/pipeline-worker/Dockerfile @@ -11,6 +11,7 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY worker.py . +COPY word_extract.py . # Pre-download PaddleOCR models during build to avoid download at runtime RUN python -c "from paddleocr import PaddleOCR; PaddleOCR(use_angle_cls=True, lang='en', show_log=False)" diff --git a/docgrok/pipeline-worker/requirements.txt b/docgrok/pipeline-worker/requirements.txt index 7fed625..b7b9733 100644 --- a/docgrok/pipeline-worker/requirements.txt +++ b/docgrok/pipeline-worker/requirements.txt @@ -8,3 +8,5 @@ PyMuPDF==1.25.3 azure-storage-blob>=12.19.0 azure-identity>=1.15.0 opencv-python-headless==4.10.0.84 +python-docx==1.1.2 +olefile==0.47 diff --git a/docgrok/pipeline-worker/word_extract.py b/docgrok/pipeline-worker/word_extract.py new file mode 100644 index 0000000..7442225 --- /dev/null +++ b/docgrok/pipeline-worker/word_extract.py @@ -0,0 +1,273 @@ +"""Microsoft Word document text extraction for the DocGrok pipeline worker. + +Two formats are supported: + +* ``.docx`` — modern Office Open XML (zip + XML). Extracted with + ``python-docx``; we read paragraphs and table cell text from the main + document and from any headers/footers/footnotes/endnotes. Returns one + "section" per Word section break so downstream chunking can still + preserve logical groupings. + +* ``.doc`` — legacy OLE compound binary. We use ``olefile`` to read the + ``WordDocument`` stream and decode the text directly from the FIB. + This is a best-effort path; if extraction yields nothing usable we + raise an explicit 415 so the caller can route or convert. + +Both functions return a ``list[str]`` of "page-like" text segments +matching the shape ``_stage_extract`` already produces for PDFs/text. +""" + +from __future__ import annotations + +from typing import List + + +def extract_docx(path: str) -> List[str]: + """Extract text from a .docx file. + + Returns a list where each element is the text of one Word section + (delimited by section breaks). For a single-section document this is + a one-element list. Paragraphs are separated by ``\\n``; tables are + flattened cell-by-cell with tab + newline separators. + """ + try: + from docx import Document + from docx.oxml.ns import qn + except ImportError as e: + raise RuntimeError( + "python-docx is not installed; cannot extract .docx. " + "Install with: pip install python-docx" + ) from e + + doc = Document(path) + + sections: List[List[str]] = [[]] + + def _emit(text: str) -> None: + if text: + sections[-1].append(text) + + def _para_text(p) -> str: + return p.text or "" + + def _table_text(tbl) -> str: + rows: List[str] = [] + for row in tbl.rows: + cells = [(cell.text or "").strip() for cell in row.cells] + cells = [c for c in cells if c] + if cells: + rows.append("\t".join(cells)) + return "\n".join(rows) + + body = doc.element.body + for child in body.iterchildren(): + tag = child.tag + if tag == qn("w:p"): + from docx.text.paragraph import Paragraph + _emit(_para_text(Paragraph(child, doc))) + # A section break lives inside a paragraph's pPr/sectPr. + pPr = child.find(qn("w:pPr")) + if pPr is not None and pPr.find(qn("w:sectPr")) is not None: + if sections[-1]: + sections.append([]) + elif tag == qn("w:tbl"): + from docx.table import Table + _emit(_table_text(Table(child, doc))) + elif tag == qn("w:sectPr"): + if sections[-1]: + sections.append([]) + + header_footer_text: List[str] = [] + for sec in doc.sections: + for hf in (sec.header, sec.footer, sec.first_page_header, + sec.first_page_footer, sec.even_page_header, + sec.even_page_footer): + for p in getattr(hf, "paragraphs", []) or []: + t = _para_text(p) + if t.strip(): + header_footer_text.append(t) + if header_footer_text: + if sections[-1]: + sections.append([]) + sections[-1].extend(header_footer_text) + + out = ["\n".join(s).strip() for s in sections] + out = [s for s in out if s] + return out or [""] + + +def extract_doc(path: str) -> List[str]: + """Best-effort text extraction from a legacy .doc file. + + The .doc binary format stores the document text in the + ``WordDocument`` OLE stream at offset ``fcMin``..``fcMin+ccpText``. + Encoding is detected from the FIB ``fExtChar`` flag (CP1252 vs + UTF-16LE). + + For documents with complex formatting (revision marks, embedded + objects, etc.) the extracted text may be imperfect. Callers that + need fidelity should ask users to convert to .docx first. + """ + try: + import olefile + except ImportError as e: + raise RuntimeError( + "olefile is not installed; cannot extract legacy .doc. " + "Install with: pip install olefile" + ) from e + + if not olefile.isOleFile(path): + raise ValueError( + f"{path!r} is not an OLE compound file; not a legacy .doc" + ) + + with olefile.OleFileIO(path) as ole: + if not ole.exists("WordDocument"): + raise ValueError( + f"{path!r} is missing the WordDocument stream; not a Word .doc" + ) + with ole.openstream("WordDocument") as fh: + wd = fh.read() + + if len(wd) < 0x200: + raise ValueError(f"WordDocument stream too short ({len(wd)} bytes)") + + flags = int.from_bytes(wd[0x000A:0x000C], "little") + is_complex = bool(flags & 0x0004) + f_table = "1Table" if (flags & 0x0200) else "0Table" + + fc_min = int.from_bytes(wd[0x0018:0x001C], "little", signed=False) + ccp_text = int.from_bytes(wd[0x004C:0x0050], "little", signed=False) + + if not is_complex and ccp_text and ccp_text < 10_000_000: + end = min(fc_min + ccp_text * 2, len(wd)) + blob = wd[fc_min:end] + text = _decode_word_text(blob) + if text.strip(): + return [text] + + if ole.exists(f_table): + with ole.openstream(f_table) as fh: + table = fh.read() + text = _extract_complex_doc_text(wd, table) + if text.strip(): + return [text] + + raise RuntimeError( + "Could not extract text from .doc; convert the file to .docx and retry." + ) + + +def _decode_word_text(blob: bytes) -> str: + """Decode a Word text run, trying UTF-16LE first then CP1252.""" + if not blob: + return "" + try: + if len(blob) % 2 == 0: + u = blob.decode("utf-16-le", errors="strict") + if sum(1 for c in u if c == "\x00") < len(u) // 4: + return _normalize_word_runs(u) + except UnicodeDecodeError: + pass + return _normalize_word_runs(blob.decode("cp1252", errors="replace")) + + +def _normalize_word_runs(text: str) -> str: + out: List[str] = [] + for ch in text: + code = ord(ch) + if ch == "\x07": + out.append("\t") + elif ch in ("\x0B", "\x0C"): + out.append("\n") + elif ch in ("\r", "\x0E"): + out.append("\n") + elif code < 0x20 and ch not in ("\n", "\t"): + continue + else: + out.append(ch) + s = "".join(out) + while "\n\n\n" in s: + s = s.replace("\n\n\n", "\n\n") + return s.strip() + + +def _extract_complex_doc_text(wd: bytes, table: bytes) -> str: + """Reconstruct text from the piece table for complex .doc files.""" + fc_clx = int.from_bytes(wd[0x01A2:0x01A6], "little", signed=False) + lcb_clx = int.from_bytes(wd[0x01A6:0x01AA], "little", signed=False) + if not lcb_clx or fc_clx + lcb_clx > len(table): + return "" + + clx = table[fc_clx:fc_clx + lcb_clx] + i = 0 + while i < len(clx): + if clx[i] == 0x02: + cb_pcd = int.from_bytes(clx[i + 1:i + 5], "little", signed=False) + pcdt = clx[i + 5:i + 5 + cb_pcd] + n = (len(pcdt) - 4) // 12 + cps = [ + int.from_bytes(pcdt[j * 4:(j + 1) * 4], "little", signed=False) + for j in range(n + 1) + ] + pcds = pcdt[4 + (n + 1) * 4:] + parts: List[str] = [] + for k in range(n): + pcd = pcds[k * 8:(k + 1) * 8] + fc_raw = int.from_bytes(pcd[2:6], "little", signed=False) + ansi = bool(fc_raw & 0x40000000) + fc = fc_raw & 0x3FFFFFFF + cp_len = cps[k + 1] - cps[k] + if ansi: + fc //= 2 + blob = wd[fc:fc + cp_len] + else: + blob = wd[fc:fc + cp_len * 2] + parts.append(_decode_word_text(blob)) + return _normalize_word_runs("".join(parts)) + elif clx[i] == 0x01: + cb = int.from_bytes(clx[i + 1:i + 3], "little", signed=False) + i += 3 + cb + else: + break + return "" + + +if __name__ == "__main__": + import sys + import tempfile + + try: + from docx import Document + except ImportError: + print("python-docx not installed; install it to run the self-test") + sys.exit(2) + + doc = Document() + doc.add_heading("DocGrok Word self-test", level=1) + doc.add_paragraph("This is paragraph one with some text.") + doc.add_paragraph("And a second paragraph.") + table = doc.add_table(rows=2, cols=2) + table.cell(0, 0).text = "alpha" + table.cell(0, 1).text = "beta" + table.cell(1, 0).text = "gamma" + table.cell(1, 1).text = "delta" + doc.add_paragraph("After-table paragraph.") + + with tempfile.NamedTemporaryFile(suffix=".docx", delete=False) as tmp: + doc.save(tmp.name) + path = tmp.name + + sections = extract_docx(path) + print(f"Extracted {len(sections)} section(s):") + for i, s in enumerate(sections): + print(f"--- section {i} ({len(s)} chars) ---") + print(s) + + assert sections, "no sections extracted" + full = "\n".join(sections) + for needle in ("paragraph one", "second paragraph", + "alpha", "beta", "gamma", "delta", + "After-table paragraph"): + assert needle in full, f"missing {needle!r}" + print("OK — docx round-trip succeeded") diff --git a/docgrok/pipeline-worker/worker.py b/docgrok/pipeline-worker/worker.py index 811c121..c9da741 100644 --- a/docgrok/pipeline-worker/worker.py +++ b/docgrok/pipeline-worker/worker.py @@ -115,7 +115,7 @@ def filter(self, record): ), "params": { "doctype": { - "type": "enum[auto|pdf|text]", "default": "auto", + "type": "enum[auto|pdf|text|docx|doc]", "default": "auto", "description": "How to interpret the input. 'auto' = detect from filename / pipeline hint.", }, "ocr_engine": { @@ -322,6 +322,37 @@ def filter(self, record): }, ], }, + { + "name": "word-transform", + "version": "1.0.0", + "description": ( + "Extract text from Microsoft Word documents (.docx via " + "python-docx, legacy .doc via olefile), split into " + "~1500-char paragraph-aware chunks, embed each chunk." + ), + "applies_to": { + "extensions": [".docx", ".doc"], + "content_types": [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/msword", + ], + }, + "priority": 25, + "stages": [ + { + "type": "extract", "name": "extract_word", + "config": {"doctype": "auto"}, + }, + { + "type": "chunk", "name": "chunk_text", + "config": {"strategy": "paragraph", "max_chars": 1500, "overlap_chars": 100, "min_chars": 0}, + }, + { + "type": "embed", "name": "embed_vectors", + "config": {"model_id": None, "router_url": None, "batch_size": 1}, + }, + ], + }, { "name": "text-transform", "version": "1.0.0", @@ -797,31 +828,52 @@ def _chunk_text(text: str, chunk_size: int = 2000) -> List[str]: ".yml", ".yaml", ".xml", ".html", ".htm", ".py", ".js", ".ts", ".go", ".java", ".c", ".cpp", ".h", ".sh", ".sql"} _PDF_EXTS = {".pdf"} +_DOCX_EXTS = {".docx"} +_DOC_EXTS = {".doc"} -def _is_text_blob(blob_name: Optional[str], pipeline_hint: Optional[str]) -> bool: - """Decide whether to treat a blob as plain text (skip OCR). +def _classify_blob_doctype(blob_name: Optional[str], pipeline_hint: Optional[str]) -> str: + """Return one of: 'docx', 'doc', 'text', 'pdf' for auto-detection. - Priority: - 1. Pipeline name hint: *-text / text-* → text; *-pdf / pdf-* → PDF. - 2. Blob filename extension: .txt/.md/.json/... → text; .pdf → PDF. - 3. Default: PDF (existing behavior for backward compat). + Order of precedence: + 1. Pipeline name hint (text, word/docx, pdf). + 2. Blob filename extension. + 3. Default 'pdf' (backward compatible). """ if pipeline_hint: p = pipeline_hint.lower() + if "docx" in p or "word" in p: + return "docx" if "text" in p or p.endswith("-txt") or p.startswith("txt-"): - return True + return "text" if "pdf" in p: - return False + return "pdf" if blob_name: name = blob_name.lower() + for ext in _DOCX_EXTS: + if name.endswith(ext): + return "docx" + for ext in _DOC_EXTS: + if name.endswith(ext): + return "doc" for ext in _TEXT_EXTS: if name.endswith(ext): - return True + return "text" for ext in _PDF_EXTS: if name.endswith(ext): - return False - return False # default → PDF path (unchanged historical behavior) + return "pdf" + return "pdf" + + +def _is_text_blob(blob_name: Optional[str], pipeline_hint: Optional[str]) -> bool: + """Decide whether to treat a blob as plain text (skip OCR). + + Priority: + 1. Pipeline name hint: *-text / text-* → text; *-pdf / pdf-* → PDF. + 2. Blob filename extension: .txt/.md/.json/... → text; .pdf → PDF. + 3. Default: PDF (existing behavior for backward compat). + """ + return _classify_blob_doctype(blob_name, pipeline_hint) == "text" # ── Embedding via DocGrok Router ─────────────────────────────────────── @@ -1074,7 +1126,7 @@ async def _stage_extract(srec: StepRecord, ctx: Dict[str, Any], cfg: Dict[str, A elif source_kind == "inline_b64": doctype = "pdf" else: - doctype = "text" if _is_text_blob(blob_name, pipeline_hint) else "pdf" + doctype = _classify_blob_doctype(blob_name, pipeline_hint) srec.notes.append(f"doctype auto-detected as '{doctype}'") else: doctype = requested @@ -1100,6 +1152,66 @@ async def _stage_extract(srec: StepRecord, ctx: Dict[str, Any], cfg: Dict[str, A srec.output["pages_total"] = 1 return + if doctype in ("docx", "doc"): + suffix = "." + doctype + if source_kind == "blob": + word_path = _download_blob_to_file( + ctx["blob_container"], blob_name, + ctx.get("blob_connection_string"), ctx.get("blob_account_url"), + ) + srec.notes.append(f"streamed blob to temp file for {doctype} extraction") + elif source_kind == "inline_b64": + try: + raw = base64.b64decode(ctx.get("inline_b64") or "") + except Exception as e: + raise HTTPException(status_code=400, detail=f"Invalid base64 data: {e}") + fd, word_path = tempfile.mkstemp(suffix=suffix, prefix="docgrok-") + with os.fdopen(fd, "wb") as f: + f.write(raw) + del raw + gc.collect() + srec.notes.append(f"decoded base64 input to temp {doctype} file") + else: + raise HTTPException(status_code=400, detail=f"{doctype} doctype requires Word input") + + try: + size = os.path.getsize(word_path) + except Exception: + size = -1 + srec.output["bytes"] = size + + from word_extract import extract_docx, extract_doc + try: + if doctype == "docx": + sections = extract_docx(word_path) + else: + sections = extract_doc(word_path) + except HTTPException: + raise + except (ValueError, RuntimeError) as e: + status = 415 if doctype == "doc" else 400 + raise HTTPException(status_code=status, + detail=f"Failed to parse {doctype}: {e}") + except Exception as e: + raise HTTPException(status_code=400, + detail=f"Failed to parse {doctype}: {e}") + finally: + if source_kind == "inline_b64": + try: + os.unlink(word_path) + except OSError: + pass + + ctx["page_texts"] = sections or [""] + total_chars = sum(len(s) for s in sections) + srec.output["chars"] = total_chars + srec.output["pages_total"] = len(sections) + srec.output["sections"] = len(sections) + srec.notes.append( + f"extracted {len(sections)} section(s), {total_chars} chars from {doctype}" + ) + return + # doctype == "pdf" pdf_path = ctx.get("pdf_path") if not pdf_path: