Upload folder using huggingface_hub

warraqa/__init__.py

@@ -0,0 +1,20 @@
+"""
+Warraqa (ورّاقة) — The Document Scribe Agent
+
+Named after the professional scribes and paper-makers of the Islamic Golden Age,
+Warraqa is a specialized agent whose sole mission is converting PDF and Word
+documents into clean, accurate Markdown with surgical precision.
+
+She reads. She transcribes. She scores her own work.
+"""
+
+__version__ = "1.0.0"
+
+AGENT_NAME = "Warraqa"
+AGENT_NAME_AR = "ورّاقة"
+AGENT_DESCRIPTION = "Document-to-Markdown conversion specialist"
+AGENT_IDENTITY = f"{AGENT_NAME} ({AGENT_NAME_AR}) v{__version__} — {AGENT_DESCRIPTION}"
+
+from warraqa.agent import Warraqa  # noqa: E402, F401
+
+__all__ = ["Warraqa", "AGENT_NAME", "AGENT_IDENTITY", "__version__"]

warraqa/agent.py

@@ -0,0 +1,801 @@
+"""
+Warraqa Agent — Core Orchestrator
+
+The brain of the Warraqa agent. Handles file discovery, engine selection,
+conversion with fallback chains, quality scoring, output saving, and
+file movement. Designed for both CLI use and programmatic inter-agent calls.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from datetime import datetime
+from pathlib import Path
+
+from warraqa.engines.base import BaseEngine
+from warraqa.engines.docx_pandoc import PandocEngine
+from warraqa.engines.office_com import OfficeCOMConverter
+from warraqa.engines.office_markitdown import MarkItDownEngine
+from warraqa.engines.pdf_marker import MarkerPDFEngine
+from warraqa.engines.pdf_pymupdf import PyMuPDFEngine
+from warraqa.models import BatchReport, ConversionResult, FileInfo
+from warraqa.scoring.scorer import ConversionScorer
+from warraqa.utils import (
+    PDFAnalyzer,
+    detect_file_type,
+    ensure_directories,
+    find_supported_files,
+    format_file_size,
+    get_mime_type,
+    open_folder_picker,
+    safe_move,
+    sanitize_filename,
+    sanitize_markdown,
+    setup_logging,
+)
+
+logger = logging.getLogger("warraqa")
+
+# Default config
+DEFAULT_CONFIG = {
+    "mode": "manual",
+    "watch": {"input_folder": None, "poll_interval_seconds": 10},
+    "output": {
+        "base_dir": "./output",
+        "md_files": "md_files",
+        "processed": "processed",
+        "failed": "failed",
+        "reports": "reports",
+        "scanned_pdfs": "scanned_pdfs",
+    },
+    "engines": {
+        "pdf_primary": "marker",
+        "pdf_fallback": "pymupdf",
+        "docx_primary": "markitdown",
+        "docx_fallback": "pandoc",
+        "pptx_primary": "markitdown",
+        "pptx_fallback": None,
+    },
+    "scoring": {"fail_threshold": 40, "warn_threshold": 70},
+    "logging": {"level": "INFO", "console": True, "file": None},
+}
+
+
+class Warraqa:
+    """
+    The Document Scribe Agent.
+
+    Converts PDF and Word documents to clean Markdown with quality scoring.
+
+    Usage:
+        agent = Warraqa()
+        result = agent.convert_file("path/to/document.pdf")
+        results = agent.process_folder("path/to/input/folder")
+    """
+
+    def __init__(self, config_path: str | Path | None = None, native_engine: str = "pymupdf"):
+        """
+        Initialize Warraqa with configuration.
+
+        Args:
+            config_path: Path to config.yaml. If None, uses defaults and
+                         looks for config.yaml in the package's parent directory.
+        """
+        self._config = self._load_config(config_path)
+        self._native_engine = native_engine
+        self._setup_logging()
+        self._output_base = Path(self._config["output"]["base_dir"]).resolve()
+        self._init_output_dirs()
+        self._engines = self._init_engines()
+        self._office_com = OfficeCOMConverter()
+        self._scorer = ConversionScorer(
+            fail_threshold=self._config["scoring"]["fail_threshold"],
+            warn_threshold=self._config["scoring"]["warn_threshold"],
+        )
+        if self._office_com.is_available():
+            logger.debug("Office COM pre-converter available (legacy .doc/.ppt support)")
+        else:
+            logger.warning(
+                "Office COM pre-converter unavailable — install pywin32 + MS Office "
+                "to support legacy .doc and .ppt files"
+            )
+        logger.info("Warraqa (ورّاقة) initialized — The Document Scribe is ready")
+
+    # --- Public API ---
+
+    def convert_file(
+        self,
+        file_path: str | Path,
+        *,
+        force_ocr: bool = False,
+        defer_pdf_fallback: bool = False,
+    ) -> ConversionResult:
+        """
+        Convert a single file to Markdown.
+
+        This is the primary API for other agents to call.
+
+        Args:
+            file_path: Path to the document file (PDF, DOCX, DOC).
+            force_ocr: If True, bypass PDF native-text analysis and force the
+                OCR engine (Marker) as primary. Used by the Phase 2 deferred
+                OCR pass for files that were already classified scanned or
+                that failed PyMuPDF extraction in Phase 1.
+            defer_pdf_fallback: If True, when a native-PDF PyMuPDF run
+                produces a poor result, return immediately with
+                `defer_to_ocr=True` instead of falling back to Marker inline.
+                Used by Phase 1 of `process_folder` to keep the fast pass
+                fast — the caller is expected to stage the file for Phase 2.
+
+        Returns:
+            ConversionResult with markdown content, score, and metadata.
+        """
+        file_path = Path(file_path).resolve()
+        start_time = time.time()
+
+        # Build file info
+        file_info = self._build_file_info(file_path)
+        if file_info is None:
+            return ConversionResult(
+                source_file=FileInfo(
+                    path=file_path, file_type="unknown",
+                    size_bytes=0, mime_type="unknown",
+                ),
+                error=f"Unsupported or undetectable file type: {file_path.suffix}",
+            )
+
+        logger.info(
+            f"Converting: [bold]{file_info.filename}[/bold] "
+            f"({file_info.file_type.upper()}, {format_file_size(file_info.size_bytes)})",
+            extra={"markup": True},
+        )
+
+        # Pre-convert legacy binary formats (.doc, .ppt) to modern .docx/.pptx
+        # via MS Office COM. The original FileInfo is kept so reports, moves,
+        # and the .md filename reflect the user's actual file.
+        original_file_info = file_info
+        working_info, temp_to_cleanup, preconvert_error = self._maybe_preconvert_legacy(file_info)
+
+        if preconvert_error is not None:
+            result = ConversionResult(
+                source_file=original_file_info,
+                error=preconvert_error,
+                engine_used="OfficeCOM",
+            )
+        else:
+            try:
+                result = self._convert_with_fallback(
+                    working_info,
+                    force_ocr=force_ocr,
+                    defer_pdf_fallback=defer_pdf_fallback,
+                )
+            finally:
+                if temp_to_cleanup is not None:
+                    try:
+                        temp_to_cleanup.unlink(missing_ok=True)
+                    except Exception as e:
+                        logger.debug(f"Temp cleanup failed for {temp_to_cleanup}: {e}")
+            # Restore original FileInfo so output/reporting reflect the user's file
+            result.source_file = original_file_info
+
+        result.processing_time_seconds = time.time() - start_time
+
+        # Log result
+        grade_color = {"A": "green", "B": "blue", "C": "yellow", "D": "red", "F": "bold red"}
+        color = grade_color.get(result.grade, "white")
+        if result.success:
+            logger.info(
+                f"Result: [{color}]{result.grade}[/{color}] "
+                f"({result.confidence_score}/100) — "
+                f"{result.engine_used} engine — "
+                f"{result.processing_time_seconds:.1f}s",
+                extra={"markup": True},
+            )
+        else:
+            logger.error(
+                f"FAILED: [{color}]{result.grade}[/{color}] "
+                f"({result.confidence_score}/100) — {result.error}",
+                extra={"markup": True},
+            )
+
+        if result.warnings:
+            for w in result.warnings:
+                logger.warning(f"  ⚠ {w}")
+
+        return result
+
+    def process_folder(
+        self,
+        input_dir: str | Path | None = None,
+        save_output: bool = True,
+        move_files: bool = True,
+    ) -> list[ConversionResult]:
+        """
+        Process all supported files in a folder.
+
+        Args:
+            input_dir: Path to the input folder. If None, opens a folder picker.
+            save_output: Whether to save .md files and reports to output/.
+            move_files: Whether to move originals to processed/failed folders.
+
+        Returns:
+            List of ConversionResult objects.
+        """
+        # Resolve input directory
+        if input_dir is None:
+            logger.info("Opening folder picker...")
+            input_dir = open_folder_picker("Select Input Folder for Warraqa")
+            if input_dir is None:
+                logger.warning("No folder selected. Aborting.")
+                return []
+        input_dir = Path(input_dir).resolve()
+
+        if not input_dir.is_dir():
+            logger.error(f"Not a valid directory: {input_dir}")
+            return []
+
+        # Find supported files
+        files = find_supported_files(input_dir)
+        if not files:
+            logger.info(f"No supported files found in: {input_dir}")
+            return []
+
+        logger.info(
+            f"Found [bold]{len(files)}[/bold] file(s) in: {input_dir}",
+            extra={"markup": True},
+        )
+
+        # --- Pre-pass: triage PDFs into native vs scanned ---
+        # Scanned PDFs are deferred to the end (OCR is slow); native PDFs and
+        # Word docs run first. Scanned PDFs are physically moved to the staging
+        # folder so they're out of the input set during phase 1.
+        fast_queue: list[Path] = []
+        scanned_queue: list[Path] = []
+        for file_path in files:
+            if file_path.suffix.lower() == ".pdf" and not PDFAnalyzer.is_native_text(file_path):
+                staged_path = safe_move(file_path, self._scanned_dir)
+                scanned_queue.append(staged_path)
+            else:
+                fast_queue.append(file_path)
+
+        if scanned_queue:
+            logger.info(
+                f"Triage: [bold]{len(fast_queue)}[/bold] fast file(s), "
+                f"[bold]{len(scanned_queue)}[/bold] scanned PDF(s) staged to "
+                f"[dim]{self._scanned_dir.name}/[/dim] for deferred OCR",
+                extra={"markup": True},
+            )
+
+        report = BatchReport(start_time=datetime.now())
+        deferred_in_phase1 = 0
+
+        def _run(
+            queue: list[Path],
+            phase_label: str,
+            *,
+            defer_pdf_fallback: bool = False,
+            force_ocr: bool = False,
+        ) -> None:
+            nonlocal deferred_in_phase1
+            for i, file_path in enumerate(queue, 1):
+                logger.info(f"\n{'='*60}")
+                logger.info(f"{phase_label} — file {i}/{len(queue)}")
+                logger.info(f"{'='*60}")
+
+                result = self.convert_file(
+                    file_path,
+                    force_ocr=force_ocr,
+                    defer_pdf_fallback=defer_pdf_fallback,
+                )
+
+                # If Phase 1 chose to defer this file to OCR, stage it and skip
+                # save/move/report — Phase 2 will produce the final result.
+                if result.defer_to_ocr:
+                    staged = safe_move(file_path, self._scanned_dir)
+                    scanned_queue.append(staged)
+                    deferred_in_phase1 += 1
+                    logger.info(
+                        f"  → Staged for Phase 2 OCR: "
+                        f"[dim]{self._scanned_dir.name}/{staged.name}[/dim]",
+                        extra={"markup": True},
+                    )
+                    continue
+
+                if save_output and result.markdown_content:
+                    self._save_output(result)
+
+                if move_files:
+                    self._move_original(result)
+
+                if save_output:
+                    self._save_report(result)
+
+                report.results.append(result)
+
+        # Phase 1: native PDFs + Word docs
+        # `defer_pdf_fallback=True` keeps the fast pass fast: any native PDF
+        # that PyMuPDF can't extract from is staged to Phase 2 instead of
+        # triggering an inline Marker call.
+        if fast_queue:
+            _run(fast_queue, "Phase 1 (fast)", defer_pdf_fallback=True)
+
+        # Phase 2: deferred OCR for staged scanned PDFs
+        # `force_ocr=True` skips the native-text re-analysis (it was either
+        # already classified scanned in triage, or got deferred from Phase 1
+        # because PyMuPDF couldn't extract text — both cases want Marker).
+        if scanned_queue:
+            logger.info(f"\n{'#'*60}")
+            logger.info(
+                f"Phase 2 (OCR): processing {len(scanned_queue)} scanned PDF(s) "
+                f"({deferred_in_phase1} deferred from Phase 1)"
+            )
+            logger.info(f"{'#'*60}")
+            _run(scanned_queue, "Phase 2 (OCR)", force_ocr=True)
+
+        report.end_time = datetime.now()
+
+        # Print batch summary
+        self._print_batch_summary(report)
+
+        return report.results
+
+    def watch_folder(
+        self,
+        input_dir: str | Path | None = None,
+        interval: int | None = None,
+    ) -> None:
+        """
+        Watch mode: continuously monitor a folder for new files.
+
+        Args:
+            input_dir: Folder to watch. Uses config if None.
+            interval: Poll interval in seconds. Uses config if None.
+        """
+        if input_dir is None:
+            input_dir = self._config["watch"].get("input_folder")
+            if input_dir is None:
+                logger.info("No watch folder configured. Opening picker...")
+                input_dir = open_folder_picker("Select Folder to Watch")
+                if input_dir is None:
+                    logger.warning("No folder selected. Aborting watch mode.")
+                    return
+
+        input_dir = Path(input_dir).resolve()
+        if interval is None:
+            interval = self._config["watch"].get("poll_interval_seconds", 10)
+
+        logger.info(
+            f"Watch mode: monitoring [bold]{input_dir}[/bold] "
+            f"every {interval}s (Ctrl+C to stop)",
+            extra={"markup": True},
+        )
+
+        processed_files: set[str] = set()
+
+        try:
+            while True:
+                files = find_supported_files(input_dir)
+                new_files = [
+                    f for f in files if str(f) not in processed_files
+                ]
+
+                if new_files:
+                    logger.info(f"Found {len(new_files)} new file(s)")
+                    for file_path in new_files:
+                        result = self.convert_file(file_path)
+                        if result.markdown_content:
+                            self._save_output(result)
+                        self._move_original(result)
+                        self._save_report(result)
+                        processed_files.add(str(file_path))
+
+                time.sleep(interval)
+
+        except KeyboardInterrupt:
+            logger.info("\nWatch mode stopped by user.")
+
+    # --- Internal Methods ---
+
+    def _load_config(self, config_path: str | Path | None) -> dict:
+        """Load configuration from YAML file, falling back to defaults."""
+        config = DEFAULT_CONFIG.copy()
+
+        if config_path is None:
+            # Look for config.yaml relative to this package
+            pkg_dir = Path(__file__).parent.parent
+            config_path = pkg_dir / "config.yaml"
+
+        config_path = Path(config_path)
+        if config_path.is_file():
+            try:
+                import yaml
+                with open(config_path, encoding="utf-8") as f:
+                    file_config = yaml.safe_load(f) or {}
+                config = self._deep_merge(config, file_config)
+            except Exception as e:
+                logger.warning(f"Failed to load config from {config_path}: {e}")
+
+        return config
+
+    def _deep_merge(self, base: dict, override: dict) -> dict:
+        """Deep merge override dict into base dict."""
+        result = base.copy()
+        for key, value in override.items():
+            if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+                result[key] = self._deep_merge(result[key], value)
+            else:
+                result[key] = value
+        return result
+
+    def _setup_logging(self) -> None:
+        """Configure logging from config."""
+        log_config = self._config.get("logging", {})
+        setup_logging(
+            level=log_config.get("level", "INFO"),
+            log_file=log_config.get("file"),
+        )
+
+    def _init_output_dirs(self) -> None:
+        """Create output directory structure."""
+        out = self._config["output"]
+        self._md_dir = self._output_base / out["md_files"]
+        self._processed_dir = self._output_base / out["processed"]
+        self._failed_dir = self._output_base / out["failed"]
+        self._reports_dir = self._output_base / out["reports"]
+        self._scanned_dir = self._output_base / out.get("scanned_pdfs", "scanned_pdfs")
+        ensure_directories(
+            self._md_dir, self._processed_dir, self._failed_dir,
+            self._reports_dir, self._scanned_dir,
+        )
+
+    def _init_engines(self) -> dict[str, dict[str, BaseEngine | None]]:
+        """Initialize conversion engines based on config."""
+        engine_config = self._config["engines"]
+
+        engine_map: dict[str, BaseEngine] = {
+            "marker": MarkerPDFEngine(),
+            "pymupdf": PyMuPDFEngine(),
+            "markitdown": MarkItDownEngine(),
+            "pandoc": PandocEngine(),
+        }
+
+        def _resolve(key: str, default: str | None) -> BaseEngine | None:
+            name = engine_config.get(key, default)
+            return engine_map.get(name) if name else None
+
+        engines = {
+            "pdf": {
+                "primary": _resolve("pdf_primary", "marker"),
+                "fallback": _resolve("pdf_fallback", "pymupdf"),
+            },
+            "docx": {
+                "primary": _resolve("docx_primary", "markitdown"),
+                "fallback": _resolve("docx_fallback", "pandoc"),
+            },
+            "pptx": {
+                "primary": _resolve("pptx_primary", "markitdown"),
+                # Pandoc cannot read .pptx — no native fallback by default.
+                "fallback": _resolve("pptx_fallback", None),
+            },
+        }
+        # Legacy formats route to their modern counterpart's engines —
+        # the OfficeCOMConverter rewrites file_info.file_type upstream,
+        # but we keep these aliases as a safety net.
+        engines["doc"] = engines["docx"]
+        engines["ppt"] = engines["pptx"]
+
+        # Log engine availability
+        for name, engine in engine_map.items():
+            status = "✓ available" if engine.is_available() else "✗ not installed"
+            logger.debug(f"Engine {name}: {status}")
+
+        return engines
+
+    def _build_file_info(self, file_path: Path) -> FileInfo | None:
+        """Build FileInfo from a file path. Returns None if unsupported."""
+        file_type = detect_file_type(file_path)
+        if file_type is None:
+            return None
+
+        return FileInfo(
+            path=file_path,
+            file_type=file_type,
+            size_bytes=file_path.stat().st_size,
+            mime_type=get_mime_type(file_path),
+        )
+
+    def _maybe_preconvert_legacy(
+        self, file_info: FileInfo
+    ) -> tuple[FileInfo, Path | None, str | None]:
+        """
+        Pre-convert legacy binary Office files (.doc, .ppt) to modern XML
+        (.docx, .pptx) via MS Office COM. Returns:
+            (working_file_info, temp_path_to_cleanup, error_message)
+
+        - For non-legacy formats: returns the original FileInfo unchanged.
+        - For legacy formats with COM available: returns a FileInfo pointing
+          at the temp converted file, plus the temp path for later cleanup.
+        - For legacy formats when COM is unavailable or conversion fails:
+          returns the original FileInfo plus a descriptive error message.
+          The caller should treat this as a hard failure rather than calling
+          the engines (MarkItDown and Pandoc both choke on the binary format).
+        """
+        ftype = file_info.file_type
+        if ftype not in ("doc", "ppt"):
+            return file_info, None, None
+
+        if not self._office_com.is_available():
+            return file_info, None, (
+                f"Legacy {ftype.upper()} format requires MS Office COM "
+                f"(pywin32 + Word/PowerPoint) — not available on this system"
+            )
+
+        try:
+            if ftype == "doc":
+                converted = self._office_com.convert_doc_to_docx(file_info.path)
+                new_type, new_mime = "docx", (
+                    "application/vnd.openxmlformats-officedocument."
+                    "wordprocessingml.document"
+                )
+            else:  # ppt
+                converted = self._office_com.convert_ppt_to_pptx(file_info.path)
+                new_type, new_mime = "pptx", (
+                    "application/vnd.openxmlformats-officedocument."
+                    "presentationml.presentation"
+                )
+        except Exception as e:
+            return file_info, None, f"Legacy {ftype.upper()} preconvert failed: {e}"
+
+        working = FileInfo(
+            path=converted,
+            file_type=new_type,
+            size_bytes=converted.stat().st_size,
+            mime_type=new_mime,
+            filename=file_info.filename,  # keep original filename for reporting
+        )
+        return working, converted, None
+
+    def _convert_with_fallback(
+        self,
+        file_info: FileInfo,
+        *,
+        force_ocr: bool = False,
+        defer_pdf_fallback: bool = False,
+    ) -> ConversionResult:
+        """
+        Try primary engine, fall back if failure or very low score.
+        Keeps the best result.
+
+        If `defer_pdf_fallback` and the primary PDF engine is PyMuPDF and it
+        produces a poor result, skip the inline Marker fallback and mark the
+        result with `defer_to_ocr=True` so the batch orchestrator can stage
+        the file for the Phase 2 OCR pass.
+        """
+        engine_set = self._engines.get(file_info.file_type)
+        if not engine_set:
+            return ConversionResult(
+                source_file=file_info,
+                error=f"No engines configured for file type: {file_info.file_type}",
+            )
+
+        primary: BaseEngine | None = engine_set.get("primary")
+        fallback: BaseEngine | None = engine_set.get("fallback")
+
+        # --- Pre-scan Document (Dynamic Engine Selection) ---
+        if file_info.file_type == "pdf":
+            if force_ocr:
+                logger.info("PDF: force_ocr=True — using Marker (OCR) as primary.")
+                primary = self._engines["pdf"]["primary"]   # Marker
+                fallback = self._engines["pdf"]["fallback"] # PyMuPDF
+            else:
+                is_native = PDFAnalyzer.is_native_text(file_info.path)
+                if is_native:
+                    logger.info(f"PDF is NATIVE TEXT. Using '{self._native_engine}' for speed/quality tradeoff.")
+                    if self._native_engine == "pymupdf":
+                        primary = self._engines["pdf"]["fallback"]  # PyMuPDF
+                        fallback = self._engines["pdf"]["primary"]  # Marker
+                    else:
+                        primary = self._engines["pdf"]["primary"]  # Marker
+                        fallback = self._engines["pdf"]["fallback"]  # PyMuPDF
+                else:
+                    logger.info("PDF is SCANNED IMAGE. Forcing 'marker' engine for OCR.")
+                    primary = self._engines["pdf"]["primary"]  # Marker
+                    fallback = self._engines["pdf"]["fallback"]  # PyMuPDF
+
+
+        # Try primary engine
+        result = self._try_engine(primary, file_info)
+
+        fail_threshold = self._config["scoring"]["fail_threshold"]
+        primary_poor = result.error or result.confidence_score < fail_threshold
+
+        # --- Phase 1 deferral: skip inline Marker fallback for poor PyMuPDF ---
+        # When called from the batch fast-pass, a poor native-PDF result should
+        # not stall Phase 1 by invoking the slow OCR engine. Stage it instead.
+        if (
+            defer_pdf_fallback
+            and file_info.file_type == "pdf"
+            and primary is self._engines["pdf"]["fallback"]  # primary was PyMuPDF
+            and primary_poor
+        ):
+            logger.info(
+                f"PyMuPDF gave a poor result "
+                f"(score={result.confidence_score}, words={len(result.markdown_content.split())}). "
+                f"Deferring to Phase 2 OCR instead of inline Marker fallback."
+            )
+            result.defer_to_ocr = True
+            return result
+
+        # If primary failed or scored very poorly, try fallback
+        if fallback and primary_poor:
+            logger.info(
+                f"Primary engine {'failed' if result.error else 'scored poorly'}, "
+                f"trying fallback: {fallback.name}"
+            )
+            fallback_result = self._try_engine(fallback, file_info)
+            fallback_result.fallback_used = True
+
+            # Keep the better result
+            if fallback_result.error is None and (
+                result.error or fallback_result.confidence_score > result.confidence_score
+            ):
+                result = fallback_result
+
+        return result
+
+    def _try_engine(self, engine: BaseEngine | None, file_info: FileInfo) -> ConversionResult:
+        """Attempt conversion with a single engine."""
+        if engine is None:
+            return ConversionResult(
+                source_file=file_info, error="No engine available"
+            )
+
+        if not engine.is_available():
+            return ConversionResult(
+                source_file=file_info,
+                error=f"Engine {engine.name} is not installed",
+                engine_used=engine.name,
+            )
+
+        try:
+            markdown_text, metadata = engine.convert(file_info.path)
+
+            # Strip unpaired surrogates so the result can actually be saved
+            # to a UTF-8 file. PyMuPDF (and others) occasionally emit them
+            # from PDFs with malformed text encodings or custom font cmaps.
+            markdown_text, surrogate_count = sanitize_markdown(markdown_text)
+
+            # Score the result
+            score, grade, warnings = self._scorer.score(
+                markdown_text, file_info, metadata
+            )
+
+            if surrogate_count > 0:
+                warnings.append(
+                    f"Replaced {surrogate_count} invalid Unicode surrogate(s) from engine output"
+                )
+                logger.debug(
+                    f"Sanitized {surrogate_count} surrogate(s) from {engine.name} output"
+                )
+
+            return ConversionResult(
+                source_file=file_info,
+                markdown_content=markdown_text,
+                confidence_score=score,
+                grade=grade,
+                engine_used=engine.name,
+                metadata=metadata,
+                warnings=warnings,
+            )
+
+        except Exception as e:
+            logger.error(f"Engine {engine.name} failed: {e}")
+            return ConversionResult(
+                source_file=file_info,
+                error=str(e),
+                engine_used=engine.name,
+            )
+
+    def _save_output(self, result: ConversionResult) -> None:
+        """Save the converted Markdown to output/md_files/."""
+        if not result.markdown_content:
+            return
+
+        filename = sanitize_filename(result.source_file.filename) + ".md"
+        output_path = self._md_dir / filename
+
+        # Handle collision
+        if output_path.exists():
+            stem = output_path.stem
+            counter = 1
+            while output_path.exists():
+                output_path = self._md_dir / f"{stem}_{counter}.md"
+                counter += 1
+
+        # `errors="replace"` is a safety net in case any odd code point slips
+        # past sanitize_markdown — one bad PDF should never crash the batch.
+        with open(output_path, "w", encoding="utf-8", errors="replace") as f:
+            f.write(result.markdown_content)
+        result.output_path = output_path
+        logger.debug(f"Saved: {output_path.name}")
+
+    def _move_original(self, result: ConversionResult) -> None:
+        """Move the original file to processed/ or failed/."""
+        src = result.source_file.path
+        if not src.exists():
+            logger.warning(f"Source file no longer exists: {src}")
+            return
+
+        dest_dir = self._processed_dir if result.success else self._failed_dir
+        safe_move(src, dest_dir)
+        logger.debug(f"Moved original to: {dest_dir.name}/")
+
+    def _save_report(self, result: ConversionResult) -> None:
+        """Save a JSON report for this conversion."""
+        report_name = sanitize_filename(result.source_file.filename) + ".json"
+        report_path = self._reports_dir / report_name
+
+        if report_path.exists():
+            stem = report_path.stem
+            counter = 1
+            while report_path.exists():
+                report_path = self._reports_dir / f"{stem}_{counter}.json"
+                counter += 1
+
+        # `errors="replace"` mirrors _save_output — one weird code point
+        # shouldn't ever take down report writing for a whole batch.
+        with open(report_path, "w", encoding="utf-8", errors="replace") as f:
+            json.dump(result.to_report_dict(), f, indent=2, ensure_ascii=False)
+
+    def _print_batch_summary(self, report: BatchReport) -> None:
+        """Print a beautiful batch summary to console."""
+        from rich.table import Table
+
+        summary = report.summary()
+
+        logger.info(f"\n{'='*60}")
+        logger.info("BATCH CONVERSION COMPLETE")
+        logger.info(f"{'='*60}")
+
+        table = Table(title="Warraqa — Conversion Summary")
+        table.add_column("File", style="cyan")
+        table.add_column("Type", style="dim")
+        table.add_column("Engine", style="blue")
+        table.add_column("Score", justify="center")
+        table.add_column("Grade", justify="center")
+        table.add_column("Time", justify="right", style="dim")
+        table.add_column("Status", justify="center")
+
+        grade_styles = {"A": "bold green", "B": "green", "C": "yellow", "D": "red", "F": "bold red"}
+
+        for r in report.results:
+            grade_style = grade_styles.get(r.grade, "white")
+            status = "✅" if r.success else "❌"
+            table.add_row(
+                r.source_file.filename,
+                r.source_file.file_type.upper(),
+                r.engine_used or "—",
+                str(r.confidence_score),
+                f"[{grade_style}]{r.grade}[/{grade_style}]",
+                f"{r.processing_time_seconds:.1f}s",
+                status,
+            )
+
+        from warraqa.utils import console
+        console.print(table)
+
+        # Stats
+        logger.info(
+            f"\nTotal: {summary['total_files']} | "
+            f"Success: {summary['successful']} | "
+            f"Failed: {summary['failed']} | "
+            f"Avg Score: {summary['average_score']} | "
+            f"Total Time: {summary['total_time_seconds']:.1f}s"
+        )
+        grades = summary["grade_distribution"]
+        logger.info(
+            f"Grades: A={grades['A']} B={grades['B']} C={grades['C']} "
+            f"D={grades['D']} F={grades['F']}"
+        )

warraqa/cli.py

@@ -0,0 +1,150 @@
+"""
+Warraqa CLI — Command-Line Entry Point
+
+Exposed as the `warraqa` console-script (see pyproject.toml). The repo-root
+`run.py` is a thin shim that imports `main` from here, so `python run.py`
+continues to work for users who clone the repo without installing.
+
+Usage:
+    warraqa                                # Manual mode: opens folder picker
+    warraqa --folder <path>                # Manual mode: process specific folder
+    warraqa --file <path>                  # Convert a single file
+    warraqa --watch [--folder <path>]      # Watch mode: continuously monitor
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import sys
+from pathlib import Path
+
+if sys.stdout.encoding and sys.stdout.encoding.lower() != "utf-8":
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
+
+def print_banner() -> None:
+    """Print the Warraqa startup banner."""
+    console = Console()
+    banner = Text()
+    banner.append("  ورّاقة  ", style="bold magenta")
+    banner.append("Warraqa", style="bold white")
+    banner.append(" — The Document Scribe\n", style="dim")
+    banner.append("  Converting documents to Markdown with precision", style="italic dim")
+
+    console.print(Panel(
+        banner,
+        border_style="magenta",
+        padding=(1, 2),
+        title="[bold]Document Scribe Agent[/bold]",
+        subtitle="v1.0.0",
+    ))
+    console.print()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="warraqa",
+        description="Warraqa (ورّاقة) — The Document Scribe Agent. "
+                    "Converts PDF, Word, and PowerPoint documents to clean Markdown.",
+    )
+    parser.add_argument(
+        "--file",
+        type=str,
+        help="Convert a single file to Markdown",
+    )
+    parser.add_argument(
+        "--folder",
+        type=str,
+        help="Process all supported files in a folder",
+    )
+    parser.add_argument(
+        "--watch",
+        action="store_true",
+        default=False,
+        help="Enable watch mode: continuously monitor for new files",
+    )
+    parser.add_argument(
+        "--interval",
+        type=int,
+        default=None,
+        help="Watch mode poll interval in seconds (default: 10)",
+    )
+    parser.add_argument(
+        "--config",
+        type=str,
+        default=None,
+        help="Path to config.yaml (default: auto-detect)",
+    )
+    parser.add_argument(
+        "--native-engine",
+        type=str,
+        choices=["pymupdf", "marker"],
+        default="pymupdf",
+        help="Engine to use for NATIVE text PDFs (default: pymupdf for speed; "
+             "'marker' for max quality).",
+    )
+    parser.add_argument(
+        "--no-move",
+        action="store_true",
+        default=False,
+        help="Don't move original files after processing",
+    )
+    parser.add_argument(
+        "--no-save",
+        action="store_true",
+        default=False,
+        help="Don't save output files (dry-run mode)",
+    )
+
+    args = parser.parse_args()
+
+    print_banner()
+
+    from warraqa.agent import Warraqa
+    agent = Warraqa(config_path=args.config, native_engine=args.native_engine)
+
+    # --- Single file mode ---
+    if args.file:
+        file_path = Path(args.file).resolve()
+        if not file_path.is_file():
+            Console().print(f"[red]Error: File not found: {file_path}[/red]")
+            sys.exit(1)
+
+        result = agent.convert_file(file_path)
+
+        if not args.no_save and result.markdown_content:
+            agent._save_output(result)
+            agent._save_report(result)
+
+        if not args.no_move:
+            agent._move_original(result)
+
+        sys.exit(0 if result.success else 1)
+
+    # --- Watch mode ---
+    if args.watch:
+        agent.watch_folder(
+            input_dir=args.folder,
+            interval=args.interval,
+        )
+        sys.exit(0)
+
+    # --- Batch/manual mode (default) ---
+    results = agent.process_folder(
+        input_dir=args.folder,
+        save_output=not args.no_save,
+        move_files=not args.no_move,
+    )
+
+    # Exit code: non-zero if any files failed
+    failed = sum(1 for r in results if not r.success)
+    sys.exit(1 if failed > 0 else 0)
+
+
+if __name__ == "__main__":
+    main()

warraqa/engines/__init__.py

@@ -0,0 +1,5 @@
+"""Warraqa conversion engines — specialized converters for each document format."""
+
+from warraqa.engines.base import BaseEngine  # noqa: F401
+
+__all__ = ["BaseEngine"]

warraqa/engines/base.py

@@ -0,0 +1,141 @@
+"""
+Base Engine Interface
+
+Abstract base class that all Warraqa conversion engines must implement.
+This ensures a consistent API across PDF and Word engines, enabling
+clean fallback chains and inter-engine substitution.
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("warraqa.engines")
+
+
+class BaseEngine(ABC):
+    """Abstract base class for all document conversion engines."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Human-readable engine name (e.g., 'Marker', 'PyMuPDF', 'MarkItDown')."""
+        ...
+
+    @property
+    @abstractmethod
+    def supported_types(self) -> list[str]:
+        """List of file types this engine supports (e.g., ['pdf'], ['docx', 'doc'])."""
+        ...
+
+    @abstractmethod
+    def convert(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """
+        Convert a document file to Markdown.
+
+        Args:
+            file_path: Path to the input document.
+
+        Returns:
+            A tuple of:
+                - markdown_text: The converted Markdown content as a string.
+                - metadata: Dictionary with engine-specific metadata, such as:
+                    - page_count: int
+                    - word_count: int
+                    - heading_count: int
+                    - table_count: int
+                    - images_detected: int
+                    - ocr_pages: int (for PDFs)
+                    - equations_detected: int (for PDFs)
+
+        Raises:
+            FileNotFoundError: If the input file does not exist.
+            ValueError: If the file type is not supported.
+            RuntimeError: If the conversion fails.
+        """
+        ...
+
+    def supports(self, file_type: str) -> bool:
+        """Check if this engine supports the given file type."""
+        return file_type.lower() in self.supported_types
+
+    def is_available(self) -> bool:
+        """
+        Check if this engine's dependencies are installed and ready.
+
+        Override this method in subclasses that have optional dependencies.
+        """
+        return True
+
+    def _validate_input(self, file_path: Path) -> None:
+        """Common input validation for all engines."""
+        if not file_path.exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+        if not file_path.is_file():
+            raise ValueError(f"Not a file: {file_path}")
+        if file_path.stat().st_size == 0:
+            raise ValueError(f"File is empty: {file_path}")
+
+    def _count_markdown_elements(self, markdown: str) -> dict[str, int]:
+        """
+        Count structural Markdown elements in the output.
+
+        Useful for engines that don't provide their own metadata about
+        the structure of the converted document.
+        """
+        lines = markdown.split("\n")
+        counts = {
+            "heading_count": 0,
+            "list_item_count": 0,
+            "table_row_count": 0,
+            "code_block_count": 0,
+            "link_count": 0,
+            "image_placeholder_count": 0,
+            "word_count": len(markdown.split()),
+        }
+
+        in_code_block = False
+        for line in lines:
+            stripped = line.strip()
+
+            # Code blocks
+            if stripped.startswith("```"):
+                in_code_block = not in_code_block
+                if not in_code_block:
+                    counts["code_block_count"] += 1
+                continue
+
+            if in_code_block:
+                continue
+
+            # Headings
+            if stripped.startswith("#"):
+                counts["heading_count"] += 1
+
+            # List items
+            if stripped.startswith(("- ", "* ", "+ ")) or (
+                len(stripped) > 2 and stripped[0].isdigit() and ". " in stripped[:5]
+            ):
+                counts["list_item_count"] += 1
+
+            # Table rows
+            if stripped.startswith("|") and stripped.endswith("|"):
+                # Skip separator rows like |---|---|
+                if not all(c in "|-: " for c in stripped):
+                    counts["table_row_count"] += 1
+
+            # Links
+            counts["link_count"] += stripped.count("](")
+
+            # Image placeholders
+            if stripped.startswith("!["):
+                counts["image_placeholder_count"] += 1
+
+        return counts
+
+    def __repr__(self) -> str:
+        available = "✓" if self.is_available() else "✗"
+        return f"<{self.name} Engine [{available}] types={self.supported_types}>"

warraqa/engines/docx_pandoc.py

@@ -0,0 +1,181 @@
+"""
+Pandoc Word Engine
+
+Fallback Word/DOCX conversion engine using Pandoc via subprocess.
+Pandoc is the gold-standard universal document converter, providing
+the most reliable handling of complex formatting edge cases.
+
+Requires Pandoc to be installed on the system.
+"""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from warraqa.engines.base import BaseEngine
+
+logger = logging.getLogger("warraqa.engines.pandoc")
+
+
+class PandocEngine(BaseEngine):
+    """Word/DOCX to Markdown converter using Pandoc (subprocess)."""
+
+    def __init__(self, pandoc_path: str | None = None):
+        """
+        Args:
+            pandoc_path: Explicit path to the pandoc executable.
+                         If None, uses system PATH.
+        """
+        self._pandoc_path = pandoc_path
+
+    @property
+    def name(self) -> str:
+        return "Pandoc"
+
+    @property
+    def supported_types(self) -> list[str]:
+        return ["docx", "doc"]
+
+    def _get_pandoc_cmd(self) -> str:
+        """Get the pandoc command path."""
+        if self._pandoc_path:
+            return self._pandoc_path
+        return "pandoc"
+
+    def is_available(self) -> bool:
+        """Check if Pandoc is installed and accessible."""
+        cmd = self._get_pandoc_cmd()
+
+        # Check explicit path
+        if self._pandoc_path:
+            return Path(self._pandoc_path).is_file()
+
+        # Check system PATH
+        if shutil.which(cmd):
+            return True
+
+        return False
+
+    def _get_pandoc_version(self) -> str | None:
+        """Get the installed Pandoc version."""
+        try:
+            result = subprocess.run(
+                [self._get_pandoc_cmd(), "--version"],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            if result.returncode == 0:
+                first_line = result.stdout.strip().split("\n")[0]
+                return first_line
+        except Exception:
+            pass
+        return None
+
+    def convert(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """
+        Convert a Word document to Markdown using Pandoc.
+
+        Runs Pandoc as a subprocess:
+            pandoc input.docx -t markdown --wrap=none
+
+        Args:
+            file_path: Path to the Word document.
+
+        Returns:
+            (markdown_text, metadata) tuple.
+        """
+        self._validate_input(file_path)
+
+        if not self.is_available():
+            raise RuntimeError(
+                "Pandoc is not installed or not found on PATH. "
+                "Install from: https://pandoc.org/installing.html"
+            )
+
+        logger.info(f"[Pandoc] Converting: {file_path.name}")
+
+        try:
+            cmd = [
+                self._get_pandoc_cmd(),
+                str(file_path),
+                "-t", "markdown",
+                "--wrap=none",           # Don't wrap long lines
+                "--extract-media=-",     # Don't extract media files
+            ]
+
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=120,  # 2-minute timeout for very large documents
+                encoding="utf-8",
+            )
+
+            if result.returncode != 0:
+                error_msg = result.stderr.strip() if result.stderr else "Unknown error"
+                raise RuntimeError(f"Pandoc exited with code {result.returncode}: {error_msg}")
+
+            markdown_text = result.stdout
+
+            if not markdown_text.strip():
+                logger.warning(f"[Pandoc] Empty output for {file_path.name}")
+
+            # Post-process: replace image references with alt-text placeholders
+            markdown_text = self._replace_images_with_placeholders(markdown_text)
+
+            # Build metadata
+            metadata: dict[str, Any] = {
+                "page_count": 0,
+                "images_detected": 0,
+                "tables_detected": 0,
+                "pandoc_version": self._get_pandoc_version(),
+            }
+
+            # Count structural elements
+            element_counts = self._count_markdown_elements(markdown_text)
+            metadata.update(element_counts)
+
+            metadata["tables_detected"] = markdown_text.count("|---|") + markdown_text.count("| ---")
+
+            logger.info(
+                f"[Pandoc] Success: {file_path.name} — "
+                f"{metadata['word_count']} words, "
+                f"{metadata['heading_count']} headings"
+            )
+
+            return markdown_text, metadata
+
+        except subprocess.TimeoutExpired as e:
+            logger.error(f"[Pandoc] Timeout for {file_path.name} (>120s)")
+            raise RuntimeError("Pandoc conversion timed out (>120 seconds)") from e
+        except Exception as e:
+            if isinstance(e, RuntimeError):
+                raise
+            logger.error(f"[Pandoc] Conversion failed for {file_path.name}: {e}")
+            raise RuntimeError(f"Pandoc conversion failed: {e}") from e
+
+    def _replace_images_with_placeholders(self, markdown: str) -> str:
+        """Replace any image references with alt-text placeholders."""
+        import re
+
+        image_count = 0
+
+        def _replace_match(match: re.Match) -> str:
+            nonlocal image_count
+            image_count += 1
+            alt_text = match.group(1).strip()
+            if alt_text and alt_text.lower() not in ("image", "img", "figure", ""):
+                return f"![Image: {alt_text}]()"
+            return f"![Image: embedded image #{image_count} in document]()"
+
+        markdown = re.sub(
+            r"!\[([^\]]*)\]\([^\)]+\)",
+            _replace_match,
+            markdown,
+        )
+        return markdown

warraqa/engines/office_com.py

@@ -0,0 +1,194 @@
+"""
+Office COM Pre-Converter
+
+Uses Microsoft Word and PowerPoint via pywin32 COM automation to convert
+legacy binary Office formats (.doc, .ppt) into their modern XML
+equivalents (.docx, .pptx). Required because MarkItDown and Pandoc cannot
+read the 1990s-era binary formats directly.
+
+Windows + Microsoft Office required. The Office applications are launched
+invisible and kept alive for the duration of the Warraqa session — opening
+Word for every file would be very slow in a large batch.
+"""
+
+from __future__ import annotations
+
+import atexit
+import logging
+import os
+import tempfile
+from pathlib import Path
+
+logger = logging.getLogger("warraqa.engines.office_com")
+
+# Word file format constants (https://docs.microsoft.com/office/vba/api/word.wdsaveformat)
+WD_FORMAT_XML_DOCUMENT = 12   # .docx
+
+# PowerPoint file format constants (https://docs.microsoft.com/office/vba/api/powerpoint.ppsaveasfiletype)
+PP_SAVE_AS_OPEN_XML_PRESENTATION = 24  # .pptx
+
+
+class OfficeCOMConverter:
+    """Legacy → modern Office file pre-converter via MS Office COM."""
+
+    def __init__(self) -> None:
+        self._word = None
+        self._powerpoint = None
+        self._com_initialized = False
+        self._available = self._check_available()
+        if self._available:
+            atexit.register(self.close)
+
+    @staticmethod
+    def _check_available() -> bool:
+        """Verify pywin32 is importable. Does not start any COM app."""
+        try:
+            import win32com.client  # noqa: F401
+            return True
+        except ImportError:
+            return False
+
+    def is_available(self) -> bool:
+        return self._available
+
+    # --- Public conversion API ---
+
+    def convert_doc_to_docx(self, src: Path) -> Path:
+        """Convert a binary .doc to a .docx in a temp file. Returns the temp path."""
+        if not src.is_file():
+            raise FileNotFoundError(src)
+        word = self._ensure_word()
+        out = self._make_temp_path(src.stem, ".docx")
+
+        logger.info(f"[OfficeCOM] Converting .doc → .docx: {src.name}")
+        doc = None
+        try:
+            doc = word.Documents.Open(
+                str(src.resolve()),
+                ReadOnly=True,
+                ConfirmConversions=False,
+                AddToRecentFiles=False,
+            )
+            doc.SaveAs2(str(out.resolve()), FileFormat=WD_FORMAT_XML_DOCUMENT)
+        except Exception as e:
+            raise RuntimeError(f"Word COM failed on {src.name}: {e}") from e
+        finally:
+            if doc is not None:
+                try:
+                    doc.Close(SaveChanges=False)
+                except Exception:
+                    pass
+
+        if not out.is_file() or out.stat().st_size == 0:
+            raise RuntimeError(f"Word produced no output for {src.name}")
+        return out
+
+    def convert_ppt_to_pptx(self, src: Path) -> Path:
+        """Convert a binary .ppt to a .pptx in a temp file. Returns the temp path."""
+        if not src.is_file():
+            raise FileNotFoundError(src)
+        ppt = self._ensure_powerpoint()
+        out = self._make_temp_path(src.stem, ".pptx")
+
+        logger.info(f"[OfficeCOM] Converting .ppt → .pptx: {src.name}")
+        pres = None
+        try:
+            pres = ppt.Presentations.Open(
+                str(src.resolve()),
+                ReadOnly=True,
+                WithWindow=False,
+            )
+            pres.SaveAs(
+                str(out.resolve()),
+                PP_SAVE_AS_OPEN_XML_PRESENTATION,
+            )
+        except Exception as e:
+            raise RuntimeError(f"PowerPoint COM failed on {src.name}: {e}") from e
+        finally:
+            if pres is not None:
+                try:
+                    pres.Close()
+                except Exception:
+                    pass
+
+        if not out.is_file() or out.stat().st_size == 0:
+            raise RuntimeError(f"PowerPoint produced no output for {src.name}")
+        return out
+
+    def close(self) -> None:
+        """Shutdown Word and PowerPoint. Safe to call multiple times."""
+        if self._word is not None:
+            try:
+                self._word.Quit()
+                logger.debug("[OfficeCOM] Word closed")
+            except Exception:
+                pass
+            self._word = None
+        if self._powerpoint is not None:
+            try:
+                self._powerpoint.Quit()
+                logger.debug("[OfficeCOM] PowerPoint closed")
+            except Exception:
+                pass
+            self._powerpoint = None
+        if self._com_initialized:
+            try:
+                import pythoncom
+                pythoncom.CoUninitialize()
+            except Exception:
+                pass
+            self._com_initialized = False
+
+    # --- Internals ---
+
+    def _ensure_com_init(self) -> None:
+        if self._com_initialized:
+            return
+        import pythoncom
+        pythoncom.CoInitialize()
+        self._com_initialized = True
+
+    def _ensure_word(self):
+        if self._word is not None:
+            return self._word
+        if not self._available:
+            raise RuntimeError("pywin32 is not installed — cannot use Office COM")
+        self._ensure_com_init()
+        import win32com.client
+        try:
+            self._word = win32com.client.Dispatch("Word.Application")
+            self._word.Visible = False
+            self._word.DisplayAlerts = 0  # wdAlertsNone
+            logger.info(f"[OfficeCOM] Word {self._word.Version} initialized")
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to start Word via COM (is Microsoft Word installed?): {e}"
+            ) from e
+        return self._word
+
+    def _ensure_powerpoint(self):
+        if self._powerpoint is not None:
+            return self._powerpoint
+        if not self._available:
+            raise RuntimeError("pywin32 is not installed — cannot use Office COM")
+        self._ensure_com_init()
+        import win32com.client
+        try:
+            self._powerpoint = win32com.client.Dispatch("PowerPoint.Application")
+            # PowerPoint's Visible property is read-only in newer versions
+            # when there's no UI thread; just leave default.
+            logger.info(
+                f"[OfficeCOM] PowerPoint {self._powerpoint.Version} initialized"
+            )
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to start PowerPoint via COM (is PowerPoint installed?): {e}"
+            ) from e
+        return self._powerpoint
+
+    @staticmethod
+    def _make_temp_path(stem: str, suffix: str) -> Path:
+        """Generate a unique temp file path. Caller is responsible for cleanup."""
+        safe_stem = "".join(c if c.isalnum() or c in "_-" else "_" for c in stem)[:60]
+        token = os.urandom(6).hex()
+        return Path(tempfile.gettempdir()) / f"warraqa_{token}_{safe_stem}{suffix}"

warraqa/engines/office_markitdown.py

@@ -0,0 +1,144 @@
+"""
+MarkItDown Office Engine
+
+Primary Office document conversion engine using Microsoft's MarkItDown
+library. Handles modern XML-based Office formats: .docx and .pptx. Legacy
+.doc and .ppt are pre-converted to the modern format by the
+OfficeCOMConverter before this engine runs.
+
+Provides clean preservation of headings, lists, tables, links, slides,
+speaker notes, and embedded content.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from warraqa.engines.base import BaseEngine
+
+logger = logging.getLogger("warraqa.engines.markitdown")
+
+
+class MarkItDownEngine(BaseEngine):
+    """Office (DOCX/PPTX) to Markdown converter using Microsoft's MarkItDown."""
+
+    def __init__(self):
+        self._converter = None
+
+    @property
+    def name(self) -> str:
+        return "MarkItDown"
+
+    @property
+    def supported_types(self) -> list[str]:
+        # Modern formats only. Legacy .doc / .ppt are normalized upstream
+        # by the OfficeCOMConverter into .docx / .pptx before reaching here.
+        return ["docx", "pptx"]
+
+    def is_available(self) -> bool:
+        """Check if markitdown is installed."""
+        try:
+            from markitdown import MarkItDown  # noqa: F401
+            return True
+        except ImportError:
+            return False
+
+    def _ensure_initialized(self) -> None:
+        """Lazy-initialize the MarkItDown converter."""
+        if self._converter is not None:
+            return
+
+        if not self.is_available():
+            raise RuntimeError(
+                "MarkItDown is not installed. Run: pip install 'markitdown[all]'"
+            )
+
+        from markitdown import MarkItDown
+        self._converter = MarkItDown()
+        logger.debug("MarkItDown engine initialized")
+
+    def convert(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """
+        Convert an Office document (.docx or .pptx) to Markdown using
+        MarkItDown.
+
+        Legacy .doc / .ppt files should be pre-converted to their modern
+        equivalents by the OfficeCOMConverter before reaching this engine
+        — MarkItDown's bundled converters do not parse the binary formats.
+
+        Args:
+            file_path: Path to the Office document.
+
+        Returns:
+            (markdown_text, metadata) tuple.
+        """
+        self._validate_input(file_path)
+        self._ensure_initialized()
+
+        logger.info(f"[MarkItDown] Converting: {file_path.name}")
+
+        try:
+            # Perform the conversion
+            result = self._converter.convert(str(file_path))
+
+            # MarkItDown returns result.text_content with the markdown
+            markdown_text = result.text_content if result.text_content else ""
+
+            if not markdown_text.strip():
+                logger.warning(
+                    f"[MarkItDown] Empty output for {file_path.name}"
+                )
+
+            # Post-process: replace image references with alt-text placeholders
+            markdown_text = self._replace_images_with_placeholders(markdown_text)
+
+            # Build metadata
+            metadata: dict[str, Any] = {
+                "page_count": 0,  # Word docs don't have fixed page counts
+                "images_detected": 0,
+                "tables_detected": 0,
+            }
+
+            # Count structural elements
+            element_counts = self._count_markdown_elements(markdown_text)
+            metadata.update(element_counts)
+
+            # Estimate table count from markdown
+            metadata["tables_detected"] = markdown_text.count("|---|") + markdown_text.count("| ---")
+
+            logger.info(
+                f"[MarkItDown] Success: {file_path.name} — "
+                f"{metadata['word_count']} words, "
+                f"{metadata['heading_count']} headings"
+            )
+
+            return markdown_text, metadata
+
+        except Exception as e:
+            logger.error(
+                f"[MarkItDown] Conversion failed for {file_path.name}: {e}"
+            )
+            raise RuntimeError(f"MarkItDown conversion failed: {e}") from e
+
+    def _replace_images_with_placeholders(self, markdown: str) -> str:
+        """Replace any image references with alt-text placeholders."""
+        import re
+
+        image_count = 0
+
+        def _replace_match(match: re.Match) -> str:
+            nonlocal image_count
+            image_count += 1
+            alt_text = match.group(1).strip()
+            if alt_text and alt_text.lower() not in ("image", "img", "figure", ""):
+                return f"![Image: {alt_text}]()"
+            return f"![Image: embedded image #{image_count} in document]()"
+
+        markdown = re.sub(
+            r"!\[([^\]]*)\]\([^\)]+\)",
+            _replace_match,
+            markdown,
+        )
+        return markdown

warraqa/engines/pdf_marker.py

@@ -0,0 +1,179 @@
+"""
+Marker PDF Engine
+
+Primary PDF conversion engine using the Marker library — a deep learning
+pipeline that handles complex layouts, tables, equations, multi-column
+formats, headers/footers, and image detection.
+
+Marker's purpose-built ML models outperform general-purpose vision LLMs
+for structural document extraction.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from warraqa.engines.base import BaseEngine
+
+logger = logging.getLogger("warraqa.engines.marker")
+
+
+class MarkerPDFEngine(BaseEngine):
+    """High-fidelity PDF to Markdown converter using Marker."""
+
+    def __init__(self):
+        self._converter = None
+        self._initialized = False
+
+    @property
+    def name(self) -> str:
+        return "Marker"
+
+    @property
+    def supported_types(self) -> list[str]:
+        return ["pdf"]
+
+    def is_available(self) -> bool:
+        """Check if marker-pdf is installed."""
+        try:
+            import marker  # noqa: F401
+            return True
+        except ImportError:
+            return False
+
+    def _ensure_initialized(self) -> None:
+        """
+        Lazy-initialize the Marker converter.
+
+        Marker loads several large ML models into memory on first use.
+        We do this lazily to avoid slow startup when the engine isn't needed,
+        but cache the converter so models are loaded only once.
+        """
+        if self._initialized:
+            return
+
+        if not self.is_available():
+            raise RuntimeError(
+                "Marker is not installed. Run: pip install marker-pdf"
+            )
+
+        try:
+            from marker.converters.pdf import PdfConverter
+            from marker.models import create_model_dict
+
+            # Initialize Marker ML models (downloads if first run)
+            logger.info("Loading Marker ML models (this may take a few minutes on first run)...")
+            models = create_model_dict()
+            
+            self._converter = PdfConverter(artifact_dict=models)
+            self._initialized = True
+            logger.info("Marker engine initialized (models loaded)")
+        except Exception as e:
+            logger.error(f"Failed to initialize Marker engine: {e}")
+            raise RuntimeError(f"Marker initialization failed: {e}") from e
+
+    def convert(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """
+        Convert a PDF file to Markdown using Marker.
+
+        Args:
+            file_path: Path to the PDF file.
+
+        Returns:
+            (markdown_text, metadata) tuple.
+        """
+        self._validate_input(file_path)
+        self._ensure_initialized()
+
+        logger.info(f"[Marker] Converting: {file_path.name}")
+
+        try:
+            # Run the conversion
+            rendered = self._converter(str(file_path))
+
+            # Extract the markdown text from the rendered output
+            markdown_text = rendered.markdown
+
+            import fitz
+            try:
+                with fitz.open(str(file_path)) as doc:
+                    page_count = len(doc)
+            except Exception:
+                page_count = 0
+
+            # Build metadata from conversion results
+            metadata: dict[str, Any] = {
+                "page_count": page_count,
+                "images_detected": 0,
+                "tables_detected": 0,
+                "equations_detected": 0,
+            }
+
+            # Count images from rendered output
+            if hasattr(rendered, 'images') and rendered.images:
+                metadata["images_detected"] = len(rendered.images)
+
+            # Post-process: replace image references with alt-text placeholders
+            markdown_text = self._replace_images_with_placeholders(
+                markdown_text, metadata
+            )
+
+            # Count structural elements from the markdown itself
+            element_counts = self._count_markdown_elements(markdown_text)
+            metadata.update(element_counts)
+
+            # Count tables and equations from the markdown
+            metadata["tables_detected"] = markdown_text.count("|---|") + markdown_text.count("| ---")
+            metadata["equations_detected"] = (
+                markdown_text.count("$$") // 2 + markdown_text.count("\\(")
+            )
+
+            logger.info(
+                f"[Marker] Success: {file_path.name} — "
+                f"{metadata['word_count']} words, "
+                f"{metadata['page_count']} pages"
+            )
+
+            return markdown_text, metadata
+
+        except Exception as e:
+            logger.error(f"[Marker] Conversion failed for {file_path.name}: {e}")
+            raise RuntimeError(f"Marker conversion failed: {e}") from e
+
+    def _replace_images_with_placeholders(
+        self, markdown: str, metadata: dict
+    ) -> str:
+        """
+        Replace image references with descriptive alt-text placeholders.
+
+        Converts patterns like:
+            ![](image_path.png)
+            ![some alt text](image_path.png)
+        To:
+            ![Image: figure/diagram detected on this page]()
+        """
+        import re
+
+        image_count = 0
+
+        def _replace_match(match: re.Match) -> str:
+            nonlocal image_count
+            image_count += 1
+            alt_text = match.group(1).strip()
+            if alt_text and alt_text.lower() not in ("image", "img", "figure"):
+                # Keep meaningful alt text
+                return f"![Image: {alt_text}]()"
+            return f"![Image: figure/diagram #{image_count} detected in document]()"
+
+        # Match markdown image syntax: ![alt text](path)
+        markdown = re.sub(
+            r"!\[([^\]]*)\]\([^\)]+\)",
+            _replace_match,
+            markdown,
+        )
+
+        metadata["images_detected"] = max(metadata.get("images_detected", 0), image_count)
+
+        return markdown

warraqa/engines/pdf_pymupdf.py

@@ -0,0 +1,123 @@
+"""
+PyMuPDF4LLM PDF Engine
+
+Fallback PDF conversion engine — fast, lightweight, CPU-only.
+Uses PyMuPDF's LLM-optimized markdown export for text-heavy documents.
+No ML model loading means instant startup and minimal resource usage.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from warraqa.engines.base import BaseEngine
+
+logger = logging.getLogger("warraqa.engines.pymupdf")
+
+
+class PyMuPDFEngine(BaseEngine):
+    """Fast, lightweight PDF to Markdown converter using PyMuPDF4LLM."""
+
+    @property
+    def name(self) -> str:
+        return "PyMuPDF"
+
+    @property
+    def supported_types(self) -> list[str]:
+        return ["pdf"]
+
+    def is_available(self) -> bool:
+        """Check if pymupdf4llm is installed."""
+        try:
+            import pymupdf4llm  # noqa: F401
+            return True
+        except ImportError:
+            return False
+
+    def convert(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """
+        Convert a PDF file to Markdown using PyMuPDF4LLM.
+
+        This is a fast, single-function conversion that works best on
+        text-heavy, simple-layout PDFs. It does not use ML models.
+
+        Args:
+            file_path: Path to the PDF file.
+
+        Returns:
+            (markdown_text, metadata) tuple.
+        """
+        self._validate_input(file_path)
+
+        if not self.is_available():
+            raise RuntimeError(
+                "PyMuPDF4LLM is not installed. Run: pip install pymupdf4llm"
+            )
+
+        logger.info(f"[PyMuPDF] Converting: {file_path.name}")
+
+        try:
+            import pymupdf  # fitz
+            import pymupdf4llm
+
+            # Get page count first
+            doc = pymupdf.open(str(file_path))
+            page_count = len(doc)
+            doc.close()
+
+            # Convert to markdown — single function call
+            markdown_text = pymupdf4llm.to_markdown(
+                str(file_path),
+                write_images=False,  # We use alt-text placeholders, not image files
+                show_progress=False,
+            )
+
+            # Post-process: replace any image references with placeholders
+            markdown_text = self._replace_images_with_placeholders(markdown_text)
+
+            # Build metadata
+            metadata: dict[str, Any] = {
+                "page_count": page_count,
+                "images_detected": 0,
+                "tables_detected": 0,
+                "equations_detected": 0,
+            }
+
+            # Count structural elements
+            element_counts = self._count_markdown_elements(markdown_text)
+            metadata.update(element_counts)
+
+            logger.info(
+                f"[PyMuPDF] Success: {file_path.name} — "
+                f"{metadata['word_count']} words, "
+                f"{page_count} pages"
+            )
+
+            return markdown_text, metadata
+
+        except Exception as e:
+            logger.error(f"[PyMuPDF] Conversion failed for {file_path.name}: {e}")
+            raise RuntimeError(f"PyMuPDF conversion failed: {e}") from e
+
+    def _replace_images_with_placeholders(self, markdown: str) -> str:
+        """Replace any image references with alt-text placeholders."""
+        import re
+
+        image_count = 0
+
+        def _replace_match(match: re.Match) -> str:
+            nonlocal image_count
+            image_count += 1
+            alt_text = match.group(1).strip()
+            if alt_text and alt_text.lower() not in ("image", "img", "figure", ""):
+                return f"![Image: {alt_text}]()"
+            return f"![Image: figure/diagram #{image_count} detected in document]()"
+
+        markdown = re.sub(
+            r"!\[([^\]]*)\]\([^\)]+\)",
+            _replace_match,
+            markdown,
+        )
+        return markdown

warraqa/models.py

@@ -0,0 +1,158 @@
+"""
+Warraqa Data Models
+
+Core data structures used throughout the Warraqa agent for representing
+file information, conversion results, and batch reports.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class FileInfo:
+    """Metadata about an input file to be converted."""
+
+    path: Path
+    file_type: str  # "pdf", "docx", "doc"
+    size_bytes: int
+    mime_type: str
+    filename: str = ""
+
+    def __post_init__(self):
+        self.path = Path(self.path)
+        if not self.filename:
+            self.filename = self.path.name
+
+
+@dataclass
+class ConversionResult:
+    """Complete result of a single file conversion."""
+
+    # Source information
+    source_file: FileInfo
+
+    # Output
+    output_path: Path | None = None
+    markdown_content: str = ""
+
+    # Quality assessment
+    confidence_score: int = 0
+    grade: str = "F"
+
+    # Engine details
+    engine_used: str = ""
+    fallback_used: bool = False
+    defer_to_ocr: bool = False  # Phase-1 batch flag: poor PyMuPDF result, defer to Phase 2 OCR
+
+    # Performance
+    processing_time_seconds: float = 0.0
+
+    # Issues
+    warnings: list[str] = field(default_factory=list)
+    error: str | None = None
+
+    # Metadata from the engine
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    # Timestamp
+    timestamp: datetime = field(default_factory=datetime.now)
+
+    @property
+    def success(self) -> bool:
+        """Whether the conversion was successful (not failed)."""
+        return self.error is None and self.confidence_score >= 40
+
+    @property
+    def moved_to(self) -> str:
+        """Where the original file was moved to."""
+        return "processed" if self.success else "failed"
+
+    def to_report_dict(self) -> dict[str, Any]:
+        """Serialize to a JSON-compatible dictionary for reporting."""
+        return {
+            "agent": "Warraqa v1.0.0",
+            "timestamp": self.timestamp.isoformat(),
+            "source": {
+                "filename": self.source_file.filename,
+                "path": str(self.source_file.path),
+                "type": self.source_file.file_type,
+                "size_bytes": self.source_file.size_bytes,
+                "mime_type": self.source_file.mime_type,
+            },
+            "result": {
+                "output_path": str(self.output_path) if self.output_path else None,
+                "engine_used": self.engine_used,
+                "fallback_used": self.fallback_used,
+                "confidence_score": self.confidence_score,
+                "grade": self.grade,
+                "processing_time_seconds": round(self.processing_time_seconds, 2),
+                "warnings": self.warnings,
+                "error": self.error,
+                "metadata": self.metadata,
+            },
+            "moved_to": self.moved_to,
+        }
+
+
+@dataclass
+class BatchReport:
+    """Summary of a batch conversion run."""
+
+    results: list[ConversionResult] = field(default_factory=list)
+    start_time: datetime = field(default_factory=datetime.now)
+    end_time: datetime | None = None
+
+    @property
+    def total_files(self) -> int:
+        return len(self.results)
+
+    @property
+    def successful(self) -> int:
+        return sum(1 for r in self.results if r.success)
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for r in self.results if not r.success)
+
+    @property
+    def average_score(self) -> float:
+        if not self.results:
+            return 0.0
+        return sum(r.confidence_score for r in self.results) / len(self.results)
+
+    @property
+    def average_time(self) -> float:
+        if not self.results:
+            return 0.0
+        return sum(r.processing_time_seconds for r in self.results) / len(self.results)
+
+    @property
+    def total_time_seconds(self) -> float:
+        if self.end_time and self.start_time:
+            return (self.end_time - self.start_time).total_seconds()
+        return sum(r.processing_time_seconds for r in self.results)
+
+    @property
+    def grade_distribution(self) -> dict[str, int]:
+        dist: dict[str, int] = {"A": 0, "B": 0, "C": 0, "D": 0, "F": 0}
+        for r in self.results:
+            if r.grade in dist:
+                dist[r.grade] += 1
+        return dist
+
+    def summary(self) -> dict[str, Any]:
+        """Generate a summary dictionary for the batch."""
+        return {
+            "total_files": self.total_files,
+            "successful": self.successful,
+            "failed": self.failed,
+            "average_score": round(self.average_score, 1),
+            "average_time_seconds": round(self.average_time, 2),
+            "total_time_seconds": round(self.total_time_seconds, 2),
+            "grade_distribution": self.grade_distribution,
+        }

warraqa/scoring/__init__.py

@@ -0,0 +1,5 @@
+"""Warraqa quality scoring — multi-dimensional conversion quality assessment."""
+
+from warraqa.scoring.scorer import ConversionScorer  # noqa: F401
+
+__all__ = ["ConversionScorer"]

warraqa/scoring/scorer.py

@@ -0,0 +1,276 @@
+"""
+Warraqa Conversion Scorer
+
+Multi-dimensional quality assessment engine that evaluates conversions
+across 5 weighted dimensions: completeness, structure, encoding, density, readability.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import unicodedata
+from collections import Counter
+from typing import Any
+
+from warraqa.models import FileInfo
+
+logger = logging.getLogger("warraqa.scoring")
+
+EXPECTED_WORDS_PER_KB = {"pdf": 40.0, "docx": 80.0, "doc": 30.0}
+
+GARBLED_PATTERNS = [
+    r"[□■◆◇►◄▲▼●○★☆♠♣♥♦]{3,}",
+    r"[\ufffd]{2,}",
+    r"[^\x00-\x7F]{10,}",
+    r"(?:[\x80-\x9f]){3,}",
+]
+
+GRADE_MAP = [(90, "A"), (75, "B"), (60, "C"), (40, "D"), (0, "F")]
+
+
+class ConversionScorer:
+    """Scores document-to-Markdown conversions on a 0-100 scale with A-F grades."""
+
+    def __init__(self, fail_threshold: int = 40, warn_threshold: int = 70):
+        self.fail_threshold = fail_threshold
+        self.warn_threshold = warn_threshold
+
+    def score(
+        self, markdown: str, source_info: FileInfo, metadata: dict[str, Any] | None = None
+    ) -> tuple[int, str, list[str]]:
+        """Score a conversion. Returns (score, grade, warnings)."""
+        if metadata is None:
+            metadata = {}
+        warnings: list[str] = []
+
+        if not markdown or not markdown.strip():
+            warnings.append("Conversion produced empty output")
+            return 0, "F", warnings
+
+        completeness = self._score_text_completeness(markdown, source_info, metadata, warnings)
+        structure = self._score_structure_integrity(markdown, metadata, warnings)
+        encoding = self._score_encoding_quality(markdown, warnings)
+        density = self._score_content_density(markdown, warnings)
+        readability = self._score_readability(markdown, warnings)
+
+        composite = (
+            completeness * 0.30 + structure * 0.25 + encoding * 0.20
+            + density * 0.15 + readability * 0.10
+        )
+        final_score = max(0, min(100, round(composite)))
+        grade = self._score_to_grade(final_score)
+
+        if final_score < self.fail_threshold:
+            warnings.insert(0, f"Score {final_score} below fail threshold ({self.fail_threshold})")
+        elif final_score < self.warn_threshold:
+            warnings.insert(0, f"Score {final_score} below warning threshold ({self.warn_threshold})")
+
+        logger.debug(
+            f"Scoring: Completeness={completeness:.0f} Structure={structure:.0f} "
+            f"Encoding={encoding:.0f} Density={density:.0f} Readability={readability:.0f} "
+            f"→ {final_score} ({grade})"
+        )
+        return final_score, grade, warnings
+
+    def _score_text_completeness(
+        self, markdown: str, source_info: FileInfo, metadata: dict, warnings: list[str]
+    ) -> float:
+        """Text Completeness (30%): word count vs expected density for file size."""
+        word_count = len(markdown.split())
+        if word_count < 10:
+            warnings.append(f"Extremely short output: only {word_count} words")
+            return 5.0
+        if word_count < 50:
+            warnings.append(f"Very short output: only {word_count} words")
+            return 20.0
+
+        file_size_kb = max(source_info.size_bytes / 1024, 0.001)
+        expected_density = EXPECTED_WORDS_PER_KB.get(source_info.file_type, 40.0)
+        expected_words = max(file_size_kb * expected_density, 1)
+        ratio = word_count / expected_words
+
+        if ratio >= 0.8:
+            return 100.0
+        elif ratio >= 0.5:
+            return 70.0 + (ratio - 0.5) * 100.0
+        elif ratio >= 0.2:
+            return 30.0 + (ratio - 0.2) * 133.3
+        elif ratio >= 0.05:
+            warnings.append(
+                f"Word count ({word_count}) low for file size ({file_size_kb:.0f} KB)"
+            )
+            return 10.0 + (ratio - 0.05) * 133.3
+        else:
+            warnings.append(
+                f"Word count ({word_count}) very low for file size ({file_size_kb:.0f} KB)"
+            )
+            return ratio * 200.0
+
+    def _score_structure_integrity(
+        self, markdown: str, metadata: dict, warnings: list[str]
+    ) -> float:
+        """Structure Integrity (25%): presence of headings, lists, tables, formatting."""
+        lines = markdown.split("\n")
+        total_lines = len(lines)
+        if total_lines == 0:
+            return 0.0
+
+        heading_count = sum(1 for ln in lines if ln.strip().startswith("#"))
+        list_count = sum(
+            1 for ln in lines
+            if ln.strip().startswith(("- ", "* ", "+ "))
+            or (len(ln.strip()) > 2 and ln.strip()[0].isdigit() and ". " in ln.strip()[:5])
+        )
+        table_rows = sum(
+            1 for ln in lines
+            if ln.strip().startswith("|") and ln.strip().endswith("|")
+        )
+        bold_italic = len(re.findall(r"\*\*[^*]+\*\*|\*[^*]+\*|__[^_]+__|_[^_]+_", markdown))
+        code_blocks = markdown.count("```")
+
+        structure_elements = heading_count + list_count + table_rows + bold_italic + code_blocks
+        word_count = len(markdown.split())
+
+        if word_count < 50:
+            return 80.0 if structure_elements > 0 else 50.0
+
+        structure_ratio = (structure_elements / word_count) * 100
+        if structure_ratio >= 2.0:
+            score = 100.0
+        elif structure_ratio >= 1.0:
+            score = 80.0
+        elif structure_ratio >= 0.5:
+            score = 65.0
+        elif structure_ratio >= 0.1:
+            score = 45.0
+        else:
+            score = 25.0
+            if word_count > 200:
+                warnings.append("Output is a wall of plain text with minimal formatting")
+
+        if heading_count >= 3:
+            score = min(100.0, score + 10.0)
+        elif heading_count >= 1:
+            score = min(100.0, score + 5.0)
+        return score
+
+    def _score_encoding_quality(self, markdown: str, warnings: list[str]) -> float:
+        """Encoding Quality (20%): garbled chars, mojibake, broken Unicode."""
+        total_chars = len(markdown)
+        if total_chars == 0:
+            return 0.0
+
+        issues = 0
+        for pattern in GARBLED_PATTERNS:
+            issues += len(re.findall(pattern, markdown))
+
+        replacement_chars = markdown.count("\ufffd")
+        if replacement_chars > 0:
+            issues += replacement_chars
+            warnings.append(f"Found {replacement_chars} Unicode replacement characters")
+
+        non_printable = sum(
+            1 for c in markdown
+            if unicodedata.category(c).startswith("C") and c not in "\n\r\t"
+        )
+        if non_printable > total_chars * 0.01:
+            issues += non_printable
+            warnings.append(f"Found {non_printable} non-printable characters")
+
+        alnum_count = sum(1 for c in markdown if c.isalnum() or c.isspace())
+        alnum_ratio = alnum_count / total_chars
+        if alnum_ratio < 0.5:
+            issues += int((0.5 - alnum_ratio) * 100)
+            warnings.append(f"Low alphanumeric ratio ({alnum_ratio:.1%})")
+
+        issue_ratio = issues / total_chars
+        if issue_ratio == 0:
+            return 100.0
+        elif issue_ratio < 0.001:
+            return 95.0
+        elif issue_ratio < 0.005:
+            return 80.0
+        elif issue_ratio < 0.01:
+            return 65.0
+        elif issue_ratio < 0.05:
+            return 40.0
+        else:
+            return 15.0
+
+    def _score_content_density(self, markdown: str, warnings: list[str]) -> float:
+        """Content Density (15%): meaningful text vs whitespace and noise."""
+        lines = markdown.split("\n")
+        total_lines = len(lines)
+        if total_lines == 0:
+            return 0.0
+
+        blank_lines = sum(1 for ln in lines if not ln.strip())
+        blank_ratio = blank_lines / total_lines
+        short_lines = sum(1 for ln in lines if 0 < len(ln.strip()) < 3)
+        short_ratio = short_lines / total_lines
+
+        line_counts = Counter(ln.strip() for ln in lines if ln.strip())
+        most_common_count = line_counts.most_common(1)[0][1] if line_counts else 0
+        repetition_ratio = most_common_count / total_lines if total_lines > 0 else 0
+
+        score = 100.0
+        if blank_ratio > 0.5:
+            score -= (blank_ratio - 0.5) * 100
+            warnings.append(f"Excessive blank lines: {blank_ratio:.0%} of output is empty")
+        elif blank_ratio > 0.3:
+            score -= (blank_ratio - 0.3) * 50
+
+        if short_ratio > 0.3:
+            score -= (short_ratio - 0.3) * 80
+            warnings.append(f"Many very short lines ({short_ratio:.0%}) — possible OCR artifacts")
+
+        if repetition_ratio > 0.1 and most_common_count > 5:
+            score -= repetition_ratio * 60
+            warnings.append(f"Repetitive content: one line appears {most_common_count} times")
+
+        return max(0.0, min(100.0, score))
+
+    def _score_readability(self, markdown: str, warnings: list[str]) -> float:
+        """Readability (10%): line length, paragraph structure, whitespace balance."""
+        lines = markdown.split("\n")
+        non_empty = [ln for ln in lines if ln.strip()]
+        if not non_empty:
+            return 0.0
+
+        avg_len = sum(len(ln) for ln in non_empty) / len(non_empty)
+        paragraphs, in_para = 0, False
+        for line in lines:
+            if line.strip():
+                if not in_para:
+                    paragraphs += 1
+                    in_para = True
+            else:
+                in_para = False
+
+        score = 100.0
+        total_words = len(markdown.split())
+
+        if avg_len > 500:
+            score -= 30
+            warnings.append(f"Very long average line length ({avg_len:.0f} chars)")
+        elif avg_len > 300:
+            score -= 15
+
+        if paragraphs < 2 and total_words > 200:
+            score -= 20
+            warnings.append("Document appears to be a single block of text")
+        elif paragraphs < 3 and total_words > 500:
+            score -= 10
+
+        if paragraphs >= 5 and total_words > 200:
+            score = min(100.0, score + 10)
+
+        return max(0.0, min(100.0, score))
+
+    @staticmethod
+    def _score_to_grade(score: int) -> str:
+        for threshold, grade in GRADE_MAP:
+            if score >= threshold:
+                return grade
+        return "F"

warraqa/utils.py

@@ -0,0 +1,310 @@
+"""
+Warraqa Utilities
+
+File detection, folder picker dialog, path helpers, and logging configuration.
+"""
+
+from __future__ import annotations
+
+import logging
+import mimetypes
+import re
+import shutil
+import tkinter as tk
+from pathlib import Path
+from tkinter import filedialog
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+logger = logging.getLogger("warraqa")
+console = Console()
+
+# Supported file extensions and their type mapping
+EXTENSION_TYPE_MAP: dict[str, str] = {
+    ".pdf": "pdf",
+    ".docx": "docx",
+    ".doc": "doc",
+    ".pptx": "pptx",
+    ".ppt": "ppt",
+}
+
+# MIME type fallback mapping
+MIME_TYPE_MAP: dict[str, str] = {
+    "application/pdf": "pdf",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+    "application/msword": "doc",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
+    "application/vnd.ms-powerpoint": "ppt",
+}
+
+
+def detect_file_type(path: Path) -> str | None:
+    """
+    Detect file type using extension first, then MIME type as fallback.
+
+    Returns:
+        File type string ("pdf", "docx", "doc") or None if unsupported.
+    """
+    # Try extension first (most reliable for common formats)
+    ext = path.suffix.lower()
+    if ext in EXTENSION_TYPE_MAP:
+        return EXTENSION_TYPE_MAP[ext]
+
+    # Fallback to MIME type detection
+    mime_type, _ = mimetypes.guess_type(str(path))
+    if mime_type and mime_type in MIME_TYPE_MAP:
+        return MIME_TYPE_MAP[mime_type]
+
+    # Try python-magic if available
+    try:
+        import magic  # type: ignore
+
+        mime_type = magic.from_file(str(path), mime=True)
+        if mime_type in MIME_TYPE_MAP:
+            return MIME_TYPE_MAP[mime_type]
+    except ImportError:
+        pass
+    except Exception as e:
+        logger.debug(f"python-magic detection failed for {path}: {e}")
+
+    return None
+
+
+def get_mime_type(path: Path) -> str:
+    """Get the MIME type of a file."""
+    # Try python-magic first for accuracy
+    try:
+        import magic  # type: ignore
+
+        return magic.from_file(str(path), mime=True)
+    except (ImportError, Exception):
+        pass
+
+    # Fallback to mimetypes module
+    mime_type, _ = mimetypes.guess_type(str(path))
+    return mime_type or "application/octet-stream"
+
+
+def open_folder_picker(title: str = "Select Input Folder") -> Path | None:
+    """
+    Open a native folder picker dialog.
+
+    Returns:
+        Selected folder path, or None if cancelled.
+    """
+    root = tk.Tk()
+    root.withdraw()  # Hide the root window
+    root.attributes("-topmost", True)  # Bring dialog to front
+
+    folder_path = filedialog.askdirectory(title=title)
+
+    root.destroy()
+
+    if folder_path:
+        return Path(folder_path)
+    return None
+
+
+def ensure_directories(*dirs: Path | str) -> None:
+    """Create directories if they don't exist."""
+    for d in dirs:
+        Path(d).mkdir(parents=True, exist_ok=True)
+
+
+def safe_move(src: Path, dest_dir: Path) -> Path:
+    """
+    Move a file to a destination directory, handling name collisions.
+
+    If a file with the same name already exists in dest_dir, appends
+    _1, _2, etc. to the filename.
+
+    Returns:
+        The final destination path.
+    """
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    dest = dest_dir / src.name
+
+    if dest.exists():
+        stem = src.stem
+        suffix = src.suffix
+        counter = 1
+        while dest.exists():
+            dest = dest_dir / f"{stem}_{counter}{suffix}"
+            counter += 1
+
+    shutil.move(str(src), str(dest))
+    logger.debug(f"Moved: {src.name} → {dest}")
+    return dest
+
+
+def sanitize_filename(name: str) -> str:
+    """
+    Clean a filename for use as a markdown output filename.
+
+    Removes special characters, replaces spaces with underscores,
+    and ensures the name is filesystem-safe.
+    """
+    # Remove file extension if present
+    name = Path(name).stem
+
+    # Replace common separators with underscores
+    name = re.sub(r"[\s\-]+", "_", name)
+
+    # Remove characters that are problematic on filesystems
+    name = re.sub(r"[^\w\.\-]", "", name)
+
+    # Collapse multiple underscores
+    name = re.sub(r"_+", "_", name)
+
+    # Strip leading/trailing underscores
+    name = name.strip("_")
+
+    # Ensure we have something
+    if not name:
+        name = "untitled"
+
+    return name
+
+
+def find_supported_files(directory: Path) -> list[Path]:
+    """
+    Find all supported files in a directory (non-recursive).
+
+    Returns:
+        List of paths to supported files, sorted alphabetically.
+    """
+    supported = []
+    if not directory.is_dir():
+        logger.warning(f"Not a directory: {directory}")
+        return supported
+
+    for file_path in sorted(directory.iterdir()):
+        if file_path.is_file() and file_path.suffix.lower() in EXTENSION_TYPE_MAP:
+            supported.append(file_path)
+
+    return supported
+
+
+def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    """
+    Configure structured logging with rich console output.
+
+    Args:
+        level: Logging level (DEBUG, INFO, WARNING, ERROR)
+        log_file: Optional path to a log file
+    """
+    log_level = getattr(logging, level.upper(), logging.INFO)
+
+    # Configure root warraqa logger
+    warraqa_logger = logging.getLogger("warraqa")
+    warraqa_logger.setLevel(log_level)
+
+    # Clear existing handlers
+    warraqa_logger.handlers.clear()
+
+    # Rich console handler
+    rich_handler = RichHandler(
+        console=console,
+        show_time=True,
+        show_path=False,
+        markup=True,
+        rich_tracebacks=True,
+    )
+    rich_handler.setLevel(log_level)
+    rich_handler.setFormatter(logging.Formatter("%(message)s"))
+    warraqa_logger.addHandler(rich_handler)
+
+    # File handler (if specified)
+    if log_file:
+        log_path = Path(log_file)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        file_handler = logging.FileHandler(str(log_path), encoding="utf-8")
+        file_handler.setLevel(log_level)
+        file_handler.setFormatter(
+            logging.Formatter(
+                "%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            )
+        )
+        warraqa_logger.addHandler(file_handler)
+
+    warraqa_logger.debug("Logging initialized at level %s", level)
+
+
+def format_file_size(size_bytes: int) -> str:
+    """Format a file size in bytes to a human-readable string."""
+    for unit in ("B", "KB", "MB", "GB"):
+        if size_bytes < 1024:
+            return f"{size_bytes:.1f} {unit}"
+        size_bytes /= 1024  # type: ignore
+    return f"{size_bytes:.1f} TB"
+
+
+# Unpaired-surrogate range — invalid in well-formed Unicode; appear in engine
+# output when a PDF has malformed text encodings or custom font cmaps that
+# don't round-trip cleanly. Python's UTF-8 encoder refuses them, which
+# crashes `Path.write_text` and breaks the whole batch.
+_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
+
+
+def sanitize_markdown(text: str) -> tuple[str, int]:
+    """
+    Replace lone Unicode surrogates with U+FFFD (replacement char).
+
+    Returns a (cleaned_text, replacement_count) tuple so callers can warn
+    about how much was sanitized.
+    """
+    if not text:
+        return text, 0
+    count = 0
+
+    def _sub(_match: re.Match) -> str:
+        nonlocal count
+        count += 1
+        return "�"
+
+    cleaned = _SURROGATE_RE.sub(_sub, text)
+    return cleaned, count
+
+class PDFAnalyzer:
+    """Analyzes PDFs to determine structural properties before conversion."""
+
+    @staticmethod
+    def is_native_text(file_path: Path, sample_pages: int = 5, threshold: int = 100) -> bool:
+        """
+        Determines if a PDF is natively digital (contains embedded text) 
+        or if it is a scanned image.
+        """
+        try:
+            import fitz
+            total_chars = 0
+            pages_checked = 0
+            
+            with fitz.open(str(file_path)) as doc:
+                pages_to_check = min(sample_pages, len(doc))
+                
+                if pages_to_check == 0:
+                    return False
+                    
+                for i in range(pages_to_check):
+                    page = doc[i]
+                    text = page.get_text()
+                    total_chars += len(text.strip())
+                    pages_checked += 1
+                    
+            if pages_checked == 0:
+                return False
+                
+            avg_chars_per_page = total_chars / pages_checked
+            
+            if avg_chars_per_page > threshold:
+                logger.debug(f"Document classified as NATIVE TEXT (avg {avg_chars_per_page:.0f} chars/page).")
+                return True
+            else:
+                logger.debug(f"Document classified as SCANNED IMAGE (avg {avg_chars_per_page:.0f} chars/page).")
+                return False
+                
+        except Exception as e:
+            logger.warning(f"Failed to analyze PDF {file_path.name}: {e}. Defaulting to SCANNED.")
+            return False