detector.py

"""
AI Writing Report — Hugging Face Space
Detector: coai/roberta-ai-detector-v2 (gated model -> add an HF_TOKEN secret whose account accepted the terms)
Input: .docx / .pdf / .txt or pasted text  ->  highlighted PDF report + on-screen preview
PDF and DOCX inputs keep their original layout: highlights are drawn onto the source pages
(DOCX is converted to PDF with LibreOffice first), so figures, equations and tables are untouched.
"""
import os, re, html, datetime, tempfile, shutil, subprocess
from dataclasses import dataclass, field

import gradio as gr
import docx_report as DX

# ---------------------------------------------------------------- config
MODEL_ID = os.getenv("DETECTOR_MODEL", "coai/roberta-ai-detector-v2")
HF_TOKEN = os.getenv("HF_TOKEN")
BRAND = os.getenv("REPORT_BRAND", "CS SOFTWARE SOLUTIONS")
MIN_PARA_WORDS = 25        # shorter blocks (headings, captions) are not analysed
WINDOW_WORDS = 90          # detector context per window (model is weak below ~50 words)
MAX_WINDOW_WORDS = 320     # keeps each window under 512 tokens
BATCH = 16
HIGH_COLOR = "#A9E4EE"     # cyan
MOD_COLOR = "#E4DCF5"      # light purple

# `spaces` is installed on every HF Space; only SPACES_ZERO_GPU=true means ZeroGPU hardware.
ZERO = os.getenv("SPACES_ZERO_GPU", "").strip().lower() in ("1", "true", "yes")
try:
    import spaces
except Exception:
    spaces = None
ZERO = ZERO and spaces is not None

def _duration(texts):
    # ~0.05 s per window on ZeroGPU (+0.1 s with the second model) + load overhead; capped for long theses
    return min(300, 30 + int(len(texts) * (0.15 if MODEL2_ID else 0.05)))

def gpu(fn):
    return spaces.GPU(duration=_duration)(fn) if ZERO else fn

# ---------------------------------------------------------------- model
# Loaded once at startup. On ZeroGPU the model must be created (and moved to
# "cuda") at module level, not inside the @spaces.GPU function, or it is
# re-downloaded / re-loaded on every request.
_tok, _model = None, None

# Second detector (ensemble). The first model was trained on older chatbot text and misses much of what current
# models (GPT-4o/5, Claude, Gemini) write; this DeBERTa-v3-large detector was trained on the RAID benchmark (11 modern
# generators, many domains, adversarial edits) and catches those. Final score = weighted mean of the two.
#   DETECTOR_MODEL2   default desklib/ai-text-detector-v1.01 ("" = single model, as before)
#   DETECTOR_WEIGHT2  default 0.7   weight of the second model
#   DETECTOR_COMBINE  mean (default) | max  (max flags text that EITHER model flags: more catches, more false alarms)
MODEL2_ID = os.getenv("DETECTOR_MODEL2", "desklib/ai-text-detector-v1.01").strip()
WEIGHT2 = float(os.getenv("DETECTOR_WEIGHT2", "0.7"))
COMBINE = os.getenv("DETECTOR_COMBINE", "mean").strip().lower()
_tok2, _model2, _m2_failed = None, None, False

# Third detector: robust to paraphrasing (what Turnitin's "AI-paraphrased" check is about). Measured with
# devtools/compare_detectors.py on 18 AI-written academic samples, 21 human abstracts from 2008 and 16 paragraphs our
# own humanizer had brought to 6%: coai + desklib flagged 18/18 AI, 0/21 human, 1/16 humanized; this RoBERTa
# (fakespot-ai) flagged 18/18, 2/21, 8/16. Averaged with the first two: 18/18 AI, 0/21 human, 6/16 humanized, so
# the score is much harder to game without flagging human writing.
#   DETECTOR_MODEL3   default fakespot-ai/roberta-base-ai-text-detection-v1 ("" = off)
#   DETECTOR_WEIGHT3  default 0.5   share of the final score
MODEL3_ID = os.getenv("DETECTOR_MODEL3", "fakespot-ai/roberta-base-ai-text-detection-v1").strip()
WEIGHT3 = float(os.getenv("DETECTOR_WEIGHT3", "0.5"))
_tok3, _model3, _m3_failed = None, None, False
_cpu_m3 = None


def load_model3():
    global _tok3, _model3, _m3_failed
    if not MODEL3_ID or _m3_failed or _model3 is not None:
        return _tok3, _model3
    try:
        import torch
        from transformers import AutoModelForSequenceClassification, AutoTokenizer
        _tok3 = AutoTokenizer.from_pretrained(MODEL3_ID, token=HF_TOKEN)
        _model3 = AutoModelForSequenceClassification.from_pretrained(MODEL3_ID, token=HF_TOKEN).eval()
        if ZERO or torch.cuda.is_available():
            _model3.to("cuda")
        print(f"[detector] third model loaded: {MODEL3_ID}", flush=True)
    except Exception as err:
        _m3_failed = True
        print(f"[detector] third model not available ({err}); first two only", flush=True)
    return _tok3, _model3


# More detectors averaged with the score above, each counting as much as the whole ensemble before it. Chosen with
# devtools/compare_detectors.py from 9 public detectors (18 AI-written academic samples, 21 human abstracts from
# 2008, 16 paragraphs our own humanizer had rewritten): ensemble before 18/18 AI, 0/21 human, 6/16 humanized flagged;
# ShantanuT01/vanguard (ModernBERT-large, MIT) 17/18, 1/21, 13/16; Oxidane/tmr (RoBERTa, MIT) 18/18, 1/21, 11/16;
# the mean of the three: 18/18, 0/21, 11/16. Rejected: the most downloaded detector flagged every human text, and
# three others flagged 7 to 18 of the 21. DETECTOR_EXTRA = comma-separated model ids ("" = none).
EXTRA_IDS = [m.strip() for m in os.getenv(
    "DETECTOR_EXTRA", "ShantanuT01/vanguard-ai-text-detector,Oxidane/tmr-ai-text-detector").split(",") if m.strip()]
_extras, _extras_failed, _extras_cpu = None, False, None


def load_extras(cpu=False):
    """[(model id, tokenizer, model, AI label index or None for a single sigmoid output)] (GPU when there is one)."""
    global _extras, _extras_failed, _extras_cpu
    if cpu and _extras_cpu is not None:
        return _extras_cpu
    if not cpu and (_extras is not None or _extras_failed):
        return _extras or []
    out = []
    for mid in EXTRA_IDS:
        try:
            import torch
            from transformers import AutoModelForSequenceClassification, AutoTokenizer
            tok = AutoTokenizer.from_pretrained(mid, token=HF_TOKEN)
            m = AutoModelForSequenceClassification.from_pretrained(mid, token=HF_TOKEN).eval()
            if not cpu and (ZERO or torch.cuda.is_available()):
                m.to("cuda")
            labels = {int(k): str(v).lower() for k, v in m.config.id2label.items()}
            ai = None if m.config.num_labels == 1 else next(
                (i for i, l in labels.items() if l in ("ai", "fake", "machine", "generated", "label_1")), 1)
            out.append((mid, tok, m, ai))
            print(f"[detector] extra model loaded: {mid}", flush=True)
        except Exception as err:
            print(f"[detector] extra model {mid} not available ({err}); left out", flush=True)
    if cpu:
        _extras_cpu = out
    else:
        _extras, _extras_failed = out, not out
    return out


def _score_extra(tok, model, ai, texts):
    import torch
    dev = next(model.parameters()).device
    bs = 32 if dev.type == "cuda" else 8
    res = []
    for i in range(0, len(texts), bs):
        enc = tok(texts[i:i + bs], return_tensors="pt", truncation=True, max_length=512, padding=True).to(dev)
        enc.pop("token_type_ids", None)
        with torch.no_grad(), _amp(dev):
            lg = model(**enc).logits.float()
        p = torch.sigmoid(lg[:, 0]) if ai is None else torch.softmax(lg, -1)[:, ai]
        res.extend(p.cpu().tolist())
    return res


def _with_extras(base, texts, cpu=False):
    ps = [base] + [_score_extra(tok, m, ai, texts) for _, tok, m, ai in load_extras(cpu)]
    return [sum(x) / len(ps) for x in zip(*ps)] if len(ps) > 1 else base


def _with3(p12, p3):
    if p3 is None:
        return p12
    return [(1 - WEIGHT3) * a + WEIGHT3 * b for a, b in zip(p12, p3)]


def _desklib_class():
    import torch
    from torch import nn
    from transformers import AutoConfig, AutoModel, PreTrainedModel

    class DesklibAIDetectionModel(PreTrainedModel):
        config_class = AutoConfig

        def __init__(self, config):
            super().__init__(config)
            self.model = AutoModel.from_config(config)
            self.classifier = nn.Linear(config.hidden_size, 1)
            self.init_weights()

        def forward(self, input_ids, attention_mask=None):
            h = self.model(input_ids, attention_mask=attention_mask)[0]
            m = attention_mask.unsqueeze(-1).expand(h.size()).float()
            pooled = (h * m).sum(1) / m.sum(1).clamp(min=1e-9)
            return self.classifier(pooled)
    return DesklibAIDetectionModel


def load_model2():
    """Second detector, or None if disabled / not available (then the first model alone is used)."""
    global _tok2, _model2, _m2_failed
    if _model2 is None and MODEL2_ID and not _m2_failed:
        try:
            import torch
            from transformers import AutoTokenizer
            _tok2 = AutoTokenizer.from_pretrained(MODEL2_ID, token=HF_TOKEN)
            cls = _desklib_class() if "desklib" in MODEL2_ID.lower() else None
            if cls is not None:
                _model2 = cls.from_pretrained(MODEL2_ID, token=HF_TOKEN).eval()
            else:
                from transformers import AutoModelForSequenceClassification
                _model2 = AutoModelForSequenceClassification.from_pretrained(MODEL2_ID, token=HF_TOKEN).eval()
            if ZERO or torch.cuda.is_available():
                _model2.to("cuda")
            print(f"[detector] second model loaded: {MODEL2_ID}", flush=True)
        except Exception as err:
            _m2_failed = True
            print(f"[detector] second model not available ({err}); first model only", flush=True)
    return _tok2, _model2


# Half precision on the GPU (about 2x faster on a T4). Used only after half_check() found that it gives the same
# verdicts as full precision on the bundled samples; DETECTOR_HALF=0 = never.
HALF = False
HALF_WANTED = os.getenv("DETECTOR_HALF", "1") != "0"


def _amp(dev):
    import contextlib
    import torch
    if HALF and dev.type == "cuda":
        return torch.autocast("cuda", dtype=torch.float16)
    return contextlib.nullcontext()


def _score_with2(tok, model, texts):
    import torch
    dev = next(model.parameters()).device
    bs = 32 if dev.type == "cuda" else 8
    out = []
    for i in range(0, len(texts), bs):
        enc = tok(texts[i:i + bs], return_tensors="pt", truncation=True, max_length=768, padding=True).to(dev)
        with torch.no_grad(), _amp(dev):
            logits = model(input_ids=enc["input_ids"], attention_mask=enc["attention_mask"])
            logits = logits.logits if hasattr(logits, "logits") else logits
            if logits.shape[-1] == 1:
                p = torch.sigmoid(logits.float()).squeeze(-1)
            else:
                p = torch.softmax(logits.float(), dim=-1)[:, 1]
        out.extend(p.cpu().tolist())
    return out


def _combine(p1, p2):
    if p2 is None:
        return p1
    if COMBINE == "max":
        return [max(a, b) for a, b in zip(p1, p2)]
    return [(1 - WEIGHT2) * a + WEIGHT2 * b for a, b in zip(p1, p2)]

def load_model():
    global _tok, _model
    if _model is None:
        import torch
        from transformers import AutoTokenizer, AutoModelForSequenceClassification
        _tok = AutoTokenizer.from_pretrained(MODEL_ID, token=HF_TOKEN)
        _model = AutoModelForSequenceClassification.from_pretrained(MODEL_ID, token=HF_TOKEN).eval()
        if ZERO or torch.cuda.is_available():
            _model.to("cuda")
    return _tok, _model

if os.getenv("SKIP_MODEL_LOAD") != "1":
    load_model()
    load_model2()
    load_model3()
    load_extras()
    DX.start_lo_init()            # fetch newest LibreOffice in the background

def _score_with(tok, model, texts):
    import torch
    dev = next(model.parameters()).device
    bs = 64 if dev.type == "cuda" else BATCH
    out = []
    for i in range(0, len(texts), bs):
        enc = tok(texts[i:i + bs], return_tensors="pt", truncation=True,
                  max_length=512, padding=True).to(dev)
        with torch.no_grad(), _amp(dev):
            p = torch.softmax(model(**enc).logits.float(), dim=-1)[:, 1]
        out.extend(p.cpu().tolist())
    return out


@gpu
def _score_gpu(texts):
    """P(AI) for each text. Label index 1 = AI-generated (per model card)."""
    tok, model = load_model()
    p1 = _score_with(tok, model, texts)
    tok2, model2 = load_model2()
    p2 = _score_with2(tok2, model2, texts) if model2 is not None else None
    tok3, model3 = load_model3()
    p3 = _score_with(tok3, model3, texts) if model3 is not None else None   # label 1 = "ai"
    return _with_extras(_with3(_combine(p1, p2), p3), texts)


_cpu_det, _cpu_det_lock = None, __import__("threading").Lock()
LAST_ENGINE = "GPU"


def _score_cpu(texts):
    global _cpu_det, _cpu_m3
    import torch
    from transformers import AutoModelForSequenceClassification
    with _cpu_det_lock:
        if _cpu_det is None:
            _cpu_det = AutoModelForSequenceClassification.from_pretrained(MODEL_ID, token=HF_TOKEN).eval()
    import runtime as RT
    with RT.cpu_job():
        p1 = _score_with(_tok or load_model()[0], _cpu_det, texts)
        tok2, model2 = load_model2()
        p2 = None
        if model2 is not None:
            if next(model2.parameters()).device.type != "cpu":
                model2 = _cpu_model2()
            p2 = _score_with2(tok2, model2, texts)
        p3 = None
        tok3, model3 = load_model3()
        if model3 is not None:
            if next(model3.parameters()).device.type != "cpu":
                with _cpu_det_lock:
                    if _cpu_m3 is None:
                        from transformers import AutoModelForSequenceClassification as _A
                        _cpu_m3 = _A.from_pretrained(MODEL3_ID, token=HF_TOKEN).eval()
                model3 = _cpu_m3
            p3 = _score_with(tok3, model3, texts)
        return _with_extras(_with3(_combine(p1, p2), p3), texts, cpu=True)


_cpu_m2 = None


def _cpu_model2():
    """CPU copy of the second detector (ZeroGPU fallback)."""
    global _cpu_m2
    with _cpu_det_lock:
        if _cpu_m2 is None:
            cls = _desklib_class() if "desklib" in MODEL2_ID.lower() else None
            if cls is not None:
                _cpu_m2 = cls.from_pretrained(MODEL2_ID, token=HF_TOKEN).eval()
            else:
                from transformers import AutoModelForSequenceClassification
                _cpu_m2 = AutoModelForSequenceClassification.from_pretrained(MODEL2_ID, token=HF_TOKEN).eval()
    return _cpu_m2


# Scores are deterministic per text, and the humanizer asks for the same texts again and again (candidates, the
# chosen version, before/after of every polish step), so they are cached. New texts are scored shortest-first:
# batches then carry little padding, which matters most on CPU.
SCORE_CACHE_MAX = int(os.getenv("DETECTOR_CACHE", "20000"))
_score_cache, _score_cache_lock = {}, __import__("threading").Lock()


def half_check():
    """On a GPU: score the bundled samples in full and in half precision; half precision is switched on only if no
    verdict (>= 0.5) changes and no score moves by more than 0.05. Logged either way."""
    global HALF
    import time as _t
    try:
        import torch
        if not HALF_WANTED or ZERO or not torch.cuda.is_available():
            return
        import sys as _sys
        from pathlib import Path as _P
        _sys.path.insert(0, str(_P(__file__).resolve().parent / "devtools"))
        from samples_domains import DOMAINS
        import samples
        texts = list(DOMAINS.values()) + [samples.HUMAN, samples.AI_TECH, samples.AI_SOCIAL]
        texts += [s for v in DOMAINS.values() for s in split_sentences(v)][:30]
        HALF = False
        t = _t.time(); a = _score_gpu(texts); ta = _t.time() - t
        HALF = True
        t = _t.time(); b = _score_gpu(texts); tb = _t.time() - t
        diff = max(abs(x - y) for x, y in zip(a, b))
        flips = sum((x >= 0.5) != (y >= 0.5) for x, y in zip(a, b))
        HALF = flips == 0 and diff <= 0.05
        print(f"[detector] half precision {'ON' if HALF else 'OFF'}: {len(texts)} texts, full {ta:.1f}s, half {tb:.1f}s, "
              f"max score difference {diff:.3f}, verdicts changed {flips}", flush=True)
    except Exception as err:
        HALF = False
        print(f"[detector] half precision check failed, full precision kept: {err}", flush=True)


def on_cpu():
    """True when the detector runs on the CPU (no GPU): every model call is then expensive."""
    try:
        return next(load_model()[1].parameters()).device.type == "cpu" and not ZERO
    except Exception:
        return True


_light_cache = {}


def score_light(texts):
    """P(AI) from the two small models only (coai + the paraphrase-robust RoBERTa), without the large DeBERTa: for
    ranking many candidate versions on the CPU, where the large model is about two thirds of the detector's time.
    The final verdict of a document always comes from score_texts (all three models)."""
    texts = list(texts)
    todo = [t for t in dict.fromkeys(texts) if t not in _light_cache]
    if todo:
        tok, model = load_model()
        p1 = _score_with(tok, model, todo)
        tok3, model3 = load_model3()
        p3 = _score_with(tok3, model3, todo) if model3 is not None else None
        p = p1 if p3 is None else [(a + b) / 2 for a, b in zip(p1, p3)]
        if len(_light_cache) > SCORE_CACHE_MAX:
            _light_cache.clear()
        _light_cache.update(zip(todo, p))
    return [_light_cache[t] for t in texts]


def score_texts(texts):
    """P(AI) per text (cached; only texts not seen before reach the model)."""
    texts = list(texts)
    with _score_cache_lock:
        known = {t: _score_cache[t] for t in texts if t in _score_cache}
    todo = sorted(set(texts) - known.keys(), key=len)
    if todo:
        fresh = dict(zip(todo, _score_uncached(todo)))
        known.update(fresh)
        with _score_cache_lock:
            if len(_score_cache) + len(fresh) > SCORE_CACHE_MAX:
                _score_cache.clear()
            _score_cache.update(fresh)
    return [known[t] for t in texts]


def _score_uncached(texts):
    """GPU when available; falls back to CPU if the ZeroGPU quota / queue is exhausted."""
    global LAST_ENGINE
    if not ZERO:
        on_gpu = next(load_model()[1].parameters()).device.type == "cuda"
        LAST_ENGINE = "GPU" if on_gpu else "CPU"
        if on_gpu:
            return _score_gpu(texts)
        import runtime as RT
        with RT.cpu_job():                   # CPU-only hardware: share cores between users
            return _score_gpu(texts)
    try:
        LAST_ENGINE = "GPU"
        return _score_gpu(texts)
    except Exception as err:
        msg = str(err).lower()
        if not any(k in msg for k in ("quota", "zerogpu", "gpu task aborted", "cuda", "no gpu",
                                      "queue is full", "exceeded", "unlogged", "sign in")):
            raise
        print(f"[detector] GPU unavailable ({err}); scoring on CPU", flush=True)
        LAST_ENGINE = "CPU"
        return _score_cpu(texts)

# ---------------------------------------------------------------- extraction
@dataclass
class Block:
    text: str
    kind: str = "para"            # para | heading | table | skip
    sents: list = field(default_factory=list)
    rows: list = field(default_factory=list)   # table cells (list of rows)
    scores: list = field(default_factory=list)
    words: list = field(default_factory=list)  # raw words (layout path)
    toks: list = field(default_factory=list)   # (page, block, line, bbox) per word
    spans: list = field(default_factory=list)  # (start, end) word index per sentence
    el: object = None                           # w:p element (docx path)
    raw: str = ""                               # exact paragraph text (docx path)
    cspans: list = field(default_factory=list)  # (start, end) char range per sentence

REF_HEAD = re.compile(r"^\s*(\d+(\.\d+)*\.?\s*|[IVX]+\.\s*|chapter\s+\w+\s*[:.\-–]?\s*)?"
                      r"(references?(\s+(list|cited))?|list\s+of\s+references|bibliography|works\s+cited|"
                      r"literature\s+cited|sources|webliography)\s*[:.]?\s*$", re.I)
# one reference-list entry (APA / Harvard / IEEE / Vancouver): never rewritten, wherever it appears
REF_ENTRY = re.compile(
    r"^\s*(?:\[\d+\]|\d+\.)?\s*"                                         # [12] / 12.
    r"(?:[A-Z][A-Za-z'’\-]+(?:\s[A-Z][A-Za-z'’\-]+)?,\s*(?:[A-Z]\.\s*-?)+"   # Kumar, R. S.
    r"|[A-Z]\.\s*(?:[A-Z]\.\s*)?[A-Z][A-Za-z'’\-]+,)"                       # R. Kumar,
    r".{0,400}?(?:\(\s*\d{4}[a-z]?\s*\)|\b(?:19|20)\d{2}[a-z]?\b)")
REF_HINT = re.compile(r"\bdoi\b|doi\.org|https?://|\bvol\.|\bpp\.|\bJournal\b|\bProceedings\b|"
                      r"\bPress\b|\b\d+\s*\(\d+\)\s*,?\s*\d+", re.I)
BULLET = re.compile(r"^\s*([•●▪\-\*–]|\(?[a-z0-9]{1,2}[\)\.])\s+")

def _norm(t):
    return re.sub(r"[ \t ]+", " ", t.replace("\r", "")).strip()

def extract_docx(path):
    import docx
    from docx.text.paragraph import Paragraph
    from docx.table import Table
    d = docx.Document(path)
    blocks = []
    for child in d.element.body.iterchildren():
        tag = child.tag.split("}")[-1]
        if tag == "p":
            p = Paragraph(child, d)
            t = _norm(p.text)
            if not t:
                continue
            style = (p.style.name or "").lower() if p.style is not None else ""
            kind = "heading" if ("heading" in style or "title" in style) else "para"
            if "list" in style and len(t.split()) < MIN_PARA_WORDS:
                kind = "skip"
            blocks.append(Block(t, kind))
        elif tag == "tbl":
            tbl = Table(child, d)
            rows = []
            for r in tbl.rows:
                cells, prev = [], None
                for c in r.cells:              # merged cells repeat the same _tc
                    if prev is not None and c._tc is prev:
                        continue
                    prev = c._tc
                    cells.append(_norm(c.text))
                rows.append(cells)
            rows = [r for r in rows if any(r)]
            if rows:
                blocks.append(Block("\n".join(" | ".join(r) for r in rows), "table", rows=rows))
    return blocks

def extract_pdf(path):
    import pymupdf as fitz
    blocks = []
    with fitz.open(path) as doc:
        for page in doc:
            h = page.rect.height
            items = []                                   # (y, Block)
            tboxes = []
            try:
                for tb in page.find_tables().tables:
                    rows = [[_norm(c or "") for c in r] for r in tb.extract()]
                    rows = [r for r in rows if any(r)]
                    if len(rows) >= 2 and max(len(r) for r in rows) >= 2:
                        tboxes.append(fitz.Rect(tb.bbox))
                        items.append((tb.bbox[1], Block("\n".join(" | ".join(r) for r in rows),
                                                        "table", rows=rows)))
            except Exception:
                pass
            for b in page.get_text("blocks", sort=True):
                x0, y0, x1, y1, t, _, btype = b
                if btype != 0:
                    continue
                if y1 < h * 0.06 or y0 > h * 0.94:     # running header / footer
                    continue
                r = fitz.Rect(x0, y0, x1, y1)
                if any(tr.intersects(r) and (tr & r).get_area() > 0.5 * r.get_area() for tr in tboxes):
                    continue                             # text already captured in a table
                t = _norm(re.sub(r"-\n(?=[a-z])", "", t).replace("\n", " "))
                if t:
                    items.append((y0, Block(t)))
            blocks.extend(b for _, b in sorted(items, key=lambda x: x[0]))
    # re-join paragraphs split across page / column breaks
    merged = []
    for b in blocks:
        if merged and merged[-1].kind == "para" and not re.search(r"[.!?:\"”)]$", merged[-1].text) \
                and b.text[:1].islower():
            merged[-1].text += " " + b.text
        else:
            merged.append(b)
    return merged

def _dehyphen(t):
    return re.sub(r"(\w)- ([a-z])", r"\1\2", t)

def extract_pdf_layout(path):
    """Paragraph blocks that remember where every word sits on the page."""
    import pymupdf as fitz
    blocks = []
    with fitz.open(path) as doc:
        for pno, page in enumerate(doc):
            h = page.rect.height
            tables, tboxes = [], []
            try:
                for tb in page.find_tables().tables:
                    rows = [[_norm(c or "") for c in r] for r in tb.extract()]
                    rows = [r for r in rows if any(r)]
                    if len(rows) >= 2 and max(len(r) for r in rows) >= 2:
                        tboxes.append(fitz.Rect(tb.bbox))
                        tables.append((tb.bbox[1], Block("\n".join(" | ".join(r) for r in rows),
                                                         "table", rows=rows)))
            except Exception:
                pass
            bywords = {}
            for w in page.get_text("words"):
                bywords.setdefault(w[5], []).append(w)
            page_blocks = []
            for b in page.get_text("blocks", sort=True):
                x0, y0, x1, y1, _, bno, btype = b
                if btype != 0 or bno not in bywords:
                    continue
                if y1 < h * 0.06 or y0 > h * 0.94:        # running header / footer
                    continue
                r = fitz.Rect(x0, y0, x1, y1)
                if any(tr.intersects(r) and (tr & r).get_area() > 0.5 * r.get_area() for tr in tboxes):
                    continue                                # part of a table
                ws = sorted(bywords[bno], key=lambda w: (w[6], w[7]))
                words = [w[4] for w in ws]
                toks = [(pno, bno, w[6], (w[0], w[1], w[2], w[3])) for w in ws]
                page_blocks.append((y0, Block(" ".join(words), words=words, toks=toks)))
            # place detected tables at their vertical position
            for ty, tblk in sorted(tables, key=lambda x: x[0]):
                idx = next((i for i, (y, _) in enumerate(page_blocks) if y > ty), len(page_blocks))
                page_blocks.insert(idx, (ty, tblk))
            blocks.extend(b for _, b in page_blocks)
    # re-join paragraphs split across page / column breaks
    merged = []
    for b in blocks:
        if merged and b.kind == "para" and merged[-1].kind == "para" and merged[-1].toks \
                and not re.search(r"[.!?:\"”)]$", merged[-1].text) and b.text[:1].islower():
            m = merged[-1]
            m.text += " " + b.text
            m.words += b.words
            m.toks += b.toks
        else:
            merged.append(b)
    return merged

def layout_sentences(b):
    """Sentence spans over the block's word list (+ clean display text)."""
    sents = split_sentences(" ".join(b.words))
    counts = [len(s.split()) for s in sents]
    if sum(counts) != len(b.words):
        sents, counts = [" ".join(b.words)], [len(b.words)]
    spans, i = [], 0
    for c in counts:
        spans.append((i, i + c))
        i += c
    b.spans = spans
    b.sents = [_dehyphen(s) for s in sents]

def extract_docx_live(doc, table_prose=False):
    """Blocks bound to the paragraphs of an open python-docx Document.
    table_prose=True: paragraphs inside table cells become separate blocks (documents laid out in tables)."""
    blocks = []
    for kind, obj in DX.iter_body(doc):
        if kind == "p":
            raw = DX.para_text(obj._p)
            t = re.sub(r"\s+", " ", raw).strip()
            if not t:
                continue
            style = (obj.style.name or "").lower() if obj.style is not None else ""
            k = "heading" if ("heading" in style or "title" in style) else "para"
            if ("list" in style or "caption" in style) and len(t.split()) < MIN_PARA_WORDS:
                k = "skip"
            blocks.append(Block(t, k, el=obj._p, raw=raw))
        elif table_prose:
            seen = set()
            for cell_p in obj._tbl.iter(DX.qn("w:p")):
                if id(cell_p) in seen:
                    continue
                seen.add(id(cell_p))
                raw = DX.para_text(cell_p)
                t = re.sub(r"\s+", " ", raw).strip()
                if t:
                    blocks.append(Block(t, "para", el=cell_p, raw=raw))
        else:
            rows = []
            for r in obj.rows:
                cells, prev = [], None
                for c in r.cells:
                    if prev is not None and c._tc is prev:
                        continue
                    prev = c._tc
                    cells.append(_norm(c.text))
                rows.append(cells)
            rows = [r for r in rows if any(r)]
            if rows:
                blocks.append(Block("\n".join(" | ".join(r) for r in rows), "table", rows=rows))
    return blocks

PASTE_MIN_WORDS = 6        # pasted text: short prose paragraphs still count (windows span paragraphs)


def docx_blocks(doc):
    """Blocks of a Word document, classified. Documents written in short paragraphs, bullet lists,
    heading-styled prose or table layouts would otherwise have "no qualifying prose", so the rules relax
    step by step until prose is found."""
    blocks = classify(extract_docx_live(doc))
    if any(b.kind == "para" for b in blocks):
        return blocks
    for min_words in (12, PASTE_MIN_WORDS):
        blocks = extract_docx_live(doc)
        for b in blocks:
            if b.kind == "heading" and len(b.text.split()) >= 15 and re.search(r"[.!?]$", b.text):
                b.kind = "para"                        # prose wrongly styled as Heading/Title
            elif b.kind == "skip" and b.el is not None:
                b.kind = "para"                        # bullet / list items written as full sentences
        blocks = classify(blocks, min_words)
        if any(b.kind == "para" for b in blocks):
            return blocks
    blocks = classify(extract_docx_live(doc, table_prose=True), PASTE_MIN_WORDS)   # text laid out in tables
    return blocks


def extract_text(raw):
    """Pasted / .txt text: single line breaks are paragraph breaks, PDF soft-wraps are re-joined."""
    import textparse as TP
    return [Block(_norm(p)) for p, _ in TP.parse_paste(raw) if _norm(p)]

def classify(blocks, min_words=MIN_PARA_WORDS):
    in_refs = False
    for b in blocks:
        if REF_HEAD.match(b.text):
            b.kind, in_refs = "heading", True
            continue
        if in_refs:
            b.kind = "skip"
            continue
        if b.kind in ("table", "skip", "heading"):
            continue
        if REF_ENTRY.match(b.text) and REF_HINT.search(b.text) and len(b.text.split()) < 120:
            b.kind = "skip"                                 # a reference-list entry: never rewritten
            continue
        wc = len(b.text.split())
        wordy = sum(1 for w in b.text.split() if re.fullmatch(r"[A-Za-z][A-Za-z'’\-]+[.,;:)]?", w))
        if re.match(r"^\s*(\d+(\.\d+)*\.?|[IVX]+\.|[A-Z]\.)\s+[A-Z]", b.text) and wc <= 14 \
                and not re.search(r"[.!?]$", b.text):
            b.kind = "heading"                              # "1. Introduction", "2.3 Results"
            continue
        if wc and wordy / wc < 0.55:                       # equations, symbols, numeric lists
            b.kind = "skip"
            continue
        if re.match(r"^(fig(ure)?|table)\s*\.?\s*\d", b.text, re.I) and wc < 40:
            b.kind = "skip"
        elif wc <= 14 and not re.search(r"[.!?:]$", b.text):
            b.kind = "heading"                          # titles / section headings
        elif BULLET.match(b.text) and wc < min_words:
            b.kind = "skip"
        elif wc < min_words:
            b.kind = "skip"
    return blocks

# ---------------------------------------------------------------- sentences
_ABBR = r"(?:et al|e\.g|i\.e|etc|vs|Fig|Figs|Eq|Eqs|Ref|Refs|No|Vol|pp|Dr|Mr|Mrs|Ms|Prof|approx|cf|Sec|Ch)"

def split_sentences(t):
    import textparse as TP                              # one rule for what ends a sentence, in every tool
    t = TP.protect_dots(t)
    # (the closing quote / bracket after the full stop belongs to the sentence: it must not be consumed by the split)
    parts = re.split(r"(?:(?<=[.!?])|(?<=[.!?][\"”’)]))\s+(?=[A-Z0-9\[\(“\"])", t)
    return [p.replace("<DOT>", ".").strip() for p in parts if p.strip()]

# ---------------------------------------------------------------- analysis
def make_windows(lengths):
    """Overlapping sentence windows of ~WINDOW_WORDS words (≈50% overlap)."""
    windows, i, n = [], 0, len(lengths)
    while i < n:
        j, wc = i + 1, lengths[i]
        while j < n and wc < WINDOW_WORDS and wc + lengths[j] <= MAX_WINDOW_WORDS:
            wc += lengths[j]
            j += 1
        windows.append((i, j))
        if j >= n:
            break
        i = max(i + 1, i + (j - i) // 2)
    return windows

def analyse(blocks, scorer=None):
    scorer = scorer or score_texts          # looked up at call time (tests may swap it)
    stream = []   # (block_idx, sent_idx, text, wordcount)
    for bi, b in enumerate(blocks):
        if b.kind != "para":
            continue
        if b.toks:
            layout_sentences(b)
        elif b.el is not None:
            raws = split_sentences(b.raw)
            b.cspans = DX.sentence_char_spans(b.raw, raws)
            b.sents = [re.sub(r"\s+", " ", x).strip() for x in raws]
        else:
            b.sents = split_sentences(b.text)
        b.scores = [0.0] * len(b.sents)
        for si, s in enumerate(b.sents):
            stream.append((bi, si, s, len(s.split())))
    if not stream:
        return blocks

    windows = make_windows([x[3] for x in stream])
    probs = scorer([" ".join(stream[k][2] for k in range(a, b)) for a, b in windows])

    acc, cnt = [0.0] * len(stream), [0] * len(stream)
    for (a, b), p in zip(windows, probs):
        for k in range(a, b):
            acc[k] += p
            cnt[k] += 1
    for k, (bi, si, _, _) in enumerate(stream):
        blocks[bi].scores[si] = acc[k] / max(cnt[k], 1)

    # light paragraph smoothing -> less speckled highlighting
    for b in blocks:
        if b.kind == "para" and b.scores:
            m = sum(b.scores) / len(b.scores)
            b.scores = [0.7 * s + 0.3 * m for s in b.scores]
    return blocks

def summarise(blocks, hi, mod):
    total = hi_w = mod_w = 0
    for b in blocks:
        if b.kind != "para":
            continue
        for s, p in zip(b.sents, b.scores):
            w = len(s.split())
            total += w
            if p >= hi:
                hi_w += w
            elif p >= mod:
                mod_w += w
    pct = round(100 * (hi_w + mod_w) / total) if total else 0
    return dict(pct=pct, analysed=total, high=hi_w, moderate=mod_w,
                all_words=sum(len(b.text.split()) for b in blocks))

def band(pct):
    if pct < 20:
        return "Low likelihood of AI-generated content"
    if pct <= 50:
        return "Moderate likelihood of AI-generated content"
    return "High likelihood of AI-generated content"

# ---------------------------------------------------------------- PDF report
def build_pdf(blocks, stats, fname, fsize_kb, hi, mod, out_path, summary_only=False):
    from reportlab.lib.pagesizes import A4
    from reportlab.lib.units import mm
    from reportlab.lib import colors
    from reportlab.lib.styles import ParagraphStyle
    from reportlab.platypus import (BaseDocTemplate, Frame, PageTemplate, Paragraph,
                                    Spacer, Table, TableStyle, PageBreak)
    from reportlab.pdfgen import canvas as rl_canvas
    from reportlab.pdfbase import pdfmetrics
    from reportlab.pdfbase.ttfonts import TTFont

    serif, serif_b, sans, sans_b = "Times-Roman", "Times-Bold", "Helvetica", "Helvetica-Bold"
    fdir = "/usr/share/fonts/truetype/dejavu/"
    try:
        for a, f in [("Serif", "DejaVuSerif.ttf"), ("SerifB", "DejaVuSerif-Bold.ttf"),
                     ("Sans", "DejaVuSans.ttf"), ("SansB", "DejaVuSans-Bold.ttf")]:
            pdfmetrics.registerFont(TTFont(a, fdir + f))
        serif, serif_b, sans, sans_b = "Serif", "SerifB", "Sans", "SansB"   # Unicode-safe
    except Exception:
        pass

    now = datetime.datetime.now(datetime.timezone.utc).strftime("%B %d, %Y at %I:%M %p UTC")

    class NumberedCanvas(rl_canvas.Canvas):
        def __init__(self, *a, **k):
            super().__init__(*a, **k)
            self._saved = []
        def showPage(self):
            self._saved.append(dict(self.__dict__))
            self._startPage()
        def save(self):
            total = len(self._saved)
            for st in self._saved:
                self.__dict__.update(st)
                w, h = A4
                self.setFont(sans, 8)
                self.setFillColor(colors.HexColor("#555555"))
                self.drawString(18 * mm, h - 12 * mm, BRAND)
                self.drawRightString(w - 18 * mm, h - 12 * mm, "AI WRITING REPORT")
                if not summary_only:
                    self.drawCentredString(w / 2, 10 * mm, f"Page {self._pageNumber} of {total}")
                super().showPage()
            super().save()

    _n = [0]
    def S(**k):
        _n[0] += 1
        return ParagraphStyle(name=f"s{_n[0]}", **k)

    body = S(fontName=serif, fontSize=12, leading=24, alignment=4, firstLineIndent=18)
    head = S(fontName=serif_b, fontSize=12, leading=22, spaceBefore=8, spaceAfter=2)
    plain = S(fontName=serif, fontSize=11, leading=16, textColor=colors.HexColor("#333333"), spaceAfter=6)
    tbls = S(fontName=serif, fontSize=9, leading=12, textColor=colors.HexColor("#555555"), spaceAfter=6)
    h1 = S(fontName=sans_b, fontSize=26, leading=32, spaceAfter=6)
    h2 = S(fontName=sans_b, fontSize=12, leading=16, spaceBefore=12, spaceAfter=4)
    small = S(fontName=sans, fontSize=9.5, leading=13.5, textColor=colors.HexColor("#333333"))
    esc = lambda t: html.escape(t, quote=False)
    cell = S(fontName=serif, fontSize=9.5, leading=12)
    cell_b = S(fontName=serif_b, fontSize=9.5, leading=12)

    def make_table(rows):
        ncol = max(len(r) for r in rows)
        rows = [r + [""] * (ncol - len(r)) for r in rows]
        # column widths proportional to content length, within page width
        lens = [max(min(len(r[c]), 60) for r in rows) + 4 for c in range(ncol)]
        W = 160 * mm
        widths = [W * l / sum(lens) for l in lens]
        data = [[Paragraph(esc(c), cell_b if i == 0 else cell) for c in r] for i, r in enumerate(rows)]
        t = Table(data, colWidths=widths, repeatRows=1, hAlign="CENTER")
        t.setStyle(TableStyle([
            ("GRID", (0, 0), (-1, -1), 0.4, colors.HexColor("#9a9a9a")),
            ("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#efefef")),
            ("VALIGN", (0, 0), (-1, -1), "TOP"),
            ("TOPPADDING", (0, 0), (-1, -1), 3), ("BOTTOMPADDING", (0, 0), (-1, -1), 3),
            ("LEFTPADDING", (0, 0), (-1, -1), 4), ("RIGHTPADDING", (0, 0), (-1, -1), 4)]))
        t.spaceBefore, t.spaceAfter = 4, 10
        return t

    story = [Spacer(1, 10), Paragraph(f"{stats['pct']}% detected as AI", h1),
             Paragraph("The percentage indicates the share of analysed text that the detector classified as "
                       "likely AI-generated (high plus moderate likelihood).", small),
             Paragraph(f"<b>{band(stats['pct'])}</b>", S(fontName=sans, fontSize=10.5, leading=16, spaceBefore=6)),
             Paragraph("Important Notice", h2),
             Paragraph("This percentage is a probabilistic estimate from a machine-learning classifier and should be "
                       "treated as an indicator, not a conclusion. Results vary with writing style, subject matter, "
                       "non-native English and document structure, and false positives occur. Use this report as "
                       "one reference point alongside human judgement.", small),
             Paragraph("Guidelines", h2),
             Paragraph("The report analyses long-form prose. Highlight colours show the likelihood of a passage being "
                       "flagged: cyan for high, light purple for moderate, unhighlighted for low. Non-qualifying "
                       "content (headings, short bullet points, tables, captions, references) is not analysed. "
                       "Scanned or image-only pages cannot be read.", small),
             Paragraph("Details", h2)]
    t = Table([["File", esc(fname)[:60]], ["Word count", f"{stats['all_words']:,}"],
               ["Words analysed", f"{stats['analysed']:,}"], ["Document size", f"{fsize_kb:.1f} KB"],
               ["Detector", "CS AI Writing Suite detector"]], colWidths=[40 * mm, 110 * mm])
    t.setStyle(TableStyle([("FONT", (0, 0), (-1, -1), sans, 9.5), ("FONT", (0, 0), (0, -1), sans_b, 9.5),
                           ("BOTTOMPADDING", (0, 0), (-1, -1), 3)]))
    story += [t, Paragraph("Score Interpretation", h2)]
    t = Table([["0% – 19%", "Low likelihood of AI-generated content"],
               ["20% – 50%", "Moderate likelihood of AI-generated content"],
               ["51% – 100%", "High likelihood of AI-generated content"]], colWidths=[30 * mm, 120 * mm])
    t.setStyle(TableStyle([("FONT", (0, 0), (-1, -1), sans, 9.5), ("BOTTOMPADDING", (0, 0), (-1, -1), 3)]))
    story += [t, Paragraph("Highlight Legend", h2)]
    t = Table([["", f"High likelihood of being flagged (≥ {hi:.0%})"],
               ["", f"Moderate likelihood of being flagged ({mod:.0%} – {hi:.0%})"],
               ["", f"Low likelihood of being flagged (< {mod:.0%})"]],
              colWidths=[10 * mm, 140 * mm], rowHeights=6 * mm)
    t.setStyle(TableStyle([("FONT", (0, 0), (-1, -1), sans, 9.5),
                           ("BACKGROUND", (0, 0), (0, 0), colors.HexColor(HIGH_COLOR)),
                           ("BACKGROUND", (0, 1), (0, 1), colors.HexColor(MOD_COLOR)),
                           ("BOX", (0, 2), (0, 2), 0.5, colors.HexColor("#999999")),
                           ("VALIGN", (0, 0), (-1, -1), "MIDDLE")]))
    story += [t, Spacer(1, 14), Paragraph(f"Report generated: {now}", small)]
    if summary_only:
        blocks = []
    else:
        story.append(PageBreak())

    for b in blocks:
        if b.kind == "para":
            parts = []
            for s, p in zip(b.sents, b.scores):
                s = esc(s)
                if p >= hi:
                    s = f'<font backColor="{HIGH_COLOR}">{s}</font>'
                elif p >= mod:
                    s = f'<font backColor="{MOD_COLOR}">{s}</font>'
                parts.append(s)
            story.append(Paragraph(" ".join(parts), body))
        elif b.kind == "heading":
            story.append(Paragraph(esc(b.text), head))
        elif b.kind == "table" and b.rows:
            story.append(make_table(b.rows))
        elif b.kind == "table":
            story.append(Paragraph(esc(b.text).replace("\n", "<br/>"), tbls))
        else:
            story.append(Paragraph(esc(b.text), plain))

    doc = BaseDocTemplate(out_path, pagesize=A4, leftMargin=25 * mm, rightMargin=25 * mm,
                          topMargin=22 * mm, bottomMargin=20 * mm, title="AI Writing Report")
    doc.addPageTemplates([PageTemplate(id="p", frames=[
        Frame(doc.leftMargin, doc.bottomMargin, doc.width, doc.height, id="f")])])
    doc.build(story, canvasmaker=NumberedCanvas)
    return out_path

def build_overlay_pdf(src_pdf, blocks, stats, fname, fsize_kb, hi, mod, out_path):
    """Summary page + the ORIGINAL pages with highlight annotations."""
    import pymupdf as fitz
    rgb = lambda hx: tuple(int(hx[i:i + 2], 16) / 255 for i in (1, 3, 5))
    hc, mc = rgb(HIGH_COLOR), rgb(MOD_COLOR)
    src = fitz.open(src_pdf)
    pages = {}                       # keep Page objects alive while annotating
    for b in blocks:
        if b.kind != "para" or not b.spans:
            continue
        for (a, e), p in zip(b.spans, b.scores):
            if p < mod:
                continue
            col = hc if p >= hi else mc
            lines = {}
            for pno, bno, lno, bbox in b.toks[a:e]:
                key = (pno, bno, lno)
                lines[key] = lines[key] | fitz.Rect(bbox) if key in lines else fitz.Rect(bbox)
            for (pno, _, _), r in lines.items():
                pg = pages.get(pno) or pages.setdefault(pno, src[pno])
                an = pg.add_highlight_annot(r)
                an.set_colors(stroke=col)
                an.update()

    summ = out_path + ".summary.pdf"
    build_pdf([], stats, fname, fsize_kb, hi, mod, summ, summary_only=True)
    final = fitz.open(summ)
    n_summary = len(final)
    final.insert_pdf(src, annots=True)
    n = len(final)
    grey = (0.35, 0.35, 0.35)
    for i, page in enumerate(final):
        if page.rotation:
            continue
        w, h = page.rect.width, page.rect.height
        label = f"Page {i + 1} of {n}"
        page.insert_text(((w - fitz.get_text_length(label, fontsize=7)) / 2, h - 8), label,
                         fontsize=7, color=grey, overlay=True)
        if i >= n_summary:
            page.insert_text((20, 11), BRAND, fontsize=6.5, color=grey, overlay=True)
            rt = "AI WRITING REPORT"
            page.insert_text((w - 20 - fitz.get_text_length(rt, fontsize=6.5), 11), rt,
                             fontsize=6.5, color=grey, overlay=True)
    final.save(out_path, garbage=3, deflate=True)
    final.close(); src.close()
    os.remove(summ)
    return out_path

# ---------------------------------------------------------------- HTML preview
def build_html(blocks, stats, hi, mod):
    col = "#c0392b" if stats["pct"] > 50 else "#d68910" if stats["pct"] >= 20 else "#1e8449"
    out = [f"""<div style="font-family:system-ui;padding:14px 18px;border:1px solid #ddd;border-radius:10px;margin-bottom:12px">
<div style="font-size:34px;font-weight:700;color:{col}">{stats['pct']}% detected as AI</div>
<div style="color:#555">{band(stats['pct'])} · {stats['analysed']:,} of {stats['all_words']:,} words analysed</div>
<div style="margin-top:6px;font-size:13px;color:#111"><span style="background:{HIGH_COLOR};padding:1px 8px">High</span>
<span style="background:{MOD_COLOR};padding:1px 8px">Moderate</span>
<span style="padding:1px 8px;border:1px solid #ccc">Low</span></div></div>
<div style="font-family:Georgia,serif;line-height:1.9;max-height:640px;overflow:auto;padding:4px 12px;color:#111;background:#fff;border-radius:8px">"""]
    for b in blocks:
        if b.kind == "para":
            segs = []
            for s, p in zip(b.sents, b.scores):
                bg = HIGH_COLOR if p >= hi else MOD_COLOR if p >= mod else "transparent"
                segs.append(f'<span title="P(AI) = {p:.2f}" style="background:{bg}">{html.escape(s)}</span>')
            out.append("<p style='text-align:justify'>" + " ".join(segs) + "</p>")
        elif b.kind == "heading":
            out.append(f"<p><b>{html.escape(b.text)}</b></p>")
        elif b.kind == "table" and b.rows:
            td = "border:1px solid #bbb;padding:3px 8px;text-align:left;vertical-align:top"
            trs = []
            for i, r in enumerate(b.rows):
                tag = "th" if i == 0 else "td"
                bg = "background:#f0f0f0;" if i == 0 else ""
                trs.append("<tr>" + "".join(f"<{tag} style='{bg}{td}'>{html.escape(c)}</{tag}>" for c in r) + "</tr>")
            out.append("<table style='border-collapse:collapse;margin:6px auto 14px;font-size:0.88em'>"
                       + "".join(trs) + "</table>")
        else:
            out.append(f"<p style='color:#777;font-size:0.9em'>{html.escape(b.text).replace(chr(10), '<br>')}</p>")
    out.append("</div>")
    return "".join(out)

# ---------------------------------------------------------------- pipeline
def load_blocks(file, pasted):
    """Returns (blocks, fname, size_kb, source_pdf_or_None, docx_document_or_None)."""
    src_pdf, doc = None, None
    if file is not None:
        path = file if isinstance(file, str) else file.name
        ext = os.path.splitext(path)[1].lower()
        fname, size_kb = os.path.basename(path), os.path.getsize(path) / 1024
        if ext == ".docx":
            try:
                doc = DX.open_clean(path)
            except Exception:
                raise gr.Error("Could not open this .docx (is it a valid Word file, not .doc?).")
            return docx_blocks(doc), fname, size_kb, None, doc
        elif ext == ".pdf":
            src_pdf = path
            blocks = extract_pdf_layout(path)
        elif ext in (".txt", ".md"):
            with open(path, encoding="utf-8", errors="ignore") as fh:
                blocks = extract_text(fh.read())
        else:
            raise gr.Error("Upload a .docx, .pdf or .txt file.")
    elif pasted and pasted.strip():
        blocks, fname, size_kb = extract_text(pasted), "Pasted text", len(pasted.encode()) / 1024
        return classify(blocks, PASTE_MIN_WORDS), fname, size_kb, None, None
    else:
        raise gr.Error("Upload a file or paste text.")
    if file is not None and os.path.splitext(fname)[1].lower() in (".txt", ".md"):
        return classify(blocks, PASTE_MIN_WORDS), fname, size_kb, src_pdf, doc
    return classify(blocks), fname, size_kb, src_pdf, doc

def run(file, pasted, hi, mod, progress=gr.Progress()):
    if mod >= hi:
        raise gr.Error("Moderate threshold must be below the high threshold.")
    progress(0.05, desc="Reading document")
    blocks, fname, size_kb, src_pdf, doc = load_blocks(file, pasted)
    if not any(b.kind == "para" for b in blocks):
        raise gr.Error("No qualifying prose found — paste full sentences (headings and fragments are "
                       "not analysed). Scanned/image-only PDFs have no readable text.")
    progress(0.2, desc="Running detector")
    blocks = analyse(blocks)
    stats = summarise(blocks, hi, mod)
    base = re.sub(r"[^\w\-]+", "_", os.path.splitext(fname)[0])[:40] or "document"
    work = tempfile.mkdtemp()
    pdf = os.path.join(work, f"AI_Report_{base}.pdf")
    warn = ""

    if doc is not None:                                   # ---- Word: inject, then render
        progress(0.7, desc="Highlighting the Word file")
        for b in blocks:
            if b.kind == "para" and b.el is not None and b.cspans:
                spans = [(sp[0], sp[1], (HIGH_COLOR if p >= hi else MOD_COLOR).lstrip("#"))
                         for sp, p in zip(b.cspans, b.scores) if sp and p >= mod]
                DX.colour_paragraph(b.el, spans)
        docx_tmp = os.path.join(work, f"{base}_highlighted.docx")
        doc.save(docx_tmp)
        progress(0.8, desc="Converting to PDF")
        rendered = DX.convert_to_pdf(docx_tmp)
        if rendered:
            want, got = DX.count_docx_pictures(doc), DX.count_pdf_images(rendered)
            if got < want:
                warn = (f"{want - got} of {want} pictures from the Word file did not render in the PDF. "
                        "For a report identical to Word, save the manuscript as PDF from Word "
                        "and upload that PDF instead.")
            progress(0.9, desc="Building report")
            build_overlay_pdf(rendered, [], stats, fname, size_kb, hi, mod, pdf)
        else:
            warn = ("Word-to-PDF conversion failed, so this report is re-typeset text without "
                    "figures/equations. Reason: " + (DX.LAST_ERROR or "unknown") +
                    " — Workaround: save the manuscript as PDF from Word and upload the PDF.")
            build_pdf(blocks, stats, fname, size_kb, hi, mod, pdf)
    elif src_pdf:                                         # ---- PDF: annotate original pages
        progress(0.85, desc="Building report")
        build_overlay_pdf(src_pdf, blocks, stats, fname, size_kb, hi, mod, pdf)
    else:                                                 # ---- plain text
        progress(0.85, desc="Building report")
        build_pdf(blocks, stats, fname, size_kb, hi, mod, pdf)

    if stats["analysed"] < 50:
        warn = ((warn + " ") if warn else "") + (
            f"Only {stats['analysed']} words could be analysed. The detector is much less reliable "
            "below ~50 words — paste a longer passage for a trustworthy score.")
    meta = getattr(progress, "meta", None)          # run log (dashboard)
    if isinstance(meta, dict):
        meta.update(name=fname, words=stats["all_words"], before=stats["pct"], output=pdf, engine=LAST_ENGINE)
    page = build_html(blocks, stats, hi, mod)
    if warn:
        page = (f"<div style='font-family:system-ui;background:#fff4e5;border:1px solid #f0b458;"
                f"color:#6b4400;padding:10px 14px;border-radius:8px;margin-bottom:10px'>⚠ {html.escape(warn)}</div>"
                + page)
    return page, pdf


# (at the end of the module: half_check and everything it uses are defined by now)
if os.getenv("SKIP_MODEL_LOAD") != "1":
    __import__("threading").Thread(target=half_check, daemon=True).start()