Upload pasita.py with huggingface_hub

pasita.py

@@ -1,471 +1,1193 @@
-import html
+"""Self-contained inference harness for OpceanAI/PASITA text-to-Markdown.
+
+No gradio/spaces imports. Works on CPU and bf16 GPU. CLI:
+    python3 harness.py --text-file doc.txt
+"""
+
+from __future__ import annotations
+
+import argparse
+import math
+import re
 import time
+from dataclasses import dataclass, field
+from html.parser import HTMLParser
+from typing import Any
 
-import spaces
 import torch
-import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-from pasita import infer_kind, convert_to_markdown
 
 MODEL_ID = "OpceanAI/PASITA"
-MODEL_CARD = "https://huggingface.co/OpceanAI/PASITA"
+CONTEXT_LEN = 2048
+DEFAULT_HARD_CAP = 512
 
-tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
-tokenizer.padding_side = "left"
-model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype="auto").to("cuda")
-model.eval()
 
-EMPTY_MD = "_The converted Markdown appears here. Paste a document on the left and press Convert._"
+def build_prompt(text: str, mode: str = "canonical", instruction: str | None = None) -> str:
+    if instruction is not None:
+        return instruction + text + "\n\n### Markdown:\n"
+    if mode == "canonical":
+        return "CONVIERTE A MARKDOWN:\n" + text + "\n\n### Markdown:\n"
+    if mode == "english":
+        return "CONVERT TO MARKDOWN:\n" + text + "\n\n### Markdown:\n"
+    if mode == "spanish_instr":
+        return (
+            "Transforma el siguiente texto en Markdown, conservando toda la información:\n\n"
+            + text
+            + "\n\n### Markdown:\n"
+        )
+    if mode == "fewshot":
+        return (
+            "CONVIERTE A MARKDOWN:\n\n"
+            "### Ejemplo 1\n"
+            "Entrada: <b>Aviso</b><p>Entrega en 24 h.</p>\n"
+            "Salida:\n"
+            "## Aviso\n\n"
+            "Entrega en 24 h.\n\n"
+            "### Ejemplo 2\n"
+            "Entrada: producto: teclado; precio: 49,90 €\n"
+            "Salida:\n"
+            "- **producto:** teclado\n"
+            "- **precio:** 49,90 €\n\n"
+            "### Documento\n"
+            + text
+            + "\n\n### Markdown:\n"
+        )
+    if mode == "plain":
+        return text
+    if mode == "md_label":
+        return "CONVIERTE A MARKDOWN:\n" + text + "\n\nMarkdown:\n"
+    if mode == "sep_only":
+        return text + "\n\n### Markdown:\n"
+    if mode == "doc_instr_es":
+        return "Convierte este documento a Markdown:\n" + text + "\n\n### Markdown:\n"
+    if mode == "doc_instr_en":
+        return "Convert this document to Markdown:\n" + text + "\n\n### Markdown:\n"
+    if mode == "csv_instr":
+        return "Convierte este CSV a tabla markdown:\n" + text + "\n\n### Markdown:\n"
+    if mode == "html_instr":
+        return "Convierte este HTML a Markdown:\n" + text + "\n\n### Markdown:\n"
+    if mode == "html_instr_en":
+        return "Convert this HTML to Markdown:\n" + text + "\n\n### Markdown:\n"
+    if mode == "fewshot2":
+        return (
+            "CONVIERTE A MARKDOWN:\n\n"
+            "Ejemplo:\n"
+            "Entrada: Puntos\n"
+            "1. Primero.\n"
+            "2. Segundo.\n"
+            "Salida:\n"
+            "## Puntos\n\n"
+            "1. Primero.\n"
+            "2. Segundo.\n\n"
+            "Documento:\n"
+            + text
+            + "\n\n### Markdown:\n"
+        )
+    if mode == "respuesta":
+        return "CONVIERTE A MARKDOWN:\n" + text + "\n\n### Respuesta:\n"
+    if mode == "output":
+        return "CONVIERTE A MARKDOWN:\n" + text + "\n\n### Output:\n"
+    raise ValueError(f"unknown prompt mode: {mode}")
+
+
+@dataclass
+class GenerationResult:
+    candidates: list[str]
+    raw_candidates: list[str]
+    prompt: str
+    prompt_tokens: int
+    max_new_tokens: int
+    seconds: float
+    strategy: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def best_index(self) -> int:
+        return self.strategy.get("best_index", 0)
+
+    @property
+    def best(self) -> str:
+        return self.candidates[self.best_index]
+
+
+def _model_device(model) -> torch.device:
+    try:
+        return next(model.parameters()).device
+    except StopIteration:
+        return torch.device("cpu")
 
-GLYPH = (
-    '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" '
-    'stroke-linecap="round" aria-hidden="true">'
-    '<path d="M9 4 7 20M17 4l-2 16M4 9h16M3 15h16"/></svg>'
-)
 
+def compute_max_new_tokens(prompt_tokens: int, ratio: float = 1.5, hard_cap: int = DEFAULT_HARD_CAP) -> int:
+    budget = CONTEXT_LEN - prompt_tokens - 1
+    if budget <= 16:
+        return 16
+    by_ratio = int(math.ceil(prompt_tokens * ratio))
+    return max(16, min(hard_cap, budget, by_ratio))
 
-def _status(text: str, state: str = "ready", detail: str = "") -> str:
-    detail_html = f'<span class="p-detail">{html.escape(detail)}</span>' if detail else ""
-    return (
-        f'<div class="p-status {state}"><span class="p-dot"></span>'
-        f'<span class="p-text">{html.escape(text)}</span>{detail_html}</div>'
+
+def _truncate_text_to_tokens(tokenizer, text: str, max_tokens: int) -> str:
+    ids = tokenizer.encode(text, add_special_tokens=False)
+    if len(ids) <= max_tokens:
+        return text
+    head = int(max_tokens * 0.6)
+    tail = max_tokens - head
+    head_text = tokenizer.decode(ids[:head])
+    tail_text = tokenizer.decode(ids[-tail:]) if tail > 0 else ""
+    return head_text + "\n...\n" + tail_text
+
+
+def generate_candidates(
+    model,
+    tokenizer,
+    text: str,
+    *,
+    mode: str = "canonical",
+    instruction: str | None = None,
+    num_candidates: int = 1,
+    num_beams: int = 1,
+    do_sample: bool = False,
+    temperature: float = 1.0,
+    top_p: float = 0.95,
+    no_repeat_ngram_size: int = 0,
+    repetition_penalty: float = 1.0,
+    max_new_tokens: int | str | None = None,
+    max_new_ratio: float = 1.5,
+    hard_cap: int = DEFAULT_HARD_CAP,
+    use_cache: bool = True,
+    early_stopping: bool | None = None,
+    seed: int = 0,
+    device: torch.device | str | None = None,
+    **gen_kwargs: Any,
+) -> GenerationResult:
+    tokenizer.padding_side = "left"
+    prompt = build_prompt(text, mode, instruction)
+    max_new = max_new_tokens
+    if max_new is None or max_new == "auto":
+        prompt_ids = tokenizer(prompt, add_special_tokens=True)["input_ids"]
+        if len(prompt_ids) > CONTEXT_LEN - 64:
+            text = _truncate_text_to_tokens(tokenizer, text, CONTEXT_LEN - 96)
+            prompt = build_prompt(text, mode, instruction)
+            prompt_ids = tokenizer(prompt, add_special_tokens=True)["input_ids"]
+        max_new = compute_max_new_tokens(len(prompt_ids), max_new_ratio, hard_cap)
+    else:
+        max_new = int(max_new)
+
+    inputs = tokenizer(
+        prompt,
+        return_tensors="pt",
+        add_special_tokens=True,
+        truncation=True,
+        max_length=CONTEXT_LEN - max_new,
+    )
+    if device is None:
+        device = _model_device(model)
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+    prompt_tokens = int(inputs["input_ids"].shape[1])
+
+    if do_sample:
+        torch.manual_seed(seed)
+    if early_stopping is None:
+        early_stopping = num_beams > 1
+
+    generate_kwargs: dict[str, Any] = dict(
+        max_new_tokens=max_new,
+        do_sample=do_sample,
+        num_beams=max(1, num_beams),
+        num_return_sequences=max(1, num_candidates),
+        use_cache=use_cache,
+        pad_token_id=tokenizer.pad_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+        early_stopping=early_stopping,
     )
+    if no_repeat_ngram_size:
+        generate_kwargs["no_repeat_ngram_size"] = no_repeat_ngram_size
+    if repetition_penalty and repetition_penalty != 1.0:
+        generate_kwargs["repetition_penalty"] = repetition_penalty
+    if do_sample:
+        generate_kwargs["temperature"] = temperature
+        generate_kwargs["top_p"] = top_p
+    generate_kwargs.update(gen_kwargs)
 
+    t0 = time.perf_counter()
+    with torch.inference_mode():
+        out = model.generate(**inputs, **generate_kwargs)
+    seconds = time.perf_counter() - t0
 
-def _counter(text: str) -> str:
-    text = text or ""
-    words = len(text.split())
-    return (
-        '<span class="p-label">PLAIN TEXT</span>'
-        f'<span class="p-count">{len(text):,} CHARS / {words:,} WORDS</span>'
+    new_tokens = out[:, prompt_tokens:]
+    raw_candidates = tokenizer.batch_decode(new_tokens, skip_special_tokens=True)
+    strategy = {
+        "mode": mode,
+        "num_candidates": num_candidates,
+        "num_beams": max(1, num_beams),
+        "do_sample": do_sample,
+        "temperature": temperature if do_sample else 0.0,
+        "no_repeat_ngram_size": no_repeat_ngram_size,
+        "repetition_penalty": repetition_penalty,
+        "max_new_tokens": max_new,
+        "max_new_ratio": max_new_ratio,
+    }
+    return GenerationResult(
+        candidates=[postprocess(c, text) for c in raw_candidates],
+        raw_candidates=raw_candidates,
+        prompt=prompt,
+        prompt_tokens=prompt_tokens,
+        max_new_tokens=max_new,
+        seconds=seconds,
+        strategy=strategy,
     )
 
 
-def _estimate_duration(text, num_candidates=3, structured_fallback=True, *args, **kwargs):
-    if structured_fallback and infer_kind((text or "").strip()) in ("html", "data"):
-        return 10
-    return min(30, max(12, 8 + len(text or "") // 200 + 4 * (int(num_candidates) - 1)))
+_MARKER_RE = re.compile(
+    r"^\s*(?:#{0,6}\s*)?(?:###\s*markdown\s*:?\s*|##\s*answer\s*:?\s*|"
+    r"conviert[ea] a markdown\s*:?\s*|convert to markdown\s*:?\s*|"
+    r"transforma el siguiente texto[^\n]*|aqu[íi] tienes[^\n]*|"
+    r"entrada\s*:|salida\s*:|\[your answer\])\s*$",
+    re.IGNORECASE,
+)
+
+
+def _norm_ws(s: str) -> str:
+    return re.sub(r"\s+", " ", s).strip()
+
+
+def _norm_cmp(s: str) -> str:
+    s = s.lower()
+    s = re.sub(r"[\u2010-\u2015\u2212]", "-", s)
+    s = s.replace("\u00a0", " ")
+    return _norm_ws(s)
+
+
+def _is_marker(line: str) -> bool:
+    return bool(_MARKER_RE.match(line))
+
+
+def _strip_prompt_echo(text: str) -> str:
+    lines = text.splitlines()
+    while lines and (not lines[0].strip() or _is_marker(lines[0])):
+        lines.pop(0)
+    while lines and (not lines[-1].strip() or _is_marker(lines[-1])):
+        lines.pop()
+    for i, line in enumerate(lines):
+        if i >= 1 and _is_marker(line) and len(_norm_ws(" ".join(lines[:i]))) >= 30:
+            lines = lines[:i]
+            break
+    return "\n".join(lines).strip()
+
+
+def _unwrap_code_fence(text: str) -> str:
+    lines = text.strip().splitlines()
+    if len(lines) < 3:
+        return text
+    if not lines[0].strip().startswith("```") or lines[-1].strip() != "```":
+        return text
+    inner = lines[1:-1]
+    if any(l.strip().startswith("```") for l in inner):
+        return text
+    return "\n".join(inner).strip()
 
 
-@spaces.GPU(duration=_estimate_duration)
-def convert(text: str, num_candidates: int = 3, structured_fallback: bool = True) -> tuple[str, str, str]:
-    """Convert a plain-text document into faithful GitHub-Flavored Markdown.
+def _dedup_leading_h1(text: str) -> str:
+    lines = text.strip().splitlines()
+    if not lines or not re.match(r"^#\s+\S", lines[0].strip()):
+        return text
+    head = _norm_cmp(lines[0].lstrip("# "))
+    rest = "\n".join(lines[1:])
+    if head and head in _norm_cmp(rest):
+        return rest.strip()
+    return text
 
-    PASITA samples the document several times, every candidate is post-processed and
-    scored for number fidelity, content coverage and GFM validity, and the best one is
-    returned. Inputs detected as HTML or delimited data are converted deterministically.
 
-    Args:
-        text: The document to convert, for example OCR output, pasted HTML, meeting notes or a report.
-        num_candidates: How many PASITA samples to generate and rerank, from 1 to 3.
-        structured_fallback: Use the deterministic converter for HTML and delimited data.
-    Returns:
-        A tuple of the rendered Markdown, the raw Markdown and a status line.
-    """
-    text = (text or "").strip()
+def _fix_double_heading(text: str) -> str:
+    out = []
+    for line in text.splitlines():
+        m = re.match(r"^\s*#{1,6}\s+(#{1,6}\s+.+?)\s*$", line)
+        if m:
+            line = m.group(1)
+        out.append(line)
+    return "\n".join(out)
+
+
+def _space_heading_markers(text: str) -> str:
+    return re.sub(r"(?m)^(\s*#{1,6})(?=[^\s#])", r"\1 ", text)
+
+
+def _cut_repeated_tail_paragraphs(text: str, input_text: str) -> str:
+    paras = re.split(r"\n\s*\n", text.strip())
+    if len(paras) < 2:
+        return text
+    in_norm = _norm_cmp(input_text) if input_text else ""
+    seen: dict[str, int] = {}
+    for i, p in enumerate(paras):
+        n = _norm_cmp(p)
+        if len(n) < 40 or (in_norm and n in in_norm):
+            continue
+        key = n[:60]
+        if key in seen:
+            return "\n\n".join(paras[: seen[key]]).strip()
+        seen[key] = i
+    for i in range(max(1, len(paras) - 3), len(paras)):
+        p = paras[i].strip()
+        if p.startswith("#") and "\n" not in p:
+            body = _norm_cmp(p.lstrip("# "))
+            if len(body) >= 6:
+                earlier = _norm_cmp("\n\n".join(paras[:i]))
+                if body in earlier:
+                    return "\n\n".join(paras[:i]).strip()
+    changed = True
+    while changed and len(paras) >= 2:
+        changed = False
+        last = _norm_cmp(paras[-1])
+        if len(last) < 40:
+            break
+        earlier = [_norm_cmp(p) for p in paras[:-1]]
+        if last in earlier or any(last[:60] == e[:60] for e in earlier if len(e) >= 60):
+            paras.pop()
+            changed = True
+    return "\n\n".join(paras).strip()
+
+
+def _cut_repeated_tail_ngram(text: str, input_text: str, min_n: int = 6) -> str:
+    paras = re.split(r"\n\s*\n", text.strip())
+    if len(paras) < 3:
+        return text
+    flat: list[str] = []
+    owner: list[int] = []
+    for pi, p in enumerate(paras):
+        for w in _norm_cmp(p).split():
+            flat.append(w)
+            owner.append(pi)
+    n = len(flat)
+    best: tuple[int, int, int] | None = None
+    for size in range(min(16, n // 2), min_n - 1, -1):
+        seen: dict[tuple[str, ...], int] = {}
+        for i in range(n - size + 1):
+            gram = tuple(flat[i : i + size])
+            if gram in seen:
+                j = seen[gram]
+                if owner[j] < owner[i] and owner[i] >= len(paras) - 3:
+                    if best is None or owner[i] < best[0] or (owner[i] == best[0] and owner[j] < best[1]):
+                        best = (owner[i], owner[j], size)
+            else:
+                seen[gram] = i
+        if best is not None:
+            break
+    if best is None:
+        return text
+    s2, f, size = best
+    in_norm = _norm_cmp(input_text) if input_text else ""
+    for k in range(f + 1, s2):
+        nk = _norm_cmp(paras[k])
+        if len(nk) >= 30 and (not in_norm or nk not in in_norm):
+            return "\n\n".join(paras[:k]).strip()
+    if size >= 12:
+        return "\n\n".join(paras[:s2]).strip()
+    return text
+
+
+def _norm_line(line: str) -> str:
+    return _norm_cmp(re.sub(r"^\s*#{1,6}\s*", "", line))
+
+
+def _cut_repeated_line_blocks(text: str) -> str:
+    lines = text.rstrip().splitlines()
+    changed = True
+    while changed and len(lines) >= 2:
+        changed = False
+        n = len(lines)
+        for k in range(n // 2, 0, -1):
+            if lines[n - k :] != lines[n - 2 * k : n - k]:
+                if [_norm_line(l) for l in lines[n - k :]] != [
+                    _norm_line(l) for l in lines[n - 2 * k : n - k]
+                ]:
+                    continue
+            block = [l for l in lines[n - k :] if l.strip()]
+            if not block:
+                continue
+            if k == 1:
+                line = lines[n - 1]
+                long_enough = len(line.split()) >= 6
+                if not long_enough or text.count(line.strip()) < 3:
+                    continue
+            lines = lines[: n - k]
+            changed = True
+            break
+    return "\n".join(lines).rstrip()
+
+
+def _content_coverage(text: str, input_words: set[str]) -> float:
+    if not input_words:
+        return 1.0
+    words = set(_content_words(text))
+    return len(input_words & words) / len(input_words)
+
+
+def _cut_after_completion(text: str, input_text: str, min_cov: float = 0.85) -> str:
+    if not input_text:
+        return text
+    in_nums = extract_number_keys(input_text)
+    if not in_nums:
+        return text
+    in_words = set(_content_words(input_text))
+    lines = text.splitlines()
+    seen: set[str] = set()
+    for i, line in enumerate(lines):
+        seen |= extract_number_keys(line)
+        if in_nums <= seen:
+            candidate = "\n".join(lines[: i + 1]).rstrip()
+            if candidate and _content_coverage(candidate, in_words) >= min_cov:
+                return candidate
+            break
+    return text
+
+
+def _cut_repeated_char_tail(text: str) -> str:
+    t = text.rstrip()
+    n = len(t)
+    for p in range(24, min(800, n // 2) + 1):
+        block = t[n - p :]
+        if not block.strip():
+            continue
+        reps = 1
+        while (reps + 1) * p <= n and t[n - (reps + 1) * p : n - reps * p] == block:
+            reps += 1
+        if reps >= 3:
+            return t[: n - reps * p].rstrip()
+        if reps >= 2 and p >= 120:
+            return t[: n - reps * p].rstrip()
+    return t
+
+
+def _cut_input_echo(text: str, input_text: str) -> str:
+    if not input_text:
+        return text
+    in_lines = [l.strip() for l in input_text.strip().splitlines() if l.strip()]
+    out_lines = text.rstrip().splitlines()
+    for k in range(min(8, len(out_lines), len(in_lines)), 1, -1):
+        tail = [l.strip() for l in out_lines[-k:]]
+        if any(not l for l in tail):
+            continue
+        joined = _norm_ws(" ".join(tail))
+        if len(joined.split()) < 8 and k < 2:
+            continue
+        for i in range(len(in_lines) - k + 1):
+            if in_lines[i : i + k] == tail:
+                return "\n".join(out_lines[: len(out_lines) - k]).rstrip()
+    if out_lines:
+        tail = _norm_ws(out_lines[-1])
+        earlier = _norm_cmp("\n".join(out_lines[:-1]))
+        if len(tail.split()) >= 8 and tail in _norm_cmp(input_text) and _norm_cmp(tail) in earlier:
+            return "\n".join(out_lines[:-1]).rstrip()
+    return text
+
+
+def _repair_headings(text: str, input_text: str) -> str:
+    if not input_text:
+        return text
+    import difflib
+
+    source_lines = [l.strip() for l in input_text.splitlines() if len(l.strip()) >= 8]
+    if not source_lines:
+        return text
+    out_lines = text.splitlines()
+    for i, line in enumerate(out_lines):
+        m = re.match(r"^(\s*#{1,6}\s+)(.+?)\s*$", line)
+        if not m:
+            continue
+        body = m.group(2).strip()
+        if len(body) < 8:
+            continue
+        best_ratio, best_src = 0.0, None
+        for src in source_lines:
+            ratio = difflib.SequenceMatcher(None, body.lower(), src.lower()).ratio()
+            if ratio > best_ratio:
+                best_ratio, best_src = ratio, src
+        if best_src and 0.88 <= best_ratio < 1.0:
+            out_lines[i] = m.group(1) + best_src
+    return "\n".join(out_lines)
+
+
+def _strip_leading_dot(text: str) -> str:
+    return re.sub(r"(?m)^(\s*)\.\s+", r"\1", text)
+
+
+def _passthrough(text: str) -> str:
+    lines = [l.rstrip() for l in text.splitlines()]
+    out: list[str] = []
+    blank = 0
+    for line in lines:
+        if line.strip():
+            if blank:
+                out.append("")
+                blank = 0
+            out.append(line)
+        else:
+            blank += 1
+    return "\n".join(out).strip()
+
+
+def postprocess(raw: str, input_text: str | None = None) -> str:
+    text = raw.strip()
     if not text:
-        return EMPTY_MD, "", _status("READY / PASTE A DOCUMENT TO CONVERT")
+        return raw.strip()
+    original = text
+    text = _unwrap_code_fence(text)
+    text = _strip_prompt_echo(text)
+    text = _cut_input_echo(text, input_text or "")
+    text = _dedup_leading_h1(text)
+    text = _repair_headings(text, input_text or "")
+    text = _fix_double_heading(text)
+    text = _space_heading_markers(text)
+    text = _cut_after_completion(text, input_text or "")
+    text = _cut_repeated_tail_paragraphs(text, input_text or "")
+    text = _cut_repeated_line_blocks(text)
+    text = _cut_repeated_tail_ngram(text, input_text or "")
+    text = _cut_repeated_char_tail(text)
+    text = _strip_leading_dot(text)
+    text = _strip_prompt_echo(text)
+    text = "\n".join(l.rstrip() for l in text.splitlines()).strip()
+    return text if text else original
+
+
+NUM_RE = re.compile(r"\d+(?:[.,]\d+)?%?")
+WORD_RE = re.compile(r"[a-zA-ZáéíóúüñçÁÉÍÓÚÜÑÇ]{3,}")
+LIST_RE = re.compile(r"^\s*(?:[-*•]|\d+[.)])\s+\S", re.M)
+TABLE_ROW_RE = re.compile(r"^\s*\|.*\|\s*$", re.M)
+FENCE_RE = re.compile(r"^\s*(```|~~~)", re.M)
 
-    started = time.perf_counter()
+STOPWORDS = {
+    "the", "and", "for", "are", "but", "not", "you", "your", "with", "that", "this",
+    "from", "have", "has", "was", "were", "will", "would", "can", "could", "should",
+    "into", "than", "then", "them", "they", "their", "there", "these", "those", "our",
+    "its", "it's", "also", "some", "any", "all", "each", "when", "where", "which",
+    "who", "what", "how", "why", "does", "did", "been", "being", "about", "over",
+    "under", "after", "before", "between", "same", "more", "most", "other", "only",
+    "los", "las", "una", "uno", "unos", "unas", "del", "para", "por", "con", "sin",
+    "que", "como", "más", "mas", "pero", "sus", "este", "esta", "estos", "estas",
+    "eso", "esa", "esos", "esas", "son", "sea", "fue", "han", "hay", "muy", "sin",
+    "sobre", "entre", "cuando", "donde", "porque", "también", "tambien", "todos",
+    "toda", "todo", "cada", "desde", "hasta", "según", "segun", "solo", "sólo",
+    "nos", "les", "nuestro", "nuestra", "vuestro", "vuestra", "del", "al", "se",
+}
+
+
+def _num_key(token: str) -> str | None:
+    t = token.rstrip("%").strip()
+    if not t:
+        return None
+    has_c, has_d = "," in t, "." in t
+    if has_c and has_d:
+        if t.rfind(",") > t.rfind("."):
+            t = t.replace(".", "").replace(",", ".")
+        else:
+            t = t.replace(",", "")
+    elif has_c:
+        parts = t.split(",")
+        if len(parts[-1]) == 3 and len(parts) == 2 and len(parts[0]) <= 3:
+            t = t.replace(",", "")
+        else:
+            t = t.replace(",", ".")
+    elif has_d:
+        parts = t.split(".")
+        if len(parts) == 2 and len(parts[-1]) == 3 and len(parts[0]) <= 3:
+            t = t.replace(".", "")
     try:
-        result = convert_to_markdown(
-            model,
-            tokenizer,
-            text,
-            num_candidates=max(1, min(3, int(num_candidates))),
-            do_sample=True,
-            temperature=0.3,
-            top_p=0.95,
-            seed=0,
-            use_cache=True,
-            structured_fallback=bool(structured_fallback),
-            skip_model_for_structured=True,
-        )
-    except Exception as exc:
-        message = html.escape(f"{type(exc).__name__}: {exc}")
-        return (
-            f"> Conversion failed.\n>\n> `{message}`",
-            "",
-            _status(f"ERROR / {type(exc).__name__}", "error", f"{time.perf_counter() - started:.1f}S"),
-        )
+        return format(float(t), ".6g")
+    except ValueError:
+        return t
+
+
+def extract_number_keys(text: str) -> set[str]:
+    keys = set()
+    for m in NUM_RE.finditer(text):
+        k = _num_key(m.group(0))
+        if k is not None and k != "0":
+            keys.add(k)
+    return keys
+
 
-    metrics = result["metrics"]
-    numbers = f"{metrics['numbers_matched']}/{metrics['numbers_in']} NUMBERS"
-    overlap = f"{metrics['content_overlap']:.0%} CONTENT"
-    routes = {
-        "model": "PASITA + RERANK",
-        "structured": "HTML/TABLE CONVERTER",
-        "passthrough": "SOURCE PASSTHROUGH",
+def _content_words(text: str) -> list[str]:
+    words = [w.lower() for w in WORD_RE.findall(text)]
+    return [w for w in words if w not in STOPWORDS]
+
+
+def _mk():
+    from markdown_it import MarkdownIt
+
+    return MarkdownIt("commonmark").enable("table").enable("strikethrough")
+
+
+_MD = None
+
+
+def _count_html_tokens(tokens) -> int:
+    total = 0
+    for t in tokens:
+        if t.type in ("html_block", "html_inline"):
+            total += 1
+        if getattr(t, "children", None):
+            total += _count_html_tokens(t.children)
+    return total
+
+
+def gfm_stats(text: str) -> dict[str, Any]:
+    global _MD
+    if _MD is None:
+        try:
+            _MD = _mk()
+        except Exception:
+            return {"html_tokens": 0, "malformed_tables": 0, "malformed_links": 0, "fences_even": True, "gfm_clean": None}
+    try:
+        tokens = _MD.parse(text)
+    except Exception:
+        return {"html_tokens": 0, "malformed_tables": 1, "malformed_links": 0, "fences_even": True, "gfm_clean": 0.0}
+    html_tokens = _count_html_tokens(tokens)
+    tr_open = sum(1 for t in tokens if t.type == "tr_open")
+    pipe_lines = TABLE_ROW_RE.findall(text)
+    malformed = 0
+    if pipe_lines and tr_open == 0:
+        malformed = 1
+    link_opens = len(re.findall(r"\]\(", text))
+    valid_links = 0
+    for link in re.finditer(r"\[[^\]]*\]\(([^)]*)\)", text):
+        dest = link.group(1)
+        if " " not in dest and "\n" not in dest:
+            valid_links += 1
+    malformed_links = max(0, link_opens - valid_links)
+    if text.count("](") > valid_links and re.search(r"\]\([^)]*$", text, re.M):
+        malformed_links = max(malformed_links, 1)
+    fences = FENCE_RE.findall(text)
+    fences_even = len(fences) % 2 == 0
+    clean = 1.0
+    if html_tokens:
+        clean -= 0.25 * min(4, html_tokens)
+    if malformed:
+        clean -= 0.5
+    if malformed_links:
+        clean -= 0.25 * min(4, malformed_links)
+    if not fences_even:
+        clean -= 0.25
+    return {
+        "html_tokens": html_tokens,
+        "malformed_tables": malformed,
+        "malformed_links": malformed_links,
+        "fences_even": fences_even,
+        "gfm_clean": max(0.0, clean),
     }
-    source_label = routes.get(result.get("route"), "PASITA + RERANK")
-    taken = result["seconds"] if result["seconds"] else time.perf_counter() - started
-    detail = f"{source_label} / {numbers} / {overlap} / {taken:.1f}S"
-    return result["markdown"], result["markdown"], _status("DONE", "ready", detail)
-
-
-def reset() -> tuple[str, str, str, str]:
-    return EMPTY_MD, "", _status("READY"), _counter("")
-
-
-BUSY_JS = """(text, num_candidates, structured_fallback, md, source, status) => {
-  const n = Number(num_candidates) || 1;
-  const label = n > 1 ? 'PASITA ' + n + ' SAMPLES / RERANK' : 'PASITA SAMPLE / RERANK';
-  return [md, source,
-    '<div class="p-status busy"><span class="p-dot"></span><span class="p-text">CONVERTING</span><span class="p-detail">' + label + '</span></div>'];
-}"""
-
-COUNTER_JS = """(text) => {
-  const t = (text || '').trim();
-  const words = t ? t.split(/\\s+/).length : 0;
-  const chars = (text || '').length;
-  return '<span class="p-label">PLAIN TEXT</span><span class="p-count">' + chars.toLocaleString() + ' CHARS / ' + words.toLocaleString() + ' WORDS</span>';
-}"""
-
-
-HEADER = f"""
-<header class="p-head">
-  <div class="p-brand">
-    <span class="p-glyph">{GLYPH}</span>
-    <span class="p-name">PASITA</span>
-    <span class="p-tag">/ 88M FROM SCRATCH</span>
-  </div>
-  <div class="p-actions">
-    <a class="p-btn" href="{MODEL_CARD}" target="_blank" rel="noopener">MODEL CARD</a>
-  </div>
-</header>
-"""
 
-FOOTER = f"""
-<footer class="p-foot">
-  <span>PASITA v1 converts plain text to Markdown, in Spanish and English. It works best on medium and long documents.</span>
-  <span>
-    <a href="{MODEL_CARD}" target="_blank" rel="noopener">MODEL</a> /
-    <a href="?view=api" target="_blank" rel="noopener">API</a> /
-    RUNS ON ZEROGPU
-  </span>
-</footer>
-"""
 
-EXAMPLES = [
-    [
-        """RESUMEN MENSUAL DE OPERACIONES: MARZO 2026
-
-Resumen ejecutivo
-Los pedidos gestionados alcanzaron 18.420 unidades, un 6,1% más que en febrero. El tiempo medio de preparación bajó a 2,4 horas.
-
-Detalle por almacén
-Madrid: 8.120 pedidos (+7,4%)
-Barcelona: 5.480 pedidos (+3,9%)
-Lisboa: 4.820 pedidos (+6,7%)
-
-Incidencias
-Se registraron 96 incidencias de embalaje, el 0,52% del total. La previsión para abril es de 19.100 pedidos."""
-    ],
-    [
-        """Team sync, 2026-08-14
-Attendees: Ana, Marc, Priya, Tom
-
-Decisions
-1. Ship the new onboarding flow on Sept 2.
-2. Postpone the pricing experiment to Q4.
-3. Marc owns the migration checklist; due Aug 22.
-
-Open questions
-- Do we need a second staging cluster?
-- Who signs off on the SOC2 evidence?"""
-    ],
-    [
-        """<h2>Shipping policy</h2><p>Orders placed before 2:00 PM CET ship the same business day.</p><ul><li>Standard delivery: 3 to 5 business days</li><li>Express delivery: 1 business day</li><li>Free shipping over 49 EUR</li></ul><p>Returns are accepted within 30 days with the original packaging.</p>"""
-    ],
-    [
-        """year,region,revenue,growth
-2019,North America,5120.4,4.1%
-2019,Europe,4380.9,3.7%
-2019,Latin America,2105.2,2.9%
-2020,North America,4875.6,-4.8%
-2020,Europe,4210.3,-3.9%
-2020,Latin America,1988.1,-5.6%
-2021,North America,5402.7,10.8%
-2021,Europe,4605.5,9.4%
-2021,Latin America,2240.8,12.7%
-2022,North America,5610.2,3.8%
-2022,Europe,4720.6,2.5%
-2022,Latin America,2385.4,6.5%
-2023,North America,5894.9,5.1%
-2023,Europe,4903.7,3.9%
-2023,Latin America,2560.3,7.3%"""
-    ],
-]
-
-CSS = """
-@import url('https://fonts.googleapis.com/css2?family=Hanken+Grotesk:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap');
-
-:root, .gradio-container {
-  --bg:#050608; --ink:#f4f5f7; --muted:#9aa0ab; --muted-2:#7f8698;
-  --warm:#ff7a30; --cool:#7aa0ff; --obsidian:#0a0a0c;
-  --hair:rgba(255,255,255,0.14); --hair-soft:rgba(255,255,255,0.08);
-  --panel:rgba(255,255,255,0.03); --panel-hover:rgba(255,255,255,0.06);
-  --danger:#ff7a6b;
-  --font-display:"Hanken Grotesk", system-ui, sans-serif;
-  --font-mono:"JetBrains Mono", ui-monospace, monospace;
-
-  --body-background-fill:transparent;
-  --background-fill-primary:transparent;
-  --background-fill-secondary:transparent;
-  --block-background-fill:transparent;
-  --block-border-color:transparent;
-  --block-border-width:0px;
-  --block-radius:0px;
-  --border-color-primary:var(--hair);
-  --body-text-color:var(--ink);
-  --body-text-color-subdued:var(--muted);
-  --input-background-fill:transparent;
-  --input-border-color:transparent;
-  --input-radius:12px;
-  --color-accent:var(--warm);
-  --color-accent-soft:rgba(255,122,48,0.15);
-  --button-primary-background-fill:var(--warm);
-  --button-primary-background-fill-hover:#ff9757;
-  --button-primary-text-color:#0b0705;
-  --button-secondary-background-fill:transparent;
-  --button-secondary-text-color:var(--muted);
-  --button-secondary-border-color:var(--hair);
-  --button-large-radius:100px;
-  --button-small-radius:100px;
-  --shadow-drop:none; --shadow-drop-lg:none;
-  --font:var(--font-display); --font-mono:var(--font-mono);
-}
+def _longest_repeated_ngram(text: str, max_n: int = 64) -> int:
+    words = [_norm_cmp(w) for w in text.split()]
+    n = len(words)
+    for size in range(min(max_n, n // 2), 4, -1):
+        seen = set()
+        for i in range(n - size + 1):
+            gram = tuple(words[i : i + size])
+            if gram in seen:
+                return size
+            seen.add(gram)
+    return 0
 
-body, .gradio-container {
-  background:
-    radial-gradient(120% 80% at 50% 118%, rgba(255,110,40,0.10), rgba(255,110,40,0) 55%),
-    radial-gradient(90% 70% at 88% -10%, rgba(80,120,230,0.10), rgba(80,120,230,0) 55%),
-    radial-gradient(140% 120% at 50% 50%, #0a0c11 0%, #050608 55%, #030305 100%) !important;
-  background-attachment: fixed !important;
-  color: var(--ink);
-  font-family: var(--font-display);
-  -webkit-font-smoothing: antialiased;
-}
-.gradio-container { max-width: 1240px !important; margin: 0 auto; padding: 0 20px 26px; }
-footer[aria-label="Gradio footer navigation"] { display: none !important; }
-:focus-visible { outline: 2px solid var(--cool) !important; outline-offset: 2px; }
 
-.p-head {
-  display: flex; align-items: center; justify-content: space-between; gap: 16px;
-  padding: 22px 0 16px; border-bottom: 1px solid var(--hair-soft); margin-bottom: 20px;
-}
-.p-brand { display: flex; align-items: center; gap: 10px; }
-.p-glyph { display: inline-flex; width: 22px; height: 22px; color: var(--warm); }
-.p-glyph svg { width: 100%; height: 100%; }
-.p-name { font-weight: 700; letter-spacing: 0.14em; font-size: 1.02rem; }
-.p-tag { font-family: var(--font-mono); font-size: 0.62rem; letter-spacing: 0.14em; color: var(--muted-2); margin-top: 2px; }
-.p-actions { display: flex; align-items: center; gap: 10px; }
-.p-btn {
-  font-family: var(--font-mono); font-size: 0.72rem; letter-spacing: 0.1em;
-  color: var(--ink); text-decoration: none; padding: 0 18px; min-height: 44px;
-  display: inline-flex; align-items: center; border-radius: 999px;
-  background: rgba(255,255,255,0.04); border: 1px solid var(--hair);
-  transition: background .25s ease, color .25s ease, border-color .25s ease;
-}
-.p-btn:hover { background: var(--ink); color: #08090c; border-color: var(--ink); }
+def degeneracy_stats(text: str) -> dict[str, Any]:
+    paras = [_norm_cmp(p) for p in re.split(r"\n\s*\n", text) if len(_norm_cmp(p)) > 80]
+    dup_para = len(paras) > len(set(paras))
+    longest = _longest_repeated_ngram(text)
+    return {"longest_repeated_ngram": longest, "duplicated_paragraph": dup_para}
 
-.p-status {
-  display: flex; align-items: center; gap: 9px; margin: -6px 0 16px;
-  font-family: var(--font-mono); font-size: 11px; letter-spacing: 0.08em;
-  text-transform: uppercase; color: var(--muted); min-height: 20px;
-}
-.p-dot { width: 7px; height: 7px; border-radius: 50%; background: rgba(255,255,255,0.2); flex-shrink: 0; }
-.p-status.ready .p-dot { background: var(--warm); box-shadow: 0 0 10px rgba(255,138,77,0.55); }
-.p-status.busy .p-dot { background: #ffcd6b; animation: p-pulse 1.1s ease-in-out infinite; }
-.p-status.error { color: var(--danger); }
-.p-status.error .p-dot { background: var(--danger); box-shadow: 0 0 10px rgba(255,122,107,0.5); }
-.p-detail { margin-left: auto; color: var(--muted-2); }
-@keyframes p-pulse { 0%,100% { opacity: 1; } 50% { opacity: 0.3; } }
-
-#p-shell { gap: 18px; align-items: stretch; }
-.p-panel {
-  background: var(--panel); border: 1px solid var(--hair); border-radius: 16px;
-  padding: 14px 14px 12px; min-width: 0;
-}
-.p-label { font-family: var(--font-mono); font-size: 0.66rem; letter-spacing: 0.14em; text-transform: uppercase; color: var(--muted); }
-.p-count { float: right; font-family: var(--font-mono); font-size: 0.66rem; letter-spacing: 0.04em; color: var(--muted-2); }
-
-#p-text, #p-text > div { background: transparent !important; border: 0 !important; box-shadow: none !important; }
-#p-text textarea {
-  background: transparent !important; border: 0 !important; color: var(--ink) !important;
-  font-family: var(--font-display) !important; font-size: 0.95rem; line-height: 1.65;
-  min-height: 360px; padding: 12px 2px 0;
-}
-#p-text textarea::placeholder { color: var(--muted-2) !important; }
-
-#p-run, #p-clear { font-family: var(--font-mono) !important; font-size: 0.75rem !important; letter-spacing: 0.12em; min-height: 46px; }
-#p-run { flex: 2; }
-#p-clear { flex: 1; background: transparent !important; color: var(--muted) !important; border: 1px solid var(--hair) !important; }
-#p-clear:hover { color: var(--ink) !important; border-color: rgba(255,255,255,0.3) !important; }
-
-#p-examples .label { font-family: var(--font-mono) !important; font-size: 0.66rem !important; letter-spacing: 0.14em; text-transform: uppercase; color: var(--muted) !important; }
-#p-examples .label svg { display: none; }
-#p-examples .gallery-item {
-  background: var(--panel) !important; border: 1px solid var(--hair) !important; border-radius: 999px !important;
-  color: var(--muted) !important; font-size: 12px !important; min-height: 44px; padding: 8px 14px !important;
-  text-align: left; transition: background .2s ease, color .2s ease, border-color .2s ease;
-}
-#p-examples .gallery-item:hover { border-color: rgba(255,255,255,0.3) !important; color: var(--ink) !important; background: var(--panel-hover) !important; }
 
-#p-tabs .tab-container[role="tablist"] {
-  display: inline-flex; gap: 4px; background: var(--panel); border: 1px solid var(--hair);
-  border-radius: 100px; padding: 3px; margin-bottom: 12px;
-}
-#p-tabs .tab-container[role="tablist"] button {
-  appearance: none; border: 0 !important; background: transparent !important; border-radius: 100px !important;
-  padding: 7px 14px !important; font-family: var(--font-mono) !important; font-size: 10.5px !important;
-  letter-spacing: 0.1em; text-transform: uppercase; color: var(--muted) !important;
-}
-#p-tabs .tab-container[role="tablist"] button.selected,
-#p-tabs .tab-container[role="tablist"] button[aria-selected="true"] {
-  background: rgba(255,255,255,0.08) !important; color: var(--warm) !important;
-}
-#p-tabs .tab-container::after,
-#p-tabs .tab-container button.selected::after { content: none !important; }
-
-#p-rendered { min-height: 360px; font-size: 0.95rem; line-height: 1.72; color: var(--ink); }
-#p-rendered .prose { font-size: 0.95rem; line-height: 1.72; }
-#p-rendered h1, #p-rendered h2, #p-rendered h3, #p-rendered h4 { color: #fff; font-weight: 600; line-height: 1.25; margin: 1.25em 0 0.55em; }
-#p-rendered h1 { font-size: 1.5em; }
-#p-rendered h2 { font-size: 1.28em; }
-#p-rendered h3 { font-size: 1.12em; }
-#p-rendered p { margin: 0 0 0.9em; }
-#p-rendered ul, #p-rendered ol { margin: 0 0 0.9em; padding-left: 1.5em; }
-#p-rendered li { margin: 0.25em 0; }
-#p-rendered li::marker { color: var(--muted-2); }
-#p-rendered a { color: #8ab4ff; text-decoration: underline; text-underline-offset: 2px; }
-#p-rendered a:hover { color: #aecbff; }
-#p-rendered blockquote { margin: 0 0 0.9em; padding: 2px 0 2px 16px; border-left: 2px solid var(--hair); color: var(--muted); }
-#p-rendered hr { border: 0; border-top: 1px solid var(--hair); margin: 1.3em 0; }
-#p-rendered code { font-family: var(--font-mono); font-size: 0.85em; background: var(--panel); border: 1px solid var(--hair); border-radius: 5px; padding: 1px 5px; }
-#p-rendered pre { margin: 0 0 0.9em; padding: 14px 16px; background: var(--obsidian); border: 1px solid var(--hair); border-radius: 12px; overflow-x: auto; }
-#p-rendered pre code { background: none; border: 0; padding: 0; font-size: 0.82em; line-height: 1.6; }
-#p-rendered table { border-collapse: collapse; margin: 0 0 0.9em; font-size: 0.92em; display: block; overflow-x: auto; }
-#p-rendered th, #p-rendered td { border: 1px solid var(--hair); padding: 6px 11px; text-align: left; }
-#p-rendered th { background: var(--panel); color: var(--ink); font-weight: 600; }
-
-#p-source {
-  background: var(--obsidian) !important; border: 1px solid var(--hair-soft) !important;
-  border-radius: 12px !important; font-family: var(--font-mono) !important; min-height: 360px;
-}
-#p-source label[data-testid="block-label"] { display: none; }
-#p-source .cm-editor { background: transparent !important; }
-#p-source .cm-content, #p-source .cm-scroller { color: var(--ink) !important; caret-color: var(--warm) !important; }
-#p-source .cm-gutters { background: transparent !important; color: var(--muted-2) !important; border-right: 1px solid var(--hair-soft) !important; }
-
-#p-settings { background: transparent !important; border: 1px solid var(--hair-soft) !important; border-radius: 12px !important; margin-top: 4px; }
-#p-settings > .label-wrap { font-family: var(--font-mono) !important; font-size: 0.68rem !important; letter-spacing: 0.12em; text-transform: uppercase; color: var(--muted) !important; }
-#p-settings > .label-wrap .icon { color: var(--muted-2) !important; }
-#p-settings .p-label { margin-bottom: 4px; }
-
-.p-foot {
-  display: flex; flex-wrap: wrap; justify-content: space-between; gap: 14px;
-  margin-top: 22px; padding-top: 16px; border-top: 1px solid var(--hair-soft);
-  font-family: var(--font-mono); font-size: 0.68rem; letter-spacing: 0.05em; line-height: 1.7; color: var(--muted-2);
-}
-.p-foot a { color: var(--muted); text-decoration: none; }
-.p-foot a:hover { color: var(--warm); }
-
-@media (max-width: 820px) {
-  .gradio-container { padding: 0 14px 20px; }
-  #p-shell { flex-direction: column !important; }
-  .p-tag { display: none; }
-  #p-text textarea { min-height: 220px; }
-  #p-rendered, #p-source { min-height: 240px; }
-  .p-status { flex-wrap: wrap; }
-  .p-detail { margin-left: 0; width: 100%; }
-  .p-foot { flex-direction: column; gap: 8px; }
-}
+def list_line_counts(text: str, kind: str = "generic") -> tuple[int, int]:
+    in_list = len(LIST_RE.findall(text))
+    if kind == "data":
+        rows = [l for l in text.splitlines() if "," in l and l.strip()]
+        if rows and "year" in rows[0].lower():
+            rows = rows[1:]
+        in_list += len(rows)
+    return in_list, 0
 
-@media (prefers-reduced-motion: reduce) {
-  *, *::before, *::after { animation: none !important; transition: none !important; }
-}
-"""
 
-theme = gr.themes.Base(
-    primary_hue=gr.themes.colors.orange,
-    neutral_hue=gr.themes.colors.gray,
-    font=[gr.themes.GoogleFont("Hanken Grotesk"), "system-ui", "sans-serif"],
-    font_mono=[gr.themes.GoogleFont("JetBrains Mono"), "ui-monospace", "monospace"],
-)
+def _preface_ratio(input_text: str, output: str) -> float:
+    in_words = set(_content_words(input_text))
+    if not in_words:
+        return 0.0
+    head_words = _content_words("\n".join(output.strip().splitlines()[:3]))[:40]
+    if len(head_words) < 8:
+        return 0.0
+    novel = sum(1 for w in head_words if w not in in_words)
+    return novel / len(head_words)
+
+
+def score_text(input_text: str, output: str, must: list[str] | None = None, kind: str = "generic") -> dict[str, Any]:
+    out_norm = _norm_cmp(output)
+    in_numbers = extract_number_keys(input_text)
+    out_numbers = extract_number_keys(output)
+    matched = in_numbers & out_numbers
+    number_fidelity = len(matched) / len(in_numbers) if in_numbers else 1.0
+
+    in_words = _content_words(input_text)
+    out_word_set = set(_content_words(output))
+    if in_words:
+        overlap = sum(1 for w in set(in_words) if w in out_word_set) / len(set(in_words))
+    else:
+        overlap = 1.0
+
+    must = must or []
+    must_hits = sum(1 for m in must if _norm_cmp(m) in out_norm)
+    must_fidelity = must_hits / len(must) if must else None
+
+    list_in, _ = list_line_counts(input_text, kind)
+    list_out = len(LIST_RE.findall(output)) + len(TABLE_ROW_RE.findall(output))
+    list_ratio = min(1.0, list_out / list_in) if list_in else 1.0
+
+    gfm = gfm_stats(output)
+    deg = degeneracy_stats(output)
+    metrics = {
+        "number_fidelity": round(number_fidelity, 4),
+        "numbers_in": len(in_numbers),
+        "numbers_matched": len(matched),
+        "spurious_numbers": len(out_numbers - in_numbers),
+        "content_overlap": round(overlap, 4),
+        "must_fidelity": None if must_fidelity is None else round(must_fidelity, 4),
+        "must_hits": must_hits,
+        "must_total": len(must),
+        "list_in": list_in,
+        "list_out": list_out,
+        "list_ratio": round(list_ratio, 4),
+        "list_marker_conservation": round(list_ratio, 4) if list_in else 1.0,
+        "missing_number_keys": sorted(in_numbers - out_numbers),
+        "chars_in": len(input_text),
+        "chars_out": len(output),
+        "preface_ratio": round(_preface_ratio(input_text, output), 4),
+        "degenerate": int(deg["duplicated_paragraph"] or deg["longest_repeated_ngram"] >= 12),
+        **gfm,
+        **deg,
+    }
+    metrics["quality"] = round(quality_score(metrics), 4)
+    return metrics
+
+
+def quality_score(metrics: dict[str, Any]) -> float:
+    score = 0.0
+    score += 0.30 * (metrics.get("must_fidelity") or 0.0)
+    score += 0.25 * metrics.get("number_fidelity", 0.0)
+    score += 0.15 * metrics.get("content_overlap", 0.0)
+    score += 0.10 * metrics.get("list_ratio", 0.0)
+    clean = metrics.get("gfm_clean")
+    score += 0.20 * (clean if clean is not None else 1.0)
+    if metrics.get("degenerate"):
+        score -= 0.15
+    if metrics.get("html_tokens", 0):
+        score -= 0.10
+    if metrics.get("malformed_tables", 0):
+        score -= 0.10
+    if metrics.get("malformed_links", 0):
+        score -= 0.05 * min(4, metrics["malformed_links"])
+    if metrics.get("preface_ratio", 0.0) > 0.5:
+        score -= 0.25
+    if metrics.get("spurious_numbers", 0):
+        score -= 0.05
+    chars_in = metrics.get("chars_in", 0)
+    chars_out = metrics.get("chars_out", 0)
+    if chars_in and chars_out < 0.2 * chars_in:
+        score -= 0.20
+    return max(0.0, min(1.5, score))
 
-with gr.Blocks(title="PASITA: plain text to Markdown") as demo:
-    gr.HTML(HEADER)
-    status = gr.HTML(_status("READY"), elem_id="p-status")
-
-    with gr.Row(elem_id="p-shell"):
-        with gr.Column(scale=1, min_width=320, elem_classes=["p-panel"]):
-            label_in = gr.HTML(_counter(""))
-            text_in = gr.Textbox(
-                show_label=False,
-                container=False,
-                placeholder="Paste OCR output, pasted HTML, meeting notes, a report or an article... (SHIFT + ENTER to convert)",
-                lines=18,
-                max_lines=32,
-                elem_id="p-text",
-            )
-            with gr.Accordion("SETTINGS", open=False, elem_id="p-settings"):
-                candidates = gr.Slider(
-                    minimum=1, maximum=3, value=3, step=1,
-                    label="PASITA CANDIDATES",
-                    info="More samples improve the reranker. 3 is the measured default.",
-                )
-                structured = gr.Checkbox(
-                    value=True,
-                    label="DETERMINISTIC CONVERTER FOR HTML AND TABLES",
-                    info="PASITA does not handle raw HTML or data dumps at this size.",
-                )
-            with gr.Row():
-                run = gr.Button("CONVERT TO MARKDOWN", variant="primary", elem_id="p-run")
-                clear = gr.Button("CLEAR", elem_id="p-clear")
-        with gr.Column(scale=1, min_width=320, elem_classes=["p-panel"]):
-            with gr.Tabs(elem_id="p-tabs"):
-                with gr.Tab("RENDERED", elem_id="p-tab-rendered"):
-                    md_out = gr.Markdown(EMPTY_MD, buttons=["copy"], height=520, elem_id="p-rendered")
-                with gr.Tab("SOURCE", elem_id="p-tab-source"):
-                    source = gr.Code(
-                        value="", language="markdown", buttons=["copy", "download"],
-                        wrap_lines=True, lines=24, max_lines=24, elem_id="p-source",
-                    )
-
-    gr.Examples(
-        examples=EXAMPLES,
-        example_labels=["ES REPORT", "EN MEETING", "HTML", "CSV"],
-        inputs=[text_in],
-        outputs=[md_out, source, status],
-        fn=convert,
-        cache_examples=True,
-        cache_mode="lazy",
-        examples_per_page=4,
-        label="EXAMPLES",
-        elem_id="p-examples",
+
+def rerank_candidates(input_text: str, candidates: list[str], must: list[str] | None = None, kind: str = "generic") -> tuple[int, list[dict]]:
+    scored = [(i, score_text(input_text, c, must, kind)) for i, c in enumerate(candidates)]
+    best = max(scored, key=lambda pair: pair[1]["quality"])[0] if scored else 0
+    return best, [m for _, m in scored]
+
+
+class _HtmlToMarkdown(HTMLParser):
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.blocks: list[str] = []
+        self.buf: list[str] = []
+        self.heading: int | None = None
+        self.list_stack: list[str] = []
+        self.pending_list: list[str] = []
+        self.pending_items: list[str] = []
+        self.in_pre = False
+        self.href: str | None = None
+        self.link_pos: int | None = None
+        self.table_rows: list[list[str]] | None = None
+        self.row: list[str] | None = None
+        self.cell_buf: list[str] | None = None
+
+    def _text(self) -> str:
+        t = "".join(self.buf)
+        self.buf = []
+        return t
+
+    def _flush_para(self) -> None:
+        t = _norm_ws(self._text())
+        if t:
+            self.blocks.append(t)
+
+    def _flush_line(self) -> None:
+        t = "".join(self.buf).strip()
+        self.buf = []
+        if t:
+            self.pending_items.append(t)
+
+    def handle_data(self, data: str) -> None:
+        if self.cell_buf is not None:
+            self.cell_buf.append(data)
+        else:
+            self.buf.append(data)
+
+    def handle_starttag(self, tag: str, attrs) -> None:
+        a = dict(attrs)
+        if tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            self._flush_para()
+            self.heading = int(tag[1])
+        elif tag in ("p", "div", "section", "article", "header", "footer", "main", "aside"):
+            self._flush_para()
+        elif tag == "br":
+            self.buf.append("\n")
+        elif tag in ("strong", "b"):
+            self.buf.append("**")
+        elif tag in ("em", "i"):
+            self.buf.append("*")
+        elif tag == "code" and not self.in_pre:
+            self.buf.append("`")
+        elif tag == "pre":
+            self._flush_para()
+            self.in_pre = True
+            self.buf.append("```\n")
+        elif tag == "a":
+            self.href = a.get("href")
+            self.link_pos = len(self.buf)
+            self.buf.append("")
+        elif tag in ("ul", "ol"):
+            self._flush_para()
+            self.list_stack.append(tag)
+        elif tag == "li":
+            self._flush_line()
+            depth = max(0, len(self.list_stack) - 1)
+            ordered = self.list_stack and self.list_stack[-1] == "ol"
+            self.buf.append("  " * depth + ("1. " if ordered else "- "))
+        elif tag == "table":
+            self._flush_para()
+            self.table_rows = []
+        elif tag == "tr":
+            self.row = []
+        elif tag in ("td", "th"):
+            self.cell_buf = []
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            t = _norm_ws(self._text())
+            if t:
+                level = self.heading or 1
+                self.blocks.append("#" * level + " " + t)
+            self.heading = None
+        elif tag in ("p", "div", "section", "article", "header", "footer", "main", "aside"):
+            self._flush_para()
+        elif tag in ("strong", "b"):
+            self.buf.append("**")
+        elif tag in ("em", "i"):
+            self.buf.append("*")
+        elif tag == "code" and not self.in_pre:
+            self.buf.append("`")
+        elif tag == "pre":
+            self.in_pre = False
+            t = "".join(self.buf)
+            self.buf = []
+            self.blocks.append("```\n" + t.strip("\n") + "\n```")
+        elif tag == "a":
+            if self.link_pos is not None:
+                text = "".join(self.buf[self.link_pos :])
+                del self.buf[self.link_pos :]
+                if self.href:
+                    self.buf.append(f"[{text}]({self.href})")
+                else:
+                    self.buf.append(text)
+            self.link_pos = None
+            self.href = None
+        elif tag in ("ul", "ol"):
+            if self.list_stack:
+                self.list_stack.pop()
+            if not self.list_stack and self.pending_items:
+                self.blocks.append("\n".join(self.pending_items))
+                self.pending_items = []
+        elif tag == "li":
+            self._flush_line()
+        elif tag in ("td", "th"):
+            if self.row is not None and self.cell_buf is not None:
+                self.row.append(_norm_ws("".join(self.cell_buf)))
+            self.cell_buf = None
+        elif tag == "tr":
+            if self.table_rows is not None and self.row is not None:
+                self.table_rows.append(self.row)
+            self.row = None
+        elif tag == "table":
+            self._emit_table()
+            self.table_rows = None
+
+    def _emit_table(self) -> None:
+        rows = [r for r in (self.table_rows or []) if any(c for c in r)]
+        if not rows:
+            return
+        width = max(len(r) for r in rows)
+        header = (rows[0] + [""] * width)[:width]
+        lines = ["| " + " | ".join(header) + " |", "| " + " | ".join(["---"] * width) + " |"]
+        for r in rows[1:]:
+            cells = (r + [""] * width)[:width]
+            lines.append("| " + " | ".join(cells) + " |")
+        self.blocks.append("\n".join(lines))
+
+    def finish(self) -> str:
+        if self.in_pre:
+            self.handle_endtag("pre")
+        if self.pending_items:
+            self.blocks.append("\n".join(self.pending_items))
+            self.pending_items = []
+        self._flush_para()
+        md = "\n\n".join(b for b in self.blocks if b.strip())
+        return re.sub(r"\n{3,}", "\n\n", md).strip()
+
+
+def looks_like_html(text: str) -> bool:
+    return bool(
+        re.search(r"</?(?:p|div|h[1-6]|ul|ol|li|table|tr|td|th|br|strong|em|a|span|img|section|article)\b[^>]*>", text, re.I)
+    )
+
+
+def html_to_markdown(text: str) -> str:
+    parser = _HtmlToMarkdown()
+    parser.feed(text)
+    parser.close()
+    return parser.finish()
+
+
+def _split_cells(line: str, delim: str) -> list[str]:
+    if delim == ",":
+        import csv as _csv
+
+        return [c.strip() for c in next(_csv.reader([line]))]
+    if delim == "\t":
+        return [c.strip() for c in line.split("\t")]
+    if delim == "|":
+        return [c.strip() for c in line.strip().strip("|").split("|")]
+    return [c.strip() for c in re.split(r"\s{2,}", line.strip())]
+
+
+def _find_tabular_block(text: str):
+    lines = [l for l in text.splitlines() if l.strip()]
+    if len(lines) < 3:
+        return None
+    for delim in (",", "\t", "|", "  "):
+        for start in range(0, min(2, len(lines))):
+            seg = lines[start:]
+            if len(seg) < 3:
+                continue
+            counts = [len(_split_cells(l, delim)) for l in seg]
+            if min(counts) >= 2 and max(counts) == min(counts):
+                return delim, lines[:start], seg
+    return None
+
+
+def delimited_to_markdown_table(text: str) -> str | None:
+    found = _find_tabular_block(text)
+    if not found:
+        return None
+    delim, pre, seg = found
+    rows = [_split_cells(l, delim) for l in seg]
+    width = len(rows[0])
+    if width < 2 or any(len(r) != width for r in rows):
+        return None
+    def esc(cells: list[str]) -> list[str]:
+        return [c.replace("|", "\\|") for c in cells]
+    out: list[str] = []
+    if pre:
+        out.append("\n".join(pre).strip())
+    out.append("| " + " | ".join(esc(rows[0])) + " |")
+    out.append("| " + " | ".join(["---"] * width) + " |")
+    for r in rows[1:]:
+        out.append("| " + " | ".join(esc(r)) + " |")
+    return "\n".join(out)
+
+
+def infer_kind(text: str) -> str:
+    if looks_like_html(text):
+        return "html"
+    if _find_tabular_block(text):
+        return "data"
+    return "generic"
+
+
+def structured_candidates(text: str) -> list[str]:
+    cands: list[str] = []
+    if looks_like_html(text):
+        try:
+            cands.append(html_to_markdown(text))
+        except Exception:
+            pass
+    table = delimited_to_markdown_table(text)
+    if table:
+        cands.append(table)
+    return cands
+
+
+def convert_to_markdown(
+    model,
+    tokenizer,
+    text: str,
+    *,
+    must: list[str] | None = None,
+    kind: str = "generic",
+    structured_fallback: bool = True,
+    skip_model_for_structured: bool = False,
+    **gen_kwargs: Any,
+) -> dict[str, Any]:
+    if kind == "generic":
+        kind = infer_kind(text)
+    if skip_model_for_structured and structured_fallback and kind in ("html", "data"):
+        candidates = [postprocess(c, text) for c in structured_candidates(text)]
+        if not candidates:
+            candidates = [postprocess(text, text)]
+        best, metrics = rerank_candidates(text, candidates, must, kind)
+        return {
+            "markdown": candidates[best],
+            "candidate_index": best,
+            "fallback_used": True,
+            "metrics": metrics[best],
+            "candidate_metrics": metrics,
+            "candidates": candidates,
+            "raw_candidates": [],
+            "prompt_tokens": 0,
+            "max_new_tokens": 0,
+            "seconds": 0.0,
+            "strategy": {"mode": "structured", "best_index": best, "structured_fallback": True},
+        }
+    result = generate_candidates(model, tokenizer, text, **gen_kwargs)
+    candidates = list(result.candidates)
+    fallback_start = len(candidates)
+    if structured_fallback:
+        for extra in structured_candidates(text):
+            candidates.append(postprocess(extra, text))
+    best, metrics = rerank_candidates(text, candidates, must, kind)
+    chosen = candidates[best]
+    route = "model"
+    if not chosen.strip():
+        chosen = text.strip()
+        metrics[best]["quality"] = 0.0
+    best_metrics = metrics[best]
+    if best_metrics.get("number_fidelity", 1.0) < 1.0:
+        passthrough = _passthrough(text)
+        pass_metrics = score_text(text, passthrough, must, kind)
+        if pass_metrics.get("number_fidelity", 0.0) > best_metrics.get("number_fidelity", 0.0) and pass_metrics.get("content_overlap", 0.0) >= 0.9:
+            chosen = passthrough
+            best_metrics = pass_metrics
+            route = "passthrough"
+    result.strategy["best_index"] = best
+    result.strategy["structured_fallback"] = structured_fallback
+    result.strategy["route"] = route
+    return {
+        "markdown": chosen,
+        "candidate_index": best,
+        "fallback_used": best >= fallback_start,
+        "route": route,
+        "metrics": best_metrics,
+        "candidate_metrics": metrics,
+        "candidates": candidates,
+        "raw_candidates": result.raw_candidates,
+        "prompt_tokens": result.prompt_tokens,
+        "max_new_tokens": result.max_new_tokens,
+        "seconds": result.seconds,
+        "strategy": result.strategy,
+    }
+
+
+def load_model(model_id: str = MODEL_ID, device: str | None = None, local_files_only: bool = True):
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(model_id, local_files_only=local_files_only)
+    model = AutoModelForCausalLM.from_pretrained(model_id, dtype="auto", local_files_only=local_files_only)
+    model.eval()
+    if device:
+        model.to(device)
+    return model, tokenizer
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description="PASITA text-to-Markdown CLI")
+    parser.add_argument("--text-file", required=True)
+    parser.add_argument("--model", default=MODEL_ID)
+    parser.add_argument(
+        "--mode",
+        default="canonical",
+        choices=[
+            "canonical",
+            "english",
+            "spanish_instr",
+            "fewshot",
+            "fewshot2",
+            "plain",
+            "md_label",
+            "respuesta",
+            "output",
+            "sep_only",
+            "doc_instr_es",
+            "doc_instr_en",
+            "csv_instr",
+            "html_instr",
+            "html_instr_en",
+        ],
     )
-    gr.HTML(FOOTER)
-
-    run.click(
-        convert,
-        inputs=[text_in, candidates, structured],
-        outputs=[md_out, source, status],
-        api_name="convert",
-        js=BUSY_JS,
-        show_progress="minimal",
+    parser.add_argument("--instruction", default=None)
+    parser.add_argument("--candidates", type=int, default=3)
+    parser.add_argument("--beams", type=int, default=1)
+    parser.add_argument("--no-repeat-ngram", type=int, default=0)
+    parser.add_argument("--repetition-penalty", type=float, default=1.0)
+    parser.add_argument("--temperature", type=float, default=0.3)
+    parser.add_argument("--min-new", type=int, default=0)
+    parser.add_argument("--max-new", default="auto")
+    parser.add_argument("--max-new-ratio", type=float, default=1.5)
+    parser.add_argument("--hard-cap", type=int, default=DEFAULT_HARD_CAP)
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--device", default=None)
+    parser.add_argument("--no-structured-fallback", action="store_true")
+    parser.add_argument("--show-candidates", action="store_true")
+    args = parser.parse_args(argv)
+
+    with open(args.text_file, encoding="utf-8") as fh:
+        text = fh.read()
+
+    model, tokenizer = load_model(args.model, args.device)
+    max_new: Any = args.max_new
+    if args.max_new != "auto":
+        max_new = int(args.max_new)
+    gen_kwargs: dict[str, Any] = dict(
+        mode=args.mode,
+        instruction=args.instruction,
+        num_candidates=max(1, args.candidates),
+        num_beams=max(1, args.beams),
+        do_sample=args.temperature > 0,
+        temperature=args.temperature if args.temperature > 0 else 1.0,
+        no_repeat_ngram_size=args.no_repeat_ngram,
+        repetition_penalty=args.repetition_penalty,
+        max_new_tokens=max_new,
+        max_new_ratio=args.max_new_ratio,
+        hard_cap=args.hard_cap,
+        seed=args.seed,
     )
-    text_in.submit(
-        convert,
-        inputs=[text_in, candidates, structured],
-        outputs=[md_out, source, status],
-        api_visibility="undocumented",
-        js=BUSY_JS,
-        show_progress="minimal",
+    if args.min_new > 0:
+        gen_kwargs["min_new_tokens"] = args.min_new
+    result = convert_to_markdown(
+        model,
+        tokenizer,
+        text,
+        structured_fallback=not args.no_structured_fallback,
+        **gen_kwargs,
     )
-    text_in.change(None, inputs=[text_in], outputs=[label_in], js=COUNTER_JS, api_visibility="undocumented")
-    clear.click(reset, inputs=None, outputs=[md_out, source, status, label_in], api_visibility="undocumented")
+    if args.show_candidates:
+        for i, (cand, met) in enumerate(zip(result["candidates"], result["candidate_metrics"])):
+            marker = " *" if i == result["candidate_index"] else ""
+            print(f"----- candidate {i}{marker} quality={met['quality']} -----")
+            print(cand)
+            print()
+    print(result["markdown"])
+
 
-demo.queue()
-demo.launch(theme=theme, css=CSS, mcp_server=True)
+if __name__ == "__main__":
+    main()