Spaces:

NagaYu
/

ClaimCheck

Running

ClaimCheck / claimcheck.js
ClaimCheck v1.1.0: client-side deterministic LLM answer verifier
b07f2d1 verified about 1 month ago
87.1 kB
/*
 * ClaimCheck - browser port of the deterministic LLM answer verifier.
 *
 * DESIGN PHILOSOPHY
 * =================
 * We cannot detect every hallucination. But dangerous hallucinations are
 * SPECIFIC, and specific claims can be matched as strings. So this verifies
 * only what it can verify deterministically, and always reports how much it
 * did NOT check (coverage).
 *
 * Two numbers, always together:
 *   grounding_score = (supported + derived) / verifiable claims
 *   coverage        = sentences that produced a claim / all sentences
 * An answer of pure hedging produces zero claims and scores 1.00 on any naive
 * groundedness metric. coverage is what stops you reading that as "verified".
 *
 * This file is a faithful port of app.py (the Python source of truth) and is
 * validated for parity against all 158 cases of NagaYu/claimcheck-eval.
 * Everything runs client-side: no network, no server, nothing leaves the tab.
 */

export const APP_VERSION = "1.1.0";
export const STATUSES = ["supported", "derived", "approximate", "unsupported", "contradicted"];
export const CLAIM_TYPES = ["NUMERIC", "DATE", "QUOTE", "ENTITY", "URL"];
export const VERDICTS = ["pass", "annotate", "retry", "block"];

export const DEFAULTS = {
  MAX_TEXT_CHARS: 20000,
  NGRAM_LEAK_N: 8,
  LEAK_THRESHOLD: 0.15,
  PRICE_IN_PER_1K: 0.0,
  PRICE_OUT_PER_1K: 0.0,
  ENTROPY_THRESHOLD: 3.6,
  ENTROPY_MIN_LEN: 24,
  CONTRADICTION_REL: 0.25,
  APPROX_RATIO: 0.82,
  QUOTE_CONTRADICTION_FLOOR: 0.65,
  DERIVE_MAX_TERMS: 3,
  DERIVE_BUDGET_MS: 120,
  DERIVE_MAX_NUMBERS: 40,
  FUZZY_BUDGET_MS: 250,
  MAX_CLAIMS: 1500,
};

/* ===================================================================== (C) NORMALIZER */

// NOTE: "ー" (U+30FC) is deliberately absent. It is a katakana letter, not a
// dash: folding it turned データセンター into デ-タセンタ- and broke Japanese
// matching wholesale. This was a real bug in v1.0.0.
const PUNCT_MAP = {
  "，": ",", "、": ",", "､": ",",
  "．": ".", "。": ".", "｡": ".",
  "：": ":", "；": ";",
  "（": "(", "）": ")", "［": "[", "］": "]", "｛": "{", "｝": "}",
  "「": '"', "」": '"', "『": '"', "』": '"',
  "“": '"', "”": '"', "„": '"', "‟": '"',
  "‘": "'", "’": "'", "‚": "'",
  "－": "-", "−": "-", "—": "-", "–": "-", "―": "-", "‐": "-", "‑": "-",
  "％": "%", "＄": "$", "￥": "¥", "／": "/", "＼": "\\", "＃": "#", "＆": "&",
  "！": "!", "？": "?", "〜": "~", "～": "~", "・": " ",
};

/** Guarantees: NFKC + lowercase + collapsed whitespace + unified punctuation; never throws. */
export function normText(s) {
  try {
    if (s === null || s === undefined) return "";
    let t = String(s).replace(/[，、､．。｡：；（）［］｛｝「」『』“”„‟‘’‚－−—–―‐‑％＄￥／＼＃＆！？〜～・]/g,
      (ch) => (PUNCT_MAP[ch] !== undefined ? PUNCT_MAP[ch] : ch));
    t = t.normalize("NFKC").toLowerCase();
    return t.replace(/\s+/g, " ").trim();
  } catch (e) { return ""; }
}

const MAG_FACTORS = { "千": 1e3, "万": 1e4, "億": 1e8, "兆": 1e12, k: 1e3, K: 1e3, m: 1e6, M: 1e6, b: 1e9, B: 1e9 };

/** Guarantees: a finite Number for any recognised numeric literal, else null. */
export function normNumber(s) {
  try {
    if (s === null || s === undefined) return null;
    if (typeof s === "number") return Number.isFinite(s) ? s : null;
    let t = String(s).trim();
    if (!t) return null;
    t = t.normalize("NFKC").replace(/[−－]/g, "-").replace(/，/g, ",");
    let sign = 1;
    if (t[0] === "▲" || t[0] === "△") { sign = -1; t = t.slice(1).trim(); }
    if (t.startsWith("(") && t.endsWith(")")) { sign = -sign; t = t.slice(1, -1).trim(); }
    if (t[0] === "+" || t[0] === "-") { if (t[0] === "-") sign = -sign; t = t.slice(1).trim(); }
    let low = t.toLowerCase();
    low = low.replace(/^(?:us\$|usd|jpy|eur|gbp|krw|cny|[$¥€£₩])/, "")
             .replace(/(?:usd|jpy|eur|gbp|円|ドル|ユーロ)$/, "").trim();
    t = low;
    for (const suf of ["%", "パーセント", "percent", "pct"]) {
      if (t.endsWith(suf)) { t = t.slice(0, -suf.length).trim(); break; }
    }
    if (t.endsWith("割")) {   // 3割 == 30%
      const base = parseNum(t.slice(0, -1).replace(/[,\s_]/g, ""));
      return base === null ? null : sign * base * 10;
    }
    let factor = 1, changed = true;
    while (changed && t) {
      changed = false;
      for (const suf of Object.keys(MAG_FACTORS)) {
        if (suf.length === 1 && t.endsWith(suf)) {
          const rest = t.slice(0, -1).trim();
          if (rest && /\d/.test(rest)) { factor *= MAG_FACTORS[suf]; t = rest; changed = true; break; }
        }
      }
    }
    t = t.replace(/[^0-9.eE+\-]+$/, "").trim().replace(/[,\s_]/g, "");
    const base = parseNum(t);
    if (base === null) return null;
    return sign * base * factor;
  } catch (e) { return null; }
}

function parseNum(t) {
  if (!t || !/^[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?$/.test(t)) return null;
  const v = Number(t);
  return Number.isFinite(v) ? v : null;
}

const MONTHS = { jan:1,january:1,feb:2,february:2,mar:3,march:3,apr:4,april:4,may:5,jun:6,june:6,
  jul:7,july:7,aug:8,august:8,sep:9,sept:9,september:9,oct:10,october:10,nov:11,november:11,dec:12,december:12 };
// Japanese era -> offset such that year = offset + eraYear. 令和1 == 2019.
const ERAS = { "令和":2018, reiwa:2018, "平成":1988, heisei:1988, "昭和":1925, showa:1925,
  "大正":1911, taisho:1911, "明治":1867, meiji:1867 };

function mkDate(y, m, d) {
  y = parseInt(y, 10);
  if (!(y >= 1000 && y <= 3000)) return null;
  const p = (n, w) => String(n).padStart(w, "0");
  if (m === undefined || m === null || m === "") return p(y, 4);
  m = parseInt(m, 10);
  if (!(m >= 1 && m <= 12)) return null;
  if (d === undefined || d === null || d === "") return `${p(y,4)}-${p(m,2)}`;
  d = parseInt(d, 10);
  if (!(d >= 1 && d <= 31)) return null;
  return `${p(y,4)}-${p(m,2)}-${p(d,2)}`;
}

/** Guarantees: an ISO string (YYYY / YYYY-MM / YYYY-MM-DD) or null. Ambiguity returns null, not a guess. */
export function normDate(s) {
  try {
    if (!s) return null;
    let t = String(s).normalize("NFKC").trim().replace(/[−]/g, "-").replace(/．/g, ".");
    t = t.replace(/\s+/g, " ").trim().replace(/[.,]+$/, "");
    if (!t) return null;
    let m;
    if ((m = /^(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日?$/.exec(t))) return mkDate(m[1], m[2], m[3]);
    if ((m = /^(\d{4})\s*年\s*(\d{1,2})\s*月$/.exec(t))) return mkDate(m[1], m[2]);
    if ((m = /^(令和|平成|昭和|大正|明治)\s*(\d{1,2}|元)\s*年(?:\s*(\d{1,2})\s*月(?:\s*(\d{1,2})\s*日?)?)?$/.exec(t))) {
      const ey = m[2] === "元" ? 1 : parseInt(m[2], 10);
      return mkDate(ERAS[m[1]] + ey, m[3], m[4]);
    }
    if ((m = /^(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})$/.exec(t))) return mkDate(m[1], m[2], m[3]);
    if ((m = /^(\d{4})[-/.](\d{1,2})$/.exec(t))) return mkDate(m[1], m[2]);
    if ((m = /^([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})$/.exec(t))) {
      const mo = MONTHS[m[1].toLowerCase().replace(/\.$/, "")];
      if (mo) return mkDate(m[3], mo, m[2]);
    }
    if ((m = /^(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(\d{4})$/.exec(t))) {
      const mo = MONTHS[m[2].toLowerCase().replace(/\.$/, "")];
      if (mo) return mkDate(m[3], mo, m[1]);
    }
    if ((m = /^([A-Za-z]{3,9})\.?,?\s+(\d{4})$/.exec(t))) {
      const mo = MONTHS[m[1].toLowerCase().replace(/\.$/, "")];
      if (mo) return mkDate(m[2], mo);
    }
    if ((m = /^(\d{4})\s*年(?:度)?$/.exec(t))) return mkDate(m[1]);
    if ((m = /^(?:fy)?(\d{4})$/i.exec(t))) return mkDate(m[1]);
    return null;
  } catch (e) { return null; }
}

/* ============================================================ SIMILARITY PRIMITIVES */

/**
 * Guarantees: the same value as Python difflib.SequenceMatcher(None,a,b,autojunk=False).ratio().
 * Ported rather than approximated, because the approximate/contradicted
 * boundary is decided by this number.
 */
function smRatio(a, b) {
  const la = a.length, lb = b.length;
  if (!la && !lb) return 1.0;
  if (!la || !lb) return 0.0;
  const b2j = new Map();
  for (let j = 0; j < lb; j++) {
    const c = b[j];
    let arr = b2j.get(c);
    if (!arr) { arr = []; b2j.set(c, arr); }
    arr.push(j);
  }
  function longest(alo, ahi, blo, bhi) {
    let besti = alo, bestj = blo, bestsize = 0;
    let j2len = new Map();
    for (let i = alo; i < ahi; i++) {
      const newj2len = new Map();
      const js = b2j.get(a[i]);
      if (js) {
        for (const j of js) {
          if (j < blo) continue;
          if (j >= bhi) break;
          const k = (j2len.get(j - 1) || 0) + 1;
          newj2len.set(j, k);
          if (k > bestsize) { besti = i - k + 1; bestj = j - k + 1; bestsize = k; }
        }
      }
      j2len = newj2len;
    }
    while (besti > alo && bestj > blo && a[besti - 1] === b[bestj - 1]) { besti--; bestj--; bestsize++; }
    while (besti + bestsize < ahi && bestj + bestsize < bhi && a[besti + bestsize] === b[bestj + bestsize]) bestsize++;
    return [besti, bestj, bestsize];
  }
  let matches = 0;
  const queue = [[0, la, 0, lb]];
  while (queue.length) {
    const [alo, ahi, blo, bhi] = queue.pop();
    const [i, j, k] = longest(alo, ahi, blo, bhi);
    if (k) {
      matches += k;
      if (alo < i && blo < j) queue.push([alo, i, blo, j]);
      if (i + k < ahi && j + k < bhi) queue.push([i + k, ahi, j + k, bhi]);
    }
  }
  return (2.0 * matches) / (la + lb);
}

/** Guarantees: candidate window starts where needle and haystack share a character k-gram. */
function fuzzyAnchors(needle, haystack, maxAnchors = 96) {
  const nl = needle.length;
  // k must stay strictly below the needle length, and short identifiers need
  // k=2: "第13条" against "第12条" shares no 3-gram, and that one-digit-off
  // article number is exactly the fabrication this tool exists to catch.
  let k = nl <= 6 ? 2 : Math.min(8, Math.max(4, Math.floor(nl / 3)));
  k = Math.min(k, Math.max(2, nl - 1));
  if (nl < 2) return haystack.includes(needle) ? [0] : [];
  const offsets = [...new Set([0, Math.floor(nl / 4), Math.floor(nl / 2), Math.floor((3 * nl) / 4), Math.max(0, nl - k)])].sort((x, y) => x - y);
  const anchors = [];
  for (const off of offsets) {
    const gram = needle.slice(off, off + k);
    if (gram.length < k) continue;
    let idx = haystack.indexOf(gram), hits = 0;
    while (idx !== -1 && hits < 20 && anchors.length < maxAnchors) {
      anchors.push(Math.max(0, idx - off));
      hits++;
      idx = haystack.indexOf(gram, idx + 1);
    }
    if (anchors.length >= maxAnchors) break;
  }
  return anchors;
}

/**
 * Guarantees: (ratio, window) for an order-preserving match, with bounded work.
 * No shared k-gram means no candidate windows and an immediate 0.0, which is
 * what keeps a 20k context from becoming a full sliding-window scan.
 */
export function bestFuzzy(needle, haystack, maxEvals = 240) {
  try {
    if (!needle || !haystack) return [0.0, ""];
    const nl = needle.length;
    const anchors = fuzzyAnchors(needle, haystack);
    if (!anchors.length) return [0.0, ""];
    const widths = [...new Set([Math.max(3, nl - 2), nl, Math.floor(nl * 1.25) + 3])].sort((a, b) => a - b);
    let best = 0.0, bestTxt = "", evals = 0;
    const seen = new Set();
    for (let st of anchors) {
      st = Math.max(0, Math.min(st, Math.max(0, haystack.length - 1)));
      for (const w of widths) {
        const key = st * 10000 + w;
        if (seen.has(key)) continue;
        seen.add(key);
        const win = haystack.slice(st, st + w);
        if (!win) continue;
        evals++;
        const r = smRatio(win, needle);
        if (r > best) { best = r; bestTxt = win; }
        if (best >= 0.995 || evals >= maxEvals) return [best, bestTxt];
      }
    }
    return [best, bestTxt];
  } catch (e) { return [0.0, ""]; }
}

// Rolling-hash longest common substring. difflib.find_longest_match was the
// original implementation and degenerates to O(n^2) on repetitive text: a 20k
// answer of one repeated character took 12.7s, a denial of service on a small box.
const LCS_MOD = 33554393, LCS_BASE = 257;   // modulus kept under 2^25 so products stay exact in float64

function rkPositions(s, k) {
  const out = new Map(), n = s.length;
  if (k <= 0 || k > n) return out;
  let h = 0;
  for (let i = 0; i < k; i++) h = (h * LCS_BASE + s.charCodeAt(i)) % LCS_MOD;
  out.set(h, 0);
  let power = 1;
  for (let i = 0; i < k; i++) power = (power * LCS_BASE) % LCS_MOD;
  for (let i = k; i < n; i++) {
    h = (h * LCS_BASE + s.charCodeAt(i) - s.charCodeAt(i - k) * power) % LCS_MOD;
    if (h < 0) h += LCS_MOD;
    if (!out.has(h)) out.set(h, i - k + 1);
  }
  return out;
}

/** Guarantees: the longest shared substring, O(n log n), bounded by `cap` characters. */
export function longestCommonFragment(a, b, cap = 3000) {
  try {
    a = (a || "").slice(0, cap); b = (b || "").slice(0, cap);
    if (!a || !b) return "";
    let lo = 1, hi = Math.min(a.length, b.length), best = "";
    while (lo <= hi) {
      const mid = (lo + hi) >> 1;
      const ha = rkPositions(a, mid), hb = rkPositions(b, mid);
      let found = null;
      if (ha.size && hb.size) {
        const [small, big, sa] = ha.size <= hb.size ? [ha, hb, a] : [hb, ha, b];
        const other = small === ha ? b : a;
        for (const [h, pos] of small) {
          const j = big.get(h);
          if (j === undefined) continue;
          const cand = sa.slice(pos, pos + mid);
          if (cand === other.slice(j, j + mid)) { found = cand; break; }  // guard against collisions
        }
      }
      if (found) { best = found; lo = mid + 1; } else hi = mid - 1;
    }
    return best;
  } catch (e) { return ""; }
}

/* ============================================================== (B) CLAIM EXTRACTOR */

const URL_RE = /(?:https?:\/\/|ftp:\/\/|www\.)[^\s<>"'()\[\]{}、。，！？]+|\b[a-z0-9][a-z0-9\-]{0,62}\.(?:com|org|net|io|ai|co|jp|dev|app|gov|edu|info|me|cloud|xyz)(?:\/[^\s<>"'()\[\]{}、。，！？]*)?/gi;

const NUM_CORE = "(?:[0-9]{1,3}(?:,[0-9]{3})+(?:\\.[0-9]+)?|[0-9]+(?:\\.[0-9]+)?|\\.[0-9]+)";
const CUR_PAT = "(?:US\\$|USD|JPY|EUR|GBP|KRW|CNY|[$¥€£₩])";
const MAG_PAT = "(?:千|万|億|兆|[kKmMbB](?![A-Za-z0-9]))";
const PCT_PAT = "(?:%|パーセント|ポイント|割)";
const UNIT_PAT = "(?:円|ドル|ユーロ|元|ウォン|件|人|名|社|店|台|個|本|枚|冊|回|倍|点|位|票|室|席|品|語|文字|字|行|列|頁|ページ|年度|年間|年|ヶ月|か月|カ月|ヵ月|箇月|週間|日間|時間|分間|秒|分|時|日|歳|才|km|cm|mm|kg|mg|GB|MB|TB|KB|PB|Mbps|Gbps|bps|kHz|MHz|GHz|Hz|kW|MW|W|mL|L|m2|㎡|平方メートル|立方メートル|℃|°C|°F|m|g|t|V|A)";
// The whitespace sits INSIDE the currency group and the letter-lookahead INSIDE
// the unit group. Outside, the match could begin on a newline ("\n2", breaking
// list-marker suppression) and every ordinal ("25th") was silently dropped.
const NUM_RE = new RegExp(
  "(?<![0-9A-Za-z._\\-])(?<sign>[-+▲△])?(?:(?<cur>" + CUR_PAT + ")\\s{0,2})?(?<num>" + NUM_CORE + ")" +
  "(?<exp>[eE][-+]?[0-9]{1,3})?\\s?(?<mag>" + MAG_PAT + ")?\\s?(?<pct>" + PCT_PAT + ")?" +
  "(?:\\s?(?<unit>" + UNIT_PAT + ")(?![A-Za-z]))?", "g");

const MON = "Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August|Sep|Sept|September|Oct|October|Nov|November|Dec|December";
const DATE_PATTERNS = [
  /(?:令和|平成|昭和|大正|明治)\s*(?:\d{1,2}|元)\s*年(?:\s*\d{1,2}\s*月(?:\s*\d{1,2}\s*日)?)?/g,
  /\d{4}\s*年\s*\d{1,2}\s*月\s*\d{1,2}\s*日/g,
  /\d{4}\s*年\s*\d{1,2}\s*月/g,
  // NOT \b: in "契約は2024-01-05に" both sides are word characters, so \b never
  // fires and ISO dates inside Japanese text were invisible.
  /(?<![0-9A-Za-z])\d{4}[-/.]\d{1,2}[-/.]\d{1,2}(?![0-9A-Za-z])/g,
  /(?<![0-9A-Za-z])\d{4}[-/]\d{1,2}(?![-/.\d])/g,
  new RegExp("(?<![0-9A-Za-z])(?:" + MON + ")\\.?\\s+\\d{1,2}(?:st|nd|rd|th)?,?\\s+\\d{4}(?![0-9A-Za-z])", "gi"),
  new RegExp("(?<![0-9A-Za-z])\\d{1,2}(?:st|nd|rd|th)?\\s+(?:" + MON + ")\\.?,?\\s+\\d{4}(?![0-9A-Za-z])", "gi"),
  new RegExp("(?<![0-9A-Za-z])(?:" + MON + ")\\.?,?\\s+\\d{4}(?![0-9A-Za-z])", "gi"),
  /(?<![0-9A-Za-z])(?:FY)\s?\d{4}(?![0-9A-Za-z])/gi,
  /\d{4}\s*年度/g,
  /(?<![0-9])\d{4}\s*年(?![0-9度])/g,
];

// Relative expressions are counted but never become claims: they cannot be
// resolved without knowing when the answer was generated.
const RELATIVE_DATE_RE = /(?:昨年|一昨年|来年|再来年|今年|本年|先月|今月|来月|先週|今週|来週|昨日|本日|今日|明日|明後日|現在|最近|近年|直近|先般|過去\d+年|今後\d+年|\byesterday\b|\btoday\b|\btomorrow\b|\blast\s+(?:year|month|week|quarter)\b|\bnext\s+(?:year|month|week|quarter)\b|\bthis\s+(?:year|month|week|quarter)\b|\brecently\b|\bcurrently\b|\bnowadays\b)/gi;

const QUOTE_BRACKET_RE = /[「『“"]([^「」『』“”"\n]{4,180})[」』”"]/g;
const QUOTE_MARKER_RE = /(?:によると|によれば|には|と記載(?:されて)?(?:い)?(?:ます|る)?|と明記|と述べ|と書かれ|と報告|と発表|states?\s+that|according\s+to|reported\s+that|said\s+that|notes?\s+that|writes?\s+that|claims?\s+that|indicates?\s+that)/gi;

const ENT_CAPWORDS_RE = /\b[A-Z][A-Za-z0-9&'’.\-]{1,30}(?:\s+(?:of|the|and|for|de|von|van)\s+|\s+)(?:[A-Z][A-Za-z0-9&'’.\-]{1,30})(?:\s+[A-Z][A-Za-z0-9&'’.\-]{1,30}){0,3}/g;
const ENT_MODEL_RE = /\b(?=[A-Za-z0-9\-]{4,24}\b)(?=[^\s]*\d)[A-Z]{1,6}[A-Za-z]{0,4}[-_]?\d{2,6}[A-Za-z0-9\-]{0,8}\b/g;
const ENT_ARTICLE_RE = /(?:第\s*\d+\s*条(?:\s*の\s*\d+)?(?:\s*第\s*\d+\s*項)?|Article\s+\d+(?:\.\d+)?|Sections?\s+\d+(?:\.\d+)?|§\s?\d+(?:\.\d+)?|ISO\s?\d{3,5}(?::\d{4})?|RFC\s?\d{3,5})/gi;
const ENT_ACRONYM_RE = /\b[A-Z]{3,8}\b/g;

// ISO currency codes are generic tokens, not entities. Left in, the acronym
// extractor claimed "USD" and its span blocked the number after it, so
// "USD 45,000" lost its number entirely.
const CURRENCY_CODES = new Set("usd jpy eur gbp krw cny chf aud cad hkd sgd inr brl rub sek nok dkk nzd mxn zar thb twd php idr vnd pln czk huf try ils aed sar qar myr clp cop ars".split(" "));

const ENTITY_STOPWORDS = new Set((`
the a an and or but if then than that this these those there here it its it's is are was were be been
i you we they he she him her his our your their us them my me not no yes of in on at to for from by with
as into over under about after before between during without within across per via such same other another
however therefore moreover furthermore additionally meanwhile although because since while when where which who whom
note notes please thank thanks hello hi ok okay yes no true false null none
january february march april may june july august september october november december
monday tuesday wednesday thursday friday saturday sunday
summary conclusion introduction overview background result results method methods discussion reference references
answer question context example examples figure table appendix section chapter page pages
company service product system data user users client customer report document file page site page
ai llm api url json html http https pdf csv xml sql cpu gpu ram ssd usb faq ceo cfo cto url uri id ids
note caution warning important warning info tip
`).trim().split(/\s+/).concat([...CURRENCY_CODES]));

const ABBREVIATIONS = new Set("mr mrs ms dr prof st vs etc e.g i.e fig no inc ltd co corp jr sr approx al u.s u.k cf vol pp est dept univ jan feb mar apr jun jul aug sep sept oct nov dec".split(" "));
const SENT_TERMINATORS = "。．！？!?\n";

class SpanMask {
  constructor(n) { this.m = new Uint8Array(Math.max(0, n)); }
  add(s, e) { if (e > s && s >= 0) this.m.fill(1, s, Math.min(e, this.m.length)); }
  addAll(spans) { for (const sp of spans) this.add(sp[0], sp[1]); }
  hits(s, e) { if (e <= s) return false; for (let i = Math.max(0, s); i < Math.min(e, this.m.length); i++) if (this.m[i]) return true; return false; }
  contains(s, e) { if (e <= s) return false; if (e > this.m.length) return false; for (let i = s; i < e; i++) if (!this.m[i]) return false; return true; }
}

function overlaps(a, b) { return !(a[1] <= b[0] || b[1] <= a[0]); }

/** Guarantees: [start, end, text] per sentence, offsets exact; '.' between digits never splits. */
export function splitSentences(text, protectedSpans = []) {
  const out = [];
  if (!text) return out;
  const n = text.length;
  let start = 0, i = 0;
  while (i < n) {
    const ch = text[i];
    if (SENT_TERMINATORS.includes(ch)) {
      let inside = false;
      for (const r of protectedSpans) if (r[0] <= i && i < r[1]) { inside = true; break; }
      if (inside) { i++; continue; }
      if (ch === "．" || ch === ".") {
        const prev = i > 0 ? text[i - 1] : "", nxt = i + 1 < n ? text[i + 1] : "";
        if (/\d/.test(prev) && /\d/.test(nxt)) { i++; continue; }
        const tail = text.slice(Math.max(0, i - 12), i).toLowerCase();
        const parts = tail.split(/[^a-z.]/);
        const word = parts.length ? parts[parts.length - 1] : "";
        if (word && ABBREVIATIONS.has(word)) { i++; continue; }
      }
      let j = i + 1;
      while (j < n && (SENT_TERMINATORS.includes(text[j]) || "」』”\")".includes(text[j]))) j++;
      const seg = text.slice(start, j);
      if (seg.trim()) out.push([start, j, seg]);
      start = j; i = j; continue;
    }
    i++;
  }
  if (start < n) { const seg = text.slice(start, n); if (seg.trim()) out.push([start, n, seg]); }
  return out;
}

function* iterUrls(text) {
  URL_RE.lastIndex = 0;
  let m;
  while ((m = URL_RE.exec(text || "")) !== null) {
    const trimmed = m[0].replace(/[.,;:!?)»”"']+$/, "");
    yield [m.index, m.index + trimmed.length, trimmed];
  }
}

/** Guarantees: a comparable URL form (lowercased, scheme/www/trailing-slash stripped). */
export function normUrl(u) {
  try { return normText(u).replace(/^(?:https?:\/\/|ftp:\/\/)/, "").replace(/^www\./, "").replace(/\/+$/, "").split("#")[0]; }
  catch (e) { return ""; }
}
function urlHost(u) { try { return normUrl(u).split("/")[0].split("?")[0]; } catch (e) { return ""; } }

function numericClaimFromMatch(m, whole) {
  const g = m.groups || {};
  const raw = m[0].trim();
  if (!raw) return null;
  let base = Number((g.num || "").replace(/,/g, "") + (g.exp || ""));
  if (!Number.isFinite(base)) return null;
  if (g.sign === "-" || g.sign === "▲" || g.sign === "△") base = -base;
  if (g.mag && MAG_FACTORS[g.mag] !== undefined) base *= MAG_FACTORS[g.mag];
  let kind = "number";
  if (g.pct === "%" || g.pct === "パーセント") kind = "percent";
  else if (g.pct === "割") { base *= 10; kind = "percent"; }
  else if (g.pct === "ポイント") kind = "point";
  else if (g.cur) kind = "currency";
  else if (g.unit) kind = "unit";

  let start = m.index;
  let end = m.index + m[0].replace(/\s+$/, "").length;
  // Parenthesised accounting negatives: "(1,200)" is -1,200 in financial tables.
  if (!g.sign && start > 0 && whole[start - 1] === "(" && end < whole.length && whole[end] === ")") {
    base = -base; start -= 1; end += 1;
  }
  return { type: "NUMERIC", value: raw, normalized: String(base), kind,
           unit: (g.pct || g.unit || g.cur || g.mag || "").trim(), span: [start, end], _num: base };
}

/** Guarantees: true for enumeration markers like "1." or "2)" at a line start. */
function isListMarker(text, m) {
  const s = m.index;
  const lineStart = text.lastIndexOf("\n", Math.max(0, s - 1)) + 1;
  const before = text.slice(lineStart, s).trim();
  if (!["", "-", "*", "・", "(", "（"].includes(before)) return false;
  const after = text.slice(m.index + m[0].length, m.index + m[0].length + 2);
  return [".", ")", "、", "．", "）", ""].includes(after.slice(0, 1));
}

/** Guarantees: typed, span-accurate claims; sentences yielding none are what lowers coverage. */
export function extractClaims(answer, options = {}) {
  try {
    if (!answer) return [];
    const o = Object.assign({ enable_quote: true, enable_entity: true, enable_url: true,
                              enable_numeric: true, enable_date: true, max_claims: DEFAULTS.MAX_CLAIMS }, options);
    const n = answer.length;
    const urlSpans = [], urls = [];
    for (const [s, e, u] of iterUrls(answer)) { urlSpans.push([s, e]); urls.push([s, e, u]); }
    let sentences = splitSentences(answer, urlSpans);
    if (!sentences.length) sentences = [[0, n, answer]];
    const sentStarts = sentences.map((x) => x[0]);
    const sentIndex = (pos) => {
      let lo = 0, hi = sentStarts.length - 1, ans = 0;
      while (lo <= hi) { const mid = (lo + hi) >> 1; if (sentStarts[mid] <= pos) { ans = mid; lo = mid + 1; } else hi = mid - 1; }
      return Math.min(ans, sentences.length - 1);
    };

    const claims = [];
    const taken = new SpanMask(n);
    if (o.enable_url) for (const [s, e, u] of urls)
      claims.push({ type: "URL", value: u, normalized: normUrl(u), span: [s, e], sentence_index: sentIndex(s) });
    taken.addAll(urlSpans);

    const quoteMask = new SpanMask(n);
    if (o.enable_quote) {
      QUOTE_BRACKET_RE.lastIndex = 0; let m;
      while ((m = QUOTE_BRACKET_RE.exec(answer)) !== null) {
        const s = m.index + m[0].indexOf(m[1]), e = s + m[1].length;
        if (taken.contains(s, e)) continue;
        const body = m[1].trim();
        if (body.length < 4) continue;
        quoteMask.add(s, e);
        claims.push({ type: "QUOTE", value: body, normalized: normText(body), span: [s, e], sentence_index: sentIndex(s), source: "bracket" });
      }
      QUOTE_MARKER_RE.lastIndex = 0;
      while ((m = QUOTE_MARKER_RE.exec(answer)) !== null) {
        const s = m.index + m[0].length;
        const si = sentIndex(s);
        const sentEnd = si < sentences.length ? sentences[si][1] : n;
        const raw = answer.slice(s, sentEnd);
        const lead = raw.length - raw.replace(/^[ 　:：,、\n]+/, "").length;
        const body = raw.trim().replace(/^[ 　:：,、。.\n"'「」『』“”]+|[ 　:：,、。.\n"'「」『』“”]+$/g, "");
        if (body.length < 8) continue;
        const bs = s + lead, be = bs + body.length;
        if (quoteMask.hits(bs, be) || taken.contains(bs, be)) continue;
        quoteMask.add(bs, be);
        claims.push({ type: "QUOTE", value: body.slice(0, 200), normalized: normText(body.slice(0, 200)),
                      span: [bs, Math.min(be, bs + 200)], sentence_index: si, source: "marker" });
      }
    }

    const dateSpans = [], dateMask = new SpanMask(n), urlMask = new SpanMask(n);
    urlMask.addAll(urlSpans);
    if (o.enable_date) {
      for (const pat of DATE_PATTERNS) {
        pat.lastIndex = 0; let m;
        while ((m = pat.exec(answer)) !== null) {
          const s = m.index, e = m.index + m[0].length;
          if (m[0].length === 0) { pat.lastIndex++; continue; }
          if (dateMask.hits(s, e) || urlMask.contains(s, e)) continue;
          const iso = normDate(m[0]);
          if (!iso) continue;
          dateSpans.push([s, e]); dateMask.add(s, e);
          claims.push({ type: "DATE", value: m[0], normalized: iso, span: [s, e], sentence_index: sentIndex(s) });
        }
      }
    }
    taken.addAll(dateSpans);

    const entSpans = [], entMask = new SpanMask(n);
    if (o.enable_entity) {
      for (const [pat, sub] of [[ENT_ARTICLE_RE, "article"], [ENT_MODEL_RE, "model"]]) {
        pat.lastIndex = 0; let m;
        while ((m = pat.exec(answer)) !== null) {
          const s = m.index, e = m.index + m[0].length;
          if (taken.contains(s, e) || entMask.hits(s, e)) continue;
          const v = m[0].trim();
          const lead = /^[A-Za-z]+/.exec(v);           // "USD45,000" is a price, not a part number
          if (sub === "model" && lead && CURRENCY_CODES.has(lead[0].toLowerCase())) continue;
          entSpans.push([s, e]); entMask.add(s, e);
          claims.push({ type: "ENTITY", value: v, normalized: normText(v), span: [s, e], sentence_index: sentIndex(s), subtype: sub });
        }
      }
      ENT_CAPWORDS_RE.lastIndex = 0; let m;
      while ((m = ENT_CAPWORDS_RE.exec(answer)) !== null) {
        let s = m.index, e = m.index + m[0].length;
        if (taken.contains(s, e) || entMask.hits(s, e)) continue;
        let v = m[0].trim();
        const toks = normText(v).split(/\s+/).filter(Boolean);
        if (!toks.length) continue;
        if (toks.every((t) => ENTITY_STOPWORDS.has(t.replace(/^[.,'’-]+|[.,'’-]+$/g, "")))) continue;
        // Trim leading/trailing function words so "The NASA report" compares as "NASA report".
        let words = v.split(/\s+/);
        while (words.length > 1 && ENTITY_STOPWORDS.has(normText(words[0]).replace(/^[.,'’-]+|[.,'’-]+$/g, ""))) { s += words[0].length + 1; words = words.slice(1); }
        while (words.length > 1 && ENTITY_STOPWORDS.has(normText(words[words.length - 1]).replace(/^[.,'’-]+|[.,'’-]+$/g, ""))) { e -= words[words.length - 1].length + 1; words = words.slice(0, -1); }
        v = words.join(" ");
        if (v.length < 4 || e <= s) continue;
        entSpans.push([s, e]); entMask.add(s, e);
        claims.push({ type: "ENTITY", value: v, normalized: normText(v), span: [s, e], sentence_index: sentIndex(s), subtype: "name" });
      }
      ENT_ACRONYM_RE.lastIndex = 0;
      while ((m = ENT_ACRONYM_RE.exec(answer)) !== null) {
        const s = m.index, e = m.index + m[0].length;
        if (taken.contains(s, e) || entMask.hits(s, e)) continue;
        if (ENTITY_STOPWORDS.has(normText(m[0]))) continue;
        entSpans.push([s, e]); entMask.add(s, e);
        claims.push({ type: "ENTITY", value: m[0], normalized: normText(m[0]), span: [s, e], sentence_index: sentIndex(s), subtype: "acronym" });
      }
    }

    if (o.enable_numeric) {
      const block = new SpanMask(n);
      block.addAll(urlSpans); block.addAll(dateSpans); block.addAll(entSpans);
      NUM_RE.lastIndex = 0; let m;
      while ((m = NUM_RE.exec(answer)) !== null) {
        if (m[0].length === 0) { NUM_RE.lastIndex++; continue; }
        const s = m.index, e = m.index + m[0].length;
        if (block.hits(s, e)) continue;
        if (isListMarker(answer, m)) continue;
        const c = numericClaimFromMatch(m, answer);
        if (!c) continue;
        c.sentence_index = sentIndex(c.span[0]);
        claims.push(c);
      }
    }

    claims.sort((a, b) => a.span[0] - b.span[0] || a.span[1] - b.span[1]);
    let dropped = 0;
    if (claims.length > o.max_claims) { dropped = claims.length - o.max_claims; claims.length = o.max_claims; }
    claims.forEach((c, i) => { c.id = "c" + String(i).padStart(4, "0"); });
    if (dropped) claims._dropped = dropped;
    return claims;
  } catch (e) { return []; }
}

/* ==================================================================== (D) VERIFIER */

/** Guarantees: a stable 9-significant-digit key, so 0.1+0.2 and 0.3 collide. */
function fkey(v) {
  const f = Number(v);
  if (!Number.isFinite(f)) return "";
  if (f === 0) return "0";
  let s = String(Number(f.toPrecision(9)));
  return s === "-0" ? "0" : s;
}

/**
 * Guarantees: a bounded, time-budgeted map from a derived value to its formula.
 * Built once per verify() call, so each numeric claim is an O(1) lookup.
 * Coincidental hits are possible (2 and 3 in context make 6 "derivable"), which
 * is why the formula is always returned as evidence for a human to reject.
 */
class DerivationIndex {
  constructor(numbers, maxTerms, budgetMs, maxNumbers) {
    this.map = new Map(); this.truncated = false;
    const t0 = Date.now(), budget = Math.max(5, budgetMs);
    const seen = new Set(), uniq = [];
    for (const pair of numbers) {
      const k = fkey(pair[0]);
      if (!k || seen.has(k)) continue;
      seen.add(k); uniq.push(pair);
      if (uniq.length >= maxNumbers) { this.truncated = true; break; }
    }
    if (uniq.length < 2) return;
    const pairs = [];
    for (let i = 0; i < uniq.length; i++) {
      for (let j = 0; j < uniq.length; j++) {
        if (i === j) continue;
        for (const r of DerivationIndex.combine(uniq[i][0], uniq[i][1], uniq[j][0], uniq[j][1], false)) {
          this.put(r[0], r[1]);
          if (pairs.length < 800) pairs.push(r);
        }
      }
      if (Date.now() - t0 > budget) { this.truncated = true; return; }
    }
    if (maxTerms >= 3 && uniq.length <= 20) {
      for (const p of pairs) {
        if (Date.now() - t0 > budget) { this.truncated = true; return; }
        for (const c of uniq)
          for (const r of DerivationIndex.combine(p[0], "(" + p[1] + ")", c[0], c[1], true)) this.put(r[0], r[1]);
      }
    }
  }
  static combine(a, ar, b, br, third) {
    const out = [[a + b, ar + " + " + br], [a - b, ar + " - " + br], [a * b, ar + " * " + br]];
    if (b !== 0) {
      out.push([a / b, ar + " / " + br]);
      if (!third) {
        out.push([(a / b) * 100, ar + " / " + br + " * 100 (percentage)"]);
        out.push([((a - b) / b) * 100, "(" + ar + " - " + br + ") / " + br + " * 100 (change rate)"]);
      }
    }
    if (!third) out.push([(a * b) / 100, ar + " * " + br + "% (percent of)"]);
    return out.filter((x) => Number.isFinite(x[0]));
  }
  put(v, ex) { if (Math.abs(v) < 1e-9) return; const k = fkey(v); if (k && !this.map.has(k)) this.map.set(k, ex); }
  lookup(v) { return this.map.get(fkey(v)) || null; }
}

function contextNumbers(context) {
  const out = [];
  try {
    context = context || "";
    const mask = new SpanMask(context.length);
    for (const u of iterUrls(context)) mask.add(u[0], u[1]);
    for (const pat of DATE_PATTERNS) {
      pat.lastIndex = 0; let m;
      while ((m = pat.exec(context)) !== null) { if (!m[0].length) { pat.lastIndex++; continue; } mask.add(m.index, m.index + m[0].length); }
    }
    NUM_RE.lastIndex = 0; let m;
    while ((m = NUM_RE.exec(context)) !== null) {
      if (!m[0].length) { NUM_RE.lastIndex++; continue; }
      if (mask.hits(m.index, m.index + m[0].length)) continue;
      const c = numericClaimFromMatch(m, context);
      if (c) out.push([c._num, c.value]);
    }
  } catch (e) {}
  return out;
}

function contextDates(context) {
  const out = new Set();
  for (const pat of DATE_PATTERNS) {
    pat.lastIndex = 0; let m;
    while ((m = pat.exec(context || "")) !== null) {
      if (!m[0].length) { pat.lastIndex++; continue; }
      const iso = normDate(m[0]); if (iso) out.add(iso);
    }
  }
  return out;
}

/** Guarantees: one pass over the context produces every lookup table the verifiers need. */
class ContextIndex {
  constructor(context, opts) {
    this.raw = context || "";
    this.norm = normText(this.raw);
    this.numbers = contextNumbers(this.raw);
    this.numMap = new Map();
    for (const p of this.numbers) { const k = fkey(p[0]); if (k && !this.numMap.has(k)) this.numMap.set(k, p[1]); }
    this.dates = contextDates(this.raw);
    // Years stated only as part of a context date still support a bare year in
    // the answer; without this, context "FY2024" vs answer "2024" was unsupported.
    this.dateYears = new Set(Array.from(this.dates).map((d) => parseInt(d.split("-")[0], 10)).filter(Number.isFinite));
    this.urls = new Map();
    for (const u of iterUrls(this.raw)) this.urls.set(normUrl(u[2]), u[2]);
    this.hosts = new Set(Array.from(this.urls.values()).map(urlHost));
    this.tokens = new Set(this.norm.split(/[^0-9a-z぀-ヿ一-鿿]+/).filter(Boolean));
    this._deriv = null;
    this._opts = opts || {};
    const fb = this._opts.fuzzy_budget_ms;
    this.fuzzyDeadline = Date.now() + (fb === undefined ? DEFAULTS.FUZZY_BUDGET_MS : fb);
    this.fuzzyExhausted = false;
  }
  get deriv() {
    if (!this._deriv) {
      const o = this._opts;
      this._deriv = new DerivationIndex(this.numbers,
        o.derive_max_terms === undefined ? DEFAULTS.DERIVE_MAX_TERMS : o.derive_max_terms,
        o.derive_budget_ms === undefined ? DEFAULTS.DERIVE_BUDGET_MS : o.derive_budget_ms,
        o.derive_max_numbers === undefined ? DEFAULTS.DERIVE_MAX_NUMBERS : o.derive_max_numbers);
    }
    return this._deriv;
  }
  /** Guarantees: a bounded fuzzy match, or [0,""] once the request budget is spent. */
  fuzzy(needle) {
    if (Date.now() > this.fuzzyDeadline) { this.fuzzyExhausted = true; return [0.0, ""]; }
    return bestFuzzy(needle, this.norm);
  }
}

function nearestNumber(target, candidates) {
  let best = null;
  for (const p of candidates) {
    const f = p[0];
    const denom = Math.max(Math.abs(target), Math.abs(f), 1e-12);
    const rel = Math.abs(target - f) / denom;
    if (!best || rel < best[2]) best = [f, p[1], rel];
  }
  return best;
}

function orderOfMagnitudeOff(a, b) {
  if (a === 0 || b === 0) return null;
  const r = Math.abs(a) / Math.abs(b);
  if (!(r > 0)) return null;
  const lg = Math.log10(r), k = Math.round(lg);
  return k !== 0 && Math.abs(lg - k) < 0.02 && Math.abs(k) <= 9 ? k : null;
}

/*
 * NUMERIC. Misfires that can happen, concretely:
 *   1) context "approx 1,200 items" vs answer "1200 items" -> supported after
 *      normalisation. But context "1,200" vs answer "1,250" is 4% off ->
 *      contradicted. To allow rounding raise numeric_tolerance, do NOT lower
 *      contradiction_rel.
 *   2) context "revenue 100, cost 40" vs answer "profit 60" -> derived (100 - 40).
 *   3) context "3.5%" vs answer "3.5 points" matches on value -> supported.
 *      Semantic unit errors are invisible to this tool.
 */
function verifyNumeric(claim, ctx, o) {
  let f = claim._num;
  if (f === undefined || f === null) f = normNumber(claim.value);
  if (f === null || !Number.isFinite(f)) { claim.status = "unsupported"; claim.evidence = null; claim.reason = "unparseable_number"; return claim; }
  const tol = o.numeric_tolerance === undefined ? 0 : o.numeric_tolerance;
  const contrRel = o.contradiction_rel === undefined ? DEFAULTS.CONTRADICTION_REL : o.contradiction_rel;

  const k = fkey(f);
  if (k && ctx.numMap.has(k)) { claim.status = "supported"; claim.evidence = "context literal: " + ctx.numMap.get(k); return claim; }
  if (tol > 0) {
    const near = nearestNumber(f, ctx.numbers);
    if (near && near[2] <= tol) { claim.status = "supported"; claim.evidence = "context literal within tolerance " + tol + ": " + near[1]; return claim; }
  }
  if (claim.kind === "percent") {
    const ak = fkey(f / 100);
    if (ak && ctx.numMap.has(ak)) { claim.status = "derived"; claim.evidence = (f / 100) + " == " + f + "% (fraction form in context) -> " + ctx.numMap.get(ak); return claim; }
  } else if (Math.abs(f) < 1) {
    const ak = fkey(f * 100);
    if (ak && ctx.numMap.has(ak)) { claim.status = "derived"; claim.evidence = f + " == " + (f * 100) + "% (percent form in context: " + ctx.numMap.get(ak) + ")"; return claim; }
  }
  // An exact sign flip is settled BEFORE derivation. Otherwise a loss reported
  // as a profit gets excused by a coincidental 3-term formula and passes.
  if (f !== 0) {
    const flip = fkey(-f);
    if (flip && ctx.numMap.has(flip)) {
      claim.status = "contradicted";
      claim.evidence = "context states " + ctx.numMap.get(flip) + ", the exact negation of this value (sign error)";
      claim.nearest_context_value = ctx.numMap.get(flip);
      return claim;
    }
  }
  if (o.enable_derivation !== false) {
    const expr = ctx.deriv.lookup(f);
    if (expr) { claim.status = "derived"; claim.evidence = "derivable from context: " + expr + " = " + fkey(f); return claim; }
  }
  if (claim.kind === "number" && !claim.unit && Number.isInteger(f) && f >= 1900 && f <= 2100 && ctx.dateYears.has(f)) {
    claim.status = "supported"; claim.evidence = "matches a year stated in the context (" + f + ")"; return claim;
  }
  const near = nearestNumber(f, ctx.numbers);
  if (near) {
    const nf = near[0], nraw = near[1], rel = near[2];
    const oom = orderOfMagnitudeOff(f, nf);
    // Bare integers 0-10 with no unit are near everything; never contradicted.
    const bareSmall = claim.kind === "number" && Math.abs(f) <= 10 && Number.isInteger(f);
    if (oom !== null && !bareSmall) {
      claim.status = "contradicted";
      claim.evidence = "closest context value " + nraw + " differs by 10^" + oom + " (order-of-magnitude error)";
      claim.nearest_context_value = nraw; return claim;
    }
    if (!bareSmall && nf !== 0 && f === -nf) {
      claim.status = "contradicted"; claim.evidence = "closest context value " + nraw + " has the opposite sign";
      claim.nearest_context_value = nraw; return claim;
    }
    if (!bareSmall && rel <= contrRel) {
      claim.status = "contradicted";
      claim.evidence = "closest context value is " + nraw + " (relative difference " + (rel * 100).toFixed(1) + "%)";
      claim.nearest_context_value = nraw; return claim;
    }
    claim.nearest_context_value = nraw;
  }
  claim.status = "unsupported"; claim.evidence = null; return claim;
}

/*
 * DATE. Misfires that can happen, concretely:
 *   - context "2024-01" vs answer "2024-01-15" -> approximate: the model added
 *     precision the source never stated.
 *   - FY2024 and calendar 2024 are treated as the same year. Fiscal-year start
 *     months differ by country and company, so this is a known false positive.
 */
function verifyDate(claim, ctx) {
  const iso = claim.normalized || normDate(claim.value);
  if (!iso) { claim.status = "unsupported"; claim.evidence = null; claim.reason = "unparseable_date"; return claim; }
  if (ctx.dates.has(iso)) { claim.status = "supported"; claim.evidence = "context date: " + iso; return claim; }
  for (const d of ctx.dates) if (d.startsWith(iso)) { claim.status = "supported"; claim.evidence = "context date " + d + " falls inside " + iso; return claim; }
  for (const d of ctx.dates) if (iso.startsWith(d)) { claim.status = "approximate"; claim.evidence = "context only states " + d + "; the answer adds precision (" + iso + ")"; return claim; }
  let best = null;
  for (const d of ctx.dates) {
    let common = 0;
    const a = iso.split("-"), b = d.split("-");
    for (let i = 0; i < Math.min(a.length, b.length); i++) { if (a[i] === b[i]) common++; else break; }
    if (!best || common > best[1]) best = [d, common];
  }
  if (best && best[1] >= 1) { claim.status = "contradicted"; claim.evidence = "context has a near but different date: " + best[0]; claim.nearest_context_value = best[0]; return claim; }
  claim.status = "unsupported"; claim.evidence = null; return claim;
}

/*
 * QUOTE. Misfires that can happen, concretely:
 *   - A politeness-level paraphrase comes out approximate: the meaning survived,
 *     the quote did not. That is intended.
 *   - KNOWN LIMITATION: a one-character semantic reversal in Japanese
 *     (increase -> decrease, approve -> reject) keeps similarity above 0.9 and
 *     stays "approximate". A hard limit of character-level matching.
 */
function verifyQuote(claim, ctx, o) {
  const q = claim.normalized || normText(claim.value);
  if (!q || q.length < 3) { claim.status = "unsupported"; claim.evidence = null; claim.reason = "quote_too_short"; return claim; }
  if (ctx.norm.includes(q)) { claim.status = "supported"; claim.evidence = "exact (normalised) substring of the context"; return claim; }
  const res = ctx.fuzzy(q), ratio = res[0], win = res[1];
  const thr = o.approx_ratio === undefined ? DEFAULTS.APPROX_RATIO : o.approx_ratio;
  claim.similarity = Math.round(ratio * 10000) / 10000;
  if (ratio >= thr) { claim.status = "approximate"; claim.evidence = "closest context passage (similarity " + ratio.toFixed(2) + "): " + win.slice(0, 180); return claim; }
  const floor = o.quote_contradiction_floor === undefined ? DEFAULTS.QUOTE_CONTRADICTION_FLOOR : o.quote_contradiction_floor;
  if (ratio >= floor) {
    claim.status = "contradicted";
    claim.evidence = "a similar but materially different passage exists (similarity " + ratio.toFixed(2) + "): " + win.slice(0, 180);
    claim.nearest_context_value = win.slice(0, 180); return claim;
  }
  claim.status = "unsupported"; claim.evidence = null; return claim;
}

/*
 * ENTITY. Misfires that can happen, concretely:
 *   - Generic noun phrases like "Machine Learning" come out unsupported although
 *     nothing was hallucinated.
 *   - Acronym expansion is not handled: context "World Health Organization" vs
 *     answer "WHO" finds no match.
 *   ENTITY is the noisiest check. Disable it first.
 */
function verifyEntity(claim, ctx) {
  const v = claim.normalized || normText(claim.value);
  if (!v) { claim.status = "unsupported"; claim.evidence = null; return claim; }
  if (ENTITY_STOPWORDS.has(v)) { claim.status = "supported"; claim.evidence = "common word, not treated as a factual entity"; claim.suppressed = true; return claim; }
  if (ctx.norm.includes(v)) { claim.status = "supported"; claim.evidence = "appears verbatim in the context"; return claim; }
  const toks = v.split(/[^0-9a-z぀-ヿ一-鿿]+/).filter(Boolean);
  const meaningful = toks.filter((t) => !ENTITY_STOPWORDS.has(t));
  if (meaningful.length && meaningful.every((t) => ctx.tokens.has(t))) {
    claim.status = "approximate"; claim.evidence = "all tokens appear in the context but not as this phrase"; return claim;
  }
  const res = ctx.fuzzy(v), ratio = res[0], win = res[1];
  claim.similarity = Math.round(ratio * 10000) / 10000;
  // Identifiers are exact-or-wrong: "Article 15" is not "approximately Article
  // 12". This branch must precede the generic near-match branch below, or a
  // one-digit-off article number is reported as a harmless approximation.
  if ((claim.subtype === "model" || claim.subtype === "article") && ratio >= 0.72) {
    claim.status = "contradicted"; claim.evidence = "a similar identifier exists in the context: " + win.slice(0, 120);
    claim.nearest_context_value = win.slice(0, 120); return claim;
  }
  if (ratio >= 0.90 && v.length >= 6) { claim.status = "approximate"; claim.evidence = "near match in context (similarity " + ratio.toFixed(2) + "): " + win.slice(0, 120); return claim; }
  claim.status = "unsupported"; claim.evidence = null; return claim;
}

/*
 * URL. Misfires that can happen, concretely:
 *   - A tracking parameter makes an otherwise identical link fail exact matching.
 *   - Host matches but path differs -> contradicted. It may be a legitimate
 *     parent page rather than a fabricated one.
 */
function verifyUrl(claim, ctx) {
  const u = claim.normalized || normUrl(claim.value);
  if (!u) { claim.status = "unsupported"; claim.evidence = null; return claim; }
  if (ctx.urls.has(u)) { claim.status = "supported"; claim.evidence = "context URL: " + ctx.urls.get(u); return claim; }
  for (const entry of ctx.urls) {
    const cu = entry[0], orig = entry[1];
    if (cu.startsWith(u) || u.startsWith(cu)) { claim.status = "approximate"; claim.evidence = "context has a related URL: " + orig; return claim; }
  }
  const h = urlHost(u);
  if (h && ctx.hosts.has(h)) {
    claim.status = "contradicted";
    claim.evidence = "host " + h + " appears in the context but this exact path does not (possible fabricated link)";
    claim.nearest_context_value = h; return claim;
  }
  claim.status = "unsupported"; claim.evidence = null; return claim;
}

const VERIFIERS = { NUMERIC: verifyNumeric, DATE: verifyDate, QUOTE: verifyQuote, ENTITY: verifyEntity, URL: verifyUrl };

function nowMs() { return (typeof performance !== "undefined" && performance.now) ? performance.now() : Date.now(); }

/** Guarantees: grounding_score AND coverage, separately, plus a status per claim. Never throws. */
export function verify(answer, context, options) {
  const t0 = nowMs();
  const o = Object.assign({}, options || {});
  const maxChars = o.max_chars === undefined ? DEFAULTS.MAX_TEXT_CHARS : o.max_chars;
  const aTrunc = (answer || "").length > maxChars, cTrunc = (context || "").length > maxChars;
  answer = (answer || "").slice(0, maxChars);
  context = (context || "").slice(0, maxChars);

  const claims = extractClaims(answer, o);
  const claimsDropped = claims._dropped || 0;
  const ctx = new ContextIndex(context, o);
  const urlSpans = [];
  for (const u of iterUrls(answer)) urlSpans.push([u[0], u[1]]);
  const sentences = splitSentences(answer, urlSpans);
  const nSent = sentences.length;
  const sentWith = new Set();

  // Identical claim text gets the identical verdict against the same context,
  // so a repetitive answer costs one verification per distinct value.
  const memo = new Map();
  const CARRY = ["status", "evidence", "reason", "similarity", "nearest_context_value", "suppressed"];
  for (const c of claims) {
    const key = [c.type, c.normalized, c.value, c.subtype].join(" |~| ");
    const cached = memo.get(key);
    if (cached) { for (const k of CARRY) if (k in cached) c[k] = cached[k]; }
    else {
      const fn = VERIFIERS[c.type];
      try { if (!fn) { c.status = "unsupported"; c.evidence = null; } else fn(c, ctx, o); }
      catch (e) { c.status = "unsupported"; c.evidence = null; c.reason = "verifier_error"; }
      const snap = {}; for (const k of CARRY) if (k in c) snap[k] = c[k];
      memo.set(key, snap);
    }
    if (!c.suppressed) sentWith.add(c.sentence_index);
  }

  const counts = {}; STATUSES.forEach((s) => (counts[s] = 0));
  const byType = {}; CLAIM_TYPES.forEach((t) => { byType[t] = {}; STATUSES.forEach((s) => (byType[t][s] = 0)); });
  const effective = [];
  for (const c of claims) {
    if (c.suppressed) continue;
    effective.push(c);
    counts[c.status] = (counts[c.status] || 0) + 1;
    if (byType[c.type]) byType[c.type][c.status] = (byType[c.type][c.status] || 0) + 1;
  }
  const total = effective.length;
  const good = counts.supported + counts.derived;
  const grounding = total ? good / total : 0;
  const coverage = nSent ? sentWith.size / nSent : 0;
  RELATIVE_DATE_RE.lastIndex = 0;
  const relDates = (answer.match(RELATIVE_DATE_RE) || []).length;

  const notes = [
    "grounding_score is computed over verifiable claims only.",
    "coverage is the share of sentences that produced at least one verifiable claim; the rest were NOT checked.",
  ];
  if (claimsDropped) notes.push(claimsDropped + " further claims were dropped by the max_claims cap and were NOT checked.");
  if (ctx.fuzzyExhausted) notes.push("The fuzzy-matching budget ran out; later QUOTE/ENTITY claims were matched exactly only.");

  const unverified = [];
  for (let i = 0; i < sentences.length && unverified.length < 50; i++)
    if (!sentWith.has(i)) unverified.push({ sentence_index: i, text: sentences[i][2].trim().slice(0, 200) });

  return {
    ok: true,
    grounding_score: Math.round(grounding * 10000) / 10000,
    coverage: Math.round(coverage * 10000) / 10000,
    claims: effective.map((c) => { const d = {}; for (const k in c) if (!k.startsWith("_")) d[k] = c[k]; if (!("evidence" in d)) d.evidence = null; return d; }),
    counts: {
      claims_total: total, sentences_total: nSent, sentences_verified: sentWith.size,
      sentences_unverified: Math.max(0, nSent - sentWith.size), relative_date_mentions: relDates,
      by_status: counts, by_type: byType,
      context_numbers: ctx.numbers.length, context_dates: ctx.dates.size,
      derivation_entries: ctx._deriv ? ctx._deriv.map.size : 0,
      derivation_truncated: ctx._deriv ? ctx._deriv.truncated : false,
      claims_dropped_by_cap: claimsDropped, distinct_claims_verified: memo.size,
      fuzzy_budget_exhausted: ctx.fuzzyExhausted,
    },
    unverified_sentences: unverified,
    truncated: { answer: aTrunc, context: cTrunc },
    latency_ms: Math.round((nowMs() - t0) * 100) / 100,
    notes,
  };
}

/* ============================================================== (E) LEAK DETECTOR */

function leakTokens(s) {
  const out = normText(s).match(/[0-9a-z]+|[぀-ヿ一-鿿가-힯]/g);
  return out || [];
}
function ngrams(tokens, n) {
  const out = new Set();
  if (n <= 0 || tokens.length < n) return out;
  for (let i = 0; i <= tokens.length - n; i++) out.add(tokens.slice(i, i + n).join(" "));
  return out;
}

/**
 * Guarantees: the share of answer n-grams that echo the system prompt, plus the
 * longest shared fragment as evidence.
 * Misfire: boilerplate such as "answer concisely in Japanese" is shared
 * legitimately and raises overlap. Tune the threshold per deployment.
 */
export function detectSystemPromptLeak(answer, systemPrompt, n, threshold) {
  try {
    n = n === undefined ? DEFAULTS.NGRAM_LEAK_N : Math.max(3, n);
    threshold = threshold === undefined ? DEFAULTS.LEAK_THRESHOLD : threshold;
    answer = (answer || "").slice(0, DEFAULTS.MAX_TEXT_CHARS);
    systemPrompt = (systemPrompt || "").slice(0, DEFAULTS.MAX_TEXT_CHARS);
    if (!answer || !systemPrompt)
      return { ok: true, leak: false, overlap: 0, n, threshold, matched_ngrams: 0, total_ngrams: 0, longest_fragment: "", fragment_len: 0, reason: "empty_input" };
    const aNg = ngrams(leakTokens(answer), n), sNg = ngrams(leakTokens(systemPrompt), n);
    let inter = 0;
    const sample = [];
    for (const g of aNg) if (sNg.has(g)) { inter++; if (sample.length < 5) sample.push(g); }
    const overlap = aNg.size ? inter / aNg.size : 0;
    const frag = longestCommonFragment(normText(answer), normText(systemPrompt));
    return { ok: true, leak: overlap >= threshold && inter > 0, overlap: Math.round(overlap * 10000) / 10000,
             n, threshold, matched_ngrams: inter, total_ngrams: aNg.size,
             longest_fragment: frag.slice(0, 400), fragment_len: frag.length, sample_ngrams: sample };
  } catch (e) { return { ok: true, leak: false, overlap: 0, n, threshold, matched_ngrams: 0, total_ngrams: 0, longest_fragment: "", fragment_len: 0 }; }
}

const INSTRUCTION_PATTERNS = [
  /ignore\s+(?:all\s+)?(?:the\s+)?(?:previous|prior|above|earlier)\s+instructions?/gi,
  /disregard\s+(?:all\s+)?(?:the\s+)?(?:previous|prior|above)/gi,
  /(?:reveal|print|repeat|output|show|dump)\s+(?:me\s+)?(?:your|the)\s+(?:system\s+)?(?:prompt|instructions?|rules?)/gi,
  /you\s+are\s+now\b/gi, /\bact\s+as\b/gi, /\bdeveloper\s+mode\b/gi, /\bjailbreak\b/gi,
  /(?:これまでの|以前の|上記の)(?:指示|命令|ルール)(?:は|を)?(?:すべて)?(?:無視|忘れ)/g,
  /システム\s*プロンプト(?:を)?(?:表示|出力|教え|見せ)/g,
  /(?:あなたは|君は)今から/g,
];

/** Guarantees: reports whether imperative strings from the user input reappear in the answer. */
export function detectInjectionEcho(answer, userInput) {
  try {
    answer = (answer || "").slice(0, DEFAULTS.MAX_TEXT_CHARS);
    userInput = (userInput || "").slice(0, DEFAULTS.MAX_TEXT_CHARS);
    if (!answer || !userInput) return { ok: true, echo: false, matches: [], reason: "empty_input" };
    const an = normText(answer);
    const found = [];
    for (const pat of INSTRUCTION_PATTERNS) {
      pat.lastIndex = 0; let m;
      while ((m = pat.exec(userInput)) !== null) {
        if (!m[0].length) { pat.lastIndex++; continue; }
        const phrase = m[0].trim(), pn = normText(phrase);
        if (!pn) continue;
        const exact = an.includes(pn);
        const ratio = exact ? 1 : bestFuzzy(pn, an)[0];
        found.push({ instruction: phrase.slice(0, 160), in_answer: exact || ratio >= 0.88, similarity: Math.round(ratio * 1000) / 1000, exact });
      }
    }
    const echoed = found.filter((f) => f.in_answer);
    return { ok: true, echo: echoed.length > 0, matches: found.slice(0, 20), echoed_count: echoed.length, instruction_like_in_input: found.length };
  } catch (e) { return { ok: true, echo: false, matches: [] }; }
}

/* ================================================================ (F) OUTPUT SAFETY */

const CC_RE = /(?<![0-9])(?:[0-9]{4}[ \-]?){3}[0-9]{1,7}(?![0-9])/g;
const JWT_RE = /\beyJ[A-Za-z0-9_\-]{6,}\.[A-Za-z0-9_\-]{6,}\.[A-Za-z0-9_\-]{0,600}/g;
const EMAIL_RE = /\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,24}\b/g;
const PHONE_RE = /(?<![0-9])(?:\+\d{1,3}[ \-]?)?(?:\(0?\d{1,4}\)|0\d{1,4})[ \-]?\d{1,4}[ \-]?\d{3,4}(?![0-9])/g;
const PRIVKEY_RE = /-----BEGIN\s+(?:RSA|DSA|EC|OPENSSH|PGP|ENCRYPTED)?\s*PRIVATE KEY(?:\s+BLOCK)?-----/g;
const HIGH_ENTROPY_RE = new RegExp("[A-Za-z0-9+/=_\\-]{" + DEFAULTS.ENTROPY_MIN_LEN + ",200}", "g");
const ENTROPY_ALLOW_RE = /^(?:[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64})$/i;

const CRED_PREFIXES = [
  ["openai_key", /\bsk-(?:proj-|svcacct-)?[A-Za-z0-9_\-]{16,}/g],
  ["anthropic_key", /\bsk-ant-[A-Za-z0-9_\-]{16,}/g],
  ["github_token", /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}|\bgithub_pat_[A-Za-z0-9_]{20,}/g],
  ["aws_access_key", /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|ANPA|ANVA)[A-Z0-9]{12,}/g],
  ["slack_token", /\bxox[abprs]-[A-Za-z0-9\-]{8,}/g],
  ["google_api_key", /\bAIza[A-Za-z0-9_\-]{30,}/g],
  ["google_oauth", /\bya29\.[A-Za-z0-9_\-]{20,}/g],
  ["gitlab_token", /\bglpat-[A-Za-z0-9_\-]{16,}/g],
  ["hf_token", /\bhf_[A-Za-z0-9]{20,}/g],
  ["npm_token", /\bnpm_[A-Za-z0-9]{30,}/g],
  ["sendgrid_key", /\bSG\.[A-Za-z0-9_\-]{16,}\.[A-Za-z0-9_\-]{16,}/g],
  ["stripe_key", /\b(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{16,}/g],
  ["digitalocean_token", /\bdop_v1_[a-f0-9]{32,}/g],
  ["twilio_sid", /\bAC[a-f0-9]{32}\b/g],
];
const SAFETY_SEVERITY = {
  credit_card: "critical", private_key: "critical", jwt: "critical", openai_key: "critical",
  anthropic_key: "critical", github_token: "critical", aws_access_key: "critical", slack_token: "critical",
  google_api_key: "critical", google_oauth: "critical", gitlab_token: "critical", hf_token: "critical",
  npm_token: "critical", sendgrid_key: "critical", stripe_key: "critical", digitalocean_token: "critical",
  twilio_sid: "high", high_entropy_string: "medium", email: "low", phone: "low",
};

/** Guarantees: Shannon entropy in bits/char, 0 for empty input. */
export function shannonEntropy(s) {
  if (!s) return 0;
  const n = s.length, counts = new Map();
  for (const ch of s) counts.set(ch, (counts.get(ch) || 0) + 1);
  let acc = 0;
  for (const c of counts.values()) { const p = c / n; acc -= p * Math.log2(p); }
  return acc;
}

/** Guarantees: true only if the digit string passes the Luhn mod-10 checksum. */
export function luhnOk(digits) {
  const ds = String(digits).replace(/\D/g, "").split("").map(Number);
  if (ds.length < 13 || ds.length > 19) return false;
  let total = 0;
  const parity = ds.length % 2;
  for (let i = 0; i < ds.length; i++) {
    let d = ds[i];
    if (i % 2 === parity) { d *= 2; if (d > 9) d -= 9; }
    total += d;
  }
  return total % 10 === 0;
}

function mask(s) {
  s = String(s);
  if (s.length <= 6) return "*".repeat(s.length);
  return s.slice(0, 3) + "*".repeat(Math.max(3, s.length - 5)) + s.slice(-2);
}

/**
 * Guarantees: a list of findings (type/span/confidence/severity); never throws.
 * Credit-card candidates are reported ONLY after passing Luhn, because a bare
 * 16-digit run is far more often an order number than a card.
 */
export function scanOutput(answer) {
  const findings = [];
  try {
    answer = (answer || "").slice(0, DEFAULTS.MAX_TEXT_CHARS);
    if (!answer) return findings;
    let m;
    PRIVKEY_RE.lastIndex = 0;
    while ((m = PRIVKEY_RE.exec(answer)) !== null)
      findings.push({ type: "private_key", value: m[0].slice(0, 30), span: [m.index, m.index + m[0].length], confidence: 0.99, severity: "critical" });
    for (const [name, pat] of CRED_PREFIXES) {
      pat.lastIndex = 0;
      while ((m = pat.exec(answer)) !== null)
        findings.push({ type: name, value: mask(m[0]), span: [m.index, m.index + m[0].length], confidence: 0.95, severity: SAFETY_SEVERITY[name] || "high" });
    }
    JWT_RE.lastIndex = 0;
    while ((m = JWT_RE.exec(answer)) !== null) {
      const parts = m[0].split(".");
      findings.push({ type: "jwt", value: mask(m[0]), span: [m.index, m.index + m[0].length], confidence: parts.length === 3 && parts[0] && parts[1] ? 0.95 : 0.6, severity: "critical" });
    }
    CC_RE.lastIndex = 0;
    while ((m = CC_RE.exec(answer)) !== null) {
      const digits = m[0].replace(/\D/g, "");
      if (digits.length < 13 || digits.length > 19) continue;
      if (!luhnOk(digits)) continue;   // the single biggest false-positive source
      findings.push({ type: "credit_card", value: mask(m[0]), span: [m.index, m.index + m[0].length], confidence: 0.9, severity: "critical" });
    }
    const claimed = () => findings.map((f) => f.span);
    HIGH_ENTROPY_RE.lastIndex = 0;
    let taken = claimed();
    while ((m = HIGH_ENTROPY_RE.exec(answer)) !== null) {
      const sp = [m.index, m.index + m[0].length];
      if (taken.some((c) => overlaps(sp, c))) continue;
      const tok = m[0];
      if (ENTROPY_ALLOW_RE.test(tok)) continue;
      if (!(/\d/.test(tok) && /[A-Za-z]/.test(tok))) continue;
      const ent = shannonEntropy(tok);
      if (ent < DEFAULTS.ENTROPY_THRESHOLD) continue;
      const conf = Math.min(0.85, 0.35 + (ent - DEFAULTS.ENTROPY_THRESHOLD) * 0.4 + Math.min(0.2, (tok.length - DEFAULTS.ENTROPY_MIN_LEN) / 100));
      findings.push({ type: "high_entropy_string", value: mask(tok), span: sp, confidence: Math.round(conf * 1000) / 1000, severity: "medium", entropy: Math.round(ent * 1000) / 1000, length: tok.length });
    }
    taken = claimed();
    EMAIL_RE.lastIndex = 0;
    while ((m = EMAIL_RE.exec(answer)) !== null) {
      const sp = [m.index, m.index + m[0].length];
      if (taken.some((c) => overlaps(sp, c))) continue;
      findings.push({ type: "email", value: mask(m[0]), span: sp, confidence: 0.9, severity: "low" });
    }
    taken = claimed();
    PHONE_RE.lastIndex = 0;
    while ((m = PHONE_RE.exec(answer)) !== null) {
      const sp = [m.index, m.index + m[0].length];
      if (taken.some((c) => overlaps(sp, c))) continue;
      const digits = m[0].replace(/\D/g, "");
      if (digits.length < 9 || digits.length > 15) continue;
      findings.push({ type: "phone", value: mask(m[0]), span: sp, confidence: 0.55, severity: "low" });
    }
    findings.sort((a, b) => a.span[0] - b.span[0]);
    return findings;
  } catch (e) { return [{ type: "scanner_error", value: "", span: [0, 0], confidence: 0, severity: "low" }]; }
}

/** Guarantees: the answer with every finding span replaced by a [REDACTED:TYPE] marker. */
export function redact(answer, findings) {
  try {
    answer = answer || "";
    if (!answer) return "";
    findings = findings || scanOutput(answer);
    const spans = findings.filter((f) => f.type !== "scanner_error" && f.span[1] > f.span[0])
      .map((f) => [f.span[0], f.span[1], f.type]).sort((a, b) => a[0] - b[0]);
    let out = "", cursor = 0;
    for (const [s, e, t] of spans) {
      if (s < cursor) continue;
      out += answer.slice(cursor, s) + "[REDACTED:" + String(t).toUpperCase() + "]";
      cursor = e;
    }
    return out + answer.slice(cursor);
  } catch (e) { return answer || ""; }
}

/** Guarantees: findings plus severity rollups and a redacted body. */
export function safetyReport(answer) {
  const findings = scanOutput(answer);
  const bySeverity = {}, byType = {};
  for (const f of findings) {
    bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1;
    byType[f.type] = (byType[f.type] || 0) + 1;
  }
  return { ok: true, findings, counts_by_severity: bySeverity, counts_by_type: byType,
           critical: bySeverity.critical || 0, redacted: redact(answer, findings) };
}

/* ================================================================== (G) SCHEMA GATE */

/** Guarantees: the most likely JSON payload substring, or null. */
export function extractJson(text) {
  try {
    if (!text) return null;
    text = String(text);
    const fence = /```[a-zA-Z0-9_+\-]*\s*\n?([\s\S]*?)```/.exec(text);
    if (fence && fence[1].trim()) {
      const inner = fence[1].trim();
      if (inner[0] === "[" || inner[0] === "{") return inner;
      text = inner;
    }
    let best = null;
    for (const [op, cl] of [["{", "}"], ["[", "]"]]) {
      let start = text.indexOf(op);
      while (start !== -1) {
        let depth = 0, inStr = false, esc = false, quote = "";
        for (let i = start; i < text.length; i++) {
          const ch = text[i];
          if (inStr) {
            if (esc) esc = false;
            else if (ch === "\\") esc = true;
            else if (ch === quote) inStr = false;
            continue;
          }
          if (ch === '"' || ch === "'") { inStr = true; quote = ch; continue; }
          if (ch === op) depth++;
          else if (ch === cl) {
            depth--;
            if (depth === 0) { const cand = text.slice(start, i + 1); if (!best || cand.length > best.length) best = cand; break; }
          }
        }
        start = text.indexOf(op, start + 1);
      }
    }
    return best;
  } catch (e) { return null; }
}

function escapeRawControl(s) {
  let out = "", inStr = false, esc = false, quote = "";
  for (const ch of s) {
    if (inStr) {
      if (esc) { esc = false; out += ch; continue; }
      if (ch === "\\") { esc = true; out += ch; continue; }
      if (ch === quote) { inStr = false; out += ch; continue; }
      if (ch === "\n") { out += "\\n"; continue; }
      if (ch === "\r") { out += "\\r"; continue; }
      if (ch === "\t") { out += "\\t"; continue; }
      out += ch; continue;
    }
    if (ch === '"' || ch === "'") { inStr = true; quote = ch; }
    out += ch;
  }
  return out;
}

function singleToDouble(s) {
  let out = "", i = 0, inDq = false, esc = false;
  while (i < s.length) {
    const ch = s[i];
    if (inDq) {
      out += ch;
      if (esc) esc = false; else if (ch === "\\") esc = true; else if (ch === '"') inDq = false;
      i++; continue;
    }
    if (ch === '"') { inDq = true; out += ch; i++; continue; }
    if (ch === "'") {
      let j = i + 1, buf = "", e = false;
      while (j < s.length) {
        const c2 = s[j];
        if (e) { buf += c2; e = false; }
        else if (c2 === "\\") { e = true; buf += c2; }
        else if (c2 === "'") break;
        else buf += c2;
        j++;
      }
      out += '"' + buf.replace(/"/g, '\\"') + '"';
      i = j + 1; continue;
    }
    out += ch; i++;
  }
  return out;
}

/**
 * Guarantees: [parsedOrNull, repairsApplied]; never throws.
 * PURPOSE - so a formatting slip never costs a retry. Re-prompting because the
 * model emitted a trailing comma or a fullwidth colon is pure waste: the content
 * was already correct. Fix syntax locally, reserve retries for wrong claims.
 */
export function repairJson(text) {
  const repairs = [];
  try {
    if (text === null || text === undefined) return [null, ["empty_input"]];
    if (typeof text === "object") return [text, []];
    let s = String(text).trim();
    if (!s) return [null, ["empty_input"]];
    try { return [JSON.parse(s), []]; } catch (e) {}
    const cand = extractJson(s);
    if (cand && cand !== s) { s = cand; repairs.push("extracted_json_block"); }
    try { return [JSON.parse(s), repairs]; } catch (e) {}
    const fw = { "：": ":", "，": ",", "、": ",", "｛": "{", "｝": "}", "［": "[", "］": "]", "“": '"', "”": '"', "‘": "'", "’": "'", "＂": '"', "　": " " };
    let before = s;
    for (const k in fw) s = s.split(k).join(fw[k]);
    if (s !== before) repairs.push("normalised_fullwidth_punctuation");
    before = s;
    s = s.replace(/\/\*[\s\S]*?\*\//g, "").replace(/(?<![:"'])\/\/[^\n]*$/gm, "");
    if (s !== before) repairs.push("removed_comments");
    before = s; s = singleToDouble(s);
    if (s !== before) repairs.push("single_to_double_quotes");
    before = s; s = escapeRawControl(s);
    if (s !== before) repairs.push("escaped_raw_control_chars");
    before = s;
    s = s.replace(/\bTrue\b/g, "true").replace(/\bFalse\b/g, "false").replace(/\b(?:None|NaN|Undefined|undefined)\b/g, "null");
    if (s !== before) repairs.push("python_literals_to_json");
    before = s; s = s.replace(/,(\s*[}\]])/g, "$1");
    if (s !== before) repairs.push("removed_trailing_commas");
    before = s; s = s.replace(/([{,]\s*)([A-Za-z_][A-Za-z0-9_\-]*)\s*:/g, '$1"$2":');
    if (s !== before) repairs.push("quoted_bare_keys");
    try { return [JSON.parse(s), repairs]; } catch (e) {}
    const opens = (s.split("{").length - 1) - (s.split("}").length - 1);
    const obr = (s.split("[").length - 1) - (s.split("]").length - 1);
    if (opens > 0 || obr > 0) {
      try { const obj = JSON.parse(s + "]".repeat(Math.max(0, obr)) + "}".repeat(Math.max(0, opens))); repairs.push("closed_unbalanced_brackets"); return [obj, repairs]; } catch (e) {}
    }
    repairs.push("unrepairable");
    return [null, repairs];
  } catch (e) { return [null, ["repair_error"]]; }
}

const COERCERS = {
  str: (v) => (typeof v === "string" ? v : typeof v === "object" && v !== null ? JSON.stringify(v) : String(v)),
  int: (v) => { const n = parseInt(v, 10); if (Number.isNaN(n)) throw new Error("nan"); return n; },
  float: (v) => { const n = Number(v); if (!Number.isFinite(n)) throw new Error("nan"); return n; },
  number: (v) => { const n = Number(v); if (!Number.isFinite(n)) throw new Error("nan"); return n; },
  bool: (v) => (typeof v === "boolean" ? v : ["1", "true", "yes", "on"].includes(String(v).trim().toLowerCase())),
  list: (v) => (Array.isArray(v) ? v : [v]),
  dict: (v) => (typeof v === "object" && v !== null && !Array.isArray(v) ? v : { value: v }),
  any: (v) => v,
};

/** Guarantees: {ok,value,missing,extra,coerced,errors}; never throws. A trailing "?" marks a key optional. */
export function coerceSchema(obj, schema) {
  const result = { ok: true, value: {}, missing: [], extra: [], coerced: [], errors: [] };
  try {
    if (!schema || typeof schema !== "object" || !Object.keys(schema).length) { result.ok = typeof obj === "object" && obj !== null; result.value = obj; return result; }
    if (Array.isArray(obj) && obj.length && typeof obj[0] === "object") { obj = obj[0]; result.coerced.push("took_first_element_of_list"); }
    if (typeof obj !== "object" || obj === null || Array.isArray(obj)) { result.ok = false; result.errors.push("payload_is_not_an_object"); result.value = obj; return result; }
    const wanted = {};
    for (const k in schema) { const optional = k.endsWith("?"); wanted[optional ? k.slice(0, -1) : k] = [schema[k], optional]; }
    for (const key in wanted) {
      const [t, optional] = wanted[key];
      if (!(key in obj)) { if (!optional) { result.missing.push(key); result.ok = false; } continue; }
      const raw = obj[key];
      if (typeof t === "object" && t !== null) {
        const sub = coerceSchema(raw, t);
        result.value[key] = sub.value;
        if (!sub.ok) { result.ok = false; result.errors.push({ [key]: { missing: sub.missing, errors: sub.errors } }); }
        continue;
      }
      const fn = COERCERS[String(t).trim().toLowerCase()];
      if (!fn) { result.value[key] = raw; continue; }
      try { const nv = fn(raw); if (nv !== raw) result.coerced.push(key); result.value[key] = nv; }
      catch (e) { result.ok = false; result.errors.push({ [key]: "cannot_coerce_to_" + t }); result.value[key] = raw; }
    }
    for (const k in obj) if (!(k in wanted)) result.extra.push(k);
    return result;
  } catch (e) { result.ok = false; result.errors.push("coerce_error"); return result; }
}

/** Guarantees: one call that extracts, repairs and optionally schema-coerces LLM JSON output. */
export function structureCheck(text, schemaJson) {
  const t0 = nowMs();
  const [obj, repairs] = repairJson(text);
  const out = { ok: true, valid_json: obj !== null, repairs, parsed: obj, schema_ok: null, schema: null,
                missing: [], extra: [], coerced: [], errors: [], value: obj };
  let schema = null;
  if (schemaJson) {
    const [sobj] = repairJson(schemaJson);
    if (sobj && typeof sobj === "object" && !Array.isArray(sobj)) { schema = sobj; out.schema = sobj; }
    else out.errors.push("schema_unparseable");
  }
  if (schema && obj !== null) {
    const c = coerceSchema(obj, schema);
    out.schema_ok = c.ok; out.missing = c.missing; out.extra = c.extra; out.coerced = c.coerced;
    out.errors = out.errors.concat(c.errors); out.value = c.value;
  }
  out.latency_ms = Math.round((nowMs() - t0) * 100) / 100;
  return out;
}

/* ================================================================ (H) RETRY ADVISOR */

/** Guarantees: dependency-free token estimate (ASCII ~4 chars/token, CJK ~1 char/token). */
export function estimateTokens(s) {
  if (!s) return 0;
  s = String(s);
  let ascii = 0, cjk = 0, other = 0;
  for (const ch of s) {
    const o = ch.codePointAt(0);
    if (o < 128) ascii++;
    else if ((o >= 0x3040 && o <= 0x30ff) || (o >= 0x4e00 && o <= 0x9fff) || (o >= 0x3400 && o <= 0x4dbf) || (o >= 0xac00 && o <= 0xd7af) || (o >= 0xff00 && o <= 0xffef)) cjk++;
    else other++;
  }
  return Math.ceil(ascii / 4 + cjk + other / 2);
}

const FAIL_STATUSES = ["unsupported", "contradicted"];

/**
 * Guarantees: a targeted correction instruction plus a blind-vs-targeted estimate.
 * A blind retry re-sends the whole system prompt + context + question and
 * regenerates everything. A targeted retry re-sends only the answer and the
 * specific failing claims. The difference is the money this saves.
 */
export function buildRetryInstruction(result, prices) {
  try {
    prices = prices || {};
    const pin = prices.in === undefined ? DEFAULTS.PRICE_IN_PER_1K : prices.in;
    const pout = prices.out === undefined ? DEFAULTS.PRICE_OUT_PER_1K : prices.out;
    if (typeof result === "string") { const [p] = repairJson(result); result = p && typeof p === "object" ? p : {}; }
    if (!result || typeof result !== "object") return { ok: true, needed: false, instruction: "", items: [], estimate: {}, reason: "no_result_supplied" };
    const ver = result.verify && typeof result.verify === "object" ? result.verify : result;
    const claims = ver.claims || [];
    const bad = claims.filter((c) => c && FAIL_STATUSES.includes(c.status));
    bad.sort((a, b) => (a.status === "contradicted" ? 0 : 1) - (b.status === "contradicted" ? 0 : 1) || (a.span ? a.span[0] : 0) - (b.span ? b.span[0] : 0));
    const items = bad.slice(0, 60).map((c) => ({ id: c.id, type: c.type, value: c.value, status: c.status,
      nearest_context_value: c.nearest_context_value, sentence_index: c.sentence_index }));

    const linesJa = [], linesEn = [];
    for (const it of items) {
      if (it.status === "contradicted" && it.nearest_context_value) {
        linesJa.push('- 「' + it.value + '」(' + it.type + '): 文脈の該当値は「' + it.nearest_context_value + '」です。');
        linesEn.push('- "' + it.value + '" (' + it.type + '): the context says "' + it.nearest_context_value + '".');
      } else {
        linesJa.push('- 「' + it.value + '」(' + it.type + '): 与えた文脈に裏付けがありません。');
        linesEn.push('- "' + it.value + '" (' + it.type + '): not supported by the provided context.');
      }
    }
    const instruction = items.length
      ? "The following statements are not supported by the context you were given. Correct them using the context, or explicitly say the context does not support them. Do not rewrite anything else.\n"
        + linesEn.join("\n") + "\n\nReturn only the corrected full text. Do not introduce new figures, dates or proper nouns."
      : "";
    const instructionJa = items.length
      ? "以下の記述には、与えた文脈での裏付けがありません。文脈に基づいて訂正するか、裏付けが無い旨を明記してください。該当箇所以外は書き換えないでください。\n"
        + linesJa.join("\n") + "\n\n訂正後の全文のみを出力してください。新しい数値・日付・固有名詞を追加しないでください。"
      : "";

    const sizes = (result.meta && result.meta.sizes) || {};
    const tokSystem = sizes.system_prompt_tokens || 0, tokContext = sizes.context_tokens || 0;
    const tokUser = sizes.user_input_tokens || 0, tokAnswer = sizes.answer_tokens || 0;
    const instrTokens = estimateTokens(instruction);
    const blindIn = tokSystem + tokContext + tokUser, blindOut = tokAnswer;
    const targetedIn = tokAnswer + instrTokens + Math.min(tokSystem, 200);
    const targetedOut = Math.max(1, Math.round(tokAnswer * Math.min(1, Math.max(0.15, items.length / Math.max(1, claims.length || 1)))));
    // Signed on purpose. For a small context the instruction can cost more than
    // the blind retry it replaces, and hiding that behind max(0, ...) would make
    // this tool lie about its own value.
    const savedIn = blindIn - targetedIn, savedOut = blindOut - targetedOut;
    const savedTokens = savedIn + savedOut;
    const savedCost = Math.round(((savedIn / 1000) * pin + (savedOut / 1000) * pout) * 1e6) / 1e6;
    const priced = pin > 0 || pout > 0;
    const cheaper = priced ? savedCost > 0 : savedTokens > 0;
    const basis = priced ? "cost" : "tokens";
    const recommendation = !items.length ? "no retry needed"
      : cheaper ? "targeted retry is cheaper by " + (priced ? savedCost.toFixed(6) : savedTokens + " tokens") + " (basis: " + basis + ")"
      : "targeted retry is NOT cheaper here (basis: " + basis + ") - the context is small relative to the instruction; retry blind, or batch several corrections into one call";

    return { ok: true, needed: items.length > 0, instruction, instruction_ja: instructionJa, items,
      failed_claims: bad.length,
      estimate: { blind_retry_tokens: blindIn + blindOut, targeted_retry_tokens: targetedIn + targetedOut,
        saved_tokens: savedTokens, saved_cost: savedCost, targeted_is_cheaper: cheaper, comparison_basis: basis,
        recommendation,
        breakdown: { blind_input_tokens: blindIn, blind_output_tokens: blindOut, targeted_input_tokens: targetedIn,
          targeted_output_tokens: targetedOut, saved_input_tokens: savedIn, saved_output_tokens: savedOut, instruction_tokens: instrTokens },
        assumptions: { price_in_per_1k: pin, price_out_per_1k: pout,
          tokenizer: "heuristic: ASCII ~4 chars/token, CJK ~1 char/token",
          note: "saved_tokens and saved_cost are signed and can disagree: output tokens are usually priced several times higher than input tokens." } } };
  } catch (e) { return { ok: false, error: String(e).slice(0, 300) }; }
}

/* ======================================================================== (I) GATE */

export const DEFAULT_POLICY = {
  pass_grounding: 0.95, pass_coverage: 0.5, annotate_grounding: 0.9, retry_grounding: 0.7,
  retry_on_contradicted: 1, block_on_critical_safety: true, block_on_leak: false, block_on_injection_echo: false,
  enable_numeric: true, enable_date: true, enable_quote: true, enable_entity: true, enable_url: true,
  enable_derivation: true, derive_max_terms: DEFAULTS.DERIVE_MAX_TERMS, derive_budget_ms: DEFAULTS.DERIVE_BUDGET_MS,
  derive_max_numbers: DEFAULTS.DERIVE_MAX_NUMBERS, fuzzy_budget_ms: DEFAULTS.FUZZY_BUDGET_MS,
  max_claims: DEFAULTS.MAX_CLAIMS, numeric_tolerance: 0, contradiction_rel: DEFAULTS.CONTRADICTION_REL,
  approx_ratio: DEFAULTS.APPROX_RATIO, quote_contradiction_floor: DEFAULTS.QUOTE_CONTRADICTION_FLOOR,
  leak_ngram: DEFAULTS.NGRAM_LEAK_N, leak_threshold: DEFAULTS.LEAK_THRESHOLD, max_chars: DEFAULTS.MAX_TEXT_CHARS,
};

let GATE_SEQ = 0;

/**
 * Guarantees: a four-valued verdict (pass/annotate/retry/block) and never throws.
 * Order is deliberate: output safety runs FIRST and a critical hit
 * short-circuits everything. Verifying the grounding of an answer that leaks an
 * API key is wasted work.
 */
export function gate(answer, context, systemPrompt, userInput, schemaJson, policyJson, tagsJson, prices) {
  const tStart = nowMs();
  GATE_SEQ += 1;
  const eventId = "e" + String(GATE_SEQ).padStart(6, "0");
  const warnings = [];
  try {
    const pol = Object.assign({}, DEFAULT_POLICY);
    if (policyJson) {
      const [obj, reps] = repairJson(policyJson);
      if (obj && typeof obj === "object") {
        for (const k in obj) { if (k in pol) pol[k] = obj[k]; else warnings.push("unknown_policy_key:" + k); }
        if (reps && reps.length) warnings.push("policy_json_repaired:" + reps.join(","));
      } else warnings.push("policy_json_unparseable_using_defaults");
    }
    const maxChars = pol.max_chars || DEFAULTS.MAX_TEXT_CHARS;
    const clip = (s, name) => { s = s === null || s === undefined ? "" : String(s); if (s.length > maxChars) { warnings.push(name + " truncated to max_chars=" + maxChars); return s.slice(0, maxChars); } return s; };
    answer = clip(answer, "answer"); context = clip(context, "context");
    systemPrompt = clip(systemPrompt, "system_prompt"); userInput = clip(userInput, "user_input");

    let tags = {};
    if (tagsJson) { const [t] = repairJson(tagsJson); if (t && typeof t === "object") tags = t; else warnings.push("tags_json_unparseable"); }

    const sizes = { answer_chars: answer.length, context_chars: context.length,
      system_prompt_chars: systemPrompt.length, user_input_chars: userInput.length,
      answer_tokens: estimateTokens(answer), context_tokens: estimateTokens(context),
      system_prompt_tokens: estimateTokens(systemPrompt), user_input_tokens: estimateTokens(userInput) };

    const meta = { sizes, tags, policy: pol, timestamp: new Date().toISOString(), app_version: APP_VERSION };

    if (!answer.trim())
      return { ok: true, event_id: eventId, verdict: "annotate", reasons: ["empty_answer"], warnings,
        safety: { ok: true, findings: [], critical: 0 }, leak: null, verify: null, schema: null, retry: null,
        meta: Object.assign(meta, { latency_ms: Math.round((nowMs() - tStart) * 100) / 100 }) };

    // 2. output safety FIRST, short-circuit on critical
    const safety = safetyReport(answer);
    const critical = safety.critical || 0;
    if (critical > 0 && pol.block_on_critical_safety)
      return { ok: true, event_id: eventId, verdict: "block",
        reasons: ["critical_output_safety_violation:" + critical],
        warnings: warnings.concat(["verification skipped: blocked before grounding checks"]),
        safety, leak: null, verify: null, schema: null, retry: null,
        meta: Object.assign(meta, { latency_ms: Math.round((nowMs() - tStart) * 100) / 100 }) };

    // 3. leak detection
    const leak = detectSystemPromptLeak(answer, systemPrompt, pol.leak_ngram, pol.leak_threshold);
    const echo = detectInjectionEcho(answer, userInput);

    // 4. verification
    const ver = verify(answer, context, pol);

    // 5. schema gate (only when a schema is supplied)
    const schemaRes = schemaJson && String(schemaJson).trim() ? structureCheck(answer, schemaJson) : null;

    // 6. retry advisor
    const retry = buildRetryInstruction({ verify: ver, meta: { sizes } }, prices);

    const bs = ver.counts.by_status || {};
    const contradicted = bs.contradicted || 0;
    const grounding = ver.grounding_score, coverage = ver.coverage;
    const claimsTotal = ver.counts.claims_total || 0;
    const reasons = [];
    let verdict = "pass";

    if (leak.leak) { reasons.push("system_prompt_leak:overlap=" + leak.overlap.toFixed(3)); verdict = pol.block_on_leak ? "block" : "annotate"; }
    if (echo.echo) { reasons.push("injection_echo:" + echo.echoed_count); if (pol.block_on_injection_echo) verdict = "block"; else if (verdict === "pass") verdict = "annotate"; }

    if (verdict !== "block") {
      if (contradicted >= (pol.retry_on_contradicted || 1)) { verdict = "retry"; reasons.push("contradicted_claims:" + contradicted); }
      else if (claimsTotal && grounding < pol.retry_grounding) { verdict = "retry"; reasons.push("grounding_score below retry threshold (" + grounding.toFixed(3) + " < " + pol.retry_grounding + ")"); }
      else if (claimsTotal && grounding < pol.annotate_grounding) { if (verdict === "pass") verdict = "annotate"; reasons.push("grounding_score below annotate threshold (" + grounding.toFixed(3) + " < " + pol.annotate_grounding + ")"); }
      else if (!claimsTotal) { if (verdict === "pass") verdict = "annotate"; reasons.push("no verifiable claims were found; nothing was checked"); }

      if (verdict === "pass") {
        if (grounding < pol.pass_grounding) { verdict = "annotate"; reasons.push("grounding_score below pass threshold"); }
        else if (coverage < pol.pass_coverage) { verdict = "annotate"; reasons.push("coverage " + coverage.toFixed(2) + " below pass threshold " + pol.pass_coverage + " - most of the answer was not checked"); }
      }
    }

    if (schemaRes) {
      if (schemaRes.valid_json === false) { reasons.push("output is not valid JSON even after repair"); if (verdict === "pass" || verdict === "annotate") verdict = "retry"; }
      else if (schemaRes.schema_ok === false) { reasons.push("JSON does not satisfy the schema: missing=" + JSON.stringify(schemaRes.missing)); if (verdict === "pass" || verdict === "annotate") verdict = "retry"; }
    }
    if (critical > 0) reasons.push("critical_output_safety_violation:" + critical + " (policy did not block)");
    const otherSafety = (safety.findings || []).length - critical;
    if (otherSafety > 0 && verdict === "pass") { verdict = "annotate"; reasons.push("non-critical safety findings:" + otherSafety); }

    meta.latency_ms = Math.round((nowMs() - tStart) * 100) / 100;
    meta.verify_latency_ms = ver.latency_ms;
    return { ok: true, event_id: eventId, verdict, reasons: reasons.length ? reasons : ["all checks passed"],
      warnings, grounding_score: grounding, coverage, safety,
      leak: { system_prompt: leak, injection_echo: echo }, verify: ver, schema: schemaRes, retry, meta };
  } catch (e) {
    return { ok: false, event_id: eventId, error: { type: "GateError", message: String(e && e.message ? e.message : e).slice(0, 500) } };
  }
}

/* ============================================================== HIGHLIGHT HELPERS */

export const STATUS_COLORS = { supported: "#16a34a", derived: "#2563eb", approximate: "#ca8a04", unsupported: "#ea580c", contradicted: "#dc2626" };
const STATUS_PRIORITY = { contradicted: 0, unsupported: 1, approximate: 2, derived: 3, supported: 4 };

/** Guarantees: [text, statusOrNull] segments covering the answer exactly once. */
export function buildHighlight(answer, claims) {
  try {
    answer = answer || "";
    if (!answer) return [["", null]];
    const spans = (claims || []).filter((c) => c && c.span && !c.suppressed && c.span[1] > c.span[0] && c.span[0] >= 0 && c.span[1] <= answer.length)
      .map((c) => [c.span[0], c.span[1], c.status || "unsupported"]);
    // The most severe (and at equal severity, the longest) claim wins an overlap.
    spans.sort((a, b) => (STATUS_PRIORITY[a[2]] ?? 9) - (STATUS_PRIORITY[b[2]] ?? 9) || (b[1] - b[0]) - (a[1] - a[0]));
    const placed = [];
    for (const sp of spans) if (!placed.some((p) => overlaps(sp, p))) placed.push(sp);
    placed.sort((a, b) => a[0] - b[0]);
    const out = [];
    let cursor = 0;
    for (const [s, e, st] of placed) {
      if (s > cursor) out.push([answer.slice(cursor, s), null]);
      out.push([answer.slice(s, e), st]);
      cursor = e;
    }
    if (cursor < answer.length) out.push([answer.slice(cursor), null]);
    return out.length ? out : [[answer, null]];
  } catch (e) { return [[answer || "", null]]; }
}