#!/usr/bin/env python3 """Convert judged TriviaQA generations into the TokenEvalRow schema. TriviaQA ships no hallucination labels, so unlike RAGTruth it cannot be converted from the source release -- it has to be generated and judged first. This is the last step of that chain: scripts/sample_triviaqa.py parquet -> raw samples hallucination_labelling/02_generate_answers.py -> generations + token spine hallucination_labelling/03_auto_label_tokens.py -> per-token G/H/U + t_star scripts/convert_triviaqa.py (this) -> TokenEvalRow JSONL The point of this file is that the labelling pipeline's row is NOT the trainer's row, and the difference is where label alignment can silently break. The judge emits `token_labels` against the generator's own token spine (`gen_tokens` / `gen_token_ids`), so the conversion must carry that spine through verbatim rather than re-tokenising the answer text. Re-tokenising is the failure mode that produced misaligned labels earlier in this project: a different tokenizer call can split the same string differently, and every label after the first divergence then describes the wrong token. So this script never calls a tokenizer. It validates that the spine it was handed is internally consistent and refuses the row otherwise. Label encoding matches v5.2: 0 grounded, 1 hallucinated, 2 uncertain. Usage: python scripts/convert_triviaqa.py \ --input hallucination_labelling/output/labeled_triviaqa_llama32_1b.jsonl \ --output ~/Desktop/datasets/hallucinations/triviaqa_llama32_1b.jsonl \ --generator-model meta-llama/Llama-3.2-1B-Instruct """ from __future__ import annotations import argparse import collections import hashlib import json from pathlib import Path GROUNDED, HALLUCINATED, UNCERTAIN = 0, 1, 2 LABEL_MAP = { "G": GROUNDED, "g": GROUNDED, 0: GROUNDED, "0": GROUNDED, "H": HALLUCINATED, "h": HALLUCINATED, 1: HALLUCINATED, "1": HALLUCINATED, "U": UNCERTAIN, "u": UNCERTAIN, 2: UNCERTAIN, "2": UNCERTAIN, } def parse_args(argv=None): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--input", type=Path, required=True, help="Output of 03_auto_label_tokens.py") p.add_argument("--output", type=Path, required=True) p.add_argument( "--generator-model", required=True, help=( "The model whose weights will SCORE these rows. The replay guard " "compares this against the model actually loaded, so it must be the " "repo id used at scoring time, not a friendly name." ), ) p.add_argument( "--reject-cap-hit", type=int, default=None, metavar="N", help=( "reject rows whose generation reached N tokens (the generation " "cap): a capped answer is amputated mid-stream, typically inside a " "repetition loop, and its judge labels are not meaningful. " "Pre-registered for v2 (docs/triviaqa_v2_spec.md); v1 was built " "without it and stays that way." ), ) p.add_argument( "--max-uncertain-fraction", type=float, default=0.5, help=( "Drop rows where the judge marked more than this fraction of tokens " "uncertain. Such rows contribute almost nothing to a G/H metric and " "their few labelled tokens are the ones the judges disagreed least " "about, which biases the sample." ), ) return p.parse_args(argv) def spans_from_labels(labels, offsets): """Contiguous hallucinated token runs -> character spans. The trainer keeps `hallucinated_spans` for reporting and for span-level baselines; it does not derive token labels from them (those come straight from the judge). Offsets can be missing for special tokens, so runs are closed on any token without a usable offset rather than bridged across it. """ spans, start_char, end_char = [], None, None for index, label in enumerate(labels): offset = offsets[index] if index < len(offsets) else None usable = ( isinstance(offset, (list, tuple)) and len(offset) == 2 and offset[0] is not None and int(offset[1]) > int(offset[0]) ) if label == HALLUCINATED and usable: if start_char is None: start_char = int(offset[0]) end_char = int(offset[1]) elif start_char is not None: spans.append({"start": start_char, "end": end_char}) start_char, end_char = None, None if start_char is not None: spans.append({"start": start_char, "end": end_char}) return spans def build_row(record, generator_model): """Return (row, reason_rejected). Exactly one of the two is None.""" answer = record.get("generated_answer") token_ids = record.get("gen_token_ids") tokens = record.get("gen_tokens") raw_labels = record.get("token_labels") if record.get("error"): return None, "generation_error" if not answer or not token_ids or not tokens: return None, "empty_generation" if not raw_labels: return None, "unjudged" # The three sequences describe the same tokens; a length mismatch means the # spine and the labels disagree, and there is no safe way to repair that. if not (len(token_ids) == len(tokens) == len(raw_labels)): return None, "spine_length_mismatch" labels = [] for value in raw_labels: if value not in LABEL_MAP: return None, "unknown_label_value" labels.append(LABEL_MAP[value]) counts = collections.Counter(labels) offsets = record.get("gen_token_offsets") or [] spans = spans_from_labels(labels, offsets) t_star = record.get("t_star_index") if t_star is None: t_star = next((i for i, v in enumerate(labels) if v == HALLUCINATED), None) row = { "id": record["id"], # Bare question. The chat template is applied at replay time via # --prompt-template, exactly as for v5.2, so the stored prompt must NOT # already be wrapped or the template would be applied twice. "prompt": record["question"], "answer": answer, "labels": labels, "hallucinated_spans": spans, "metadata": { "source": record.get("source", "triviaqa"), "dataset": "TriviaQA (rc.nocontext)", "generator_model": generator_model, "generator_backend": record.get("generator_backend"), "generated_token_ids": list(token_ids), "generated_tokens": list(tokens), "generated_logprobs": record.get("gen_logprobs"), "judge_model": record.get("judge_model"), "judge_rationale": record.get("judge_rationale"), "auto_label_method": record.get("auto_label_method"), "reference_answer": record.get("reference_answer"), "t_star_index": t_star, "original_label_counts": { "grounded": counts.get(GROUNDED, 0), "hallucinated": counts.get(HALLUCINATED, 0), "uncertain": counts.get(UNCERTAIN, 0), }, "extra": { **(record.get("extra") or {}), "token_label_reasons": record.get("token_label_reasons"), # Recorded for provenance only. make_eval_split reads an # explicit group from extra.v5_1.split_group and otherwise # groups on normalised prompt -- which for TriviaQA is the # question, and every question is unique, so grouping is # already identity and needs no override here. "split_group": record["id"], }, }, } # The uncertain-fraction filter is applied by the caller, which owns the # threshold; everything rejectable for structural reasons is caught above. return row, None def main(argv=None): args = parse_args(argv) rejected = collections.Counter() rows = [] total = 0 with args.input.open(encoding="utf-8") as handle: for line in handle: line = line.strip() if not line: continue total += 1 record = json.loads(line) row, reason = build_row(record, args.generator_model) if row is None: rejected[reason or "unknown"] += 1 continue if args.reject_cap_hit and len(row["metadata"]["generated_token_ids"]) >= args.reject_cap_hit: rejected["cap_hit"] += 1 continue labels = row["labels"] uncertain = sum(1 for v in labels if v == UNCERTAIN) if labels and uncertain / len(labels) > args.max_uncertain_fraction: rejected["mostly_uncertain"] += 1 continue rows.append(row) if not rows: raise SystemExit("no usable rows -- check that the judge actually ran") args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("w", encoding="utf-8") as handle: for row in rows: handle.write(json.dumps(row, ensure_ascii=False) + "\n") n_tokens = sum(len(r["labels"]) for r in rows) n_h = sum(sum(1 for v in r["labels"] if v == HALLUCINATED) for r in rows) n_u = sum(sum(1 for v in r["labels"] if v == UNCERTAIN) for r in rows) n_g = n_tokens - n_h - n_u with_h = sum(1 for r in rows if any(v == HALLUCINATED for v in r["labels"])) print(f"wrote {len(rows):,} of {total:,} rows -> {args.output}") print(f" tokens {n_tokens:,} | G {n_g:,} | H {n_h:,} | U {n_u:,}") # This is the number that decides whether the corpus is usable at all: the # AP floor is the G/H prevalence, and a token-level detector needs enough # positives for the metric to have resolution. if n_g + n_h: print(f" H prevalence over G/H tokens: {n_h / (n_g + n_h):.4%}") print(f" rows containing >=1 hallucinated token: {with_h:,} ({with_h/len(rows):.1%})") if rejected: print(" rejected:", dict(rejected)) print(f" sha256: {hashlib.sha256(args.output.read_bytes()).hexdigest()}") return 0 if __name__ == "__main__": raise SystemExit(main())