Md Saiful Islam Sajol
Claude Opus 4.8
Sync to GitHub main fd62f0f: THRIVE v6.1 weights, v6/TriviaQA pipeline, labelling tooling
ac442f8 | #!/usr/bin/env python3 | |
| """Convert judged TriviaQA generations into the TokenEvalRow schema. | |
| TriviaQA ships no hallucination labels, so unlike RAGTruth it cannot be | |
| converted from the source release -- it has to be generated and judged first. | |
| This is the last step of that chain: | |
| scripts/sample_triviaqa.py parquet -> raw samples | |
| hallucination_labelling/02_generate_answers.py -> generations + token spine | |
| hallucination_labelling/03_auto_label_tokens.py -> per-token G/H/U + t_star | |
| scripts/convert_triviaqa.py (this) -> TokenEvalRow JSONL | |
| The point of this file is that the labelling pipeline's row is NOT the trainer's | |
| row, and the difference is where label alignment can silently break. The judge | |
| emits `token_labels` against the generator's own token spine (`gen_tokens` / | |
| `gen_token_ids`), so the conversion must carry that spine through verbatim | |
| rather than re-tokenising the answer text. Re-tokenising is the failure mode | |
| that produced misaligned labels earlier in this project: a different tokenizer | |
| call can split the same string differently, and every label after the first | |
| divergence then describes the wrong token. | |
| So this script never calls a tokenizer. It validates that the spine it was | |
| handed is internally consistent and refuses the row otherwise. | |
| Label encoding matches v5.2: 0 grounded, 1 hallucinated, 2 uncertain. | |
| Usage: | |
| python scripts/convert_triviaqa.py \ | |
| --input hallucination_labelling/output/labeled_triviaqa_llama32_1b.jsonl \ | |
| --output ~/Desktop/datasets/hallucinations/triviaqa_llama32_1b.jsonl \ | |
| --generator-model meta-llama/Llama-3.2-1B-Instruct | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| GROUNDED, HALLUCINATED, UNCERTAIN = 0, 1, 2 | |
| LABEL_MAP = { | |
| "G": GROUNDED, "g": GROUNDED, 0: GROUNDED, "0": GROUNDED, | |
| "H": HALLUCINATED, "h": HALLUCINATED, 1: HALLUCINATED, "1": HALLUCINATED, | |
| "U": UNCERTAIN, "u": UNCERTAIN, 2: UNCERTAIN, "2": UNCERTAIN, | |
| } | |
| def parse_args(argv=None): | |
| p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| p.add_argument("--input", type=Path, required=True, help="Output of 03_auto_label_tokens.py") | |
| p.add_argument("--output", type=Path, required=True) | |
| p.add_argument( | |
| "--generator-model", | |
| required=True, | |
| help=( | |
| "The model whose weights will SCORE these rows. The replay guard " | |
| "compares this against the model actually loaded, so it must be the " | |
| "repo id used at scoring time, not a friendly name." | |
| ), | |
| ) | |
| p.add_argument( | |
| "--reject-cap-hit", | |
| type=int, | |
| default=None, | |
| metavar="N", | |
| help=( | |
| "reject rows whose generation reached N tokens (the generation " | |
| "cap): a capped answer is amputated mid-stream, typically inside a " | |
| "repetition loop, and its judge labels are not meaningful. " | |
| "Pre-registered for v2 (docs/triviaqa_v2_spec.md); v1 was built " | |
| "without it and stays that way." | |
| ), | |
| ) | |
| p.add_argument( | |
| "--max-uncertain-fraction", | |
| type=float, | |
| default=0.5, | |
| help=( | |
| "Drop rows where the judge marked more than this fraction of tokens " | |
| "uncertain. Such rows contribute almost nothing to a G/H metric and " | |
| "their few labelled tokens are the ones the judges disagreed least " | |
| "about, which biases the sample." | |
| ), | |
| ) | |
| return p.parse_args(argv) | |
| def spans_from_labels(labels, offsets): | |
| """Contiguous hallucinated token runs -> character spans. | |
| The trainer keeps `hallucinated_spans` for reporting and for span-level | |
| baselines; it does not derive token labels from them (those come straight | |
| from the judge). Offsets can be missing for special tokens, so runs are | |
| closed on any token without a usable offset rather than bridged across it. | |
| """ | |
| spans, start_char, end_char = [], None, None | |
| for index, label in enumerate(labels): | |
| offset = offsets[index] if index < len(offsets) else None | |
| usable = ( | |
| isinstance(offset, (list, tuple)) | |
| and len(offset) == 2 | |
| and offset[0] is not None | |
| and int(offset[1]) > int(offset[0]) | |
| ) | |
| if label == HALLUCINATED and usable: | |
| if start_char is None: | |
| start_char = int(offset[0]) | |
| end_char = int(offset[1]) | |
| elif start_char is not None: | |
| spans.append({"start": start_char, "end": end_char}) | |
| start_char, end_char = None, None | |
| if start_char is not None: | |
| spans.append({"start": start_char, "end": end_char}) | |
| return spans | |
| def build_row(record, generator_model): | |
| """Return (row, reason_rejected). Exactly one of the two is None.""" | |
| answer = record.get("generated_answer") | |
| token_ids = record.get("gen_token_ids") | |
| tokens = record.get("gen_tokens") | |
| raw_labels = record.get("token_labels") | |
| if record.get("error"): | |
| return None, "generation_error" | |
| if not answer or not token_ids or not tokens: | |
| return None, "empty_generation" | |
| if not raw_labels: | |
| return None, "unjudged" | |
| # The three sequences describe the same tokens; a length mismatch means the | |
| # spine and the labels disagree, and there is no safe way to repair that. | |
| if not (len(token_ids) == len(tokens) == len(raw_labels)): | |
| return None, "spine_length_mismatch" | |
| labels = [] | |
| for value in raw_labels: | |
| if value not in LABEL_MAP: | |
| return None, "unknown_label_value" | |
| labels.append(LABEL_MAP[value]) | |
| counts = collections.Counter(labels) | |
| offsets = record.get("gen_token_offsets") or [] | |
| spans = spans_from_labels(labels, offsets) | |
| t_star = record.get("t_star_index") | |
| if t_star is None: | |
| t_star = next((i for i, v in enumerate(labels) if v == HALLUCINATED), None) | |
| row = { | |
| "id": record["id"], | |
| # Bare question. The chat template is applied at replay time via | |
| # --prompt-template, exactly as for v5.2, so the stored prompt must NOT | |
| # already be wrapped or the template would be applied twice. | |
| "prompt": record["question"], | |
| "answer": answer, | |
| "labels": labels, | |
| "hallucinated_spans": spans, | |
| "metadata": { | |
| "source": record.get("source", "triviaqa"), | |
| "dataset": "TriviaQA (rc.nocontext)", | |
| "generator_model": generator_model, | |
| "generator_backend": record.get("generator_backend"), | |
| "generated_token_ids": list(token_ids), | |
| "generated_tokens": list(tokens), | |
| "generated_logprobs": record.get("gen_logprobs"), | |
| "judge_model": record.get("judge_model"), | |
| "judge_rationale": record.get("judge_rationale"), | |
| "auto_label_method": record.get("auto_label_method"), | |
| "reference_answer": record.get("reference_answer"), | |
| "t_star_index": t_star, | |
| "original_label_counts": { | |
| "grounded": counts.get(GROUNDED, 0), | |
| "hallucinated": counts.get(HALLUCINATED, 0), | |
| "uncertain": counts.get(UNCERTAIN, 0), | |
| }, | |
| "extra": { | |
| **(record.get("extra") or {}), | |
| "token_label_reasons": record.get("token_label_reasons"), | |
| # Recorded for provenance only. make_eval_split reads an | |
| # explicit group from extra.v5_1.split_group and otherwise | |
| # groups on normalised prompt -- which for TriviaQA is the | |
| # question, and every question is unique, so grouping is | |
| # already identity and needs no override here. | |
| "split_group": record["id"], | |
| }, | |
| }, | |
| } | |
| # The uncertain-fraction filter is applied by the caller, which owns the | |
| # threshold; everything rejectable for structural reasons is caught above. | |
| return row, None | |
| def main(argv=None): | |
| args = parse_args(argv) | |
| rejected = collections.Counter() | |
| rows = [] | |
| total = 0 | |
| with args.input.open(encoding="utf-8") as handle: | |
| for line in handle: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| total += 1 | |
| record = json.loads(line) | |
| row, reason = build_row(record, args.generator_model) | |
| if row is None: | |
| rejected[reason or "unknown"] += 1 | |
| continue | |
| if args.reject_cap_hit and len(row["metadata"]["generated_token_ids"]) >= args.reject_cap_hit: | |
| rejected["cap_hit"] += 1 | |
| continue | |
| labels = row["labels"] | |
| uncertain = sum(1 for v in labels if v == UNCERTAIN) | |
| if labels and uncertain / len(labels) > args.max_uncertain_fraction: | |
| rejected["mostly_uncertain"] += 1 | |
| continue | |
| rows.append(row) | |
| if not rows: | |
| raise SystemExit("no usable rows -- check that the judge actually ran") | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| with args.output.open("w", encoding="utf-8") as handle: | |
| for row in rows: | |
| handle.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| n_tokens = sum(len(r["labels"]) for r in rows) | |
| n_h = sum(sum(1 for v in r["labels"] if v == HALLUCINATED) for r in rows) | |
| n_u = sum(sum(1 for v in r["labels"] if v == UNCERTAIN) for r in rows) | |
| n_g = n_tokens - n_h - n_u | |
| with_h = sum(1 for r in rows if any(v == HALLUCINATED for v in r["labels"])) | |
| print(f"wrote {len(rows):,} of {total:,} rows -> {args.output}") | |
| print(f" tokens {n_tokens:,} | G {n_g:,} | H {n_h:,} | U {n_u:,}") | |
| # This is the number that decides whether the corpus is usable at all: the | |
| # AP floor is the G/H prevalence, and a token-level detector needs enough | |
| # positives for the metric to have resolution. | |
| if n_g + n_h: | |
| print(f" H prevalence over G/H tokens: {n_h / (n_g + n_h):.4%}") | |
| print(f" rows containing >=1 hallucinated token: {with_h:,} ({with_h/len(rows):.1%})") | |
| if rejected: | |
| print(" rejected:", dict(rejected)) | |
| print(f" sha256: {hashlib.sha256(args.output.read_bytes()).hexdigest()}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |