#!/usr/bin/env bash # MVA Transformer-CRF baseline, single seed, on v5.2 and RAGTruth. # # MVA (Ogamon et al., *SEM 2025, arXiv:2504.04335) is the closest published # method to ours -- attention features, hallucinated span detection -- so its # absence would be the omission a reviewer notices first. # # TWO SETTINGS THAT ARE NOT DEFAULTS, both deliberate: # # 1. --mva-max-length 512 on RAGTruth. The default is 300 and RAGTruth answers # reach 511 tokens (p95 384). Truncation is applied per row # (limit = min(len(labels), mva_max_length)), so at 300 MVA would score a # SMALLER token set than every other method and the table would stop being a # comparison. v5.2 answers average 26 tokens, so 300 is already ample there. # # 2. Published hyperparameters on BOTH corpora -- the script defaults, which are # the paper's configuration. No lock is used. # # An earlier version of this script passed # model_weights/fair_baselines/mva_causal_v5.3_hyperparameter_lock.json on # v5.2. That failed loudly: # # ValueError: MVA hyperparameter lock does not match the input dataset # # because the lock is bound by sha256 to v5.3's file, not v5.2's. The guard # was right, and so is the outcome: that lock was OUR tune-once selection on # OUR corpus, which would have given MVA more tuning than any other baseline # receives. Every other baseline here runs at its authors' settings, and the # MVA paper evaluated on RAGTruth with its own Optuna search, so the defaults # are the faithful choice on both datasets. # # MEMORY. MVA features are 3 x L x H per token = 3 x 32 x 32 = 3072 dims on # Llama-2-7b, built from the full attention tensor. Attention extraction on # RAGTruth is what previously took the HOST to 47.9 GB and got OOM-killed; # normalize_attentions was fixed to stream per layer, but MVA's feature builder # is a separate consumer and has never been run at these sequence lengths. The # RAGTruth pass therefore runs SECOND and its host RSS is sampled, so a failure # costs the v5.2 result nothing. # # bash scripts/run_mva.sh # both datasets # ONLY=v52 bash scripts/run_mva.sh # one of them # ONLY=ragtruth bash scripts/run_mva.sh set -u -o pipefail REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"; cd "$REPO" PY="${PY:-$REPO/.venv/bin/python}" SEED="${SEED:-42}" ROOT="${ROOT:-$REPO/outputs/mva}" ONLY="${ONLY:-both}" DET="multi_view_attention_transformer_crf" mkdir -p "$ROOT"; STATUS="$ROOT/status.tsv"; : > "$STATUS" echo "### MVA Transformer-CRF, seed $SEED" echo "started: $(date -Is)"; echo # Sample host RSS while a pid lives, so an OOM is diagnosed rather than guessed. watch_host () { local pid="$1" out="$2" peak=0 base cur base=$(( ( $(awk '/MemTotal/{print $2}' /proc/meminfo) - $(awk '/MemAvailable/{print $2}' /proc/meminfo) ) / 1024 )) while kill -0 "$pid" 2>/dev/null; do cur=$(( ( $(awk '/MemTotal/{print $2}' /proc/meminfo) - $(awk '/MemAvailable/{print $2}' /proc/meminfo) ) / 1024 )) cur=$(( cur - base )); (( cur > peak )) && peak=$cur sleep 20 done echo "$peak" > "$out" } run () { local name="$1"; shift local out="$ROOT/$name" if [[ -f "$out/summary.json" ]]; then echo "[skip] $name"; printf '%s\tSKIP\t0\n' "$name" >> "$STATUS"; return 0 fi mkdir -p "$out" echo "=== [$name] start $(date -Is)" local start rc el; start=$(date +%s) "$PY" -u scripts/train_faithful_baselines_dgx.py "$@" \ --detector "$DET" \ --suite causal_matched \ --seed "$SEED" --device cuda --dtype bfloat16 \ --allow-unverified-replay --strict-labels \ --output "$out/predictions.jsonl" \ --summary "$out/summary.json" \ --save-dir "$out/models" > "$out/run.log" 2>&1 & local pid=$! watch_host "$pid" "$out/host_peak_mb.txt" & local wpid=$! wait "$pid"; rc=$?; wait "$wpid" 2>/dev/null || true el=$(( $(date +%s) - start )) local peak; peak=$(cat "$out/host_peak_mb.txt" 2>/dev/null || echo "?") if [[ $rc -eq 0 ]]; then printf '%s\tOK\t%s\n' "$name" "$el" >> "$STATUS" echo "=== [$name] OK in $((el/60))m | host peak ${peak} MiB" "$PY" - "$out/summary.json" <<'PYEOF' import json,sys s=json.load(open(sys.argv[1])) for r in s.get("eval") or []: print(" %-38s AP %.4f AUROC %.4f tokens %s" % (r["method"], r.get("auprc") or 0, r.get("auroc") or 0, r.get("n_metric_tokens"))) PYEOF else local why="rc=$rc"; [[ $rc -eq 137 ]] && why="OOM-KILLED (host peak ${peak} MiB)" printf '%s\tFAILED(%s)\t%s\n' "$name" "$why" "$el" >> "$STATUS" echo "=== [$name] FAILED ($why)" tail -6 "$out/run.log" | sed "s/^/ /" fi echo } # ---- v5.2 first: 1B model, 26-token answers, lowest risk -------------------- if [[ "$ONLY" == "both" || "$ONLY" == "v52" ]]; then run v52 \ --model meta-llama/Llama-3.2-1B-Instruct \ --model-revision 9213176726f574b556790deb65791e0c5aa438b6 \ --input "$HOME/datasets/hallucinations_custom/hallu_dataset_v5.2_uncertain_eval.jsonl" \ --split-summary "$HOME/datasets/hallucinations_custom/hallu_v5_2_split.json" \ --prompt-template tanazzah-llama-chat \ --hidden-layer 8 --llm-check-attention-layer 8 \ --feature-cache-dir "$ROOT/cache_v52" fi # ---- RAGTruth second: 7B, 511-token answers, the memory risk ---------------- if [[ "$ONLY" == "both" || "$ONLY" == "ragtruth" ]]; then run ragtruth \ --model NousResearch/Llama-2-7b-chat-hf --allow-unpinned-model \ --input "$HOME/datasets/hallucinations_custom/ragtruth_llama2_7b_chat.jsonl" \ --split-summary "$HOME/datasets/hallucinations_custom/ragtruth_llama2_7b_chat_split.json" \ --prompt-template raw \ --hidden-layer 16 --llm-check-attention-layer 16 \ --mva-max-length 512 \ --feature-cache-dir "$ROOT/cache_ragtruth" fi echo "=== finished $(date -Is) ==="; sort "$STATUS" echo echo "Marks on the same token sets:" echo " v5.2 (72,678 tok, floor 0.0158): LatticeSpike 0.2741 | lookback 0.2305 | probe 0.2286" echo " RAGTruth (87,678 tok, floor 0.0625): LatticeSpike bs12 0.4168 | probe 0.4075 | lookback 0.4036" echo echo "MVA is stochastic (transformer + CRF) and measured +/-0.133 AP on v5.2," echo "so a single seed here is a smoke test, not a result. Five seeds before it" echo "enters any table."