Lattice_Spike / scripts /run_mva.sh
Md Saiful Islam Sajol
Upload Lattice_Spike project with full LFS weights
e2c9d42 verified
Raw History Blame Contribute Delete
6.3 kB
#!/usr/bin/env bash
# MVA Transformer-CRF baseline, single seed, on v5.2 and RAGTruth.
#
# MVA (Ogamon et al., *SEM 2025, arXiv:2504.04335) is the closest published
# method to ours -- attention features, hallucinated span detection -- so its
# absence would be the omission a reviewer notices first.
#
# TWO SETTINGS THAT ARE NOT DEFAULTS, both deliberate:
#
# 1. --mva-max-length 512 on RAGTruth. The default is 300 and RAGTruth answers
# reach 511 tokens (p95 384). Truncation is applied per row
# (limit = min(len(labels), mva_max_length)), so at 300 MVA would score a
# SMALLER token set than every other method and the table would stop being a
# comparison. v5.2 answers average 26 tokens, so 300 is already ample there.
#
# 2. Published hyperparameters on BOTH corpora -- the script defaults, which are
# the paper's configuration. No lock is used.
#
# An earlier version of this script passed
# model_weights/fair_baselines/mva_causal_v5.3_hyperparameter_lock.json on
# v5.2. That failed loudly:
#
# ValueError: MVA hyperparameter lock does not match the input dataset
#
# because the lock is bound by sha256 to v5.3's file, not v5.2's. The guard
# was right, and so is the outcome: that lock was OUR tune-once selection on
# OUR corpus, which would have given MVA more tuning than any other baseline
# receives. Every other baseline here runs at its authors' settings, and the
# MVA paper evaluated on RAGTruth with its own Optuna search, so the defaults
# are the faithful choice on both datasets.
#
# MEMORY. MVA features are 3 x L x H per token = 3 x 32 x 32 = 3072 dims on
# Llama-2-7b, built from the full attention tensor. Attention extraction on
# RAGTruth is what previously took the HOST to 47.9 GB and got OOM-killed;
# normalize_attentions was fixed to stream per layer, but MVA's feature builder
# is a separate consumer and has never been run at these sequence lengths. The
# RAGTruth pass therefore runs SECOND and its host RSS is sampled, so a failure
# costs the v5.2 result nothing.
#
# bash scripts/run_mva.sh # both datasets
# ONLY=v52 bash scripts/run_mva.sh # one of them
# ONLY=ragtruth bash scripts/run_mva.sh
set -u -o pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"; cd "$REPO"
PY="${PY:-$REPO/.venv/bin/python}"
SEED="${SEED:-42}"
ROOT="${ROOT:-$REPO/outputs/mva}"
ONLY="${ONLY:-both}"
DET="multi_view_attention_transformer_crf"
mkdir -p "$ROOT"; STATUS="$ROOT/status.tsv"; : > "$STATUS"
echo "### MVA Transformer-CRF, seed $SEED"
echo "started: $(date -Is)"; echo
# Sample host RSS while a pid lives, so an OOM is diagnosed rather than guessed.
watch_host () {
local pid="$1" out="$2" peak=0 base cur
base=$(( ( $(awk '/MemTotal/{print $2}' /proc/meminfo) - $(awk '/MemAvailable/{print $2}' /proc/meminfo) ) / 1024 ))
while kill -0 "$pid" 2>/dev/null; do
cur=$(( ( $(awk '/MemTotal/{print $2}' /proc/meminfo) - $(awk '/MemAvailable/{print $2}' /proc/meminfo) ) / 1024 ))
cur=$(( cur - base )); (( cur > peak )) && peak=$cur
sleep 20
done
echo "$peak" > "$out"
}
run () {
local name="$1"; shift
local out="$ROOT/$name"
if [[ -f "$out/summary.json" ]]; then
echo "[skip] $name"; printf '%s\tSKIP\t0\n' "$name" >> "$STATUS"; return 0
fi
mkdir -p "$out"
echo "=== [$name] start $(date -Is)"
local start rc el; start=$(date +%s)
"$PY" -u scripts/train_faithful_baselines_dgx.py "$@" \
--detector "$DET" \
--suite causal_matched \
--seed "$SEED" --device cuda --dtype bfloat16 \
--allow-unverified-replay --strict-labels \
--output "$out/predictions.jsonl" \
--summary "$out/summary.json" \
--save-dir "$out/models" > "$out/run.log" 2>&1 &
local pid=$!
watch_host "$pid" "$out/host_peak_mb.txt" &
local wpid=$!
wait "$pid"; rc=$?; wait "$wpid" 2>/dev/null || true
el=$(( $(date +%s) - start ))
local peak; peak=$(cat "$out/host_peak_mb.txt" 2>/dev/null || echo "?")
if [[ $rc -eq 0 ]]; then
printf '%s\tOK\t%s\n' "$name" "$el" >> "$STATUS"
echo "=== [$name] OK in $((el/60))m | host peak ${peak} MiB"
"$PY" - "$out/summary.json" <<'PYEOF'
import json,sys
s=json.load(open(sys.argv[1]))
for r in s.get("eval") or []:
print(" %-38s AP %.4f AUROC %.4f tokens %s"
% (r["method"], r.get("auprc") or 0, r.get("auroc") or 0,
r.get("n_metric_tokens")))
PYEOF
else
local why="rc=$rc"; [[ $rc -eq 137 ]] && why="OOM-KILLED (host peak ${peak} MiB)"
printf '%s\tFAILED(%s)\t%s\n' "$name" "$why" "$el" >> "$STATUS"
echo "=== [$name] FAILED ($why)"
tail -6 "$out/run.log" | sed "s/^/ /"
fi
echo
}
# ---- v5.2 first: 1B model, 26-token answers, lowest risk --------------------
if [[ "$ONLY" == "both" || "$ONLY" == "v52" ]]; then
run v52 \
--model meta-llama/Llama-3.2-1B-Instruct \
--model-revision 9213176726f574b556790deb65791e0c5aa438b6 \
--input "$HOME/datasets/hallucinations_custom/hallu_dataset_v5.2_uncertain_eval.jsonl" \
--split-summary "$HOME/datasets/hallucinations_custom/hallu_v5_2_split.json" \
--prompt-template tanazzah-llama-chat \
--hidden-layer 8 --llm-check-attention-layer 8 \
--feature-cache-dir "$ROOT/cache_v52"
fi
# ---- RAGTruth second: 7B, 511-token answers, the memory risk ----------------
if [[ "$ONLY" == "both" || "$ONLY" == "ragtruth" ]]; then
run ragtruth \
--model NousResearch/Llama-2-7b-chat-hf --allow-unpinned-model \
--input "$HOME/datasets/hallucinations_custom/ragtruth_llama2_7b_chat.jsonl" \
--split-summary "$HOME/datasets/hallucinations_custom/ragtruth_llama2_7b_chat_split.json" \
--prompt-template raw \
--hidden-layer 16 --llm-check-attention-layer 16 \
--mva-max-length 512 \
--feature-cache-dir "$ROOT/cache_ragtruth"
fi
echo "=== finished $(date -Is) ==="; sort "$STATUS"
echo
echo "Marks on the same token sets:"
echo " v5.2 (72,678 tok, floor 0.0158): LatticeSpike 0.2741 | lookback 0.2305 | probe 0.2286"
echo " RAGTruth (87,678 tok, floor 0.0625): LatticeSpike bs12 0.4168 | probe 0.4075 | lookback 0.4036"
echo
echo "MVA is stochastic (transformer + CRF) and measured +/-0.133 AP on v5.2,"
echo "so a single seed here is a smoke test, not a result. Five seeds before it"
echo "enters any table."