Download scripts/run_mva.sh from Anonymous1611/Lattice_Spike: direct link, hf CLI and curl.
- Browser
- Download file 6.3 kB
-
https://huggingface.co/Anonymous1611/Lattice_Spike/resolve/main/scripts/run_mva.sh
- Command line
-
hf download hf://Anonymous1611/Lattice_Spike/scripts/run_mva.sh
-
curl -L -o run_mva.sh https://huggingface.co/Anonymous1611/Lattice_Spike/resolve/main/scripts/run_mva.sh
6.3 kB
| # MVA Transformer-CRF baseline, single seed, on v5.2 and RAGTruth. | |
| # | |
| # MVA (Ogamon et al., *SEM 2025, arXiv:2504.04335) is the closest published | |
| # method to ours -- attention features, hallucinated span detection -- so its | |
| # absence would be the omission a reviewer notices first. | |
| # | |
| # TWO SETTINGS THAT ARE NOT DEFAULTS, both deliberate: | |
| # | |
| # 1. --mva-max-length 512 on RAGTruth. The default is 300 and RAGTruth answers | |
| # reach 511 tokens (p95 384). Truncation is applied per row | |
| # (limit = min(len(labels), mva_max_length)), so at 300 MVA would score a | |
| # SMALLER token set than every other method and the table would stop being a | |
| # comparison. v5.2 answers average 26 tokens, so 300 is already ample there. | |
| # | |
| # 2. Published hyperparameters on BOTH corpora -- the script defaults, which are | |
| # the paper's configuration. No lock is used. | |
| # | |
| # An earlier version of this script passed | |
| # model_weights/fair_baselines/mva_causal_v5.3_hyperparameter_lock.json on | |
| # v5.2. That failed loudly: | |
| # | |
| # ValueError: MVA hyperparameter lock does not match the input dataset | |
| # | |
| # because the lock is bound by sha256 to v5.3's file, not v5.2's. The guard | |
| # was right, and so is the outcome: that lock was OUR tune-once selection on | |
| # OUR corpus, which would have given MVA more tuning than any other baseline | |
| # receives. Every other baseline here runs at its authors' settings, and the | |
| # MVA paper evaluated on RAGTruth with its own Optuna search, so the defaults | |
| # are the faithful choice on both datasets. | |
| # | |
| # MEMORY. MVA features are 3 x L x H per token = 3 x 32 x 32 = 3072 dims on | |
| # Llama-2-7b, built from the full attention tensor. Attention extraction on | |
| # RAGTruth is what previously took the HOST to 47.9 GB and got OOM-killed; | |
| # normalize_attentions was fixed to stream per layer, but MVA's feature builder | |
| # is a separate consumer and has never been run at these sequence lengths. The | |
| # RAGTruth pass therefore runs SECOND and its host RSS is sampled, so a failure | |
| # costs the v5.2 result nothing. | |
| # | |
| # bash scripts/run_mva.sh # both datasets | |
| # ONLY=v52 bash scripts/run_mva.sh # one of them | |
| # ONLY=ragtruth bash scripts/run_mva.sh | |
| set -u -o pipefail | |
| REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"; cd "$REPO" | |
| PY="${PY:-$REPO/.venv/bin/python}" | |
| SEED="${SEED:-42}" | |
| ROOT="${ROOT:-$REPO/outputs/mva}" | |
| ONLY="${ONLY:-both}" | |
| DET="multi_view_attention_transformer_crf" | |
| mkdir -p "$ROOT"; STATUS="$ROOT/status.tsv"; : > "$STATUS" | |
| echo "### MVA Transformer-CRF, seed $SEED" | |
| echo "started: $(date -Is)"; echo | |
| # Sample host RSS while a pid lives, so an OOM is diagnosed rather than guessed. | |
| watch_host () { | |
| local pid="$1" out="$2" peak=0 base cur | |
| base=$(( ( $(awk '/MemTotal/{print $2}' /proc/meminfo) - $(awk '/MemAvailable/{print $2}' /proc/meminfo) ) / 1024 )) | |
| while kill -0 "$pid" 2>/dev/null; do | |
| cur=$(( ( $(awk '/MemTotal/{print $2}' /proc/meminfo) - $(awk '/MemAvailable/{print $2}' /proc/meminfo) ) / 1024 )) | |
| cur=$(( cur - base )); (( cur > peak )) && peak=$cur | |
| sleep 20 | |
| done | |
| echo "$peak" > "$out" | |
| } | |
| run () { | |
| local name="$1"; shift | |
| local out="$ROOT/$name" | |
| if [[ -f "$out/summary.json" ]]; then | |
| echo "[skip] $name"; printf '%s\tSKIP\t0\n' "$name" >> "$STATUS"; return 0 | |
| fi | |
| mkdir -p "$out" | |
| echo "=== [$name] start $(date -Is)" | |
| local start rc el; start=$(date +%s) | |
| "$PY" -u scripts/train_faithful_baselines_dgx.py "$@" \ | |
| --detector "$DET" \ | |
| --suite causal_matched \ | |
| --seed "$SEED" --device cuda --dtype bfloat16 \ | |
| --allow-unverified-replay --strict-labels \ | |
| --output "$out/predictions.jsonl" \ | |
| --summary "$out/summary.json" \ | |
| --save-dir "$out/models" > "$out/run.log" 2>&1 & | |
| local pid=$! | |
| watch_host "$pid" "$out/host_peak_mb.txt" & | |
| local wpid=$! | |
| wait "$pid"; rc=$?; wait "$wpid" 2>/dev/null || true | |
| el=$(( $(date +%s) - start )) | |
| local peak; peak=$(cat "$out/host_peak_mb.txt" 2>/dev/null || echo "?") | |
| if [[ $rc -eq 0 ]]; then | |
| printf '%s\tOK\t%s\n' "$name" "$el" >> "$STATUS" | |
| echo "=== [$name] OK in $((el/60))m | host peak ${peak} MiB" | |
| "$PY" - "$out/summary.json" <<'PYEOF' | |
| import json,sys | |
| s=json.load(open(sys.argv[1])) | |
| for r in s.get("eval") or []: | |
| print(" %-38s AP %.4f AUROC %.4f tokens %s" | |
| % (r["method"], r.get("auprc") or 0, r.get("auroc") or 0, | |
| r.get("n_metric_tokens"))) | |
| PYEOF | |
| else | |
| local why="rc=$rc"; [[ $rc -eq 137 ]] && why="OOM-KILLED (host peak ${peak} MiB)" | |
| printf '%s\tFAILED(%s)\t%s\n' "$name" "$why" "$el" >> "$STATUS" | |
| echo "=== [$name] FAILED ($why)" | |
| tail -6 "$out/run.log" | sed "s/^/ /" | |
| fi | |
| echo | |
| } | |
| # ---- v5.2 first: 1B model, 26-token answers, lowest risk -------------------- | |
| if [[ "$ONLY" == "both" || "$ONLY" == "v52" ]]; then | |
| run v52 \ | |
| --model meta-llama/Llama-3.2-1B-Instruct \ | |
| --model-revision 9213176726f574b556790deb65791e0c5aa438b6 \ | |
| --input "$HOME/datasets/hallucinations_custom/hallu_dataset_v5.2_uncertain_eval.jsonl" \ | |
| --split-summary "$HOME/datasets/hallucinations_custom/hallu_v5_2_split.json" \ | |
| --prompt-template tanazzah-llama-chat \ | |
| --hidden-layer 8 --llm-check-attention-layer 8 \ | |
| --feature-cache-dir "$ROOT/cache_v52" | |
| fi | |
| # ---- RAGTruth second: 7B, 511-token answers, the memory risk ---------------- | |
| if [[ "$ONLY" == "both" || "$ONLY" == "ragtruth" ]]; then | |
| run ragtruth \ | |
| --model NousResearch/Llama-2-7b-chat-hf --allow-unpinned-model \ | |
| --input "$HOME/datasets/hallucinations_custom/ragtruth_llama2_7b_chat.jsonl" \ | |
| --split-summary "$HOME/datasets/hallucinations_custom/ragtruth_llama2_7b_chat_split.json" \ | |
| --prompt-template raw \ | |
| --hidden-layer 16 --llm-check-attention-layer 16 \ | |
| --mva-max-length 512 \ | |
| --feature-cache-dir "$ROOT/cache_ragtruth" | |
| fi | |
| echo "=== finished $(date -Is) ==="; sort "$STATUS" | |
| echo | |
| echo "Marks on the same token sets:" | |
| echo " v5.2 (72,678 tok, floor 0.0158): LatticeSpike 0.2741 | lookback 0.2305 | probe 0.2286" | |
| echo " RAGTruth (87,678 tok, floor 0.0625): LatticeSpike bs12 0.4168 | probe 0.4075 | lookback 0.4036" | |
| echo | |
| echo "MVA is stochastic (transformer + CRF) and measured +/-0.133 AP on v5.2," | |
| echo "so a single seed here is a smoke test, not a result. Five seeds before it" | |
| echo "enters any table." | |