Buckets:

AsianPlayer
/

TriDrive

Files

xet

AsianPlayer/TriDrive / code /eval /run_k4_evals_v2.sh

AsianPlayer

16 days ago

download

raw

6.26 kB

#!/usr/bin/env bash
# cwm3/eval/run_k4_evals_v2.sh — K4 (claim B1 / K4) evaluation of the U0 runs on the v2 causal-kinematics packs (docs/k4_handoff.md v1 -> v2), CPU.
#   bash eval/run_k4_evals_v2.sh "0 1 2" [render=1]
# Per seed k and regime (all tags idempotent on results/paired_compare.csv; driver populations from results/targets_kin_causal_routes_v2, --kin-version v2):
#   K4v2in_s<k>_causal_vs_oldin      A = U0_kin_can_noK_s<k>_kcv2 (causal v2 inputs)  B = …_oldin_ctgt_v2 (OLD route-median inputs, same v2 targets) — input effect (claim K4)
#   K4v2c_s<k>_causal_vs_k0          A = …_kcv2                    B = k0 heuristic from the v2 causal K0 (+ persistence base-rate rows)   — claim B1, all rows
#   K4v2c_s<k>_oldin_vs_k0           A = …_oldin_ctgt_v2           B = k0                                                                 — control level
#   K4v2cm_s<k>_causalm_vs_k0        A = …_kcv2_masked             B = k0; v2 suspect routes censored from every population (--mask-routes) — claim B1, masked
#   K4v2v1_s<k>_causalv2_vs_causalv1 A = …_kcv2  B = …_kcv1 (v1 causal packs), both scored on the v2 causal targets, identical rows       — v2 vs v1 build
#   K4v2v1_s<k>_causalv2m_vs_causalv1m  the masked runs (each trained without its own version's suspects), scored outside the v2 suspects
#   K4v2s1_s<k>_causalstats1_vs_causal  A = …_kcv2_stats1 (v2 packs, PRE-fix float32 statistics)  B = …_kcv2 (stats v2) — the stats-version arm (coordinator 2026-09-11)
#   K4v2c_s<k>_causalstats1_vs_k0       A = …_kcv2_stats1  B = k0                                                             — its level vs the reference
#   Recipe arm (coordinator 2026-09-11): A = …_kcv2_clip8 (clamp 8) / …_kcv2_clip8_ln (clamp 8 + input LN), stats v2, all rows, v2 targets:
#   K4v2c_s<k>_{clip8,clip8ln}_vs_k0; K4v2clip_s<k>_{clip8,clip8ln}_vs_causal (B = …_kcv2); K4v2clip_s<k>_{clip8,clip8ln}_vs_causalstats1 (B = …_kcv2_stats1);
#   K4v2clip_s<k>_clip8ln_vs_clip8 (the LN effect at clamp 8).
# Cross-target rows vs the non-causal noK (old targets) are NOT repeated (declared DESCRIPTIVE / definition mismatch in the v1 K4 row).
# Rendering: report_joint.py (--claim K4 / B1) -> results/tables/k4_v2_{input_effect,b1_vs_k0,b1_vs_k0_masked,vs_v1}.md; report_k4_summary.py --kin-version v2 -> k4_v2_summary.md
set -u
cd "$(dirname "$0")/.."
PY=python
export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4
MASK=results/targets_kin_causal_routes_v2/driver_box_suspect_routes.json
have_tag() { grep -q ",$1,$2,test," results/paired_compare.csv 2>/dev/null; }
pc() {  # tag regime a b [extra args]
  local tag="$1" reg="$2" a="$3" b="$4"; shift 4
  have_tag "$tag" "$reg" && return
  echo "== $tag $reg"
  $PY eval/paired_compare_joint.py --regime "$reg" --a "$a" --b "$b" --tag "$tag" --kin-targets causal_routes --kin-version v2 "$@" 2>&1 | grep -E "appended|Error|Traceback|File \""
}
for s in ${1:-0 1 2}; do
  A=results/runs/U0_kin_can_noK_s${s}_kcv2; AM=results/runs/U0_kin_can_noK_s${s}_kcv2_masked; C=results/runs/U0_kin_can_noK_s${s}_oldin_ctgt_v2
  V1=results/runs/U0_kin_can_noK_s${s}_kcv1; V1M=results/runs/U0_kin_can_noK_s${s}_kcv1_masked; S1=results/runs/U0_kin_can_noK_s${s}_kcv2_stats1
  C8=results/runs/U0_kin_can_noK_s${s}_kcv2_clip8; C8L=results/runs/U0_kin_can_noK_s${s}_kcv2_clip8_ln
  for reg in manual engaged; do
    if [[ -f $A/PRED_DONE && -f $A/TRAIN_DONE ]]; then
      pc "K4v2c_s${s}_causal_vs_k0" $reg $A none --b-k0
      [[ -f $C/PRED_DONE && -f $C/TRAIN_DONE ]] && pc "K4v2in_s${s}_causal_vs_oldin" $reg $A $C
      [[ -f $V1/PRED_DONE ]] && pc "K4v2v1_s${s}_causalv2_vs_causalv1" $reg $A $V1
    else echo "skip s$s $reg (causal v2 run not finished)"; fi
    [[ -f $C/PRED_DONE && -f $C/TRAIN_DONE ]] && pc "K4v2c_s${s}_oldin_vs_k0" $reg $C none --b-k0
    if [[ -f $S1/PRED_DONE ]]; then
      pc "K4v2c_s${s}_causalstats1_vs_k0" $reg $S1 none --b-k0
      [[ -f $A/PRED_DONE && -f $A/TRAIN_DONE ]] && pc "K4v2s1_s${s}_causalstats1_vs_causal" $reg $S1 $A
    fi
    for arm in clip8 clip8ln; do
      D=$C8; [[ $arm == clip8ln ]] && D=$C8L
      [[ -f $D/PRED_DONE ]] || continue
      pc "K4v2c_s${s}_${arm}_vs_k0" $reg $D none --b-k0
      [[ -f $A/PRED_DONE && -f $A/TRAIN_DONE ]] && pc "K4v2clip_s${s}_${arm}_vs_causal" $reg $D $A
      [[ -f $S1/PRED_DONE ]] && pc "K4v2clip_s${s}_${arm}_vs_causalstats1" $reg $D $S1
    done
    [[ -f $C8/PRED_DONE && -f $C8L/PRED_DONE ]] && pc "K4v2clip_s${s}_clip8ln_vs_clip8" $reg $C8L $C8
    if [[ -f $AM/PRED_DONE && -f $AM/TRAIN_DONE ]]; then
      pc "K4v2cm_s${s}_causalm_vs_k0" $reg $AM none --b-k0 --mask-routes $MASK
      [[ -f $V1M/PRED_DONE ]] && pc "K4v2v1_s${s}_causalv2m_vs_causalv1m" $reg $AM $V1M --mask-routes $MASK
    else echo "skip masked s$s $reg (run not finished)"; fi
  done
done
if [[ "${2:-1}" == "1" ]]; then
  S=${1:-0 1 2}
  $PY eval/report_joint.py --stage K4v2in --a causal --b oldin --claim K4 --seeds ${S// /,} --out results/tables/k4_v2_input_effect.md 2>&1 | grep -E "written|verdict"
  $PY eval/report_joint.py --stage K4v2c --a causal --b k0 --claim B1 --seeds ${S// /,} --extra oldin:k0 --out results/tables/k4_v2_b1_vs_k0.md 2>&1 | grep -E "written|verdict"
  $PY eval/report_joint.py --stage K4v2cm --a causalm --b k0 --claim B1 --seeds ${S// /,} --out results/tables/k4_v2_b1_vs_k0_masked.md 2>&1 | grep -E "written|verdict"
  $PY eval/report_joint.py --stage K4v2v1 --a causalv2 --b causalv1 --claim K4 --seeds ${S// /,} --extra causalv2m:causalv1m --out results/tables/k4_v2_vs_v1.md 2>&1 | grep -E "written|verdict"
  [[ -f results/runs/U0_kin_can_noK_s0_kcv2_stats1/PRED_DONE ]] && $PY eval/report_joint.py --stage K4v2s1 --a causalstats1 --b causal --claim K4 --seeds ${S// /,} --out results/tables/k4_v2_stats1_vs_stats2.md 2>&1 | grep -E "written|verdict"
  [[ -f results/runs/U0_kin_can_noK_s0_kcv2_clip8_ln/PRED_DONE ]] && $PY eval/report_joint.py --stage K4v2clip --a clip8ln --b causal --claim K4 --seeds ${S// /,} --extra clip8:causal clip8:causalstats1 clip8ln:causalstats1 clip8ln:clip8 --out results/tables/k4_v2_recipe_arm.md 2>&1 | grep -E "written|verdict"
  $PY eval/report_k4_summary.py --kin-version v2 --out results/tables/k4_v2_summary.md 2>&1 | tail -3
fi
echo K4V2_EVALS_DONE

Xet Storage Details

Size:: 6.26 kB
Xet hash:: 143d5cdf425e416fd2af01fe52c8445c231b8274910b0fdfdd96988d61b31128

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	#!/usr/bin/env bash
	# cwm3/eval/run_k4_evals_v2.sh — K4 (claim B1 / K4) evaluation of the U0 runs on the v2 causal-kinematics packs (docs/k4_handoff.md v1 -> v2), CPU.
	# bash eval/run_k4_evals_v2.sh "0 1 2" [render=1]
	# Per seed k and regime (all tags idempotent on results/paired_compare.csv; driver populations from results/targets_kin_causal_routes_v2, --kin-version v2):
	# K4v2in_s<k>_causal_vs_oldin A = U0_kin_can_noK_s<k>_kcv2 (causal v2 inputs) B = …_oldin_ctgt_v2 (OLD route-median inputs, same v2 targets) — input effect (claim K4)
	# K4v2c_s<k>_causal_vs_k0 A = …_kcv2 B = k0 heuristic from the v2 causal K0 (+ persistence base-rate rows) — claim B1, all rows
	# K4v2c_s<k>_oldin_vs_k0 A = …_oldin_ctgt_v2 B = k0 — control level
	# K4v2cm_s<k>_causalm_vs_k0 A = …_kcv2_masked B = k0; v2 suspect routes censored from every population (--mask-routes) — claim B1, masked
	# K4v2v1_s<k>_causalv2_vs_causalv1 A = …_kcv2 B = …_kcv1 (v1 causal packs), both scored on the v2 causal targets, identical rows — v2 vs v1 build
	# K4v2v1_s<k>_causalv2m_vs_causalv1m the masked runs (each trained without its own version's suspects), scored outside the v2 suspects
	# K4v2s1_s<k>_causalstats1_vs_causal A = …_kcv2_stats1 (v2 packs, PRE-fix float32 statistics) B = …_kcv2 (stats v2) — the stats-version arm (coordinator 2026-09-11)
	# K4v2c_s<k>_causalstats1_vs_k0 A = …_kcv2_stats1 B = k0 — its level vs the reference
	# Recipe arm (coordinator 2026-09-11): A = …_kcv2_clip8 (clamp 8) / …_kcv2_clip8_ln (clamp 8 + input LN), stats v2, all rows, v2 targets:
	# K4v2c_s<k>_{clip8,clip8ln}_vs_k0; K4v2clip_s<k>_{clip8,clip8ln}_vs_causal (B = …_kcv2); K4v2clip_s<k>_{clip8,clip8ln}_vs_causalstats1 (B = …_kcv2_stats1);
	# K4v2clip_s<k>_clip8ln_vs_clip8 (the LN effect at clamp 8).
	# Cross-target rows vs the non-causal noK (old targets) are NOT repeated (declared DESCRIPTIVE / definition mismatch in the v1 K4 row).
	# Rendering: report_joint.py (--claim K4 / B1) -> results/tables/k4_v2_{input_effect,b1_vs_k0,b1_vs_k0_masked,vs_v1}.md; report_k4_summary.py --kin-version v2 -> k4_v2_summary.md
	set -u
	cd "$(dirname "$0")/.."
	PY=python
	export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4
	MASK=results/targets_kin_causal_routes_v2/driver_box_suspect_routes.json
	have_tag() { grep -q ",$1,$2,test," results/paired_compare.csv 2>/dev/null; }
	pc() { # tag regime a b [extra args]
	local tag="$1" reg="$2" a="$3" b="$4"; shift 4
	have_tag "$tag" "$reg" && return
	echo "== $tag $reg"
	$PY eval/paired_compare_joint.py --regime "$reg" --a "$a" --b "$b" --tag "$tag" --kin-targets causal_routes --kin-version v2 "$@" 2>&1 \| grep -E "appended\|Error\|Traceback\|File \""
	}
	for s in ${1:-0 1 2}; do
	A=results/runs/U0_kin_can_noK_s${s}_kcv2; AM=results/runs/U0_kin_can_noK_s${s}_kcv2_masked; C=results/runs/U0_kin_can_noK_s${s}_oldin_ctgt_v2
	V1=results/runs/U0_kin_can_noK_s${s}_kcv1; V1M=results/runs/U0_kin_can_noK_s${s}_kcv1_masked; S1=results/runs/U0_kin_can_noK_s${s}_kcv2_stats1
	C8=results/runs/U0_kin_can_noK_s${s}_kcv2_clip8; C8L=results/runs/U0_kin_can_noK_s${s}_kcv2_clip8_ln
	for reg in manual engaged; do
	if [[ -f $A/PRED_DONE && -f $A/TRAIN_DONE ]]; then
	pc "K4v2c_s${s}_causal_vs_k0" $reg $A none --b-k0
	[[ -f $C/PRED_DONE && -f $C/TRAIN_DONE ]] && pc "K4v2in_s${s}_causal_vs_oldin" $reg $A $C
	[[ -f $V1/PRED_DONE ]] && pc "K4v2v1_s${s}_causalv2_vs_causalv1" $reg $A $V1
	else echo "skip s$s $reg (causal v2 run not finished)"; fi
	[[ -f $C/PRED_DONE && -f $C/TRAIN_DONE ]] && pc "K4v2c_s${s}_oldin_vs_k0" $reg $C none --b-k0
	if [[ -f $S1/PRED_DONE ]]; then
	pc "K4v2c_s${s}_causalstats1_vs_k0" $reg $S1 none --b-k0
	[[ -f $A/PRED_DONE && -f $A/TRAIN_DONE ]] && pc "K4v2s1_s${s}_causalstats1_vs_causal" $reg $S1 $A
	fi
	for arm in clip8 clip8ln; do
	D=$C8; [[ $arm == clip8ln ]] && D=$C8L
	[[ -f $D/PRED_DONE ]] \|\| continue
	pc "K4v2c_s${s}_${arm}_vs_k0" $reg $D none --b-k0
	[[ -f $A/PRED_DONE && -f $A/TRAIN_DONE ]] && pc "K4v2clip_s${s}_${arm}_vs_causal" $reg $D $A
	[[ -f $S1/PRED_DONE ]] && pc "K4v2clip_s${s}_${arm}_vs_causalstats1" $reg $D $S1
	done
	[[ -f $C8/PRED_DONE && -f $C8L/PRED_DONE ]] && pc "K4v2clip_s${s}_clip8ln_vs_clip8" $reg $C8L $C8
	if [[ -f $AM/PRED_DONE && -f $AM/TRAIN_DONE ]]; then
	pc "K4v2cm_s${s}_causalm_vs_k0" $reg $AM none --b-k0 --mask-routes $MASK
	[[ -f $V1M/PRED_DONE ]] && pc "K4v2v1_s${s}_causalv2m_vs_causalv1m" $reg $AM $V1M --mask-routes $MASK
	else echo "skip masked s$s $reg (run not finished)"; fi
	done
	done
	if [[ "${2:-1}" == "1" ]]; then
	S=${1:-0 1 2}
	$PY eval/report_joint.py --stage K4v2in --a causal --b oldin --claim K4 --seeds ${S// /,} --out results/tables/k4_v2_input_effect.md 2>&1 \| grep -E "written\|verdict"
	$PY eval/report_joint.py --stage K4v2c --a causal --b k0 --claim B1 --seeds ${S// /,} --extra oldin:k0 --out results/tables/k4_v2_b1_vs_k0.md 2>&1 \| grep -E "written\|verdict"
	$PY eval/report_joint.py --stage K4v2cm --a causalm --b k0 --claim B1 --seeds ${S// /,} --out results/tables/k4_v2_b1_vs_k0_masked.md 2>&1 \| grep -E "written\|verdict"
	$PY eval/report_joint.py --stage K4v2v1 --a causalv2 --b causalv1 --claim K4 --seeds ${S// /,} --extra causalv2m:causalv1m --out results/tables/k4_v2_vs_v1.md 2>&1 \| grep -E "written\|verdict"
	[[ -f results/runs/U0_kin_can_noK_s0_kcv2_stats1/PRED_DONE ]] && $PY eval/report_joint.py --stage K4v2s1 --a causalstats1 --b causal --claim K4 --seeds ${S// /,} --out results/tables/k4_v2_stats1_vs_stats2.md 2>&1 \| grep -E "written\|verdict"
	[[ -f results/runs/U0_kin_can_noK_s0_kcv2_clip8_ln/PRED_DONE ]] && $PY eval/report_joint.py --stage K4v2clip --a clip8ln --b causal --claim K4 --seeds ${S// /,} --extra clip8:causal clip8:causalstats1 clip8ln:causalstats1 clip8ln:clip8 --out results/tables/k4_v2_recipe_arm.md 2>&1 \| grep -E "written\|verdict"
	$PY eval/report_k4_summary.py --kin-version v2 --out results/tables/k4_v2_summary.md 2>&1 \| tail -3
	fi
	echo K4V2_EVALS_DONE