Buckets:
| # cwm3/eval/run_identity_twins.sh — evaluation of the identity-invariance arms (innovation 2; docs/innovations_report.md §1) against the survivor baseline | |
| # U0_kin_can_noK_s<k>_kcv2_clip8_ln, per seed / arm / regime (idempotent on the csv tags): | |
| # IDtw_s<k>_<arm>_vs_ln eval/shift_twin.py (same-route twin pair-AUROC + random-negative reference; identity probe top-1 on the frozen GRU state; centred behaviour probe) | |
| # on the causal v2 kin targets (--kin-source causal_routes --kin-version v2), events hand_return / hand_leave / head_turn_onset | |
| # IDk4_s<k>_<arm>_vs_ln eval/paired_compare_joint.py driver cells arm − baseline on identical rows, causal v2 targets, all rows | |
| # IDk4m_s<k>_<arm>_vs_ln the same with the v2 suspect routes removed from every population (--mask-routes; evaluation-side masking, both models trained on all rows) | |
| # IDk0_s<k>_<arm>_vs_k0 / IDk0m_… arm vs the k0 current-state heuristic (all rows / masked) | |
| # then eval/report_identity_invariance.py -> results/tables/identity_invariance.md. bash eval/run_identity_twins.sh "0 1 2" "adv01 adv03 devnorm" [render=1] | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| MASK=results/targets_kin_causal_routes_v2/driver_box_suspect_routes.json | |
| have_pc() { grep -q ",$1,$2,test," results/paired_compare.csv 2>/dev/null; } | |
| pc() { local tag="$1" reg="$2" a="$3" b="$4"; shift 4; have_pc "$tag" "$reg" && return; echo "== $tag $reg" | |
| $PY eval/paired_compare_joint.py --regime "$reg" --a "$a" --b "$b" --tag "$tag" --kin-targets causal_routes --kin-version v2 "$@" 2>&1 | grep -E "appended|Error|Traceback|File \"" | cut -c1-200; } | |
| for s in ${1:-0 1 2}; do | |
| B=results/runs/U0_kin_can_noK_s${s}_kcv2_clip8_ln | |
| for arm in ${2:-adv01 adv03 devnorm}; do | |
| A=results/runs/U0_kin_can_noK_s${s}_kcv2_clip8_ln_${arm} | |
| [[ -f $A/PRED_DONE && -f $B/PRED_DONE ]] || { echo "skip s$s $arm (runs not finished)"; continue; } | |
| for reg in manual engaged; do | |
| tag="IDtw_s${s}_${arm}_vs_ln" | |
| if ! grep -q ",${tag},${reg}," results/shift_twin.csv 2>/dev/null; then | |
| echo "== $tag $reg" | |
| $PY eval/shift_twin.py --a $A --b $B --regime $reg --events hand_return,hand_leave,head_turn_onset --tag "$tag" --kin-source causal_routes --kin-version v2 2>&1 | grep -E "pair_auroc|identity_probe|latent_probe|Error|Traceback|File \"|done in" | cut -c1-220 | |
| fi | |
| pc "IDk4_s${s}_${arm}_vs_ln" $reg $A $B | |
| pc "IDk4m_s${s}_${arm}_vs_ln" $reg $A $B --mask-routes $MASK | |
| pc "IDk0_s${s}_${arm}_vs_k0" $reg $A none --b-k0 | |
| pc "IDk0m_s${s}_${arm}_vs_k0" $reg $A none --b-k0 --mask-routes $MASK | |
| done | |
| done | |
| # the baseline's own twin statistics / identity probe (A = B = baseline gives the levels; the delta is 0 by construction) and its k0 rows exist from the K4 recipe arm | |
| for reg in manual engaged; do | |
| tag="IDtw_s${s}_ln_vs_ln" | |
| [[ -f $B/PRED_DONE ]] && ! grep -q ",${tag},${reg}," results/shift_twin.csv 2>/dev/null && { echo "== $tag $reg" | |
| $PY eval/shift_twin.py --a $B --b $B --regime $reg --events hand_return,hand_leave,head_turn_onset --tag "$tag" --kin-source causal_routes --kin-version v2 2>&1 | grep -E "pair_auroc|identity_probe|Error|Traceback|File \"" | cut -c1-220; } | |
| [[ -f $B/PRED_DONE ]] && pc "IDk0m_s${s}_ln_vs_k0" $reg $B none --b-k0 --mask-routes $MASK | |
| done | |
| done | |
| [[ "${3:-1}" == "1" ]] && $PY eval/report_identity_invariance.py 2>&1 | tail -5 | |
| echo IDENTITY_EVALS_DONE | |
Xet Storage Details
- Size:
- 3.51 kB
- Xet hash:
- 5a0fc89316a2a1879e94a8a3f563e23ebae9b639737810b14a1a998859970607
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.