File size: 6,295 Bytes
9b7b80d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | #!/bin/bash
# Every run of the label-efficiency study in docs/design.md ("Label
# efficiency on GamGam"), with its exact overrides, one after another on one
# GPU (the study ran them as parallel jobs, 3.5 hours on 4 GPUs):
# bash scripts/label_efficiency_study.sh
# OUT (default outputs/study) holds runs/, label_efficiency/ and probes/;
# DELPHES is the Delphes event cache. Restartable: finished runs are skipped.
# TabPFN-3.5 needs its weights in TABPFN_MODEL_CACHE_DIR (docs/usage.md).
# Prepare the Delphes and GamGam event caches first.
set -euo pipefail
cd "$(dirname "$0")/.."
OUT="${OUT:-outputs/study}"
D="${DELPHES:-${SCRATCH:-outputs}/gnn4colliders/delphes/events.pt}"
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}"
export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}"
export TABPFN_MODEL_CACHE_DIR="${TABPFN_MODEL_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/tabpfn}"
G4C=(uv run --all-extras gnn4colliders)
EVAL=data.eval_events=20000
PGNN=Pretrained_GNN/multiclass_pretrained_model_12/model_epoch_71.pt
# The encoder sizes of configs/delphes_{jepa,masked}.yaml.
TRANSFORMER=(model.backbone=transformer model.hid_size=128 model.n_layers=4
model.n_heads=8 model.ffn_size=512 model.pair_bias=true)
sweep() { # <config> [key=value ...]: N = 100, 1000, 10000
uv run --all-extras python scripts/label_efficiency.py "$1" \
--events 100 1000 10000 --out "${OUT}/label_efficiency" --override "${EVAL}" "${@:2}"
}
steps() { # <config> <name> [key=value ...]: a fixed number of steps per N
local n batch epochs
while read -r n batch epochs; do
uv run --all-extras python scripts/label_efficiency.py "$1" --events "${n}" \
--out "${OUT}/label_efficiency" --override name="$2" "${EVAL}" \
data.batch_size="${batch}" train.max_epochs="${epochs}" train.early_stopping=null "${@:3}"
done <<< $'100 100 1000\n1000 128 400\n10000 256 100'
}
train() { # <config> <run> [key=value ...]
[[ -e "${OUT}/runs/$2/last.pt" ]] ||
"${G4C[@]}" train "$1" data.cache="${D}" "${EVAL}" run_dir="${OUT}/runs/$2" "${@:3}"
}
probes() { # <run>: k-NN and linear probes of its last epoch
for n in 100 1000 10000; do
local out="${OUT}/probes/$1/n${n}.json"
[[ -e "${out}" ]] || "${G4C[@]}" probe configs/gamgam_jepa_finetune.yaml split=validation \
model.pretrained="${OUT}/runs/$1/last.pt" data.train_events="${n}" "${EVAL}" output="${out}"
done
}
# Supervised, from scratch and from Pretrained_GNN.
steps configs/gamgam.yaml gamgam_steps
steps configs/gamgam_finetune.yaml gamgam_finetune_steps
steps configs/gamgam_transformer.yaml gamgam_transformer_steps
# Label-free pretraining on Delphes, probes, and fine-tuning.
for objective in jepa masked contrastive; do
train "configs/delphes_${objective}.yaml" "delphes_${objective}" \
data.train_events=4000000 train.max_epochs=8
probes "delphes_${objective}"
done
train configs/delphes_contrastive.yaml delphes_contrastive_noscale \
data.train_events=4000000 train.max_epochs=4 model.scale_jitter=0.0
probes delphes_contrastive_noscale
for n in 100 1000 10000; do # the random encoder
out="${OUT}/probes/random/n${n}.json"
[[ -e "${out}" ]] || "${G4C[@]}" probe configs/gamgam_jepa_finetune.yaml split=validation \
model.pretrained=null data.train_events="${n}" "${EVAL}" output="${out}"
done
for objective in jepa masked; do
steps configs/gamgam_jepa_finetune.yaml "gamgam_${objective}_finetune_steps" \
model.pretrained="${OUT}/runs/delphes_${objective}/last.pt"
done
# In-context models as released, and TabPFN with other inputs.
sweep configs/gamgam_tabpfn.yaml
sweep configs/gamgam_tabicl.yaml
sweep configs/gamgam_tabdpt.yaml
for config in configs/gamgam_tabpfn.yaml configs/gamgam_tabicl.yaml; do
uv run --all-extras python scripts/label_efficiency.py "${config}" --events 50000 \
--out "${OUT}/label_efficiency" --override "${EVAL}"
done
sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_physics model.relative_phi=true model.pair_objects=4
sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_pgnn model.tabular_inputs=both \
model.backbone=edge_network model.hid_size=64 model.n_layers=4 model.pretrained="${PGNN}"
for objective in jepa masked; do
sweep configs/gamgam_tabpfn.yaml name="gamgam_tabpfn_${objective}" model.tabular_inputs=both \
"${TRANSFORMER[@]}" model.pretrained="${OUT}/runs/delphes_${objective}/last.pt"
done
# Meta-training on Delphes tasks. One step at learning rate 0 keeps the LoRA
# updates at zero: the released network by its direct forward pass.
train configs/delphes_tabpfn.yaml delphes_tabpfn_zero data.eval_events=2000 \
train.max_epochs=1 train.learning_rate=0.0 task.steps_per_epoch=1 task.validation_steps=1
sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_direct checkpoint="${OUT}/runs/delphes_tabpfn_zero/best.pt"
train configs/delphes_tabpfn.yaml delphes_tabpfn_all12 train.max_epochs=6
sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_meta_all12 checkpoint="${OUT}/runs/delphes_tabpfn_all12/best.pt"
train configs/delphes_pfn.yaml delphes_pfn_all12 train.max_epochs=25
sweep configs/gamgam_pfn.yaml name=gamgam_pfn_all12 checkpoint="${OUT}/runs/delphes_pfn_all12/best.pt"
# The supervised runs' kept epochs and the leading in-context runs on all
# validation events.
full() { # <run> [<config> key=value ...]; by default the run's own config
local n dir
for n in 100 1000 10000; do
dir="${OUT}/label_efficiency/$1/n${n}"
[[ -e "${dir}/validation_full_metrics.json" ]] ||
"${G4C[@]}" evaluate "${2:-${dir}/config.yaml}" split=validation \
data.train_events="${n}" data.eval_events=null run_dir="${dir}" \
output="${dir}/validation_full_metrics.json" "${@:3}"
done
}
for run in gamgam_steps gamgam_finetune_steps gamgam_transformer_steps \
gamgam_jepa_finetune_steps gamgam_masked_finetune_steps; do
full "${run}"
done
full gamgam_tabpfn configs/gamgam_tabpfn.yaml
full gamgam_tabpfn_pgnn configs/gamgam_tabpfn.yaml model.tabular_inputs=both \
model.backbone=edge_network model.hid_size=64 model.n_layers=4 model.pretrained="${PGNN}"
full gamgam_tabpfn_masked configs/gamgam_tabpfn.yaml model.tabular_inputs=both \
"${TRANSFORMER[@]}" model.pretrained="${OUT}/runs/delphes_masked/last.pt"
|