#!/bin/bash # Every run of the label-efficiency study in docs/design.md ("Label # efficiency on GamGam"), with its exact overrides, one after another on one # GPU (the study ran them as parallel jobs, 3.5 hours on 4 GPUs): # bash scripts/label_efficiency_study.sh # OUT (default outputs/study) holds runs/, label_efficiency/ and probes/; # DELPHES is the Delphes event cache. Restartable: finished runs are skipped. # TabPFN-3.5 needs its weights in TABPFN_MODEL_CACHE_DIR (docs/usage.md). # Prepare the Delphes and GamGam event caches first. set -euo pipefail cd "$(dirname "$0")/.." OUT="${OUT:-outputs/study}" D="${DELPHES:-${SCRATCH:-outputs}/gnn4colliders/delphes/events.pt}" export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}" export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}" export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}" export TABPFN_MODEL_CACHE_DIR="${TABPFN_MODEL_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/tabpfn}" G4C=(uv run --all-extras gnn4colliders) EVAL=data.eval_events=20000 PGNN=Pretrained_GNN/multiclass_pretrained_model_12/model_epoch_71.pt # The encoder sizes of configs/delphes_{jepa,masked}.yaml. TRANSFORMER=(model.backbone=transformer model.hid_size=128 model.n_layers=4 model.n_heads=8 model.ffn_size=512 model.pair_bias=true) sweep() { # [key=value ...]: N = 100, 1000, 10000 uv run --all-extras python scripts/label_efficiency.py "$1" \ --events 100 1000 10000 --out "${OUT}/label_efficiency" --override "${EVAL}" "${@:2}" } steps() { # [key=value ...]: a fixed number of steps per N local n batch epochs while read -r n batch epochs; do uv run --all-extras python scripts/label_efficiency.py "$1" --events "${n}" \ --out "${OUT}/label_efficiency" --override name="$2" "${EVAL}" \ data.batch_size="${batch}" train.max_epochs="${epochs}" train.early_stopping=null "${@:3}" done <<< $'100 100 1000\n1000 128 400\n10000 256 100' } train() { # [key=value ...] [[ -e "${OUT}/runs/$2/last.pt" ]] || "${G4C[@]}" train "$1" data.cache="${D}" "${EVAL}" run_dir="${OUT}/runs/$2" "${@:3}" } probes() { # : k-NN and linear probes of its last epoch for n in 100 1000 10000; do local out="${OUT}/probes/$1/n${n}.json" [[ -e "${out}" ]] || "${G4C[@]}" probe configs/gamgam_jepa_finetune.yaml split=validation \ model.pretrained="${OUT}/runs/$1/last.pt" data.train_events="${n}" "${EVAL}" output="${out}" done } # Supervised, from scratch and from Pretrained_GNN. steps configs/gamgam.yaml gamgam_steps steps configs/gamgam_finetune.yaml gamgam_finetune_steps steps configs/gamgam_transformer.yaml gamgam_transformer_steps # Label-free pretraining on Delphes, probes, and fine-tuning. for objective in jepa masked contrastive; do train "configs/delphes_${objective}.yaml" "delphes_${objective}" \ data.train_events=4000000 train.max_epochs=8 probes "delphes_${objective}" done train configs/delphes_contrastive.yaml delphes_contrastive_noscale \ data.train_events=4000000 train.max_epochs=4 model.scale_jitter=0.0 probes delphes_contrastive_noscale for n in 100 1000 10000; do # the random encoder out="${OUT}/probes/random/n${n}.json" [[ -e "${out}" ]] || "${G4C[@]}" probe configs/gamgam_jepa_finetune.yaml split=validation \ model.pretrained=null data.train_events="${n}" "${EVAL}" output="${out}" done for objective in jepa masked; do steps configs/gamgam_jepa_finetune.yaml "gamgam_${objective}_finetune_steps" \ model.pretrained="${OUT}/runs/delphes_${objective}/last.pt" done # In-context models as released, and TabPFN with other inputs. sweep configs/gamgam_tabpfn.yaml sweep configs/gamgam_tabicl.yaml sweep configs/gamgam_tabdpt.yaml for config in configs/gamgam_tabpfn.yaml configs/gamgam_tabicl.yaml; do uv run --all-extras python scripts/label_efficiency.py "${config}" --events 50000 \ --out "${OUT}/label_efficiency" --override "${EVAL}" done sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_physics model.relative_phi=true model.pair_objects=4 sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_pgnn model.tabular_inputs=both \ model.backbone=edge_network model.hid_size=64 model.n_layers=4 model.pretrained="${PGNN}" for objective in jepa masked; do sweep configs/gamgam_tabpfn.yaml name="gamgam_tabpfn_${objective}" model.tabular_inputs=both \ "${TRANSFORMER[@]}" model.pretrained="${OUT}/runs/delphes_${objective}/last.pt" done # Meta-training on Delphes tasks. One step at learning rate 0 keeps the LoRA # updates at zero: the released network by its direct forward pass. train configs/delphes_tabpfn.yaml delphes_tabpfn_zero data.eval_events=2000 \ train.max_epochs=1 train.learning_rate=0.0 task.steps_per_epoch=1 task.validation_steps=1 sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_direct checkpoint="${OUT}/runs/delphes_tabpfn_zero/best.pt" train configs/delphes_tabpfn.yaml delphes_tabpfn_all12 train.max_epochs=6 sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_meta_all12 checkpoint="${OUT}/runs/delphes_tabpfn_all12/best.pt" train configs/delphes_pfn.yaml delphes_pfn_all12 train.max_epochs=25 sweep configs/gamgam_pfn.yaml name=gamgam_pfn_all12 checkpoint="${OUT}/runs/delphes_pfn_all12/best.pt" # The supervised runs' kept epochs and the leading in-context runs on all # validation events. full() { # [ key=value ...]; by default the run's own config local n dir for n in 100 1000 10000; do dir="${OUT}/label_efficiency/$1/n${n}" [[ -e "${dir}/validation_full_metrics.json" ]] || "${G4C[@]}" evaluate "${2:-${dir}/config.yaml}" split=validation \ data.train_events="${n}" data.eval_events=null run_dir="${dir}" \ output="${dir}/validation_full_metrics.json" "${@:3}" done } for run in gamgam_steps gamgam_finetune_steps gamgam_transformer_steps \ gamgam_jepa_finetune_steps gamgam_masked_finetune_steps; do full "${run}" done full gamgam_tabpfn configs/gamgam_tabpfn.yaml full gamgam_tabpfn_pgnn configs/gamgam_tabpfn.yaml model.tabular_inputs=both \ model.backbone=edge_network model.hid_size=64 model.n_layers=4 model.pretrained="${PGNN}" full gamgam_tabpfn_masked configs/gamgam_tabpfn.yaml model.tabular_inputs=both \ "${TRANSFORMER[@]}" model.pretrained="${OUT}/runs/delphes_masked/last.pt"