Download scripts/label_efficiency_study.sh from HWresearch/GNN4Colliders: direct link, hf CLI and curl.
- Browser
- Download file 6.3 kB
-
https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/scripts/label_efficiency_study.sh
- Command line
-
hf download hf://HWresearch/GNN4Colliders/scripts/label_efficiency_study.sh
-
curl -L -o label_efficiency_study.sh https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/scripts/label_efficiency_study.sh
6.3 kB
| # Every run of the label-efficiency study in docs/design.md ("Label | |
| # efficiency on GamGam"), with its exact overrides, one after another on one | |
| # GPU (the study ran them as parallel jobs, 3.5 hours on 4 GPUs): | |
| # bash scripts/label_efficiency_study.sh | |
| # OUT (default outputs/study) holds runs/, label_efficiency/ and probes/; | |
| # DELPHES is the Delphes event cache. Restartable: finished runs are skipped. | |
| # TabPFN-3.5 needs its weights in TABPFN_MODEL_CACHE_DIR (docs/usage.md). | |
| # Prepare the Delphes and GamGam event caches first. | |
| set -euo pipefail | |
| cd "$(dirname "$0")/.." | |
| OUT="${OUT:-outputs/study}" | |
| D="${DELPHES:-${SCRATCH:-outputs}/gnn4colliders/delphes/events.pt}" | |
| export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}" | |
| export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}" | |
| export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}" | |
| export TABPFN_MODEL_CACHE_DIR="${TABPFN_MODEL_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/tabpfn}" | |
| G4C=(uv run --all-extras gnn4colliders) | |
| EVAL=data.eval_events=20000 | |
| PGNN=Pretrained_GNN/multiclass_pretrained_model_12/model_epoch_71.pt | |
| # The encoder sizes of configs/delphes_{jepa,masked}.yaml. | |
| TRANSFORMER=(model.backbone=transformer model.hid_size=128 model.n_layers=4 | |
| model.n_heads=8 model.ffn_size=512 model.pair_bias=true) | |
| sweep() { # <config> [key=value ...]: N = 100, 1000, 10000 | |
| uv run --all-extras python scripts/label_efficiency.py "$1" \ | |
| --events 100 1000 10000 --out "${OUT}/label_efficiency" --override "${EVAL}" "${@:2}" | |
| } | |
| steps() { # <config> <name> [key=value ...]: a fixed number of steps per N | |
| local n batch epochs | |
| while read -r n batch epochs; do | |
| uv run --all-extras python scripts/label_efficiency.py "$1" --events "${n}" \ | |
| --out "${OUT}/label_efficiency" --override name="$2" "${EVAL}" \ | |
| data.batch_size="${batch}" train.max_epochs="${epochs}" train.early_stopping=null "${@:3}" | |
| done <<< $'100 100 1000\n1000 128 400\n10000 256 100' | |
| } | |
| train() { # <config> <run> [key=value ...] | |
| [[ -e "${OUT}/runs/$2/last.pt" ]] || | |
| "${G4C[@]}" train "$1" data.cache="${D}" "${EVAL}" run_dir="${OUT}/runs/$2" "${@:3}" | |
| } | |
| probes() { # <run>: k-NN and linear probes of its last epoch | |
| for n in 100 1000 10000; do | |
| local out="${OUT}/probes/$1/n${n}.json" | |
| [[ -e "${out}" ]] || "${G4C[@]}" probe configs/gamgam_jepa_finetune.yaml split=validation \ | |
| model.pretrained="${OUT}/runs/$1/last.pt" data.train_events="${n}" "${EVAL}" output="${out}" | |
| done | |
| } | |
| # Supervised, from scratch and from Pretrained_GNN. | |
| steps configs/gamgam.yaml gamgam_steps | |
| steps configs/gamgam_finetune.yaml gamgam_finetune_steps | |
| steps configs/gamgam_transformer.yaml gamgam_transformer_steps | |
| # Label-free pretraining on Delphes, probes, and fine-tuning. | |
| for objective in jepa masked contrastive; do | |
| train "configs/delphes_${objective}.yaml" "delphes_${objective}" \ | |
| data.train_events=4000000 train.max_epochs=8 | |
| probes "delphes_${objective}" | |
| done | |
| train configs/delphes_contrastive.yaml delphes_contrastive_noscale \ | |
| data.train_events=4000000 train.max_epochs=4 model.scale_jitter=0.0 | |
| probes delphes_contrastive_noscale | |
| for n in 100 1000 10000; do # the random encoder | |
| out="${OUT}/probes/random/n${n}.json" | |
| [[ -e "${out}" ]] || "${G4C[@]}" probe configs/gamgam_jepa_finetune.yaml split=validation \ | |
| model.pretrained=null data.train_events="${n}" "${EVAL}" output="${out}" | |
| done | |
| for objective in jepa masked; do | |
| steps configs/gamgam_jepa_finetune.yaml "gamgam_${objective}_finetune_steps" \ | |
| model.pretrained="${OUT}/runs/delphes_${objective}/last.pt" | |
| done | |
| # In-context models as released, and TabPFN with other inputs. | |
| sweep configs/gamgam_tabpfn.yaml | |
| sweep configs/gamgam_tabicl.yaml | |
| sweep configs/gamgam_tabdpt.yaml | |
| for config in configs/gamgam_tabpfn.yaml configs/gamgam_tabicl.yaml; do | |
| uv run --all-extras python scripts/label_efficiency.py "${config}" --events 50000 \ | |
| --out "${OUT}/label_efficiency" --override "${EVAL}" | |
| done | |
| sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_physics model.relative_phi=true model.pair_objects=4 | |
| sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_pgnn model.tabular_inputs=both \ | |
| model.backbone=edge_network model.hid_size=64 model.n_layers=4 model.pretrained="${PGNN}" | |
| for objective in jepa masked; do | |
| sweep configs/gamgam_tabpfn.yaml name="gamgam_tabpfn_${objective}" model.tabular_inputs=both \ | |
| "${TRANSFORMER[@]}" model.pretrained="${OUT}/runs/delphes_${objective}/last.pt" | |
| done | |
| # Meta-training on Delphes tasks. One step at learning rate 0 keeps the LoRA | |
| # updates at zero: the released network by its direct forward pass. | |
| train configs/delphes_tabpfn.yaml delphes_tabpfn_zero data.eval_events=2000 \ | |
| train.max_epochs=1 train.learning_rate=0.0 task.steps_per_epoch=1 task.validation_steps=1 | |
| sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_direct checkpoint="${OUT}/runs/delphes_tabpfn_zero/best.pt" | |
| train configs/delphes_tabpfn.yaml delphes_tabpfn_all12 train.max_epochs=6 | |
| sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_meta_all12 checkpoint="${OUT}/runs/delphes_tabpfn_all12/best.pt" | |
| train configs/delphes_pfn.yaml delphes_pfn_all12 train.max_epochs=25 | |
| sweep configs/gamgam_pfn.yaml name=gamgam_pfn_all12 checkpoint="${OUT}/runs/delphes_pfn_all12/best.pt" | |
| # The supervised runs' kept epochs and the leading in-context runs on all | |
| # validation events. | |
| full() { # <run> [<config> key=value ...]; by default the run's own config | |
| local n dir | |
| for n in 100 1000 10000; do | |
| dir="${OUT}/label_efficiency/$1/n${n}" | |
| [[ -e "${dir}/validation_full_metrics.json" ]] || | |
| "${G4C[@]}" evaluate "${2:-${dir}/config.yaml}" split=validation \ | |
| data.train_events="${n}" data.eval_events=null run_dir="${dir}" \ | |
| output="${dir}/validation_full_metrics.json" "${@:3}" | |
| done | |
| } | |
| for run in gamgam_steps gamgam_finetune_steps gamgam_transformer_steps \ | |
| gamgam_jepa_finetune_steps gamgam_masked_finetune_steps; do | |
| full "${run}" | |
| done | |
| full gamgam_tabpfn configs/gamgam_tabpfn.yaml | |
| full gamgam_tabpfn_pgnn configs/gamgam_tabpfn.yaml model.tabular_inputs=both \ | |
| model.backbone=edge_network model.hid_size=64 model.n_layers=4 model.pretrained="${PGNN}" | |
| full gamgam_tabpfn_masked configs/gamgam_tabpfn.yaml model.tabular_inputs=both \ | |
| "${TRANSFORMER[@]}" model.pretrained="${OUT}/runs/delphes_masked/last.pt" | |