File size: 6,295 Bytes
9b7b80d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
#!/bin/bash
# Every run of the label-efficiency study in docs/design.md ("Label
# efficiency on GamGam"), with its exact overrides, one after another on one
# GPU (the study ran them as parallel jobs, 3.5 hours on 4 GPUs):
#   bash scripts/label_efficiency_study.sh
# OUT (default outputs/study) holds runs/, label_efficiency/ and probes/;
# DELPHES is the Delphes event cache. Restartable: finished runs are skipped.
# TabPFN-3.5 needs its weights in TABPFN_MODEL_CACHE_DIR (docs/usage.md).
# Prepare the Delphes and GamGam event caches first.

set -euo pipefail
cd "$(dirname "$0")/.."
OUT="${OUT:-outputs/study}"
D="${DELPHES:-${SCRATCH:-outputs}/gnn4colliders/delphes/events.pt}"
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}"
export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}"
export TABPFN_MODEL_CACHE_DIR="${TABPFN_MODEL_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/tabpfn}"
G4C=(uv run --all-extras gnn4colliders)
EVAL=data.eval_events=20000
PGNN=Pretrained_GNN/multiclass_pretrained_model_12/model_epoch_71.pt
# The encoder sizes of configs/delphes_{jepa,masked}.yaml.
TRANSFORMER=(model.backbone=transformer model.hid_size=128 model.n_layers=4
  model.n_heads=8 model.ffn_size=512 model.pair_bias=true)

sweep() {  # <config> [key=value ...]: N = 100, 1000, 10000
  uv run --all-extras python scripts/label_efficiency.py "$1" \
    --events 100 1000 10000 --out "${OUT}/label_efficiency" --override "${EVAL}" "${@:2}"
}
steps() {  # <config> <name> [key=value ...]: a fixed number of steps per N
  local n batch epochs
  while read -r n batch epochs; do
    uv run --all-extras python scripts/label_efficiency.py "$1" --events "${n}" \
      --out "${OUT}/label_efficiency" --override name="$2" "${EVAL}" \
      data.batch_size="${batch}" train.max_epochs="${epochs}" train.early_stopping=null "${@:3}"
  done <<< $'100 100 1000\n1000 128 400\n10000 256 100'
}
train() {  # <config> <run> [key=value ...]
  [[ -e "${OUT}/runs/$2/last.pt" ]] ||
    "${G4C[@]}" train "$1" data.cache="${D}" "${EVAL}" run_dir="${OUT}/runs/$2" "${@:3}"
}
probes() {  # <run>: k-NN and linear probes of its last epoch
  for n in 100 1000 10000; do
    local out="${OUT}/probes/$1/n${n}.json"
    [[ -e "${out}" ]] || "${G4C[@]}" probe configs/gamgam_jepa_finetune.yaml split=validation \
      model.pretrained="${OUT}/runs/$1/last.pt" data.train_events="${n}" "${EVAL}" output="${out}"
  done
}

# Supervised, from scratch and from Pretrained_GNN.
steps configs/gamgam.yaml gamgam_steps
steps configs/gamgam_finetune.yaml gamgam_finetune_steps
steps configs/gamgam_transformer.yaml gamgam_transformer_steps

# Label-free pretraining on Delphes, probes, and fine-tuning.
for objective in jepa masked contrastive; do
  train "configs/delphes_${objective}.yaml" "delphes_${objective}" \
    data.train_events=4000000 train.max_epochs=8
  probes "delphes_${objective}"
done
train configs/delphes_contrastive.yaml delphes_contrastive_noscale \
  data.train_events=4000000 train.max_epochs=4 model.scale_jitter=0.0
probes delphes_contrastive_noscale
for n in 100 1000 10000; do  # the random encoder
  out="${OUT}/probes/random/n${n}.json"
  [[ -e "${out}" ]] || "${G4C[@]}" probe configs/gamgam_jepa_finetune.yaml split=validation \
    model.pretrained=null data.train_events="${n}" "${EVAL}" output="${out}"
done
for objective in jepa masked; do
  steps configs/gamgam_jepa_finetune.yaml "gamgam_${objective}_finetune_steps" \
    model.pretrained="${OUT}/runs/delphes_${objective}/last.pt"
done

# In-context models as released, and TabPFN with other inputs.
sweep configs/gamgam_tabpfn.yaml
sweep configs/gamgam_tabicl.yaml
sweep configs/gamgam_tabdpt.yaml
for config in configs/gamgam_tabpfn.yaml configs/gamgam_tabicl.yaml; do
  uv run --all-extras python scripts/label_efficiency.py "${config}" --events 50000 \
    --out "${OUT}/label_efficiency" --override "${EVAL}"
done
sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_physics model.relative_phi=true model.pair_objects=4
sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_pgnn model.tabular_inputs=both \
  model.backbone=edge_network model.hid_size=64 model.n_layers=4 model.pretrained="${PGNN}"
for objective in jepa masked; do
  sweep configs/gamgam_tabpfn.yaml name="gamgam_tabpfn_${objective}" model.tabular_inputs=both \
    "${TRANSFORMER[@]}" model.pretrained="${OUT}/runs/delphes_${objective}/last.pt"
done

# Meta-training on Delphes tasks. One step at learning rate 0 keeps the LoRA
# updates at zero: the released network by its direct forward pass.
train configs/delphes_tabpfn.yaml delphes_tabpfn_zero data.eval_events=2000 \
  train.max_epochs=1 train.learning_rate=0.0 task.steps_per_epoch=1 task.validation_steps=1
sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_direct checkpoint="${OUT}/runs/delphes_tabpfn_zero/best.pt"
train configs/delphes_tabpfn.yaml delphes_tabpfn_all12 train.max_epochs=6
sweep configs/gamgam_tabpfn.yaml name=gamgam_tabpfn_meta_all12 checkpoint="${OUT}/runs/delphes_tabpfn_all12/best.pt"
train configs/delphes_pfn.yaml delphes_pfn_all12 train.max_epochs=25
sweep configs/gamgam_pfn.yaml name=gamgam_pfn_all12 checkpoint="${OUT}/runs/delphes_pfn_all12/best.pt"

# The supervised runs' kept epochs and the leading in-context runs on all
# validation events.
full() {  # <run> [<config> key=value ...]; by default the run's own config
  local n dir
  for n in 100 1000 10000; do
    dir="${OUT}/label_efficiency/$1/n${n}"
    [[ -e "${dir}/validation_full_metrics.json" ]] ||
      "${G4C[@]}" evaluate "${2:-${dir}/config.yaml}" split=validation \
      data.train_events="${n}" data.eval_events=null run_dir="${dir}" \
      output="${dir}/validation_full_metrics.json" "${@:3}"
  done
}
for run in gamgam_steps gamgam_finetune_steps gamgam_transformer_steps \
  gamgam_jepa_finetune_steps gamgam_masked_finetune_steps; do
  full "${run}"
done
full gamgam_tabpfn configs/gamgam_tabpfn.yaml
full gamgam_tabpfn_pgnn configs/gamgam_tabpfn.yaml model.tabular_inputs=both \
  model.backbone=edge_network model.hid_size=64 model.n_layers=4 model.pretrained="${PGNN}"
full gamgam_tabpfn_masked configs/gamgam_tabpfn.yaml model.tabular_inputs=both \
  "${TRANSFORMER[@]}" model.pretrained="${OUT}/runs/delphes_masked/last.pt"