Hugging Face's logo

amorarc
/
func-jepa

PyTorch
Safetensors
symbolic-mathematics
jepa
representation-learning
symbolic-regression
function-encoder
Model card Files
xet
 Community
func-jepa
File size: 2,138 Bytes
522b012
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
{
 "step": 38000,
 "config": {
  "run_name": "e05",
  "data": "data/v05",
  "seed": 0,
  "model": {
   "hidden": 512,
   "layers": 10,
   "heads": 8,
   "intermediate": 2176,
   "max_pos": 130,
   "dropout": 0.0,
   "latent_dim": 256,
   "pooling": "cls",
   "A_init": "identity",
   "A_inv": false,
   "deriv": "factored",
   "deriv_hidden": 1024,
   "gain_hidden": 256,
   "proj_init": 1.0,
   "init": "scaled_normal",
   "init_std": 0.02,
   "vocab_size": null,
   "magnitude": "computed"
  },
  "train": {
   "batch_size": 384,
   "steps": 40000,
   "lr": 0.0003,
   "A_lr": 0.001,
   "weight_decay": 0.01,
   "warmup": 700,
   "min_lr_frac": 0.1,
   "grad_clip": 1.0,
   "bf16": true,
   "mask_p": 0.15,
   "log_every": 50,
   "eval_every": 1000,
   "ckpt_every": 1000,
   "batch_size_after_mlm": 384,
   "grad_ckpt": true,
   "len_buckets": 4,
   "p_group": 0.15,
   "p_sweep": 0.25,
   "p_combo": 0.15,
   "p_single": 0.5,
   "combo_coefs": [
    1,
    -1,
    2,
    -2,
    3,
    -3,
    5,
    -5,
    10,
    -10,
    "1/2",
    "-1/2"
   ],
   "combo_min_rel": 0.1
  },
  "loss": {
   "w_mlm": 0.0,
   "w_jepa": 1.0,
   "w_gram": 1.0,
   "gram_mode": "cosine",
   "jepa_start": 1000,
   "jepa_ramp": 1000,
   "jepa_enc_grad": 1.0,
   "norm_eps": 0.05,
   "jepa_stopgrad": true,
   "combine": "sum",
   "jepa_max_ratio": 0.5,
   "lam_K": 0.1,
   "guard": false,
   "w_ijepa": 0.0,
   "w_inv_reg": 0.0,
   "mlm_hold": 0,
   "mlm_anneal": 0,
   "w_lin": 1.0,
   "lin_ramp": 1000,
   "lin_max_ratio": 1.0,
   "w_jepa_d2": 0.0,
   "w_jepa_pow": 0.0,
   "gram_inner": "H1",
   "sobolev_lambda": 0.5,
   "lambda_ramp": 0,
   "magnitude_ramp": 0,
   "w_vnorm": 0.05,
   "vnorm_ramp": 0,
   "fprime_nograd": false,
   "w_cent": 1.5,
   "w_var": 1.0,
   "var_gamma": 0.5,
   "w_lrvar": 0.0,
   "lin_start": 1000,
   "jepa_norm": "split",
   "jepa_gain_w": 1.0
  }
 },
 "K": null,
 "r": 0.5,
 "sobolev_lambda": 0.5,
 "mag_t": 1.0,
 "component": "encoder_05",
 "description": "encoder + derivative operator D on [0, 5] (paper: E_[0,5])",
 "source_checkpoint": "runs/e05/best.pt",
 "domain": [
  0.0,
  5.0
 ],
 "format": "func-jepa-1"
}