Hugging Face's logo

AdaptiveChunking
/
pilot_D_s1

PyTorch
Safetensors
dynamic-chunking
tokenization
hnet
byte-level
beetle-hnet
legacy-trainer
Model card Files
xet
 Community
pilot_D_s1
File size: 2,398 Bytes
008a38f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
{
  "run": "pilot_D_s1",
  "step": 6000,
  "model_config": {
    "vocab_size": 256,
    "d_enc": 256,
    "d_main": 512,
    "n_enc": 2,
    "n_dec": 2,
    "n_main": 6,
    "n_heads_enc": 4,
    "n_heads_main": 8,
    "seq_len": 1024,
    "dropout": 0.0,
    "target_ratio": 6.0,
    "max_chunk_mult": 3.0,
    "dechunk": "ema",
    "router_noise": 0.0,
    "rope_base": 10000.0
  },
  "tied_weights": {
    "head.weight": "emb.weight"
  },
  "train_config": {
    "condition": "D",
    "seed": 1,
    "out_dir": null,
    "run_name": "pilot_D_s1",
    "corpus": "corpus",
    "langs": null,
    "seq_len": 1024,
    "batch_size": 32,
    "grad_accum": 1,
    "steps": 6000,
    "lr": 0.0003,
    "min_lr_frac": 0.1,
    "warmup": 300,
    "weight_decay": 0.1,
    "grad_clip": 1.0,
    "betas": [
      0.9,
      0.95
    ],
    "router_lr_mult": 1.0,
    "d_enc": 256,
    "d_main": 512,
    "n_enc": 2,
    "n_dec": 2,
    "n_main": 6,
    "n_heads_enc": 4,
    "n_heads_main": 8,
    "dechunk": "ema",
    "target_ratio": 6.0,
    "alpha": 0.03,
    "alpha_parity_cap": 0.06,
    "global_anchor": 0.0,
    "premium_clip": 4.0,
    "premium_kind": "flores_per_sent",
    "mixture": null,
    "doc_filters": null,
    "tau1": 0.4,
    "tau2": 0.7,
    "freeze_scope": "router",
    "ckpt_every": 200,
    "eval_every": 200,
    "log_every": 50,
    "probe_sents": 128,
    "eval_batches": 8,
    "keep_ckpt": false,
    "amp": "bf16",
    "compile": false,
    "cpc_decay": 0.9,
    "device": "cuda",
    "log_spaced_probe": true,
    "probe_bpb_batches": 4,
    "dump_reps": true,
    "dump_reps_thresholds": [
      0.3,
      0.4,
      0.5,
      0.6,
      0.7
    ],
    "router_noise": 0.0,
    "genreg_target_entropy": null,
    "genreg_eta": 0.05
  },
  "probe_agg": {
    "gini_cpc": 0.19931583660905217,
    "gini_cps": 0.07285079635143443,
    "bpic": 10.88277328796257,
    "worst_lang": "zh",
    "cpc_parity_vs_en": {
      "en": 1.0,
      "de": 0.9606295246240699,
      "vi": 1.3587862507309212,
      "tr": 0.9946425759826852,
      "fi": 0.8716158469228481,
      "ru": 0.9186823295794272,
      "el": 0.7862724474330963,
      "hi": 0.8917088902824416,
      "ta": 0.8041269219926616,
      "th": 0.9889663604926271,
      "zh": 3.1580155984998517,
      "my": 0.9464667936802198
    }
  },
  "source": "AdaptiveChunking/hnet-chunking-pilots/pilot_D_s1/model.safetensors"
}