Hugging Face's logo

AdaptiveChunking
/
ali_mem-split_s1

PyTorch
Safetensors
dynamic-chunking
tokenization
hnet
byte-level
beetle-hnet
Model card Files
xet
 Community
ali_mem-split_s1
File size: 2,091 Bytes
fe28acd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
model:
  architecture: ali
  d_enc: 256
  d_main: 512
  n_enc: 2
  n_dec: 2
  n_main: 6
  n_heads_enc: 4
  n_heads_main: 8
  seq_len: 1024
  dropout: 0.0
  target_ratio: 6.0
  max_chunk_mult: 3.0
  dechunk: ema
  router_noise: 0.0
  rope_base: 10000.0
memory:
  ali:
    budget_B: 65536
    staging_capacity: 262144
    policy: frequency
    value_mode: learned
    candidate_types:
    - split
    write_rule: mean
    eta_min: 0.0
    memory_scope: persistent
    consolidate_every: 50
    max_span: 64
    gate: vector
    retrieval: true
    read_dropout: 0.1
    gate_hidden: 0
    type_dim: 16
    gate_bias_init: -2.0
    probe_seqs: 2
    probe_reps: 8
    horizon: 32
    predictor: mlp
    lambda_util: 0.1
    enabled: true
    inventory_kwargs: {}
    probe_mix: null
    util_batch_slots: 1024
  pkm:
    n_keys: 256
    topk: 8
    heads: 1
    d_key: null
    value_lr_mult: 1.0
objective:
  name: ratio
  schedule: null
  alpha: 0.03
  alpha_parity_cap: 0.06
  global_anchor: 0.0
  premium_clip: 4.0
  premium_kind: flores_per_sent
  tau1: 0.4
  tau2: 0.7
  freeze_scope: router
  cpc_decay: 0.9
  genreg_target_entropy: null
  genreg_eta: 0.05
data:
  corpus: corpus
  langs:
  - en
  - tr
  - fi
  mixture: null
  doc_filters: null
  byte_classes: scratch/data/byte_classes.json
training:
  seed: 1
  max_steps: 6000
  batch_size: 32
  grad_accum: 1
  lr: 0.0003
  min_lr_frac: 0.1
  warmup: 300
  weight_decay: 0.1
  grad_clip: 1.0
  betas:
  - 0.9
  - 0.95
  router_lr_mult: 1.0
  log_every: 50
  amp: bf16
  compile: false
  device: cuda
checkpointing:
  run_name: ali_mem-split_s1
  runs_dir: runs
  ckpt_every: 200
  keep: false
  milestones:
  - 1000
  - 3000
  resume: auto
  keep_schedule: log2
  keep_extra_offsets:
  - 2
  - 4
  - 6
  - 8
  hub_repo: AdaptiveChunking/{run_name}
  hub_private: false
  keep_local: false
evaluation:
  eval_every: 200
  eval_batches: 8
  probe_sents: 128
  probe_bpb_batches: 4
  log_spaced_probe: true
  dense_early_every: 0
  dense_early_until: 0
  dump_reps: true
  dump_reps_thresholds:
  - 0.3
  - 0.4
  - 0.5
  - 0.6
  - 0.7