File size: 2,687 Bytes
2de2d16
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
model:
  architecture: hnet_moe_router
  d_enc: 32
  d_main: 48
  n_enc: 1
  n_dec: 1
  n_main: 1
  n_heads_enc: 2
  n_heads_main: 2
  seq_len: 128
  dropout: 0.0
  target_ratio: 6.0
  max_chunk_mult: 3.0
  dechunk: ema
  router_noise: 0.0
  rope_base: 10000.0
  moe_router: {n_experts: 5, gate: script, expert_families: null, common: inherit,
    default_family: latin, expert_init: clone, target_ratios: null, unilid_path: null,
    unilid_window: 32, unilid_temperature: 1.0, unilid_lang_experts: null, gate_temperature: 1.0,
    balance_coef: 0.01, detach_gate_in_ratio: true}
memory:
  ali:
    budget_B: 512
    staging_capacity: 2048
    policy: utility
    value_mode: learned
    candidate_types: [surface, split, merge]
    write_rule: mean
    eta_min: 0.0
    memory_scope: persistent
    consolidate_every: 5
    max_span: 64
    gate: vector
    retrieval: true
    read_dropout: 0.1
    gate_hidden: 0
    type_dim: 16
    gate_bias_init: -2.0
    probe_seqs: 1
    probe_reps: 3
    horizon: 32
    predictor: mlp
    lambda_util: 0.1
    enabled: true
    inventory_kwargs: {min_obs: 1, min_age: 0}
    probe_mix: null
    util_batch_slots: 1024
  pkm: {n_keys: 16, topk: 4, heads: 1, d_key: null, value_lr_mult: 1.0}
objective: {name: ratio, schedule: null, alpha: 0.03, alpha_parity_cap: 0.06, global_anchor: 0.0,
  premium_clip: 4.0, premium_kind: flores_per_sent, tau1: 0.4, tau2: 0.7, freeze_scope: router,
  cpc_decay: 0.9, genreg_target_entropy: null, genreg_eta: 0.05, flex_targets: premium,
  flex_schedule: inverse_linear, flex_start_ratio: null, flex_ramp_start: 0.0, flex_ramp_end: 0.8,
  flex_freeze_at: null, flex_bits_source: flores_ngram, flex_bits_order: 5, flex_bits_anchor: en}
data:
  corpus: synthetic
  langs: [en, de, hi]
  mixture: null
  doc_filters: null
  byte_classes: null
training:
  seed: 0
  max_steps: 20
  batch_size: 4
  grad_accum: 1
  lr: 0.0003
  min_lr_frac: 0.1
  warmup: 2
  weight_decay: 0.1
  grad_clip: 1.0
  betas: [0.9, 0.95]
  router_lr_mult: 1.0
  log_every: 5
  amp: none
  compile: false
  device: cpu
checkpointing:
  run_name: smoke_moe-router-script
  runs_dir: /private/tmp/claude-501/-Users-suchirsalhan-Documents-PHD-PHD-beetle-hnet/3f003e5e-086f-4ff2-a93f-1f86ed43e9bd/scratchpad/runs
  ckpt_every: 10
  keep: false
  milestones: []
  resume: auto
  keep_schedule: log2
  keep_extra_offsets: [2, 4, 6, 8]
  hub_repo: AdaptiveChunking/{run_name}
  hub_private: false
  keep_local: false
evaluation:
  eval_every: 10
  eval_batches: 1
  probe_sents: 4
  probe_bpb_batches: 1
  log_spaced_probe: true
  dense_early_every: 0
  dense_early_until: 0
  dump_reps: false
  dump_reps_thresholds: [0.3, 0.4, 0.5, 0.6, 0.7]