KitsuVp commited on
Commit
d6370f5
·
verified ·
1 Parent(s): e93dbf7

Update configuration_neollm.py

Browse files
Files changed (1) hide show
  1. configuration_neollm.py +29 -2
configuration_neollm.py CHANGED
@@ -251,6 +251,15 @@ class NeoLLMConfig(PretrainedConfig):
251
  Enable SiameseNorm as a two-stream residual topology using RMSNorm
252
  only. When ``True``, decoder layers instantiate the SiameseNorm RMS
253
  stream norms instead of the standard decoder pre-norm modules.
 
 
 
 
 
 
 
 
 
254
  siamese_normalized_input (:obj:`bool`, *optional*, defaults to ``True``):
255
  Apply an additional RMSNorm to the fused SiameseNorm input before
256
  the shared Attention and MLP blocks. This corresponds to the
@@ -731,6 +740,8 @@ class NeoLLMConfig(PretrainedConfig):
731
  use_lns=True,
732
  use_gpas=False,
733
  use_siamesenorm=True,
 
 
734
  siamese_normalized_input=True,
735
  siamese_depth_scaling=True,
736
  siamese_attn_x_scale_init=1.0,
@@ -746,13 +757,13 @@ class NeoLLMConfig(PretrainedConfig):
746
  generator_krank=64,
747
  # ── Leviathan-JTok / JTok-M continuous modulation ─────────────────
748
  use_jtok=True,
749
- use_jtokm=False,
750
  jtok_num_modes=4,
751
  jtok_num_experts=5,
752
  jtok_top_k=2,
753
  jtok_norm_eps=1e-6,
754
  jtok_aux_loss_weight=1e-4,
755
- jtok_kernel_backend="torch",
756
  # ── Hadamard output projection (Aggarwal & Kumar, 2026) ───────────
757
  use_hadamard_o_proj=True,
758
  # ── PolyNorm exclusivity ──────────────────────────────────────────
@@ -1311,6 +1322,22 @@ class NeoLLMConfig(PretrainedConfig):
1311
  self.use_lns = bool(use_lns)
1312
  self.use_gpas = bool(use_gpas)
1313
  self.use_siamesenorm = bool(use_siamesenorm)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1314
  self.siamese_normalized_input = bool(siamese_normalized_input)
1315
  self.siamese_depth_scaling = bool(siamese_depth_scaling)
1316
  self.siamese_attn_x_scale_init = float(siamese_attn_x_scale_init)
 
251
  Enable SiameseNorm as a two-stream residual topology using RMSNorm
252
  only. When ``True``, decoder layers instantiate the SiameseNorm RMS
253
  stream norms instead of the standard decoder pre-norm modules.
254
+ use_dagformer (:obj:`bool`, *optional*, defaults to ``False``):
255
+ Add causal, signed depth routing for Q/K/V and residual reads.
256
+ Requires JTok and JTok-M disabled; preserves the embedding producer.
257
+ dagformer (:obj:`dict`, *optional*):
258
+ Grouped overrides: predictor_dim=256, predictor_layers=2,
259
+ predictor_heads=4, local_dim=128, input_source="tokens".
260
+ input_source="embeddings" shares the composed backbone embeddings,
261
+ including Leviathan when active, instead of a separate token table.
262
+ Kernel shapes derive from actual tensors; no batch tuning required.
263
  siamese_normalized_input (:obj:`bool`, *optional*, defaults to ``True``):
264
  Apply an additional RMSNorm to the fused SiameseNorm input before
265
  the shared Attention and MLP blocks. This corresponds to the
 
740
  use_lns=True,
741
  use_gpas=False,
742
  use_siamesenorm=True,
743
+ use_dagformer=False,
744
+ dagformer=None,
745
  siamese_normalized_input=True,
746
  siamese_depth_scaling=True,
747
  siamese_attn_x_scale_init=1.0,
 
757
  generator_krank=64,
758
  # ── Leviathan-JTok / JTok-M continuous modulation ─────────────────
759
  use_jtok=True,
760
+ use_jtokm=True,
761
  jtok_num_modes=4,
762
  jtok_num_experts=5,
763
  jtok_top_k=2,
764
  jtok_norm_eps=1e-6,
765
  jtok_aux_loss_weight=1e-4,
766
+ jtok_kernel_backend="triton",
767
  # ── Hadamard output projection (Aggarwal & Kumar, 2026) ───────────
768
  use_hadamard_o_proj=True,
769
  # ── PolyNorm exclusivity ──────────────────────────────────────────
 
1322
  self.use_lns = bool(use_lns)
1323
  self.use_gpas = bool(use_gpas)
1324
  self.use_siamesenorm = bool(use_siamesenorm)
1325
+ self.use_dagformer = bool(use_dagformer)
1326
+ if use_dagformer and (use_jtok or use_jtokm):
1327
+ raise ValueError("DAGFormer requires use_jtok=False and use_jtokm=False; choose one method.")
1328
+ dag_defaults = dict(predictor_dim=256, predictor_layers=2, predictor_heads=4,
1329
+ local_dim=128, input_source="tokens")
1330
+ if dagformer is not None:
1331
+ if not isinstance(dagformer, dict) or set(dagformer) - set(dag_defaults):
1332
+ raise ValueError("dagformer accepts predictor_dim, predictor_layers, predictor_heads, local_dim and input_source")
1333
+ dag_defaults.update(dagformer)
1334
+ if (not isinstance(dag_defaults["input_source"], str)
1335
+ or dag_defaults["input_source"] not in {"tokens", "embeddings"}):
1336
+ raise ValueError("dagformer.input_source must be tokens or embeddings")
1337
+ if (any(type(value) is not int or value < 1 for key,value in dag_defaults.items() if key != "input_source")
1338
+ or dag_defaults["predictor_dim"] % dag_defaults["predictor_heads"]):
1339
+ raise ValueError("DAGFormer dimensions must be positive and predictor_dim divisible by heads")
1340
+ self.dagformer = dag_defaults
1341
  self.siamese_normalized_input = bool(siamese_normalized_input)
1342
  self.siamese_depth_scaling = bool(siamese_depth_scaling)
1343
  self.siamese_attn_x_scale_init = float(siamese_attn_x_scale_init)