Hugging Face's logo

OLAResearchX
/
memoryathena-main-reader-20260922

Text Generation
PyTorch
English
memoryathena
paper-aligned
control-main
Model card Files
xet
 Community
memoryathena-main-reader-20260922
File size: 3,758 Bytes
c2338a4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
{
  "config": null,
  "condition": "transferred",
  "target_model": "mistralai/Mistral-7B-v0.3",
  "source_memory": "/scratch/project_462001050/mingyl/llm-memory-transfer/publish_bundle/models/hf_repos/olaresearchx-xmemtransfer-llama2-source-20m/memory.pt",
  "memory_config": "/scratch/project_462001050/mingyl/llm-memory-transfer/publish_bundle/models/hf_repos/olaresearchx-xmemtransfer-llama2-source-20m/memory_config.json",
  "init_adaptor": null,
  "train_generated_branch_only": false,
  "dual_reader_mode": "both",
  "max_tokens": 20000000,
  "seq_len": 2048,
  "batch_size": 1,
  "lr": 0.0003,
  "warmup_steps": 200,
  "seed": 42,
  "output_dir": "/scratch/project_462001493/mingyl/ATHENA/run/tri_advantage_formal_20260908T133625Z/reader_20m_train",
  "log_every": 50,
  "eval_every": 2000,
  "validation_max_tokens": 2000000,
  "injection_layers": "2:10",
  "adaptor_branches": 4,
  "architecture": "generative",
  "reader_type": "cross_attention",
  "generator_cue_source": "hybrid",
  "generator_num_latents": 4,
  "generator_hidden_size": 256,
  "generator_layers": 2,
  "generator_heads": 4,
  "generator_cue_window": 3,
  "generator_adaptive_router": true,
  "generator_router_hidden_size": 64,
  "generator_router_semantic_size": 64,
  "generator_source_adapter_rank": 16,
  "generator_router_expert_mode": "residual",
  "generator_fusion_type": "tri_reader",
  "memory_dim": 512,
  "canon_mode": "word_boundary",
  "corpus": "wikipedia-2021",
  "corpus_subset": "hq-dqa",
  "wikipedia2021_dataset": null,
  "wikipedia2021_source_tokenizer": "mistralai/Mistral-7B-v0.3",
  "wikipedia2021_require_tokenizer_match": true,
  "early_stopping_patience": 0,
  "grad_accum_steps": 1,
  "gradient_checkpointing": true,
  "joint_engram_generated_router": false,
  "joint_source_reader": false,
  "joint_tri_reader": false,
  "joint_tri_subset_reader": true,
  "engram_source_loss_weight": 1.0,
  "generated_source_loss_weight": 1.0,
  "ge_source_loss_weight": 1.0,
  "gh_source_loss_weight": 1.0,
  "routed_source_loss_weight": 1.0,
  "subset_routed_source_loss_weight": 1.0,
  "tri_router_distillation_weight": 0.5,
  "deterministic_engram_init_seed": 42,
  "router_lr": 0.001,
  "router_start_tokens": 2000000,
  "router_init_alpha": 0.95,
  "generated_residual_penalty": 0.0,
  "skip_final_test_eval": true,
  "adaptor_dir": "/scratch/project_462001493/mingyl/ATHENA/run/tri_memory_joint_softsubset_20260903T053716Z/tri_20m_train",
  "adaptor_checkpoint": null,
  "router_hidden_size": 64,
  "router_semantic_size": 64,
  "router_temperature": 1.0,
  "span_lengths": [
    1,
    4,
    8,
    16,
    32
  ],
  "teacher_temperature": 0.15,
  "advantage_weight_floor": 0.01,
  "advantage_weight_cap": 2.0,
  "router_thresholds": [
    0.5,
    0.55,
    0.6,
    0.65,
    0.7,
    0.75,
    0.8,
    0.85,
    0.9,
    0.95
  ],
  "advantage_thresholds": [
    0.0,
    0.01,
    0.02,
    0.05,
    0.1,
    0.2,
    0.3,
    0.5
  ],
  "candidate_space": "sources",
  "advantage_threshold": 0.0,
  "confidence_threshold": 0.5,
  "advantage_max_scale": 1.0,
  "advantage_regression_weight": 1.0,
  "advantage_confidence_weight": 0.25,
  "advantage_reader": {
    "enabled": true,
    "candidates": "sources",
    "threshold": 0.0,
    "confidence_threshold": 0.5,
    "temperature": 0.15,
    "max_scale": 1.0
  },
  "deployment_reader_mode": "tri_advantage_routed",
  "router_strategy": "wikipedia_counterfactual_tri_advantage_distillation",
  "router_training_data": "wikipedia-2021-causal-next-token-only",
  "downstream_training_examples": 0,
  "source_expert_checkpoint": "/scratch/project_462001493/mingyl/ATHENA/run/tri_memory_joint_softsubset_20260903T053716Z/tri_20m_train/adaptor_best.pt",
  "experts_frozen": true,
  "best_step": 4000
}