ATOM (atom-ai): code snapshot (100fae0 + batched-train), README with measured numbers, curated checkpoints (part 3)
969c687 verified | { | |
| "val2048_ce_train_labels": 1.2471501433821601, | |
| "val2048_ce_gen_labels": 3.489207078371329, | |
| "P(next==input | input is space/punct) train-label": 0.00022220478790635267, | |
| "P(next==input | input is space/punct) gen-label": 0.7932385270193427, | |
| "corpus P(next==input | space/punct)": 0.0, | |
| "efference_argmax_acc_all": 0.75244140625, | |
| "efference_argmax_acc_when_target_space_punct": 0.8051643192488263, | |
| "perturbation_KL_by_lag": { | |
| "normal": { | |
| "1": 7.563288368284702, | |
| "2": 4.332693288591504, | |
| "3": 0.5186640677547265, | |
| "4": 0.28677583388096356, | |
| "6": 0.13789069090411962, | |
| "8": 0.08064216026137287, | |
| "12": 0.05657446289211343, | |
| "16": 0.007557341790871826, | |
| "24": 0.0003628401454953088, | |
| "32": 8.886801428234041e-05, | |
| "48": 6.453894268853427e-07, | |
| "64": 1.0171765136198404e-08 | |
| }, | |
| "field_alone": { | |
| "1": 0.8334241298337778, | |
| "2": 0.3138229798823886, | |
| "3": 0.27025706580025144, | |
| "4": 0.2091712161588172, | |
| "6": 0.05581894152540675, | |
| "8": 0.04037778945833755, | |
| "12": 0.010343110077883466, | |
| "16": 0.0015113500040039678, | |
| "24": 0.0004311490147725176, | |
| "32": 3.1171161309314264e-05, | |
| "48": 8.987319392398756e-08, | |
| "64": 2.4325379636517635e-08 | |
| } | |
| }, | |
| "atomizer_feature_diff_by_lag": { | |
| "1": 0.15000000596046448, | |
| "2": 0.141309916973114, | |
| "4": 0.0921187698841095, | |
| "8": 0.04808657988905907, | |
| "16": 0.013103140518069267, | |
| "32": 0.0009729340672492981, | |
| "64": 5.364418029785156e-06 | |
| }, | |
| "grad_norm_wrt_atom_at_lag (lag0=current input)": { | |
| "0": 1.933836281299591, | |
| "1": 0.0, | |
| "2": 0.0, | |
| "4": 0.0, | |
| "8": 0.0, | |
| "16": 0.0, | |
| "32": 0.0, | |
| "64": 0.0 | |
| }, | |
| "params_total": 479516, | |
| "params_requires_grad": 266796, | |
| "params_receiving_grad": 163396, | |
| "params_receiving_grad_list": { | |
| "core.dynamics.dynamics.coupling_scale": 1, | |
| "core.dynamics.dynamics.energy_decay": 1, | |
| "core.dynamics.dynamics.phase_sync": 1, | |
| "compiler.mapping.0.weight": 20480, | |
| "compiler.mapping.0.bias": 64, | |
| "compiler.mapping.2.weight": 16384, | |
| "compiler.mapping.2.bias": 256, | |
| "surface.obl_gate": 1, | |
| "surface.field_feat_norm.weight": 128, | |
| "surface.field_feat_norm.bias": 128, | |
| "surface.alpha_byte_proj.fc1.weight": 8192, | |
| "surface.alpha_byte_proj.fc1.bias": 128, | |
| "surface.alpha_byte_proj.fc2.weight": 32768, | |
| "surface.last_atom.byte_embed.weight": 8192, | |
| "surface.last_atom.expand.weight": 16384, | |
| "surface.last_atom.expand.bias": 256, | |
| "surface.last_atom.contract.weight": 8192, | |
| "surface.last_atom.contract.bias": 32, | |
| "surface.last_atom.phase_proj.weight": 2048, | |
| "surface.last_atom.phase_proj.bias": 32, | |
| "surface.last_atom.head.weight": 8192, | |
| "surface.last_atom.head.bias": 256, | |
| "field_probe.weight": 40960, | |
| "field_probe.bias": 320 | |
| }, | |
| "params_requires_grad_but_zero_grad": { | |
| "compiler.operation_gate.weight": 2240, | |
| "compiler.operation_gate.bias": 7, | |
| "surface.payload_copy_scale": 1, | |
| "surface.field_hard_couple_byte.weight": 32768, | |
| "surface.field_hard_couple_length.weight": 128, | |
| "surface.alpha_length_proj.fc1.weight": 8192, | |
| "surface.alpha_length_proj.fc1.bias": 128, | |
| "surface.alpha_length_proj.fc2.weight": 128, | |
| "surface.state_norm.weight": 32, | |
| "surface.state_norm.bias": 32, | |
| "surface.byte_embed.weight": 8192, | |
| "surface.payload_alpha_gate.weight": 2048, | |
| "surface.payload_to_byte.weight": 8192, | |
| "surface.payload_to_length.weight": 32, | |
| "field_next_probe.weight": 40960, | |
| "field_next_probe.bias": 320 | |
| }, | |
| "ce_only_note": "field_probe/field_next_probe get grad only from the 0.05 field aux loss, not from CE" | |
| } |