{"schema_version": 3, "notes": {"loss_delta": "ablated NLL minus normal NLL; positive means the removed component helps these probes", "token_accuracy_gain": "normal minus ablated next-token accuracy, expressed as a fraction", "ppl_reduction_pct": "100 * (1 - exp(-loss_delta)); relative to the ablated model on scored probe tokens", "output_influence": "local RMS(output - main-only output) / RMS(main-only output), after normalization and projection", "residual_influence": "same local output difference / RMS(incoming block residual)", "long_context_memory_gain": "memory-removal loss increase at full context minus that at short context, on identical targets", "scrambled": "memory value channels rotated; evidence gate recomputed from the corrupted read", "evidence_weights": "per-head weights for log active RMS, log memory RMS, and read agreement", "agreement_weight": "per-head weight u on per-channel agreement a_d*m_d/(RMS a*RMS m) in the gate; starts at 0", "orthogonal_weight": "per-head weight rho adding gate*(memory minus its projection on active); starts at 0", "orthogonal_to_active": "RMS(rho*gate*orthogonal memory) / RMS(active): realized size of the fill-in term", "no_evidence": "all read-dependent gate evidence removed (3 head features and channel agreement)", "no_agreement": "only the per-channel agreement evidence removed", "no_orthogonal": "only the orthogonal memory fill-in removed", "scope": "fixed WikiText probes; inference ablations of the trained model, not a separately trained baseline or proof of retrieval", "nonfinite": "worst non-finite fraction across tracked reads/states; null denotes a non-finite metric"}, "history": [{"step": 1000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 18.30347241299978, "memory_to_active": 1.1254302263259888, "contribution_to_active": 0.17087247967720032, "mix_mean": 0.09319590777158737, "mix_low": 0.12976455688476562, "mix_high": 0.0013084709644317627, "read_cosine": 0.47456222772598267, "beta_mean": 0.16284582018852234, "nonfinite": 0.0, "weighted_memory_to_active": 0.1365586221218109, "mix_std": 0.09810812026262283, "beta_std": 0.10103181004524231, "beta_low": 0.04873276129364967, "beta_high": 3.1789145396032836e-06, "orthogonal_to_active": 0.007730647921562195, "samples": 4, "scored_tokens": 899, "normal_loss": 3.8809218406677246, "normal_token_accuracy": 0.38153505325317383, "state_active": 0.05034780874848366, "state_memory": 0.04095778986811638, "state_peak": 32.77020263671875, "growth_active": 1.1459711790084839, "growth_memory": 1.7329604625701904, "output_influence": 0.19075635075569153, "residual_influence": 0.043929241597652435, "probes": [{"index": 0, "tokens": 256, "normal_loss": 4.049933910369873, "no_memory_loss_delta": 0.057084038853645325, "scrambled_loss_delta": 0.2554522156715393, "memory_only_loss_delta": 1.103989839553833, "no_evidence_loss_delta": 0.002187761478126049, "no_agreement_loss_delta": 0.0012014060048386455, "no_orthogonal_loss_delta": -0.00531424256041646}, {"index": 94, "tokens": 256, "normal_loss": 3.9506988525390625, "no_memory_loss_delta": 0.17945094406604767, "scrambled_loss_delta": 0.40680935978889465, "memory_only_loss_delta": 1.0493055582046509, "no_evidence_loss_delta": 0.020134318619966507, "no_agreement_loss_delta": -0.0031188828870654106, "no_orthogonal_loss_delta": 0.003815312869846821}, {"index": 188, "tokens": 256, "normal_loss": 4.300835132598877, "no_memory_loss_delta": 0.07685204595327377, "scrambled_loss_delta": 0.11938802152872086, "memory_only_loss_delta": 0.9473724365234375, "no_evidence_loss_delta": 0.016642728820443153, "no_agreement_loss_delta": -0.0006234513712115586, "no_orthogonal_loss_delta": 0.0020949668250977993}, {"index": 283, "tokens": 131, "normal_loss": 2.593687057495117, "no_memory_loss_delta": 0.11968938261270523, "scrambled_loss_delta": 0.3704429268836975, "memory_only_loss_delta": 1.0231000185012817, "no_evidence_loss_delta": 0.03777654841542244, "no_agreement_loss_delta": 0.0037628761492669582, "no_orthogonal_loss_delta": 0.007187840528786182}], "agreement_weight_abs_mean": 0.04583337903022766, "agreement_weight_abs_max": 0.12710890173912048, "agreement_weight_mean": 0.002788061508908868, "orthogonal_weight_abs_mean": 0.0669613853096962, "orthogonal_weight_abs_max": 0.22606532275676727, "orthogonal_weight_mean": 0.05576256290078163, "no_memory_loss": 3.9876034259796143, "no_memory_loss_delta": 0.10668119788169861, "no_memory_token_accuracy_gain": 0.013348164968192577, "no_memory_token_win_fraction": 0.5728587508201599, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 10.118782945621001, "scrambled_loss": 4.157485485076904, "scrambled_loss_delta": 0.27656322717666626, "scrambled_token_accuracy_gain": 0.030033370479941368, "scrambled_token_win_fraction": 0.6151279211044312, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 24.1614332968802, "memory_only_loss": 4.912954330444336, "memory_only_loss_delta": 1.0320323705673218, "memory_only_token_accuracy_gain": 0.1268075704574585, "memory_only_token_win_fraction": 0.7964404821395874, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 64.37178730550848, "no_evidence_loss": 3.897522211074829, "no_evidence_loss_delta": 0.0166003555059433, "no_evidence_token_accuracy_gain": 0.0022246940061450005, "no_evidence_token_win_fraction": 0.535038948059082, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 1.6463328882447903, "no_agreement_loss": 3.880746841430664, "no_agreement_loss_delta": -0.0001752402022248134, "no_agreement_token_accuracy_gain": 0.0011123470030725002, "no_agreement_token_win_fraction": 0.5317018628120422, "no_agreement_probe_win_fraction": 0.5, "no_agreement_ppl_reduction_pct": -0.01752555576860029, "no_orthogonal_loss": 3.882138967514038, "no_orthogonal_loss_delta": 0.0012171221897006035, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5417129993438721, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.1216381796901572, "short_no_memory_loss_delta": 0.06437418609857559, "long_context_memory_gain": 0.042307011783123016, "normal_context_loss_gain": 0.10457105189561844, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 1.5117685794830322, "contribution_to_active": 0.19285067915916443, "mix_mean": 0.10553234070539474, "mix_low": 0.012334506027400494, "mix_high": 0.0, "read_cosine": 0.6306785345077515, "beta_mean": 0.37753862142562866, "nonfinite": 0.0, "weighted_memory_to_active": 0.19436496496200562, "mix_std": 0.07134319841861725, "beta_std": 0.13617722690105438, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.020971450954675674, "output_influence": 0.1606207937002182, "residual_influence": 0.05395475775003433, "state_active": 0.03936564177274704, "state_memory": 0.053337693214416504, "state_peak": 16.30777931213379, "growth_active": 0.9816148281097412, "growth_memory": 1.3239558935165405, "heads": [{"head": 0, "memory_to_active": 2.034700870513916, "contribution_to_active": 0.14469677209854126, "mix_mean": 0.056166376918554306, "mix_low": 0.01864147186279297, "mix_high": 0.0, "read_cosine": 0.4961021840572357, "beta_mean": 0.38015735149383545, "nonfinite": 0.0, "weighted_memory_to_active": 0.14815986156463623, "mix_std": 0.0443427599966526, "beta_std": 0.1278824806213379, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01306382566690445, "evidence_weights": [-0.08524852246046066, -0.08632869273424149, 0.12506404519081116], "agreement_weight": -0.048554278910160065, "orthogonal_weight": 0.10251951962709427}, {"head": 1, "memory_to_active": 1.3262494802474976, "contribution_to_active": 0.2948301136493683, "mix_mean": 0.10637465119361877, "mix_low": 0.012478828430175781, "mix_high": 0.0, "read_cosine": 0.6564260125160217, "beta_mean": 0.3998090326786041, "nonfinite": 0.0, "weighted_memory_to_active": 0.32013165950775146, "mix_std": 0.0662195086479187, "beta_std": 0.1328236311674118, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03964695334434509, "evidence_weights": [-0.16492897272109985, -0.1685759425163269, 0.18441985547542572], "agreement_weight": -0.04211108386516571, "orthogonal_weight": 0.1793963462114334}, {"head": 2, "memory_to_active": 1.3401267528533936, "contribution_to_active": 0.10307388007640839, "mix_mean": 0.07626378536224365, "mix_low": 0.008724212646484375, "mix_high": 0.0, "read_cosine": 0.6516422033309937, "beta_mean": 0.4168844223022461, "nonfinite": 0.0, "weighted_memory_to_active": 0.11457648128271103, "mix_std": 0.05119140446186066, "beta_std": 0.11490868031978607, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.011676608584821224, "evidence_weights": [-0.12563255429267883, -0.13039298355579376, 0.13042201101779938], "agreement_weight": -0.07439012080430984, "orthogonal_weight": 0.13250097632408142}, {"head": 3, "memory_to_active": 1.5933822393417358, "contribution_to_active": 0.22517189383506775, "mix_mean": 0.11191520094871521, "mix_low": 0.007306098937988281, "mix_high": 0.0, "read_cosine": 0.6208474040031433, "beta_mean": 0.38866108655929565, "nonfinite": 0.0, "weighted_memory_to_active": 0.2592208683490753, "mix_std": 0.06624513119459152, "beta_std": 0.12024959921836853, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.027010831981897354, "evidence_weights": [-0.16289949417114258, -0.16759194433689117, 0.17834614217281342], "agreement_weight": -0.053912531584501266, "orthogonal_weight": 0.15407025814056396}, {"head": 4, "memory_to_active": 2.2726478576660156, "contribution_to_active": 0.14944161474704742, "mix_mean": 0.05627879500389099, "mix_low": 0.016101837158203125, "mix_high": 0.0, "read_cosine": 0.5233297348022461, "beta_mean": 0.35564538836479187, "nonfinite": 0.0, "weighted_memory_to_active": 0.1485714316368103, "mix_std": 0.04073551297187805, "beta_std": 0.13145431876182556, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01666906476020813, "evidence_weights": [-0.0895020142197609, -0.09131546318531036, 0.12962445616722107], "agreement_weight": -0.07897825539112091, "orthogonal_weight": 0.1314396858215332}, {"head": 5, "memory_to_active": 0.503503680229187, "contribution_to_active": 0.23988980054855347, "mix_mean": 0.22619521617889404, "mix_low": 0.010754585266113281, "mix_high": 0.0, "read_cosine": 0.8357236981391907, "beta_mean": 0.32407432794570923, "nonfinite": 0.0, "weighted_memory_to_active": 0.17552947998046875, "mix_std": 0.15932486951351166, "beta_std": 0.18974462151527405, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01776142045855522, "evidence_weights": [-0.26389554142951965, -0.2561887204647064, 0.27413007616996765], "agreement_weight": 0.09660986810922623, "orthogonal_weight": 0.22606532275676727}]}, {"layer": 1, "memory_to_active": 1.5604782104492188, "contribution_to_active": 0.11304756253957748, "mix_mean": 0.06301547586917877, "mix_low": 0.04415464773774147, "mix_high": 0.0, "read_cosine": 0.462346613407135, "beta_mean": 0.2262749969959259, "nonfinite": 0.0, "weighted_memory_to_active": 0.11378175765275955, "mix_std": 0.044633686542510986, "beta_std": 0.09595614671707153, "beta_low": 0.005625407211482525, "beta_high": 0.0, "orthogonal_to_active": 0.008438188582658768, "output_influence": 0.10865269601345062, "residual_influence": 0.023075079545378685, "state_active": 0.03180061653256416, "state_memory": 0.04786168783903122, "state_peak": 16.67510223388672, "growth_active": 0.9918466210365295, "growth_memory": 1.687711238861084, "heads": [{"head": 0, "memory_to_active": 2.60282039642334, "contribution_to_active": 0.09272409975528717, "mix_mean": 0.024126291275024414, "mix_low": 0.09544754028320312, "mix_high": 0.0, "read_cosine": 0.1710355281829834, "beta_mean": 0.2881757616996765, "nonfinite": 0.0, "weighted_memory_to_active": 0.07540814578533173, "mix_std": 0.015503648668527603, "beta_std": 0.10299960523843765, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.002215005923062563, "evidence_weights": [-0.033468544483184814, -0.029603010043501854, -0.028367415070533752], "agreement_weight": -0.10016506165266037, "orthogonal_weight": 0.029696160927414894}, {"head": 1, "memory_to_active": 1.226422667503357, "contribution_to_active": 0.12346063554286957, "mix_mean": 0.07661363482475281, "mix_low": 0.004939079284667969, "mix_high": 0.0, "read_cosine": 0.601112425327301, "beta_mean": 0.1751299351453781, "nonfinite": 0.0, "weighted_memory_to_active": 0.13053648173809052, "mix_std": 0.05581389740109444, "beta_std": 0.0636804923415184, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012950100004673004, "evidence_weights": [-0.10846500098705292, -0.1001993715763092, 0.08163914084434509], "agreement_weight": -0.047403957694768906, "orthogonal_weight": 0.12375570088624954}, {"head": 2, "memory_to_active": 0.761913537979126, "contribution_to_active": 0.14447224140167236, "mix_mean": 0.11396860331296921, "mix_low": 0.006411552429199219, "mix_high": 0.0, "read_cosine": 0.7008106708526611, "beta_mean": 0.16913828253746033, "nonfinite": 0.0, "weighted_memory_to_active": 0.16214802861213684, "mix_std": 0.0861339196562767, "beta_std": 0.060018714517354965, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013397927395999432, "evidence_weights": [-0.1435006558895111, -0.13751909136772156, 0.15083777904510498], "agreement_weight": 0.03872077539563179, "orthogonal_weight": 0.15166428685188293}, {"head": 3, "memory_to_active": 1.4856398105621338, "contribution_to_active": 0.058030206710100174, "mix_mean": 0.0329417884349823, "mix_low": 0.05822944641113281, "mix_high": 0.0, "read_cosine": 0.4110480844974518, "beta_mean": 0.24382908642292023, "nonfinite": 0.0, "weighted_memory_to_active": 0.056532420217990875, "mix_std": 0.0184482354670763, "beta_std": 0.08936350047588348, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0034027635119855404, "evidence_weights": [-0.05319041386246681, -0.05045095458626747, 0.04525056853890419], "agreement_weight": -0.029092835262417793, "orthogonal_weight": 0.06561696529388428}, {"head": 4, "memory_to_active": 1.2357349395751953, "contribution_to_active": 0.16478325426578522, "mix_mean": 0.09997289627790451, "mix_low": 0.003856658935546875, "mix_high": 0.0, "read_cosine": 0.5614390969276428, "beta_mean": 0.13905766606330872, "nonfinite": 0.0, "weighted_memory_to_active": 0.1629096120595932, "mix_std": 0.061334237456321716, "beta_std": 0.07981449365615845, "beta_low": 0.03375244140625, "beta_high": 0.0, "orthogonal_to_active": 0.016245659440755844, "evidence_weights": [-0.11893657594919205, -0.11613268405199051, 0.14477454125881195], "agreement_weight": -0.04838193953037262, "orthogonal_weight": 0.11980244517326355}, {"head": 5, "memory_to_active": 2.050337791442871, "contribution_to_active": 0.09481493383646011, "mix_mean": 0.03046964295208454, "mix_low": 0.09604358673095703, "mix_high": 0.0, "read_cosine": 0.32863378524780273, "beta_mean": 0.34231922030448914, "nonfinite": 0.0, "weighted_memory_to_active": 0.09515583515167236, "mix_std": 0.03056817129254341, "beta_std": 0.17986008524894714, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024176735896617174, "evidence_weights": [-0.03940070420503616, -0.037831418216228485, 0.045657604932785034], "agreement_weight": -0.006878205109387636, "orthogonal_weight": 0.028172334656119347}]}, {"layer": 2, "memory_to_active": 1.6729798316955566, "contribution_to_active": 0.14012864232063293, "mix_mean": 0.045456260442733765, "mix_low": 0.0918685644865036, "mix_high": 0.0, "read_cosine": 0.440112441778183, "beta_mean": 0.21748623251914978, "nonfinite": 0.0, "weighted_memory_to_active": 0.13786862790584564, "mix_std": 0.049319155514240265, "beta_std": 0.11763788014650345, "beta_low": 0.0067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.008871374651789665, "output_influence": 0.26981204748153687, "residual_influence": 0.0525018535554409, "state_active": 0.025642868131399155, "state_memory": 0.04441012442111969, "state_peak": 24.33842658996582, "growth_active": 0.9815218448638916, "growth_memory": 1.6076253652572632, "heads": [{"head": 0, "memory_to_active": 1.9115673303604126, "contribution_to_active": 0.1377030462026596, "mix_mean": 0.0323842391371727, "mix_low": 0.1896820068359375, "mix_high": 0.0, "read_cosine": 0.3089676797389984, "beta_mean": 0.12937083840370178, "nonfinite": 0.0, "weighted_memory_to_active": 0.12940824031829834, "mix_std": 0.03727094829082489, "beta_std": 0.061925169080495834, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008852816186845303, "evidence_weights": [-0.058210983872413635, -0.053727228194475174, 0.038795266300439835], "agreement_weight": -0.04550204053521156, "orthogonal_weight": 0.07211325317621231}, {"head": 1, "memory_to_active": 2.086857795715332, "contribution_to_active": 0.16780075430870056, "mix_mean": 0.04709482192993164, "mix_low": 0.05980873107910156, "mix_high": 0.0, "read_cosine": 0.36844077706336975, "beta_mean": 0.2709788382053375, "nonfinite": 0.0, "weighted_memory_to_active": 0.16487738490104675, "mix_std": 0.04244506359100342, "beta_std": 0.16060250997543335, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00903579406440258, "evidence_weights": [-0.06247995048761368, -0.0602257065474987, 0.0509285032749176], "agreement_weight": -0.036471493542194366, "orthogonal_weight": 0.059104565531015396}, {"head": 2, "memory_to_active": 0.8615380525588989, "contribution_to_active": 0.09969912469387054, "mix_mean": 0.06149986386299133, "mix_low": 0.04984569549560547, "mix_high": 0.0, "read_cosine": 0.5965182781219482, "beta_mean": 0.10302107036113739, "nonfinite": 0.0, "weighted_memory_to_active": 0.10703542828559875, "mix_std": 0.06374691426753998, "beta_std": 0.04651573300361633, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005142420530319214, "evidence_weights": [-0.07642791420221329, -0.07085227221250534, 0.07346849888563156], "agreement_weight": 0.016298120841383934, "orthogonal_weight": 0.08332496881484985}, {"head": 3, "memory_to_active": 1.0320509672164917, "contribution_to_active": 0.07705442607402802, "mix_mean": 0.0524381659924984, "mix_low": 0.07040119171142578, "mix_high": 0.0, "read_cosine": 0.5442616939544678, "beta_mean": 0.33876246213912964, "nonfinite": 0.0, "weighted_memory_to_active": 0.08038477599620819, "mix_std": 0.04792006313800812, "beta_std": 0.16945293545722961, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.003384801559150219, "evidence_weights": [-0.07464028149843216, -0.07606763392686844, 0.08431178331375122], "agreement_weight": 0.013314093463122845, "orthogonal_weight": 0.055754002183675766}, {"head": 4, "memory_to_active": 2.951814651489258, "contribution_to_active": 0.2810760736465454, "mix_mean": 0.03465666621923447, "mix_low": 0.11928749084472656, "mix_high": 0.0, "read_cosine": 0.3296867311000824, "beta_mean": 0.3750869631767273, "nonfinite": 0.0, "weighted_memory_to_active": 0.2697947323322296, "mix_std": 0.06549809873104095, "beta_std": 0.21580441296100616, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.021933216601610184, "evidence_weights": [-0.03173137456178665, -0.029652124270796776, 0.03720677271485329], "agreement_weight": -0.004815624561160803, "orthogonal_weight": 0.08695096522569656}, {"head": 5, "memory_to_active": 1.1940490007400513, "contribution_to_active": 0.07743845880031586, "mix_mean": 0.04466380551457405, "mix_low": 0.062186241149902344, "mix_high": 0.0, "read_cosine": 0.4927995204925537, "beta_mean": 0.08769714087247849, "nonfinite": 0.0, "weighted_memory_to_active": 0.07571117579936981, "mix_std": 0.03903382271528244, "beta_std": 0.05152656137943268, "beta_low": 0.04022216796875, "beta_high": 0.0, "orthogonal_to_active": 0.004879198968410492, "evidence_weights": [-0.06386411190032959, -0.0563647635281086, 0.04384433850646019], "agreement_weight": -0.030739637091755867, "orthogonal_weight": 0.07522373646497726}]}, {"layer": 3, "memory_to_active": 1.8052823543548584, "contribution_to_active": 0.09958098083734512, "mix_mean": 0.03604817017912865, "mix_low": 0.10809072107076645, "mix_high": 0.0, "read_cosine": 0.39936134219169617, "beta_mean": 0.18985086679458618, "nonfinite": 0.0, "weighted_memory_to_active": 0.09869866073131561, "mix_std": 0.040898531675338745, "beta_std": 0.10814685374498367, "beta_low": 0.0020039877854287624, "beta_high": 0.0, "orthogonal_to_active": 0.0034524588845670223, "output_influence": 0.14215174317359924, "residual_influence": 0.02357439510524273, "state_active": 0.03276269882917404, "state_memory": 0.045357055962085724, "state_peak": 30.003633499145508, "growth_active": 1.007768154144287, "growth_memory": 1.8449362516403198, "heads": [{"head": 0, "memory_to_active": 1.3872623443603516, "contribution_to_active": 0.05244278162717819, "mix_mean": 0.019927693530917168, "mix_low": 0.25040531158447266, "mix_high": 0.0, "read_cosine": 0.28849881887435913, "beta_mean": 0.06556054949760437, "nonfinite": 0.0, "weighted_memory_to_active": 0.045378610491752625, "mix_std": 0.026151634752750397, "beta_std": 0.048972636461257935, "beta_low": 0.0062255859375, "beta_high": 0.0, "orthogonal_to_active": 0.0003641046932898462, "evidence_weights": [-0.006896649021655321, -0.008991251699626446, -0.017700517550110817], "agreement_weight": -0.08947199583053589, "orthogonal_weight": 0.008377510122954845}, {"head": 1, "memory_to_active": 1.9175691604614258, "contribution_to_active": 0.07467420399188995, "mix_mean": 0.03225110471248627, "mix_low": 0.08569908142089844, "mix_high": 0.0, "read_cosine": 0.35203999280929565, "beta_mean": 0.15813125669956207, "nonfinite": 0.0, "weighted_memory_to_active": 0.07072477042675018, "mix_std": 0.026502598077058792, "beta_std": 0.05855536460876465, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026486252900213003, "evidence_weights": [-0.04528431221842766, -0.04412305727601051, 0.02691652998328209], "agreement_weight": -0.05509966239333153, "orthogonal_weight": 0.03931611776351929}, {"head": 2, "memory_to_active": 1.9836149215698242, "contribution_to_active": 0.13339927792549133, "mix_mean": 0.043657660484313965, "mix_low": 0.03447532653808594, "mix_high": 0.0, "read_cosine": 0.472640722990036, "beta_mean": 0.21338552236557007, "nonfinite": 0.0, "weighted_memory_to_active": 0.14044257998466492, "mix_std": 0.04441238194704056, "beta_std": 0.1325325071811676, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007114625535905361, "evidence_weights": [-0.06701023131608963, -0.0709010660648346, 0.0896078422665596], "agreement_weight": -0.0397479347884655, "orthogonal_weight": 0.0587073415517807}, {"head": 3, "memory_to_active": 1.6081819534301758, "contribution_to_active": 0.06480413675308228, "mix_mean": 0.022719528526067734, "mix_low": 0.12147808074951172, "mix_high": 0.0, "read_cosine": 0.41928237676620483, "beta_mean": 0.1318608969449997, "nonfinite": 0.0, "weighted_memory_to_active": 0.06424278020858765, "mix_std": 0.026396073400974274, "beta_std": 0.07521107792854309, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009416736429557204, "evidence_weights": [-0.017259890213608742, -0.01579868793487549, 0.006541264243423939], "agreement_weight": -0.04972367733716965, "orthogonal_weight": 0.015760473906993866}, {"head": 4, "memory_to_active": 3.095309257507324, "contribution_to_active": 0.0752568393945694, "mix_mean": 0.020750701427459717, "mix_low": 0.13577651977539062, "mix_high": 0.0, "read_cosine": 0.27938035130500793, "beta_mean": 0.3292759358882904, "nonfinite": 0.0, "weighted_memory_to_active": 0.0707332119345665, "mix_std": 0.013898990117013454, "beta_std": 0.14476974308490753, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005607694038189948, "evidence_weights": [-0.005679269786924124, -0.005655416753143072, -0.023027772083878517], "agreement_weight": -0.12710890173912048, "orthogonal_weight": 0.0077977548353374004}, {"head": 5, "memory_to_active": 0.8397566676139832, "contribution_to_active": 0.196908637881279, "mix_mean": 0.07698231935501099, "mix_low": 0.020709991455078125, "mix_high": 0.0, "read_cosine": 0.5843257308006287, "beta_mean": 0.24089094996452332, "nonfinite": 0.0, "weighted_memory_to_active": 0.20066998898983002, "mix_std": 0.10802951455116272, "beta_std": 0.18883979320526123, "beta_low": 0.00579833984375, "beta_high": 0.0, "orthogonal_to_active": 0.009084954857826233, "evidence_weights": [-0.09395249933004379, -0.1018003299832344, 0.07420765608549118], "agreement_weight": 0.0699828714132309, "orthogonal_weight": 0.0830351859331131}]}, {"layer": 4, "memory_to_active": 1.1567280292510986, "contribution_to_active": 0.11478142440319061, "mix_mean": 0.0610661506652832, "mix_low": 0.1228439062833786, "mix_high": 0.0, "read_cosine": 0.5092046856880188, "beta_mean": 0.16375017166137695, "nonfinite": 0.0, "weighted_memory_to_active": 0.12536945939064026, "mix_std": 0.056105293333530426, "beta_std": 0.09603685140609741, "beta_low": 0.0009969075908884406, "beta_high": 0.0, "orthogonal_to_active": 0.008487749844789505, "output_influence": 0.12645260989665985, "residual_influence": 0.026023387908935547, "state_active": 0.03119116835296154, "state_memory": 0.041949499398469925, "state_peak": 23.514936447143555, "growth_active": 1.0199646949768066, "growth_memory": 1.936954379081726, "heads": [{"head": 0, "memory_to_active": 0.7613071203231812, "contribution_to_active": 0.10276615619659424, "mix_mean": 0.09828590601682663, "mix_low": 0.0060882568359375, "mix_high": 0.0, "read_cosine": 0.6739318370819092, "beta_mean": 0.11469273269176483, "nonfinite": 0.0, "weighted_memory_to_active": 0.12181933969259262, "mix_std": 0.08929581195116043, "beta_std": 0.11448337137699127, "beta_low": 0.00543212890625, "beta_high": 0.0, "orthogonal_to_active": 0.00412856787443161, "evidence_weights": [-0.07955467700958252, -0.08189456909894943, 0.10088597983121872], "agreement_weight": 0.0287938229739666, "orthogonal_weight": 0.05594412237405777}, {"head": 1, "memory_to_active": 0.7135986089706421, "contribution_to_active": 0.09486286342144012, "mix_mean": 0.04158905893564224, "mix_low": 0.18462562561035156, "mix_high": 0.0, "read_cosine": 0.4666501581668854, "beta_mean": 0.1286623328924179, "nonfinite": 0.0, "weighted_memory_to_active": 0.06810495257377625, "mix_std": 0.049592338502407074, "beta_std": 0.08572491258382797, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.0032872864976525307, "evidence_weights": [-0.05954482778906822, -0.061395272612571716, 0.06127132847905159], "agreement_weight": 0.0292519498616457, "orthogonal_weight": 0.06529273837804794}, {"head": 2, "memory_to_active": 2.1345415115356445, "contribution_to_active": 0.07441265881061554, "mix_mean": 0.019993919879198074, "mix_low": 0.3104438781738281, "mix_high": 0.0, "read_cosine": 0.31054556369781494, "beta_mean": 0.3300788104534149, "nonfinite": 0.0, "weighted_memory_to_active": 0.07610137015581131, "mix_std": 0.01857486553490162, "beta_std": 0.08691401779651642, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030183957424014807, "evidence_weights": [-0.016537733376026154, -0.018155565485358238, 0.008172200992703438], "agreement_weight": -0.01649048924446106, "orthogonal_weight": 0.044627267867326736}, {"head": 3, "memory_to_active": 1.3106420040130615, "contribution_to_active": 0.058472104370594025, "mix_mean": 0.026245765388011932, "mix_low": 0.1562490463256836, "mix_high": 0.0, "read_cosine": 0.28555434942245483, "beta_mean": 0.17738063633441925, "nonfinite": 0.0, "weighted_memory_to_active": 0.04583505541086197, "mix_std": 0.023885615170001984, "beta_std": 0.16110634803771973, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.00019827738287858665, "evidence_weights": [-0.021912846714258194, -0.021285684779286385, -0.047031935304403305], "agreement_weight": -0.00620677974075079, "orthogonal_weight": 0.004813183099031448}, {"head": 4, "memory_to_active": 0.9948686957359314, "contribution_to_active": 0.11214880645275116, "mix_mean": 0.08331339061260223, "mix_low": 0.02425098419189453, "mix_high": 0.0, "read_cosine": 0.6594685912132263, "beta_mean": 0.0863107293844223, "nonfinite": 0.0, "weighted_memory_to_active": 0.16158704459667206, "mix_std": 0.07170321047306061, "beta_std": 0.03404100239276886, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00873585045337677, "evidence_weights": [-0.07611515372991562, -0.06785909831523895, 0.129691019654274], "agreement_weight": -0.003172646276652813, "orthogonal_weight": 0.10043743252754211}, {"head": 5, "memory_to_active": 1.0254101753234863, "contribution_to_active": 0.246025949716568, "mix_mean": 0.09696885943412781, "mix_low": 0.055405616760253906, "mix_high": 0.0, "read_cosine": 0.659077525138855, "beta_mean": 0.1453758329153061, "nonfinite": 0.0, "weighted_memory_to_active": 0.2787688970565796, "mix_std": 0.0835798904299736, "beta_std": 0.09395141899585724, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.031558115035295486, "evidence_weights": [-0.11349572241306305, -0.10121162980794907, 0.13646216690540314], "agreement_weight": -0.03687392175197601, "orthogonal_weight": 0.17134536802768707}]}, {"layer": 5, "memory_to_active": 1.0199416875839233, "contribution_to_active": 0.1788899153470993, "mix_mean": 0.10944749414920807, "mix_low": 0.1635534018278122, "mix_high": 0.00019200643873773515, "read_cosine": 0.4971182942390442, "beta_mean": 0.14997828006744385, "nonfinite": 0.0, "weighted_memory_to_active": 0.13335570693016052, "mix_std": 0.11429329216480255, "beta_std": 0.09397906064987183, "beta_low": 0.0091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.009061984717845917, "output_influence": 0.18052420020103455, "residual_influence": 0.03188811242580414, "state_active": 0.06981596350669861, "state_memory": 0.04581035301089287, "state_peak": 29.87325668334961, "growth_active": 1.0962451696395874, "growth_memory": 1.7901066541671753, "heads": [{"head": 0, "memory_to_active": 0.8528079986572266, "contribution_to_active": 0.22161395847797394, "mix_mean": 0.18033815920352936, "mix_low": 0.004246711730957031, "mix_high": 1.9073486328125e-06, "read_cosine": 0.5005165934562683, "beta_mean": 0.22605910897254944, "nonfinite": 0.0, "weighted_memory_to_active": 0.2024349570274353, "mix_std": 0.1575603038072586, "beta_std": 0.09570439904928207, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01430180948227644, "evidence_weights": [-0.11998923867940903, -0.11277690529823303, 0.11144284904003143], "agreement_weight": 0.011944805271923542, "orthogonal_weight": 0.0876605287194252}, {"head": 1, "memory_to_active": 1.628050684928894, "contribution_to_active": 0.05799517408013344, "mix_mean": 0.014546901918947697, "mix_low": 0.582127571105957, "mix_high": 0.0, "read_cosine": 0.2646457254886627, "beta_mean": 0.12411569058895111, "nonfinite": 0.0, "weighted_memory_to_active": 0.050751104950904846, "mix_std": 0.025569090619683266, "beta_std": 0.1021450012922287, "beta_low": 0.0107421875, "beta_high": 0.0, "orthogonal_to_active": 0.0005969308549538255, "evidence_weights": [0.008841353468596935, 0.020138179883360863, -0.07844530791044235], "agreement_weight": -0.019852451980113983, "orthogonal_weight": -0.012462133541703224}, {"head": 2, "memory_to_active": 0.41032516956329346, "contribution_to_active": 0.18536855280399323, "mix_mean": 0.15824325382709503, "mix_low": 0.04058265686035156, "mix_high": 0.0005655288696289062, "read_cosine": 0.8033432364463806, "beta_mean": 0.15800228714942932, "nonfinite": 0.0, "weighted_memory_to_active": 0.11545690894126892, "mix_std": 0.1742589920759201, "beta_std": 0.08976060897111893, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01320677064359188, "evidence_weights": [-0.13247115910053253, -0.1305253654718399, 0.16783013939857483], "agreement_weight": 0.05251815915107727, "orthogonal_weight": 0.19781331717967987}, {"head": 3, "memory_to_active": 1.565362572669983, "contribution_to_active": 0.08376073837280273, "mix_mean": 0.027573131024837494, "mix_low": 0.2769737243652344, "mix_high": 0.0, "read_cosine": 0.3811084032058716, "beta_mean": 0.14799463748931885, "nonfinite": 0.0, "weighted_memory_to_active": 0.09113417565822601, "mix_std": 0.03600258380174637, "beta_std": 0.1086120530962944, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.001581041608005762, "evidence_weights": [-0.02320617437362671, -0.023937903344631195, -0.0038080033846199512], "agreement_weight": -0.02200755849480629, "orthogonal_weight": 0.020268693566322327}, {"head": 4, "memory_to_active": 0.40372204780578613, "contribution_to_active": 0.2797456681728363, "mix_mean": 0.15915602445602417, "mix_low": 0.043768882751464844, "mix_high": 0.0005846023559570312, "read_cosine": 0.6610792875289917, "beta_mean": 0.12506085634231567, "nonfinite": 0.0, "weighted_memory_to_active": 0.12317969650030136, "mix_std": 0.17835497856140137, "beta_std": 0.12432825565338135, "beta_low": 0.044189453125, "beta_high": 0.0, "orthogonal_to_active": 0.013172178529202938, "evidence_weights": [-0.08114694058895111, -0.09106718003749847, 0.12042637914419174], "agreement_weight": 0.058432649821043015, "orthogonal_weight": 0.1503008008003235}, {"head": 5, "memory_to_active": 1.2593820095062256, "contribution_to_active": 0.24485531449317932, "mix_mean": 0.11682748049497604, "mix_low": 0.03362083435058594, "mix_high": 0.0, "read_cosine": 0.3720164895057678, "beta_mean": 0.11863715946674347, "nonfinite": 0.0, "weighted_memory_to_active": 0.2171773761510849, "mix_std": 0.11401379108428955, "beta_std": 0.043324001133441925, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.011513175442814827, "evidence_weights": [-0.08839328587055206, -0.08341897279024124, 0.10381017625331879], "agreement_weight": -0.05806361511349678, "orthogonal_weight": 0.058902863413095474}]}, {"layer": 6, "memory_to_active": 0.8235504031181335, "contribution_to_active": 0.16640181839466095, "mix_mean": 0.07615190744400024, "mix_low": 0.20041832327842712, "mix_high": 0.002631346695125103, "read_cosine": 0.36848708987236023, "beta_mean": 0.12593913078308105, "nonfinite": 0.0, "weighted_memory_to_active": 0.10573551803827286, "mix_std": 0.1186642125248909, "beta_std": 0.10681974142789841, "beta_low": 0.0368245467543602, "beta_high": 0.0, "orthogonal_to_active": 0.0030428082682192326, "output_influence": 0.2971770167350769, "residual_influence": 0.07056359946727753, "state_active": 0.05732712522149086, "state_memory": 0.034692347049713135, "state_peak": 19.663602828979492, "growth_active": 1.2609107494354248, "growth_memory": 1.8020715713500977, "heads": [{"head": 0, "memory_to_active": 0.49694323539733887, "contribution_to_active": 0.17348743975162506, "mix_mean": 0.07906300574541092, "mix_low": 0.055779457092285156, "mix_high": 0.0024957656860351562, "read_cosine": 0.6210991740226746, "beta_mean": 0.2737487554550171, "nonfinite": 0.0, "weighted_memory_to_active": 0.1325318068265915, "mix_std": 0.12723614275455475, "beta_std": 0.17682719230651855, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005750072188675404, "evidence_weights": [-0.0766557976603508, -0.07525480538606644, 0.09480855613946915], "agreement_weight": 0.07311481237411499, "orthogonal_weight": 0.08091863989830017}, {"head": 1, "memory_to_active": 0.5615568161010742, "contribution_to_active": 0.1297767460346222, "mix_mean": 0.11109913885593414, "mix_low": 0.013356208801269531, "mix_high": 0.00043392181396484375, "read_cosine": 0.4502409100532532, "beta_mean": 0.061705660074949265, "nonfinite": 0.0, "weighted_memory_to_active": 0.08811480551958084, "mix_std": 0.11158347129821777, "beta_std": 0.03507441282272339, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.004904525820165873, "evidence_weights": [-0.13344132900238037, -0.12272073328495026, 0.06873954832553864], "agreement_weight": 0.026150673627853394, "orthogonal_weight": 0.07052509486675262}, {"head": 2, "memory_to_active": 0.9989885091781616, "contribution_to_active": 0.1114528700709343, "mix_mean": 0.03565690666437149, "mix_low": 0.11614131927490234, "mix_high": 0.0003376007080078125, "read_cosine": 0.2704851031303406, "beta_mean": 0.09615267813205719, "nonfinite": 0.0, "weighted_memory_to_active": 0.08366924524307251, "mix_std": 0.07279790937900543, "beta_std": 0.08743385970592499, "beta_low": 0.00262451171875, "beta_high": 0.0, "orthogonal_to_active": 0.00023032054014038295, "evidence_weights": [-0.02626446820795536, -0.023765617981553078, -0.028197426348924637], "agreement_weight": -0.09627609699964523, "orthogonal_weight": -0.0029260532464832067}, {"head": 3, "memory_to_active": 0.5592331886291504, "contribution_to_active": 0.34765923023223877, "mix_mean": 0.10442180931568146, "mix_low": 0.1744709014892578, "mix_high": 0.009890556335449219, "read_cosine": 0.3279092609882355, "beta_mean": 0.09290459752082825, "nonfinite": 0.0, "weighted_memory_to_active": 0.19185014069080353, "mix_std": 0.19607362151145935, "beta_std": 0.15270671248435974, "beta_low": 0.14398193359375, "beta_high": 0.0, "orthogonal_to_active": 0.0016278158873319626, "evidence_weights": [-0.05391531437635422, -0.08691446483135223, 0.05874519422650337], "agreement_weight": 0.021041780710220337, "orthogonal_weight": 0.010532664135098457}, {"head": 4, "memory_to_active": 0.49533727765083313, "contribution_to_active": 0.20869868993759155, "mix_mean": 0.11813536286354065, "mix_low": 0.036449432373046875, "mix_high": 0.0026302337646484375, "read_cosine": 0.3352507948875427, "beta_mean": 0.10362842679023743, "nonfinite": 0.0, "weighted_memory_to_active": 0.11088570952415466, "mix_std": 0.18277789652347565, "beta_std": 0.12271951884031296, "beta_low": 0.07366943359375, "beta_high": 0.0, "orthogonal_to_active": 0.0030607925727963448, "evidence_weights": [-0.09929198026657104, -0.11837662011384964, 0.05757264420390129], "agreement_weight": 0.018654178828001022, "orthogonal_weight": 0.03224311023950577}, {"head": 5, "memory_to_active": 1.8292429447174072, "contribution_to_active": 0.027335885912179947, "mix_mean": 0.008535187691450119, "mix_low": 0.8063125610351562, "mix_high": 0.0, "read_cosine": 0.20593728125095367, "beta_mean": 0.1274946928024292, "nonfinite": 0.0, "weighted_memory_to_active": 0.027361396700143814, "mix_std": 0.021516242995858192, "beta_std": 0.06615674495697021, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026833228766918182, "evidence_weights": [0.06057814508676529, 0.06313656270503998, -0.12178753316402435], "agreement_weight": -0.040022626519203186, "orthogonal_weight": -0.09621114283800125}]}, {"layer": 7, "memory_to_active": 1.5746853351593018, "contribution_to_active": 0.20252570509910583, "mix_mean": 0.04609004035592079, "mix_low": 0.1580062061548233, "mix_high": 0.0, "read_cosine": 0.38995298743247986, "beta_mean": 0.19013498723506927, "nonfinite": 0.0, "weighted_memory_to_active": 0.21698671579360962, "mix_std": 0.04747496172785759, "beta_std": 0.09275511652231216, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0161453764885664, "output_influence": 0.20910242199897766, "residual_influence": 0.03309318423271179, "state_active": 0.017677780240774155, "state_memory": 0.039449580013751984, "state_peak": 30.61307716369629, "growth_active": 0.9376987218856812, "growth_memory": 1.5431395769119263, "heads": [{"head": 0, "memory_to_active": 2.0416040420532227, "contribution_to_active": 0.7210560441017151, "mix_mean": 0.08235098421573639, "mix_low": 0.018848419189453125, "mix_high": 0.0, "read_cosine": 0.4382385313510895, "beta_mean": 0.17716704308986664, "nonfinite": 0.0, "weighted_memory_to_active": 0.8174067139625549, "mix_std": 0.08415849506855011, "beta_std": 0.1015949547290802, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07365664094686508, "evidence_weights": [-0.07535520941019058, -0.07516124844551086, 0.07326523214578629], "agreement_weight": -0.022190028801560402, "orthogonal_weight": 0.15030473470687866}, {"head": 1, "memory_to_active": 1.5492870807647705, "contribution_to_active": 0.10512048751115799, "mix_mean": 0.047748759388923645, "mix_low": 0.046179771423339844, "mix_high": 0.0, "read_cosine": 0.26938971877098083, "beta_mean": 0.14835673570632935, "nonfinite": 0.0, "weighted_memory_to_active": 0.09399949759244919, "mix_std": 0.043893251568078995, "beta_std": 0.0738004520535469, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.003602870274335146, "evidence_weights": [-0.041126977652311325, -0.03982691466808319, 0.014346328563988209], "agreement_weight": -0.042710065841674805, "orthogonal_weight": 0.04008922353386879}, {"head": 2, "memory_to_active": 1.5150114297866821, "contribution_to_active": 0.16138431429862976, "mix_mean": 0.05061354115605354, "mix_low": 0.19689559936523438, "mix_high": 0.0, "read_cosine": 0.396299809217453, "beta_mean": 0.25239264965057373, "nonfinite": 0.0, "weighted_memory_to_active": 0.1651022881269455, "mix_std": 0.056876931339502335, "beta_std": 0.12527978420257568, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.009599493816494942, "evidence_weights": [-0.0562715008854866, -0.05388825386762619, 0.04677979275584221], "agreement_weight": 0.008113271556794643, "orthogonal_weight": 0.07396142184734344}, {"head": 3, "memory_to_active": 1.694802165031433, "contribution_to_active": 0.07862918823957443, "mix_mean": 0.03100290149450302, "mix_low": 0.1892223358154297, "mix_high": 0.0, "read_cosine": 0.3351594805717468, "beta_mean": 0.16425283253192902, "nonfinite": 0.0, "weighted_memory_to_active": 0.07719287276268005, "mix_std": 0.030426057055592537, "beta_std": 0.09916935861110687, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029968987219035625, "evidence_weights": [-0.02232772298157215, -0.01614472083747387, 0.003990331199020147], "agreement_weight": -0.08485499769449234, "orthogonal_weight": 0.042345643043518066}, {"head": 4, "memory_to_active": 1.293303370475769, "contribution_to_active": 0.08569923788309097, "mix_mean": 0.04615761712193489, "mix_low": 0.09019088745117188, "mix_high": 0.0, "read_cosine": 0.4966488182544708, "beta_mean": 0.14866778254508972, "nonfinite": 0.0, "weighted_memory_to_active": 0.09343327581882477, "mix_std": 0.046867869794368744, "beta_std": 0.05732209235429764, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00592407351359725, "evidence_weights": [-0.04533062502741814, -0.0397644005715847, 0.07128198444843292], "agreement_weight": -0.003722223686054349, "orthogonal_weight": 0.08468472957611084}, {"head": 5, "memory_to_active": 1.3541035652160645, "contribution_to_active": 0.06326491385698318, "mix_mean": 0.01866643875837326, "mix_low": 0.40670013427734375, "mix_high": 0.0, "read_cosine": 0.4039815366268158, "beta_mean": 0.2499728500843048, "nonfinite": 0.0, "weighted_memory_to_active": 0.054785579442977905, "mix_std": 0.022627171128988266, "beta_std": 0.0993640124797821, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010922858491539955, "evidence_weights": [-0.01455326285213232, -0.008842159062623978, -0.026227792724967003], "agreement_weight": -0.034102216362953186, "orthogonal_weight": 0.02315635420382023}]}, {"layer": 8, "memory_to_active": 0.945662260055542, "contribution_to_active": 0.2203679382801056, "mix_mean": 0.16550779342651367, "mix_low": 0.05308564752340317, "mix_high": 0.0013825098285451531, "read_cosine": 0.616972804069519, "beta_mean": 0.13005167245864868, "nonfinite": 0.0, "weighted_memory_to_active": 0.20547941327095032, "mix_std": 0.15200801193714142, "beta_std": 0.0921308696269989, "beta_low": 0.0216776542365551, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.013478200882673264, "output_influence": 0.24371568858623505, "residual_influence": 0.04445744305849075, "state_active": 0.047649990767240524, "state_memory": 0.03342745453119278, "state_peak": 18.41265106201172, "growth_active": 1.202721118927002, "growth_memory": 1.8533010482788086, "heads": [{"head": 0, "memory_to_active": 0.6224160194396973, "contribution_to_active": 0.08321762830018997, "mix_mean": 0.07933259755373001, "mix_low": 0.04582786560058594, "mix_high": 9.5367431640625e-07, "read_cosine": 0.7863842248916626, "beta_mean": 0.06733990460634232, "nonfinite": 0.0, "weighted_memory_to_active": 0.08339578658342361, "mix_std": 0.0758623331785202, "beta_std": 0.055744193494319916, "beta_low": 0.01043701171875, "beta_high": 0.0, "orthogonal_to_active": 0.004436152055859566, "evidence_weights": [-0.08457750827074051, -0.08097110688686371, 0.1099025160074234], "agreement_weight": 0.025004319846630096, "orthogonal_weight": 0.10078152269124985}, {"head": 1, "memory_to_active": 0.6203776001930237, "contribution_to_active": 0.24245774745941162, "mix_mean": 0.27414822578430176, "mix_low": 0.009705543518066406, "mix_high": 1.71661376953125e-05, "read_cosine": 0.7746248245239258, "beta_mean": 0.06790029257535934, "nonfinite": 0.0, "weighted_memory_to_active": 0.24365869164466858, "mix_std": 0.1815907210111618, "beta_std": 0.03684190288186073, "beta_low": 0.00079345703125, "beta_high": 0.0, "orthogonal_to_active": 0.021354060620069504, "evidence_weights": [-0.17182697355747223, -0.16179874539375305, 0.1754506230354309], "agreement_weight": 0.018611088395118713, "orthogonal_weight": 0.16663968563079834}, {"head": 2, "memory_to_active": 1.4534248113632202, "contribution_to_active": 0.13201551139354706, "mix_mean": 0.06094270199537277, "mix_low": 0.15270042419433594, "mix_high": 0.0, "read_cosine": 0.45542946457862854, "beta_mean": 0.22892656922340393, "nonfinite": 0.0, "weighted_memory_to_active": 0.13671234250068665, "mix_std": 0.07479190081357956, "beta_std": 0.08377158641815186, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0046657477505505085, "evidence_weights": [-0.061084382236003876, -0.06220746785402298, 0.036935385316610336], "agreement_weight": -0.02269374392926693, "orthogonal_weight": 0.0382734015583992}, {"head": 3, "memory_to_active": 1.0320593118667603, "contribution_to_active": 0.2100931853055954, "mix_mean": 0.13950616121292114, "mix_low": 0.0644073486328125, "mix_high": 0.0, "read_cosine": 0.59417325258255, "beta_mean": 0.10616099834442139, "nonfinite": 0.0, "weighted_memory_to_active": 0.2400636374950409, "mix_std": 0.14633069932460785, "beta_std": 0.07621695101261139, "beta_low": 0.0010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.01938653364777565, "evidence_weights": [-0.07455935329198837, -0.07025499641895294, 0.13217273354530334], "agreement_weight": 0.014591534622013569, "orthogonal_weight": 0.1122957095503807}, {"head": 4, "memory_to_active": 1.493660807609558, "contribution_to_active": 0.2516392469406128, "mix_mean": 0.13091696798801422, "mix_low": 0.02337360382080078, "mix_high": 0.0, "read_cosine": 0.5333513021469116, "beta_mean": 0.1304205060005188, "nonfinite": 0.0, "weighted_memory_to_active": 0.3096892237663269, "mix_std": 0.11232194304466248, "beta_std": 0.06626572459936142, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.011195417493581772, "evidence_weights": [-0.08846012502908707, -0.08413166552782059, 0.07972424477338791], "agreement_weight": -0.008130681701004505, "orthogonal_weight": 0.05111749470233917}, {"head": 5, "memory_to_active": 0.45203471183776855, "contribution_to_active": 0.4027842879295349, "mix_mean": 0.3082001507282257, "mix_low": 0.02249908447265625, "mix_high": 0.008276939392089844, "read_cosine": 0.5578733682632446, "beta_mean": 0.1795618236064911, "nonfinite": 0.0, "weighted_memory_to_active": 0.2193567156791687, "mix_std": 0.3211504817008972, "beta_std": 0.233944833278656, "beta_low": 0.11773681640625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.01983128860592842, "evidence_weights": [-0.10228201746940613, -0.10943176597356796, 0.12508521974086761], "agreement_weight": 0.006706996355205774, "orthogonal_weight": 0.12287578731775284}]}, {"layer": 9, "memory_to_active": 1.5093053579330444, "contribution_to_active": 0.1595071405172348, "mix_mean": 0.06795687973499298, "mix_low": 0.37541502714157104, "mix_high": 0.0, "read_cosine": 0.3736903667449951, "beta_mean": 0.15407109260559082, "nonfinite": 0.0, "weighted_memory_to_active": 0.1698601245880127, "mix_std": 0.07247383892536163, "beta_std": 0.06834493577480316, "beta_low": 0.0113932304084301, "beta_high": 0.0, "orthogonal_to_active": 0.006297608371824026, "output_influence": 0.11130371689796448, "residual_influence": 0.020761463791131973, "state_active": 0.01654212363064289, "state_memory": 0.04851624369621277, "state_peak": 23.6389217376709, "growth_active": 0.8790090084075928, "growth_memory": 1.8656034469604492, "heads": [{"head": 0, "memory_to_active": 1.4941906929016113, "contribution_to_active": 0.08874063938856125, "mix_mean": 0.03276066482067108, "mix_low": 0.45022106170654297, "mix_high": 0.0, "read_cosine": 0.31198370456695557, "beta_mean": 0.23317492008209229, "nonfinite": 0.0, "weighted_memory_to_active": 0.09092524647712708, "mix_std": 0.055508360266685486, "beta_std": 0.11049510538578033, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029312395490705967, "evidence_weights": [-0.007727079559117556, -0.008615938946604729, 0.03041013330221176], "agreement_weight": -0.025977658107876778, "orthogonal_weight": 0.0367242805659771}, {"head": 1, "memory_to_active": 1.6526601314544678, "contribution_to_active": 0.030012404546141624, "mix_mean": 0.009994711726903915, "mix_low": 0.726496696472168, "mix_high": 0.0, "read_cosine": 0.2581053376197815, "beta_mean": 0.04831261932849884, "nonfinite": 0.0, "weighted_memory_to_active": 0.0306945089250803, "mix_std": 0.015205282717943192, "beta_std": 0.03884682059288025, "beta_low": 0.068359375, "beta_high": 0.0, "orthogonal_to_active": 0.0003085954813286662, "evidence_weights": [0.020131994038820267, 0.02380376309156418, -0.032916586846113205], "agreement_weight": -0.0034521538764238358, "orthogonal_weight": -0.010789811611175537}, {"head": 2, "memory_to_active": 0.962882936000824, "contribution_to_active": 0.13693037629127502, "mix_mean": 0.03546863794326782, "mix_low": 0.4121894836425781, "mix_high": 0.0, "read_cosine": 0.19379538297653198, "beta_mean": 0.14416542649269104, "nonfinite": 0.0, "weighted_memory_to_active": 0.08652203530073166, "mix_std": 0.059010472148656845, "beta_std": 0.08534637838602066, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030248272232711315, "evidence_weights": [-0.03148090839385986, -0.025299977511167526, 0.010339704342186451], "agreement_weight": 0.040830470621585846, "orthogonal_weight": 0.04339071735739708}, {"head": 3, "memory_to_active": 1.3585717678070068, "contribution_to_active": 0.251711905002594, "mix_mean": 0.13044553995132446, "mix_low": 0.03821563720703125, "mix_high": 0.0, "read_cosine": 0.4691113829612732, "beta_mean": 0.09169338643550873, "nonfinite": 0.0, "weighted_memory_to_active": 0.25591468811035156, "mix_std": 0.10033681988716125, "beta_std": 0.035013578832149506, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013359016738831997, "evidence_weights": [-0.06769842654466629, -0.06270154565572739, 0.05156915634870529], "agreement_weight": -0.05640807002782822, "orthogonal_weight": 0.06036091223359108}, {"head": 4, "memory_to_active": 1.273144006729126, "contribution_to_active": 0.335532009601593, "mix_mean": 0.17377513647079468, "mix_low": 0.060894012451171875, "mix_high": 0.0, "read_cosine": 0.511694073677063, "beta_mean": 0.11597221344709396, "nonfinite": 0.0, "weighted_memory_to_active": 0.42004165053367615, "mix_std": 0.1524398922920227, "beta_std": 0.04330024868249893, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.017415089532732964, "evidence_weights": [-0.07974901050329208, -0.07599534094333649, 0.08552542328834534], "agreement_weight": -0.008617553859949112, "orthogonal_weight": 0.05844144895672798}, {"head": 5, "memory_to_active": 2.314382553100586, "contribution_to_active": 0.11411552131175995, "mix_mean": 0.025296591222286224, "mix_low": 0.5644731521606445, "mix_high": 0.0, "read_cosine": 0.49745213985443115, "beta_mean": 0.2911079525947571, "nonfinite": 0.0, "weighted_memory_to_active": 0.13506260514259338, "mix_std": 0.05234219878911972, "beta_std": 0.09706747531890869, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007468830444850028, "evidence_weights": [-0.0008958112448453903, 0.0002934187068603933, -0.027470940724015236], "agreement_weight": -0.0008693499257788062, "orthogonal_weight": 0.006656501907855272}]}, {"layer": 10, "memory_to_active": 0.9403178691864014, "contribution_to_active": 0.20971153676509857, "mix_mean": 0.0917426198720932, "mix_low": 0.08187390118837357, "mix_high": 0.0036287307739257812, "read_cosine": 0.38509422540664673, "beta_mean": 0.11548592150211334, "nonfinite": 0.0, "weighted_memory_to_active": 0.14461418986320496, "mix_std": 0.13476665318012238, "beta_std": 0.1022031158208847, "beta_low": 0.07000732421875, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.0037157684564590454, "output_influence": 0.2940627932548523, "residual_influence": 0.07186158001422882, "state_active": 0.06681288778781891, "state_memory": 0.028610792011022568, "state_peak": 16.858495712280273, "growth_active": 1.3124356269836426, "growth_memory": 1.7253538370132446, "heads": [{"head": 0, "memory_to_active": 0.7438116073608398, "contribution_to_active": 0.3336372971534729, "mix_mean": 0.12104257941246033, "mix_low": 0.16582489013671875, "mix_high": 0.01863861083984375, "read_cosine": 0.13415905833244324, "beta_mean": 0.11109457910060883, "nonfinite": 0.0, "weighted_memory_to_active": 0.1560683697462082, "mix_std": 0.20055244863033295, "beta_std": 0.16182607412338257, "beta_low": 0.13336181640625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0017990643391385674, "evidence_weights": [-0.02647276781499386, -0.0510968342423439, 0.11260659247636795], "agreement_weight": 0.040607910603284836, "orthogonal_weight": 0.013151595368981361}, {"head": 1, "memory_to_active": 1.2634830474853516, "contribution_to_active": 0.11192349344491959, "mix_mean": 0.05744898319244385, "mix_low": 0.025483131408691406, "mix_high": 0.0, "read_cosine": 0.39863938093185425, "beta_mean": 0.08156877756118774, "nonfinite": 0.0, "weighted_memory_to_active": 0.12191872298717499, "mix_std": 0.07646115124225616, "beta_std": 0.041762739419937134, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008245991775766015, "evidence_weights": [-0.05905941501259804, -0.05472048744559288, 0.0649324283003807], "agreement_weight": -0.01985447108745575, "orthogonal_weight": 0.008291206322610378}, {"head": 2, "memory_to_active": 1.4751304388046265, "contribution_to_active": 0.36094802618026733, "mix_mean": 0.06580961495637894, "mix_low": 0.05832958221435547, "mix_high": 0.00048065185546875, "read_cosine": 0.2715243399143219, "beta_mean": 0.149158775806427, "nonfinite": 0.0, "weighted_memory_to_active": 0.3600706458091736, "mix_std": 0.10571566224098206, "beta_std": 0.06918329000473022, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013866976834833622, "evidence_weights": [-0.06331101804971695, -0.05793902277946472, 0.02714851126074791], "agreement_weight": -0.05595601350069046, "orthogonal_weight": 0.041904743760824203}, {"head": 3, "memory_to_active": 0.23275671899318695, "contribution_to_active": 0.26858535408973694, "mix_mean": 0.18925023078918457, "mix_low": 0.15073394775390625, "mix_high": 0.00258636474609375, "read_cosine": 0.6818022727966309, "beta_mean": 0.08222435414791107, "nonfinite": 0.0, "weighted_memory_to_active": 0.08215533941984177, "mix_std": 0.26730647683143616, "beta_std": 0.1352517008781433, "beta_low": 0.239501953125, "beta_high": 0.0, "orthogonal_to_active": 0.004756412003189325, "evidence_weights": [-0.08539076149463654, -0.08783653378486633, 0.14057035744190216], "agreement_weight": 0.009492117911577225, "orthogonal_weight": 0.08093588054180145}, {"head": 4, "memory_to_active": 1.3401135206222534, "contribution_to_active": 0.06044293940067291, "mix_mean": 0.033014874905347824, "mix_low": 0.04662609100341797, "mix_high": 0.0, "read_cosine": 0.32596340775489807, "beta_mean": 0.16726790368556976, "nonfinite": 0.0, "weighted_memory_to_active": 0.06183667108416557, "mix_std": 0.034398872405290604, "beta_std": 0.07205575704574585, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 5.892532135476358e-05, "evidence_weights": [-0.0341515839099884, -0.031011072918772697, -0.019306741654872894], "agreement_weight": -0.06367263942956924, "orthogonal_weight": -0.0010469204280525446}, {"head": 5, "memory_to_active": 0.5866116285324097, "contribution_to_active": 0.12273206561803818, "mix_mean": 0.0838894248008728, "mix_low": 0.04424571990966797, "mix_high": 6.67572021484375e-05, "read_cosine": 0.4984769821166992, "beta_mean": 0.10160112380981445, "nonfinite": 0.0, "weighted_memory_to_active": 0.08563531190156937, "mix_std": 0.1241653636097908, "beta_std": 0.13313910365104675, "beta_low": 0.04718017578125, "beta_high": 0.0, "orthogonal_to_active": 0.000988634186796844, "evidence_weights": [-0.07364897429943085, -0.06882991641759872, 0.06700205057859421], "agreement_weight": 0.04240773618221283, "orthogonal_weight": 0.014415696263313293}]}, {"layer": 11, "memory_to_active": 0.5558773279190063, "contribution_to_active": 0.21666763722896576, "mix_mean": 0.13697195053100586, "mix_low": 0.13202938437461853, "mix_high": 0.012343406677246094, "read_cosine": 0.5216681957244873, "beta_mean": 0.08774951100349426, "nonfinite": 0.0, "weighted_memory_to_active": 0.10927225649356842, "mix_std": 0.18913236260414124, "beta_std": 0.10687252879142761, "beta_low": 0.0862426832318306, "beta_high": 0.0, "orthogonal_to_active": 0.0062565929256379604, "output_influence": 0.38822850584983826, "residual_influence": 0.10827123373746872, "state_active": 0.06888125836849213, "state_memory": 0.028393980115652084, "state_peak": 17.224809646606445, "growth_active": 1.2232288122177124, "growth_memory": 1.6942226886749268, "heads": [{"head": 0, "memory_to_active": 0.716224193572998, "contribution_to_active": 0.053470708429813385, "mix_mean": 0.033375125378370285, "mix_low": 0.16443443298339844, "mix_high": 0.0, "read_cosine": 0.47329628467559814, "beta_mean": 0.09083917737007141, "nonfinite": 0.0, "weighted_memory_to_active": 0.050430238246917725, "mix_std": 0.04526186361908913, "beta_std": 0.07416189461946487, "beta_low": 0.01025390625, "beta_high": 0.0, "orthogonal_to_active": 0.0004721285367850214, "evidence_weights": [-0.02133983001112938, -0.01006980799138546, -0.04745129868388176], "agreement_weight": 0.0842709168791771, "orthogonal_weight": 0.014276544563472271}, {"head": 1, "memory_to_active": 0.7638885974884033, "contribution_to_active": 0.10271376371383667, "mix_mean": 0.08116282522678375, "mix_low": 0.03737831115722656, "mix_high": 0.0006961822509765625, "read_cosine": 0.48685723543167114, "beta_mean": 0.08770914375782013, "nonfinite": 0.0, "weighted_memory_to_active": 0.09161119163036346, "mix_std": 0.11645182967185974, "beta_std": 0.08460090309381485, "beta_low": 0.01318359375, "beta_high": 0.0, "orthogonal_to_active": 0.003560079727321863, "evidence_weights": [-0.06851416826248169, -0.06283014267683029, 0.03658159449696541], "agreement_weight": 0.040141645818948746, "orthogonal_weight": 0.050762224942445755}, {"head": 2, "memory_to_active": 0.19406813383102417, "contribution_to_active": 0.3926902413368225, "mix_mean": 0.21326759457588196, "mix_low": 0.3336143493652344, "mix_high": 0.02789783477783203, "read_cosine": 0.4815709888935089, "beta_mean": 0.08618263155221939, "nonfinite": 0.0, "weighted_memory_to_active": 0.104837566614151, "mix_std": 0.3290334939956665, "beta_std": 0.16184186935424805, "beta_low": 0.29229736328125, "beta_high": 0.0, "orthogonal_to_active": 0.013811253011226654, "evidence_weights": [-0.07064895331859589, -0.07777974009513855, 0.14379197359085083], "agreement_weight": -0.03396414592862129, "orthogonal_weight": 0.16066975891590118}, {"head": 3, "memory_to_active": 0.32251301407814026, "contribution_to_active": 0.4044575095176697, "mix_mean": 0.2859423756599426, "mix_low": 0.18082904815673828, "mix_high": 0.043877601623535156, "read_cosine": 0.44753018021583557, "beta_mean": 0.13459981977939606, "nonfinite": 0.0, "weighted_memory_to_active": 0.15435060858726501, "mix_std": 0.3519240617752075, "beta_std": 0.21668127179145813, "beta_low": 0.1728515625, "beta_high": 0.0, "orthogonal_to_active": 0.012199717573821545, "evidence_weights": [-0.1407303363084793, -0.13356059789657593, 0.1350124180316925], "agreement_weight": -0.03974566236138344, "orthogonal_weight": 0.09333091974258423}, {"head": 4, "memory_to_active": 0.8484745025634766, "contribution_to_active": 0.22458824515342712, "mix_mean": 0.10327188670635223, "mix_low": 0.04642009735107422, "mix_high": 0.0015726089477539062, "read_cosine": 0.46660107374191284, "beta_mean": 0.0581999272108078, "nonfinite": 0.0, "weighted_memory_to_active": 0.1656048595905304, "mix_std": 0.15461008250713348, "beta_std": 0.04986026510596275, "beta_low": 0.0211181640625, "beta_high": 0.0, "orthogonal_to_active": 0.004106538370251656, "evidence_weights": [-0.07672670483589172, -0.0735289677977562, 0.06003795191645622], "agreement_weight": -0.014483288861811161, "orthogonal_weight": 0.030702631920576096}, {"head": 5, "memory_to_active": 0.49009546637535095, "contribution_to_active": 0.12208528816699982, "mix_mean": 0.10481192171573639, "mix_low": 0.02950000762939453, "mix_high": 1.621246337890625e-05, "read_cosine": 0.774153470993042, "beta_mean": 0.068966343998909, "nonfinite": 0.0, "weighted_memory_to_active": 0.08879902958869934, "mix_std": 0.13751283288002014, "beta_std": 0.05408892780542374, "beta_low": 0.00775146484375, "beta_high": 0.0, "orthogonal_to_active": 0.0033898388501256704, "evidence_weights": [-0.10288047045469284, -0.10298144817352295, 0.11689693480730057], "agreement_weight": 0.02427237294614315, "orthogonal_weight": 0.06155763193964958}]}, {"layer": 12, "memory_to_active": 1.298872947692871, "contribution_to_active": 0.10254543274641037, "mix_mean": 0.04074009507894516, "mix_low": 0.17793132364749908, "mix_high": 0.0, "read_cosine": 0.3643713593482971, "beta_mean": 0.18923239409923553, "nonfinite": 0.0, "weighted_memory_to_active": 0.12820446491241455, "mix_std": 0.039624035358428955, "beta_std": 0.1262802928686142, "beta_low": 0.0047607421875, "beta_high": 0.0, "orthogonal_to_active": 0.004168949089944363, "output_influence": 0.08059296011924744, "residual_influence": 0.01787503995001316, "state_active": 0.03740713745355606, "state_memory": 0.050248220562934875, "state_peak": 19.15057945251465, "growth_active": 1.1494522094726562, "growth_memory": 1.803122878074646, "heads": [{"head": 0, "memory_to_active": 1.2667073011398315, "contribution_to_active": 0.24768926203250885, "mix_mean": 0.09730274975299835, "mix_low": 0.010480880737304688, "mix_high": 0.0, "read_cosine": 0.6582118272781372, "beta_mean": 0.12213216722011566, "nonfinite": 0.0, "weighted_memory_to_active": 0.41752535104751587, "mix_std": 0.08630087226629257, "beta_std": 0.13263864815235138, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.01473489124327898, "evidence_weights": [-0.08140332996845245, -0.08396507054567337, 0.09905026108026505], "agreement_weight": 0.03865474835038185, "orthogonal_weight": 0.08750944584608078}, {"head": 1, "memory_to_active": 1.3161975145339966, "contribution_to_active": 0.04185407608747482, "mix_mean": 0.016941722482442856, "mix_low": 0.3666343688964844, "mix_high": 0.0, "read_cosine": 0.2684754729270935, "beta_mean": 0.23250126838684082, "nonfinite": 0.0, "weighted_memory_to_active": 0.039418548345565796, "mix_std": 0.016129687428474426, "beta_std": 0.18862596154212952, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.00037791390786878765, "evidence_weights": [0.0054217432625591755, 0.0034320896957069635, 0.06016073748469353], "agreement_weight": 0.10428306460380554, "orthogonal_weight": -0.01334924716502428}, {"head": 2, "memory_to_active": 0.6068501472473145, "contribution_to_active": 0.06483371555805206, "mix_mean": 0.04635543003678322, "mix_low": 0.0532989501953125, "mix_high": 0.0, "read_cosine": 0.29085761308670044, "beta_mean": 0.047830283641815186, "nonfinite": 0.0, "weighted_memory_to_active": 0.03697216510772705, "mix_std": 0.043636471033096313, "beta_std": 0.03951901197433472, "beta_low": 0.02783203125, "beta_high": 0.0, "orthogonal_to_active": 0.0005160169093869627, "evidence_weights": [-0.03212103992700577, -0.029018832370638847, 0.010677684098482132], "agreement_weight": 0.04507596790790558, "orthogonal_weight": 0.015172146260738373}, {"head": 3, "memory_to_active": 2.082873821258545, "contribution_to_active": 0.11301837861537933, "mix_mean": 0.03360767289996147, "mix_low": 0.11211776733398438, "mix_high": 0.0, "read_cosine": 0.21288777887821198, "beta_mean": 0.24521538615226746, "nonfinite": 0.0, "weighted_memory_to_active": 0.10596659779548645, "mix_std": 0.028328636661171913, "beta_std": 0.1000705361366272, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012770072789862752, "evidence_weights": [-0.027189940214157104, -0.030119145289063454, 0.005047478247433901], "agreement_weight": -0.10137967765331268, "orthogonal_weight": 0.012269394472241402}, {"head": 4, "memory_to_active": 1.6209888458251953, "contribution_to_active": 0.05349840223789215, "mix_mean": 0.025385677814483643, "mix_low": 0.17501258850097656, "mix_high": 0.0, "read_cosine": 0.3127594292163849, "beta_mean": 0.21697956323623657, "nonfinite": 0.0, "weighted_memory_to_active": 0.049647435545921326, "mix_std": 0.020649297162890434, "beta_std": 0.10148521512746811, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006593517609871924, "evidence_weights": [-0.021584324538707733, -0.016699345782399178, -0.032375648617744446], "agreement_weight": -0.08811115473508835, "orthogonal_weight": 0.013651969842612743}, {"head": 5, "memory_to_active": 0.8996198177337646, "contribution_to_active": 0.09437873214483261, "mix_mean": 0.024847283959388733, "mix_low": 0.35004329681396484, "mix_high": 0.0, "read_cosine": 0.44303590059280396, "beta_mean": 0.27073565125465393, "nonfinite": 0.0, "weighted_memory_to_active": 0.11969669908285141, "mix_std": 0.04269925132393837, "beta_std": 0.19534233212471008, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007448513060808182, "evidence_weights": [0.002172280801460147, -0.010271475650370121, 0.054060276597738266], "agreement_weight": 0.11956866085529327, "orthogonal_weight": -0.15206407010555267}]}, {"layer": 13, "memory_to_active": 0.9358019828796387, "contribution_to_active": 0.12419701367616653, "mix_mean": 0.08538739383220673, "mix_low": 0.23361842334270477, "mix_high": 8.50359647301957e-05, "read_cosine": 0.3295585513114929, "beta_mean": 0.17130225896835327, "nonfinite": 0.0, "weighted_memory_to_active": 0.08480361104011536, "mix_std": 0.09011908620595932, "beta_std": 0.13693267107009888, "beta_low": 0.1290181577205658, "beta_high": 0.0, "orthogonal_to_active": 0.00514183659106493, "output_influence": 0.17115473747253418, "residual_influence": 0.045224882662296295, "state_active": 0.07432448118925095, "state_memory": 0.04294530302286148, "state_peak": 15.868112564086914, "growth_active": 1.460447907447815, "growth_memory": 1.5739896297454834, "heads": [{"head": 0, "memory_to_active": 2.0690970420837402, "contribution_to_active": 0.02364744246006012, "mix_mean": 0.009654787369072437, "mix_low": 0.6484594345092773, "mix_high": 0.0, "read_cosine": 0.23883642256259918, "beta_mean": 0.507077693939209, "nonfinite": 0.0, "weighted_memory_to_active": 0.02862360328435898, "mix_std": 0.010335740633308887, "beta_std": 0.26973724365234375, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.0014632614329457283, "evidence_weights": [0.04214927926659584, 0.043858323246240616, -0.041704315692186356], "agreement_weight": 0.05613204464316368, "orthogonal_weight": -0.05989733338356018}, {"head": 1, "memory_to_active": 1.3840546607971191, "contribution_to_active": 0.034138888120651245, "mix_mean": 0.015021534636616707, "mix_low": 0.45496082305908203, "mix_high": 0.0, "read_cosine": 0.2651556432247162, "beta_mean": 0.1897987425327301, "nonfinite": 0.0, "weighted_memory_to_active": 0.0698726549744606, "mix_std": 0.017540911212563515, "beta_std": 0.15594223141670227, "beta_low": 0.00146484375, "beta_high": 0.0, "orthogonal_to_active": 0.0009942523902282119, "evidence_weights": [0.013075475580990314, 0.011973245069384575, -0.001246152212843299], "agreement_weight": 0.09234149008989334, "orthogonal_weight": -0.03494422137737274}, {"head": 2, "memory_to_active": 0.4598506689071655, "contribution_to_active": 0.1471279263496399, "mix_mean": 0.14085328578948975, "mix_low": 0.014246940612792969, "mix_high": 0.0, "read_cosine": 0.6300516128540039, "beta_mean": 0.0522192120552063, "nonfinite": 0.0, "weighted_memory_to_active": 0.10985498875379562, "mix_std": 0.11954817920923233, "beta_std": 0.09589241445064545, "beta_low": 0.2764892578125, "beta_high": 0.0, "orthogonal_to_active": 0.01089406292885542, "evidence_weights": [-0.1376171112060547, -0.13354337215423584, 0.1442873626947403], "agreement_weight": 0.07499328255653381, "orthogonal_weight": 0.1484268456697464}, {"head": 3, "memory_to_active": 0.14886882901191711, "contribution_to_active": 0.15207555890083313, "mix_mean": 0.08702310919761658, "mix_low": 0.05034446716308594, "mix_high": 0.0, "read_cosine": -0.07318875938653946, "beta_mean": 0.017827030271291733, "nonfinite": 0.0, "weighted_memory_to_active": 0.03647788241505623, "mix_std": 0.10126754641532898, "beta_std": 0.013498252257704735, "beta_low": 0.33905029296875, "beta_high": 0.0, "orthogonal_to_active": 0.00026447034906595945, "evidence_weights": [-0.10446624457836151, -0.10695088654756546, 0.07759088277816772], "agreement_weight": 0.08062078803777695, "orthogonal_weight": 0.007713285740464926}, {"head": 4, "memory_to_active": 0.4083033800125122, "contribution_to_active": 0.2379460632801056, "mix_mean": 0.21113812923431396, "mix_low": 0.027564048767089844, "mix_high": 0.0005102157592773438, "read_cosine": 0.6643826961517334, "beta_mean": 0.0912991464138031, "nonfinite": 0.0, "weighted_memory_to_active": 0.1263050138950348, "mix_std": 0.22117188572883606, "beta_std": 0.12201869487762451, "beta_low": 0.147705078125, "beta_high": 0.0, "orthogonal_to_active": 0.009806607849895954, "evidence_weights": [-0.09894034266471863, -0.102674201130867, 0.12132451683282852], "agreement_weight": 0.02508760616183281, "orthogonal_weight": 0.1118633896112442}, {"head": 5, "memory_to_active": 1.1446372270584106, "contribution_to_active": 0.15024620294570923, "mix_mean": 0.048633474856615067, "mix_low": 0.20613479614257812, "mix_high": 0.0, "read_cosine": 0.2521136403083801, "beta_mean": 0.1695917546749115, "nonfinite": 0.0, "weighted_memory_to_active": 0.13768747448921204, "mix_std": 0.07085023075342178, "beta_std": 0.16450713574886322, "beta_low": 0.00933837890625, "beta_high": 0.0, "orthogonal_to_active": 0.007428365759551525, "evidence_weights": [-0.03666318207979202, -0.03235756605863571, 0.00029434741009026766], "agreement_weight": -0.027763979509472847, "orthogonal_weight": 0.05753828212618828}]}, {"layer": 14, "memory_to_active": 0.4540847837924957, "contribution_to_active": 0.2311563491821289, "mix_mean": 0.18831846117973328, "mix_low": 0.07146644592285156, "mix_high": 0.00014209747314453125, "read_cosine": 0.6323144435882568, "beta_mean": 0.07575929909944534, "nonfinite": 0.0, "weighted_memory_to_active": 0.129116952419281, "mix_std": 0.17031662166118622, "beta_std": 0.08538475632667542, "beta_low": 0.0617268905043602, "beta_high": 0.0, "orthogonal_to_active": 0.004032636061310768, "output_influence": 0.13342693448066711, "residual_influence": 0.03285325691103935, "state_active": 0.08254703879356384, "state_memory": 0.036780185997486115, "state_peak": 23.68768882751465, "growth_active": 1.4069744348526, "growth_memory": 2.001615285873413, "heads": [{"head": 0, "memory_to_active": 0.5043478012084961, "contribution_to_active": 0.3026500344276428, "mix_mean": 0.2479017823934555, "mix_low": 0.00998687744140625, "mix_high": 0.00014209747314453125, "read_cosine": 0.6283482313156128, "beta_mean": 0.07924382388591766, "nonfinite": 0.0, "weighted_memory_to_active": 0.17775331437587738, "mix_std": 0.19972588121891022, "beta_std": 0.10615117102861404, "beta_low": 0.0426025390625, "beta_high": 0.0, "orthogonal_to_active": 0.006256652995944023, "evidence_weights": [-0.1299353837966919, -0.13878069818019867, 0.13045643270015717], "agreement_weight": 0.06371406465768814, "orthogonal_weight": 0.0504019632935524}, {"head": 1, "memory_to_active": 0.39791905879974365, "contribution_to_active": 0.3184913992881775, "mix_mean": 0.2711309492588043, "mix_low": 0.040230751037597656, "mix_high": 8.96453857421875e-05, "read_cosine": 0.5047937631607056, "beta_mean": 0.06229403614997864, "nonfinite": 0.0, "weighted_memory_to_active": 0.14537343382835388, "mix_std": 0.22857700288295746, "beta_std": 0.08194579184055328, "beta_low": 0.1146240234375, "beta_high": 0.0, "orthogonal_to_active": 0.003956885077059269, "evidence_weights": [-0.10032646358013153, -0.10628031194210052, 0.12105609476566315], "agreement_weight": 0.06336544454097748, "orthogonal_weight": -0.03761081025004387}, {"head": 2, "memory_to_active": 0.5035257339477539, "contribution_to_active": 0.17992626130580902, "mix_mean": 0.11858916282653809, "mix_low": 0.1291332244873047, "mix_high": 0.0, "read_cosine": 0.5761762261390686, "beta_mean": 0.11353138089179993, "nonfinite": 0.0, "weighted_memory_to_active": 0.14190930128097534, "mix_std": 0.13298986852169037, "beta_std": 0.08886094391345978, "beta_low": 0.0008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.0009423464071005583, "evidence_weights": [-0.07182743400335312, -0.0760284811258316, 0.059374235570430756], "agreement_weight": 0.0963440090417862, "orthogonal_weight": 0.011508703231811523}, {"head": 3, "memory_to_active": 0.24678447842597961, "contribution_to_active": 0.33063948154449463, "mix_mean": 0.2607697546482086, "mix_low": 0.043562889099121094, "mix_high": 0.0006208419799804688, "read_cosine": 0.7809845209121704, "beta_mean": 0.04998134821653366, "nonfinite": 0.0, "weighted_memory_to_active": 0.09955824911594391, "mix_std": 0.2279115617275238, "beta_std": 0.04755299165844917, "beta_low": 0.03369140625, "beta_high": 0.0, "orthogonal_to_active": 0.001815147465094924, "evidence_weights": [-0.15070201456546783, -0.1506868302822113, 0.15359841287136078], "agreement_weight": 0.05320984125137329, "orthogonal_weight": 0.03424384444952011}, {"head": 4, "memory_to_active": 0.4628545045852661, "contribution_to_active": 0.16145312786102295, "mix_mean": 0.15174415707588196, "mix_low": 0.049083709716796875, "mix_high": 0.0, "read_cosine": 0.6750195026397705, "beta_mean": 0.07620932906866074, "nonfinite": 0.0, "weighted_memory_to_active": 0.12198728322982788, "mix_std": 0.14522084593772888, "beta_std": 0.09802023321390152, "beta_low": 0.031005859375, "beta_high": 0.0, "orthogonal_to_active": 0.007450855802744627, "evidence_weights": [-0.06620434671640396, -0.06541670113801956, 0.05178007483482361], "agreement_weight": 0.04878160357475281, "orthogonal_weight": 0.08956056088209152}, {"head": 5, "memory_to_active": 0.6090771555900574, "contribution_to_active": 0.09377774596214294, "mix_mean": 0.07977497577667236, "mix_low": 0.1568012237548828, "mix_high": 0.0, "read_cosine": 0.6285642385482788, "beta_mean": 0.07329587638378143, "nonfinite": 0.0, "weighted_memory_to_active": 0.08812005072832108, "mix_std": 0.087474524974823, "beta_std": 0.0897774025797844, "beta_low": 0.1475830078125, "beta_high": 0.0, "orthogonal_to_active": 0.003773928852751851, "evidence_weights": [-0.0637165829539299, -0.0603828988969326, 0.04144253581762314], "agreement_weight": 0.05822183936834335, "orthogonal_weight": 0.06280692666769028}]}, {"layer": 15, "memory_to_active": 0.2415475994348526, "contribution_to_active": 0.2615998387336731, "mix_mean": 0.1717013716697693, "mix_low": 0.049542587250471115, "mix_high": 0.0005304018850438297, "read_cosine": 0.6720632910728455, "beta_mean": 0.04092768579721451, "nonfinite": 0.0, "weighted_memory_to_active": 0.08742557466030121, "mix_std": 0.17855703830718994, "beta_std": 0.050850965082645416, "beta_low": 0.3335774838924408, "beta_high": 0.0, "orthogonal_to_active": 0.0021273719612509012, "output_influence": 0.1351228654384613, "residual_influence": 0.04688861221075058, "state_active": 0.10581614077091217, "state_memory": 0.03353411331772804, "state_peak": 32.77020263671875, "growth_active": 1.4236985445022583, "growth_memory": 1.6736561059951782, "heads": [{"head": 0, "memory_to_active": 0.23998621106147766, "contribution_to_active": 0.293423056602478, "mix_mean": 0.26751214265823364, "mix_low": 0.004837989807128906, "mix_high": 1.239776611328125e-05, "read_cosine": 0.7392483949661255, "beta_mean": 0.0876530259847641, "nonfinite": 0.0, "weighted_memory_to_active": 0.07923099398612976, "mix_std": 0.2021283358335495, "beta_std": 0.10689405351877213, "beta_low": 0.0611572265625, "beta_high": 0.0, "orthogonal_to_active": 0.004866770934313536, "evidence_weights": [-0.17619776725769043, -0.1882602870464325, 0.19651998579502106], "agreement_weight": 0.05603979155421257, "orthogonal_weight": 0.0950867235660553}, {"head": 1, "memory_to_active": 0.43725818395614624, "contribution_to_active": 0.18652290105819702, "mix_mean": 0.10913890600204468, "mix_low": 0.05715465545654297, "mix_high": 9.5367431640625e-07, "read_cosine": 0.6729634404182434, "beta_mean": 0.04035421088337898, "nonfinite": 0.0, "weighted_memory_to_active": 0.10765095800161362, "mix_std": 0.13245251774787903, "beta_std": 0.02587798610329628, "beta_low": 0.0225830078125, "beta_high": 0.0, "orthogonal_to_active": 0.0004447204992175102, "evidence_weights": [-0.08704158663749695, -0.09117493778467178, 0.09907971322536469], "agreement_weight": 0.09742612391710281, "orthogonal_weight": -0.007510444615036249}, {"head": 2, "memory_to_active": 0.14770323038101196, "contribution_to_active": 0.26021432876586914, "mix_mean": 0.16507753729820251, "mix_low": 0.06178569793701172, "mix_high": 0.0008392333984375, "read_cosine": 0.807927131652832, "beta_mean": 0.012020151130855083, "nonfinite": 0.0, "weighted_memory_to_active": 0.04459017515182495, "mix_std": 0.19453831017017365, "beta_std": 0.01018854882568121, "beta_low": 0.54962158203125, "beta_high": 0.0, "orthogonal_to_active": 0.0005629053339362144, "evidence_weights": [-0.1442512720823288, -0.14225803315639496, 0.15407463908195496], "agreement_weight": 0.06167301908135414, "orthogonal_weight": -0.025324035435914993}, {"head": 3, "memory_to_active": 0.39873746037483215, "contribution_to_active": 0.2869640290737152, "mix_mean": 0.10442547500133514, "mix_low": 0.06177330017089844, "mix_high": 5.7220458984375e-06, "read_cosine": 0.7429389953613281, "beta_mean": 0.07888224720954895, "nonfinite": 0.0, "weighted_memory_to_active": 0.24478894472122192, "mix_std": 0.1357191801071167, "beta_std": 0.13003525137901306, "beta_low": 0.11871337890625, "beta_high": 0.0, "orthogonal_to_active": 0.005391882266849279, "evidence_weights": [-0.08250829577445984, -0.10470578819513321, 0.12145710736513138], "agreement_weight": 0.07303216308355331, "orthogonal_weight": 0.0691058337688446}, {"head": 4, "memory_to_active": 0.1817060261964798, "contribution_to_active": 0.17880623042583466, "mix_mean": 0.1297595351934433, "mix_low": 0.07156181335449219, "mix_high": 0.00045299530029296875, "read_cosine": 0.4532736837863922, "beta_mean": 0.023164238780736923, "nonfinite": 0.0, "weighted_memory_to_active": 0.03365051746368408, "mix_std": 0.15700215101242065, "beta_std": 0.026434164494276047, "beta_low": 0.31866455078125, "beta_high": 0.0, "orthogonal_to_active": 0.000982365570962429, "evidence_weights": [-0.1422916054725647, -0.13661497831344604, 0.13489080965518951], "agreement_weight": 0.07355381548404694, "orthogonal_weight": -0.03469863533973694}, {"head": 5, "memory_to_active": 0.043894488364458084, "contribution_to_active": 0.3636682629585266, "mix_mean": 0.25429457426071167, "mix_low": 0.040142059326171875, "mix_high": 0.0018711090087890625, "read_cosine": 0.61602783203125, "beta_mean": 0.0034922496415674686, "nonfinite": 0.0, "weighted_memory_to_active": 0.014641856774687767, "mix_std": 0.24950172007083893, "beta_std": 0.005675774533301592, "beta_low": 0.93072509765625, "beta_high": 0.0, "orthogonal_to_active": 0.0005155864637345076, "evidence_weights": [-0.2027525156736374, -0.1938769370317459, 0.20430023968219757], "agreement_weight": 0.041830793023109436, "orthogonal_weight": -0.04870852828025818}]}]}, {"step": 2000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8113520959996094, "memory_to_active": 1.1554800271987915, "contribution_to_active": 0.23050424456596375, "mix_mean": 0.11619294434785843, "mix_low": 0.20069080591201782, "mix_high": 0.0024956665001809597, "read_cosine": 0.43156591057777405, "beta_mean": 0.139984592795372, "nonfinite": 0.0, "weighted_memory_to_active": 0.16060417890548706, "mix_std": 0.12117066979408264, "beta_std": 0.09151624888181686, "beta_low": 0.12995976209640503, "beta_high": 2.543131586207892e-06, "orthogonal_to_active": 0.011750968173146248, "samples": 4, "scored_tokens": 899, "normal_loss": 3.591984510421753, "normal_token_accuracy": 0.40266963839530945, "state_active": 0.06552737206220627, "state_memory": 0.0422172024846077, "state_peak": 42.93809127807617, "growth_active": 1.1564066410064697, "growth_memory": 1.6863257884979248, "output_influence": 0.2269592583179474, "residual_influence": 0.05788205564022064, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.7260959148406982, "no_memory_loss_delta": 0.08356726169586182, "scrambled_loss_delta": 0.4978235363960266, "memory_only_loss_delta": 1.1165937185287476, "no_evidence_loss_delta": 0.020706787705421448, "no_agreement_loss_delta": 0.016333622857928276, "no_orthogonal_loss_delta": 0.0043076337315142155}, {"index": 94, "tokens": 256, "normal_loss": 3.73533034324646, "no_memory_loss_delta": 0.14959029853343964, "scrambled_loss_delta": 0.4298478364944458, "memory_only_loss_delta": 1.2059398889541626, "no_evidence_loss_delta": 0.025123639032244682, "no_agreement_loss_delta": 0.019693035632371902, "no_orthogonal_loss_delta": -0.0025801402516663074}, {"index": 188, "tokens": 256, "normal_loss": 3.9206838607788086, "no_memory_loss_delta": 0.040923550724983215, "scrambled_loss_delta": 0.10860754549503326, "memory_only_loss_delta": 1.3206337690353394, "no_evidence_loss_delta": 0.0011219661682844162, "no_agreement_loss_delta": -0.012284116819500923, "no_orthogonal_loss_delta": -0.0018548708176240325}, {"index": 283, "tokens": 131, "normal_loss": 2.4074366092681885, "no_memory_loss_delta": 0.11333587020635605, "scrambled_loss_delta": 0.3993604779243469, "memory_only_loss_delta": 1.380405306816101, "no_evidence_loss_delta": 0.016966886818408966, "no_agreement_loss_delta": -0.014239334501326084, "no_orthogonal_loss_delta": 0.003811219474300742}], "agreement_weight_abs_mean": 0.0509067066013813, "agreement_weight_abs_max": 0.1448127180337906, "agreement_weight_mean": 0.013217180036008358, "orthogonal_weight_abs_mean": 0.09748291224241257, "orthogonal_weight_abs_max": 0.4855039417743683, "orthogonal_weight_mean": 0.029207676649093628, "no_memory_loss": 3.6865475177764893, "no_memory_loss_delta": 0.0945625975728035, "no_memory_token_accuracy_gain": 0.01223581749945879, "no_memory_token_win_fraction": 0.5561735033988953, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 9.022921659930407, "scrambled_loss": 3.94527006149292, "scrambled_loss_delta": 0.35328546166419983, "scrambled_token_accuracy_gain": 0.046718575060367584, "scrambled_token_win_fraction": 0.6017797589302063, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 29.762333685440023, "memory_only_loss": 4.830565452575684, "memory_only_loss_delta": 1.2385804653167725, "memory_only_token_accuracy_gain": 0.1535038948059082, "memory_only_token_win_fraction": 0.8242491483688354, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 71.02046994228053, "no_evidence_loss": 3.6078274250030518, "no_evidence_loss_delta": 0.01584257371723652, "no_evidence_token_accuracy_gain": 0.0, "no_evidence_token_win_fraction": 0.5261401534080505, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 1.5717740243481975, "no_agreement_loss": 3.596670627593994, "no_agreement_loss_delta": 0.004686026833951473, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.5083426237106323, "no_agreement_probe_win_fraction": 0.5, "no_agreement_ppl_reduction_pct": 0.46750645400928365, "no_orthogonal_loss": 3.592503786087036, "no_orthogonal_loss_delta": 0.0005190891679376364, "no_orthogonal_token_accuracy_gain": 0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5216907858848572, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 0.05189544644642141, "short_no_memory_loss_delta": 0.0502241775393486, "long_context_memory_gain": 0.044338420033454895, "normal_context_loss_gain": 0.12026944756507874, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 1.6152276992797852, "contribution_to_active": 0.34772875905036926, "mix_mean": 0.1198432594537735, "mix_low": 0.018851280212402344, "mix_high": 0.00024716061307117343, "read_cosine": 0.6173242330551147, "beta_mean": 0.29380157589912415, "nonfinite": 0.0, "weighted_memory_to_active": 0.33626189827919006, "mix_std": 0.08655655384063721, "beta_std": 0.11334124952554703, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04462466388940811, "output_influence": 0.23873326182365417, "residual_influence": 0.16369159519672394, "state_active": 0.07066553831100464, "state_memory": 0.06380428373813629, "state_peak": 23.72179412841797, "growth_active": 1.0241039991378784, "growth_memory": 1.4391196966171265, "heads": [{"head": 0, "memory_to_active": 2.232405662536621, "contribution_to_active": 0.391584575176239, "mix_mean": 0.0547097809612751, "mix_low": 0.014367103576660156, "mix_high": 0.0, "read_cosine": 0.46572548151016235, "beta_mean": 0.28895631432533264, "nonfinite": 0.0, "weighted_memory_to_active": 0.4065417945384979, "mix_std": 0.05105604976415634, "beta_std": 0.08528195321559906, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03992719575762749, "evidence_weights": [-0.07625209540128708, -0.08420030027627945, 0.17278485000133514], "agreement_weight": -0.060160137712955475, "orthogonal_weight": 0.12105625122785568}, {"head": 1, "memory_to_active": 1.4942728281021118, "contribution_to_active": 0.6491847038269043, "mix_mean": 0.09084278345108032, "mix_low": 0.015267372131347656, "mix_high": 4.100799560546875e-05, "read_cosine": 0.6413099765777588, "beta_mean": 0.31546488404273987, "nonfinite": 0.0, "weighted_memory_to_active": 0.7397494316101074, "mix_std": 0.07085443288087845, "beta_std": 0.13225506246089935, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0999266728758812, "evidence_weights": [-0.14774461090564728, -0.1644948422908783, 0.22892791032791138], "agreement_weight": -0.02322533167898655, "orthogonal_weight": 0.2563217580318451}, {"head": 2, "memory_to_active": 1.4086799621582031, "contribution_to_active": 0.14049863815307617, "mix_mean": 0.06147532910108566, "mix_low": 0.027849197387695312, "mix_high": 0.0, "read_cosine": 0.5988146066665649, "beta_mean": 0.3215150237083435, "nonfinite": 0.0, "weighted_memory_to_active": 0.14618340134620667, "mix_std": 0.04653683677315712, "beta_std": 0.10351534187793732, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01754014566540718, "evidence_weights": [-0.10015920549631119, -0.11271616816520691, 0.1409880667924881], "agreement_weight": -0.06828376650810242, "orthogonal_weight": 0.15759554505348206}, {"head": 3, "memory_to_active": 1.6996257305145264, "contribution_to_active": 0.38601261377334595, "mix_mean": 0.13749387860298157, "mix_low": 0.0098876953125, "mix_high": 0.0, "read_cosine": 0.6374584436416626, "beta_mean": 0.32268089056015015, "nonfinite": 0.0, "weighted_memory_to_active": 0.4139379560947418, "mix_std": 0.08993706107139587, "beta_std": 0.13705319166183472, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05795666575431824, "evidence_weights": [-0.17881688475608826, -0.19327372312545776, 0.2547457814216614], "agreement_weight": -0.060402389615774155, "orthogonal_weight": 0.19925230741500854}, {"head": 4, "memory_to_active": 2.5689382553100586, "contribution_to_active": 0.16905459761619568, "mix_mean": 0.04221928492188454, "mix_low": 0.040741920471191406, "mix_high": 0.0, "read_cosine": 0.49390703439712524, "beta_mean": 0.3107417821884155, "nonfinite": 0.0, "weighted_memory_to_active": 0.15725979208946228, "mix_std": 0.03381439670920372, "beta_std": 0.13920339941978455, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.026374075561761856, "evidence_weights": [-0.05700184404850006, -0.06591637432575226, 0.16369104385375977], "agreement_weight": -0.06946790963411331, "orthogonal_weight": 0.19445791840553284}, {"head": 5, "memory_to_active": 0.28744378685951233, "contribution_to_active": 0.35003745555877686, "mix_mean": 0.33231842517852783, "mix_low": 0.004994392395019531, "mix_high": 0.00144195556640625, "read_cosine": 0.8667296171188354, "beta_mean": 0.2034505158662796, "nonfinite": 0.0, "weighted_memory_to_active": 0.1538989245891571, "mix_std": 0.22714054584503174, "beta_std": 0.08273850381374359, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02602320909500122, "evidence_weights": [-0.37534618377685547, -0.3565063774585724, 0.39389312267303467], "agreement_weight": 0.12246192991733551, "orthogonal_weight": 0.34301847219467163}]}, {"layer": 1, "memory_to_active": 2.1152029037475586, "contribution_to_active": 0.15437674522399902, "mix_mean": 0.055193014442920685, "mix_low": 0.22300037741661072, "mix_high": 0.0, "read_cosine": 0.43594890832901, "beta_mean": 0.2737695276737213, "nonfinite": 0.0, "weighted_memory_to_active": 0.15424028038978577, "mix_std": 0.051412433385849, "beta_std": 0.12581582367420197, "beta_low": 0.00010172525799134746, "beta_high": 0.0, "orthogonal_to_active": 0.013877121731638908, "output_influence": 0.11532975733280182, "residual_influence": 0.021379947662353516, "state_active": 0.04629124328494072, "state_memory": 0.06616265326738358, "state_peak": 20.102794647216797, "growth_active": 1.0321286916732788, "growth_memory": 1.768631935119629, "heads": [{"head": 0, "memory_to_active": 3.061467170715332, "contribution_to_active": 0.09490342438220978, "mix_mean": 0.013654053211212158, "mix_low": 0.5055141448974609, "mix_high": 0.0, "read_cosine": 0.13015376031398773, "beta_mean": 0.25237560272216797, "nonfinite": 0.0, "weighted_memory_to_active": 0.07086323201656342, "mix_std": 0.016651462763547897, "beta_std": 0.10800666362047195, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020320331677794456, "evidence_weights": [0.011745466850697994, 0.013934066519141197, -0.07686220854520798], "agreement_weight": -0.0928560197353363, "orthogonal_weight": -0.029460128396749496}, {"head": 1, "memory_to_active": 1.9861459732055664, "contribution_to_active": 0.189295694231987, "mix_mean": 0.05988912656903267, "mix_low": 0.036510467529296875, "mix_high": 0.0, "read_cosine": 0.574069082736969, "beta_mean": 0.264432430267334, "nonfinite": 0.0, "weighted_memory_to_active": 0.19040030241012573, "mix_std": 0.05645336955785751, "beta_std": 0.09129113703966141, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01939348131418228, "evidence_weights": [-0.10119616985321045, -0.0960649773478508, 0.09348027408123016], "agreement_weight": -0.07874442636966705, "orthogonal_weight": 0.12359067797660828}, {"head": 2, "memory_to_active": 0.828657329082489, "contribution_to_active": 0.24608595669269562, "mix_mean": 0.15445908904075623, "mix_low": 0.012354850769042969, "mix_high": 0.0, "read_cosine": 0.6994694471359253, "beta_mean": 0.14635369181632996, "nonfinite": 0.0, "weighted_memory_to_active": 0.2779560089111328, "mix_std": 0.12966234982013702, "beta_std": 0.060554344207048416, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.032249923795461655, "evidence_weights": [-0.18356798589229584, -0.18295231461524963, 0.20162415504455566], "agreement_weight": 0.020666154101490974, "orthogonal_weight": 0.18416105210781097}, {"head": 3, "memory_to_active": 2.1540465354919434, "contribution_to_active": 0.05162844434380531, "mix_mean": 0.018154945224523544, "mix_low": 0.3689155578613281, "mix_high": 0.0, "read_cosine": 0.36370372772216797, "beta_mean": 0.44158846139907837, "nonfinite": 0.0, "weighted_memory_to_active": 0.05248834937810898, "mix_std": 0.016508445143699646, "beta_std": 0.19947291910648346, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.002269346732646227, "evidence_weights": [-0.0251042190939188, -0.023492781445384026, -0.007624747231602669], "agreement_weight": -0.01947646588087082, "orthogonal_weight": 0.04767819866538048}, {"head": 4, "memory_to_active": 1.578859567642212, "contribution_to_active": 0.25312602519989014, "mix_mean": 0.06717190146446228, "mix_low": 0.025526046752929688, "mix_high": 0.0, "read_cosine": 0.5310126543045044, "beta_mean": 0.13143715262413025, "nonfinite": 0.0, "weighted_memory_to_active": 0.24253332614898682, "mix_std": 0.06170796602964401, "beta_std": 0.06442499905824661, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.027194377034902573, "evidence_weights": [-0.10565973818302155, -0.10424716025590897, 0.16857683658599854], "agreement_weight": -0.033379774540662766, "orthogonal_weight": 0.12345936894416809}, {"head": 5, "memory_to_active": 3.082040786743164, "contribution_to_active": 0.0912209153175354, "mix_mean": 0.017828935757279396, "mix_low": 0.38918113708496094, "mix_high": 0.0, "read_cosine": 0.31728440523147583, "beta_mean": 0.4064297676086426, "nonfinite": 0.0, "weighted_memory_to_active": 0.09120054543018341, "mix_std": 0.027491014450788498, "beta_std": 0.23114481568336487, "beta_low": 0.0006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.00012356421211734414, "evidence_weights": [-0.009955359622836113, -0.009341149590909481, 0.0756540298461914], "agreement_weight": -0.05430920049548149, "orthogonal_weight": -0.0014914125204086304}]}, {"layer": 2, "memory_to_active": 1.5996079444885254, "contribution_to_active": 0.11264816671609879, "mix_mean": 0.04020991176366806, "mix_low": 0.1846882551908493, "mix_high": 2.5590263248886913e-05, "read_cosine": 0.4431022107601166, "beta_mean": 0.16198116540908813, "nonfinite": 0.0, "weighted_memory_to_active": 0.10350585728883743, "mix_std": 0.051451511681079865, "beta_std": 0.09245707094669342, "beta_low": 0.1727091521024704, "beta_high": 0.0, "orthogonal_to_active": 0.004781048744916916, "output_influence": 0.19372615218162537, "residual_influence": 0.03107782080769539, "state_active": 0.03101624920964241, "state_memory": 0.045801371335983276, "state_peak": 30.532886505126953, "growth_active": 0.9708555340766907, "growth_memory": 1.7076836824417114, "heads": [{"head": 0, "memory_to_active": 1.8339523077011108, "contribution_to_active": 0.07302771508693695, "mix_mean": 0.018706824630498886, "mix_low": 0.4126701354980469, "mix_high": 0.0, "read_cosine": 0.27703627943992615, "beta_mean": 0.07909141480922699, "nonfinite": 0.0, "weighted_memory_to_active": 0.06577292829751968, "mix_std": 0.025202181190252304, "beta_std": 0.04738355427980423, "beta_low": 0.05084228515625, "beta_high": 0.0, "orthogonal_to_active": 0.0036048064939677715, "evidence_weights": [-0.02143188938498497, -0.019456099718809128, 0.05607709288597107], "agreement_weight": -0.05740327388048172, "orthogonal_weight": 0.05691003426909447}, {"head": 1, "memory_to_active": 1.798563003540039, "contribution_to_active": 0.08386600017547607, "mix_mean": 0.028035469353199005, "mix_low": 0.15036678314208984, "mix_high": 0.0, "read_cosine": 0.3618038296699524, "beta_mean": 0.16756823658943176, "nonfinite": 0.0, "weighted_memory_to_active": 0.08307836204767227, "mix_std": 0.024699300527572632, "beta_std": 0.08600535988807678, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.002655887743458152, "evidence_weights": [-0.0356728695333004, -0.03449009358882904, 0.0473020076751709], "agreement_weight": -0.02434070035815239, "orthogonal_weight": 0.03504304215312004}, {"head": 2, "memory_to_active": 0.8496609926223755, "contribution_to_active": 0.08854065090417862, "mix_mean": 0.060520656406879425, "mix_low": 0.04849052429199219, "mix_high": 0.0, "read_cosine": 0.5924789905548096, "beta_mean": 0.05844564735889435, "nonfinite": 0.0, "weighted_memory_to_active": 0.1183818131685257, "mix_std": 0.05509594455361366, "beta_std": 0.027007946744561195, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.0034367237240076065, "evidence_weights": [-0.07496093958616257, -0.0667719617486, 0.08288030326366425], "agreement_weight": 0.047875482589006424, "orthogonal_weight": 0.05986277014017105}, {"head": 3, "memory_to_active": 1.5053532123565674, "contribution_to_active": 0.06023329496383667, "mix_mean": 0.03698955103754997, "mix_low": 0.12775039672851562, "mix_high": 0.0, "read_cosine": 0.5528576374053955, "beta_mean": 0.37093108892440796, "nonfinite": 0.0, "weighted_memory_to_active": 0.08463893830776215, "mix_std": 0.033633213490247726, "beta_std": 0.19021497666835785, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018613273277878761, "evidence_weights": [-0.0463077574968338, -0.05998220667243004, 0.14106683433055878], "agreement_weight": 0.035201720893383026, "orthogonal_weight": 0.03347909450531006}, {"head": 4, "memory_to_active": 3.091259002685547, "contribution_to_active": 0.2252756655216217, "mix_mean": 0.022888440638780594, "mix_low": 0.30292510986328125, "mix_high": 0.0, "read_cosine": 0.2653595209121704, "beta_mean": 0.2931925058364868, "nonfinite": 0.0, "weighted_memory_to_active": 0.22211769223213196, "mix_std": 0.050938982516527176, "beta_std": 0.201494038105011, "beta_low": 0.0059814453125, "beta_high": 0.0, "orthogonal_to_active": 0.015540183521807194, "evidence_weights": [-0.016987767070531845, -0.015137099660933018, 0.07440836727619171], "agreement_weight": -0.0035694569814950228, "orthogonal_weight": 0.07613299787044525}, {"head": 5, "memory_to_active": 0.5188583731651306, "contribution_to_active": 0.14494560658931732, "mix_mean": 0.07411850988864899, "mix_low": 0.06592655181884766, "mix_high": 0.00015354156494140625, "read_cosine": 0.6090769171714783, "beta_mean": 0.0026580693665891886, "nonfinite": 0.0, "weighted_memory_to_active": 0.04704541712999344, "mix_std": 0.11913945525884628, "beta_std": 0.002636550460010767, "beta_low": 0.979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.0015873630763962865, "evidence_weights": [-0.07749450951814651, -0.06595039367675781, 0.1092313602566719], "agreement_weight": -0.0019669781904667616, "orthogonal_weight": 0.031268030405044556}]}, {"layer": 3, "memory_to_active": 1.9830414056777954, "contribution_to_active": 0.11859278380870819, "mix_mean": 0.025557782500982285, "mix_low": 0.33147287368774414, "mix_high": 0.00021457672119140625, "read_cosine": 0.336691290140152, "beta_mean": 0.19886469841003418, "nonfinite": 0.0, "weighted_memory_to_active": 0.09848064929246902, "mix_std": 0.0418642982840538, "beta_std": 0.11271193623542786, "beta_low": 0.0145670585334301, "beta_high": 0.0, "orthogonal_to_active": 0.0018891912186518312, "output_influence": 0.16005297005176544, "residual_influence": 0.022279739379882812, "state_active": 0.037369515746831894, "state_memory": 0.05022231861948967, "state_peak": 30.30024528503418, "growth_active": 0.997590959072113, "growth_memory": 1.7260950803756714, "heads": [{"head": 0, "memory_to_active": 1.7671483755111694, "contribution_to_active": 0.04358051344752312, "mix_mean": 0.01636234112083912, "mix_low": 0.48668575286865234, "mix_high": 0.0, "read_cosine": 0.22061534225940704, "beta_mean": 0.04033592715859413, "nonfinite": 0.0, "weighted_memory_to_active": 0.041722703725099564, "mix_std": 0.028012778609991074, "beta_std": 0.028154689818620682, "beta_low": 0.05322265625, "beta_high": 0.0, "orthogonal_to_active": 0.0007107434794306755, "evidence_weights": [0.013472247868776321, 0.0036494156811386347, -0.05473236367106438], "agreement_weight": -0.08040720969438553, "orthogonal_weight": -0.017771270126104355}, {"head": 1, "memory_to_active": 1.5574114322662354, "contribution_to_active": 0.05082418769598007, "mix_mean": 0.02394815906882286, "mix_low": 0.26479434967041016, "mix_high": 0.0, "read_cosine": 0.36955875158309937, "beta_mean": 0.06956686079502106, "nonfinite": 0.0, "weighted_memory_to_active": 0.045501165091991425, "mix_std": 0.028856324031949043, "beta_std": 0.04666011780500412, "beta_low": 0.02764892578125, "beta_high": 0.0, "orthogonal_to_active": 0.00021034153178334236, "evidence_weights": [-0.02164953388273716, -0.018409060314297676, -0.00971940252929926], "agreement_weight": -0.05671211704611778, "orthogonal_weight": 0.0050168707966804504}, {"head": 2, "memory_to_active": 2.628391742706299, "contribution_to_active": 0.17992034554481506, "mix_mean": 0.03195742517709732, "mix_low": 0.15624427795410156, "mix_high": 0.0, "read_cosine": 0.38990867137908936, "beta_mean": 0.37606316804885864, "nonfinite": 0.0, "weighted_memory_to_active": 0.2058933526277542, "mix_std": 0.05104648321866989, "beta_std": 0.16709546744823456, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.003077305853366852, "evidence_weights": [-0.04802064970135689, -0.05017422139644623, 0.10466964542865753], "agreement_weight": -0.026192542165517807, "orthogonal_weight": 0.019211813807487488}, {"head": 3, "memory_to_active": 1.4442390203475952, "contribution_to_active": 0.03990587592124939, "mix_mean": 0.018766656517982483, "mix_low": 0.3911857604980469, "mix_high": 0.0, "read_cosine": 0.411678671836853, "beta_mean": 0.09073755145072937, "nonfinite": 0.0, "weighted_memory_to_active": 0.04105500131845474, "mix_std": 0.026410439983010292, "beta_std": 0.049418047070503235, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0002743698132690042, "evidence_weights": [0.006342653185129166, 0.01027658861130476, 0.01130259595811367], "agreement_weight": -0.05418875813484192, "orthogonal_weight": -0.007421587128192186}, {"head": 4, "memory_to_active": 3.5177135467529297, "contribution_to_active": 0.0499877966940403, "mix_mean": 0.010869812220335007, "mix_low": 0.5792331695556641, "mix_high": 0.0, "read_cosine": 0.22352102398872375, "beta_mean": 0.33821529150009155, "nonfinite": 0.0, "weighted_memory_to_active": 0.04971866309642792, "mix_std": 0.010288778692483902, "beta_std": 0.18273845314979553, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0022789048962295055, "evidence_weights": [0.04640812799334526, 0.04336213320493698, -0.08354219049215317], "agreement_weight": -0.1448127180337906, "orthogonal_weight": -0.04535388574004173}, {"head": 5, "memory_to_active": 0.9833434820175171, "contribution_to_active": 0.34733790159225464, "mix_mean": 0.05144229158759117, "mix_low": 0.11069393157958984, "mix_high": 0.0012874603271484375, "read_cosine": 0.404865026473999, "beta_mean": 0.2782692611217499, "nonfinite": 0.0, "weighted_memory_to_active": 0.2069929838180542, "mix_std": 0.10657098144292831, "beta_std": 0.20220480859279633, "beta_low": 0.00653076171875, "beta_high": 0.0, "orthogonal_to_active": 0.004783482290804386, "evidence_weights": [-0.03697003424167633, -0.07906807959079742, 0.07264890521764755], "agreement_weight": 0.08995359390974045, "orthogonal_weight": 0.033701807260513306}]}, {"layer": 4, "memory_to_active": 1.3192572593688965, "contribution_to_active": 0.12476307153701782, "mix_mean": 0.06215225160121918, "mix_low": 0.191966712474823, "mix_high": 1.1126201115985168e-06, "read_cosine": 0.4628886580467224, "beta_mean": 0.18081419169902802, "nonfinite": 0.0, "weighted_memory_to_active": 0.12418863922357559, "mix_std": 0.07085513323545456, "beta_std": 0.1150469034910202, "beta_low": 0.02330525778234005, "beta_high": 0.0, "orthogonal_to_active": 0.010173154063522816, "output_influence": 0.12274059653282166, "residual_influence": 0.02215469628572464, "state_active": 0.04209069162607193, "state_memory": 0.04999702423810959, "state_peak": 23.12236976623535, "growth_active": 1.042678713798523, "growth_memory": 1.9166781902313232, "heads": [{"head": 0, "memory_to_active": 0.8091039657592773, "contribution_to_active": 0.13748681545257568, "mix_mean": 0.0918915867805481, "mix_low": 0.05163288116455078, "mix_high": 0.0, "read_cosine": 0.5751941800117493, "beta_mean": 0.13280513882637024, "nonfinite": 0.0, "weighted_memory_to_active": 0.14036598801612854, "mix_std": 0.11905268579721451, "beta_std": 0.12265579402446747, "beta_low": 0.0235595703125, "beta_high": 0.0, "orthogonal_to_active": 0.0032354644499719143, "evidence_weights": [-0.08130645006895065, -0.09764014184474945, 0.11619981378316879], "agreement_weight": 0.03310438618063927, "orthogonal_weight": 0.034911148250103}, {"head": 1, "memory_to_active": 0.7890884280204773, "contribution_to_active": 0.12434443831443787, "mix_mean": 0.06605177372694016, "mix_low": 0.16295719146728516, "mix_high": 0.0, "read_cosine": 0.48182910680770874, "beta_mean": 0.13927537202835083, "nonfinite": 0.0, "weighted_memory_to_active": 0.12171900272369385, "mix_std": 0.0823885127902031, "beta_std": 0.10870756208896637, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.004461472854018211, "evidence_weights": [-0.07857154309749603, -0.08647901564836502, 0.13815370202064514], "agreement_weight": 0.04447997361421585, "orthogonal_weight": 0.0551028698682785}, {"head": 2, "memory_to_active": 2.3304226398468018, "contribution_to_active": 0.08914514631032944, "mix_mean": 0.015614725649356842, "mix_low": 0.522883415222168, "mix_high": 0.0, "read_cosine": 0.2712666690349579, "beta_mean": 0.26811057329177856, "nonfinite": 0.0, "weighted_memory_to_active": 0.09889114648103714, "mix_std": 0.02008524350821972, "beta_std": 0.11094959080219269, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005375394131988287, "evidence_weights": [-0.0037574241869151592, -0.007344995625317097, -0.018516583368182182], "agreement_weight": 0.0216376893222332, "orthogonal_weight": 0.06618642061948776}, {"head": 3, "memory_to_active": 1.961132526397705, "contribution_to_active": 0.07210889458656311, "mix_mean": 0.024964282289147377, "mix_low": 0.28779029846191406, "mix_high": 0.0, "read_cosine": 0.24465639889240265, "beta_mean": 0.32734858989715576, "nonfinite": 0.0, "weighted_memory_to_active": 0.07148697972297668, "mix_std": 0.029894117265939713, "beta_std": 0.20637157559394836, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.002661698032170534, "evidence_weights": [-0.0352536141872406, -0.026811815798282623, -0.09106507897377014], "agreement_weight": -0.004768847022205591, "orthogonal_weight": -0.04277121275663376}, {"head": 4, "memory_to_active": 0.8012277483940125, "contribution_to_active": 0.13200080394744873, "mix_mean": 0.10697861015796661, "mix_low": 0.023428916931152344, "mix_high": 0.0, "read_cosine": 0.5944415330886841, "beta_mean": 0.03584336116909981, "nonfinite": 0.0, "weighted_memory_to_active": 0.14233079552650452, "mix_std": 0.10247130692005157, "beta_std": 0.025229791179299355, "beta_low": 0.11566162109375, "beta_high": 0.0, "orthogonal_to_active": 0.008838461712002754, "evidence_weights": [-0.09802110493183136, -0.08762181550264359, 0.19585800170898438], "agreement_weight": -0.014200138859450817, "orthogonal_weight": 0.10411947220563889}, {"head": 5, "memory_to_active": 1.2245680093765259, "contribution_to_active": 0.1934923231601715, "mix_mean": 0.06741252541542053, "mix_low": 0.10310745239257812, "mix_high": 6.67572021484375e-06, "read_cosine": 0.6099441051483154, "beta_mean": 0.1815020591020584, "nonfinite": 0.0, "weighted_memory_to_active": 0.17033788561820984, "mix_std": 0.0712389200925827, "beta_std": 0.11636706441640854, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.036466434597969055, "evidence_weights": [-0.10043316334486008, -0.08780965954065323, 0.14757077395915985], "agreement_weight": -0.05582868307828903, "orthogonal_weight": 0.25248563289642334}]}, {"layer": 5, "memory_to_active": 1.0867226123809814, "contribution_to_active": 0.20235003530979156, "mix_mean": 0.1112561970949173, "mix_low": 0.19992780685424805, "mix_high": 0.0003121694317087531, "read_cosine": 0.398078054189682, "beta_mean": 0.11871408671140671, "nonfinite": 0.0, "weighted_memory_to_active": 0.13712938129901886, "mix_std": 0.1255166083574295, "beta_std": 0.09520422667264938, "beta_low": 0.0391947440803051, "beta_high": 0.0, "orthogonal_to_active": 0.010659358464181423, "output_influence": 0.2103969305753708, "residual_influence": 0.032670244574546814, "state_active": 0.08057266473770142, "state_memory": 0.0554082915186882, "state_peak": 38.39942932128906, "growth_active": 1.1257476806640625, "growth_memory": 2.2191219329833984, "heads": [{"head": 0, "memory_to_active": 1.2978532314300537, "contribution_to_active": 0.16946378350257874, "mix_mean": 0.06779316067695618, "mix_low": 0.06251049041748047, "mix_high": 0.0, "read_cosine": 0.20612120628356934, "beta_mean": 0.24717199802398682, "nonfinite": 0.0, "weighted_memory_to_active": 0.16849753260612488, "mix_std": 0.08715129643678665, "beta_std": 0.15349315106868744, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00568295456469059, "evidence_weights": [-0.07588107138872147, -0.07367105036973953, 0.05241867154836655], "agreement_weight": 0.0007618764648213983, "orthogonal_weight": 0.037107545882463455}, {"head": 1, "memory_to_active": 1.4754399061203003, "contribution_to_active": 0.07311389595270157, "mix_mean": 0.018864504992961884, "mix_low": 0.6151533126831055, "mix_high": 0.0, "read_cosine": 0.22932417690753937, "beta_mean": 0.11106526851654053, "nonfinite": 0.0, "weighted_memory_to_active": 0.04935616999864578, "mix_std": 0.0519857220351696, "beta_std": 0.1241668313741684, "beta_low": 0.08233642578125, "beta_high": 0.0, "orthogonal_to_active": 0.006901703774929047, "evidence_weights": [0.017433080822229385, 0.026829266920685768, -0.10201504081487656], "agreement_weight": 0.013435039669275284, "orthogonal_weight": -0.13458067178726196}, {"head": 2, "memory_to_active": 0.4878076910972595, "contribution_to_active": 0.22690996527671814, "mix_mean": 0.21699926257133484, "mix_low": 0.0139923095703125, "mix_high": 1.430511474609375e-05, "read_cosine": 0.7388227581977844, "beta_mean": 0.08469462394714355, "nonfinite": 0.0, "weighted_memory_to_active": 0.16002622246742249, "mix_std": 0.18512064218521118, "beta_std": 0.04528285562992096, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016555961221456528, "evidence_weights": [-0.18964767456054688, -0.19169752299785614, 0.2795637249946594], "agreement_weight": 0.08544189482927322, "orthogonal_weight": 0.18573494255542755}, {"head": 3, "memory_to_active": 1.9734989404678345, "contribution_to_active": 0.09142984449863434, "mix_mean": 0.02259398065507412, "mix_low": 0.401092529296875, "mix_high": 0.0, "read_cosine": 0.3231983184814453, "beta_mean": 0.13669733703136444, "nonfinite": 0.0, "weighted_memory_to_active": 0.09260290116071701, "mix_std": 0.03433022275567055, "beta_std": 0.1012284904718399, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.0020039137452840805, "evidence_weights": [0.002833534963428974, -0.0016691605560481548, -0.002884982619434595], "agreement_weight": -0.03625132516026497, "orthogonal_weight": -0.022839415818452835}, {"head": 4, "memory_to_active": 0.40138909220695496, "contribution_to_active": 0.3921402096748352, "mix_mean": 0.16669344902038574, "mix_low": 0.05935859680175781, "mix_high": 0.0018587112426757812, "read_cosine": 0.579430103302002, "beta_mean": 0.09336058795452118, "nonfinite": 0.0, "weighted_memory_to_active": 0.16859275102615356, "mix_std": 0.21173134446144104, "beta_std": 0.1263664960861206, "beta_low": 0.13690185546875, "beta_high": 0.0, "orthogonal_to_active": 0.02743031457066536, "evidence_weights": [-0.09700702875852585, -0.12532727420330048, 0.21363697946071625], "agreement_weight": 0.0726916640996933, "orthogonal_weight": 0.19922645390033722}, {"head": 5, "memory_to_active": 0.8843461275100708, "contribution_to_active": 0.26104244589805603, "mix_mean": 0.17459285259246826, "mix_low": 0.04745960235595703, "mix_high": 0.0, "read_cosine": 0.3115715980529785, "beta_mean": 0.03929468244314194, "nonfinite": 0.0, "weighted_memory_to_active": 0.18370068073272705, "mix_std": 0.1827804148197174, "beta_std": 0.02068750560283661, "beta_low": 0.015380859375, "beta_high": 0.0, "orthogonal_to_active": 0.00538130197674036, "evidence_weights": [-0.113508440554142, -0.11594212055206299, 0.15189656615257263], "agreement_weight": -0.015690961852669716, "orthogonal_weight": -0.03176986798644066}]}, {"layer": 6, "memory_to_active": 0.6467334032058716, "contribution_to_active": 0.24211865663528442, "mix_mean": 0.12709413468837738, "mix_low": 0.23663172125816345, "mix_high": 0.003951708786189556, "read_cosine": 0.4070890545845032, "beta_mean": 0.1038692519068718, "nonfinite": 0.0, "weighted_memory_to_active": 0.1388901323080063, "mix_std": 0.15494193136692047, "beta_std": 0.0874793753027916, "beta_low": 0.1924845576286316, "beta_high": 0.0, "orthogonal_to_active": 0.005510169081389904, "output_influence": 0.2983081340789795, "residual_influence": 0.061493881046772, "state_active": 0.06932757049798965, "state_memory": 0.03103168122470379, "state_peak": 30.530654907226562, "growth_active": 1.3118339776992798, "growth_memory": 1.6798603534698486, "heads": [{"head": 0, "memory_to_active": 0.5070229172706604, "contribution_to_active": 0.25992244482040405, "mix_mean": 0.08786597847938538, "mix_low": 0.1268768310546875, "mix_high": 0.002269744873046875, "read_cosine": 0.5202725529670715, "beta_mean": 0.25983408093452454, "nonfinite": 0.0, "weighted_memory_to_active": 0.17310689389705658, "mix_std": 0.15008045732975006, "beta_std": 0.14429086446762085, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0056760720908641815, "evidence_weights": [-0.06516069918870926, -0.07268145680427551, 0.12109559774398804], "agreement_weight": 0.10578647255897522, "orthogonal_weight": -0.04985760152339935}, {"head": 1, "memory_to_active": 0.5988935232162476, "contribution_to_active": 0.16118086874485016, "mix_mean": 0.09997822344303131, "mix_low": 0.052321434020996094, "mix_high": 8.58306884765625e-06, "read_cosine": 0.5385947227478027, "beta_mean": 0.043947357684373856, "nonfinite": 0.0, "weighted_memory_to_active": 0.15271010994911194, "mix_std": 0.11338009685277939, "beta_std": 0.027520090341567993, "beta_low": 0.0538330078125, "beta_high": 0.0, "orthogonal_to_active": 0.0005974264931865036, "evidence_weights": [-0.11567763984203339, -0.10641998052597046, 0.04029039666056633], "agreement_weight": 0.04428854584693909, "orthogonal_weight": -0.007933713495731354}, {"head": 2, "memory_to_active": 1.5703203678131104, "contribution_to_active": 0.14555777609348297, "mix_mean": 0.02885441854596138, "mix_low": 0.39492034912109375, "mix_high": 0.0, "read_cosine": 0.23713141679763794, "beta_mean": 0.14986073970794678, "nonfinite": 0.0, "weighted_memory_to_active": 0.12836280465126038, "mix_std": 0.064633309841156, "beta_std": 0.11787723004817963, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.009939474985003471, "evidence_weights": [0.00524887815117836, 0.006440364755690098, -0.06541737914085388], "agreement_weight": -0.03258884698152542, "orthogonal_weight": -0.08088676631450653}, {"head": 3, "memory_to_active": 0.3658919930458069, "contribution_to_active": 0.425771027803421, "mix_mean": 0.1925995796918869, "mix_low": 0.06696224212646484, "mix_high": 0.014005661010742188, "read_cosine": 0.5667349100112915, "beta_mean": 0.07877561450004578, "nonfinite": 0.0, "weighted_memory_to_active": 0.19225303828716278, "mix_std": 0.26171600818634033, "beta_std": 0.12325706332921982, "beta_low": 0.14404296875, "beta_high": 0.0, "orthogonal_to_active": 0.0062881275080144405, "evidence_weights": [-0.06430596113204956, -0.1313888281583786, 0.12852375209331512], "agreement_weight": 0.025533635169267654, "orthogonal_weight": 0.04467443749308586}, {"head": 4, "memory_to_active": 0.4049757122993469, "contribution_to_active": 0.4094390273094177, "mix_mean": 0.33622369170188904, "mix_low": 0.00402069091796875, "mix_high": 0.007426261901855469, "read_cosine": 0.359444260597229, "beta_mean": 0.08541791886091232, "nonfinite": 0.0, "weighted_memory_to_active": 0.1623980700969696, "mix_std": 0.2849452793598175, "beta_std": 0.10682713985443115, "beta_low": 0.08447265625, "beta_high": 0.0, "orthogonal_to_active": 0.005413290113210678, "evidence_weights": [-0.15384404361248016, -0.19891631603240967, -0.04160269722342491], "agreement_weight": -0.018984317779541016, "orthogonal_weight": 0.037346452474594116}, {"head": 5, "memory_to_active": 0.43329548835754395, "contribution_to_active": 0.050840798765420914, "mix_mean": 0.01704293303191662, "mix_low": 0.774688720703125, "mix_high": 0.0, "read_cosine": 0.22035636007785797, "beta_mean": 0.005379769951105118, "nonfinite": 0.0, "weighted_memory_to_active": 0.024509914219379425, "mix_std": 0.05489644408226013, "beta_std": 0.005103833042085171, "beta_low": 0.87255859375, "beta_high": 0.0, "orthogonal_to_active": 0.00514662079513073, "evidence_weights": [0.05133616179227829, 0.040627893060445786, -0.11916513741016388], "agreement_weight": 0.09608771651983261, "orthogonal_weight": -0.22593951225280762}]}, {"layer": 7, "memory_to_active": 1.9315952062606812, "contribution_to_active": 0.2145080268383026, "mix_mean": 0.035361066460609436, "mix_low": 0.3689464330673218, "mix_high": 3.178914482759865e-07, "read_cosine": 0.31716084480285645, "beta_mean": 0.23694245517253876, "nonfinite": 0.0, "weighted_memory_to_active": 0.22090351581573486, "mix_std": 0.05127577483654022, "beta_std": 0.106820248067379, "beta_low": 0.0003865560283884406, "beta_high": 0.0, "orthogonal_to_active": 0.01790536567568779, "output_influence": 0.19529269635677338, "residual_influence": 0.03223716467618942, "state_active": 0.018978625535964966, "state_memory": 0.0445440374314785, "state_peak": 24.154319763183594, "growth_active": 0.8992198705673218, "growth_memory": 1.4721529483795166, "heads": [{"head": 0, "memory_to_active": 2.03995418548584, "contribution_to_active": 0.6497268080711365, "mix_mean": 0.05740918964147568, "mix_low": 0.1596813201904297, "mix_high": 1.9073486328125e-06, "read_cosine": 0.37302327156066895, "beta_mean": 0.16525554656982422, "nonfinite": 0.0, "weighted_memory_to_active": 0.6774430274963379, "mix_std": 0.07433117926120758, "beta_std": 0.10711876302957535, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08859262615442276, "evidence_weights": [-0.05243277549743652, -0.06121734902262688, 0.08728449791669846], "agreement_weight": -0.032862454652786255, "orthogonal_weight": 0.19530823826789856}, {"head": 1, "memory_to_active": 2.1861464977264404, "contribution_to_active": 0.18331658840179443, "mix_mean": 0.04984539747238159, "mix_low": 0.15000629425048828, "mix_high": 0.0, "read_cosine": 0.1596870869398117, "beta_mean": 0.26003649830818176, "nonfinite": 0.0, "weighted_memory_to_active": 0.18663917481899261, "mix_std": 0.058390092104673386, "beta_std": 0.10459433495998383, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019633760675787926, "evidence_weights": [-0.05373726412653923, -0.04919779673218727, 0.052780747413635254], "agreement_weight": -0.022068368270993233, "orthogonal_weight": 0.011080553755164146}, {"head": 2, "memory_to_active": 1.4280699491500854, "contribution_to_active": 0.18689964711666107, "mix_mean": 0.05006773769855499, "mix_low": 0.37650108337402344, "mix_high": 0.0, "read_cosine": 0.33297306299209595, "beta_mean": 0.2291613072156906, "nonfinite": 0.0, "weighted_memory_to_active": 0.18739280104637146, "mix_std": 0.08790291100740433, "beta_std": 0.10389558970928192, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008589958772063255, "evidence_weights": [-0.05693957582116127, -0.06455116719007492, 0.08803020417690277], "agreement_weight": 0.017670758068561554, "orthogonal_weight": 0.05311855301260948}, {"head": 3, "memory_to_active": 2.4752163887023926, "contribution_to_active": 0.06848549842834473, "mix_mean": 0.016722306609153748, "mix_low": 0.4468679428100586, "mix_high": 0.0, "read_cosine": 0.27513426542282104, "beta_mean": 0.31284019351005554, "nonfinite": 0.0, "weighted_memory_to_active": 0.07003198564052582, "mix_std": 0.021004237234592438, "beta_std": 0.12183838337659836, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012327043805271387, "evidence_weights": [0.0072537194937467575, 0.015952182933688164, -0.035183049738407135], "agreement_weight": -0.07665800303220749, "orthogonal_weight": 0.018821770325303078}, {"head": 4, "memory_to_active": 1.3408772945404053, "contribution_to_active": 0.09137803316116333, "mix_mean": 0.023852262645959854, "mix_low": 0.4471445083618164, "mix_high": 0.0, "read_cosine": 0.3930679261684418, "beta_mean": 0.11822909116744995, "nonfinite": 0.0, "weighted_memory_to_active": 0.09360278397798538, "mix_std": 0.03843267261981964, "beta_std": 0.07355612516403198, "beta_low": 0.0023193359375, "beta_high": 0.0, "orthogonal_to_active": 0.00523561704903841, "evidence_weights": [-0.006202427204698324, -0.0011071646586060524, 0.0513484813272953], "agreement_weight": 0.04232798144221306, "orthogonal_weight": 0.08504387736320496}, {"head": 5, "memory_to_active": 2.1193063259124756, "contribution_to_active": 0.10724157094955444, "mix_mean": 0.014269499108195305, "mix_low": 0.6334772109985352, "mix_high": 0.0, "read_cosine": 0.3690793812274933, "beta_mean": 0.3361319899559021, "nonfinite": 0.0, "weighted_memory_to_active": 0.11031121015548706, "mix_std": 0.027593543753027916, "beta_std": 0.12991830706596375, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018179102335125208, "evidence_weights": [-0.005322644487023354, 0.004207316320389509, -0.04943263903260231], "agreement_weight": -0.025657393038272858, "orthogonal_weight": 0.021102821454405785}]}, {"layer": 8, "memory_to_active": 0.9762989282608032, "contribution_to_active": 0.24632342159748077, "mix_mean": 0.17438003420829773, "mix_low": 0.09438356012105942, "mix_high": 0.0054752035066485405, "read_cosine": 0.5757588148117065, "beta_mean": 0.09436008334159851, "nonfinite": 0.0, "weighted_memory_to_active": 0.18636196851730347, "mix_std": 0.16143681108951569, "beta_std": 0.07258159667253494, "beta_low": 0.1290384978055954, "beta_high": 0.0, "orthogonal_to_active": 0.013296356424689293, "output_influence": 0.26748132705688477, "residual_influence": 0.04682483896613121, "state_active": 0.06104626506567001, "state_memory": 0.03213953971862793, "state_peak": 15.738080024719238, "growth_active": 1.079911231994629, "growth_memory": 1.5928934812545776, "heads": [{"head": 0, "memory_to_active": 0.37366634607315063, "contribution_to_active": 0.19305412471294403, "mix_mean": 0.14421561360359192, "mix_low": 0.04219245910644531, "mix_high": 0.0, "read_cosine": 0.7417515516281128, "beta_mean": 0.013847353868186474, "nonfinite": 0.0, "weighted_memory_to_active": 0.1000136286020279, "mix_std": 0.15115493535995483, "beta_std": 0.013614149764180183, "beta_low": 0.47845458984375, "beta_high": 0.0, "orthogonal_to_active": 0.0033303566742688417, "evidence_weights": [-0.11295801401138306, -0.10733495652675629, 0.1767338067293167], "agreement_weight": 0.0627100020647049, "orthogonal_weight": 0.057802844792604446}, {"head": 1, "memory_to_active": 0.625378429889679, "contribution_to_active": 0.25560450553894043, "mix_mean": 0.280215859413147, "mix_low": 0.001338958740234375, "mix_high": 0.0, "read_cosine": 0.7598714828491211, "beta_mean": 0.04766582325100899, "nonfinite": 0.0, "weighted_memory_to_active": 0.2347913682460785, "mix_std": 0.17384549975395203, "beta_std": 0.026367589831352234, "beta_low": 0.00872802734375, "beta_high": 0.0, "orthogonal_to_active": 0.020632971078157425, "evidence_weights": [-0.18715406954288483, -0.17023903131484985, 0.22162511944770813], "agreement_weight": 0.005015003494918346, "orthogonal_weight": 0.15450207889080048}, {"head": 2, "memory_to_active": 1.9877985715866089, "contribution_to_active": 0.13393534719944, "mix_mean": 0.04379567876458168, "mix_low": 0.2594413757324219, "mix_high": 0.0, "read_cosine": 0.3897813856601715, "beta_mean": 0.2167423963546753, "nonfinite": 0.0, "weighted_memory_to_active": 0.14207102358341217, "mix_std": 0.0608413890004158, "beta_std": 0.10661687701940536, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00016522521036677063, "evidence_weights": [-0.04215230792760849, -0.050687965005636215, 0.02900497056543827], "agreement_weight": -0.039980243891477585, "orthogonal_weight": -0.0012656019534915686}, {"head": 3, "memory_to_active": 0.9343826174736023, "contribution_to_active": 0.1957816481590271, "mix_mean": 0.12980881333351135, "mix_low": 0.08731269836425781, "mix_high": 0.0, "read_cosine": 0.5727939009666443, "beta_mean": 0.05811088904738426, "nonfinite": 0.0, "weighted_memory_to_active": 0.21759355068206787, "mix_std": 0.12611353397369385, "beta_std": 0.049971066415309906, "beta_low": 0.02777099609375, "beta_high": 0.0, "orthogonal_to_active": 0.023524085059762, "evidence_weights": [-0.08400565385818481, -0.07754833996295929, 0.218580424785614], "agreement_weight": 0.02377961575984955, "orthogonal_weight": 0.1644809991121292}, {"head": 4, "memory_to_active": 1.6589405536651611, "contribution_to_active": 0.23240874707698822, "mix_mean": 0.08486413955688477, "mix_low": 0.07399845123291016, "mix_high": 0.0, "read_cosine": 0.4416179358959198, "beta_mean": 0.11748260259628296, "nonfinite": 0.0, "weighted_memory_to_active": 0.2655025124549866, "mix_std": 0.08145473897457123, "beta_std": 0.06269077956676483, "beta_low": 0.0010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.00413616094738245, "evidence_weights": [-0.07872164994478226, -0.07442277669906616, 0.06515296548604965], "agreement_weight": -0.029736125841736794, "orthogonal_weight": 0.019190963357686996}, {"head": 5, "memory_to_active": 0.2776269316673279, "contribution_to_active": 0.4671560525894165, "mix_mean": 0.3633800745010376, "mix_low": 0.10201740264892578, "mix_high": 0.032851219177246094, "read_cosine": 0.548736572265625, "beta_mean": 0.11231148988008499, "nonfinite": 0.0, "weighted_memory_to_active": 0.15819969773292542, "mix_std": 0.3752107620239258, "beta_std": 0.1762290745973587, "beta_low": 0.2581787109375, "beta_high": 0.0, "orthogonal_to_active": 0.02798933908343315, "evidence_weights": [-0.11130332201719284, -0.176203653216362, 0.026848673820495605], "agreement_weight": 0.02114514447748661, "orthogonal_weight": 0.25208351016044617}]}, {"layer": 9, "memory_to_active": 1.46702241897583, "contribution_to_active": 0.2666374742984772, "mix_mean": 0.062371350824832916, "mix_low": 0.48600658774375916, "mix_high": 6.882349407533184e-05, "read_cosine": 0.35337674617767334, "beta_mean": 0.1168174147605896, "nonfinite": 0.0, "weighted_memory_to_active": 0.2620265781879425, "mix_std": 0.10272137075662613, "beta_std": 0.06447595357894897, "beta_low": 0.0864969938993454, "beta_high": 0.0, "orthogonal_to_active": 0.006768024526536465, "output_influence": 0.19724923372268677, "residual_influence": 0.03606101870536804, "state_active": 0.023281095549464226, "state_memory": 0.046554528176784515, "state_peak": 19.47195816040039, "growth_active": 0.9363527297973633, "growth_memory": 1.8781249523162842, "heads": [{"head": 0, "memory_to_active": 1.2835801839828491, "contribution_to_active": 0.10897542536258698, "mix_mean": 0.03202475607395172, "mix_low": 0.603245735168457, "mix_high": 0.0, "read_cosine": 0.29354485869407654, "beta_mean": 0.11091361939907074, "nonfinite": 0.0, "weighted_memory_to_active": 0.10926087200641632, "mix_std": 0.08676809072494507, "beta_std": 0.06737763434648514, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.0006377262179739773, "evidence_weights": [0.021586544811725616, 0.006145945750176907, 0.08444260060787201], "agreement_weight": -0.04911290109157562, "orthogonal_weight": -0.006495587062090635}, {"head": 1, "memory_to_active": 1.0169812440872192, "contribution_to_active": 0.01513234805315733, "mix_mean": 0.005558200646191835, "mix_low": 0.8755359649658203, "mix_high": 0.0, "read_cosine": 0.2952656149864197, "beta_mean": 0.016774091869592667, "nonfinite": 0.0, "weighted_memory_to_active": 0.011137925088405609, "mix_std": 0.01533725019544363, "beta_std": 0.021683506667613983, "beta_low": 0.51739501953125, "beta_high": 0.0, "orthogonal_to_active": 0.0004269621567800641, "evidence_weights": [0.03791333734989166, 0.04502496123313904, -0.0512973926961422], "agreement_weight": 0.022012822329998016, "orthogonal_weight": -0.03913586959242821}, {"head": 2, "memory_to_active": 0.7387909889221191, "contribution_to_active": 0.30855172872543335, "mix_mean": 0.06984279304742813, "mix_low": 0.48900508880615234, "mix_high": 9.5367431640625e-07, "read_cosine": 0.1559414565563202, "beta_mean": 0.06203147768974304, "nonfinite": 0.0, "weighted_memory_to_active": 0.16847576200962067, "mix_std": 0.14568744599819183, "beta_std": 0.07361504435539246, "beta_low": 0.00146484375, "beta_high": 0.0, "orthogonal_to_active": 0.006317869294434786, "evidence_weights": [-0.04428475350141525, -0.035833198577165604, 0.0876428559422493], "agreement_weight": 0.05073343589901924, "orthogonal_weight": 0.0439734049141407}, {"head": 3, "memory_to_active": 1.8811161518096924, "contribution_to_active": 0.26033496856689453, "mix_mean": 0.08908629417419434, "mix_low": 0.15955734252929688, "mix_high": 0.0, "read_cosine": 0.4681176543235779, "beta_mean": 0.13204193115234375, "nonfinite": 0.0, "weighted_memory_to_active": 0.2749507427215576, "mix_std": 0.09729181975126266, "beta_std": 0.07156320661306381, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007206116337329149, "evidence_weights": [-0.03672179952263832, -0.03187090903520584, 0.017269665375351906], "agreement_weight": -0.04721863195300102, "orthogonal_weight": 0.029513945803046227}, {"head": 4, "memory_to_active": 2.0768539905548096, "contribution_to_active": 0.7221345901489258, "mix_mean": 0.14128704369068146, "mix_low": 0.14031696319580078, "mix_high": 0.0004119873046875, "read_cosine": 0.4232597053050995, "beta_mean": 0.171218603849411, "nonfinite": 0.0, "weighted_memory_to_active": 0.7660492062568665, "mix_std": 0.1842055469751358, "beta_std": 0.07013583928346634, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02105558291077614, "evidence_weights": [-0.06433800607919693, -0.0635068267583847, 0.09559962898492813], "agreement_weight": -0.054779425263404846, "orthogonal_weight": 0.0312229972332716}, {"head": 5, "memory_to_active": 1.8048107624053955, "contribution_to_active": 0.18469566106796265, "mix_mean": 0.03642899915575981, "mix_low": 0.6483783721923828, "mix_high": 0.0, "read_cosine": 0.48413118720054626, "beta_mean": 0.20792469382286072, "nonfinite": 0.0, "weighted_memory_to_active": 0.2422848641872406, "mix_std": 0.0870380625128746, "beta_std": 0.08248046040534973, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.004963889252394438, "evidence_weights": [0.010119020007550716, -0.0005242988700047135, -0.03135601431131363], "agreement_weight": -0.010302964597940445, "orthogonal_weight": 0.029121804982423782}]}, {"layer": 10, "memory_to_active": 0.9740488529205322, "contribution_to_active": 0.2639848589897156, "mix_mean": 0.11692776530981064, "mix_low": 0.21382173895835876, "mix_high": 0.003926753997802734, "read_cosine": 0.3693373501300812, "beta_mean": 0.08893520385026932, "nonfinite": 0.0, "weighted_memory_to_active": 0.17608708143234253, "mix_std": 0.14408992230892181, "beta_std": 0.09060238301753998, "beta_low": 0.1016031950712204, "beta_high": 0.0, "orthogonal_to_active": 0.016263218596577644, "output_influence": 0.33329904079437256, "residual_influence": 0.07618958503007889, "state_active": 0.09269417077302933, "state_memory": 0.02730391174554825, "state_peak": 42.57518005371094, "growth_active": 1.3090250492095947, "growth_memory": 1.4270507097244263, "heads": [{"head": 0, "memory_to_active": 0.9352350234985352, "contribution_to_active": 0.6894375681877136, "mix_mean": 0.2585050165653229, "mix_low": 0.052555084228515625, "mix_high": 0.017952919006347656, "read_cosine": 0.10730721801519394, "beta_mean": 0.12044720351696014, "nonfinite": 0.0, "weighted_memory_to_active": 0.4700997769832611, "mix_std": 0.26397740840911865, "beta_std": 0.16537797451019287, "beta_low": 0.106201171875, "beta_high": 0.0, "orthogonal_to_active": 0.06196138262748718, "evidence_weights": [-0.07710853219032288, -0.13796895742416382, 0.12469793111085892], "agreement_weight": 0.005221338476985693, "orthogonal_weight": 0.14058244228363037}, {"head": 1, "memory_to_active": 1.6664462089538574, "contribution_to_active": 0.10113301873207092, "mix_mean": 0.03196663409471512, "mix_low": 0.2832908630371094, "mix_high": 0.0, "read_cosine": 0.33187204599380493, "beta_mean": 0.10847865045070648, "nonfinite": 0.0, "weighted_memory_to_active": 0.11085095256567001, "mix_std": 0.04778710752725601, "beta_std": 0.07926160097122192, "beta_low": 0.001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.006055068224668503, "evidence_weights": [-0.024754906073212624, -0.015922091901302338, 0.01720220223069191], "agreement_weight": -0.015160748735070229, "orthogonal_weight": -0.06390734761953354}, {"head": 2, "memory_to_active": 1.6904633045196533, "contribution_to_active": 0.2905638515949249, "mix_mean": 0.04211273789405823, "mix_low": 0.28787994384765625, "mix_high": 2.86102294921875e-06, "read_cosine": 0.24872422218322754, "beta_mean": 0.10719303041696548, "nonfinite": 0.0, "weighted_memory_to_active": 0.3012332618236542, "mix_std": 0.07783414423465729, "beta_std": 0.06420689076185226, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00796443596482277, "evidence_weights": [-0.02766755409538746, -0.021411335095763206, -0.021478747949004173], "agreement_weight": -0.014536820352077484, "orthogonal_weight": -0.02773859165608883}, {"head": 3, "memory_to_active": 0.16642169654369354, "contribution_to_active": 0.3156752586364746, "mix_mean": 0.25023365020751953, "mix_low": 0.15511798858642578, "mix_high": 0.005526542663574219, "read_cosine": 0.6899243593215942, "beta_mean": 0.07713577151298523, "nonfinite": 0.0, "weighted_memory_to_active": 0.07333173602819443, "mix_std": 0.30756914615631104, "beta_std": 0.1183239296078682, "beta_low": 0.27276611328125, "beta_high": 0.0, "orthogonal_to_active": 0.014820434153079987, "evidence_weights": [-0.1285346895456314, -0.13237324357032776, 0.27491915225982666], "agreement_weight": 0.0149220647290349, "orthogonal_weight": 0.28120657801628113}, {"head": 4, "memory_to_active": 1.0499463081359863, "contribution_to_active": 0.040753308683633804, "mix_mean": 0.02062763273715973, "mix_low": 0.39312076568603516, "mix_high": 0.0, "read_cosine": 0.26337188482284546, "beta_mean": 0.0715588629245758, "nonfinite": 0.0, "weighted_memory_to_active": 0.042045921087265015, "mix_std": 0.029671208932995796, "beta_std": 0.05414154380559921, "beta_low": 0.0233154296875, "beta_high": 0.0, "orthogonal_to_active": 0.0035338629968464375, "evidence_weights": [0.009070112369954586, 0.018794681876897812, -0.10125240683555603], "agreement_weight": -0.01719796098768711, "orthogonal_weight": -0.09214773774147034}, {"head": 5, "memory_to_active": 0.3357802629470825, "contribution_to_active": 0.14634615182876587, "mix_mean": 0.09812086075544357, "mix_low": 0.11096572875976562, "mix_high": 7.82012939453125e-05, "read_cosine": 0.574824333190918, "beta_mean": 0.04879770427942276, "nonfinite": 0.0, "weighted_memory_to_active": 0.058960750699043274, "mix_std": 0.13770049810409546, "beta_std": 0.062302350997924805, "beta_low": 0.20611572265625, "beta_high": 0.0, "orthogonal_to_active": 0.003244119230657816, "evidence_weights": [-0.0778183564543724, -0.07481510937213898, 0.12369220703840256], "agreement_weight": 0.0805424228310585, "orthogonal_weight": -0.07717785239219666}]}, {"layer": 11, "memory_to_active": 0.43260157108306885, "contribution_to_active": 0.2909334897994995, "mix_mean": 0.18921686708927155, "mix_low": 0.12316402792930603, "mix_high": 0.023091476410627365, "read_cosine": 0.525297224521637, "beta_mean": 0.05556940287351608, "nonfinite": 0.0, "weighted_memory_to_active": 0.11161007732152939, "mix_std": 0.22175104916095734, "beta_std": 0.07338914275169373, "beta_low": 0.323577880859375, "beta_high": 0.0, "orthogonal_to_active": 0.009196050465106964, "output_influence": 0.4783788323402405, "residual_influence": 0.1237354427576065, "state_active": 0.10885220021009445, "state_memory": 0.021506136283278465, "state_peak": 21.94505500793457, "growth_active": 1.256115198135376, "growth_memory": 1.4736121892929077, "heads": [{"head": 0, "memory_to_active": 0.16323992609977722, "contribution_to_active": 0.21980996429920197, "mix_mean": 0.07881421595811844, "mix_low": 0.2044677734375, "mix_high": 0.0, "read_cosine": 0.48728156089782715, "beta_mean": 0.0015629174886271358, "nonfinite": 0.0, "weighted_memory_to_active": 0.046397287398576736, "mix_std": 0.13331329822540283, "beta_std": 0.0021234715823084116, "beta_low": 0.98858642578125, "beta_high": 0.0, "orthogonal_to_active": 0.005085189361125231, "evidence_weights": [-0.03887021541595459, -0.03198117017745972, 0.03327196091413498], "agreement_weight": 0.14376698434352875, "orthogonal_weight": -0.16711971163749695}, {"head": 1, "memory_to_active": 0.6255517601966858, "contribution_to_active": 0.1378125250339508, "mix_mean": 0.11932389438152313, "mix_low": 0.054653167724609375, "mix_high": 0.0, "read_cosine": 0.4638633728027344, "beta_mean": 0.039586275815963745, "nonfinite": 0.0, "weighted_memory_to_active": 0.11267514526844025, "mix_std": 0.1510435938835144, "beta_std": 0.05040469020605087, "beta_low": 0.21490478515625, "beta_high": 0.0, "orthogonal_to_active": 0.0011727489763870835, "evidence_weights": [-0.06739123165607452, -0.05487120896577835, -0.009633113630115986], "agreement_weight": 0.07522068917751312, "orthogonal_weight": 0.014198937453329563}, {"head": 2, "memory_to_active": 0.11213262379169464, "contribution_to_active": 0.5604053735733032, "mix_mean": 0.32639044523239136, "mix_low": 0.20973682403564453, "mix_high": 0.04746055603027344, "read_cosine": 0.4800631105899811, "beta_mean": 0.06760969012975693, "nonfinite": 0.0, "weighted_memory_to_active": 0.07752341032028198, "mix_std": 0.37615495920181274, "beta_std": 0.11610113084316254, "beta_low": 0.3868408203125, "beta_high": 0.0, "orthogonal_to_active": 0.030935490503907204, "evidence_weights": [-0.1271074414253235, -0.1369110643863678, 0.12902472913265228], "agreement_weight": -0.027104368433356285, "orthogonal_weight": 0.4855039417743683}, {"head": 3, "memory_to_active": 0.10471642762422562, "contribution_to_active": 0.4179709553718567, "mix_mean": 0.4074321389198303, "mix_low": 0.12335777282714844, "mix_high": 0.09108352661132812, "read_cosine": 0.5244550704956055, "beta_mean": 0.0969531461596489, "nonfinite": 0.0, "weighted_memory_to_active": 0.0578141063451767, "mix_std": 0.40002402663230896, "beta_std": 0.14731384813785553, "beta_low": 0.27801513671875, "beta_high": 0.0, "orthogonal_to_active": 0.014327412471175194, "evidence_weights": [-0.2159973382949829, -0.1995135098695755, 0.18890447914600372], "agreement_weight": -0.011143112555146217, "orthogonal_weight": 0.2989780008792877}, {"head": 4, "memory_to_active": 1.10219407081604, "contribution_to_active": 0.15755949914455414, "mix_mean": 0.0876052975654602, "mix_low": 0.08590888977050781, "mix_high": 0.0, "read_cosine": 0.36411213874816895, "beta_mean": 0.06147417426109314, "nonfinite": 0.0, "weighted_memory_to_active": 0.15683570504188538, "mix_std": 0.12067366391420364, "beta_std": 0.0587444081902504, "beta_low": 0.0413818359375, "beta_high": 0.0, "orthogonal_to_active": 0.0009588134707883, "evidence_weights": [-0.04358739033341408, -0.031493205577135086, 0.09768476337194443], "agreement_weight": 0.013622918166220188, "orthogonal_weight": 0.0077173784375190735}, {"head": 5, "memory_to_active": 0.48777443170547485, "contribution_to_active": 0.25204259157180786, "mix_mean": 0.11573521792888641, "mix_low": 0.06085968017578125, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8320077657699585, "beta_mean": 0.06623020768165588, "nonfinite": 0.0, "weighted_memory_to_active": 0.218414768576622, "mix_std": 0.14929673075675964, "beta_std": 0.06564730405807495, "beta_low": 0.03173828125, "beta_high": 0.0, "orthogonal_to_active": 0.0026966454461216927, "evidence_weights": [-0.08414939045906067, -0.0988365039229393, 0.14875534176826477], "agreement_weight": 0.0600319467484951, "orthogonal_weight": -0.044104669243097305}]}, {"layer": 12, "memory_to_active": 1.0016937255859375, "contribution_to_active": 0.21280881762504578, "mix_mean": 0.07856070250272751, "mix_low": 0.21003279089927673, "mix_high": 2.463658711349126e-05, "read_cosine": 0.2917789816856384, "beta_mean": 0.06893201172351837, "nonfinite": 0.0, "weighted_memory_to_active": 0.18828871846199036, "mix_std": 0.1038595512509346, "beta_std": 0.06627148389816284, "beta_low": 0.0886332243680954, "beta_high": 0.0, "orthogonal_to_active": 0.010294844396412373, "output_influence": 0.15327352285385132, "residual_influence": 0.03411918506026268, "state_active": 0.04835955426096916, "state_memory": 0.036767370998859406, "state_peak": 25.190088272094727, "growth_active": 1.1654177904129028, "growth_memory": 1.8796801567077637, "heads": [{"head": 0, "memory_to_active": 1.3312666416168213, "contribution_to_active": 0.3116110563278198, "mix_mean": 0.10853800177574158, "mix_low": 0.04308319091796875, "mix_high": 0.000133514404296875, "read_cosine": 0.43354538083076477, "beta_mean": 0.06817205250263214, "nonfinite": 0.0, "weighted_memory_to_active": 0.3578515946865082, "mix_std": 0.12392012774944305, "beta_std": 0.09405127167701721, "beta_low": 0.03448486328125, "beta_high": 0.0, "orthogonal_to_active": 0.011502464301884174, "evidence_weights": [-0.07778410613536835, -0.0902111604809761, 0.13543298840522766], "agreement_weight": 0.054565269500017166, "orthogonal_weight": 0.04297516122460365}, {"head": 1, "memory_to_active": 0.8115650415420532, "contribution_to_active": 0.07812269777059555, "mix_mean": 0.03861543908715248, "mix_low": 0.3505544662475586, "mix_high": 0.0, "read_cosine": 0.4074733853340149, "beta_mean": 0.056565094739198685, "nonfinite": 0.0, "weighted_memory_to_active": 0.08141075074672699, "mix_std": 0.062460966408252716, "beta_std": 0.050054311752319336, "beta_low": 0.079833984375, "beta_high": 0.0, "orthogonal_to_active": 0.008153821341693401, "evidence_weights": [0.009096737951040268, -0.013849958777427673, 0.10624084621667862], "agreement_weight": 0.12980854511260986, "orthogonal_weight": -0.15529213845729828}, {"head": 2, "memory_to_active": 0.5208575129508972, "contribution_to_active": 0.3012409210205078, "mix_mean": 0.15396010875701904, "mix_low": 0.04948902130126953, "mix_high": 1.430511474609375e-05, "read_cosine": 0.1769130527973175, "beta_mean": 0.033934615552425385, "nonfinite": 0.0, "weighted_memory_to_active": 0.18060684204101562, "mix_std": 0.1745099127292633, "beta_std": 0.05397922545671463, "beta_low": 0.354736328125, "beta_high": 0.0, "orthogonal_to_active": 0.00029272065148688853, "evidence_weights": [-0.0687318667769432, -0.06727588176727295, 0.07634153217077255], "agreement_weight": 0.05383320525288582, "orthogonal_weight": -0.0017685617785900831}, {"head": 3, "memory_to_active": 1.1238315105438232, "contribution_to_active": 0.1582101583480835, "mix_mean": 0.05496269837021828, "mix_low": 0.18114852905273438, "mix_high": 0.0, "read_cosine": 0.11706903576850891, "beta_mean": 0.049346402287483215, "nonfinite": 0.0, "weighted_memory_to_active": 0.13573938608169556, "mix_std": 0.07152867317199707, "beta_std": 0.03821119666099548, "beta_low": 0.037109375, "beta_high": 0.0, "orthogonal_to_active": 0.006543125957250595, "evidence_weights": [-0.019364740699529648, -0.030482038855552673, -0.015068955719470978], "agreement_weight": -0.08369557559490204, "orthogonal_weight": -0.05000807344913483}, {"head": 4, "memory_to_active": 1.70273756980896, "contribution_to_active": 0.25646913051605225, "mix_mean": 0.027977272868156433, "mix_low": 0.43622589111328125, "mix_high": 0.0, "read_cosine": 0.2556014955043793, "beta_mean": 0.13886824250221252, "nonfinite": 0.0, "weighted_memory_to_active": 0.26747292280197144, "mix_std": 0.057082369923591614, "beta_std": 0.08840222656726837, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006412717513740063, "evidence_weights": [-0.00696905143558979, -0.006331480108201504, -0.02182704210281372], "agreement_weight": -0.05495165288448334, "orthogonal_weight": -0.025617104023694992}, {"head": 5, "memory_to_active": 0.519904375076294, "contribution_to_active": 0.17119891941547394, "mix_mean": 0.08731069415807724, "mix_low": 0.19969558715820312, "mix_high": 0.0, "read_cosine": 0.36007148027420044, "beta_mean": 0.06670564413070679, "nonfinite": 0.0, "weighted_memory_to_active": 0.1066506952047348, "mix_std": 0.13365522027015686, "beta_std": 0.07293065637350082, "beta_low": 0.025634765625, "beta_high": 0.0, "orthogonal_to_active": 0.028864216059446335, "evidence_weights": [-0.04034079983830452, -0.05521159619092941, 0.1384049504995346], "agreement_weight": 0.09975408762693405, "orthogonal_weight": -0.3456226885318756}]}, {"layer": 13, "memory_to_active": 0.7980684041976929, "contribution_to_active": 0.20602591335773468, "mix_mean": 0.143813818693161, "mix_low": 0.24604067206382751, "mix_high": 0.0007613500347360969, "read_cosine": 0.26166582107543945, "beta_mean": 0.10855086147785187, "nonfinite": 0.0, "weighted_memory_to_active": 0.11198790371417999, "mix_std": 0.1314820796251297, "beta_std": 0.10290741920471191, "beta_low": 0.2588908076286316, "beta_high": 0.0, "orthogonal_to_active": 0.008465049788355827, "output_influence": 0.2389102578163147, "residual_influence": 0.06586745381355286, "state_active": 0.08787547051906586, "state_memory": 0.040269020944833755, "state_peak": 24.928401947021484, "growth_active": 1.3077120780944824, "growth_memory": 1.6045225858688354, "heads": [{"head": 0, "memory_to_active": 1.240392804145813, "contribution_to_active": 0.0715503841638565, "mix_mean": 0.015731429681181908, "mix_low": 0.5719375610351562, "mix_high": 0.0, "read_cosine": 0.20308314263820648, "beta_mean": 0.13607379794120789, "nonfinite": 0.0, "weighted_memory_to_active": 0.06900466978549957, "mix_std": 0.02849762886762619, "beta_std": 0.1337733268737793, "beta_low": 0.08416748046875, "beta_high": 0.0, "orthogonal_to_active": 0.012456738390028477, "evidence_weights": [0.04149870574474335, 0.033590227365493774, -0.0811149999499321], "agreement_weight": 0.13991135358810425, "orthogonal_weight": -0.27396538853645325}, {"head": 1, "memory_to_active": 1.4318183660507202, "contribution_to_active": 0.05018889158964157, "mix_mean": 0.016321834176778793, "mix_low": 0.5567789077758789, "mix_high": 0.0, "read_cosine": 0.3006937503814697, "beta_mean": 0.2463015615940094, "nonfinite": 0.0, "weighted_memory_to_active": 0.08141133189201355, "mix_std": 0.026973068714141846, "beta_std": 0.1471947282552719, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006350921466946602, "evidence_weights": [0.041509244590997696, 0.021979384124279022, -0.019835565239191055], "agreement_weight": 0.11471779644489288, "orthogonal_weight": -0.15643209218978882}, {"head": 2, "memory_to_active": 0.4535942077636719, "contribution_to_active": 0.18754489719867706, "mix_mean": 0.16914823651313782, "mix_low": 0.035491943359375, "mix_high": 3.814697265625e-06, "read_cosine": 0.5729679465293884, "beta_mean": 0.03795979917049408, "nonfinite": 0.0, "weighted_memory_to_active": 0.14687059819698334, "mix_std": 0.15517641603946686, "beta_std": 0.057720135897397995, "beta_low": 0.306396484375, "beta_high": 0.0, "orthogonal_to_active": 0.00808394979685545, "evidence_weights": [-0.12400389462709427, -0.11806593835353851, 0.20468057692050934], "agreement_weight": 0.09072189778089523, "orthogonal_weight": 0.07454116642475128}, {"head": 3, "memory_to_active": 0.0629744827747345, "contribution_to_active": 0.3489874005317688, "mix_mean": 0.2635129690170288, "mix_low": 0.0071849822998046875, "mix_high": 0.00013256072998046875, "read_cosine": -0.4156738817691803, "beta_mean": 0.0012358874082565308, "nonfinite": 0.0, "weighted_memory_to_active": 0.020380927249789238, "mix_std": 0.18490785360336304, "beta_std": 0.0017319659236818552, "beta_low": 0.99420166015625, "beta_high": 0.0, "orthogonal_to_active": 0.0010446413652971387, "evidence_weights": [-0.21545326709747314, -0.20176832377910614, 0.021015668287873268], "agreement_weight": 0.08530785888433456, "orthogonal_weight": -0.055146440863609314}, {"head": 4, "memory_to_active": 0.31418055295944214, "contribution_to_active": 0.3462367653846741, "mix_mean": 0.3414579927921295, "mix_low": 0.029577255249023438, "mix_high": 0.004431724548339844, "read_cosine": 0.6751574873924255, "beta_mean": 0.08659671992063522, "nonfinite": 0.0, "weighted_memory_to_active": 0.13905782997608185, "mix_std": 0.30279362201690674, "beta_std": 0.11212101578712463, "beta_low": 0.16314697265625, "beta_high": 0.0, "orthogonal_to_active": 0.01383474376052618, "evidence_weights": [-0.16289418935775757, -0.18239569664001465, 0.21718712151050568], "agreement_weight": 0.012335115112364292, "orthogonal_weight": 0.1385451853275299}, {"head": 5, "memory_to_active": 1.2854499816894531, "contribution_to_active": 0.2316470444202423, "mix_mean": 0.05671045929193497, "mix_low": 0.27527332305908203, "mix_high": 0.0, "read_cosine": 0.23376640677452087, "beta_mean": 0.14313744008541107, "nonfinite": 0.0, "weighted_memory_to_active": 0.21520203351974487, "mix_std": 0.09054385870695114, "beta_std": 0.16490329802036285, "beta_low": 0.00543212890625, "beta_high": 0.0, "orthogonal_to_active": 0.009019298478960991, "evidence_weights": [-0.057793695479631424, -0.04857754334807396, -0.02921290509402752], "agreement_weight": -0.008718355558812618, "orthogonal_weight": 0.04465485364198685}]}, {"layer": 14, "memory_to_active": 0.3925924599170685, "contribution_to_active": 0.32779955863952637, "mix_mean": 0.27037039399147034, "mix_low": 0.05531787872314453, "mix_high": 0.0009509723167866468, "read_cosine": 0.6261086463928223, "beta_mean": 0.11041439324617386, "nonfinite": 0.0, "weighted_memory_to_active": 0.1532931625843048, "mix_std": 0.22427378594875336, "beta_std": 0.10945083945989609, "beta_low": 0.066192626953125, "beta_high": 4.069010537932627e-05, "orthogonal_to_active": 0.011405833065509796, "output_influence": 0.19364088773727417, "residual_influence": 0.0566234216094017, "state_active": 0.10727636516094208, "state_memory": 0.03531939908862114, "state_peak": 35.7127799987793, "growth_active": 1.4701253175735474, "growth_memory": 1.6095044612884521, "heads": [{"head": 0, "memory_to_active": 0.3859965205192566, "contribution_to_active": 0.39507484436035156, "mix_mean": 0.328120619058609, "mix_low": 0.015367507934570312, "mix_high": 0.0014219284057617188, "read_cosine": 0.6090102791786194, "beta_mean": 0.13140742480754852, "nonfinite": 0.0, "weighted_memory_to_active": 0.18875882029533386, "mix_std": 0.2701345980167389, "beta_std": 0.17394289374351501, "beta_low": 0.152099609375, "beta_high": 0.0, "orthogonal_to_active": 0.016177505254745483, "evidence_weights": [-0.11004885286092758, -0.1515463888645172, 0.19203625619411469], "agreement_weight": 0.08494431525468826, "orthogonal_weight": -0.12141366302967072}, {"head": 1, "memory_to_active": 0.41992226243019104, "contribution_to_active": 0.4307747781276703, "mix_mean": 0.34373846650123596, "mix_low": 0.04311561584472656, "mix_high": 0.0030498504638671875, "read_cosine": 0.47047239542007446, "beta_mean": 0.24644465744495392, "nonfinite": 0.0, "weighted_memory_to_active": 0.21310989558696747, "mix_std": 0.29968318343162537, "beta_std": 0.25244948267936707, "beta_low": 0.05145263671875, "beta_high": 0.000244140625, "orthogonal_to_active": 0.01885947212576866, "evidence_weights": [-0.10431624948978424, -0.14912283420562744, 0.20235894620418549], "agreement_weight": 0.05671076476573944, "orthogonal_weight": -0.1127210333943367}, {"head": 2, "memory_to_active": 0.4865502715110779, "contribution_to_active": 0.24547265470027924, "mix_mean": 0.1693534255027771, "mix_low": 0.0673532485961914, "mix_high": 9.5367431640625e-06, "read_cosine": 0.5957291126251221, "beta_mean": 0.10613753646612167, "nonfinite": 0.0, "weighted_memory_to_active": 0.16781498491764069, "mix_std": 0.17193607985973358, "beta_std": 0.05866155028343201, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016277829185128212, "evidence_weights": [-0.11642973870038986, -0.12830723822116852, 0.038226768374443054], "agreement_weight": 0.10245627164840698, "orthogonal_weight": -0.14991821348667145}, {"head": 3, "memory_to_active": 0.20112985372543335, "contribution_to_active": 0.4679701030254364, "mix_mean": 0.45702606439590454, "mix_low": 0.004279136657714844, "mix_high": 0.0012159347534179688, "read_cosine": 0.734147846698761, "beta_mean": 0.045466795563697815, "nonfinite": 0.0, "weighted_memory_to_active": 0.11230473220348358, "mix_std": 0.271086722612381, "beta_std": 0.037425700575113297, "beta_low": 0.02459716796875, "beta_high": 0.0, "orthogonal_to_active": 0.005850129760801792, "evidence_weights": [-0.25472792983055115, -0.25642240047454834, 0.27855631709098816], "agreement_weight": 0.06469523906707764, "orthogonal_weight": -0.09273690730333328}, {"head": 4, "memory_to_active": 0.3317949175834656, "contribution_to_active": 0.3126017451286316, "mix_mean": 0.23078998923301697, "mix_low": 0.029909133911132812, "mix_high": 8.58306884765625e-06, "read_cosine": 0.6813417077064514, "beta_mean": 0.07499746978282928, "nonfinite": 0.0, "weighted_memory_to_active": 0.148455411195755, "mix_std": 0.21113930642604828, "beta_std": 0.06941846013069153, "beta_low": 0.00396728515625, "beta_high": 0.0, "orthogonal_to_active": 0.009912384673953056, "evidence_weights": [-0.11094110459089279, -0.12534193694591522, 0.0909256637096405], "agreement_weight": 0.08703486621379852, "orthogonal_weight": 0.10516426712274551}, {"head": 5, "memory_to_active": 0.5301608443260193, "contribution_to_active": 0.11490324139595032, "mix_mean": 0.09319378435611725, "mix_low": 0.17188262939453125, "mix_high": 0.0, "read_cosine": 0.6659506559371948, "beta_mean": 0.058032430708408356, "nonfinite": 0.0, "weighted_memory_to_active": 0.08931513130664825, "mix_std": 0.12166278064250946, "beta_std": 0.06480689346790314, "beta_low": 0.1650390625, "beta_high": 0.0, "orthogonal_to_active": 0.0013576778583228588, "evidence_weights": [-0.08146553486585617, -0.07518570870161057, 0.020445028319954872], "agreement_weight": 0.08146613836288452, "orthogonal_weight": -0.022726155817508698}]}, {"layer": 15, "memory_to_active": 0.14796647429466248, "contribution_to_active": 0.35646817088127136, "mix_mean": 0.24677857756614685, "mix_low": 0.026800155639648438, "mix_high": 0.0008788109407760203, "read_cosine": 0.48344793915748596, "beta_mean": 0.02741716057062149, "nonfinite": 0.0, "weighted_memory_to_active": 0.06641080230474472, "mix_std": 0.21524187922477722, "beta_std": 0.0357043519616127, "beta_low": 0.5821737051010132, "beta_high": 0.0, "orthogonal_to_active": 0.002906035166233778, "output_influence": 0.2345343828201294, "residual_influence": 0.09970682859420776, "state_active": 0.1227407231926918, "state_memory": 0.028643645346164703, "state_peak": 42.93809127807617, "growth_active": 1.5736881494522095, "growth_memory": 1.586480736732483, "heads": [{"head": 0, "memory_to_active": 0.16961221396923065, "contribution_to_active": 0.45592495799064636, "mix_mean": 0.32318538427352905, "mix_low": 0.0031728744506835938, "mix_high": 5.626678466796875e-05, "read_cosine": 0.74112868309021, "beta_mean": 0.0775143951177597, "nonfinite": 0.0, "weighted_memory_to_active": 0.08661193400621414, "mix_std": 0.23856768012046814, "beta_std": 0.07743668556213379, "beta_low": 0.098388671875, "beta_high": 0.0, "orthogonal_to_active": 0.0011743095237761736, "evidence_weights": [-0.1855381727218628, -0.21656689047813416, 0.22025275230407715], "agreement_weight": 0.06506788730621338, "orthogonal_weight": 0.03360625356435776}, {"head": 1, "memory_to_active": 0.2987217307090759, "contribution_to_active": 0.27189239859580994, "mix_mean": 0.19728785753250122, "mix_low": 0.025918960571289062, "mix_high": 0.0006122589111328125, "read_cosine": 0.7362205386161804, "beta_mean": 0.010885624215006828, "nonfinite": 0.0, "weighted_memory_to_active": 0.10400798916816711, "mix_std": 0.19108186662197113, "beta_std": 0.008845395408570766, "beta_low": 0.572998046875, "beta_high": 0.0, "orthogonal_to_active": 0.00511839659884572, "evidence_weights": [-0.1465483158826828, -0.158095121383667, 0.1772228330373764], "agreement_weight": 0.0778447836637497, "orthogonal_weight": -0.08889855444431305}, {"head": 2, "memory_to_active": 0.047809042036533356, "contribution_to_active": 0.3307994306087494, "mix_mean": 0.25212985277175903, "mix_low": 0.03377819061279297, "mix_high": 0.0008411407470703125, "read_cosine": 0.693814754486084, "beta_mean": 0.002363910898566246, "nonfinite": 0.0, "weighted_memory_to_active": 0.017731821164488792, "mix_std": 0.2335393726825714, "beta_std": 0.0033646374940872192, "beta_low": 0.9744873046875, "beta_high": 0.0, "orthogonal_to_active": 0.0024870308116078377, "evidence_weights": [-0.1970216929912567, -0.20083196461200714, 0.22762735188007355], "agreement_weight": 0.05638957768678665, "orthogonal_weight": -0.20339304208755493}, {"head": 3, "memory_to_active": 0.2969778776168823, "contribution_to_active": 0.3355690836906433, "mix_mean": 0.1316748559474945, "mix_low": 0.05227184295654297, "mix_high": 0.00020503997802734375, "read_cosine": 0.5342652201652527, "beta_mean": 0.06233777850866318, "nonfinite": 0.0, "weighted_memory_to_active": 0.16205833852291107, "mix_std": 0.15144985914230347, "beta_std": 0.10307423770427704, "beta_low": 0.1500244140625, "beta_high": 0.0, "orthogonal_to_active": 0.0040566460229456425, "evidence_weights": [-0.07993398606777191, -0.14048472046852112, 0.19782595336437225], "agreement_weight": 0.07997477799654007, "orthogonal_weight": 0.06925773620605469}, {"head": 4, "memory_to_active": 0.06426611542701721, "contribution_to_active": 0.3433060944080353, "mix_mean": 0.23191529512405396, "mix_low": 0.034430503845214844, "mix_high": 0.000431060791015625, "read_cosine": -0.14927789568901062, "beta_mean": 0.011108318343758583, "nonfinite": 0.0, "weighted_memory_to_active": 0.023922476917505264, "mix_std": 0.21521608531475067, "beta_std": 0.02067640982568264, "beta_low": 0.697998046875, "beta_high": 0.0, "orthogonal_to_active": 0.004379530902951956, "evidence_weights": [-0.21845702826976776, -0.20776969194412231, 0.19100235402584076], "agreement_weight": 0.0254240483045578, "orthogonal_weight": -0.20438092947006226}, {"head": 5, "memory_to_active": 0.010411817580461502, "contribution_to_active": 0.40131694078445435, "mix_mean": 0.34447816014289856, "mix_low": 0.011228561401367188, "mix_high": 0.0031270980834960938, "read_cosine": 0.3445361852645874, "beta_mean": 0.0002929397451225668, "nonfinite": 0.0, "weighted_memory_to_active": 0.004132220055907965, "mix_std": 0.2615964114665985, "beta_std": 0.0008287374512292445, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.00022029604588169605, "evidence_weights": [-0.27441081404685974, -0.2501800060272217, 0.2368684709072113], "agreement_weight": 0.016851957887411118, "orthogonal_weight": 0.059734147042036057}]}]}, {"step": 3000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8083483860009437, "memory_to_active": 1.1804418563842773, "contribution_to_active": 0.2707062363624573, "mix_mean": 0.1348552703857422, "mix_low": 0.2165193408727646, "mix_high": 0.002417872427031398, "read_cosine": 0.42297428846359253, "beta_mean": 0.13084684312343597, "nonfinite": 0.0, "weighted_memory_to_active": 0.18330995738506317, "mix_std": 0.13835932314395905, "beta_std": 0.0873822569847107, "beta_low": 0.2144673764705658, "beta_high": 6.35782896551973e-07, "orthogonal_to_active": 0.01682128943502903, "samples": 4, "scored_tokens": 899, "normal_loss": 3.4541685581207275, "normal_token_accuracy": 0.40600666403770447, "state_active": 0.06844565272331238, "state_memory": 0.041492484509944916, "state_peak": 45.484310150146484, "growth_active": 1.1427149772644043, "growth_memory": 1.6994357109069824, "output_influence": 0.22707144916057587, "residual_influence": 0.06253382563591003, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.4630022048950195, "no_memory_loss_delta": 0.06783858686685562, "scrambled_loss_delta": 0.3832107484340668, "memory_only_loss_delta": 1.4275894165039062, "no_evidence_loss_delta": 0.02504269964993, "no_agreement_loss_delta": 0.012981833890080452, "no_orthogonal_loss_delta": 0.00042800395749509335}, {"index": 94, "tokens": 256, "normal_loss": 3.603224992752075, "no_memory_loss_delta": 0.19418951869010925, "scrambled_loss_delta": 0.3507813811302185, "memory_only_loss_delta": 1.3348686695098877, "no_evidence_loss_delta": 0.05873381346464157, "no_agreement_loss_delta": 0.02674216590821743, "no_orthogonal_loss_delta": 0.004129782319068909}, {"index": 188, "tokens": 256, "normal_loss": 3.848341941833496, "no_memory_loss_delta": 0.03596019744873047, "scrambled_loss_delta": 0.05504554510116577, "memory_only_loss_delta": 1.3586360216140747, "no_evidence_loss_delta": 0.013243427500128746, "no_agreement_loss_delta": -0.004563785158097744, "no_orthogonal_loss_delta": -0.0002738120383583009}, {"index": 283, "tokens": 131, "normal_loss": 2.3753278255462646, "no_memory_loss_delta": 0.09193740785121918, "scrambled_loss_delta": 0.30437171459198, "memory_only_loss_delta": 1.7220200300216675, "no_evidence_loss_delta": 0.010986244305968285, "no_agreement_loss_delta": 0.003179579973220825, "no_orthogonal_loss_delta": 0.0028286082670092583}], "agreement_weight_abs_mean": 0.051183875650167465, "agreement_weight_abs_max": 0.2006615251302719, "agreement_weight_mean": 0.019181212410330772, "orthogonal_weight_abs_mean": 0.14018477499485016, "orthogonal_weight_abs_max": 0.6528850793838501, "orthogonal_weight_mean": -0.007698118221014738, "no_memory_loss": 3.5524210929870605, "no_memory_loss_delta": 0.0982522964477539, "no_memory_token_accuracy_gain": 0.01223581749945879, "no_memory_token_win_fraction": 0.5528364777565002, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 9.357981169059586, "scrambled_loss": 3.723207712173462, "scrambled_loss_delta": 0.2690393030643463, "scrambled_token_accuracy_gain": 0.03448275849223137, "scrambled_token_win_fraction": 0.5806451439857483, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 23.588677693315752, "memory_only_loss": 4.8786234855651855, "memory_only_loss_delta": 1.424454689025879, "memory_only_token_accuracy_gain": 0.1446051150560379, "memory_only_token_win_fraction": 0.8220244646072388, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 75.93603491215141, "no_evidence_loss": 3.4833970069885254, "no_evidence_loss_delta": 0.029228365048766136, "no_evidence_token_accuracy_gain": 0.0022246940061450005, "no_evidence_token_win_fraction": 0.5239154696464539, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 2.880534777362982, "no_agreement_loss": 3.464644432067871, "no_agreement_loss_delta": 0.010475573129951954, "no_agreement_token_accuracy_gain": -0.0033370412420481443, "no_agreement_token_win_fraction": 0.5094549655914307, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.0420895407135278, "no_orthogonal_loss": 3.455801010131836, "no_orthogonal_loss_delta": 0.001632085768505931, "no_orthogonal_token_accuracy_gain": -0.007786429487168789, "no_orthogonal_token_win_fraction": 0.48275861144065857, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.16307546407980622, "short_no_memory_loss_delta": 0.05470488220453262, "long_context_memory_gain": 0.04354741424322128, "normal_context_loss_gain": 0.11476175487041473, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 1.75091552734375, "contribution_to_active": 0.47649437189102173, "mix_mean": 0.11050489544868469, "mix_low": 0.03843943402171135, "mix_high": 0.00040165585232898593, "read_cosine": 0.6087958812713623, "beta_mean": 0.26669541001319885, "nonfinite": 0.0, "weighted_memory_to_active": 0.45044654607772827, "mix_std": 0.08753052353858948, "beta_std": 0.115165114402771, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06076301634311676, "output_influence": 0.2284664511680603, "residual_influence": 0.20999206602573395, "state_active": 0.09635789692401886, "state_memory": 0.06618063896894455, "state_peak": 24.769397735595703, "growth_active": 1.1816253662109375, "growth_memory": 1.466120958328247, "heads": [{"head": 0, "memory_to_active": 2.2925617694854736, "contribution_to_active": 0.5246227979660034, "mix_mean": 0.05044981837272644, "mix_low": 0.021231651306152344, "mix_high": 0.0, "read_cosine": 0.5072643160820007, "beta_mean": 0.2550289034843445, "nonfinite": 0.0, "weighted_memory_to_active": 0.5557511448860168, "mix_std": 0.05491776764392853, "beta_std": 0.09452654421329498, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04457872360944748, "evidence_weights": [-0.06562229990959167, -0.07445981353521347, 0.1713024377822876], "agreement_weight": -0.05377117916941643, "orthogonal_weight": 0.10675575584173203}, {"head": 1, "memory_to_active": 1.866614818572998, "contribution_to_active": 0.9831997156143188, "mix_mean": 0.056538984179496765, "mix_low": 0.025048255920410156, "mix_high": 2.574920654296875e-05, "read_cosine": 0.5832213163375854, "beta_mean": 0.3487491309642792, "nonfinite": 0.0, "weighted_memory_to_active": 1.03017258644104, "mix_std": 0.05912025272846222, "beta_std": 0.1343039721250534, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14895518124103546, "evidence_weights": [-0.0943797305226326, -0.11872979998588562, 0.1771775484085083], "agreement_weight": -0.01924271695315838, "orthogonal_weight": 0.2675640881061554}, {"head": 2, "memory_to_active": 1.4540272951126099, "contribution_to_active": 0.306522011756897, "mix_mean": 0.04718037694692612, "mix_low": 0.05814170837402344, "mix_high": 0.0, "read_cosine": 0.5705952644348145, "beta_mean": 0.24713991582393646, "nonfinite": 0.0, "weighted_memory_to_active": 0.30059295892715454, "mix_std": 0.04861924424767494, "beta_std": 0.09620349854230881, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.032073013484478, "evidence_weights": [-0.07247322052717209, -0.08302300423383713, 0.06643090397119522], "agreement_weight": -0.05284971743822098, "orthogonal_weight": 0.14501722157001495}, {"head": 3, "memory_to_active": 1.814842700958252, "contribution_to_active": 0.5670285224914551, "mix_mean": 0.13569533824920654, "mix_low": 0.010602951049804688, "mix_high": 4.76837158203125e-06, "read_cosine": 0.6183822154998779, "beta_mean": 0.31114834547042847, "nonfinite": 0.0, "weighted_memory_to_active": 0.5659092664718628, "mix_std": 0.09581238776445389, "beta_std": 0.13183534145355225, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.09348919987678528, "evidence_weights": [-0.17688964307308197, -0.19355498254299164, 0.2747074365615845], "agreement_weight": -0.05526528134942055, "orthogonal_weight": 0.22594338655471802}, {"head": 4, "memory_to_active": 2.8637940883636475, "contribution_to_active": 0.15724602341651917, "mix_mean": 0.02937447279691696, "mix_low": 0.11004352569580078, "mix_high": 0.0, "read_cosine": 0.4718754291534424, "beta_mean": 0.2984330654144287, "nonfinite": 0.0, "weighted_memory_to_active": 0.14265459775924683, "mix_std": 0.025696203112602234, "beta_std": 0.14689534902572632, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02562783658504486, "evidence_weights": [-0.02238689921796322, -0.033003050833940506, 0.1498204916715622], "agreement_weight": -0.05026913434267044, "orthogonal_weight": 0.2076573222875595}, {"head": 5, "memory_to_active": 0.21365180611610413, "contribution_to_active": 0.32034704089164734, "mix_mean": 0.34379035234451294, "mix_low": 0.005568504333496094, "mix_high": 0.0023794174194335938, "read_cosine": 0.9014369249343872, "beta_mean": 0.13967299461364746, "nonfinite": 0.0, "weighted_memory_to_active": 0.10759875923395157, "mix_std": 0.24101728200912476, "beta_std": 0.08722598105669022, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.019854124635457993, "evidence_weights": [-0.38209572434425354, -0.36828070878982544, 0.44311708211898804], "agreement_weight": 0.1189260184764862, "orthogonal_weight": 0.3889881372451782}]}, {"layer": 1, "memory_to_active": 2.6215929985046387, "contribution_to_active": 0.22042524814605713, "mix_mean": 0.051609739661216736, "mix_low": 0.347312331199646, "mix_high": 0.0, "read_cosine": 0.3940272629261017, "beta_mean": 0.30763688683509827, "nonfinite": 0.0, "weighted_memory_to_active": 0.2203986942768097, "mix_std": 0.05885697156190872, "beta_std": 0.15504297614097595, "beta_low": 0.00014241537428461015, "beta_high": 0.0, "orthogonal_to_active": 0.021697331219911575, "output_influence": 0.13074876368045807, "residual_influence": 0.025220604613423347, "state_active": 0.05456973612308502, "state_memory": 0.07740166783332825, "state_peak": 19.12519073486328, "growth_active": 1.036495327949524, "growth_memory": 1.6910241842269897, "heads": [{"head": 0, "memory_to_active": 3.166071891784668, "contribution_to_active": 0.09146194905042648, "mix_mean": 0.01014634408056736, "mix_low": 0.6567134857177734, "mix_high": 0.0, "read_cosine": 0.10378105938434601, "beta_mean": 0.19171956181526184, "nonfinite": 0.0, "weighted_memory_to_active": 0.06744415313005447, "mix_std": 0.017146773636341095, "beta_std": 0.11122298240661621, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005067221354693174, "evidence_weights": [0.039004795253276825, 0.03650073707103729, -0.12842783331871033], "agreement_weight": -0.09958751499652863, "orthogonal_weight": -0.08104690909385681}, {"head": 1, "memory_to_active": 2.632103443145752, "contribution_to_active": 0.24171960353851318, "mix_mean": 0.049225352704524994, "mix_low": 0.110870361328125, "mix_high": 0.0, "read_cosine": 0.5284066796302795, "beta_mean": 0.3291143476963043, "nonfinite": 0.0, "weighted_memory_to_active": 0.24006502330303192, "mix_std": 0.06043395400047302, "beta_std": 0.13122597336769104, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.021164806559681892, "evidence_weights": [-0.08004916459321976, -0.07797084748744965, 0.10523439943790436], "agreement_weight": -0.05510991811752319, "orthogonal_weight": 0.10457849502563477}, {"head": 2, "memory_to_active": 0.8847025632858276, "contribution_to_active": 0.31828194856643677, "mix_mean": 0.16942206025123596, "mix_low": 0.0149993896484375, "mix_high": 0.0, "read_cosine": 0.7050294876098633, "beta_mean": 0.1882840245962143, "nonfinite": 0.0, "weighted_memory_to_active": 0.3846782147884369, "mix_std": 0.15960337221622467, "beta_std": 0.07864390313625336, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04791276529431343, "evidence_weights": [-0.20032835006713867, -0.2147800624370575, 0.215185284614563], "agreement_weight": -0.012464557774364948, "orthogonal_weight": 0.1897953748703003}, {"head": 3, "memory_to_active": 2.6034953594207764, "contribution_to_active": 0.05208750069141388, "mix_mean": 0.012612087652087212, "mix_low": 0.5941238403320312, "mix_high": 0.0, "read_cosine": 0.28928306698799133, "beta_mean": 0.41433829069137573, "nonfinite": 0.0, "weighted_memory_to_active": 0.05300014838576317, "mix_std": 0.01499049924314022, "beta_std": 0.2365790456533432, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.0012082186294719577, "evidence_weights": [-0.009534790180623531, -0.006762692704796791, -0.04476785659790039], "agreement_weight": -0.015724780037999153, "orthogonal_weight": 0.024621663615107536}, {"head": 4, "memory_to_active": 2.1120753288269043, "contribution_to_active": 0.5303621292114258, "mix_mean": 0.05540170520544052, "mix_low": 0.08872604370117188, "mix_high": 0.0, "read_cosine": 0.4836964011192322, "beta_mean": 0.20622922480106354, "nonfinite": 0.0, "weighted_memory_to_active": 0.4879325330257416, "mix_std": 0.06964025646448135, "beta_std": 0.09406650811433792, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.054177213460206985, "evidence_weights": [-0.08912912011146545, -0.09019706398248672, 0.19011181592941284], "agreement_weight": -0.03483737260103226, "orthogonal_weight": 0.11571218073368073}, {"head": 5, "memory_to_active": 4.331108093261719, "contribution_to_active": 0.0886382907629013, "mix_mean": 0.012850901111960411, "mix_low": 0.6184406280517578, "mix_high": 0.0, "read_cosine": 0.25396668910980225, "beta_mean": 0.5161356925964355, "nonfinite": 0.0, "weighted_memory_to_active": 0.08927202224731445, "mix_std": 0.03132697939872742, "beta_std": 0.2785193920135498, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.0006537525332532823, "evidence_weights": [0.014225400984287262, 0.011456507258117199, 0.07939237356185913], "agreement_weight": -0.06033455953001976, "orthogonal_weight": -0.0077729374170303345}]}, {"layer": 2, "memory_to_active": 1.6891125440597534, "contribution_to_active": 0.14748495817184448, "mix_mean": 0.05922804772853851, "mix_low": 0.27476075291633606, "mix_high": 9.632110595703125e-05, "read_cosine": 0.4012948274612427, "beta_mean": 0.1744997203350067, "nonfinite": 0.0, "weighted_memory_to_active": 0.11264742910861969, "mix_std": 0.0816209614276886, "beta_std": 0.10358448326587677, "beta_low": 0.302764892578125, "beta_high": 0.0, "orthogonal_to_active": 0.0031396779231727123, "output_influence": 0.15578895807266235, "residual_influence": 0.025729971006512642, "state_active": 0.03261202946305275, "state_memory": 0.0448315292596817, "state_peak": 32.18906021118164, "growth_active": 0.9312072396278381, "growth_memory": 1.6489300727844238, "heads": [{"head": 0, "memory_to_active": 1.7107198238372803, "contribution_to_active": 0.048005230724811554, "mix_mean": 0.013753049075603485, "mix_low": 0.6336488723754883, "mix_high": 0.0, "read_cosine": 0.235690176486969, "beta_mean": 0.09529437124729156, "nonfinite": 0.0, "weighted_memory_to_active": 0.04425698518753052, "mix_std": 0.02593354880809784, "beta_std": 0.06166435778141022, "beta_low": 0.0743408203125, "beta_high": 0.0, "orthogonal_to_active": 0.0011723280185833573, "evidence_weights": [0.018237264826893806, 0.016944855451583862, 0.06384000927209854], "agreement_weight": -0.04938575252890587, "orthogonal_weight": 0.027417393401265144}, {"head": 1, "memory_to_active": 2.590625762939453, "contribution_to_active": 0.12023374438285828, "mix_mean": 0.02153278887271881, "mix_low": 0.3637046813964844, "mix_high": 0.0, "read_cosine": 0.2570616602897644, "beta_mean": 0.25645512342453003, "nonfinite": 0.0, "weighted_memory_to_active": 0.11881192028522491, "mix_std": 0.027784977108240128, "beta_std": 0.1354900598526001, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023756243754178286, "evidence_weights": [-0.013275752775371075, -0.00934866163879633, 0.011533807031810284], "agreement_weight": -0.015507460571825504, "orthogonal_weight": 0.020837243646383286}, {"head": 2, "memory_to_active": 0.5577839612960815, "contribution_to_active": 0.1716221421957016, "mix_mean": 0.091340571641922, "mix_low": 0.059173583984375, "mix_high": 0.0, "read_cosine": 0.5527271032333374, "beta_mean": 0.007717910222709179, "nonfinite": 0.0, "weighted_memory_to_active": 0.11391501873731613, "mix_std": 0.11233019828796387, "beta_std": 0.00643064035102725, "beta_low": 0.74041748046875, "beta_high": 0.0, "orthogonal_to_active": 0.003961854614317417, "evidence_weights": [-0.08273131400346756, -0.0758623480796814, 0.11807367205619812], "agreement_weight": 0.0653647780418396, "orthogonal_weight": 0.057603877037763596}, {"head": 3, "memory_to_active": 1.5435335636138916, "contribution_to_active": 0.11110369116067886, "mix_mean": 0.05429261177778244, "mix_low": 0.1287708282470703, "mix_high": 0.0, "read_cosine": 0.5142049789428711, "beta_mean": 0.2556886374950409, "nonfinite": 0.0, "weighted_memory_to_active": 0.14480701088905334, "mix_std": 0.07415524125099182, "beta_std": 0.19377732276916504, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.0002511909115128219, "evidence_weights": [-0.03165978193283081, -0.06419949233531952, 0.22340689599514008], "agreement_weight": 0.05769600346684456, "orthogonal_weight": 0.0028301964048296213}, {"head": 4, "memory_to_active": 3.607404947280884, "contribution_to_active": 0.22942902147769928, "mix_mean": 0.021873746067285538, "mix_low": 0.4214897155761719, "mix_high": 6.771087646484375e-05, "read_cosine": 0.2265755534172058, "beta_mean": 0.43159055709838867, "nonfinite": 0.0, "weighted_memory_to_active": 0.23372703790664673, "mix_std": 0.05600479990243912, "beta_std": 0.22385597229003906, "beta_low": 0.00140380859375, "beta_high": 0.0, "orthogonal_to_active": 0.010308661498129368, "evidence_weights": [-0.001395370694808662, -0.00199575861915946, 0.08776690065860748], "agreement_weight": -0.007174887228757143, "orthogonal_weight": 0.04894637688994408}, {"head": 5, "memory_to_active": 0.12460650503635406, "contribution_to_active": 0.20451584458351135, "mix_mean": 0.1525755226612091, "mix_low": 0.04177665710449219, "mix_high": 0.0005102157592773438, "read_cosine": 0.6215094327926636, "beta_mean": 0.000251686607953161, "nonfinite": 0.0, "weighted_memory_to_active": 0.02036656253039837, "mix_std": 0.19351698458194733, "beta_std": 0.00028852815739810467, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007684081792831421, "evidence_weights": [-0.1186336874961853, -0.10021162033081055, 0.18348248302936554], "agreement_weight": -0.0032662313897162676, "orthogonal_weight": 0.05022937059402466}]}, {"layer": 3, "memory_to_active": 1.9418721199035645, "contribution_to_active": 0.1326608955860138, "mix_mean": 0.02696268819272518, "mix_low": 0.43581995368003845, "mix_high": 0.0002770423889160156, "read_cosine": 0.3166567087173462, "beta_mean": 0.15993592143058777, "nonfinite": 0.0, "weighted_memory_to_active": 0.10921283066272736, "mix_std": 0.05368047207593918, "beta_std": 0.10414187610149384, "beta_low": 0.1657918393611908, "beta_high": 0.0, "orthogonal_to_active": 0.0029258797876536846, "output_influence": 0.1342800408601761, "residual_influence": 0.018057536333799362, "state_active": 0.040984053164720535, "state_memory": 0.04960814118385315, "state_peak": 33.44113540649414, "growth_active": 0.9964455366134644, "growth_memory": 1.8219878673553467, "heads": [{"head": 0, "memory_to_active": 1.9083385467529297, "contribution_to_active": 0.058713655918836594, "mix_mean": 0.014396706596016884, "mix_low": 0.6355571746826172, "mix_high": 0.0, "read_cosine": 0.18054741621017456, "beta_mean": 0.037148743867874146, "nonfinite": 0.0, "weighted_memory_to_active": 0.0621829517185688, "mix_std": 0.03224452584981918, "beta_std": 0.028845317661762238, "beta_low": 0.05963134765625, "beta_high": 0.0, "orthogonal_to_active": 0.0017870271112769842, "evidence_weights": [0.033117711544036865, 0.01735607348382473, -0.08001284301280975], "agreement_weight": -0.08028074353933334, "orthogonal_weight": -0.029891662299633026}, {"head": 1, "memory_to_active": 1.2094173431396484, "contribution_to_active": 0.060842469334602356, "mix_mean": 0.027580007910728455, "mix_low": 0.3648490905761719, "mix_high": 0.0, "read_cosine": 0.3414647877216339, "beta_mean": 0.03108731471002102, "nonfinite": 0.0, "weighted_memory_to_active": 0.05656658485531807, "mix_std": 0.042823560535907745, "beta_std": 0.02319438010454178, "beta_low": 0.187255859375, "beta_high": 0.0, "orthogonal_to_active": 0.000286340742604807, "evidence_weights": [-0.010972362942993641, -0.008538863621652126, 0.025394372642040253], "agreement_weight": -0.03921187296509743, "orthogonal_weight": -0.005670574959367514}, {"head": 2, "memory_to_active": 2.8913469314575195, "contribution_to_active": 0.18854469060897827, "mix_mean": 0.028704442083835602, "mix_low": 0.3417787551879883, "mix_high": 0.0, "read_cosine": 0.33435821533203125, "beta_mean": 0.3301914930343628, "nonfinite": 0.0, "weighted_memory_to_active": 0.2150474488735199, "mix_std": 0.05572653189301491, "beta_std": 0.18013310432434082, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.006725511513650417, "evidence_weights": [-0.03859768807888031, -0.037255723029375076, 0.1090761348605156], "agreement_weight": -0.02498486079275608, "orthogonal_weight": -0.03655703365802765}, {"head": 3, "memory_to_active": 0.7652862668037415, "contribution_to_active": 0.10364413261413574, "mix_mean": 0.03366090729832649, "mix_low": 0.31220245361328125, "mix_high": 9.5367431640625e-07, "read_cosine": 0.4267807900905609, "beta_mean": 0.007548139430582523, "nonfinite": 0.0, "weighted_memory_to_active": 0.03562837839126587, "mix_std": 0.07168601453304291, "beta_std": 0.00700536509975791, "beta_low": 0.7432861328125, "beta_high": 0.0, "orthogonal_to_active": 0.0021910935174673796, "evidence_weights": [0.00015886241453699768, -0.002333312528207898, 0.0388459675014019], "agreement_weight": -0.05548053979873657, "orthogonal_weight": -0.05376182869076729}, {"head": 4, "memory_to_active": 3.8956801891326904, "contribution_to_active": 0.04421060532331467, "mix_mean": 0.008398941718041897, "mix_low": 0.7458124160766602, "mix_high": 0.0, "read_cosine": 0.20323175191879272, "beta_mean": 0.32347816228866577, "nonfinite": 0.0, "weighted_memory_to_active": 0.04635521024465561, "mix_std": 0.010299094952642918, "beta_std": 0.1903165578842163, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0033906828612089157, "evidence_weights": [0.07075239717960358, 0.06536907702684402, -0.11497019976377487], "agreement_weight": -0.12357097864151001, "orthogonal_weight": -0.0735408365726471}, {"head": 5, "memory_to_active": 0.9811638593673706, "contribution_to_active": 0.34000974893569946, "mix_mean": 0.04903513193130493, "mix_low": 0.2147197723388672, "mix_high": 0.0016613006591796875, "read_cosine": 0.413557231426239, "beta_mean": 0.23016156256198883, "nonfinite": 0.0, "weighted_memory_to_active": 0.2394963502883911, "mix_std": 0.1093030795454979, "beta_std": 0.195356547832489, "beta_low": 0.00445556640625, "beta_high": 0.0, "orthogonal_to_active": 0.0031746206805109978, "evidence_weights": [-0.01631004363298416, -0.07805407792329788, 0.07358082383871078], "agreement_weight": 0.09433925151824951, "orthogonal_weight": -0.02038840390741825}]}, {"layer": 4, "memory_to_active": 1.3683838844299316, "contribution_to_active": 0.13607922196388245, "mix_mean": 0.07792610675096512, "mix_low": 0.22547626495361328, "mix_high": 3.3378603347955504e-06, "read_cosine": 0.43494337797164917, "beta_mean": 0.16911886632442474, "nonfinite": 0.0, "weighted_memory_to_active": 0.11725016683340073, "mix_std": 0.08801134675741196, "beta_std": 0.10681658983230591, "beta_low": 0.1340840756893158, "beta_high": 0.0, "orthogonal_to_active": 0.009306087158620358, "output_influence": 0.11144696176052094, "residual_influence": 0.019710756838321686, "state_active": 0.043595533818006516, "state_memory": 0.047632813453674316, "state_peak": 25.36573600769043, "growth_active": 1.050960659980774, "growth_memory": 1.7813001871109009, "heads": [{"head": 0, "memory_to_active": 0.6570897698402405, "contribution_to_active": 0.1437539905309677, "mix_mean": 0.08630423992872238, "mix_low": 0.09277915954589844, "mix_high": 0.0, "read_cosine": 0.4965774118900299, "beta_mean": 0.08658179640769958, "nonfinite": 0.0, "weighted_memory_to_active": 0.1315421462059021, "mix_std": 0.12205294519662857, "beta_std": 0.09475447237491608, "beta_low": 0.04864501953125, "beta_high": 0.0, "orthogonal_to_active": 0.0013614766066893935, "evidence_weights": [-0.0778447762131691, -0.10826622694730759, 0.14285892248153687], "agreement_weight": 0.06044936552643776, "orthogonal_weight": 0.01553502306342125}, {"head": 1, "memory_to_active": 0.8214223980903625, "contribution_to_active": 0.126868337392807, "mix_mean": 0.09376087784767151, "mix_low": 0.12422370910644531, "mix_high": 0.0, "read_cosine": 0.49702203273773193, "beta_mean": 0.10458631813526154, "nonfinite": 0.0, "weighted_memory_to_active": 0.12173375487327576, "mix_std": 0.11239850521087646, "beta_std": 0.08208119124174118, "beta_low": 0.00433349609375, "beta_high": 0.0, "orthogonal_to_active": 0.0020259045995771885, "evidence_weights": [-0.08508847653865814, -0.09616821259260178, 0.21816009283065796], "agreement_weight": 0.06357775628566742, "orthogonal_weight": -0.023060621693730354}, {"head": 2, "memory_to_active": 2.972681760787964, "contribution_to_active": 0.10227847099304199, "mix_mean": 0.01374976709485054, "mix_low": 0.5958890914916992, "mix_high": 0.0, "read_cosine": 0.25911417603492737, "beta_mean": 0.3654137849807739, "nonfinite": 0.0, "weighted_memory_to_active": 0.11146466434001923, "mix_std": 0.018162745982408524, "beta_std": 0.1481514573097229, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007318067364394665, "evidence_weights": [0.005092371720820665, -0.000959086581133306, -0.024296265095472336], "agreement_weight": 0.019787564873695374, "orthogonal_weight": 0.08133470267057419}, {"head": 3, "memory_to_active": 1.7809033393859863, "contribution_to_active": 0.061787985265254974, "mix_mean": 0.021214444190263748, "mix_low": 0.3711109161376953, "mix_high": 0.0, "read_cosine": 0.2713623344898224, "beta_mean": 0.18986937403678894, "nonfinite": 0.0, "weighted_memory_to_active": 0.06654928624629974, "mix_std": 0.02741445228457451, "beta_std": 0.17240692675113678, "beta_low": 0.02178955078125, "beta_high": 0.0, "orthogonal_to_active": 0.005745151545852423, "evidence_weights": [-0.029087383300065994, -0.016841188073158264, -0.13410484790802002], "agreement_weight": -0.008477183058857918, "orthogonal_weight": -0.09934691339731216}, {"head": 4, "memory_to_active": 0.475792795419693, "contribution_to_active": 0.21070818603038788, "mix_mean": 0.19832554459571838, "mix_low": 0.010951042175292969, "mix_high": 1.71661376953125e-05, "read_cosine": 0.5219305753707886, "beta_mean": 0.007808181457221508, "nonfinite": 0.0, "weighted_memory_to_active": 0.12547580897808075, "mix_std": 0.1863134354352951, "beta_std": 0.00689128739759326, "beta_low": 0.729736328125, "beta_high": 0.0, "orthogonal_to_active": 0.003154188394546509, "evidence_weights": [-0.13631121814250946, -0.13383488357067108, 0.24309591948986053], "agreement_weight": -0.01523701474070549, "orthogonal_weight": 0.033896442502737045}, {"head": 5, "memory_to_active": 1.5024135112762451, "contribution_to_active": 0.1710783839225769, "mix_mean": 0.05420173704624176, "mix_low": 0.15790367126464844, "mix_high": 2.86102294921875e-06, "read_cosine": 0.5636535882949829, "beta_mean": 0.2604537010192871, "nonfinite": 0.0, "weighted_memory_to_active": 0.1467352956533432, "mix_std": 0.06172595173120499, "beta_std": 0.13661418855190277, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03623173385858536, "evidence_weights": [-0.08443661779165268, -0.08541172742843628, 0.14719153940677643], "agreement_weight": -0.04413394629955292, "orthogonal_weight": 0.290920615196228}]}, {"layer": 5, "memory_to_active": 0.9898471832275391, "contribution_to_active": 0.25997138023376465, "mix_mean": 0.17720568180084229, "mix_low": 0.17964810132980347, "mix_high": 0.0005089442129246891, "read_cosine": 0.3389754891395569, "beta_mean": 0.09496349096298218, "nonfinite": 0.0, "weighted_memory_to_active": 0.15379828214645386, "mix_std": 0.16836071014404297, "beta_std": 0.08186205476522446, "beta_low": 0.3431905210018158, "beta_high": 0.0, "orthogonal_to_active": 0.014889046549797058, "output_influence": 0.24726316332817078, "residual_influence": 0.039384953677654266, "state_active": 0.07774899899959564, "state_memory": 0.0506676509976387, "state_peak": 32.213401794433594, "growth_active": 1.095430612564087, "growth_memory": 2.1588103771209717, "heads": [{"head": 0, "memory_to_active": 2.2847471237182617, "contribution_to_active": 0.16837447881698608, "mix_mean": 0.04434545338153839, "mix_low": 0.19133663177490234, "mix_high": 0.0, "read_cosine": 0.06312691420316696, "beta_mean": 0.23241877555847168, "nonfinite": 0.0, "weighted_memory_to_active": 0.16451872885227203, "mix_std": 0.06668931990861893, "beta_std": 0.16124628484249115, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001558715448481962, "evidence_weights": [-0.043075162917375565, -0.04535377398133278, -0.007782304659485817], "agreement_weight": -0.009781193919479847, "orthogonal_weight": -0.0009970745304599404}, {"head": 1, "memory_to_active": 0.3335597515106201, "contribution_to_active": 0.15728838741779327, "mix_mean": 0.057507023215293884, "mix_low": 0.4467630386352539, "mix_high": 8.678436279296875e-05, "read_cosine": 0.15445920825004578, "beta_mean": 0.002453165128827095, "nonfinite": 0.0, "weighted_memory_to_active": 0.02491593547165394, "mix_std": 0.1263411045074463, "beta_std": 0.006863757967948914, "beta_low": 0.9541015625, "beta_high": 0.0, "orthogonal_to_active": 0.0057290541008114815, "evidence_weights": [-0.013034236617386341, -0.009802799671888351, -0.07615786045789719], "agreement_weight": 0.01434479933232069, "orthogonal_weight": -0.2380930781364441}, {"head": 2, "memory_to_active": 0.42433345317840576, "contribution_to_active": 0.2972913384437561, "mix_mean": 0.292837917804718, "mix_low": 0.01128387451171875, "mix_high": 0.00016117095947265625, "read_cosine": 0.6380077600479126, "beta_mean": 0.051127586513757706, "nonfinite": 0.0, "weighted_memory_to_active": 0.1613629311323166, "mix_std": 0.2283148616552353, "beta_std": 0.027938783168792725, "beta_low": 0.0013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.012379308231174946, "evidence_weights": [-0.22117328643798828, -0.2230546474456787, 0.3441070318222046], "agreement_weight": 0.08440554141998291, "orthogonal_weight": 0.11754745990037918}, {"head": 3, "memory_to_active": 2.0218822956085205, "contribution_to_active": 0.12360842525959015, "mix_mean": 0.027457382529973984, "mix_low": 0.3921070098876953, "mix_high": 0.0, "read_cosine": 0.2701541781425476, "beta_mean": 0.12683770060539246, "nonfinite": 0.0, "weighted_memory_to_active": 0.12963610887527466, "mix_std": 0.0472884327173233, "beta_std": 0.12416225671768188, "beta_low": 0.01275634765625, "beta_high": 0.0, "orthogonal_to_active": 0.008715610019862652, "evidence_weights": [0.007708132266998291, 0.0007337074493989348, -0.008430082350969315], "agreement_weight": -0.017302559688687325, "orthogonal_weight": -0.07176844775676727}, {"head": 4, "memory_to_active": 0.5599603652954102, "contribution_to_active": 0.40663665533065796, "mix_mean": 0.28236278891563416, "mix_low": 0.020280838012695312, "mix_high": 0.00189208984375, "read_cosine": 0.6315501928329468, "beta_mean": 0.15433979034423828, "nonfinite": 0.0, "weighted_memory_to_active": 0.31245818734169006, "mix_std": 0.26465049386024475, "beta_std": 0.16872745752334595, "beta_low": 0.1036376953125, "beta_high": 0.0, "orthogonal_to_active": 0.04068063199520111, "evidence_weights": [-0.1331024020910263, -0.17433667182922363, 0.2776072919368744], "agreement_weight": 0.06666495651006699, "orthogonal_weight": 0.19680719077587128}, {"head": 5, "memory_to_active": 0.31459951400756836, "contribution_to_active": 0.40662881731987, "mix_mean": 0.3587234914302826, "mix_low": 0.016117095947265625, "mix_high": 0.0009136199951171875, "read_cosine": 0.27655458450317383, "beta_mean": 0.0026039343792945147, "nonfinite": 0.0, "weighted_memory_to_active": 0.1298978477716446, "mix_std": 0.2768799662590027, "beta_std": 0.0022337508853524923, "beta_low": 0.9873046875, "beta_high": 0.0, "orthogonal_to_active": 0.02167380228638649, "evidence_weights": [-0.17273396253585815, -0.17696140706539154, 0.16563796997070312], "agreement_weight": 0.051019199192523956, "orthogonal_weight": -0.1954924613237381}]}, {"layer": 6, "memory_to_active": 0.6343638896942139, "contribution_to_active": 0.3009490370750427, "mix_mean": 0.1632615476846695, "mix_low": 0.19784706830978394, "mix_high": 0.004750887863337994, "read_cosine": 0.42260849475860596, "beta_mean": 0.0849834531545639, "nonfinite": 0.0, "weighted_memory_to_active": 0.18409861624240875, "mix_std": 0.19414064288139343, "beta_std": 0.08562271296977997, "beta_low": 0.2649434506893158, "beta_high": 0.0, "orthogonal_to_active": 0.01514134556055069, "output_influence": 0.29536986351013184, "residual_influence": 0.061428118497133255, "state_active": 0.07348685711622238, "state_memory": 0.032407768070697784, "state_peak": 27.27461814880371, "growth_active": 1.3129022121429443, "growth_memory": 1.7942607402801514, "heads": [{"head": 0, "memory_to_active": 0.6756306886672974, "contribution_to_active": 0.2611770033836365, "mix_mean": 0.08324790745973587, "mix_low": 0.21376514434814453, "mix_high": 0.0013036727905273438, "read_cosine": 0.45936310291290283, "beta_mean": 0.17156663537025452, "nonfinite": 0.0, "weighted_memory_to_active": 0.23572120070457458, "mix_std": 0.151413232088089, "beta_std": 0.13021717965602875, "beta_low": 0.00653076171875, "beta_high": 0.0, "orthogonal_to_active": 0.03223925083875656, "evidence_weights": [-0.05221030116081238, -0.0690697431564331, 0.14333553612232208], "agreement_weight": 0.11259502172470093, "orthogonal_weight": -0.20685622096061707}, {"head": 1, "memory_to_active": 0.47014111280441284, "contribution_to_active": 0.21200041472911835, "mix_mean": 0.13670405745506287, "mix_low": 0.07198047637939453, "mix_high": 0.000118255615234375, "read_cosine": 0.5405535101890564, "beta_mean": 0.017464663833379745, "nonfinite": 0.0, "weighted_memory_to_active": 0.13808874785900116, "mix_std": 0.16540923714637756, "beta_std": 0.0160348042845726, "beta_low": 0.4056396484375, "beta_high": 0.0, "orthogonal_to_active": 0.006833825726062059, "evidence_weights": [-0.12441998720169067, -0.11430999636650085, 0.03133366256952286], "agreement_weight": 0.04457756131887436, "orthogonal_weight": -0.09432022273540497}, {"head": 2, "memory_to_active": 1.6438145637512207, "contribution_to_active": 0.09292159974575043, "mix_mean": 0.024278417229652405, "mix_low": 0.4879026412963867, "mix_high": 0.0, "read_cosine": 0.2297883927822113, "beta_mean": 0.10450447350740433, "nonfinite": 0.0, "weighted_memory_to_active": 0.09246741235256195, "mix_std": 0.05858016014099121, "beta_std": 0.1080123633146286, "beta_low": 0.00128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.014358732849359512, "evidence_weights": [0.033091116696596146, 0.03394478186964989, -0.07462960481643677], "agreement_weight": 0.004568641074001789, "orthogonal_weight": -0.15630871057510376}, {"head": 3, "memory_to_active": 0.4035208225250244, "contribution_to_active": 0.4405461549758911, "mix_mean": 0.2342432737350464, "mix_low": 0.053053855895996094, "mix_high": 0.013607025146484375, "read_cosine": 0.5935707688331604, "beta_mean": 0.11598707735538483, "nonfinite": 0.0, "weighted_memory_to_active": 0.23613522946834564, "mix_std": 0.28104454278945923, "beta_std": 0.14218199253082275, "beta_low": 0.098388671875, "beta_high": 0.0, "orthogonal_to_active": 0.016088301315903664, "evidence_weights": [-0.07924049347639084, -0.1628931611776352, 0.20076043903827667], "agreement_weight": 0.014497892931103706, "orthogonal_weight": 0.10595432668924332}, {"head": 4, "memory_to_active": 0.5582533478736877, "contribution_to_active": 0.5511401891708374, "mix_mean": 0.3853485584259033, "mix_low": 0.008210182189941406, "mix_high": 0.013476371765136719, "read_cosine": 0.4161309599876404, "beta_mean": 0.09999553859233856, "nonfinite": 0.0, "weighted_memory_to_active": 0.3911464512348175, "mix_std": 0.3130345344543457, "beta_std": 0.11687685549259186, "beta_low": 0.07781982421875, "beta_high": 0.0, "orthogonal_to_active": 0.017117494717240334, "evidence_weights": [-0.17559294402599335, -0.2509286403656006, -0.11371307075023651], "agreement_weight": -0.015086190775036812, "orthogonal_weight": 0.056112077087163925}, {"head": 5, "memory_to_active": 0.05482279881834984, "contribution_to_active": 0.2479088008403778, "mix_mean": 0.11574704945087433, "mix_low": 0.3521699905395508, "mix_high": 0.0, "read_cosine": 0.29624420404434204, "beta_mean": 0.000382341182557866, "nonfinite": 0.0, "weighted_memory_to_active": 0.011032575741410255, "mix_std": 0.19536203145980835, "beta_std": 0.00041300198063254356, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0042104581370949745, "evidence_weights": [-0.02403714507818222, -0.05409999191761017, -0.17432880401611328], "agreement_weight": 0.0970279723405838, "orthogonal_weight": -0.41035497188568115}]}, {"layer": 7, "memory_to_active": 2.076176643371582, "contribution_to_active": 0.2190423309803009, "mix_mean": 0.028584856539964676, "mix_low": 0.42750200629234314, "mix_high": 3.178914482759865e-07, "read_cosine": 0.275992751121521, "beta_mean": 0.23744958639144897, "nonfinite": 0.0, "weighted_memory_to_active": 0.22295869886875153, "mix_std": 0.04301443696022034, "beta_std": 0.11874863505363464, "beta_low": 0.0002848307485692203, "beta_high": 0.0, "orthogonal_to_active": 0.023952048271894455, "output_influence": 0.18863356113433838, "residual_influence": 0.03215889632701874, "state_active": 0.02027658186852932, "state_memory": 0.047408923506736755, "state_peak": 32.262508392333984, "growth_active": 0.9471359252929688, "growth_memory": 1.4944058656692505, "heads": [{"head": 0, "memory_to_active": 2.2638039588928223, "contribution_to_active": 0.7790390849113464, "mix_mean": 0.04704631119966507, "mix_low": 0.19828128814697266, "mix_high": 1.9073486328125e-06, "read_cosine": 0.3420369625091553, "beta_mean": 0.18672913312911987, "nonfinite": 0.0, "weighted_memory_to_active": 0.7876530289649963, "mix_std": 0.06707087904214859, "beta_std": 0.10969331115484238, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12619872391223907, "evidence_weights": [-0.05080057308077812, -0.07005332410335541, 0.09432293474674225], "agreement_weight": -0.03725570812821388, "orthogonal_weight": 0.23755525052547455}, {"head": 1, "memory_to_active": 2.170973777770996, "contribution_to_active": 0.15517933666706085, "mix_mean": 0.041136499494314194, "mix_low": 0.16222667694091797, "mix_high": 0.0, "read_cosine": 0.03394781053066254, "beta_mean": 0.2855084538459778, "nonfinite": 0.0, "weighted_memory_to_active": 0.1537894904613495, "mix_std": 0.046575047075748444, "beta_std": 0.12668412923812866, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0061584762297570705, "evidence_weights": [-0.05945533886551857, -0.055841512978076935, 0.0995110273361206], "agreement_weight": -0.03457574546337128, "orthogonal_weight": -0.040733274072408676}, {"head": 2, "memory_to_active": 1.3818891048431396, "contribution_to_active": 0.18884369730949402, "mix_mean": 0.04336966574192047, "mix_low": 0.3749198913574219, "mix_high": 0.0, "read_cosine": 0.3358570337295532, "beta_mean": 0.2297210395336151, "nonfinite": 0.0, "weighted_memory_to_active": 0.19291171431541443, "mix_std": 0.08100005239248276, "beta_std": 0.1031493991613388, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005734085571020842, "evidence_weights": [-0.05753573030233383, -0.07443564385175705, 0.11516382545232773], "agreement_weight": 0.018005259335041046, "orthogonal_weight": 0.03556089848279953}, {"head": 3, "memory_to_active": 2.737863063812256, "contribution_to_active": 0.04405876621603966, "mix_mean": 0.011223959736526012, "mix_low": 0.6602621078491211, "mix_high": 0.0, "read_cosine": 0.22819240391254425, "beta_mean": 0.32859688997268677, "nonfinite": 0.0, "weighted_memory_to_active": 0.04554232209920883, "mix_std": 0.014730622060596943, "beta_std": 0.13294467329978943, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007641076808795333, "evidence_weights": [0.0377483144402504, 0.046112388372421265, -0.07079911977052689], "agreement_weight": -0.050982557237148285, "orthogonal_weight": -0.01741926185786724}, {"head": 4, "memory_to_active": 1.5053532123565674, "contribution_to_active": 0.06707067787647247, "mix_mean": 0.01712212711572647, "mix_low": 0.5318508148193359, "mix_high": 0.0, "read_cosine": 0.3231140375137329, "beta_mean": 0.18335817754268646, "nonfinite": 0.0, "weighted_memory_to_active": 0.0688784271478653, "mix_std": 0.031050648540258408, "beta_std": 0.12802498042583466, "beta_low": 0.001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.0037644249387085438, "evidence_weights": [0.008289932273328304, 0.009684736840426922, 0.04457655921578407], "agreement_weight": 0.05516970157623291, "orthogonal_weight": 0.07640216499567032}, {"head": 5, "memory_to_active": 2.397176504135132, "contribution_to_active": 0.08006241917610168, "mix_mean": 0.011610577814280987, "mix_low": 0.6374711990356445, "mix_high": 0.0, "read_cosine": 0.3928082287311554, "beta_mean": 0.21078377962112427, "nonfinite": 0.0, "weighted_memory_to_active": 0.08897718787193298, "mix_std": 0.017659367993474007, "beta_std": 0.11199527978897095, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010924673406407237, "evidence_weights": [0.004187346436083317, 0.018808143213391304, -0.08119522035121918], "agreement_weight": -0.01772991009056568, "orthogonal_weight": 0.016080565750598907}]}, {"layer": 8, "memory_to_active": 1.063410758972168, "contribution_to_active": 0.2741240859031677, "mix_mean": 0.17813055217266083, "mix_low": 0.1038869246840477, "mix_high": 0.003803571220487356, "read_cosine": 0.5602889060974121, "beta_mean": 0.1111224889755249, "nonfinite": 0.0, "weighted_memory_to_active": 0.21053750813007355, "mix_std": 0.16402818262577057, "beta_std": 0.07977969944477081, "beta_low": 0.1655680388212204, "beta_high": 0.0, "orthogonal_to_active": 0.018701985478401184, "output_influence": 0.25083988904953003, "residual_influence": 0.0452844500541687, "state_active": 0.06118971109390259, "state_memory": 0.03670790046453476, "state_peak": 18.429176330566406, "growth_active": 1.0523453950881958, "growth_memory": 1.6560425758361816, "heads": [{"head": 0, "memory_to_active": 0.23578281700611115, "contribution_to_active": 0.2947474718093872, "mix_mean": 0.21549972891807556, "mix_low": 0.020801544189453125, "mix_high": 7.152557373046875e-05, "read_cosine": 0.745180070400238, "beta_mean": 0.008366771973669529, "nonfinite": 0.0, "weighted_memory_to_active": 0.08950251340866089, "mix_std": 0.19259165227413177, "beta_std": 0.007236267905682325, "beta_low": 0.71075439453125, "beta_high": 0.0, "orthogonal_to_active": 0.0016285261372104287, "evidence_weights": [-0.16716906428337097, -0.15230244398117065, 0.20359165966510773], "agreement_weight": 0.07365605235099792, "orthogonal_weight": -0.03269409015774727}, {"head": 1, "memory_to_active": 0.7933919429779053, "contribution_to_active": 0.2511430084705353, "mix_mean": 0.26980483531951904, "mix_low": 0.0008296966552734375, "mix_high": 0.0, "read_cosine": 0.7389609813690186, "beta_mean": 0.08171069622039795, "nonfinite": 0.0, "weighted_memory_to_active": 0.29435279965400696, "mix_std": 0.169008269906044, "beta_std": 0.04558473825454712, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.026993941515684128, "evidence_weights": [-0.1865498423576355, -0.1663891226053238, 0.2569302022457123], "agreement_weight": 0.0037373278755694628, "orthogonal_weight": 0.1595015823841095}, {"head": 2, "memory_to_active": 2.274390697479248, "contribution_to_active": 0.18387901782989502, "mix_mean": 0.03163344785571098, "mix_low": 0.3197460174560547, "mix_high": 0.0, "read_cosine": 0.32918164134025574, "beta_mean": 0.30242374539375305, "nonfinite": 0.0, "weighted_memory_to_active": 0.2226974070072174, "mix_std": 0.04555989429354668, "beta_std": 0.16414441168308258, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.009388823993504047, "evidence_weights": [-0.01991397701203823, -0.03430546447634697, 0.002625878434628248], "agreement_weight": -0.01535209733992815, "orthogonal_weight": -0.05058582127094269}, {"head": 3, "memory_to_active": 0.922568678855896, "contribution_to_active": 0.19792979955673218, "mix_mean": 0.145334392786026, "mix_low": 0.05600166320800781, "mix_high": 0.0, "read_cosine": 0.6113598346710205, "beta_mean": 0.06334865838289261, "nonfinite": 0.0, "weighted_memory_to_active": 0.22680209577083588, "mix_std": 0.12931182980537415, "beta_std": 0.05118027329444885, "beta_low": 0.03668212890625, "beta_high": 0.0, "orthogonal_to_active": 0.028463559225201607, "evidence_weights": [-0.09985020011663437, -0.09632320702075958, 0.2894938290119171], "agreement_weight": 0.021965408697724342, "orthogonal_weight": 0.19543242454528809}, {"head": 4, "memory_to_active": 1.8956398963928223, "contribution_to_active": 0.2765576243400574, "mix_mean": 0.07510822266340256, "mix_low": 0.09602832794189453, "mix_high": 0.0, "read_cosine": 0.38450902700424194, "beta_mean": 0.11464429646730423, "nonfinite": 0.0, "weighted_memory_to_active": 0.2946876883506775, "mix_std": 0.08821457624435425, "beta_std": 0.06469035148620605, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.0038627618923783302, "evidence_weights": [-0.06594357639551163, -0.0666031539440155, 0.05033775046467781], "agreement_weight": -0.018218787387013435, "orthogonal_weight": 0.015049630776047707}, {"head": 5, "memory_to_active": 0.2586904466152191, "contribution_to_active": 0.44048744440078735, "mix_mean": 0.33140259981155396, "mix_low": 0.1299142837524414, "mix_high": 0.022749900817871094, "read_cosine": 0.5525415539741516, "beta_mean": 0.09624077379703522, "nonfinite": 0.0, "weighted_memory_to_active": 0.13518249988555908, "mix_std": 0.35948285460472107, "beta_std": 0.14584214985370636, "beta_low": 0.2454833984375, "beta_high": 0.0, "orthogonal_to_active": 0.04187428951263428, "evidence_weights": [-0.1176125630736351, -0.22803251445293427, -0.03239598870277405], "agreement_weight": 0.028801951557397842, "orthogonal_weight": 0.4148447513580322}]}, {"layer": 9, "memory_to_active": 1.47048819065094, "contribution_to_active": 0.3430916368961334, "mix_mean": 0.06497423350811005, "mix_low": 0.4646972417831421, "mix_high": 0.0002330144343432039, "read_cosine": 0.3548901081085205, "beta_mean": 0.12187102437019348, "nonfinite": 0.0, "weighted_memory_to_active": 0.33211106061935425, "mix_std": 0.11695358157157898, "beta_std": 0.060298241674900055, "beta_low": 0.16876220703125, "beta_high": 0.0, "orthogonal_to_active": 0.007825937122106552, "output_influence": 0.24595515429973602, "residual_influence": 0.04460762441158295, "state_active": 0.02860333025455475, "state_memory": 0.04713457077741623, "state_peak": 28.230201721191406, "growth_active": 0.9647527933120728, "growth_memory": 1.818894624710083, "heads": [{"head": 0, "memory_to_active": 1.1132004261016846, "contribution_to_active": 0.11993777751922607, "mix_mean": 0.037995655089616776, "mix_low": 0.5571994781494141, "mix_high": 0.0, "read_cosine": 0.2583779990673065, "beta_mean": 0.06852798163890839, "nonfinite": 0.0, "weighted_memory_to_active": 0.13595086336135864, "mix_std": 0.09547458589076996, "beta_std": 0.0607527419924736, "beta_low": 0.02435302734375, "beta_high": 0.0, "orthogonal_to_active": 0.008642353117465973, "evidence_weights": [0.01997806504368782, -0.008844457566738129, 0.15628862380981445], "agreement_weight": -0.04895363748073578, "orthogonal_weight": -0.07198516279459}, {"head": 1, "memory_to_active": 0.3264099955558777, "contribution_to_active": 0.06522715836763382, "mix_mean": 0.017321819439530373, "mix_low": 0.8136806488037109, "mix_high": 0.0, "read_cosine": 0.37494319677352905, "beta_mean": 0.0025067091919481754, "nonfinite": 0.0, "weighted_memory_to_active": 0.016406012699007988, "mix_std": 0.0663108080625534, "beta_std": 0.003442020621150732, "beta_low": 0.96478271484375, "beta_high": 0.0, "orthogonal_to_active": 0.0011857287026941776, "evidence_weights": [0.018581200391054153, 0.024112777784466743, -0.04053901508450508], "agreement_weight": 0.05148047208786011, "orthogonal_weight": -0.0717013031244278}, {"head": 2, "memory_to_active": 0.7664869427680969, "contribution_to_active": 0.3160899877548218, "mix_mean": 0.08570730686187744, "mix_low": 0.4132204055786133, "mix_high": 1.239776611328125e-05, "read_cosine": 0.19126543402671814, "beta_mean": 0.03920736163854599, "nonfinite": 0.0, "weighted_memory_to_active": 0.17405742406845093, "mix_std": 0.1621159166097641, "beta_std": 0.04164201766252518, "beta_low": 0.0234375, "beta_high": 0.0, "orthogonal_to_active": 0.006929377093911171, "evidence_weights": [-0.05228118970990181, -0.04195620119571686, 0.11481025069952011], "agreement_weight": 0.05331345275044441, "orthogonal_weight": 0.048309795558452606}, {"head": 3, "memory_to_active": 1.987642526626587, "contribution_to_active": 0.19709165394306183, "mix_mean": 0.06593474745750427, "mix_low": 0.21816158294677734, "mix_high": 0.0, "read_cosine": 0.4407344460487366, "beta_mean": 0.15403646230697632, "nonfinite": 0.0, "weighted_memory_to_active": 0.20336629450321198, "mix_std": 0.08118975162506104, "beta_std": 0.0779777392745018, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009938490111380816, "evidence_weights": [-0.008279393427073956, -0.006491481326520443, 0.0030795468483120203], "agreement_weight": -0.05308861285448074, "orthogonal_weight": 0.005196514539420605}, {"head": 4, "memory_to_active": 2.8363351821899414, "contribution_to_active": 1.1524980068206787, "mix_mean": 0.1451207399368286, "mix_low": 0.15606403350830078, "mix_high": 0.0013856887817382812, "read_cosine": 0.4068911373615265, "beta_mean": 0.2424091249704361, "nonfinite": 0.0, "weighted_memory_to_active": 1.1764277219772339, "mix_std": 0.2032734900712967, "beta_std": 0.0787346288561821, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02078675664961338, "evidence_weights": [-0.05734231695532799, -0.06324484199285507, 0.11213561147451401], "agreement_weight": -0.0637788325548172, "orthogonal_weight": 0.019310375675559044}, {"head": 5, "memory_to_active": 1.7928540706634521, "contribution_to_active": 0.20770519971847534, "mix_mean": 0.03776510804891586, "mix_low": 0.629857063293457, "mix_high": 0.0, "read_cosine": 0.4571281671524048, "beta_mean": 0.22453849017620087, "nonfinite": 0.0, "weighted_memory_to_active": 0.286457896232605, "mix_std": 0.09335692226886749, "beta_std": 0.0992402732372284, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00841755885630846, "evidence_weights": [0.020929083228111267, -0.00028135362663306296, -0.021678829565644264], "agreement_weight": -0.0159771628677845, "orthogonal_weight": 0.042688269168138504}]}, {"layer": 10, "memory_to_active": 0.8411193490028381, "contribution_to_active": 0.2704358398914337, "mix_mean": 0.13840769231319427, "mix_low": 0.2699238657951355, "mix_high": 0.0038177173119038343, "read_cosine": 0.3867585062980652, "beta_mean": 0.07964983582496643, "nonfinite": 0.0, "weighted_memory_to_active": 0.156650111079216, "mix_std": 0.1583840548992157, "beta_std": 0.07942117750644684, "beta_low": 0.2123311460018158, "beta_high": 0.0, "orthogonal_to_active": 0.019577842205762863, "output_influence": 0.32843124866485596, "residual_influence": 0.07605774700641632, "state_active": 0.09498348832130432, "state_memory": 0.027686113491654396, "state_peak": 36.92164611816406, "growth_active": 1.3369481563568115, "growth_memory": 1.577770709991455, "heads": [{"head": 0, "memory_to_active": 0.5291455984115601, "contribution_to_active": 0.6289559602737427, "mix_mean": 0.37699586153030396, "mix_low": 0.0331878662109375, "mix_high": 0.018207550048828125, "read_cosine": 0.20291467010974884, "beta_mean": 0.14743757247924805, "nonfinite": 0.0, "weighted_memory_to_active": 0.3004069924354553, "mix_std": 0.30847036838531494, "beta_std": 0.18055272102355957, "beta_low": 0.0904541015625, "beta_high": 0.0, "orthogonal_to_active": 0.053110744804143906, "evidence_weights": [-0.10295957326889038, -0.18794625997543335, 0.07779516279697418], "agreement_weight": 0.009506921283900738, "orthogonal_weight": 0.19379253685474396}, {"head": 1, "memory_to_active": 1.6635546684265137, "contribution_to_active": 0.09927934408187866, "mix_mean": 0.028638238087296486, "mix_low": 0.43065834045410156, "mix_high": 0.0, "read_cosine": 0.2930687665939331, "beta_mean": 0.07998790591955185, "nonfinite": 0.0, "weighted_memory_to_active": 0.09095774590969086, "mix_std": 0.055132102221250534, "beta_std": 0.059456247836351395, "beta_low": 0.015869140625, "beta_high": 0.0, "orthogonal_to_active": 0.010131508111953735, "evidence_weights": [-0.0046808053739368916, 0.0027871523052453995, -0.05397531017661095], "agreement_weight": -0.038408536463975906, "orthogonal_weight": -0.12374833226203918}, {"head": 2, "memory_to_active": 1.8371371030807495, "contribution_to_active": 0.3404942452907562, "mix_mean": 0.03050229884684086, "mix_low": 0.5234670639038086, "mix_high": 0.0, "read_cosine": 0.22661878168582916, "beta_mean": 0.13465389609336853, "nonfinite": 0.0, "weighted_memory_to_active": 0.37040388584136963, "mix_std": 0.07440105825662613, "beta_std": 0.07385696470737457, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02302035316824913, "evidence_weights": [-0.004792256746441126, -0.005140502937138081, -0.035378821194171906], "agreement_weight": -0.02653651125729084, "orthogonal_weight": -0.06438729912042618}, {"head": 3, "memory_to_active": 0.15419526398181915, "contribution_to_active": 0.330178439617157, "mix_mean": 0.2432408332824707, "mix_low": 0.12179374694824219, "mix_high": 0.0045642852783203125, "read_cosine": 0.6939067840576172, "beta_mean": 0.072911337018013, "nonfinite": 0.0, "weighted_memory_to_active": 0.07225287705659866, "mix_std": 0.29934415221214294, "beta_std": 0.11237278580665588, "beta_low": 0.255126953125, "beta_high": 0.0, "orthogonal_to_active": 0.019844336435198784, "evidence_weights": [-0.13891799747943878, -0.14889033138751984, 0.31555449962615967], "agreement_weight": 0.019742535427212715, "orthogonal_weight": 0.38391348719596863}, {"head": 4, "memory_to_active": 0.519662618637085, "contribution_to_active": 0.05522872135043144, "mix_mean": 0.028671134263277054, "mix_low": 0.44305419921875, "mix_high": 0.0, "read_cosine": 0.2734142541885376, "beta_mean": 0.009387722238898277, "nonfinite": 0.0, "weighted_memory_to_active": 0.033436279743909836, "mix_std": 0.05777602270245552, "beta_std": 0.011450784280896187, "beta_low": 0.69134521484375, "beta_high": 0.0, "orthogonal_to_active": 0.004967693705111742, "evidence_weights": [-0.003381401998922229, 0.016117263585329056, -0.12164653092622757], "agreement_weight": 0.031068257987499237, "orthogonal_weight": -0.16157864034175873}, {"head": 5, "memory_to_active": 0.3430207371711731, "contribution_to_active": 0.16847820580005646, "mix_mean": 0.12239779531955719, "mix_low": 0.0673818588256836, "mix_high": 0.00013446807861328125, "read_cosine": 0.6306276321411133, "beta_mean": 0.033520497381687164, "nonfinite": 0.0, "weighted_memory_to_active": 0.0724429041147232, "mix_std": 0.15518055856227875, "beta_std": 0.038837581872940063, "beta_low": 0.22119140625, "beta_high": 0.0, "orthogonal_to_active": 0.006392418406903744, "evidence_weights": [-0.09732265025377274, -0.08250147104263306, 0.13441665470600128], "agreement_weight": 0.0796712189912796, "orthogonal_weight": -0.1320870965719223}]}, {"layer": 11, "memory_to_active": 0.4136730134487152, "contribution_to_active": 0.32936444878578186, "mix_mean": 0.21234145760536194, "mix_low": 0.11662530899047852, "mix_high": 0.022908847779035568, "read_cosine": 0.5505306720733643, "beta_mean": 0.048329249024391174, "nonfinite": 0.0, "weighted_memory_to_active": 0.10972055792808533, "mix_std": 0.23802559077739716, "beta_std": 0.06567423045635223, "beta_low": 0.37799072265625, "beta_high": 0.0, "orthogonal_to_active": 0.013515202328562737, "output_influence": 0.5011826157569885, "residual_influence": 0.12362051010131836, "state_active": 0.11281143128871918, "state_memory": 0.019903266802430153, "state_peak": 26.668838500976562, "growth_active": 1.1755454540252686, "growth_memory": 1.4892889261245728, "heads": [{"head": 0, "memory_to_active": 0.11824482679367065, "contribution_to_active": 0.30171167850494385, "mix_mean": 0.1419616937637329, "mix_low": 0.13546466827392578, "mix_high": 0.00034809112548828125, "read_cosine": 0.5041424036026001, "beta_mean": 0.0008503894787281752, "nonfinite": 0.0, "weighted_memory_to_active": 0.04637221246957779, "mix_std": 0.1958272010087967, "beta_std": 0.0013824107591062784, "beta_low": 0.99725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.009316917508840561, "evidence_weights": [-0.07105012238025665, -0.07455266267061234, 0.07773260772228241], "agreement_weight": 0.1434430480003357, "orthogonal_weight": -0.30078139901161194}, {"head": 1, "memory_to_active": 0.5528343319892883, "contribution_to_active": 0.16374334692955017, "mix_mean": 0.1263871192932129, "mix_low": 0.07835006713867188, "mix_high": 0.0, "read_cosine": 0.5562723875045776, "beta_mean": 0.025024235248565674, "nonfinite": 0.0, "weighted_memory_to_active": 0.10650306940078735, "mix_std": 0.16405434906482697, "beta_std": 0.03454047814011574, "beta_low": 0.3707275390625, "beta_high": 0.0, "orthogonal_to_active": 0.0007139615481719375, "evidence_weights": [-0.07372265309095383, -0.054900649935007095, -0.008470600470900536], "agreement_weight": 0.09028603136539459, "orthogonal_weight": -0.009352769702672958}, {"head": 2, "memory_to_active": 0.10136374086141586, "contribution_to_active": 0.6465266942977905, "mix_mean": 0.3674537241458893, "mix_low": 0.15109634399414062, "mix_high": 0.053612709045410156, "read_cosine": 0.46317192912101746, "beta_mean": 0.06397458165884018, "nonfinite": 0.0, "weighted_memory_to_active": 0.0760120302438736, "mix_std": 0.3897651731967926, "beta_std": 0.10757293552160263, "beta_low": 0.4283447265625, "beta_high": 0.0, "orthogonal_to_active": 0.04154914617538452, "evidence_weights": [-0.1713210642337799, -0.19516761600971222, 0.055851422250270844], "agreement_weight": -0.025962216779589653, "orthogonal_weight": 0.6528850793838501}, {"head": 3, "memory_to_active": 0.08551640808582306, "contribution_to_active": 0.4192827343940735, "mix_mean": 0.42520445585250854, "mix_low": 0.10899734497070312, "mix_high": 0.08348464965820312, "read_cosine": 0.5877649188041687, "beta_mean": 0.08829735219478607, "nonfinite": 0.0, "weighted_memory_to_active": 0.047070637345314026, "mix_std": 0.4056869447231293, "beta_std": 0.13005398213863373, "beta_low": 0.29931640625, "beta_high": 0.0, "orthogonal_to_active": 0.019228141754865646, "evidence_weights": [-0.2588502764701843, -0.2474193274974823, 0.21028873324394226], "agreement_weight": 0.002753285923972726, "orthogonal_weight": 0.5075904726982117}, {"head": 4, "memory_to_active": 1.1463401317596436, "contribution_to_active": 0.1435098797082901, "mix_mean": 0.070955291390419, "mix_low": 0.15956401824951172, "mix_high": 0.0, "read_cosine": 0.3541324734687805, "beta_mean": 0.05953620374202728, "nonfinite": 0.0, "weighted_memory_to_active": 0.1334928572177887, "mix_std": 0.1032872200012207, "beta_std": 0.057589780539274216, "beta_low": 0.0478515625, "beta_high": 0.0, "orthogonal_to_active": 2.974635935970582e-05, "evidence_weights": [-0.023296840488910675, -0.01225265208631754, 0.16226078569889069], "agreement_weight": -0.0020531208720058203, "orthogonal_weight": -0.0002813432365655899}, {"head": 5, "memory_to_active": 0.477738618850708, "contribution_to_active": 0.30141234397888184, "mix_mean": 0.1420864462852478, "mix_low": 0.06627941131591797, "mix_high": 7.62939453125e-06, "read_cosine": 0.837699830532074, "beta_mean": 0.052292726933956146, "nonfinite": 0.0, "weighted_memory_to_active": 0.2488725483417511, "mix_std": 0.16953259706497192, "beta_std": 0.06290577352046967, "beta_low": 0.12445068359375, "beta_high": 0.0, "orthogonal_to_active": 0.010253301821649075, "evidence_weights": [-0.09334779530763626, -0.11842869222164154, 0.17027562856674194], "agreement_weight": 0.06313115358352661, "orthogonal_weight": -0.1443520188331604}]}, {"layer": 12, "memory_to_active": 0.9229854941368103, "contribution_to_active": 0.2919137179851532, "mix_mean": 0.12918227910995483, "mix_low": 0.1648203581571579, "mix_high": 3.2583873689873144e-05, "read_cosine": 0.3180727958679199, "beta_mean": 0.05973780155181885, "nonfinite": 0.0, "weighted_memory_to_active": 0.23299740254878998, "mix_std": 0.15391504764556885, "beta_std": 0.05486878752708435, "beta_low": 0.2613118588924408, "beta_high": 0.0, "orthogonal_to_active": 0.01959843374788761, "output_influence": 0.1513548195362091, "residual_influence": 0.03426305577158928, "state_active": 0.050939157605171204, "state_memory": 0.03132639825344086, "state_peak": 26.084110260009766, "growth_active": 1.1333956718444824, "growth_memory": 1.7652432918548584, "heads": [{"head": 0, "memory_to_active": 1.5137499570846558, "contribution_to_active": 0.2830512225627899, "mix_mean": 0.11802953481674194, "mix_low": 0.05098533630371094, "mix_high": 0.0001316070556640625, "read_cosine": 0.410578191280365, "beta_mean": 0.07742896676063538, "nonfinite": 0.0, "weighted_memory_to_active": 0.3414653241634369, "mix_std": 0.134990394115448, "beta_std": 0.09039776027202606, "beta_low": 0.009765625, "beta_high": 0.0, "orthogonal_to_active": 0.01136926282197237, "evidence_weights": [-0.07318408042192459, -0.09302579611539841, 0.1661558598279953], "agreement_weight": 0.06900199502706528, "orthogonal_weight": -0.04686383903026581}, {"head": 1, "memory_to_active": 0.5953207612037659, "contribution_to_active": 0.1330973505973816, "mix_mean": 0.08139526844024658, "mix_low": 0.20963478088378906, "mix_high": 0.0, "read_cosine": 0.48073074221611023, "beta_mean": 0.014651907607913017, "nonfinite": 0.0, "weighted_memory_to_active": 0.09700179100036621, "mix_std": 0.12355653941631317, "beta_std": 0.011316146701574326, "beta_low": 0.38739013671875, "beta_high": 0.0, "orthogonal_to_active": 0.021148454397916794, "evidence_weights": [-0.020644547417759895, -0.05923064798116684, 0.11891748756170273], "agreement_weight": 0.12045637518167496, "orthogonal_weight": -0.3232337534427643}, {"head": 2, "memory_to_active": 0.465420663356781, "contribution_to_active": 0.3199536204338074, "mix_mean": 0.19960469007492065, "mix_low": 0.024735450744628906, "mix_high": 3.0517578125e-05, "read_cosine": 0.22357787191867828, "beta_mean": 0.042186714708805084, "nonfinite": 0.0, "weighted_memory_to_active": 0.18054896593093872, "mix_std": 0.1954241842031479, "beta_std": 0.0537724532186985, "beta_low": 0.22186279296875, "beta_high": 0.0, "orthogonal_to_active": 0.0040605328977108, "evidence_weights": [-0.08804485201835632, -0.08470985293388367, 0.19006648659706116], "agreement_weight": 0.05582315847277641, "orthogonal_weight": -0.024433333426713943}, {"head": 3, "memory_to_active": 0.3816385567188263, "contribution_to_active": 0.25780338048934937, "mix_mean": 0.17865288257598877, "mix_low": 0.07441902160644531, "mix_high": 1.52587890625e-05, "read_cosine": 0.10480184853076935, "beta_mean": 0.011039389297366142, "nonfinite": 0.0, "weighted_memory_to_active": 0.11253800988197327, "mix_std": 0.19347219169139862, "beta_std": 0.013166683726012707, "beta_low": 0.66375732421875, "beta_high": 0.0, "orthogonal_to_active": 0.014864141121506691, "evidence_weights": [-0.07707089185714722, -0.10219094902276993, 0.02255328930914402], "agreement_weight": -0.03893204405903816, "orthogonal_weight": -0.1392855942249298}, {"head": 4, "memory_to_active": 2.2009778022766113, "contribution_to_active": 0.5209220051765442, "mix_mean": 0.026163499802350998, "mix_low": 0.5324983596801758, "mix_high": 9.5367431640625e-06, "read_cosine": 0.22788290679454803, "beta_mean": 0.18917544186115265, "nonfinite": 0.0, "weighted_memory_to_active": 0.5514516234397888, "mix_std": 0.06857208907604218, "beta_std": 0.1372928023338318, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01943185180425644, "evidence_weights": [0.021069781854748726, 0.011886950582265854, 0.02991967834532261], "agreement_weight": -0.02998436614871025, "orthogonal_weight": -0.037936899811029434}, {"head": 5, "memory_to_active": 0.3808049261569977, "contribution_to_active": 0.23665466904640198, "mix_mean": 0.17124782502651215, "mix_low": 0.096649169921875, "mix_high": 8.58306884765625e-06, "read_cosine": 0.46086519956588745, "beta_mean": 0.023944364860653877, "nonfinite": 0.0, "weighted_memory_to_active": 0.11497879773378372, "mix_std": 0.20747482776641846, "beta_std": 0.02326686680316925, "beta_low": 0.28509521484375, "beta_high": 0.0, "orthogonal_to_active": 0.046716347336769104, "evidence_weights": [-0.09790299832820892, -0.11578001081943512, 0.2035999745130539], "agreement_weight": 0.06658519804477692, "orthogonal_weight": -0.5096101760864258}]}, {"layer": 13, "memory_to_active": 0.5721377730369568, "contribution_to_active": 0.22775140404701233, "mix_mean": 0.176348477602005, "mix_low": 0.1496402472257614, "mix_high": 0.0004914601449854672, "read_cosine": 0.2913375496864319, "beta_mean": 0.052808213979005814, "nonfinite": 0.0, "weighted_memory_to_active": 0.10049393028020859, "mix_std": 0.1538597047328949, "beta_std": 0.062028419226408005, "beta_low": 0.2834269404411316, "beta_high": 0.0, "orthogonal_to_active": 0.017272822558879852, "output_influence": 0.2080610692501068, "residual_influence": 0.06056270748376846, "state_active": 0.09522300958633423, "state_memory": 0.03312372416257858, "state_peak": 28.175827026367188, "growth_active": 1.2241746187210083, "growth_memory": 1.7211264371871948, "heads": [{"head": 0, "memory_to_active": 0.7035999298095703, "contribution_to_active": 0.12155577540397644, "mix_mean": 0.0552423894405365, "mix_low": 0.2849464416503906, "mix_high": 0.0, "read_cosine": 0.20737649500370026, "beta_mean": 0.038475021719932556, "nonfinite": 0.0, "weighted_memory_to_active": 0.09038414061069489, "mix_std": 0.09802035987377167, "beta_std": 0.03624855726957321, "beta_low": 0.20538330078125, "beta_high": 0.0, "orthogonal_to_active": 0.046565547585487366, "evidence_weights": [-0.014008657075464725, -0.038308221846818924, -0.06974434852600098], "agreement_weight": 0.10505257546901703, "orthogonal_weight": -0.5505568981170654}, {"head": 1, "memory_to_active": 1.072988748550415, "contribution_to_active": 0.09270387142896652, "mix_mean": 0.039686478674411774, "mix_low": 0.3532228469848633, "mix_high": 0.0, "read_cosine": 0.2421909123659134, "beta_mean": 0.07517523318529129, "nonfinite": 0.0, "weighted_memory_to_active": 0.08682577311992645, "mix_std": 0.06954366713762283, "beta_std": 0.0638074204325676, "beta_low": 0.00701904296875, "beta_high": 0.0, "orthogonal_to_active": 0.029505541548132896, "evidence_weights": [0.022537175565958023, -0.013921746052801609, -0.0042451475746929646], "agreement_weight": 0.102533720433712, "orthogonal_weight": -0.40120649337768555}, {"head": 2, "memory_to_active": 0.37107834219932556, "contribution_to_active": 0.20796553790569305, "mix_mean": 0.15917804837226868, "mix_low": 0.029275894165039062, "mix_high": 9.5367431640625e-07, "read_cosine": 0.6266391277313232, "beta_mean": 0.04670459032058716, "nonfinite": 0.0, "weighted_memory_to_active": 0.11762513220310211, "mix_std": 0.14981424808502197, "beta_std": 0.07222308963537216, "beta_low": 0.27850341796875, "beta_high": 0.0, "orthogonal_to_active": 0.002095166128128767, "evidence_weights": [-0.12027406692504883, -0.11318208277225494, 0.27190354466438293], "agreement_weight": 0.09439542144536972, "orthogonal_weight": -0.024718929082155228}, {"head": 3, "memory_to_active": 0.03141959011554718, "contribution_to_active": 0.40128445625305176, "mix_mean": 0.37187618017196655, "mix_low": 0.0013532638549804688, "mix_high": 7.62939453125e-06, "read_cosine": -0.25744470953941345, "beta_mean": 0.00024258648045361042, "nonfinite": 0.0, "weighted_memory_to_active": 0.014045625925064087, "mix_std": 0.1942829191684723, "beta_std": 0.0006011682562530041, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.0025826189666986465, "evidence_weights": [-0.26495301723480225, -0.23606736958026886, 0.0038587385788559914], "agreement_weight": 0.2006615251302719, "orthogonal_weight": -0.18939921259880066}, {"head": 4, "memory_to_active": 0.29539310932159424, "contribution_to_active": 0.34265926480293274, "mix_mean": 0.3587745428085327, "mix_low": 0.01687335968017578, "mix_high": 0.0029401779174804688, "read_cosine": 0.7076356410980225, "beta_mean": 0.08230587095022202, "nonfinite": 0.0, "weighted_memory_to_active": 0.13416452705860138, "mix_std": 0.2927059531211853, "beta_std": 0.09536732733249664, "beta_low": 0.11236572265625, "beta_high": 0.0, "orthogonal_to_active": 0.01725461520254612, "evidence_weights": [-0.21072496473789215, -0.23413848876953125, 0.2770151197910309], "agreement_weight": 0.013278582133352757, "orthogonal_weight": 0.19069784879684448}, {"head": 5, "memory_to_active": 0.9583465456962585, "contribution_to_active": 0.20033952593803406, "mix_mean": 0.07333317399024963, "mix_low": 0.21216964721679688, "mix_high": 0.0, "read_cosine": 0.22162771224975586, "beta_mean": 0.07394596934318542, "nonfinite": 0.0, "weighted_memory_to_active": 0.15991833806037903, "mix_std": 0.11879103630781174, "beta_std": 0.10392294079065323, "beta_low": 0.09765625, "beta_high": 0.0, "orthogonal_to_active": 0.005633448716253042, "evidence_weights": [-0.06523668766021729, -0.061851486563682556, -0.05883434787392616], "agreement_weight": 0.0003181997162755579, "orthogonal_weight": -0.03792624548077583}]}, {"layer": 14, "memory_to_active": 0.36563676595687866, "contribution_to_active": 0.34471070766448975, "mix_mean": 0.28213730454444885, "mix_low": 0.048045478761196136, "mix_high": 0.00040944418287836015, "read_cosine": 0.5948100686073303, "beta_mean": 0.09938167780637741, "nonfinite": 0.0, "weighted_memory_to_active": 0.14028862118721008, "mix_std": 0.22313399612903595, "beta_std": 0.09325339645147324, "beta_low": 0.1171875, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.015545870177447796, "output_influence": 0.1950673907995224, "residual_influence": 0.06153865158557892, "state_active": 0.1047714501619339, "state_memory": 0.030706752091646194, "state_peak": 30.229061126708984, "growth_active": 1.3758084774017334, "growth_memory": 1.6340560913085938, "heads": [{"head": 0, "memory_to_active": 0.35067468881607056, "contribution_to_active": 0.3428230285644531, "mix_mean": 0.2990407347679138, "mix_low": 0.022586822509765625, "mix_high": 0.0009279251098632812, "read_cosine": 0.6328122615814209, "beta_mean": 0.0988025963306427, "nonfinite": 0.0, "weighted_memory_to_active": 0.15333472192287445, "mix_std": 0.25305116176605225, "beta_std": 0.13623324036598206, "beta_low": 0.182861328125, "beta_high": 0.0, "orthogonal_to_active": 0.009621281176805496, "evidence_weights": [-0.10790044814348221, -0.16276425123214722, 0.27902838587760925], "agreement_weight": 0.07262135297060013, "orthogonal_weight": -0.08891899883747101}, {"head": 1, "memory_to_active": 0.4334699213504791, "contribution_to_active": 0.4287618398666382, "mix_mean": 0.3210105299949646, "mix_low": 0.06319713592529297, "mix_high": 0.0010557174682617188, "read_cosine": 0.46165966987609863, "beta_mean": 0.2790403962135315, "nonfinite": 0.0, "weighted_memory_to_active": 0.20998965203762054, "mix_std": 0.2942017614841461, "beta_std": 0.2585115134716034, "beta_low": 0.07061767578125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.018967725336551666, "evidence_weights": [-0.09242004156112671, -0.17273183166980743, 0.25974413752555847], "agreement_weight": 0.060604896396398544, "orthogonal_weight": -0.1135188415646553}, {"head": 2, "memory_to_active": 0.46820932626724243, "contribution_to_active": 0.27686607837677, "mix_mean": 0.19811470806598663, "mix_low": 0.030470848083496094, "mix_high": 2.47955322265625e-05, "read_cosine": 0.5610386729240417, "beta_mean": 0.08169512450695038, "nonfinite": 0.0, "weighted_memory_to_active": 0.16043011844158173, "mix_std": 0.19067904353141785, "beta_std": 0.0502127930521965, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.038485847413539886, "evidence_weights": [-0.13857708871364594, -0.14919833838939667, 0.02011336199939251], "agreement_weight": 0.11965063959360123, "orthogonal_weight": -0.33266931772232056}, {"head": 3, "memory_to_active": 0.15737485885620117, "contribution_to_active": 0.5126643180847168, "mix_mean": 0.5262671113014221, "mix_low": 0.0024976730346679688, "mix_high": 0.00030517578125, "read_cosine": 0.7063825130462646, "beta_mean": 0.038064319640398026, "nonfinite": 0.0, "weighted_memory_to_active": 0.09330545365810394, "mix_std": 0.2476041465997696, "beta_std": 0.029156237840652466, "beta_low": 0.03497314453125, "beta_high": 0.0, "orthogonal_to_active": 0.011571699753403664, "evidence_weights": [-0.31336158514022827, -0.3024103343486786, 0.3235357999801636], "agreement_weight": 0.06070137396454811, "orthogonal_weight": -0.200980082154274}, {"head": 4, "memory_to_active": 0.3369513154029846, "contribution_to_active": 0.3699885308742523, "mix_mean": 0.24416880309581757, "mix_low": 0.019687652587890625, "mix_high": 0.0001430511474609375, "read_cosine": 0.5413661003112793, "beta_mean": 0.07798705995082855, "nonfinite": 0.0, "weighted_memory_to_active": 0.14000259339809418, "mix_std": 0.21711093187332153, "beta_std": 0.06227662041783333, "beta_low": 0.00433349609375, "beta_high": 0.0, "orthogonal_to_active": 0.0073679061606526375, "evidence_weights": [-0.1260223090648651, -0.15478770434856415, 0.10134873539209366], "agreement_weight": 0.10241999477148056, "orthogonal_weight": 0.07476837933063507}, {"head": 5, "memory_to_active": 0.4471404552459717, "contribution_to_active": 0.13716048002243042, "mix_mean": 0.1042219027876854, "mix_low": 0.14983272552490234, "mix_high": 0.0, "read_cosine": 0.6656011343002319, "beta_mean": 0.020700542256236076, "nonfinite": 0.0, "weighted_memory_to_active": 0.08466919511556625, "mix_std": 0.1361568719148636, "beta_std": 0.023129964247345924, "beta_low": 0.4102783203125, "beta_high": 0.0, "orthogonal_to_active": 0.00726075004786253, "evidence_weights": [-0.09663863480091095, -0.08414537459611893, 0.012694166973233223], "agreement_weight": 0.09482823312282562, "orthogonal_weight": -0.1355990171432495}]}, {"layer": 15, "memory_to_active": 0.16535302996635437, "contribution_to_active": 0.35680079460144043, "mix_mean": 0.28087881207466125, "mix_low": 0.01986408419907093, "mix_high": 0.0009508132934570312, "read_cosine": 0.5176053047180176, "beta_mean": 0.025365717709064484, "nonfinite": 0.0, "weighted_memory_to_active": 0.07934882491827011, "mix_std": 0.23023295402526855, "beta_std": 0.03180771321058273, "beta_low": 0.633697509765625, "beta_high": 0.0, "orthogonal_to_active": 0.005288105923682451, "output_influence": 0.2602532207965851, "residual_influence": 0.12292353808879852, "state_active": 0.10697714239358902, "state_memory": 0.02115190587937832, "state_peak": 45.484310150146484, "growth_active": 1.4682660102844238, "growth_memory": 1.671707272529602, "heads": [{"head": 0, "memory_to_active": 0.16192252933979034, "contribution_to_active": 0.41824978590011597, "mix_mean": 0.3313429355621338, "mix_low": 0.008258819580078125, "mix_high": 0.0001583099365234375, "read_cosine": 0.6470838785171509, "beta_mean": 0.08578234910964966, "nonfinite": 0.0, "weighted_memory_to_active": 0.0725376158952713, "mix_std": 0.24689823389053345, "beta_std": 0.07837230712175369, "beta_low": 0.0986328125, "beta_high": 0.0, "orthogonal_to_active": 0.002523615024983883, "evidence_weights": [-0.1848110556602478, -0.22393403947353363, 0.2343427687883377], "agreement_weight": 0.07397950440645218, "orthogonal_weight": -0.06245654821395874}, {"head": 1, "memory_to_active": 0.26129305362701416, "contribution_to_active": 0.36093729734420776, "mix_mean": 0.25792592763900757, "mix_low": 0.015451431274414062, "mix_high": 0.00208282470703125, "read_cosine": 0.7890966534614563, "beta_mean": 0.005524656269699335, "nonfinite": 0.0, "weighted_memory_to_active": 0.11875725537538528, "mix_std": 0.225848987698555, "beta_std": 0.006366218905895948, "beta_low": 0.87213134765625, "beta_high": 0.0, "orthogonal_to_active": 0.00803739670664072, "evidence_weights": [-0.17473231256008148, -0.18946954607963562, 0.2212066799402237], "agreement_weight": 0.07591281086206436, "orthogonal_weight": -0.1405152976512909}, {"head": 2, "memory_to_active": 0.046910665929317474, "contribution_to_active": 0.3654518723487854, "mix_mean": 0.289775013923645, "mix_low": 0.0246429443359375, "mix_high": 0.00038051605224609375, "read_cosine": 0.7251611948013306, "beta_mean": 0.0018632786814123392, "nonfinite": 0.0, "weighted_memory_to_active": 0.018665244802832603, "mix_std": 0.24689799547195435, "beta_std": 0.0021547966171056032, "beta_low": 0.987060546875, "beta_high": 0.0, "orthogonal_to_active": 0.004301262553781271, "evidence_weights": [-0.21699729561805725, -0.2209940254688263, 0.26428285241127014], "agreement_weight": 0.062137383967638016, "orthogonal_weight": -0.3417646586894989}, {"head": 3, "memory_to_active": 0.41163554787635803, "contribution_to_active": 0.22356818616390228, "mix_mean": 0.1548108458518982, "mix_low": 0.04084968566894531, "mix_high": 4.00543212890625e-05, "read_cosine": 0.6618964672088623, "beta_mean": 0.04322081804275513, "nonfinite": 0.0, "weighted_memory_to_active": 0.2237851619720459, "mix_std": 0.16316208243370056, "beta_std": 0.06797664612531662, "beta_low": 0.1910400390625, "beta_high": 0.0, "orthogonal_to_active": 0.0026510395109653473, "evidence_weights": [-0.08877143263816833, -0.16376495361328125, 0.23882368206977844], "agreement_weight": 0.0811084434390068, "orthogonal_weight": 0.02010667882859707}, {"head": 4, "memory_to_active": 0.10693809390068054, "contribution_to_active": 0.3884707987308502, "mix_mean": 0.26802390813827515, "mix_low": 0.021653175354003906, "mix_high": 7.915496826171875e-05, "read_cosine": -0.20446933805942535, "beta_mean": 0.01573863811790943, "nonfinite": 0.0, "weighted_memory_to_active": 0.04115748032927513, "mix_std": 0.23148560523986816, "beta_std": 0.03572271019220352, "beta_low": 0.65338134765625, "beta_high": 0.0, "orthogonal_to_active": 0.013997873291373253, "evidence_weights": [-0.2400747537612915, -0.2301245927810669, 0.14405469596385956], "agreement_weight": 0.010292191989719868, "orthogonal_weight": -0.3698679804801941}, {"head": 5, "memory_to_active": 0.0034182302188128233, "contribution_to_active": 0.38412681221961975, "mix_mean": 0.3833940625190735, "mix_low": 0.008328437805175781, "mix_high": 0.002964019775390625, "read_cosine": 0.48686277866363525, "beta_mean": 6.456767732743174e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0011901904363185167, "mix_std": 0.2671048641204834, "beta_std": 0.00025358848506584764, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0002174472901970148, "evidence_weights": [-0.30265340209007263, -0.26475468277931213, 0.28543004393577576], "agreement_weight": 0.01958634704351425, "orthogonal_weight": 0.18706706166267395}]}]}, {"step": 4000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8006178630002978, "memory_to_active": 1.1876063346862793, "contribution_to_active": 0.30025583505630493, "mix_mean": 0.1570095419883728, "mix_low": 0.21407587826251984, "mix_high": 0.0025207698345184326, "read_cosine": 0.41253429651260376, "beta_mean": 0.1258978694677353, "nonfinite": 0.0, "weighted_memory_to_active": 0.18998229503631592, "mix_std": 0.1515299677848816, "beta_std": 0.08883802592754364, "beta_low": 0.2611020505428314, "beta_high": 4.9591064453125e-05, "orthogonal_to_active": 0.0213452335447073, "samples": 4, "scored_tokens": 899, "normal_loss": 3.4027748107910156, "normal_token_accuracy": 0.41713014245033264, "state_active": 0.06978881359100342, "state_memory": 0.040548767894506454, "state_peak": 46.52994918823242, "growth_active": 1.1125092506408691, "growth_memory": 1.6351817846298218, "output_influence": 0.23675251007080078, "residual_influence": 0.06627412140369415, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.517042636871338, "no_memory_loss_delta": 0.06323323398828506, "scrambled_loss_delta": 0.34215283393859863, "memory_only_loss_delta": 1.2976816892623901, "no_evidence_loss_delta": 0.028374062851071358, "no_agreement_loss_delta": 0.01666717603802681, "no_orthogonal_loss_delta": -0.006288506556302309}, {"index": 94, "tokens": 256, "normal_loss": 3.5154507160186768, "no_memory_loss_delta": 0.23883415758609772, "scrambled_loss_delta": 0.4393064081668854, "memory_only_loss_delta": 1.570796251296997, "no_evidence_loss_delta": 0.0655059665441513, "no_agreement_loss_delta": 0.020966803655028343, "no_orthogonal_loss_delta": 0.003269373904913664}, {"index": 188, "tokens": 256, "normal_loss": 3.724432945251465, "no_memory_loss_delta": 0.05749799683690071, "scrambled_loss_delta": 0.1124812662601471, "memory_only_loss_delta": 1.603188395500183, "no_evidence_loss_delta": 0.0157308466732502, "no_agreement_loss_delta": 0.007624060846865177, "no_orthogonal_loss_delta": -0.012068523094058037}, {"index": 283, "tokens": 131, "normal_loss": 2.330697774887085, "no_memory_loss_delta": 0.15024244785308838, "scrambled_loss_delta": 0.30028775334358215, "memory_only_loss_delta": 1.9151502847671509, "no_evidence_loss_delta": 0.027193989604711533, "no_agreement_loss_delta": 0.019800160080194473, "no_orthogonal_loss_delta": -0.01742645725607872}], "agreement_weight_abs_mean": 0.05450502410531044, "agreement_weight_abs_max": 0.29515179991722107, "agreement_weight_mean": 0.025380654260516167, "orthogonal_weight_abs_mean": 0.18710221350193024, "orthogonal_weight_abs_max": 0.7598381042480469, "orthogonal_weight_mean": -0.04468332603573799, "no_memory_loss": 3.5270581245422363, "no_memory_loss_delta": 0.1242830902338028, "no_memory_token_accuracy_gain": 0.017797552049160004, "no_memory_token_win_fraction": 0.5750834345817566, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 11.687019992921547, "scrambled_loss": 3.7010915279388428, "scrambled_loss_delta": 0.29831641912460327, "scrambled_token_accuracy_gain": 0.03781979903578758, "scrambled_token_win_fraction": 0.6273637413978577, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 25.793350143626014, "memory_only_loss": 4.955200672149658, "memory_only_loss_delta": 1.5524263381958008, "memory_only_token_accuracy_gain": 0.17241379618644714, "memory_only_token_win_fraction": 0.8286985754966736, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 78.82663872797924, "no_evidence_loss": 3.437950372695923, "no_evidence_loss_delta": 0.03517552465200424, "no_evidence_token_accuracy_gain": 0.0055617354810237885, "no_evidence_token_win_fraction": 0.5317018628120422, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 3.4564056423343765, "no_agreement_loss": 3.418548107147217, "no_agreement_loss_delta": 0.015772948041558266, "no_agreement_token_accuracy_gain": 0.0033370412420481443, "no_agreement_token_win_fraction": 0.5161290168762207, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 1.5649206540263298, "no_orthogonal_loss": 3.3959391117095947, "no_orthogonal_loss_delta": -0.006835713516920805, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.46496108174324036, "no_orthogonal_probe_win_fraction": 0.25, "no_orthogonal_ppl_reduction_pct": -0.6859130333037455, "short_no_memory_loss_delta": 0.08731891214847565, "long_context_memory_gain": 0.03696417808532715, "normal_context_loss_gain": 0.11588642001152039, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 1.9413845539093018, "contribution_to_active": 0.5781790018081665, "mix_mean": 0.10150981694459915, "mix_low": 0.07628043740987778, "mix_high": 0.0004143714904785156, "read_cosine": 0.589614748954773, "beta_mean": 0.26351815462112427, "nonfinite": 0.0, "weighted_memory_to_active": 0.5394525527954102, "mix_std": 0.08562979847192764, "beta_std": 0.12385313212871552, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06942485272884369, "output_influence": 0.1982605755329132, "residual_influence": 0.21672135591506958, "state_active": 0.11199027299880981, "state_memory": 0.06646478921175003, "state_peak": 28.99557113647461, "growth_active": 1.25971257686615, "growth_memory": 1.4582469463348389, "heads": [{"head": 0, "memory_to_active": 2.452192544937134, "contribution_to_active": 0.6182265281677246, "mix_mean": 0.04176850616931915, "mix_low": 0.03372478485107422, "mix_high": 0.0, "read_cosine": 0.4943828582763672, "beta_mean": 0.24120773375034332, "nonfinite": 0.0, "weighted_memory_to_active": 0.6643407940864563, "mix_std": 0.05228104069828987, "beta_std": 0.08868654817342758, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04186641797423363, "evidence_weights": [-0.05888617783784866, -0.06771191209554672, 0.15711872279644012], "agreement_weight": -0.04488310217857361, "orthogonal_weight": 0.08432380855083466}, {"head": 1, "memory_to_active": 2.0965538024902344, "contribution_to_active": 1.1117284297943115, "mix_mean": 0.046771563589572906, "mix_low": 0.05536842346191406, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5835083723068237, "beta_mean": 0.3243124783039093, "nonfinite": 0.0, "weighted_memory_to_active": 1.1504580974578857, "mix_std": 0.05904313176870346, "beta_std": 0.12179459631443024, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16131451725959778, "evidence_weights": [-0.07023243606090546, -0.10379130393266678, 0.13650666177272797], "agreement_weight": -0.01040904875844717, "orthogonal_weight": 0.2603205144405365}, {"head": 2, "memory_to_active": 1.5994220972061157, "contribution_to_active": 0.4059801697731018, "mix_mean": 0.035878539085388184, "mix_low": 0.11129474639892578, "mix_high": 0.0, "read_cosine": 0.5173337459564209, "beta_mean": 0.2699415683746338, "nonfinite": 0.0, "weighted_memory_to_active": 0.39766499400138855, "mix_std": 0.045247387140989304, "beta_std": 0.13270853459835052, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03159896656870842, "evidence_weights": [-0.0548878088593483, -0.061614226549863815, -0.009677527472376823], "agreement_weight": -0.03461660072207451, "orthogonal_weight": 0.11141270399093628}, {"head": 3, "memory_to_active": 2.250004768371582, "contribution_to_active": 0.8607581257820129, "mix_mean": 0.11853775382041931, "mix_low": 0.010545730590820312, "mix_high": 9.5367431640625e-07, "read_cosine": 0.591254472732544, "beta_mean": 0.3281906545162201, "nonfinite": 0.0, "weighted_memory_to_active": 0.813614547252655, "mix_std": 0.09546688199043274, "beta_std": 0.16255083680152893, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14307290315628052, "evidence_weights": [-0.1663421243429184, -0.18480229377746582, 0.26850807666778564], "agreement_weight": -0.04946380481123924, "orthogonal_weight": 0.23095934092998505}, {"head": 4, "memory_to_active": 3.0634870529174805, "contribution_to_active": 0.14193111658096313, "mix_mean": 0.020970288664102554, "mix_low": 0.23952388763427734, "mix_high": 0.0, "read_cosine": 0.42781126499176025, "beta_mean": 0.2917872965335846, "nonfinite": 0.0, "weighted_memory_to_active": 0.12338877469301224, "mix_std": 0.019635677337646484, "beta_std": 0.13371174037456512, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02539292350411415, "evidence_weights": [0.0023610007483512163, -0.01256994903087616, 0.11087774485349655], "agreement_weight": -0.05312580242753029, "orthogonal_weight": 0.23017528653144836}, {"head": 5, "memory_to_active": 0.18664664030075073, "contribution_to_active": 0.33044925332069397, "mix_mean": 0.3451322317123413, "mix_low": 0.00722503662109375, "mix_high": 0.0024843215942382812, "read_cosine": 0.9233977794647217, "beta_mean": 0.12566904723644257, "nonfinite": 0.0, "weighted_memory_to_active": 0.08724776655435562, "mix_std": 0.2421046495437622, "beta_std": 0.1036665141582489, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01330332737416029, "evidence_weights": [-0.36482924222946167, -0.36325472593307495, 0.46244919300079346], "agreement_weight": 0.12013746798038483, "orthogonal_weight": 0.3566283881664276}]}, {"layer": 1, "memory_to_active": 2.940887928009033, "contribution_to_active": 0.23441475629806519, "mix_mean": 0.04817354306578636, "mix_low": 0.38766244053840637, "mix_high": 9.5367431640625e-07, "read_cosine": 0.38185861706733704, "beta_mean": 0.3053548038005829, "nonfinite": 0.0, "weighted_memory_to_active": 0.23908957839012146, "mix_std": 0.05336377024650574, "beta_std": 0.1593550443649292, "beta_low": 0.0004374186391942203, "beta_high": 0.0007731119985692203, "orthogonal_to_active": 0.025330934673547745, "output_influence": 0.1269703507423401, "residual_influence": 0.025655582547187805, "state_active": 0.05508206784725189, "state_memory": 0.07938799262046814, "state_peak": 18.182924270629883, "growth_active": 1.0496551990509033, "growth_memory": 1.652559518814087, "heads": [{"head": 0, "memory_to_active": 3.87715220451355, "contribution_to_active": 0.08380117267370224, "mix_mean": 0.009479244239628315, "mix_low": 0.7046356201171875, "mix_high": 0.0, "read_cosine": 0.089666947722435, "beta_mean": 0.1821569949388504, "nonfinite": 0.0, "weighted_memory_to_active": 0.0791289284825325, "mix_std": 0.015944452956318855, "beta_std": 0.12662273645401, "beta_low": 0.0010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.010205667465925217, "evidence_weights": [0.05501176044344902, 0.045766960829496384, -0.1617192029953003], "agreement_weight": -0.11397329717874527, "orthogonal_weight": -0.13707885146141052}, {"head": 1, "memory_to_active": 2.555098056793213, "contribution_to_active": 0.15603944659233093, "mix_mean": 0.03609135374426842, "mix_low": 0.16917991638183594, "mix_high": 0.0, "read_cosine": 0.5232388377189636, "beta_mean": 0.3180117607116699, "nonfinite": 0.0, "weighted_memory_to_active": 0.15733638405799866, "mix_std": 0.04099176451563835, "beta_std": 0.15067842602729797, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012065241113305092, "evidence_weights": [-0.06481952965259552, -0.06383084505796432, 0.11186541616916656], "agreement_weight": -0.0519711971282959, "orthogonal_weight": 0.0934198722243309}, {"head": 2, "memory_to_active": 0.9984238147735596, "contribution_to_active": 0.38406556844711304, "mix_mean": 0.1703098714351654, "mix_low": 0.008028984069824219, "mix_high": 2.86102294921875e-06, "read_cosine": 0.7117148637771606, "beta_mean": 0.2131475806236267, "nonfinite": 0.0, "weighted_memory_to_active": 0.4740069508552551, "mix_std": 0.15568923950195312, "beta_std": 0.09845700114965439, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06031543016433716, "evidence_weights": [-0.20952948927879333, -0.2383783608675003, 0.2253907471895218], "agreement_weight": -0.02426294982433319, "orthogonal_weight": 0.19503386318683624}, {"head": 3, "memory_to_active": 2.945899724960327, "contribution_to_active": 0.053143665194511414, "mix_mean": 0.010510324500501156, "mix_low": 0.6791133880615234, "mix_high": 0.0, "read_cosine": 0.24287457764148712, "beta_mean": 0.46801602840423584, "nonfinite": 0.0, "weighted_memory_to_active": 0.05608448386192322, "mix_std": 0.0151801947504282, "beta_std": 0.22468332946300507, "beta_low": 0.0, "beta_high": 0.00457763671875, "orthogonal_to_active": 0.0003845996398013085, "evidence_weights": [-0.003815100295469165, -0.00019322731532156467, -0.0258542038500309], "agreement_weight": -0.009092146530747414, "orthogonal_weight": 0.007582957856357098}, {"head": 4, "memory_to_active": 2.4341368675231934, "contribution_to_active": 0.6519349217414856, "mix_mean": 0.052213795483112335, "mix_low": 0.10425949096679688, "mix_high": 0.0, "read_cosine": 0.45178982615470886, "beta_mean": 0.20416982471942902, "nonfinite": 0.0, "weighted_memory_to_active": 0.589111328125, "mix_std": 0.0680464655160904, "beta_std": 0.08969767391681671, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06844417005777359, "evidence_weights": [-0.08065659552812576, -0.0826946347951889, 0.2082216739654541], "agreement_weight": -0.048321980983018875, "orthogonal_weight": 0.11969385296106339}, {"head": 5, "memory_to_active": 4.834615230560303, "contribution_to_active": 0.07750366628170013, "mix_mean": 0.010436653159558773, "mix_low": 0.6607570648193359, "mix_high": 2.86102294921875e-06, "read_cosine": 0.2718665301799774, "beta_mean": 0.4466266930103302, "nonfinite": 0.0, "weighted_memory_to_active": 0.07886933535337448, "mix_std": 0.024330493062734604, "beta_std": 0.26599112153053284, "beta_low": 0.00152587890625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0005704936338588595, "evidence_weights": [0.029211025685071945, 0.021580597385764122, 0.10868705809116364], "agreement_weight": -0.06616924703121185, "orthogonal_weight": 0.00783277302980423}]}, {"layer": 2, "memory_to_active": 1.7403926849365234, "contribution_to_active": 0.16834209859371185, "mix_mean": 0.07756033539772034, "mix_low": 0.31299370527267456, "mix_high": 0.00041913989116437733, "read_cosine": 0.3721955418586731, "beta_mean": 0.15906542539596558, "nonfinite": 0.0, "weighted_memory_to_active": 0.105341836810112, "mix_std": 0.0977231040596962, "beta_std": 0.10777515172958374, "beta_low": 0.3370259702205658, "beta_high": 0.0, "orthogonal_to_active": 0.00304587883874774, "output_influence": 0.16920366883277893, "residual_influence": 0.02835700660943985, "state_active": 0.03848961740732193, "state_memory": 0.043500397354364395, "state_peak": 27.8492374420166, "growth_active": 0.974774956703186, "growth_memory": 1.6999093294143677, "heads": [{"head": 0, "memory_to_active": 2.3665342330932617, "contribution_to_active": 0.07128827273845673, "mix_mean": 0.015120584517717361, "mix_low": 0.6455554962158203, "mix_high": 0.0, "read_cosine": 0.2086731195449829, "beta_mean": 0.1778748780488968, "nonfinite": 0.0, "weighted_memory_to_active": 0.07037073373794556, "mix_std": 0.03297729790210724, "beta_std": 0.12078645080327988, "beta_low": 0.04156494140625, "beta_high": 0.0, "orthogonal_to_active": 0.00012151362170698121, "evidence_weights": [0.03499164059758186, 0.030683083459734917, 0.025901474058628082], "agreement_weight": -0.06607718765735626, "orthogonal_weight": 0.0017579898703843355}, {"head": 1, "memory_to_active": 2.7072134017944336, "contribution_to_active": 0.08340241014957428, "mix_mean": 0.013655939139425755, "mix_low": 0.53350830078125, "mix_high": 0.0, "read_cosine": 0.23526543378829956, "beta_mean": 0.26666998863220215, "nonfinite": 0.0, "weighted_memory_to_active": 0.08256740868091583, "mix_std": 0.016421323642134666, "beta_std": 0.1235022321343422, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023682615719735622, "evidence_weights": [0.012921864166855812, 0.019500307738780975, -0.017833812162280083], "agreement_weight": 0.00701784947887063, "orthogonal_weight": 0.03005724586546421}, {"head": 2, "memory_to_active": 0.45108890533447266, "contribution_to_active": 0.18749688565731049, "mix_mean": 0.12402431666851044, "mix_low": 0.04049873352050781, "mix_high": 2.288818359375e-05, "read_cosine": 0.5303446054458618, "beta_mean": 0.0030571166425943375, "nonfinite": 0.0, "weighted_memory_to_active": 0.09076894819736481, "mix_std": 0.15498197078704834, "beta_std": 0.003082114737480879, "beta_low": 0.959228515625, "beta_high": 0.0, "orthogonal_to_active": 0.0013377114664763212, "evidence_weights": [-0.09183593839406967, -0.08999352902173996, 0.17260725796222687], "agreement_weight": 0.07149233669042587, "orthogonal_weight": 0.022397637367248535}, {"head": 3, "memory_to_active": 1.0080252885818481, "contribution_to_active": 0.18369948863983154, "mix_mean": 0.05225936695933342, "mix_low": 0.13966941833496094, "mix_high": 0.0, "read_cosine": 0.518949568271637, "beta_mean": 0.20880964398384094, "nonfinite": 0.0, "weighted_memory_to_active": 0.13993138074874878, "mix_std": 0.08481068909168243, "beta_std": 0.19343823194503784, "beta_low": 0.0035400390625, "beta_high": 0.0, "orthogonal_to_active": 0.003573841881006956, "evidence_weights": [-0.017122283577919006, -0.06311722099781036, 0.2647988498210907], "agreement_weight": 0.0815587043762207, "orthogonal_weight": -0.0463077574968338}, {"head": 4, "memory_to_active": 3.8685765266418457, "contribution_to_active": 0.23572397232055664, "mix_mean": 0.01742643117904663, "mix_low": 0.5055532455444336, "mix_high": 2.288818359375e-05, "read_cosine": 0.21168267726898193, "beta_mean": 0.29790306091308594, "nonfinite": 0.0, "weighted_memory_to_active": 0.23875856399536133, "mix_std": 0.05219964683055878, "beta_std": 0.2057279646396637, "beta_low": 0.017822265625, "beta_high": 0.0, "orthogonal_to_active": 0.010567968711256981, "evidence_weights": [0.013522360473871231, 0.01104530319571495, 0.10869044810533524], "agreement_weight": 0.013993806205689907, "orthogonal_weight": 0.04820520803332329}, {"head": 5, "memory_to_active": 0.04091717302799225, "contribution_to_active": 0.24844160676002502, "mix_mean": 0.242875337600708, "mix_low": 0.013176918029785156, "mix_high": 0.0024690628051757812, "read_cosine": 0.5282575488090515, "beta_mean": 7.780597661621869e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009653947316110134, "mix_std": 0.24494770169258118, "beta_std": 0.00011393322347430512, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003059757873415947, "evidence_weights": [-0.1586836576461792, -0.13863560557365417, 0.20551007986068726], "agreement_weight": 0.00874906312674284, "orthogonal_weight": 0.050343260169029236}]}, {"layer": 3, "memory_to_active": 1.7461163997650146, "contribution_to_active": 0.15966801345348358, "mix_mean": 0.036784376949071884, "mix_low": 0.4563163220882416, "mix_high": 0.0003325144643895328, "read_cosine": 0.3127952814102173, "beta_mean": 0.15501634776592255, "nonfinite": 0.0, "weighted_memory_to_active": 0.0910266637802124, "mix_std": 0.06944670528173447, "beta_std": 0.1016092598438263, "beta_low": 0.2365926206111908, "beta_high": 0.0, "orthogonal_to_active": 0.005446024239063263, "output_influence": 0.1286102831363678, "residual_influence": 0.017622582614421844, "state_active": 0.04198313504457474, "state_memory": 0.048730261623859406, "state_peak": 29.440452575683594, "growth_active": 0.9852408170700073, "growth_memory": 1.8044146299362183, "heads": [{"head": 0, "memory_to_active": 1.9501492977142334, "contribution_to_active": 0.08237110823392868, "mix_mean": 0.0239542406052351, "mix_low": 0.5529394149780273, "mix_high": 0.0, "read_cosine": 0.21608129143714905, "beta_mean": 0.04560859128832817, "nonfinite": 0.0, "weighted_memory_to_active": 0.0777786523103714, "mix_std": 0.04999968409538269, "beta_std": 0.03520844131708145, "beta_low": 0.01727294921875, "beta_high": 0.0, "orthogonal_to_active": 0.0032631841022521257, "evidence_weights": [0.037827134132385254, 0.015493783168494701, -0.08025939017534256], "agreement_weight": -0.09744558483362198, "orthogonal_weight": -0.0402107760310173}, {"head": 1, "memory_to_active": 0.9440092444419861, "contribution_to_active": 0.056719712913036346, "mix_mean": 0.03264912590384483, "mix_low": 0.40665626525878906, "mix_high": 0.0, "read_cosine": 0.3256092965602875, "beta_mean": 0.016278952360153198, "nonfinite": 0.0, "weighted_memory_to_active": 0.04232778772711754, "mix_std": 0.06541620194911957, "beta_std": 0.015592644922435284, "beta_low": 0.4012451171875, "beta_high": 0.0, "orthogonal_to_active": 0.001936077605932951, "evidence_weights": [-0.007762578781694174, -0.0076109240762889385, 0.01358699705451727], "agreement_weight": -0.05709083378314972, "orthogonal_weight": -0.05198116600513458}, {"head": 2, "memory_to_active": 2.725773334503174, "contribution_to_active": 0.09355097264051437, "mix_mean": 0.016490712761878967, "mix_low": 0.5340824127197266, "mix_high": 0.0, "read_cosine": 0.286480575799942, "beta_mean": 0.2288101315498352, "nonfinite": 0.0, "weighted_memory_to_active": 0.1125919371843338, "mix_std": 0.02507268637418747, "beta_std": 0.16057364642620087, "beta_low": 0.00140380859375, "beta_high": 0.0, "orthogonal_to_active": 0.008680010214447975, "evidence_weights": [-0.03474409878253937, -0.02774694561958313, 0.12189367413520813], "agreement_weight": -0.027349160984158516, "orthogonal_weight": -0.09337157756090164}, {"head": 3, "memory_to_active": 0.23178862035274506, "contribution_to_active": 0.2113848328590393, "mix_mean": 0.09257706999778748, "mix_low": 0.1770792007446289, "mix_high": 0.0007266998291015625, "read_cosine": 0.36725375056266785, "beta_mean": 0.0007484813686460257, "nonfinite": 0.0, "weighted_memory_to_active": 0.0289907306432724, "mix_std": 0.15218263864517212, "beta_std": 0.0011941391276195645, "beta_low": 0.99725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.002499503782019019, "evidence_weights": [-0.06880226731300354, -0.07335318624973297, 0.14651326835155487], "agreement_weight": 0.08356872946023941, "orthogonal_weight": -0.09744330495595932}, {"head": 4, "memory_to_active": 3.8529229164123535, "contribution_to_active": 0.03635361045598984, "mix_mean": 0.0074720224365592, "mix_low": 0.7933578491210938, "mix_high": 0.0, "read_cosine": 0.20312964916229248, "beta_mean": 0.42529892921447754, "nonfinite": 0.0, "weighted_memory_to_active": 0.03892694413661957, "mix_std": 0.009578165598213673, "beta_std": 0.21340139210224152, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.004174836445599794, "evidence_weights": [0.08952894061803818, 0.08294481784105301, -0.1414470672607422], "agreement_weight": -0.1275843381881714, "orthogonal_weight": -0.10807719826698303}, {"head": 5, "memory_to_active": 0.7720547914505005, "contribution_to_active": 0.47762778401374817, "mix_mean": 0.0475630946457386, "mix_low": 0.27378273010253906, "mix_high": 0.0012683868408203125, "read_cosine": 0.4782170057296753, "beta_mean": 0.21335303783416748, "nonfinite": 0.0, "weighted_memory_to_active": 0.24554388225078583, "mix_std": 0.11443084478378296, "beta_std": 0.183685302734375, "beta_low": 0.00238037109375, "beta_high": 0.0, "orthogonal_to_active": 0.012122534215450287, "evidence_weights": [-0.015043538995087147, -0.09066648036241531, 0.06676304340362549], "agreement_weight": 0.11829645931720734, "orthogonal_weight": -0.07771098613739014}]}, {"layer": 4, "memory_to_active": 1.3109798431396484, "contribution_to_active": 0.16971571743488312, "mix_mean": 0.099411740899086, "mix_low": 0.2255050390958786, "mix_high": 7.05718994140625e-05, "read_cosine": 0.4228505492210388, "beta_mean": 0.1433199644088745, "nonfinite": 0.0, "weighted_memory_to_active": 0.1389906406402588, "mix_std": 0.10362759977579117, "beta_std": 0.10405635833740234, "beta_low": 0.17864990234375, "beta_high": 0.0, "orthogonal_to_active": 0.013048848137259483, "output_influence": 0.13068121671676636, "residual_influence": 0.02353520691394806, "state_active": 0.04334719106554985, "state_memory": 0.04741319268941879, "state_peak": 32.572715759277344, "growth_active": 0.9922853708267212, "growth_memory": 1.7920658588409424, "heads": [{"head": 0, "memory_to_active": 0.6143872737884521, "contribution_to_active": 0.22834177315235138, "mix_mean": 0.13562382757663727, "mix_low": 0.044681549072265625, "mix_high": 6.198883056640625e-05, "read_cosine": 0.5806733965873718, "beta_mean": 0.07120409607887268, "nonfinite": 0.0, "weighted_memory_to_active": 0.1959039270877838, "mix_std": 0.16637292504310608, "beta_std": 0.08591508865356445, "beta_low": 0.06597900390625, "beta_high": 0.0, "orthogonal_to_active": 0.0014590476639568806, "evidence_weights": [-0.09200900793075562, -0.13040520250797272, 0.18736322224140167], "agreement_weight": 0.07530650496482849, "orthogonal_weight": 0.011211821809411049}, {"head": 1, "memory_to_active": 0.7878656387329102, "contribution_to_active": 0.13373738527297974, "mix_mean": 0.10446741431951523, "mix_low": 0.09624862670898438, "mix_high": 0.0, "read_cosine": 0.4598124027252197, "beta_mean": 0.09232331812381744, "nonfinite": 0.0, "weighted_memory_to_active": 0.11369967460632324, "mix_std": 0.12123384326696396, "beta_std": 0.0765683725476265, "beta_low": 0.0072021484375, "beta_high": 0.0, "orthogonal_to_active": 0.010624952614307404, "evidence_weights": [-0.0892002284526825, -0.09879060834646225, 0.26302284002304077], "agreement_weight": 0.059447281062603, "orthogonal_weight": -0.11681067943572998}, {"head": 2, "memory_to_active": 2.3346667289733887, "contribution_to_active": 0.08443146198987961, "mix_mean": 0.013234371319413185, "mix_low": 0.6471776962280273, "mix_high": 0.0, "read_cosine": 0.2881587743759155, "beta_mean": 0.18193566799163818, "nonfinite": 0.0, "weighted_memory_to_active": 0.09414796531200409, "mix_std": 0.019963400438427925, "beta_std": 0.12017977237701416, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0056448765099048615, "evidence_weights": [0.011794503778219223, 0.002467014128342271, -0.033272553235292435], "agreement_weight": 0.021832510828971863, "orthogonal_weight": 0.07439534366130829}, {"head": 3, "memory_to_active": 2.0744142532348633, "contribution_to_active": 0.11161042749881744, "mix_mean": 0.025704748928546906, "mix_low": 0.34616851806640625, "mix_high": 0.0, "read_cosine": 0.26960983872413635, "beta_mean": 0.2561481297016144, "nonfinite": 0.0, "weighted_memory_to_active": 0.15985214710235596, "mix_std": 0.03897181898355484, "beta_std": 0.19703656435012817, "beta_low": 0.02398681640625, "beta_high": 0.0, "orthogonal_to_active": 0.012152569368481636, "evidence_weights": [-0.030975142493844032, -0.014299381524324417, -0.11878526210784912], "agreement_weight": -0.005626213736832142, "orthogonal_weight": -0.11888900399208069}, {"head": 4, "memory_to_active": 0.3451816439628601, "contribution_to_active": 0.2723886966705322, "mix_mean": 0.2774788737297058, "mix_low": 0.006622314453125, "mix_high": 0.0003299713134765625, "read_cosine": 0.4283522367477417, "beta_mean": 0.00260319747030735, "nonfinite": 0.0, "weighted_memory_to_active": 0.11522040516138077, "mix_std": 0.227471262216568, "beta_std": 0.002854876918718219, "beta_low": 0.9747314453125, "beta_high": 0.0, "orthogonal_to_active": 0.007258748635649681, "evidence_weights": [-0.16912217438220978, -0.17281986773014069, 0.26969024538993835], "agreement_weight": -0.016995364800095558, "orthogonal_weight": -0.07619725912809372}, {"head": 5, "memory_to_active": 1.7093629837036133, "contribution_to_active": 0.18778449296951294, "mix_mean": 0.03996120020747185, "mix_low": 0.21213150024414062, "mix_high": 3.147125244140625e-05, "read_cosine": 0.5104966163635254, "beta_mean": 0.25570541620254517, "nonfinite": 0.0, "weighted_memory_to_active": 0.15511974692344666, "mix_std": 0.04775229096412659, "beta_std": 0.1417834758758545, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.041152894496917725, "evidence_weights": [-0.056979984045028687, -0.0717027485370636, 0.1484929472208023], "agreement_weight": -0.05323486402630806, "orthogonal_weight": 0.33456486463546753}]}, {"layer": 5, "memory_to_active": 0.9340713024139404, "contribution_to_active": 0.3080580532550812, "mix_mean": 0.22067973017692566, "mix_low": 0.13998238742351532, "mix_high": 0.0015560786705464125, "read_cosine": 0.3408423066139221, "beta_mean": 0.09963396936655045, "nonfinite": 0.0, "weighted_memory_to_active": 0.16744178533554077, "mix_std": 0.195811927318573, "beta_std": 0.08483786135911942, "beta_low": 0.3514608144760132, "beta_high": 0.0, "orthogonal_to_active": 0.01813761703670025, "output_influence": 0.303150475025177, "residual_influence": 0.04746003448963165, "state_active": 0.07432487607002258, "state_memory": 0.05168696865439415, "state_peak": 37.54184341430664, "growth_active": 1.0749982595443726, "growth_memory": 1.9735960960388184, "heads": [{"head": 0, "memory_to_active": 2.161933183670044, "contribution_to_active": 0.19215500354766846, "mix_mean": 0.04690070450305939, "mix_low": 0.21464061737060547, "mix_high": 0.0, "read_cosine": 0.18310195207595825, "beta_mean": 0.1904502809047699, "nonfinite": 0.0, "weighted_memory_to_active": 0.20152103900909424, "mix_std": 0.07833146303892136, "beta_std": 0.13612444698810577, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007076967507600784, "evidence_weights": [-0.03919996693730354, -0.044573429971933365, -0.07908447086811066], "agreement_weight": -0.019292928278446198, "orthogonal_weight": -0.038008954375982285}, {"head": 1, "memory_to_active": 0.05169954523444176, "contribution_to_active": 0.24418577551841736, "mix_mean": 0.14428746700286865, "mix_low": 0.18160533905029297, "mix_high": 8.0108642578125e-05, "read_cosine": 0.18688705563545227, "beta_mean": 0.0001488716807216406, "nonfinite": 0.0, "weighted_memory_to_active": 0.007852268405258656, "mix_std": 0.216705784201622, "beta_std": 0.00041027378756552935, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.002480784198269248, "evidence_weights": [-0.07367216050624847, -0.0653860867023468, 0.02724180370569229], "agreement_weight": 0.0893590971827507, "orthogonal_weight": -0.32236412167549133}, {"head": 2, "memory_to_active": 0.5266172289848328, "contribution_to_active": 0.31071555614471436, "mix_mean": 0.32258257269859314, "mix_low": 0.01163482666015625, "mix_high": 0.00015544891357421875, "read_cosine": 0.6031238436698914, "beta_mean": 0.06484178453683853, "nonfinite": 0.0, "weighted_memory_to_active": 0.23245017230510712, "mix_std": 0.24072569608688354, "beta_std": 0.03952110558748245, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.011874984949827194, "evidence_weights": [-0.23877589404582977, -0.2385266125202179, 0.38429462909698486], "agreement_weight": 0.07934367656707764, "orthogonal_weight": 0.07163726538419724}, {"head": 3, "memory_to_active": 2.1134607791900635, "contribution_to_active": 0.11611790955066681, "mix_mean": 0.0282658152282238, "mix_low": 0.4011964797973633, "mix_high": 0.0, "read_cosine": 0.25079959630966187, "beta_mean": 0.14796453714370728, "nonfinite": 0.0, "weighted_memory_to_active": 0.13466446101665497, "mix_std": 0.053201351314783096, "beta_std": 0.1296883374452591, "beta_low": 0.0069580078125, "beta_high": 0.0, "orthogonal_to_active": 0.01749749854207039, "evidence_weights": [0.016889236867427826, 0.007314037997275591, 0.05201982706785202], "agreement_weight": -0.004550848621875048, "orthogonal_weight": -0.13793911039829254}, {"head": 4, "memory_to_active": 0.58307284116745, "contribution_to_active": 0.5165261626243591, "mix_mean": 0.3313795328140259, "mix_low": 0.01910400390625, "mix_high": 0.006763458251953125, "read_cosine": 0.5851418972015381, "beta_mean": 0.19367057085037231, "nonfinite": 0.0, "weighted_memory_to_active": 0.3512166738510132, "mix_std": 0.2953178584575653, "beta_std": 0.20249320566654205, "beta_low": 0.10247802734375, "beta_high": 0.0, "orthogonal_to_active": 0.05265949293971062, "evidence_weights": [-0.15288762748241425, -0.20146985352039337, 0.28951090574264526], "agreement_weight": 0.06773136556148529, "orthogonal_weight": 0.20410852134227753}, {"head": 5, "memory_to_active": 0.16764403879642487, "contribution_to_active": 0.4686478078365326, "mix_mean": 0.4506622552871704, "mix_low": 0.011713027954101562, "mix_high": 0.0023374557495117188, "read_cosine": 0.23599961400032043, "beta_mean": 0.0007277569966390729, "nonfinite": 0.0, "weighted_memory_to_active": 0.07694604992866516, "mix_std": 0.2905893325805664, "beta_std": 0.0007897946052253246, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.017235971987247467, "evidence_weights": [-0.2095784693956375, -0.2123185098171234, 0.2277689278125763], "agreement_weight": 0.09257269650697708, "orthogonal_weight": -0.2611329257488251}]}, {"layer": 6, "memory_to_active": 0.7390307784080505, "contribution_to_active": 0.3571852445602417, "mix_mean": 0.20635610818862915, "mix_low": 0.17323224246501923, "mix_high": 0.006270408630371094, "read_cosine": 0.3928409814834595, "beta_mean": 0.09471450001001358, "nonfinite": 0.0, "weighted_memory_to_active": 0.21693016588687897, "mix_std": 0.21565015614032745, "beta_std": 0.09188123047351837, "beta_low": 0.35235595703125, "beta_high": 0.0, "orthogonal_to_active": 0.027510203421115875, "output_influence": 0.3295513391494751, "residual_influence": 0.06814215332269669, "state_active": 0.07102020829916, "state_memory": 0.03166022151708603, "state_peak": 27.900712966918945, "growth_active": 1.2084137201309204, "growth_memory": 1.728702187538147, "heads": [{"head": 0, "memory_to_active": 0.6684587001800537, "contribution_to_active": 0.32120007276535034, "mix_mean": 0.09638892114162445, "mix_low": 0.20977497100830078, "mix_high": 0.0017862319946289062, "read_cosine": 0.33230558037757874, "beta_mean": 0.12228158116340637, "nonfinite": 0.0, "weighted_memory_to_active": 0.2821555435657501, "mix_std": 0.16664350032806396, "beta_std": 0.12499605119228363, "beta_low": 0.00335693359375, "beta_high": 0.0, "orthogonal_to_active": 0.06487596035003662, "evidence_weights": [-0.05124152824282646, -0.07654569298028946, 0.14429892599582672], "agreement_weight": 0.11518024653196335, "orthogonal_weight": -0.36483749747276306}, {"head": 1, "memory_to_active": 0.3793417811393738, "contribution_to_active": 0.21999232470989227, "mix_mean": 0.16354544460773468, "mix_low": 0.08448028564453125, "mix_high": 0.0006647109985351562, "read_cosine": 0.5406019687652588, "beta_mean": 0.0050861709751188755, "nonfinite": 0.0, "weighted_memory_to_active": 0.1006159633398056, "mix_std": 0.19417640566825867, "beta_std": 0.005428486969321966, "beta_low": 0.8841552734375, "beta_high": 0.0, "orthogonal_to_active": 0.012980948202311993, "evidence_weights": [-0.13251681625843048, -0.12350445985794067, 0.036380864679813385], "agreement_weight": 0.01515122875571251, "orthogonal_weight": -0.22509513795375824}, {"head": 2, "memory_to_active": 2.308323621749878, "contribution_to_active": 0.11958779394626617, "mix_mean": 0.0239260233938694, "mix_low": 0.530247688293457, "mix_high": 5.435943603515625e-05, "read_cosine": 0.23554879426956177, "beta_mean": 0.23538009822368622, "nonfinite": 0.0, "weighted_memory_to_active": 0.14929276704788208, "mix_std": 0.0620785653591156, "beta_std": 0.1659158170223236, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02888016402721405, "evidence_weights": [0.045693930238485336, 0.04540092498064041, -0.0844731405377388], "agreement_weight": 0.014355208724737167, "orthogonal_weight": -0.20487761497497559}, {"head": 3, "memory_to_active": 0.3629683554172516, "contribution_to_active": 0.5110688209533691, "mix_mean": 0.3240583837032318, "mix_low": 0.026315689086914062, "mix_high": 0.015184402465820312, "read_cosine": 0.5716882348060608, "beta_mean": 0.10988646745681763, "nonfinite": 0.0, "weighted_memory_to_active": 0.21769949793815613, "mix_std": 0.3127596974372864, "beta_std": 0.13174693286418915, "beta_low": 0.12664794921875, "beta_high": 0.0, "orthogonal_to_active": 0.02530587837100029, "evidence_weights": [-0.09556086361408234, -0.19245392084121704, 0.25574374198913574], "agreement_weight": 0.014733710326254368, "orthogonal_weight": 0.16145059466362}, {"head": 4, "memory_to_active": 0.6834166049957275, "contribution_to_active": 0.644797146320343, "mix_mean": 0.44603848457336426, "mix_low": 0.008497238159179688, "mix_high": 0.019931793212890625, "read_cosine": 0.39506328105926514, "beta_mean": 0.09547000378370285, "nonfinite": 0.0, "weighted_memory_to_active": 0.5437941551208496, "mix_std": 0.32058852910995483, "beta_std": 0.12287069112062454, "beta_low": 0.1002197265625, "beta_high": 0.0, "orthogonal_to_active": 0.028408782556653023, "evidence_weights": [-0.19560962915420532, -0.3000909686088562, -0.13546648621559143], "agreement_weight": -0.010940656997263432, "orthogonal_weight": 0.06710176169872284}, {"head": 5, "memory_to_active": 0.03167551010847092, "contribution_to_active": 0.3264652490615845, "mix_mean": 0.18417930603027344, "mix_low": 0.18007755279541016, "mix_high": 9.5367431640625e-07, "read_cosine": 0.281838059425354, "beta_mean": 0.00018267575069330633, "nonfinite": 0.0, "weighted_memory_to_active": 0.008023020811378956, "mix_std": 0.23765422403812408, "beta_std": 0.00032939951051957905, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.004609485622495413, "evidence_weights": [-0.07035993039608002, -0.11078440397977829, -0.20167456567287445], "agreement_weight": 0.1106586679816246, "orthogonal_weight": -0.6026787757873535}]}, {"layer": 7, "memory_to_active": 2.2121686935424805, "contribution_to_active": 0.14778000116348267, "mix_mean": 0.023389048874378204, "mix_low": 0.5358861684799194, "mix_high": 4.76837158203125e-07, "read_cosine": 0.24805967509746552, "beta_mean": 0.23033307492733002, "nonfinite": 0.0, "weighted_memory_to_active": 0.1477828025817871, "mix_std": 0.038302231580019, "beta_std": 0.12730145454406738, "beta_low": 0.0014038087101653218, "beta_high": 0.0, "orthogonal_to_active": 0.013496330007910728, "output_influence": 0.1831616312265396, "residual_influence": 0.03240494430065155, "state_active": 0.01920969970524311, "state_memory": 0.04776803404092789, "state_peak": 15.687496185302734, "growth_active": 0.9073976874351501, "growth_memory": 1.5225802659988403, "heads": [{"head": 0, "memory_to_active": 1.841011881828308, "contribution_to_active": 0.3207666873931885, "mix_mean": 0.03265383467078209, "mix_low": 0.34749317169189453, "mix_high": 2.86102294921875e-06, "read_cosine": 0.37419256567955017, "beta_mean": 0.19951236248016357, "nonfinite": 0.0, "weighted_memory_to_active": 0.27908608317375183, "mix_std": 0.05441076681017876, "beta_std": 0.1219603642821312, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.058881644159555435, "evidence_weights": [-0.040248919278383255, -0.06600183993577957, 0.11627569794654846], "agreement_weight": -0.02887306734919548, "orthogonal_weight": 0.2585177719593048}, {"head": 1, "memory_to_active": 2.6248276233673096, "contribution_to_active": 0.17592613399028778, "mix_mean": 0.036526381969451904, "mix_low": 0.25277137756347656, "mix_high": 0.0, "read_cosine": -0.019658293575048447, "beta_mean": 0.3395581543445587, "nonfinite": 0.0, "weighted_memory_to_active": 0.1882270723581314, "mix_std": 0.04164048284292221, "beta_std": 0.14587050676345825, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01421903632581234, "evidence_weights": [-0.0649992898106575, -0.06339935958385468, 0.12034416198730469], "agreement_weight": -0.008078331127762794, "orthogonal_weight": -0.07735313475131989}, {"head": 2, "memory_to_active": 1.508183479309082, "contribution_to_active": 0.22426670789718628, "mix_mean": 0.040901847183704376, "mix_low": 0.4451589584350586, "mix_high": 0.0, "read_cosine": 0.28734317421913147, "beta_mean": 0.21029454469680786, "nonfinite": 0.0, "weighted_memory_to_active": 0.24125757813453674, "mix_std": 0.08230879157781601, "beta_std": 0.09671767801046371, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0034781249705702066, "evidence_weights": [-0.05769689381122589, -0.08468888700008392, 0.14287307858467102], "agreement_weight": 0.022120049223303795, "orthogonal_weight": 0.017482643947005272}, {"head": 3, "memory_to_active": 2.8354973793029785, "contribution_to_active": 0.03825141116976738, "mix_mean": 0.007854187861084938, "mix_low": 0.7801246643066406, "mix_high": 0.0, "read_cosine": 0.16665412485599518, "beta_mean": 0.23656445741653442, "nonfinite": 0.0, "weighted_memory_to_active": 0.03981615602970123, "mix_std": 0.010665366426110268, "beta_std": 0.13671082258224487, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020487383008003235, "evidence_weights": [0.05454469844698906, 0.06373468786478043, -0.09294667840003967], "agreement_weight": -0.05628858879208565, "orthogonal_weight": -0.052333928644657135}, {"head": 4, "memory_to_active": 1.4830666780471802, "contribution_to_active": 0.042458344250917435, "mix_mean": 0.013210725039243698, "mix_low": 0.6316823959350586, "mix_high": 0.0, "read_cosine": 0.31490558385849, "beta_mean": 0.17801210284233093, "nonfinite": 0.0, "weighted_memory_to_active": 0.043340958654880524, "mix_std": 0.023333998396992683, "beta_std": 0.12872062623500824, "beta_low": 0.0084228515625, "beta_high": 0.0, "orthogonal_to_active": 0.0016175768105313182, "evidence_weights": [0.018456634134054184, 0.019101737067103386, 0.021796975284814835], "agreement_weight": 0.05101536214351654, "orthogonal_weight": 0.04765910282731056}, {"head": 5, "memory_to_active": 2.980424165725708, "contribution_to_active": 0.08501062542200089, "mix_mean": 0.009187312796711922, "mix_low": 0.7580862045288086, "mix_high": 0.0, "read_cosine": 0.36492085456848145, "beta_mean": 0.2180568277835846, "nonfinite": 0.0, "weighted_memory_to_active": 0.09496897459030151, "mix_std": 0.017453977838158607, "beta_std": 0.13382866978645325, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007328501669690013, "evidence_weights": [0.014140169136226177, 0.03270604833960533, -0.11768855899572372], "agreement_weight": -0.020508214831352234, "orthogonal_weight": 0.009512835182249546}]}, {"layer": 8, "memory_to_active": 1.0064849853515625, "contribution_to_active": 0.2814357280731201, "mix_mean": 0.19447991251945496, "mix_low": 0.10125128924846649, "mix_high": 0.0032633147202432156, "read_cosine": 0.5486023426055908, "beta_mean": 0.09947581589221954, "nonfinite": 0.0, "weighted_memory_to_active": 0.19486606121063232, "mix_std": 0.16397297382354736, "beta_std": 0.07952255010604858, "beta_low": 0.1737467646598816, "beta_high": 0.0, "orthogonal_to_active": 0.024463362991809845, "output_influence": 0.2583450675010681, "residual_influence": 0.04931408166885376, "state_active": 0.062361858785152435, "state_memory": 0.0363161638379097, "state_peak": 21.865379333496094, "growth_active": 1.0212092399597168, "growth_memory": 1.6143090724945068, "heads": [{"head": 0, "memory_to_active": 0.1757810413837433, "contribution_to_active": 0.35924407839775085, "mix_mean": 0.28653764724731445, "mix_low": 0.004637718200683594, "mix_high": 2.86102294921875e-06, "read_cosine": 0.7290751934051514, "beta_mean": 0.006134625989943743, "nonfinite": 0.0, "weighted_memory_to_active": 0.07521407306194305, "mix_std": 0.20363479852676392, "beta_std": 0.006546109914779663, "beta_low": 0.8341064453125, "beta_high": 0.0, "orthogonal_to_active": 0.006809933111071587, "evidence_weights": [-0.21207742393016815, -0.19527806341648102, 0.22954419255256653], "agreement_weight": 0.07373929023742676, "orthogonal_weight": -0.16035175323486328}, {"head": 1, "memory_to_active": 0.778042197227478, "contribution_to_active": 0.27537527680397034, "mix_mean": 0.28234270215034485, "mix_low": 0.0022668838500976562, "mix_high": 3.814697265625e-05, "read_cosine": 0.7270250916481018, "beta_mean": 0.08746989071369171, "nonfinite": 0.0, "weighted_memory_to_active": 0.29071900248527527, "mix_std": 0.17424564063549042, "beta_std": 0.053093571215867996, "beta_low": 0.00250244140625, "beta_high": 0.0, "orthogonal_to_active": 0.028205081820487976, "evidence_weights": [-0.18579235672950745, -0.1671929955482483, 0.2884783446788788], "agreement_weight": 0.006488034036010504, "orthogonal_weight": 0.15561175346374512}, {"head": 2, "memory_to_active": 1.9363040924072266, "contribution_to_active": 0.1785314977169037, "mix_mean": 0.031004857271909714, "mix_low": 0.33399200439453125, "mix_high": 0.0, "read_cosine": 0.2716894745826721, "beta_mean": 0.18606621026992798, "nonfinite": 0.0, "weighted_memory_to_active": 0.2255762219429016, "mix_std": 0.049478430300951004, "beta_std": 0.12640580534934998, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01843368448317051, "evidence_weights": [-0.009541044943034649, -0.030254919081926346, 0.0024779043160378933], "agreement_weight": 0.025679711252450943, "orthogonal_weight": -0.09428628534078598}, {"head": 3, "memory_to_active": 0.92364501953125, "contribution_to_active": 0.19915953278541565, "mix_mean": 0.14224472641944885, "mix_low": 0.05964946746826172, "mix_high": 0.0, "read_cosine": 0.5938009023666382, "beta_mean": 0.0680089145898819, "nonfinite": 0.0, "weighted_memory_to_active": 0.20311032235622406, "mix_std": 0.12587526440620422, "beta_std": 0.05323512852191925, "beta_low": 0.0244140625, "beta_high": 0.0, "orthogonal_to_active": 0.03281358629465103, "evidence_weights": [-0.10119441896677017, -0.10182147473096848, 0.34033679962158203], "agreement_weight": 0.008884324692189693, "orthogonal_weight": 0.22701451182365417}, {"head": 4, "memory_to_active": 1.9716655015945435, "contribution_to_active": 0.20406553149223328, "mix_mean": 0.05809305980801582, "mix_low": 0.13605976104736328, "mix_high": 0.0, "read_cosine": 0.3869614899158478, "beta_mean": 0.13920342922210693, "nonfinite": 0.0, "weighted_memory_to_active": 0.2315412163734436, "mix_std": 0.07009220868349075, "beta_std": 0.08391453325748444, "beta_low": 0.0008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.0031461315229535103, "evidence_weights": [-0.043698206543922424, -0.05191044136881828, 0.04664211347699165], "agreement_weight": -0.017568185925483704, "orthogonal_weight": 0.016353510320186615}, {"head": 5, "memory_to_active": 0.2534710764884949, "contribution_to_active": 0.47223860025405884, "mix_mean": 0.36665642261505127, "mix_low": 0.07090187072753906, "mix_high": 0.01953887939453125, "read_cosine": 0.583061933517456, "beta_mean": 0.1099717915058136, "nonfinite": 0.0, "weighted_memory_to_active": 0.14303547143936157, "mix_std": 0.36051154136657715, "beta_std": 0.1539401412010193, "beta_low": 0.18060302734375, "beta_high": 0.0, "orthogonal_to_active": 0.05737174674868584, "evidence_weights": [-0.12515152990818024, -0.2670915424823761, -0.07409326732158661], "agreement_weight": 0.02832040563225746, "orthogonal_weight": 0.5488945245742798}]}, {"layer": 9, "memory_to_active": 1.4262001514434814, "contribution_to_active": 0.3544458746910095, "mix_mean": 0.07305796444416046, "mix_low": 0.43686074018478394, "mix_high": 0.0002498626708984375, "read_cosine": 0.35348036885261536, "beta_mean": 0.11423492431640625, "nonfinite": 0.0, "weighted_memory_to_active": 0.3359999656677246, "mix_std": 0.13450300693511963, "beta_std": 0.06123378500342369, "beta_low": 0.2331746518611908, "beta_high": 0.0, "orthogonal_to_active": 0.009003585204482079, "output_influence": 0.23790040612220764, "residual_influence": 0.04454652592539787, "state_active": 0.03284512087702751, "state_memory": 0.04310636222362518, "state_peak": 29.0340576171875, "growth_active": 0.9871801137924194, "growth_memory": 1.7507466077804565, "heads": [{"head": 0, "memory_to_active": 0.8111324310302734, "contribution_to_active": 0.12585709989070892, "mix_mean": 0.06306477636098862, "mix_low": 0.4501018524169922, "mix_high": 0.0, "read_cosine": 0.24791404604911804, "beta_mean": 0.031161269173026085, "nonfinite": 0.0, "weighted_memory_to_active": 0.132391557097435, "mix_std": 0.1301887333393097, "beta_std": 0.04556795209646225, "beta_low": 0.34930419921875, "beta_high": 0.0, "orthogonal_to_active": 0.018411971628665924, "evidence_weights": [0.005352412816137075, -0.032774463295936584, 0.18549863994121552], "agreement_weight": -0.03241374343633652, "orthogonal_weight": -0.15809562802314758}, {"head": 1, "memory_to_active": 0.2976360321044922, "contribution_to_active": 0.1092911958694458, "mix_mean": 0.03718814253807068, "mix_low": 0.7047739028930664, "mix_high": 0.0, "read_cosine": 0.3909824788570404, "beta_mean": 0.0024442996364086866, "nonfinite": 0.0, "weighted_memory_to_active": 0.03224698454141617, "mix_std": 0.10835312306880951, "beta_std": 0.00409244280308485, "beta_low": 0.96441650390625, "beta_high": 0.0, "orthogonal_to_active": 0.0027681193314492702, "evidence_weights": [-0.01199402753263712, -0.0036961452569812536, -0.0030195654835551977], "agreement_weight": 0.07828054577112198, "orthogonal_weight": -0.0916968435049057}, {"head": 2, "memory_to_active": 0.7700763940811157, "contribution_to_active": 0.33569300174713135, "mix_mean": 0.09851789474487305, "mix_low": 0.3558988571166992, "mix_high": 0.0, "read_cosine": 0.21262633800506592, "beta_mean": 0.03799019008874893, "nonfinite": 0.0, "weighted_memory_to_active": 0.1836027204990387, "mix_std": 0.17248621582984924, "beta_std": 0.04562518745660782, "beta_low": 0.0853271484375, "beta_high": 0.0, "orthogonal_to_active": 0.006066184490919113, "evidence_weights": [-0.06044287979602814, -0.049678534269332886, 0.11755586415529251], "agreement_weight": 0.05915425345301628, "orthogonal_weight": 0.0389099083840847}, {"head": 3, "memory_to_active": 2.062222480773926, "contribution_to_active": 0.2201952338218689, "mix_mean": 0.059294141829013824, "mix_low": 0.26531124114990234, "mix_high": 0.0, "read_cosine": 0.4146932363510132, "beta_mean": 0.14610983431339264, "nonfinite": 0.0, "weighted_memory_to_active": 0.216286301612854, "mix_std": 0.08141855150461197, "beta_std": 0.08227008581161499, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007778079016134143, "evidence_weights": [0.007357613183557987, 0.00631731329485774, 0.003118518041446805], "agreement_weight": -0.03000265173614025, "orthogonal_weight": 0.003652502316981554}, {"head": 4, "memory_to_active": 2.699430465698242, "contribution_to_active": 1.027387022972107, "mix_mean": 0.13785412907600403, "mix_low": 0.21040725708007812, "mix_high": 0.001499176025390625, "read_cosine": 0.3897984027862549, "beta_mean": 0.22092115879058838, "nonfinite": 0.0, "weighted_memory_to_active": 1.051632285118103, "mix_std": 0.20862233638763428, "beta_std": 0.0844181701540947, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.009074464440345764, "evidence_weights": [-0.04874758422374725, -0.06181207299232483, 0.11848755180835724], "agreement_weight": -0.06166832521557808, "orthogonal_weight": 0.009301813319325447}, {"head": 5, "memory_to_active": 1.9167025089263916, "contribution_to_active": 0.30825161933898926, "mix_mean": 0.04242873936891556, "mix_low": 0.6346712112426758, "mix_high": 0.0, "read_cosine": 0.46486759185791016, "beta_mean": 0.24678276479244232, "nonfinite": 0.0, "weighted_memory_to_active": 0.39983975887298584, "mix_std": 0.10594908893108368, "beta_std": 0.10542885214090347, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016922958195209503, "evidence_weights": [0.025057215243577957, -0.00616069370880723, 0.008105642162263393], "agreement_weight": -0.021845269948244095, "orthogonal_weight": 0.05991646647453308}]}, {"layer": 10, "memory_to_active": 0.7528454065322876, "contribution_to_active": 0.30357110500335693, "mix_mean": 0.1639867126941681, "mix_low": 0.2555685341358185, "mix_high": 0.004299799911677837, "read_cosine": 0.36966732144355774, "beta_mean": 0.07927589118480682, "nonfinite": 0.0, "weighted_memory_to_active": 0.16561096906661987, "mix_std": 0.1768466830253601, "beta_std": 0.08007705956697464, "beta_low": 0.3033854365348816, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.027152737602591515, "output_influence": 0.3370281755924225, "residual_influence": 0.08211025595664978, "state_active": 0.09658627212047577, "state_memory": 0.027548959478735924, "state_peak": 42.989749908447266, "growth_active": 1.2762917280197144, "growth_memory": 1.504317045211792, "heads": [{"head": 0, "memory_to_active": 0.5399236679077148, "contribution_to_active": 0.6576507687568665, "mix_mean": 0.43778154253959656, "mix_low": 0.02366352081298828, "mix_high": 0.02170848846435547, "read_cosine": 0.1649995744228363, "beta_mean": 0.1701013445854187, "nonfinite": 0.0, "weighted_memory_to_active": 0.32274937629699707, "mix_std": 0.32402503490448, "beta_std": 0.1938093900680542, "beta_low": 0.075927734375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.07180824875831604, "evidence_weights": [-0.11695253103971481, -0.22285354137420654, 0.044242966920137405], "agreement_weight": 0.02180159091949463, "orthogonal_weight": 0.24452361464500427}, {"head": 1, "memory_to_active": 1.2926762104034424, "contribution_to_active": 0.11928406357765198, "mix_mean": 0.03513580933213234, "mix_low": 0.4743843078613281, "mix_high": 9.5367431640625e-07, "read_cosine": 0.2490941286087036, "beta_mean": 0.034525640308856964, "nonfinite": 0.0, "weighted_memory_to_active": 0.08813918381929398, "mix_std": 0.08128427714109421, "beta_std": 0.03198924660682678, "beta_low": 0.18280029296875, "beta_high": 0.0, "orthogonal_to_active": 0.016112864017486572, "evidence_weights": [-0.0001232823997270316, 0.0038828111719340086, -0.12661078572273254], "agreement_weight": -0.036592815071344376, "orthogonal_weight": -0.19069220125675201}, {"head": 2, "memory_to_active": 2.080477714538574, "contribution_to_active": 0.36739736795425415, "mix_mean": 0.022149844095110893, "mix_low": 0.6327972412109375, "mix_high": 0.0, "read_cosine": 0.2078898549079895, "beta_mean": 0.15725284814834595, "nonfinite": 0.0, "weighted_memory_to_active": 0.4129660129547119, "mix_std": 0.05851854011416435, "beta_std": 0.102385014295578, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03574566915631294, "evidence_weights": [0.017795564606785774, 0.007200432941317558, -0.032576784491539], "agreement_weight": -0.027018100023269653, "orthogonal_weight": -0.08980044722557068}, {"head": 3, "memory_to_active": 0.16270503401756287, "contribution_to_active": 0.3337307572364807, "mix_mean": 0.26576346158981323, "mix_low": 0.0956735610961914, "mix_high": 0.004071235656738281, "read_cosine": 0.6933408379554749, "beta_mean": 0.09135106205940247, "nonfinite": 0.0, "weighted_memory_to_active": 0.08139868825674057, "mix_std": 0.3063340187072754, "beta_std": 0.12500102818012238, "beta_low": 0.1807861328125, "beta_high": 0.0, "orthogonal_to_active": 0.026400435715913773, "evidence_weights": [-0.1381485015153885, -0.15473054349422455, 0.3004399240016937], "agreement_weight": 0.0209745392203331, "orthogonal_weight": 0.4495992958545685}, {"head": 4, "memory_to_active": 0.22773608565330505, "contribution_to_active": 0.107807956635952, "mix_mean": 0.06524921953678131, "mix_low": 0.2746391296386719, "mix_high": 9.5367431640625e-07, "read_cosine": 0.24773448705673218, "beta_mean": 0.002094082999974489, "nonfinite": 0.0, "weighted_memory_to_active": 0.030805345624685287, "mix_std": 0.11519511044025421, "beta_std": 0.003940282855182886, "beta_low": 0.970458984375, "beta_high": 0.0, "orthogonal_to_active": 0.005874818656593561, "evidence_weights": [-0.047292035073041916, -0.024004153907299042, -0.10910823196172714], "agreement_weight": 0.03974486142396927, "orthogonal_weight": -0.21942372620105743}, {"head": 5, "memory_to_active": 0.2135532796382904, "contribution_to_active": 0.2355557382106781, "mix_mean": 0.1578403115272522, "mix_low": 0.032253265380859375, "mix_high": 1.71661376953125e-05, "read_cosine": 0.6549451351165771, "beta_mean": 0.020330362021923065, "nonfinite": 0.0, "weighted_memory_to_active": 0.05760721117258072, "mix_std": 0.17572301626205444, "beta_std": 0.02333739399909973, "beta_low": 0.41033935546875, "beta_high": 0.0, "orthogonal_to_active": 0.006974388845264912, "evidence_weights": [-0.12214261293411255, -0.09833325445652008, 0.14923492074012756], "agreement_weight": 0.08625264465808868, "orthogonal_weight": -0.1879560798406601}]}, {"layer": 11, "memory_to_active": 0.4234504699707031, "contribution_to_active": 0.35586729645729065, "mix_mean": 0.2424967885017395, "mix_low": 0.07426993548870087, "mix_high": 0.020183246582746506, "read_cosine": 0.5480028986930847, "beta_mean": 0.05553438514471054, "nonfinite": 0.0, "weighted_memory_to_active": 0.11181668937206268, "mix_std": 0.24723953008651733, "beta_std": 0.072285957634449, "beta_low": 0.329315185546875, "beta_high": 0.0, "orthogonal_to_active": 0.01807360351085663, "output_influence": 0.49776023626327515, "residual_influence": 0.12118980288505554, "state_active": 0.12236867845058441, "state_memory": 0.020822826772928238, "state_peak": 24.989341735839844, "growth_active": 1.186672568321228, "growth_memory": 1.418914794921875, "heads": [{"head": 0, "memory_to_active": 0.131185844540596, "contribution_to_active": 0.356087863445282, "mix_mean": 0.19728383421897888, "mix_low": 0.07148361206054688, "mix_high": 0.00040340423583984375, "read_cosine": 0.4330512285232544, "beta_mean": 0.0009045489132404327, "nonfinite": 0.0, "weighted_memory_to_active": 0.057610973715782166, "mix_std": 0.22909733653068542, "beta_std": 0.0027473438531160355, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.01520724967122078, "evidence_weights": [-0.10055965930223465, -0.11134590953588486, 0.12166722863912582], "agreement_weight": 0.13919685781002045, "orthogonal_weight": -0.3808246850967407}, {"head": 1, "memory_to_active": 0.7129605412483215, "contribution_to_active": 0.16326911747455597, "mix_mean": 0.14102327823638916, "mix_low": 0.051140785217285156, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5654251575469971, "beta_mean": 0.04735260456800461, "nonfinite": 0.0, "weighted_memory_to_active": 0.14913174510002136, "mix_std": 0.16744619607925415, "beta_std": 0.066045381128788, "beta_low": 0.1612548828125, "beta_high": 0.0, "orthogonal_to_active": 0.003880839329212904, "evidence_weights": [-0.0903913751244545, -0.06507056206464767, 0.014565404504537582], "agreement_weight": 0.08803189545869827, "orthogonal_weight": -0.035211168229579926}, {"head": 2, "memory_to_active": 0.10483209043741226, "contribution_to_active": 0.7303372621536255, "mix_mean": 0.4213560223579407, "mix_low": 0.0793142318725586, "mix_high": 0.055970191955566406, "read_cosine": 0.4906887412071228, "beta_mean": 0.07779262959957123, "nonfinite": 0.0, "weighted_memory_to_active": 0.08355991542339325, "mix_std": 0.39294490218162537, "beta_std": 0.11237022280693054, "beta_low": 0.36279296875, "beta_high": 0.0, "orthogonal_to_active": 0.05260339751839638, "evidence_weights": [-0.20020076632499695, -0.24203352630138397, -0.013295340351760387], "agreement_weight": -0.025363050401210785, "orthogonal_weight": 0.7598381042480469}, {"head": 3, "memory_to_active": 0.07007569074630737, "contribution_to_active": 0.40550288558006287, "mix_mean": 0.4438951015472412, "mix_low": 0.08180046081542969, "mix_high": 0.06472492218017578, "read_cosine": 0.6009165644645691, "beta_mean": 0.1047043651342392, "nonfinite": 0.0, "weighted_memory_to_active": 0.038500264286994934, "mix_std": 0.4046000838279724, "beta_std": 0.13971173763275146, "beta_low": 0.25640869140625, "beta_high": 0.0, "orthogonal_to_active": 0.021206675097346306, "evidence_weights": [-0.27769529819488525, -0.27944689989089966, 0.19774149358272552], "agreement_weight": 0.006374194286763668, "orthogonal_weight": 0.6849046945571899}, {"head": 4, "memory_to_active": 1.1479215621948242, "contribution_to_active": 0.17311425507068634, "mix_mean": 0.08283834159374237, "mix_low": 0.11554336547851562, "mix_high": 0.0, "read_cosine": 0.36274242401123047, "beta_mean": 0.06012847274541855, "nonfinite": 0.0, "weighted_memory_to_active": 0.16476498544216156, "mix_std": 0.11334120482206345, "beta_std": 0.059672318398952484, "beta_low": 0.0379638671875, "beta_high": 0.0, "orthogonal_to_active": 0.0014290274120867252, "evidence_weights": [-0.02198702283203602, -0.012043708004057407, 0.22751818597316742], "agreement_weight": -0.012503557838499546, "orthogonal_weight": -0.0113930469378829}, {"head": 5, "memory_to_active": 0.373727023601532, "contribution_to_active": 0.30689236521720886, "mix_mean": 0.16858422756195068, "mix_low": 0.046337127685546875, "mix_high": 0.0, "read_cosine": 0.8351931571960449, "beta_mean": 0.04232369735836983, "nonfinite": 0.0, "weighted_memory_to_active": 0.17733217775821686, "mix_std": 0.17600750923156738, "beta_std": 0.053168706595897675, "beta_low": 0.16448974609375, "beta_high": 0.0, "orthogonal_to_active": 0.014114437624812126, "evidence_weights": [-0.11011180281639099, -0.1411433070898056, 0.19361455738544464], "agreement_weight": 0.0641283318400383, "orthogonal_weight": -0.2457815259695053}]}, {"layer": 12, "memory_to_active": 0.8993366956710815, "contribution_to_active": 0.3565909266471863, "mix_mean": 0.18321886658668518, "mix_low": 0.1234741285443306, "mix_high": 0.0001041094510583207, "read_cosine": 0.3244490623474121, "beta_mean": 0.05777018889784813, "nonfinite": 0.0, "weighted_memory_to_active": 0.2638684809207916, "mix_std": 0.17977391183376312, "beta_std": 0.057566508650779724, "beta_low": 0.4517924189567566, "beta_high": 0.0, "orthogonal_to_active": 0.027529001235961914, "output_influence": 0.17417588829994202, "residual_influence": 0.039974961429834366, "state_active": 0.053242236375808716, "state_memory": 0.029394086450338364, "state_peak": 35.692378997802734, "growth_active": 1.1277282238006592, "growth_memory": 1.5872632265090942, "heads": [{"head": 0, "memory_to_active": 1.3805301189422607, "contribution_to_active": 0.27853918075561523, "mix_mean": 0.1169278621673584, "mix_low": 0.05820941925048828, "mix_high": 0.00012874603271484375, "read_cosine": 0.40242382884025574, "beta_mean": 0.0526651106774807, "nonfinite": 0.0, "weighted_memory_to_active": 0.3673815131187439, "mix_std": 0.13602088391780853, "beta_std": 0.08913467079401016, "beta_low": 0.1231689453125, "beta_high": 0.0, "orthogonal_to_active": 0.03750378638505936, "evidence_weights": [-0.06872862577438354, -0.0946311354637146, 0.1857164204120636], "agreement_weight": 0.07679979503154755, "orthogonal_weight": -0.1409180462360382}, {"head": 1, "memory_to_active": 0.4381885528564453, "contribution_to_active": 0.1896180659532547, "mix_mean": 0.137192040681839, "mix_low": 0.08627700805664062, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5303900837898254, "beta_mean": 0.006719646509736776, "nonfinite": 0.0, "weighted_memory_to_active": 0.10381521284580231, "mix_std": 0.16902902722358704, "beta_std": 0.005868756212294102, "beta_low": 0.7958984375, "beta_high": 0.0, "orthogonal_to_active": 0.035824961960315704, "evidence_weights": [-0.06233490630984306, -0.10606623440980911, 0.15286310017108917], "agreement_weight": 0.11194205284118652, "orthogonal_weight": -0.5052662491798401}, {"head": 2, "memory_to_active": 0.5796827077865601, "contribution_to_active": 0.3313716650009155, "mix_mean": 0.24399778246879578, "mix_low": 0.01703357696533203, "mix_high": 9.441375732421875e-05, "read_cosine": 0.23699703812599182, "beta_mean": 0.06984730809926987, "nonfinite": 0.0, "weighted_memory_to_active": 0.2330050766468048, "mix_std": 0.21248243749141693, "beta_std": 0.07800771296024323, "beta_low": 0.08819580078125, "beta_high": 0.0, "orthogonal_to_active": 0.002604791661724448, "evidence_weights": [-0.10415662825107574, -0.09124896675348282, 0.25333836674690247], "agreement_weight": 0.04768836870789528, "orthogonal_weight": -0.0125894695520401}, {"head": 3, "memory_to_active": 0.20906710624694824, "contribution_to_active": 0.3835640251636505, "mix_mean": 0.32156527042388916, "mix_low": 0.02082347869873047, "mix_high": 8.96453857421875e-05, "read_cosine": 0.09164488315582275, "beta_mean": 0.004004639573395252, "nonfinite": 0.0, "weighted_memory_to_active": 0.08951219171285629, "mix_std": 0.2499287724494934, "beta_std": 0.00565507635474205, "beta_low": 0.92437744140625, "beta_high": 0.0, "orthogonal_to_active": 0.019869288429617882, "evidence_weights": [-0.14528408646583557, -0.1638810932636261, 0.06929375231266022], "agreement_weight": 0.015628349035978317, "orthogonal_weight": -0.23252059519290924}, {"head": 4, "memory_to_active": 2.5344653129577637, "contribution_to_active": 0.670205295085907, "mix_mean": 0.02675972506403923, "mix_low": 0.5302200317382812, "mix_high": 8.0108642578125e-05, "read_cosine": 0.196961909532547, "beta_mean": 0.20566006004810333, "nonfinite": 0.0, "weighted_memory_to_active": 0.7033252120018005, "mix_std": 0.06956358253955841, "beta_std": 0.15709294378757477, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.02542404644191265, "evidence_weights": [0.03533173352479935, 0.013754351995885372, 0.07410936057567596], "agreement_weight": -0.026768742129206657, "orthogonal_weight": -0.03955068066716194}, {"head": 5, "memory_to_active": 0.2540861964225769, "contribution_to_active": 0.2862471044063568, "mix_mean": 0.2528705894947052, "mix_low": 0.028281211853027344, "mix_high": 0.0002307891845703125, "read_cosine": 0.4882764518260956, "beta_mean": 0.007724338676780462, "nonfinite": 0.0, "weighted_memory_to_active": 0.08617165684700012, "mix_std": 0.24161863327026367, "beta_std": 0.00963984988629818, "beta_low": 0.7781982421875, "beta_high": 0.0, "orthogonal_to_active": 0.043947137892246246, "evidence_weights": [-0.14528746902942657, -0.15872348845005035, 0.2349197268486023], "agreement_weight": 0.05331196263432503, "orthogonal_weight": -0.6435508728027344}]}, {"layer": 13, "memory_to_active": 0.41136476397514343, "contribution_to_active": 0.2770349979400635, "mix_mean": 0.23062312602996826, "mix_low": 0.0714055746793747, "mix_high": 0.0013256072998046875, "read_cosine": 0.285380482673645, "beta_mean": 0.03444759547710419, "nonfinite": 0.0, "weighted_memory_to_active": 0.10729283094406128, "mix_std": 0.1881224513053894, "beta_std": 0.047650404274463654, "beta_low": 0.4540202021598816, "beta_high": 0.0, "orthogonal_to_active": 0.030657339841127396, "output_influence": 0.2546749711036682, "residual_influence": 0.07478150725364685, "state_active": 0.09322774410247803, "state_memory": 0.02837301231920719, "state_peak": 28.627357482910156, "growth_active": 1.1165391206741333, "growth_memory": 1.4454776048660278, "heads": [{"head": 0, "memory_to_active": 0.4861961007118225, "contribution_to_active": 0.1824491024017334, "mix_mean": 0.11192305386066437, "mix_low": 0.1127462387084961, "mix_high": 0.0, "read_cosine": 0.22909000515937805, "beta_mean": 0.017419494688510895, "nonfinite": 0.0, "weighted_memory_to_active": 0.09518541395664215, "mix_std": 0.15540136396884918, "beta_std": 0.015793584287166595, "beta_low": 0.3861083984375, "beta_high": 0.0, "orthogonal_to_active": 0.06596435606479645, "evidence_weights": [-0.0688931792974472, -0.09818638861179352, 0.012604374438524246], "agreement_weight": 0.08361001312732697, "orthogonal_weight": -0.7313934564590454}, {"head": 1, "memory_to_active": 0.5912495255470276, "contribution_to_active": 0.16728122532367706, "mix_mean": 0.0940103828907013, "mix_low": 0.14840984344482422, "mix_high": 0.0, "read_cosine": 0.27536535263061523, "beta_mean": 0.011912085115909576, "nonfinite": 0.0, "weighted_memory_to_active": 0.08530404418706894, "mix_std": 0.13579925894737244, "beta_std": 0.011294005438685417, "beta_low": 0.5560302734375, "beta_high": 0.0, "orthogonal_to_active": 0.04666822776198387, "evidence_weights": [-0.03845452517271042, -0.0716259777545929, 0.03262575715780258], "agreement_weight": 0.07729902863502502, "orthogonal_weight": -0.6043767333030701}, {"head": 2, "memory_to_active": 0.3310595452785492, "contribution_to_active": 0.25253963470458984, "mix_mean": 0.1797475516796112, "mix_low": 0.02385711669921875, "mix_high": 9.5367431640625e-06, "read_cosine": 0.5579714179039001, "beta_mean": 0.03740439563989639, "nonfinite": 0.0, "weighted_memory_to_active": 0.1192350834608078, "mix_std": 0.16398313641548157, "beta_std": 0.06324838846921921, "beta_low": 0.38616943359375, "beta_high": 0.0, "orthogonal_to_active": 0.01233787089586258, "evidence_weights": [-0.1299554705619812, -0.12383856624364853, 0.30428481101989746], "agreement_weight": 0.09094387292861938, "orthogonal_weight": -0.13712137937545776}, {"head": 3, "memory_to_active": 0.031160298734903336, "contribution_to_active": 0.4455530643463135, "mix_mean": 0.43940621614456177, "mix_low": 0.0019102096557617188, "mix_high": 9.632110595703125e-05, "read_cosine": -0.30029112100601196, "beta_mean": 0.00024310732260346413, "nonfinite": 0.0, "weighted_memory_to_active": 0.015016615390777588, "mix_std": 0.20523673295974731, "beta_std": 0.0006290446617640555, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.0053847786039114, "evidence_weights": [-0.290043443441391, -0.2484079748392105, 0.032287176698446274], "agreement_weight": 0.29515179991722107, "orthogonal_weight": -0.3692910075187683}, {"head": 4, "memory_to_active": 0.2995474636554718, "contribution_to_active": 0.3566213846206665, "mix_mean": 0.4461157023906708, "mix_low": 0.006178855895996094, "mix_high": 0.007843017578125, "read_cosine": 0.732512354850769, "beta_mean": 0.09472940862178802, "nonfinite": 0.0, "weighted_memory_to_active": 0.14728395640850067, "mix_std": 0.3003292381763458, "beta_std": 0.10374085605144501, "beta_low": 0.0772705078125, "beta_high": 0.0, "orthogonal_to_active": 0.02543548494577408, "evidence_weights": [-0.2369043081998825, -0.2619265615940094, 0.3084222078323364], "agreement_weight": 0.007897618226706982, "orthogonal_weight": 0.24751821160316467}, {"head": 5, "memory_to_active": 0.7289754748344421, "contribution_to_active": 0.25776541233062744, "mix_mean": 0.11253580451011658, "mix_low": 0.1353311538696289, "mix_high": 4.76837158203125e-06, "read_cosine": 0.2176348865032196, "beta_mean": 0.04497705399990082, "nonfinite": 0.0, "weighted_memory_to_active": 0.18173187971115112, "mix_std": 0.16798490285873413, "beta_std": 0.09119652956724167, "beta_low": 0.31939697265625, "beta_high": 0.0, "orthogonal_to_active": 0.0281533170491457, "evidence_weights": [-0.08302488923072815, -0.0917448028922081, -0.05121435597538948], "agreement_weight": 0.03432881832122803, "orthogonal_weight": -0.1654389500617981}]}, {"layer": 14, "memory_to_active": 0.3658784031867981, "contribution_to_active": 0.37100452184677124, "mix_mean": 0.31180498003959656, "mix_low": 0.038510166108608246, "mix_high": 0.0005677541485056281, "read_cosine": 0.5859020948410034, "beta_mean": 0.09906025975942612, "nonfinite": 0.0, "weighted_memory_to_active": 0.14755697548389435, "mix_std": 0.2355678826570511, "beta_std": 0.09276412427425385, "beta_low": 0.1340738981962204, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.020533090457320213, "output_influence": 0.21081361174583435, "residual_influence": 0.0694369375705719, "state_active": 0.0959777757525444, "state_memory": 0.029755614697933197, "state_peak": 30.28428840637207, "growth_active": 1.2156286239624023, "growth_memory": 1.6097313165664673, "heads": [{"head": 0, "memory_to_active": 0.3501819372177124, "contribution_to_active": 0.3356594145298004, "mix_mean": 0.31760454177856445, "mix_low": 0.02316761016845703, "mix_high": 0.0007333755493164062, "read_cosine": 0.6210379600524902, "beta_mean": 0.10927866399288177, "nonfinite": 0.0, "weighted_memory_to_active": 0.15736180543899536, "mix_std": 0.2679831087589264, "beta_std": 0.14306147396564484, "beta_low": 0.16204833984375, "beta_high": 0.0, "orthogonal_to_active": 0.007556102238595486, "evidence_weights": [-0.10622221231460571, -0.17135335505008698, 0.34490928053855896], "agreement_weight": 0.06472461670637131, "orthogonal_weight": -0.06517619639635086}, {"head": 1, "memory_to_active": 0.45145973563194275, "contribution_to_active": 0.4528213143348694, "mix_mean": 0.3349480628967285, "mix_low": 0.06305408477783203, "mix_high": 0.0008373260498046875, "read_cosine": 0.42825764417648315, "beta_mean": 0.3004293441772461, "nonfinite": 0.0, "weighted_memory_to_active": 0.223663330078125, "mix_std": 0.2982872724533081, "beta_std": 0.2693614661693573, "beta_low": 0.072265625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.018717341125011444, "evidence_weights": [-0.08695098757743835, -0.19511911273002625, 0.27747422456741333], "agreement_weight": 0.061449144035577774, "orthogonal_weight": -0.10247761756181717}, {"head": 2, "memory_to_active": 0.46871721744537354, "contribution_to_active": 0.31023287773132324, "mix_mean": 0.24947020411491394, "mix_low": 0.020517349243164062, "mix_high": 3.24249267578125e-05, "read_cosine": 0.5122134685516357, "beta_mean": 0.05976710096001625, "nonfinite": 0.0, "weighted_memory_to_active": 0.17665186524391174, "mix_std": 0.22062237560749054, "beta_std": 0.040931105613708496, "beta_low": 0.0018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.06496657431125641, "evidence_weights": [-0.15767055749893188, -0.16636814177036285, 0.0329262837767601], "agreement_weight": 0.11729746311903, "orthogonal_weight": -0.46604371070861816}, {"head": 3, "memory_to_active": 0.11162476241588593, "contribution_to_active": 0.5719877481460571, "mix_mean": 0.5756535530090332, "mix_low": 0.0021390914916992188, "mix_high": 0.0015039443969726562, "read_cosine": 0.6531360149383545, "beta_mean": 0.019662830978631973, "nonfinite": 0.0, "weighted_memory_to_active": 0.07361239939928055, "mix_std": 0.2447810173034668, "beta_std": 0.015494892373681068, "beta_low": 0.23944091796875, "beta_high": 0.0, "orthogonal_to_active": 0.01320329774171114, "evidence_weights": [-0.33275318145751953, -0.3071499466896057, 0.3254718780517578], "agreement_weight": 0.061591994017362595, "orthogonal_weight": -0.27194201946258545}, {"head": 4, "memory_to_active": 0.33883100748062134, "contribution_to_active": 0.4013902246952057, "mix_mean": 0.27133214473724365, "mix_low": 0.012197494506835938, "mix_high": 0.0002727508544921875, "read_cosine": 0.6363770961761475, "beta_mean": 0.08299539983272552, "nonfinite": 0.0, "weighted_memory_to_active": 0.1479741334915161, "mix_std": 0.226482555270195, "beta_std": 0.06610001623630524, "beta_low": 0.00213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.0020271698012948036, "evidence_weights": [-0.1437373161315918, -0.17814213037490845, 0.10889054089784622], "agreement_weight": 0.10683220624923706, "orthogonal_weight": 0.021414827555418015}, {"head": 5, "memory_to_active": 0.47445571422576904, "contribution_to_active": 0.1539355367422104, "mix_mean": 0.121821328997612, "mix_low": 0.1099853515625, "mix_high": 2.6702880859375e-05, "read_cosine": 0.6643902659416199, "beta_mean": 0.022228185087442398, "nonfinite": 0.0, "weighted_memory_to_active": 0.10607831180095673, "mix_std": 0.1552508920431137, "beta_std": 0.02163579873740673, "beta_low": 0.32672119140625, "beta_high": 0.0, "orthogonal_to_active": 0.016728051006793976, "evidence_weights": [-0.11269015818834305, -0.0957333892583847, 0.012737875804305077], "agreement_weight": 0.09699846804141998, "orthogonal_weight": -0.2472490519285202}]}, {"layer": 15, "memory_to_active": 0.15110790729522705, "contribution_to_active": 0.3807997703552246, "mix_mean": 0.29861941933631897, "mix_low": 0.01601489447057247, "mix_high": 0.00127410888671875, "read_cosine": 0.524006724357605, "beta_mean": 0.023610590025782585, "nonfinite": 0.0, "weighted_memory_to_active": 0.06664890050888062, "mix_std": 0.23889759182929993, "beta_std": 0.029638532549142838, "beta_low": 0.64019775390625, "beta_high": 0.0, "orthogonal_to_active": 0.008670303970575333, "output_influence": 0.24775245785713196, "residual_influence": 0.11913299560546875, "state_active": 0.10456426441669464, "state_memory": 0.016851363703608513, "state_peak": 46.52994918823242, "growth_active": 1.416420340538025, "growth_memory": 1.6000734567642212, "heads": [{"head": 0, "memory_to_active": 0.15765562653541565, "contribution_to_active": 0.411342054605484, "mix_mean": 0.3213474750518799, "mix_low": 0.01019287109375, "mix_high": 0.00046825408935546875, "read_cosine": 0.5983783006668091, "beta_mean": 0.08291961252689362, "nonfinite": 0.0, "weighted_memory_to_active": 0.06302805244922638, "mix_std": 0.24442264437675476, "beta_std": 0.07476618140935898, "beta_low": 0.11480712890625, "beta_high": 0.0, "orthogonal_to_active": 0.005165741313248873, "evidence_weights": [-0.18323540687561035, -0.22242499887943268, 0.2455158233642578], "agreement_weight": 0.0818805992603302, "orthogonal_weight": -0.12681347131729126}, {"head": 1, "memory_to_active": 0.2566538155078888, "contribution_to_active": 0.36018702387809753, "mix_mean": 0.28548890352249146, "mix_low": 0.01198577880859375, "mix_high": 0.0035085678100585938, "read_cosine": 0.7925116419792175, "beta_mean": 0.004662052728235722, "nonfinite": 0.0, "weighted_memory_to_active": 0.10972786694765091, "mix_std": 0.2375640720129013, "beta_std": 0.009045192040503025, "beta_low": 0.91058349609375, "beta_high": 0.0, "orthogonal_to_active": 0.009069755673408508, "evidence_weights": [-0.18357127904891968, -0.19795583188533783, 0.2345801144838333], "agreement_weight": 0.081895612180233, "orthogonal_weight": -0.1623026728630066}, {"head": 2, "memory_to_active": 0.03839760273694992, "contribution_to_active": 0.3869568705558777, "mix_mean": 0.3053075969219208, "mix_low": 0.023546218872070312, "mix_high": 0.00033092498779296875, "read_cosine": 0.6766420602798462, "beta_mean": 0.001337818568572402, "nonfinite": 0.0, "weighted_memory_to_active": 0.015523409470915794, "mix_std": 0.25560736656188965, "beta_std": 0.0017226311611011624, "beta_low": 0.99261474609375, "beta_high": 0.0, "orthogonal_to_active": 0.004921647720038891, "evidence_weights": [-0.21989035606384277, -0.2296430766582489, 0.2669124901294708], "agreement_weight": 0.054916854947805405, "orthogonal_weight": -0.4352916479110718}, {"head": 3, "memory_to_active": 0.30527302622795105, "contribution_to_active": 0.29437506198883057, "mix_mean": 0.1820330023765564, "mix_low": 0.02394390106201172, "mix_high": 4.482269287109375e-05, "read_cosine": 0.6577816605567932, "beta_mean": 0.02528778463602066, "nonfinite": 0.0, "weighted_memory_to_active": 0.1497340202331543, "mix_std": 0.17763958871364594, "beta_std": 0.0381658673286438, "beta_low": 0.34759521484375, "beta_high": 0.0, "orthogonal_to_active": 0.0037487191148102283, "evidence_weights": [-0.0998852401971817, -0.18012402951717377, 0.27363237738609314], "agreement_weight": 0.09057784080505371, "orthogonal_weight": -0.0380285382270813}, {"head": 4, "memory_to_active": 0.14650529623031616, "contribution_to_active": 0.42941004037857056, "mix_mean": 0.2872996926307678, "mix_low": 0.018265724182128906, "mix_high": 7.43865966796875e-05, "read_cosine": -0.1414322853088379, "beta_mean": 0.027415979653596878, "nonfinite": 0.0, "weighted_memory_to_active": 0.06107594072818756, "mix_std": 0.24371418356895447, "beta_std": 0.05401653051376343, "beta_low": 0.4755859375, "beta_high": 0.0, "orthogonal_to_active": 0.028926804661750793, "evidence_weights": [-0.2488471418619156, -0.243837371468544, 0.1333635002374649], "agreement_weight": 0.008008868433535099, "orthogonal_weight": -0.5121503472328186}, {"head": 5, "memory_to_active": 0.0021619941107928753, "contribution_to_active": 0.40252751111984253, "mix_mean": 0.4102398753166199, "mix_low": 0.008154869079589844, "mix_high": 0.0032176971435546875, "read_cosine": 0.5601589679718018, "beta_mean": 4.029456977150403e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.000804065028205514, "mix_std": 0.27443772554397583, "beta_std": 0.00011479089880594984, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001891556748887524, "evidence_weights": [-0.3194458484649658, -0.27385833859443665, 0.32879263162612915], "agreement_weight": 0.03306376188993454, "orthogonal_weight": 0.2608656585216522}]}]}, {"step": 5000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.813842070998362, "memory_to_active": 1.175445318222046, "contribution_to_active": 0.3292551040649414, "mix_mean": 0.17493999004364014, "mix_low": 0.2145458608865738, "mix_high": 0.0025482177734375, "read_cosine": 0.4060126841068268, "beta_mean": 0.11760267615318298, "nonfinite": 0.0, "weighted_memory_to_active": 0.19669833779335022, "mix_std": 0.16213086247444153, "beta_std": 0.08469821512699127, "beta_low": 0.2943407893180847, "beta_high": 0.0, "orthogonal_to_active": 0.02543150633573532, "samples": 4, "scored_tokens": 899, "normal_loss": 3.374234914779663, "normal_token_accuracy": 0.41490545868873596, "state_active": 0.07127925753593445, "state_memory": 0.03858327493071556, "state_peak": 48.13330841064453, "growth_active": 1.123725175857544, "growth_memory": 1.5960341691970825, "output_influence": 0.245695561170578, "residual_influence": 0.07086853682994843, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.473862886428833, "no_memory_loss_delta": 0.12740911543369293, "scrambled_loss_delta": 0.32715001702308655, "memory_only_loss_delta": 1.4569213390350342, "no_evidence_loss_delta": 0.05093390867114067, "no_agreement_loss_delta": 0.024621248245239258, "no_orthogonal_loss_delta": 0.002140988130122423}, {"index": 94, "tokens": 256, "normal_loss": 3.4986822605133057, "no_memory_loss_delta": 0.20161645114421844, "scrambled_loss_delta": 0.33880481123924255, "memory_only_loss_delta": 1.5511337518692017, "no_evidence_loss_delta": 0.05835231393575668, "no_agreement_loss_delta": 0.020582757890224457, "no_orthogonal_loss_delta": 0.0029503386467695236}, {"index": 188, "tokens": 256, "normal_loss": 3.7734830379486084, "no_memory_loss_delta": 0.030363846570253372, "scrambled_loss_delta": 0.08703099936246872, "memory_only_loss_delta": 1.6602946519851685, "no_evidence_loss_delta": 0.0011911671608686447, "no_agreement_loss_delta": -0.0012097666040062904, "no_orthogonal_loss_delta": -0.011190459132194519}, {"index": 283, "tokens": 131, "normal_loss": 2.156137466430664, "no_memory_loss_delta": 0.25425243377685547, "scrambled_loss_delta": 0.36158448457717896, "memory_only_loss_delta": 2.0018749237060547, "no_evidence_loss_delta": 0.09100857377052307, "no_agreement_loss_delta": 0.024481086060404778, "no_orthogonal_loss_delta": 0.006713733542710543}], "agreement_weight_abs_mean": 0.05626526102423668, "agreement_weight_abs_max": 0.3866252899169922, "agreement_weight_mean": 0.03034510649740696, "orthogonal_weight_abs_mean": 0.2261587381362915, "orthogonal_weight_abs_max": 0.8791537880897522, "orthogonal_weight_mean": -0.07457432895898819, "no_memory_loss": 3.5136241912841797, "no_memory_loss_delta": 0.1393890380859375, "no_memory_token_accuracy_gain": 0.0, "no_memory_token_win_fraction": 0.6028921008110046, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 13.011045754202696, "scrambled_loss": 3.6413450241088867, "scrambled_loss_delta": 0.26711004972457886, "scrambled_token_accuracy_gain": 0.041156839579343796, "scrambled_token_win_fraction": 0.6184649467468262, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 23.44111860089173, "memory_only_loss": 4.995306968688965, "memory_only_loss_delta": 1.621071457862854, "memory_only_token_accuracy_gain": 0.15572859346866608, "memory_only_token_win_fraction": 0.8387096524238586, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 80.2313227478102, "no_evidence_loss": 3.4189562797546387, "no_evidence_loss_delta": 0.044721171259880066, "no_evidence_token_accuracy_gain": -0.0033370412420481443, "no_evidence_token_win_fraction": 0.5528364777565002, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 4.373592142779352, "no_agreement_loss": 3.3903300762176514, "no_agreement_loss_delta": 0.016095159575343132, "no_agreement_token_accuracy_gain": -0.0022246940061450005, "no_agreement_token_win_fraction": 0.5472747683525085, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.596632462691016, "no_orthogonal_loss": 3.37347674369812, "no_orthogonal_loss_delta": -0.0007584859849885106, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.517241358757019, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": -0.07587737082233048, "short_no_memory_loss_delta": 0.07848000526428223, "long_context_memory_gain": 0.06090903282165527, "normal_context_loss_gain": 0.11469221115112305, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.082796335220337, "contribution_to_active": 0.6442781686782837, "mix_mean": 0.09720532596111298, "mix_low": 0.14635993540287018, "mix_high": 0.0007333755493164062, "read_cosine": 0.5713579654693604, "beta_mean": 0.2595430016517639, "nonfinite": 0.0, "weighted_memory_to_active": 0.595264196395874, "mix_std": 0.08318614214658737, "beta_std": 0.12405703961849213, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07190461456775665, "output_influence": 0.18474522233009338, "residual_influence": 0.22614039480686188, "state_active": 0.11012719571590424, "state_memory": 0.06448175758123398, "state_peak": 34.40238571166992, "growth_active": 1.225794792175293, "growth_memory": 1.444214105606079, "heads": [{"head": 0, "memory_to_active": 2.4074413776397705, "contribution_to_active": 0.6338856220245361, "mix_mean": 0.03696015104651451, "mix_low": 0.09134197235107422, "mix_high": 0.0, "read_cosine": 0.47324442863464355, "beta_mean": 0.22861553728580475, "nonfinite": 0.0, "weighted_memory_to_active": 0.6762333512306213, "mix_std": 0.05235976725816727, "beta_std": 0.08912284672260284, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03657926991581917, "evidence_weights": [-0.05976492911577225, -0.06865493953227997, 0.13477823138237], "agreement_weight": -0.04294026270508766, "orthogonal_weight": 0.06920958310365677}, {"head": 1, "memory_to_active": 2.124436140060425, "contribution_to_active": 1.1581662893295288, "mix_mean": 0.0372588112950325, "mix_low": 0.09579944610595703, "mix_high": 6.67572021484375e-06, "read_cosine": 0.5665205121040344, "beta_mean": 0.3164750933647156, "nonfinite": 0.0, "weighted_memory_to_active": 1.1998791694641113, "mix_std": 0.058905139565467834, "beta_std": 0.12286718189716339, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1514003574848175, "evidence_weights": [-0.04995008185505867, -0.08811093866825104, 0.10415511578321457], "agreement_weight": -0.010897167026996613, "orthogonal_weight": 0.2261325865983963}, {"head": 2, "memory_to_active": 1.6601049900054932, "contribution_to_active": 0.3771601617336273, "mix_mean": 0.026182372123003006, "mix_low": 0.2278594970703125, "mix_high": 0.0, "read_cosine": 0.4916972517967224, "beta_mean": 0.28579849004745483, "nonfinite": 0.0, "weighted_memory_to_active": 0.3745478689670563, "mix_std": 0.037241436541080475, "beta_std": 0.16929355263710022, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.017296144738793373, "evidence_weights": [-0.036393433809280396, -0.03747401759028435, -0.07869575917720795], "agreement_weight": -0.029968006536364555, "orthogonal_weight": 0.06533320993185043}, {"head": 3, "memory_to_active": 2.898188591003418, "contribution_to_active": 1.2286174297332764, "mix_mean": 0.1030341163277626, "mix_low": 0.011195182800292969, "mix_high": 4.76837158203125e-06, "read_cosine": 0.569202184677124, "beta_mean": 0.2981470823287964, "nonfinite": 0.0, "weighted_memory_to_active": 1.1409376859664917, "mix_std": 0.092623770236969, "beta_std": 0.1489803045988083, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.196732297539711, "evidence_weights": [-0.15939095616340637, -0.1782878339290619, 0.2571103870868683], "agreement_weight": -0.0452166311442852, "orthogonal_weight": 0.2258169800043106}, {"head": 4, "memory_to_active": 3.2234959602355957, "contribution_to_active": 0.1053595095872879, "mix_mean": 0.015194514766335487, "mix_low": 0.4430866241455078, "mix_high": 0.0, "read_cosine": 0.3913666307926178, "beta_mean": 0.31632930040359497, "nonfinite": 0.0, "weighted_memory_to_active": 0.09073368459939957, "mix_std": 0.015382281504571438, "beta_std": 0.13650038838386536, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.019326794892549515, "evidence_weights": [0.021526439115405083, 0.0031362022273242474, 0.060126569122076035], "agreement_weight": -0.043842341750860214, "orthogonal_weight": 0.2395918071269989}, {"head": 5, "memory_to_active": 0.1831100732088089, "contribution_to_active": 0.36248016357421875, "mix_mean": 0.3646019697189331, "mix_low": 0.008876800537109375, "mix_high": 0.0043888092041015625, "read_cosine": 0.9361164569854736, "beta_mean": 0.11189248412847519, "nonfinite": 0.0, "weighted_memory_to_active": 0.08925338834524155, "mix_std": 0.24260443449020386, "beta_std": 0.07757799327373505, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.010092825628817081, "evidence_weights": [-0.35603663325309753, -0.36072415113449097, 0.4649757742881775], "agreement_weight": 0.1268029808998108, "orthogonal_weight": 0.2920082211494446}]}, {"layer": 1, "memory_to_active": 2.8813915252685547, "contribution_to_active": 0.2580811381340027, "mix_mean": 0.04904921352863312, "mix_low": 0.4415402412414551, "mix_high": 1.6530355424038135e-05, "read_cosine": 0.3597809672355652, "beta_mean": 0.28154483437538147, "nonfinite": 0.0, "weighted_memory_to_active": 0.2592296600341797, "mix_std": 0.05913946405053139, "beta_std": 0.14832694828510284, "beta_low": 0.0014038087101653218, "beta_high": 0.0, "orthogonal_to_active": 0.02796613983809948, "output_influence": 0.1276281774044037, "residual_influence": 0.026310227811336517, "state_active": 0.058047086000442505, "state_memory": 0.07740845531225204, "state_peak": 21.839284896850586, "growth_active": 1.073434829711914, "growth_memory": 1.664465308189392, "heads": [{"head": 0, "memory_to_active": 3.2000105381011963, "contribution_to_active": 0.05647224560379982, "mix_mean": 0.009347669780254364, "mix_low": 0.7457056045532227, "mix_high": 0.0, "read_cosine": 0.07516248524188995, "beta_mean": 0.13700789213180542, "nonfinite": 0.0, "weighted_memory_to_active": 0.058755215257406235, "mix_std": 0.01611839234828949, "beta_std": 0.10484454035758972, "beta_low": 0.00347900390625, "beta_high": 0.0, "orthogonal_to_active": 0.01073247566819191, "evidence_weights": [0.07091668248176575, 0.053181570023298264, -0.14715628325939178], "agreement_weight": -0.11608849465847015, "orthogonal_weight": -0.18537765741348267}, {"head": 1, "memory_to_active": 3.078239679336548, "contribution_to_active": 0.17214486002922058, "mix_mean": 0.03447816148400307, "mix_low": 0.24082469940185547, "mix_high": 2.86102294921875e-06, "read_cosine": 0.5039926767349243, "beta_mean": 0.37087857723236084, "nonfinite": 0.0, "weighted_memory_to_active": 0.1748262196779251, "mix_std": 0.047323793172836304, "beta_std": 0.16245388984680176, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.011638779193162918, "evidence_weights": [-0.05804530158638954, -0.05664122477173805, 0.13379842042922974], "agreement_weight": -0.03813660144805908, "orthogonal_weight": 0.08040361106395721}, {"head": 2, "memory_to_active": 0.9679869413375854, "contribution_to_active": 0.3857036530971527, "mix_mean": 0.18071553111076355, "mix_low": 0.009825706481933594, "mix_high": 9.059906005859375e-05, "read_cosine": 0.6956286430358887, "beta_mean": 0.15115395188331604, "nonfinite": 0.0, "weighted_memory_to_active": 0.4639545679092407, "mix_std": 0.1752360612154007, "beta_std": 0.0683949887752533, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05636173114180565, "evidence_weights": [-0.21394632756710052, -0.2584894895553589, 0.23277881741523743], "agreement_weight": -0.02224556729197502, "orthogonal_weight": 0.18417829275131226}, {"head": 3, "memory_to_active": 3.0246057510375977, "contribution_to_active": 0.05190776661038399, "mix_mean": 0.008955884724855423, "mix_low": 0.7306852340698242, "mix_high": 0.0, "read_cosine": 0.2406911551952362, "beta_mean": 0.4691619873046875, "nonfinite": 0.0, "weighted_memory_to_active": 0.054617900401353836, "mix_std": 0.01452628429979086, "beta_std": 0.24448764324188232, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.0009845163440331817, "evidence_weights": [0.00964195653796196, 0.01222538948059082, -0.02214646153151989], "agreement_weight": -0.014338525012135506, "orthogonal_weight": -0.0201762393116951}, {"head": 4, "memory_to_active": 2.5685739517211914, "contribution_to_active": 0.8123414516448975, "mix_mean": 0.052932851016521454, "mix_low": 0.14249897003173828, "mix_high": 2.86102294921875e-06, "read_cosine": 0.4477096199989319, "beta_mean": 0.21968582272529602, "nonfinite": 0.0, "weighted_memory_to_active": 0.734755277633667, "mix_std": 0.07885002344846725, "beta_std": 0.10204025357961655, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08689422905445099, "evidence_weights": [-0.07121136784553528, -0.0734303891658783, 0.22131091356277466], "agreement_weight": -0.04054322466254234, "orthogonal_weight": 0.12352839857339859}, {"head": 5, "memory_to_active": 4.448931694030762, "contribution_to_active": 0.06991692632436752, "mix_mean": 0.007865182124078274, "mix_low": 0.7797012329101562, "mix_high": 2.86102294921875e-06, "read_cosine": 0.19550111889839172, "beta_mean": 0.3413807153701782, "nonfinite": 0.0, "weighted_memory_to_active": 0.06846868991851807, "mix_std": 0.022782213985919952, "beta_std": 0.20774036645889282, "beta_low": 0.004638671875, "beta_high": 0.0, "orthogonal_to_active": 0.0011850984301418066, "evidence_weights": [0.05390429496765137, 0.0424053817987442, 0.11125394701957703], "agreement_weight": -0.07505390793085098, "orthogonal_weight": 0.018104854971170425}]}, {"layer": 2, "memory_to_active": 1.6747580766677856, "contribution_to_active": 0.22512118518352509, "mix_mean": 0.08970152586698532, "mix_low": 0.3474653661251068, "mix_high": 0.00024525323533453047, "read_cosine": 0.3634698688983917, "beta_mean": 0.1667192280292511, "nonfinite": 0.0, "weighted_memory_to_active": 0.12140295654535294, "mix_std": 0.1056114211678505, "beta_std": 0.10286390781402588, "beta_low": 0.3602193593978882, "beta_high": 0.0, "orthogonal_to_active": 0.004207327961921692, "output_influence": 0.23809143900871277, "residual_influence": 0.042269252240657806, "state_active": 0.03910079225897789, "state_memory": 0.04121794551610947, "state_peak": 37.86663055419922, "growth_active": 0.9951659440994263, "growth_memory": 1.6126257181167603, "heads": [{"head": 0, "memory_to_active": 1.612344741821289, "contribution_to_active": 0.029965952038764954, "mix_mean": 0.010169871151447296, "mix_low": 0.7518796920776367, "mix_high": 0.0, "read_cosine": 0.21084117889404297, "beta_mean": 0.08622801303863525, "nonfinite": 0.0, "weighted_memory_to_active": 0.029942713677883148, "mix_std": 0.0223073847591877, "beta_std": 0.06721474230289459, "beta_low": 0.09503173828125, "beta_high": 0.0, "orthogonal_to_active": 0.0013491439167410135, "evidence_weights": [0.060209497809410095, 0.051753606647253036, 0.008092782460153103], "agreement_weight": -0.06852783262729645, "orthogonal_weight": -0.04648708552122116}, {"head": 1, "memory_to_active": 3.5848000049591064, "contribution_to_active": 0.07755263149738312, "mix_mean": 0.011684163473546505, "mix_low": 0.6220607757568359, "mix_high": 0.0, "read_cosine": 0.21357978880405426, "beta_mean": 0.44578713178634644, "nonfinite": 0.0, "weighted_memory_to_active": 0.07481496036052704, "mix_std": 0.016184823587536812, "beta_std": 0.17442946135997772, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.004356737714260817, "evidence_weights": [0.030797559767961502, 0.03777113929390907, 0.009133713319897652], "agreement_weight": -0.0051724412478506565, "orthogonal_weight": 0.06104900315403938}, {"head": 2, "memory_to_active": 0.3472435474395752, "contribution_to_active": 0.25133442878723145, "mix_mean": 0.15274041891098022, "mix_low": 0.04290294647216797, "mix_high": 3.147125244140625e-05, "read_cosine": 0.4955739378929138, "beta_mean": 0.0014699783641844988, "nonfinite": 0.0, "weighted_memory_to_active": 0.08591477572917938, "mix_std": 0.17361339926719666, "beta_std": 0.002098863711580634, "beta_low": 0.9908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.001071764505468309, "evidence_weights": [-0.11176495254039764, -0.11243663728237152, 0.20004406571388245], "agreement_weight": 0.07563101500272751, "orthogonal_weight": -0.017496252432465553}, {"head": 3, "memory_to_active": 0.6861125230789185, "contribution_to_active": 0.2624434530735016, "mix_mean": 0.07456947863101959, "mix_low": 0.10038089752197266, "mix_high": 0.00036144256591796875, "read_cosine": 0.5547308921813965, "beta_mean": 0.09412205219268799, "nonfinite": 0.0, "weighted_memory_to_active": 0.13055726885795593, "mix_std": 0.11306163668632507, "beta_std": 0.13418616354465485, "beta_low": 0.066162109375, "beta_high": 0.0, "orthogonal_to_active": 0.009347779676318169, "evidence_weights": [-0.016084909439086914, -0.07097457349300385, 0.28126221895217896], "agreement_weight": 0.0863880142569542, "orthogonal_weight": -0.11289367079734802}, {"head": 4, "memory_to_active": 3.8016462326049805, "contribution_to_active": 0.38545531034469604, "mix_mean": 0.018323756754398346, "mix_low": 0.5565328598022461, "mix_high": 1.9073486328125e-06, "read_cosine": 0.18601490557193756, "beta_mean": 0.37267202138900757, "nonfinite": 0.0, "weighted_memory_to_active": 0.40088510513305664, "mix_std": 0.055909886956214905, "beta_std": 0.23917356133460999, "beta_low": 0.00927734375, "beta_high": 0.0, "orthogonal_to_active": 0.008882134221494198, "evidence_weights": [0.016529714688658714, 0.011986197903752327, 0.13179509341716766], "agreement_weight": 0.025665896013379097, "orthogonal_weight": 0.024537088349461555}, {"head": 5, "memory_to_active": 0.016401231288909912, "contribution_to_active": 0.34397536516189575, "mix_mean": 0.2707214951515198, "mix_low": 0.011034965515136719, "mix_high": 0.0010766983032226562, "read_cosine": 0.5200785398483276, "beta_mean": 3.6196925066178665e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006302914582192898, "mix_std": 0.2525913715362549, "beta_std": 8.062423148658127e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00023640849394723773, "evidence_weights": [-0.1844395101070404, -0.16195616126060486, 0.24770110845565796], "agreement_weight": 0.007921231910586357, "orthogonal_weight": 0.07181157171726227}]}, {"layer": 3, "memory_to_active": 1.7950026988983154, "contribution_to_active": 0.20793575048446655, "mix_mean": 0.050455816090106964, "mix_low": 0.42879199981689453, "mix_high": 0.0007009506807662547, "read_cosine": 0.2943575382232666, "beta_mean": 0.15428707003593445, "nonfinite": 0.0, "weighted_memory_to_active": 0.12684080004692078, "mix_std": 0.08506539463996887, "beta_std": 0.11013342440128326, "beta_low": 0.2618917226791382, "beta_high": 0.0, "orthogonal_to_active": 0.011765711940824986, "output_influence": 0.15309211611747742, "residual_influence": 0.021492812782526016, "state_active": 0.04608805850148201, "state_memory": 0.04758913069963455, "state_peak": 33.57183837890625, "growth_active": 1.0280274152755737, "growth_memory": 1.7435206174850464, "heads": [{"head": 0, "memory_to_active": 1.7111774682998657, "contribution_to_active": 0.1068955510854721, "mix_mean": 0.02163781225681305, "mix_low": 0.5930318832397461, "mix_high": 0.0, "read_cosine": 0.21897009015083313, "beta_mean": 0.03137907385826111, "nonfinite": 0.0, "weighted_memory_to_active": 0.10317185521125793, "mix_std": 0.04756072908639908, "beta_std": 0.026421597227454185, "beta_low": 0.15045166015625, "beta_high": 0.0, "orthogonal_to_active": 0.0024583942722529173, "evidence_weights": [0.04737132415175438, 0.020314985886216164, -0.0633440613746643], "agreement_weight": -0.07717970758676529, "orthogonal_weight": -0.024262135848402977}, {"head": 1, "memory_to_active": 0.9130507111549377, "contribution_to_active": 0.12714803218841553, "mix_mean": 0.03953449800610542, "mix_low": 0.3718452453613281, "mix_high": 0.0, "read_cosine": 0.29114145040512085, "beta_mean": 0.018917128443717957, "nonfinite": 0.0, "weighted_memory_to_active": 0.05569911375641823, "mix_std": 0.07623432576656342, "beta_std": 0.02163425274193287, "beta_low": 0.41656494140625, "beta_high": 0.0, "orthogonal_to_active": 0.004254838917404413, "evidence_weights": [-0.012592732906341553, -0.011977877467870712, 0.02970600686967373], "agreement_weight": -0.05060283839702606, "orthogonal_weight": -0.08314287662506104}, {"head": 2, "memory_to_active": 3.2638752460479736, "contribution_to_active": 0.21121756732463837, "mix_mean": 0.024694737046957016, "mix_low": 0.4376955032348633, "mix_high": 0.0, "read_cosine": 0.2508227229118347, "beta_mean": 0.2846187949180603, "nonfinite": 0.0, "weighted_memory_to_active": 0.27271178364753723, "mix_std": 0.05361528322100639, "beta_std": 0.19033974409103394, "beta_low": 0.00177001953125, "beta_high": 0.0, "orthogonal_to_active": 0.029609929770231247, "evidence_weights": [-0.03387758508324623, -0.020532958209514618, 0.11247839033603668], "agreement_weight": -0.019419243559241295, "orthogonal_weight": -0.13353131711483002}, {"head": 3, "memory_to_active": 0.1322336345911026, "contribution_to_active": 0.2817031145095825, "mix_mean": 0.1474866271018982, "mix_low": 0.08043766021728516, "mix_high": 0.0017375946044921875, "read_cosine": 0.33540868759155273, "beta_mean": 0.00039569323416799307, "nonfinite": 0.0, "weighted_memory_to_active": 0.03164450824260712, "mix_std": 0.1876296103000641, "beta_std": 0.0010332848178222775, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.0036871074698865414, "evidence_weights": [-0.10871240496635437, -0.11569787561893463, 0.1937837302684784], "agreement_weight": 0.16070306301116943, "orthogonal_weight": -0.15570715069770813}, {"head": 4, "memory_to_active": 4.054305076599121, "contribution_to_active": 0.04677348583936691, "mix_mean": 0.005718141328543425, "mix_low": 0.8664302825927734, "mix_high": 0.0, "read_cosine": 0.20208734273910522, "beta_mean": 0.3495638966560364, "nonfinite": 0.0, "weighted_memory_to_active": 0.053869280964136124, "mix_std": 0.008839989081025124, "beta_std": 0.22088640928268433, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.007523005362600088, "evidence_weights": [0.11217305064201355, 0.10354354977607727, -0.14199356734752655], "agreement_weight": -0.1227223128080368, "orthogonal_weight": -0.14127472043037415}, {"head": 5, "memory_to_active": 0.6953736543655396, "contribution_to_active": 0.4738766849040985, "mix_mean": 0.06366308033466339, "mix_low": 0.2233114242553711, "mix_high": 0.002468109130859375, "read_cosine": 0.46771490573883057, "beta_mean": 0.2408478856086731, "nonfinite": 0.0, "weighted_memory_to_active": 0.24394825100898743, "mix_std": 0.1365123987197876, "beta_std": 0.2004852294921875, "beta_low": 0.0042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.023060990497469902, "evidence_weights": [-0.010571535676717758, -0.09283679723739624, 0.04475812613964081], "agreement_weight": 0.1363537609577179, "orthogonal_weight": -0.15215003490447998}]}, {"layer": 4, "memory_to_active": 1.37919282913208, "contribution_to_active": 0.21471184492111206, "mix_mean": 0.12410926818847656, "mix_low": 0.24447982013225555, "mix_high": 0.00030295056058093905, "read_cosine": 0.41596364974975586, "beta_mean": 0.14808134734630585, "nonfinite": 0.0, "weighted_memory_to_active": 0.13844811916351318, "mix_std": 0.11453326046466827, "beta_std": 0.10456115007400513, "beta_low": 0.1866963803768158, "beta_high": 0.0, "orthogonal_to_active": 0.020522352308034897, "output_influence": 0.14595584571361542, "residual_influence": 0.026312805712223053, "state_active": 0.050610922276973724, "state_memory": 0.04791625589132309, "state_peak": 36.051666259765625, "growth_active": 1.0884631872177124, "growth_memory": 1.7458159923553467, "heads": [{"head": 0, "memory_to_active": 0.5072646141052246, "contribution_to_active": 0.2751684784889221, "mix_mean": 0.16466124355793, "mix_low": 0.03286933898925781, "mix_high": 1.049041748046875e-05, "read_cosine": 0.6304300427436829, "beta_mean": 0.07224947214126587, "nonfinite": 0.0, "weighted_memory_to_active": 0.19483187794685364, "mix_std": 0.1804211139678955, "beta_std": 0.08872085809707642, "beta_low": 0.060791015625, "beta_high": 0.0, "orthogonal_to_active": 0.0011053805937990546, "evidence_weights": [-0.10888149589300156, -0.1496652513742447, 0.22972092032432556], "agreement_weight": 0.07709818333387375, "orthogonal_weight": 0.008962259627878666}, {"head": 1, "memory_to_active": 0.7654513120651245, "contribution_to_active": 0.16147089004516602, "mix_mean": 0.12809911370277405, "mix_low": 0.0791778564453125, "mix_high": 0.0, "read_cosine": 0.4550405442714691, "beta_mean": 0.07471015304327011, "nonfinite": 0.0, "weighted_memory_to_active": 0.14043071866035461, "mix_std": 0.14536064863204956, "beta_std": 0.06597334146499634, "beta_low": 0.012939453125, "beta_high": 0.0, "orthogonal_to_active": 0.021939527243375778, "evidence_weights": [-0.10160550475120544, -0.10820204019546509, 0.3101552724838257], "agreement_weight": 0.06095685809850693, "orthogonal_weight": -0.1894138753414154}, {"head": 2, "memory_to_active": 2.9718539714813232, "contribution_to_active": 0.09068061411380768, "mix_mean": 0.011485687457025051, "mix_low": 0.7025127410888672, "mix_high": 0.0, "read_cosine": 0.30492982268333435, "beta_mean": 0.3028859496116638, "nonfinite": 0.0, "weighted_memory_to_active": 0.0977056622505188, "mix_std": 0.018344031646847725, "beta_std": 0.14809919893741608, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005086459685117006, "evidence_weights": [0.020689841359853745, 0.010919510386884212, -0.05091925337910652], "agreement_weight": 0.018580783158540726, "orthogonal_weight": 0.06408234685659409}, {"head": 3, "memory_to_active": 2.0138871669769287, "contribution_to_active": 0.07936259359121323, "mix_mean": 0.02422020211815834, "mix_low": 0.39226341247558594, "mix_high": 0.0, "read_cosine": 0.2173299491405487, "beta_mean": 0.19070976972579956, "nonfinite": 0.0, "weighted_memory_to_active": 0.10139584541320801, "mix_std": 0.03554575890302658, "beta_std": 0.1718975156545639, "beta_low": 0.0467529296875, "beta_high": 0.0, "orthogonal_to_active": 0.012470134533941746, "evidence_weights": [-0.02851959876716137, -0.00764106772840023, -0.13077446818351746], "agreement_weight": 0.006479203235358, "orthogonal_weight": -0.1549578756093979}, {"head": 4, "memory_to_active": 0.10167636722326279, "contribution_to_active": 0.3732929527759552, "mix_mean": 0.3802216351032257, "mix_low": 0.0044269561767578125, "mix_high": 0.0017852783203125, "read_cosine": 0.40427857637405396, "beta_mean": 0.0002993686939589679, "nonfinite": 0.0, "weighted_memory_to_active": 0.04526187479496002, "mix_std": 0.2574716806411743, "beta_std": 0.0005385976983234286, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.005200944375246763, "evidence_weights": [-0.20514129102230072, -0.21241384744644165, 0.33888015151023865], "agreement_weight": -0.011473366990685463, "orthogonal_weight": -0.14301103353500366}, {"head": 5, "memory_to_active": 1.9150233268737793, "contribution_to_active": 0.30829548835754395, "mix_mean": 0.03596772253513336, "mix_low": 0.2556285858154297, "mix_high": 2.193450927734375e-05, "read_cosine": 0.48377302289009094, "beta_mean": 0.24763327836990356, "nonfinite": 0.0, "weighted_memory_to_active": 0.25106266140937805, "mix_std": 0.05005629360675812, "beta_std": 0.1521373689174652, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07733166217803955, "evidence_weights": [-0.04217654466629028, -0.06795744597911835, 0.15065817534923553], "agreement_weight": -0.044881414622068405, "orthogonal_weight": 0.3754642605781555}]}, {"layer": 5, "memory_to_active": 0.8924483060836792, "contribution_to_active": 0.3429972231388092, "mix_mean": 0.2515110969543457, "mix_low": 0.12196159362792969, "mix_high": 0.0023322105407714844, "read_cosine": 0.28348463773727417, "beta_mean": 0.08596836775541306, "nonfinite": 0.0, "weighted_memory_to_active": 0.17022979259490967, "mix_std": 0.20855429768562317, "beta_std": 0.08341377973556519, "beta_low": 0.3648783564567566, "beta_high": 0.0, "orthogonal_to_active": 0.01939530298113823, "output_influence": 0.31134843826293945, "residual_influence": 0.052914880216121674, "state_active": 0.0770052820444107, "state_memory": 0.046279147267341614, "state_peak": 33.6943244934082, "growth_active": 1.1124298572540283, "growth_memory": 1.9331412315368652, "heads": [{"head": 0, "memory_to_active": 2.1074042320251465, "contribution_to_active": 0.2046017050743103, "mix_mean": 0.05013803392648697, "mix_low": 0.19840049743652344, "mix_high": 0.0, "read_cosine": 0.04314080998301506, "beta_mean": 0.16099731624126434, "nonfinite": 0.0, "weighted_memory_to_active": 0.20499958097934723, "mix_std": 0.08056202530860901, "beta_std": 0.15074554085731506, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.015384403988718987, "evidence_weights": [-0.036534383893013, -0.04350640997290611, -0.125633105635643], "agreement_weight": -0.016551548615098, "orthogonal_weight": -0.07863104343414307}, {"head": 1, "memory_to_active": 0.05399202182888985, "contribution_to_active": 0.2955760955810547, "mix_mean": 0.2137252390384674, "mix_low": 0.0859823226928711, "mix_high": 0.0009174346923828125, "read_cosine": 0.16287845373153687, "beta_mean": 0.00014348849072121084, "nonfinite": 0.0, "weighted_memory_to_active": 0.011481251567602158, "mix_std": 0.25423723459243774, "beta_std": 0.0005016601644456387, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.004422531463205814, "evidence_weights": [-0.11788216233253479, -0.10582906007766724, 0.056386079639196396], "agreement_weight": 0.09768813103437424, "orthogonal_weight": -0.405102014541626}, {"head": 2, "memory_to_active": 0.4777512550354004, "contribution_to_active": 0.35203123092651367, "mix_mean": 0.3501818776130676, "mix_low": 0.006442070007324219, "mix_high": 0.0002288818359375, "read_cosine": 0.4917304515838623, "beta_mean": 0.046512290835380554, "nonfinite": 0.0, "weighted_memory_to_active": 0.23312430083751678, "mix_std": 0.2487659752368927, "beta_std": 0.02454224042594433, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.0012684749672189355, "evidence_weights": [-0.25945529341697693, -0.25925710797309875, 0.4038670063018799], "agreement_weight": 0.07293234765529633, "orthogonal_weight": -0.0070556579157710075}, {"head": 3, "memory_to_active": 2.048339605331421, "contribution_to_active": 0.13202646374702454, "mix_mean": 0.031111929565668106, "mix_low": 0.4018220901489258, "mix_high": 0.0, "read_cosine": 0.2063649296760559, "beta_mean": 0.08905623108148575, "nonfinite": 0.0, "weighted_memory_to_active": 0.165788933634758, "mix_std": 0.058619506657123566, "beta_std": 0.10489746928215027, "beta_low": 0.0860595703125, "beta_high": 0.0, "orthogonal_to_active": 0.029662318527698517, "evidence_weights": [0.01947326771914959, 0.003813298884779215, 0.09960338473320007], "agreement_weight": 0.03466497361660004, "orthogonal_weight": -0.2061452865600586}, {"head": 4, "memory_to_active": 0.5933297872543335, "contribution_to_active": 0.5428389310836792, "mix_mean": 0.34555506706237793, "mix_low": 0.034328460693359375, "mix_high": 0.007954597473144531, "read_cosine": 0.6043381690979004, "beta_mean": 0.21886548399925232, "nonfinite": 0.0, "weighted_memory_to_active": 0.36749523878097534, "mix_std": 0.3164341151714325, "beta_std": 0.21932055056095123, "beta_low": 0.10308837890625, "beta_high": 0.0, "orthogonal_to_active": 0.05678346008062363, "evidence_weights": [-0.16380539536476135, -0.21607165038585663, 0.28680869936943054], "agreement_weight": 0.06863726675510406, "orthogonal_weight": 0.217147096991539}, {"head": 5, "memory_to_active": 0.07387277483940125, "contribution_to_active": 0.5309087038040161, "mix_mean": 0.5183543562889099, "mix_low": 0.004794120788574219, "mix_high": 0.0048923492431640625, "read_cosine": 0.19245506823062897, "beta_mean": 0.0002353986637899652, "nonfinite": 0.0, "weighted_memory_to_active": 0.03848946839570999, "mix_std": 0.29270681738853455, "beta_std": 0.0004752293461933732, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.00885063037276268, "evidence_weights": [-0.23528844118118286, -0.23381522297859192, 0.32021743059158325], "agreement_weight": 0.13075442612171173, "orthogonal_weight": -0.2718845307826996}]}, {"layer": 6, "memory_to_active": 0.6638969779014587, "contribution_to_active": 0.3897170424461365, "mix_mean": 0.24520722031593323, "mix_low": 0.15520159900188446, "mix_high": 0.004305045120418072, "read_cosine": 0.3498087525367737, "beta_mean": 0.08373984694480896, "nonfinite": 0.0, "weighted_memory_to_active": 0.19179990887641907, "mix_std": 0.23049046099185944, "beta_std": 0.09180974215269089, "beta_low": 0.36163330078125, "beta_high": 0.0, "orthogonal_to_active": 0.0289355106651783, "output_influence": 0.3176422119140625, "residual_influence": 0.07038454711437225, "state_active": 0.08117759972810745, "state_memory": 0.025872938334941864, "state_peak": 25.792993545532227, "growth_active": 1.3314822912216187, "growth_memory": 1.5946077108383179, "heads": [{"head": 0, "memory_to_active": 0.3956739902496338, "contribution_to_active": 0.3683256506919861, "mix_mean": 0.1383444368839264, "mix_low": 0.1259775161743164, "mix_high": 0.00107574462890625, "read_cosine": 0.2426125407218933, "beta_mean": 0.055764492601156235, "nonfinite": 0.0, "weighted_memory_to_active": 0.19618159532546997, "mix_std": 0.1893271505832672, "beta_std": 0.10929474234580994, "beta_low": 0.077880859375, "beta_high": 0.0, "orthogonal_to_active": 0.06067778915166855, "evidence_weights": [-0.07779266685247421, -0.10248409956693649, 0.1456698179244995], "agreement_weight": 0.10907357931137085, "orthogonal_weight": -0.47185733914375305}, {"head": 1, "memory_to_active": 0.39619043469429016, "contribution_to_active": 0.24259456992149353, "mix_mean": 0.2111586630344391, "mix_low": 0.06914138793945312, "mix_high": 0.00327301025390625, "read_cosine": 0.4948781728744507, "beta_mean": 0.004847235977649689, "nonfinite": 0.0, "weighted_memory_to_active": 0.09081432223320007, "mix_std": 0.23698794841766357, "beta_std": 0.007479382213205099, "beta_low": 0.87872314453125, "beta_high": 0.0, "orthogonal_to_active": 0.016229767352342606, "evidence_weights": [-0.147508442401886, -0.13818922638893127, 0.08254542946815491], "agreement_weight": 0.004133343696594238, "orthogonal_weight": -0.2266678661108017}, {"head": 2, "memory_to_active": 2.141763925552368, "contribution_to_active": 0.07484960556030273, "mix_mean": 0.01728687435388565, "mix_low": 0.6188325881958008, "mix_high": 0.0, "read_cosine": 0.21814019978046417, "beta_mean": 0.1882237195968628, "nonfinite": 0.0, "weighted_memory_to_active": 0.09623732417821884, "mix_std": 0.04815997928380966, "beta_std": 0.14439845085144043, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.019799713045358658, "evidence_weights": [0.06308744102716446, 0.058605097234249115, -0.1056833267211914], "agreement_weight": 0.016316596418619156, "orthogonal_weight": -0.21620424091815948}, {"head": 3, "memory_to_active": 0.32202136516571045, "contribution_to_active": 0.524024248123169, "mix_mean": 0.3758779764175415, "mix_low": 0.017792701721191406, "mix_high": 0.01016998291015625, "read_cosine": 0.5965427160263062, "beta_mean": 0.1610986441373825, "nonfinite": 0.0, "weighted_memory_to_active": 0.18797747790813446, "mix_std": 0.3231149911880493, "beta_std": 0.17308074235916138, "beta_low": 0.08782958984375, "beta_high": 0.0, "orthogonal_to_active": 0.03070889785885811, "evidence_weights": [-0.1096845343708992, -0.217275932431221, 0.28547045588493347], "agreement_weight": -0.007981108501553535, "orthogonal_weight": 0.21697692573070526}, {"head": 4, "memory_to_active": 0.6818227767944336, "contribution_to_active": 0.7790313959121704, "mix_mean": 0.4691265821456909, "mix_low": 0.005440711975097656, "mix_high": 0.011307716369628906, "read_cosine": 0.24921149015426636, "beta_mean": 0.0922727957367897, "nonfinite": 0.0, "weighted_memory_to_active": 0.5638965368270874, "mix_std": 0.3200477957725525, "beta_std": 0.11631211638450623, "beta_low": 0.1253662109375, "beta_high": 0.0, "orthogonal_to_active": 0.0353793166577816, "evidence_weights": [-0.20409846305847168, -0.33722466230392456, -0.1329122930765152], "agreement_weight": -0.014660322107374668, "orthogonal_weight": 0.07462405413389206}, {"head": 5, "memory_to_active": 0.04590914398431778, "contribution_to_active": 0.34947654604911804, "mix_mean": 0.259448766708374, "mix_low": 0.094024658203125, "mix_high": 3.814697265625e-06, "read_cosine": 0.29746735095977783, "beta_mean": 0.0002321441425010562, "nonfinite": 0.0, "weighted_memory_to_active": 0.01569219119846821, "mix_std": 0.2653048634529114, "beta_std": 0.00029295828426256776, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010817576199769974, "evidence_weights": [-0.10726192593574524, -0.15208588540554047, -0.19894243776798248], "agreement_weight": 0.10736251622438431, "orthogonal_weight": -0.743706464767456}]}, {"layer": 7, "memory_to_active": 2.129051923751831, "contribution_to_active": 0.13968144357204437, "mix_mean": 0.022483965381979942, "mix_low": 0.5711128115653992, "mix_high": 4.450480446394067e-06, "read_cosine": 0.23701660335063934, "beta_mean": 0.1783430576324463, "nonfinite": 0.0, "weighted_memory_to_active": 0.13927125930786133, "mix_std": 0.04080667719244957, "beta_std": 0.10863998532295227, "beta_low": 0.0044453940354287624, "beta_high": 0.0, "orthogonal_to_active": 0.0136478366330266, "output_influence": 0.15325117111206055, "residual_influence": 0.027782917022705078, "state_active": 0.02188895083963871, "state_memory": 0.044705379754304886, "state_peak": 19.341184616088867, "growth_active": 0.9664416313171387, "growth_memory": 1.5156701803207397, "heads": [{"head": 0, "memory_to_active": 2.0029351711273193, "contribution_to_active": 0.29127785563468933, "mix_mean": 0.02873900718986988, "mix_low": 0.4257526397705078, "mix_high": 2.6702880859375e-05, "read_cosine": 0.40066373348236084, "beta_mean": 0.18487249314785004, "nonfinite": 0.0, "weighted_memory_to_active": 0.28089725971221924, "mix_std": 0.05411841720342636, "beta_std": 0.11823499947786331, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05503195896744728, "evidence_weights": [-0.03607721999287605, -0.06534415483474731, 0.11592107266187668], "agreement_weight": -0.04298119246959686, "orthogonal_weight": 0.26665303111076355}, {"head": 1, "memory_to_active": 2.7865757942199707, "contribution_to_active": 0.19291415810585022, "mix_mean": 0.03986433148384094, "mix_low": 0.25550079345703125, "mix_high": 0.0, "read_cosine": -0.011506849899888039, "beta_mean": 0.29014572501182556, "nonfinite": 0.0, "weighted_memory_to_active": 0.20413674414157867, "mix_std": 0.04733718931674957, "beta_std": 0.1356593668460846, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0211962778121233, "evidence_weights": [-0.07115336507558823, -0.06980395317077637, 0.12540338933467865], "agreement_weight": -0.007213366683572531, "orthogonal_weight": -0.10614164173603058}, {"head": 2, "memory_to_active": 1.5154519081115723, "contribution_to_active": 0.21548494696617126, "mix_mean": 0.03791458159685135, "mix_low": 0.4910163879394531, "mix_high": 0.0, "read_cosine": 0.22511401772499084, "beta_mean": 0.20127898454666138, "nonfinite": 0.0, "weighted_memory_to_active": 0.1978265941143036, "mix_std": 0.08662034571170807, "beta_std": 0.10240933299064636, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015362192643806338, "evidence_weights": [-0.053176745772361755, -0.08843206614255905, 0.15801331400871277], "agreement_weight": 0.015298672951757908, "orthogonal_weight": 0.008647188544273376}, {"head": 3, "memory_to_active": 2.6065847873687744, "contribution_to_active": 0.029468338936567307, "mix_mean": 0.005926730576902628, "mix_low": 0.8639459609985352, "mix_high": 0.0, "read_cosine": 0.12883257865905762, "beta_mean": 0.13112986087799072, "nonfinite": 0.0, "weighted_memory_to_active": 0.03163658455014229, "mix_std": 0.01235788781195879, "beta_std": 0.09373315423727036, "beta_low": 0.00811767578125, "beta_high": 0.0, "orthogonal_to_active": 0.003249664790928364, "evidence_weights": [0.07738476991653442, 0.08622004836797714, -0.05662921071052551], "agreement_weight": -0.010906334966421127, "orthogonal_weight": -0.10265204310417175}, {"head": 4, "memory_to_active": 1.4428000450134277, "contribution_to_active": 0.045560941100120544, "mix_mean": 0.01270710676908493, "mix_low": 0.6476249694824219, "mix_high": 0.0, "read_cosine": 0.32810693979263306, "beta_mean": 0.13044795393943787, "nonfinite": 0.0, "weighted_memory_to_active": 0.05097593367099762, "mix_std": 0.025703173130750656, "beta_std": 0.10585404187440872, "beta_low": 0.0179443359375, "beta_high": 0.0, "orthogonal_to_active": 0.000508583034388721, "evidence_weights": [0.025195760652422905, 0.022329576313495636, -0.004321018233895302], "agreement_weight": 0.054449521005153656, "orthogonal_weight": 0.013675641268491745}, {"head": 5, "memory_to_active": 2.4199628829956055, "contribution_to_active": 0.06338243186473846, "mix_mean": 0.009752023033797741, "mix_low": 0.7428359985351562, "mix_high": 0.0, "read_cosine": 0.35088926553726196, "beta_mean": 0.1321832835674286, "nonfinite": 0.0, "weighted_memory_to_active": 0.07015439122915268, "mix_std": 0.018703049048781395, "beta_std": 0.09594901651144028, "beta_low": 0.0006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.0003643140953499824, "evidence_weights": [0.01744280569255352, 0.039241425693035126, -0.1317361742258072], "agreement_weight": -0.011272099800407887, "orthogonal_weight": 0.006471177563071251}]}, {"layer": 8, "memory_to_active": 1.0925350189208984, "contribution_to_active": 0.28470873832702637, "mix_mean": 0.19824716448783875, "mix_low": 0.11818060278892517, "mix_high": 0.0024166107177734375, "read_cosine": 0.5302319526672363, "beta_mean": 0.09890474379062653, "nonfinite": 0.0, "weighted_memory_to_active": 0.19345015287399292, "mix_std": 0.16391509771347046, "beta_std": 0.07808934152126312, "beta_low": 0.1963907927274704, "beta_high": 0.0, "orthogonal_to_active": 0.028238482773303986, "output_influence": 0.25309741497039795, "residual_influence": 0.05039138346910477, "state_active": 0.0645831972360611, "state_memory": 0.03678152710199356, "state_peak": 21.772937774658203, "growth_active": 1.018919825553894, "growth_memory": 1.5961695909500122, "heads": [{"head": 0, "memory_to_active": 0.12378799915313721, "contribution_to_active": 0.3938247859477997, "mix_mean": 0.35116976499557495, "mix_low": 0.0018672943115234375, "mix_high": 9.5367431640625e-07, "read_cosine": 0.7292696237564087, "beta_mean": 0.004170594736933708, "nonfinite": 0.0, "weighted_memory_to_active": 0.05584472417831421, "mix_std": 0.2171432375907898, "beta_std": 0.005337749607861042, "beta_low": 0.9173583984375, "beta_high": 0.0, "orthogonal_to_active": 0.009438427165150642, "evidence_weights": [-0.2398330420255661, -0.22768737375736237, 0.28168562054634094], "agreement_weight": 0.07184800505638123, "orthogonal_weight": -0.283387154340744}, {"head": 1, "memory_to_active": 0.8165387511253357, "contribution_to_active": 0.2715434730052948, "mix_mean": 0.267191618680954, "mix_low": 0.00405120849609375, "mix_high": 1.621246337890625e-05, "read_cosine": 0.6978518962860107, "beta_mean": 0.08335036784410477, "nonfinite": 0.0, "weighted_memory_to_active": 0.2970825433731079, "mix_std": 0.17150217294692993, "beta_std": 0.054183751344680786, "beta_low": 0.0069580078125, "beta_high": 0.0, "orthogonal_to_active": 0.028622252866625786, "evidence_weights": [-0.1792556494474411, -0.16278867423534393, 0.3027099668979645], "agreement_weight": 0.006945292931050062, "orthogonal_weight": 0.1506039947271347}, {"head": 2, "memory_to_active": 2.142577648162842, "contribution_to_active": 0.15596002340316772, "mix_mean": 0.028665833175182343, "mix_low": 0.3866758346557617, "mix_high": 0.0, "read_cosine": 0.2618081569671631, "beta_mean": 0.2319156527519226, "nonfinite": 0.0, "weighted_memory_to_active": 0.21709832549095154, "mix_std": 0.04862666875123978, "beta_std": 0.14915546774864197, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.026551103219389915, "evidence_weights": [-0.0021733101457357407, -0.029557552188634872, 0.0033367404248565435], "agreement_weight": 0.06913968175649643, "orthogonal_weight": -0.14923390746116638}, {"head": 3, "memory_to_active": 0.8947319388389587, "contribution_to_active": 0.19750896096229553, "mix_mean": 0.14811426401138306, "mix_low": 0.051026344299316406, "mix_high": 0.0, "read_cosine": 0.6257808208465576, "beta_mean": 0.05644731596112251, "nonfinite": 0.0, "weighted_memory_to_active": 0.22186721861362457, "mix_std": 0.1384809911251068, "beta_std": 0.045867741107940674, "beta_low": 0.0369873046875, "beta_high": 0.0, "orthogonal_to_active": 0.03731921687722206, "evidence_weights": [-0.10297761857509613, -0.10744715481996536, 0.3811805844306946], "agreement_weight": 0.017109498381614685, "orthogonal_weight": 0.26154860854148865}, {"head": 4, "memory_to_active": 2.3490445613861084, "contribution_to_active": 0.22227001190185547, "mix_mean": 0.045135848224163055, "mix_low": 0.18505477905273438, "mix_high": 0.0, "read_cosine": 0.3012033700942993, "beta_mean": 0.13136586546897888, "nonfinite": 0.0, "weighted_memory_to_active": 0.24105462431907654, "mix_std": 0.055303141474723816, "beta_std": 0.0860104113817215, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.0051224129274487495, "evidence_weights": [-0.028595680370926857, -0.04375714063644409, 0.0402236208319664], "agreement_weight": -0.008216660469770432, "orthogonal_weight": 0.024891523644328117}, {"head": 5, "memory_to_active": 0.22852903604507446, "contribution_to_active": 0.46714499592781067, "mix_mean": 0.3492056429386139, "mix_low": 0.08040809631347656, "mix_high": 0.014482498168945312, "read_cosine": 0.5654776692390442, "beta_mean": 0.08617869764566422, "nonfinite": 0.0, "weighted_memory_to_active": 0.12775345146656036, "mix_std": 0.35243430733680725, "beta_std": 0.12798090279102325, "beta_low": 0.21697998046875, "beta_high": 0.0, "orthogonal_to_active": 0.06237747520208359, "evidence_weights": [-0.12358864396810532, -0.28867778182029724, -0.1114044040441513], "agreement_weight": 0.032729506492614746, "orthogonal_weight": 0.643324613571167}]}, {"layer": 9, "memory_to_active": 1.3806774616241455, "contribution_to_active": 0.3778175413608551, "mix_mean": 0.08432409167289734, "mix_low": 0.3791869580745697, "mix_high": 0.0006327629089355469, "read_cosine": 0.36034679412841797, "beta_mean": 0.10517988353967667, "nonfinite": 0.0, "weighted_memory_to_active": 0.3330155611038208, "mix_std": 0.14602620899677277, "beta_std": 0.052170414477586746, "beta_low": 0.3443807065486908, "beta_high": 0.0, "orthogonal_to_active": 0.007602511905133724, "output_influence": 0.24506081640720367, "residual_influence": 0.04675310105085373, "state_active": 0.03648146614432335, "state_memory": 0.038264159113168716, "state_peak": 36.554603576660156, "growth_active": 1.0174235105514526, "growth_memory": 1.6120972633361816, "heads": [{"head": 0, "memory_to_active": 0.41509711742401123, "contribution_to_active": 0.16440454125404358, "mix_mean": 0.1210167407989502, "mix_low": 0.3008232116699219, "mix_high": 9.5367431640625e-07, "read_cosine": 0.26845037937164307, "beta_mean": 0.004292674828320742, "nonfinite": 0.0, "weighted_memory_to_active": 0.0715697780251503, "mix_std": 0.19354012608528137, "beta_std": 0.008482522331178188, "beta_low": 0.89813232421875, "beta_high": 0.0, "orthogonal_to_active": 0.016159480437636375, "evidence_weights": [-0.042326848953962326, -0.07685013860464096, 0.29616379737854004], "agreement_weight": -0.007261333521455526, "orthogonal_weight": -0.24148425459861755}, {"head": 1, "memory_to_active": 0.17558419704437256, "contribution_to_active": 0.16792316734790802, "mix_mean": 0.051469944417476654, "mix_low": 0.5682258605957031, "mix_high": 0.0, "read_cosine": 0.38784679770469666, "beta_mean": 0.0008775821188464761, "nonfinite": 0.0, "weighted_memory_to_active": 0.02250959537923336, "mix_std": 0.12628675997257233, "beta_std": 0.001289109350182116, "beta_low": 0.99700927734375, "beta_high": 0.0, "orthogonal_to_active": 0.002943316474556923, "evidence_weights": [-0.039822034537792206, -0.029339851811528206, 0.07802846282720566], "agreement_weight": 0.12650072574615479, "orthogonal_weight": -0.13451167941093445}, {"head": 2, "memory_to_active": 0.7176990509033203, "contribution_to_active": 0.318104088306427, "mix_mean": 0.10427148640155792, "mix_low": 0.29737186431884766, "mix_high": 9.5367431640625e-07, "read_cosine": 0.20628197491168976, "beta_mean": 0.03091450035572052, "nonfinite": 0.0, "weighted_memory_to_active": 0.17115435004234314, "mix_std": 0.17137880623340607, "beta_std": 0.033394817262887955, "beta_low": 0.171142578125, "beta_high": 0.0, "orthogonal_to_active": 0.0021601321641355753, "evidence_weights": [-0.06713839620351791, -0.05700435861945152, 0.1193331629037857], "agreement_weight": 0.06484122574329376, "orthogonal_weight": 0.015231502242386341}, {"head": 3, "memory_to_active": 2.040156364440918, "contribution_to_active": 0.214990496635437, "mix_mean": 0.05139503628015518, "mix_low": 0.29760265350341797, "mix_high": 0.0, "read_cosine": 0.41350144147872925, "beta_mean": 0.1330549120903015, "nonfinite": 0.0, "weighted_memory_to_active": 0.20502106845378876, "mix_std": 0.07867830991744995, "beta_std": 0.07022374868392944, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.001897095120511949, "evidence_weights": [0.015715952962636948, 0.011795517057180405, 0.0017232490936294198], "agreement_weight": -0.04344446584582329, "orthogonal_weight": 0.00929145235568285}, {"head": 4, "memory_to_active": 3.0627293586730957, "contribution_to_active": 1.084376573562622, "mix_mean": 0.13666149973869324, "mix_low": 0.2264385223388672, "mix_high": 0.0037946701049804688, "read_cosine": 0.4249700903892517, "beta_mean": 0.2637752294540405, "nonfinite": 0.0, "weighted_memory_to_active": 1.1198397874832153, "mix_std": 0.20988480746746063, "beta_std": 0.10086417198181152, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0034250200260430574, "evidence_weights": [-0.04657527059316635, -0.06681888550519943, 0.11569273471832275], "agreement_weight": -0.06836363673210144, "orthogonal_weight": 0.003277104115113616}, {"head": 5, "memory_to_active": 1.8727984428405762, "contribution_to_active": 0.3171062767505646, "mix_mean": 0.04112980514764786, "mix_low": 0.5846595764160156, "mix_high": 0.0, "read_cosine": 0.46103012561798096, "beta_mean": 0.19816440343856812, "nonfinite": 0.0, "weighted_memory_to_active": 0.4079987704753876, "mix_std": 0.09638839215040207, "beta_std": 0.09876808524131775, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.019030023366212845, "evidence_weights": [0.028079669922590256, -0.00939527340233326, 0.007595257833600044], "agreement_weight": -0.025521179661154747, "orthogonal_weight": 0.06399175524711609}]}, {"layer": 10, "memory_to_active": 0.6998932361602783, "contribution_to_active": 0.3227885365486145, "mix_mean": 0.18124769628047943, "mix_low": 0.22585837543010712, "mix_high": 0.003597736358642578, "read_cosine": 0.38115841150283813, "beta_mean": 0.07125504314899445, "nonfinite": 0.0, "weighted_memory_to_active": 0.15362277626991272, "mix_std": 0.1932642161846161, "beta_std": 0.07285290211439133, "beta_low": 0.43865966796875, "beta_high": 0.0, "orthogonal_to_active": 0.0317767933011055, "output_influence": 0.32332104444503784, "residual_influence": 0.07634417712688446, "state_active": 0.09255557507276535, "state_memory": 0.027714844793081284, "state_peak": 48.13330841064453, "growth_active": 1.207920789718628, "growth_memory": 1.4896044731140137, "heads": [{"head": 0, "memory_to_active": 0.6038696765899658, "contribution_to_active": 0.75601726770401, "mix_mean": 0.4697926640510559, "mix_low": 0.03529071807861328, "mix_high": 0.019124984741210938, "read_cosine": 0.15943412482738495, "beta_mean": 0.1638278365135193, "nonfinite": 0.0, "weighted_memory_to_active": 0.3714415431022644, "mix_std": 0.34590989351272583, "beta_std": 0.18449971079826355, "beta_low": 0.08587646484375, "beta_high": 0.0, "orthogonal_to_active": 0.09594377875328064, "evidence_weights": [-0.1253288984298706, -0.2516590356826782, -0.01657562144100666], "agreement_weight": 0.01415039412677288, "orthogonal_weight": 0.28150051832199097}, {"head": 1, "memory_to_active": 0.8608064651489258, "contribution_to_active": 0.15753959119319916, "mix_mean": 0.06461739540100098, "mix_low": 0.34507179260253906, "mix_high": 0.000194549560546875, "read_cosine": 0.18886348605155945, "beta_mean": 0.011442625895142555, "nonfinite": 0.0, "weighted_memory_to_active": 0.0743919387459755, "mix_std": 0.13161978125572205, "beta_std": 0.017324645072221756, "beta_low": 0.669677734375, "beta_high": 0.0, "orthogonal_to_active": 0.018574586138129234, "evidence_weights": [-0.019731128588318825, -0.02176785282790661, -0.0790570080280304], "agreement_weight": -0.0023539606481790543, "orthogonal_weight": -0.25842058658599854}, {"head": 2, "memory_to_active": 2.175778388977051, "contribution_to_active": 0.2512176036834717, "mix_mean": 0.018073715269565582, "mix_low": 0.6843967437744141, "mix_high": 0.0, "read_cosine": 0.22789213061332703, "beta_mean": 0.168587788939476, "nonfinite": 0.0, "weighted_memory_to_active": 0.29165858030319214, "mix_std": 0.05125946179032326, "beta_std": 0.11083024740219116, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.030960239470005035, "evidence_weights": [0.03446321189403534, 0.012521681375801563, -0.024563532322645187], "agreement_weight": -0.036356668919324875, "orthogonal_weight": -0.11052636057138443}, {"head": 3, "memory_to_active": 0.16791421175003052, "contribution_to_active": 0.30268558859825134, "mix_mean": 0.24790886044502258, "mix_low": 0.1015462875366211, "mix_high": 0.0021200180053710938, "read_cosine": 0.7163528800010681, "beta_mean": 0.07241938263177872, "nonfinite": 0.0, "weighted_memory_to_active": 0.07754065096378326, "mix_std": 0.290976881980896, "beta_std": 0.10794389247894287, "beta_low": 0.20843505859375, "beta_high": 0.0, "orthogonal_to_active": 0.02880123257637024, "evidence_weights": [-0.13309377431869507, -0.15802113711833954, 0.28065523505210876], "agreement_weight": 0.02194288559257984, "orthogonal_weight": 0.5061571598052979}, {"head": 4, "memory_to_active": 0.1910972148180008, "contribution_to_active": 0.15022267401218414, "mix_mean": 0.09994575381278992, "mix_low": 0.17052364349365234, "mix_high": 0.0001373291015625, "read_cosine": 0.2585141658782959, "beta_mean": 0.0012411712668836117, "nonfinite": 0.0, "weighted_memory_to_active": 0.03266073018312454, "mix_std": 0.1502629667520523, "beta_std": 0.003847230924293399, "beta_low": 0.9832763671875, "beta_high": 0.0, "orthogonal_to_active": 0.006814321503043175, "evidence_weights": [-0.07696565985679626, -0.051228802651166916, -0.08080276846885681], "agreement_weight": 0.03585347160696983, "orthogonal_weight": -0.247187077999115}, {"head": 5, "memory_to_active": 0.19989334046840668, "contribution_to_active": 0.31904828548431396, "mix_mean": 0.18714776635169983, "mix_low": 0.01832103729248047, "mix_high": 9.5367431640625e-06, "read_cosine": 0.7358936071395874, "beta_mean": 0.010011439211666584, "nonfinite": 0.0, "weighted_memory_to_active": 0.07404326647520065, "mix_std": 0.1895562708377838, "beta_std": 0.01267164759337902, "beta_low": 0.6846923828125, "beta_high": 0.0, "orthogonal_to_active": 0.009566592052578926, "evidence_weights": [-0.14647243916988373, -0.11724625527858734, 0.15611398220062256], "agreement_weight": 0.08698392659425735, "orthogonal_weight": -0.25435423851013184}]}, {"layer": 11, "memory_to_active": 0.3683141767978668, "contribution_to_active": 0.36838996410369873, "mix_mean": 0.26751238107681274, "mix_low": 0.06699562072753906, "mix_high": 0.021030109375715256, "read_cosine": 0.5388907194137573, "beta_mean": 0.0445517972111702, "nonfinite": 0.0, "weighted_memory_to_active": 0.10146459937095642, "mix_std": 0.2620824873447418, "beta_std": 0.06381195038557053, "beta_low": 0.4003092646598816, "beta_high": 0.0, "orthogonal_to_active": 0.020777247846126556, "output_influence": 0.49747052788734436, "residual_influence": 0.12252708524465561, "state_active": 0.1182030737400055, "state_memory": 0.018515342846512794, "state_peak": 23.971460342407227, "growth_active": 1.1363537311553955, "growth_memory": 1.4220908880233765, "heads": [{"head": 0, "memory_to_active": 0.10166335850954056, "contribution_to_active": 0.3989238739013672, "mix_mean": 0.2623459994792938, "mix_low": 0.03892230987548828, "mix_high": 0.000579833984375, "read_cosine": 0.4468757212162018, "beta_mean": 0.0006556216394528747, "nonfinite": 0.0, "weighted_memory_to_active": 0.05040159076452255, "mix_std": 0.2502824664115906, "beta_std": 0.00227805832400918, "beta_low": 0.99591064453125, "beta_high": 0.0, "orthogonal_to_active": 0.01590815931558609, "evidence_weights": [-0.12099400162696838, -0.14296920597553253, 0.16120348870754242], "agreement_weight": 0.1300356239080429, "orthogonal_weight": -0.457972913980484}, {"head": 1, "memory_to_active": 0.6228595972061157, "contribution_to_active": 0.1712372601032257, "mix_mean": 0.15585218369960785, "mix_low": 0.049083709716796875, "mix_high": 3.814697265625e-06, "read_cosine": 0.5743289589881897, "beta_mean": 0.0409952774643898, "nonfinite": 0.0, "weighted_memory_to_active": 0.13759879767894745, "mix_std": 0.1774829626083374, "beta_std": 0.06653531640768051, "beta_low": 0.27117919921875, "beta_high": 0.0, "orthogonal_to_active": 0.006060440093278885, "evidence_weights": [-0.10682454705238342, -0.0765768364071846, 0.04255471006035805], "agreement_weight": 0.08556617051362991, "orthogonal_weight": -0.06004784256219864}, {"head": 2, "memory_to_active": 0.09423113614320755, "contribution_to_active": 0.7549390196800232, "mix_mean": 0.42320388555526733, "mix_low": 0.07873821258544922, "mix_high": 0.06141853332519531, "read_cosine": 0.4324234127998352, "beta_mean": 0.06835076957941055, "nonfinite": 0.0, "weighted_memory_to_active": 0.07590922713279724, "mix_std": 0.39748960733413696, "beta_std": 0.10381276905536652, "beta_low": 0.42236328125, "beta_high": 0.0, "orthogonal_to_active": 0.05770108848810196, "evidence_weights": [-0.2131497710943222, -0.27972158789634705, -0.08440356701612473], "agreement_weight": -0.02655823901295662, "orthogonal_weight": 0.8791537880897522}, {"head": 3, "memory_to_active": 0.062101997435092926, "contribution_to_active": 0.399284303188324, "mix_mean": 0.4447771906852722, "mix_low": 0.07087898254394531, "mix_high": 0.063934326171875, "read_cosine": 0.5747233629226685, "beta_mean": 0.08341841399669647, "nonfinite": 0.0, "weighted_memory_to_active": 0.033595792949199677, "mix_std": 0.40403681993484497, "beta_std": 0.12278477102518082, "beta_low": 0.35418701171875, "beta_high": 0.0, "orthogonal_to_active": 0.023272834718227386, "evidence_weights": [-0.28464651107788086, -0.303989440202713, 0.17814627289772034], "agreement_weight": 0.008729243651032448, "orthogonal_weight": 0.8450544476509094}, {"head": 4, "memory_to_active": 0.9568556547164917, "contribution_to_active": 0.1830727756023407, "mix_mean": 0.09445907920598984, "mix_low": 0.13758563995361328, "mix_high": 0.0, "read_cosine": 0.3356545567512512, "beta_mean": 0.04092703387141228, "nonfinite": 0.0, "weighted_memory_to_active": 0.14523059129714966, "mix_std": 0.1328505426645279, "beta_std": 0.04475177079439163, "beta_low": 0.12921142578125, "beta_high": 0.0, "orthogonal_to_active": 0.0023087214212864637, "evidence_weights": [-0.026130765676498413, -0.015947887673974037, 0.2861867845058441], "agreement_weight": -0.013221653178334236, "orthogonal_weight": -0.01972879096865654}, {"head": 5, "memory_to_active": 0.3721732199192047, "contribution_to_active": 0.30288252234458923, "mix_mean": 0.22443583607673645, "mix_low": 0.026764869689941406, "mix_high": 0.000244140625, "read_cosine": 0.8693381547927856, "beta_mean": 0.03296365216374397, "nonfinite": 0.0, "weighted_memory_to_active": 0.16605152189731598, "mix_std": 0.21035243570804596, "beta_std": 0.04270903021097183, "beta_low": 0.22900390625, "beta_high": 0.0, "orthogonal_to_active": 0.01941222883760929, "evidence_weights": [-0.13035814464092255, -0.16465498507022858, 0.20769135653972626], "agreement_weight": 0.059351783245801926, "orthogonal_weight": -0.33044588565826416}]}, {"layer": 12, "memory_to_active": 0.9052202701568604, "contribution_to_active": 0.4050130844116211, "mix_mean": 0.22453142702579498, "mix_low": 0.10866276919841766, "mix_high": 0.00018628439283929765, "read_cosine": 0.32210904359817505, "beta_mean": 0.04673510417342186, "nonfinite": 0.0, "weighted_memory_to_active": 0.2959115505218506, "mix_std": 0.1977967917919159, "beta_std": 0.04873214662075043, "beta_low": 0.4560750424861908, "beta_high": 0.0, "orthogonal_to_active": 0.03951564431190491, "output_influence": 0.20028255879878998, "residual_influence": 0.04608150199055672, "state_active": 0.0544486939907074, "state_memory": 0.027856290340423584, "state_peak": 32.672035217285156, "growth_active": 1.1209579706192017, "growth_memory": 1.5537456274032593, "heads": [{"head": 0, "memory_to_active": 1.2055712938308716, "contribution_to_active": 0.27186858654022217, "mix_mean": 0.14199496805667877, "mix_low": 0.057373046875, "mix_high": 0.00013828277587890625, "read_cosine": 0.4075623154640198, "beta_mean": 0.03897320479154587, "nonfinite": 0.0, "weighted_memory_to_active": 0.3486683666706085, "mix_std": 0.1639673262834549, "beta_std": 0.059483058750629425, "beta_low": 0.18792724609375, "beta_high": 0.0, "orthogonal_to_active": 0.053906213492155075, "evidence_weights": [-0.07634129375219345, -0.1081513836979866, 0.212068572640419], "agreement_weight": 0.08529162406921387, "orthogonal_weight": -0.21504414081573486}, {"head": 1, "memory_to_active": 0.48469552397727966, "contribution_to_active": 0.24548643827438354, "mix_mean": 0.19216381013393402, "mix_low": 0.03774452209472656, "mix_high": 2.86102294921875e-05, "read_cosine": 0.5453251600265503, "beta_mean": 0.009359358809888363, "nonfinite": 0.0, "weighted_memory_to_active": 0.1580648124217987, "mix_std": 0.20522469282150269, "beta_std": 0.009672412648797035, "beta_low": 0.673828125, "beta_high": 0.0, "orthogonal_to_active": 0.06541958451271057, "evidence_weights": [-0.08859018236398697, -0.13630211353302002, 0.19110365211963654], "agreement_weight": 0.09747852385044098, "orthogonal_weight": -0.6516622304916382}, {"head": 2, "memory_to_active": 0.6201293468475342, "contribution_to_active": 0.33125269412994385, "mix_mean": 0.24744240939617157, "mix_low": 0.018354415893554688, "mix_high": 0.00010204315185546875, "read_cosine": 0.2697391211986542, "beta_mean": 0.06569072604179382, "nonfinite": 0.0, "weighted_memory_to_active": 0.267610639333725, "mix_std": 0.21814270317554474, "beta_std": 0.073985256254673, "beta_low": 0.080322265625, "beta_high": 0.0, "orthogonal_to_active": 0.004996344912797213, "evidence_weights": [-0.12877710163593292, -0.09954272955656052, 0.2691204845905304], "agreement_weight": 0.044953539967536926, "orthogonal_weight": 0.02220757305622101}, {"head": 3, "memory_to_active": 0.18261820077896118, "contribution_to_active": 0.4541797339916229, "mix_mean": 0.4024762511253357, "mix_low": 0.008597373962402344, "mix_high": 4.673004150390625e-05, "read_cosine": 0.040977105498313904, "beta_mean": 0.0041253697127103806, "nonfinite": 0.0, "weighted_memory_to_active": 0.0919385701417923, "mix_std": 0.2581954598426819, "beta_std": 0.005419336259365082, "beta_low": 0.9091796875, "beta_high": 0.0, "orthogonal_to_active": 0.029296550899744034, "evidence_weights": [-0.19863948225975037, -0.20450463891029358, 0.09206779301166534], "agreement_weight": 0.03546343371272087, "orthogonal_weight": -0.3298441469669342}, {"head": 4, "memory_to_active": 2.7427353858947754, "contribution_to_active": 0.8021360635757446, "mix_mean": 0.029475126415491104, "mix_low": 0.5172147750854492, "mix_high": 0.000255584716796875, "read_cosine": 0.20679128170013428, "beta_mean": 0.15715889632701874, "nonfinite": 0.0, "weighted_memory_to_active": 0.84002685546875, "mix_std": 0.07668192684650421, "beta_std": 0.13604328036308289, "beta_low": 0.01824951171875, "beta_high": 0.0, "orthogonal_to_active": 0.03880707919597626, "evidence_weights": [0.04876700043678284, 0.012099456042051315, 0.08110012859106064], "agreement_weight": -0.022053638473153114, "orthogonal_weight": -0.05007980018854141}, {"head": 5, "memory_to_active": 0.19557160139083862, "contribution_to_active": 0.32515501976013184, "mix_mean": 0.33363592624664307, "mix_low": 0.012692451477050781, "mix_high": 0.0005464553833007812, "read_cosine": 0.4622591733932495, "beta_mean": 0.005103080999106169, "nonfinite": 0.0, "weighted_memory_to_active": 0.06915992498397827, "mix_std": 0.2645686864852905, "beta_std": 0.007789525203406811, "beta_low": 0.866943359375, "beta_high": 0.0, "orthogonal_to_active": 0.044668056070804596, "evidence_weights": [-0.1795620173215866, -0.18851891160011292, 0.2634941339492798], "agreement_weight": 0.04390649124979973, "orthogonal_weight": -0.7411169409751892}]}, {"layer": 13, "memory_to_active": 0.3418661057949066, "contribution_to_active": 0.3063885569572449, "mix_mean": 0.26946595311164856, "mix_low": 0.03047148510813713, "mix_high": 0.001975377555936575, "read_cosine": 0.34459713101387024, "beta_mean": 0.027858871966600418, "nonfinite": 0.0, "weighted_memory_to_active": 0.10558921098709106, "mix_std": 0.2123003453016281, "beta_std": 0.03739223629236221, "beta_low": 0.5823771357536316, "beta_high": 0.0, "orthogonal_to_active": 0.041545819491147995, "output_influence": 0.28497377038002014, "residual_influence": 0.08542483299970627, "state_active": 0.09259790182113647, "state_memory": 0.028436850756406784, "state_peak": 34.51526641845703, "growth_active": 1.0909241437911987, "growth_memory": 1.380506157875061, "heads": [{"head": 0, "memory_to_active": 0.4344891905784607, "contribution_to_active": 0.23138494789600372, "mix_mean": 0.17818759381771088, "mix_low": 0.035729408264160156, "mix_high": 1.1444091796875e-05, "read_cosine": 0.2769891619682312, "beta_mean": 0.011510217562317848, "nonfinite": 0.0, "weighted_memory_to_active": 0.11732112616300583, "mix_std": 0.1967577338218689, "beta_std": 0.011755568906664848, "beta_low": 0.57244873046875, "beta_high": 0.0, "orthogonal_to_active": 0.08798687160015106, "evidence_weights": [-0.10870872437953949, -0.13666115701198578, 0.11030575633049011], "agreement_weight": 0.06840083003044128, "orthogonal_weight": -0.8085688352584839}, {"head": 1, "memory_to_active": 0.4397561550140381, "contribution_to_active": 0.24319349229335785, "mix_mean": 0.16951888799667358, "mix_low": 0.04985523223876953, "mix_high": 0.0, "read_cosine": 0.3140528202056885, "beta_mean": 0.006671681068837643, "nonfinite": 0.0, "weighted_memory_to_active": 0.10735853016376495, "mix_std": 0.19088603556156158, "beta_std": 0.0066220723092556, "beta_low": 0.8212890625, "beta_high": 0.0, "orthogonal_to_active": 0.07110090553760529, "evidence_weights": [-0.09329377114772797, -0.11822043359279633, 0.1210421696305275], "agreement_weight": 0.06952809542417526, "orthogonal_weight": -0.7312204241752625}, {"head": 2, "memory_to_active": 0.32052117586135864, "contribution_to_active": 0.2730310559272766, "mix_mean": 0.20329317450523376, "mix_low": 0.017075538635253906, "mix_high": 2.86102294921875e-06, "read_cosine": 0.6459066867828369, "beta_mean": 0.03602604568004608, "nonfinite": 0.0, "weighted_memory_to_active": 0.11251200735569, "mix_std": 0.17831765115261078, "beta_std": 0.059322867542505264, "beta_low": 0.39923095703125, "beta_high": 0.0, "orthogonal_to_active": 0.016099857166409492, "evidence_weights": [-0.14486753940582275, -0.13450919091701508, 0.3281680643558502], "agreement_weight": 0.08495155721902847, "orthogonal_weight": -0.2311617136001587}, {"head": 3, "memory_to_active": 0.0423055998980999, "contribution_to_active": 0.48370686173439026, "mix_mean": 0.48307275772094727, "mix_low": 0.0026302337646484375, "mix_high": 0.00010204315185546875, "read_cosine": -0.17996221780776978, "beta_mean": 0.0002651988761499524, "nonfinite": 0.0, "weighted_memory_to_active": 0.021880455315113068, "mix_std": 0.2114141881465912, "beta_std": 0.0008052305784076452, "beta_low": 0.998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.011706707067787647, "evidence_weights": [-0.30573004484176636, -0.25221264362335205, 0.10204261541366577], "agreement_weight": 0.3866252899169922, "orthogonal_weight": -0.540962278842926}, {"head": 4, "memory_to_active": 0.31163233518600464, "contribution_to_active": 0.330573171377182, "mix_mean": 0.43598607182502747, "mix_low": 0.007681846618652344, "mix_high": 0.011724472045898438, "read_cosine": 0.7370604276657104, "beta_mean": 0.09488534927368164, "nonfinite": 0.0, "weighted_memory_to_active": 0.14556607604026794, "mix_std": 0.3076860308647156, "beta_std": 0.10653520375490189, "beta_low": 0.06915283203125, "beta_high": 0.0, "orthogonal_to_active": 0.03146938607096672, "evidence_weights": [-0.24960479140281677, -0.27322015166282654, 0.32576701045036316], "agreement_weight": 0.006021040491759777, "orthogonal_weight": 0.30209073424339294}, {"head": 5, "memory_to_active": 0.5024919509887695, "contribution_to_active": 0.27644169330596924, "mix_mean": 0.14673709869384766, "mix_low": 0.06985664367675781, "mix_high": 1.1444091796875e-05, "read_cosine": 0.2735357880592346, "beta_mean": 0.017794732004404068, "nonfinite": 0.0, "weighted_memory_to_active": 0.12889699637889862, "mix_std": 0.18874040246009827, "beta_std": 0.0393124558031559, "beta_low": 0.63336181640625, "beta_high": 0.0, "orthogonal_to_active": 0.030911169946193695, "evidence_weights": [-0.10357266664505005, -0.11723358929157257, -0.053600091487169266], "agreement_weight": 0.03953973203897476, "orthogonal_weight": -0.2770581543445587}]}, {"layer": 14, "memory_to_active": 0.35266751050949097, "contribution_to_active": 0.37949302792549133, "mix_mean": 0.32301780581474304, "mix_low": 0.03329944610595703, "mix_high": 0.0006052653188817203, "read_cosine": 0.5770334601402283, "beta_mean": 0.09735312312841415, "nonfinite": 0.0, "weighted_memory_to_active": 0.14413124322891235, "mix_std": 0.24302715063095093, "beta_std": 0.08965666592121124, "beta_low": 0.1583150327205658, "beta_high": 0.0, "orthogonal_to_active": 0.024212580174207687, "output_influence": 0.21696247160434723, "residual_influence": 0.070954829454422, "state_active": 0.09474144876003265, "state_memory": 0.02796340361237526, "state_peak": 29.092498779296875, "growth_active": 1.1942195892333984, "growth_memory": 1.5942720174789429, "heads": [{"head": 0, "memory_to_active": 0.308642715215683, "contribution_to_active": 0.3241446018218994, "mix_mean": 0.2942197620868683, "mix_low": 0.029921531677246094, "mix_high": 0.000335693359375, "read_cosine": 0.622184157371521, "beta_mean": 0.07946290820837021, "nonfinite": 0.0, "weighted_memory_to_active": 0.12791699171066284, "mix_std": 0.2629261314868927, "beta_std": 0.11203955113887787, "beta_low": 0.2041015625, "beta_high": 0.0, "orthogonal_to_active": 0.0017202456947416067, "evidence_weights": [-0.09884549677371979, -0.17185160517692566, 0.4150526821613312], "agreement_weight": 0.060109589248895645, "orthogonal_weight": -0.01808110438287258}, {"head": 1, "memory_to_active": 0.4365517199039459, "contribution_to_active": 0.4531993269920349, "mix_mean": 0.3245197534561157, "mix_low": 0.07135868072509766, "mix_high": 0.0007410049438476562, "read_cosine": 0.45182377099990845, "beta_mean": 0.3238200545310974, "nonfinite": 0.0, "weighted_memory_to_active": 0.21661385893821716, "mix_std": 0.2996535897254944, "beta_std": 0.27629274129867554, "beta_low": 0.07464599609375, "beta_high": 0.0, "orthogonal_to_active": 0.012703011743724346, "evidence_weights": [-0.07756035774946213, -0.21113836765289307, 0.27559423446655273], "agreement_weight": 0.06162680312991142, "orthogonal_weight": -0.07312111556529999}, {"head": 2, "memory_to_active": 0.465340793132782, "contribution_to_active": 0.3398573100566864, "mix_mean": 0.28118589520454407, "mix_low": 0.01631927490234375, "mix_high": 4.38690185546875e-05, "read_cosine": 0.4702072739601135, "beta_mean": 0.05727916583418846, "nonfinite": 0.0, "weighted_memory_to_active": 0.18425533175468445, "mix_std": 0.23692931234836578, "beta_std": 0.044116679579019547, "beta_low": 0.0067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.08678023517131805, "evidence_weights": [-0.1688682585954666, -0.17564329504966736, 0.06825420260429382], "agreement_weight": 0.10793637484312057, "orthogonal_weight": -0.5750150084495544}, {"head": 3, "memory_to_active": 0.09244772046804428, "contribution_to_active": 0.5903002619743347, "mix_mean": 0.5888651609420776, "mix_low": 0.0020971298217773438, "mix_high": 0.0016927719116210938, "read_cosine": 0.6459190249443054, "beta_mean": 0.015795979648828506, "nonfinite": 0.0, "weighted_memory_to_active": 0.06106310337781906, "mix_std": 0.24469736218452454, "beta_std": 0.011748924851417542, "beta_low": 0.35150146484375, "beta_high": 0.0, "orthogonal_to_active": 0.012874741107225418, "evidence_weights": [-0.34814196825027466, -0.31261011958122253, 0.34598270058631897], "agreement_weight": 0.06490809470415115, "orthogonal_weight": -0.31776201725006104}, {"head": 4, "memory_to_active": 0.30772578716278076, "contribution_to_active": 0.4030073285102844, "mix_mean": 0.3022824823856354, "mix_low": 0.008997917175292969, "mix_high": 0.0008039474487304688, "read_cosine": 0.6191893815994263, "beta_mean": 0.08007947355508804, "nonfinite": 0.0, "weighted_memory_to_active": 0.14062923192977905, "mix_std": 0.2402503490447998, "beta_std": 0.06103429198265076, "beta_low": 0.002685546875, "beta_high": 0.0, "orthogonal_to_active": 0.0029217773117125034, "evidence_weights": [-0.15543650090694427, -0.1938471794128418, 0.11705496162176132], "agreement_weight": 0.1084057092666626, "orthogonal_weight": -0.030414201319217682}, {"head": 5, "memory_to_active": 0.5052962899208069, "contribution_to_active": 0.16644930839538574, "mix_mean": 0.14703373610973358, "mix_low": 0.07110214233398438, "mix_high": 1.430511474609375e-05, "read_cosine": 0.6528767943382263, "beta_mean": 0.027681123465299606, "nonfinite": 0.0, "weighted_memory_to_active": 0.1343088448047638, "mix_std": 0.17370609939098358, "beta_std": 0.0327078178524971, "beta_low": 0.31024169921875, "beta_high": 0.0, "orthogonal_to_active": 0.028275471180677414, "evidence_weights": [-0.1368304044008255, -0.10893825441598892, 0.014153123833239079], "agreement_weight": 0.09448147565126419, "orthogonal_weight": -0.32675430178642273}]}, {"layer": 15, "memory_to_active": 0.16741394996643066, "contribution_to_active": 0.4009583592414856, "mix_mean": 0.32096999883651733, "mix_low": 0.013164997100830078, "mix_high": 0.0016865730285644531, "read_cosine": 0.5665956735610962, "beta_mean": 0.03157755732536316, "nonfinite": 0.0, "weighted_memory_to_active": 0.07750184088945389, "mix_std": 0.24829432368278503, "beta_std": 0.03865988925099373, "beta_low": 0.5917765498161316, "beta_high": 0.0, "orthogonal_to_active": 0.014890234917402267, "output_influence": 0.278205543756485, "residual_influence": 0.141811802983284, "state_active": 0.10281092673540115, "state_memory": 0.01632893644273281, "state_peak": 42.3096809387207, "growth_active": 1.3716440200805664, "growth_memory": 1.6340017318725586, "heads": [{"head": 0, "memory_to_active": 0.1879473775625229, "contribution_to_active": 0.4426061511039734, "mix_mean": 0.33593127131462097, "mix_low": 0.012104988098144531, "mix_high": 0.0009851455688476562, "read_cosine": 0.6657899022102356, "beta_mean": 0.08520840108394623, "nonfinite": 0.0, "weighted_memory_to_active": 0.09676163643598557, "mix_std": 0.2553013265132904, "beta_std": 0.07045864313840866, "beta_low": 0.07818603515625, "beta_high": 0.0, "orthogonal_to_active": 0.009046655148267746, "evidence_weights": [-0.18670205771923065, -0.22344236075878143, 0.26654553413391113], "agreement_weight": 0.08514055609703064, "orthogonal_weight": -0.17577989399433136}, {"head": 1, "memory_to_active": 0.24679481983184814, "contribution_to_active": 0.37882766127586365, "mix_mean": 0.30446770787239075, "mix_low": 0.009164810180664062, "mix_high": 0.0045986175537109375, "read_cosine": 0.8096688985824585, "beta_mean": 0.0033694440498948097, "nonfinite": 0.0, "weighted_memory_to_active": 0.11282429099082947, "mix_std": 0.24992039799690247, "beta_std": 0.006284315139055252, "beta_low": 0.94744873046875, "beta_high": 0.0, "orthogonal_to_active": 0.010164765641093254, "evidence_weights": [-0.18926291167736053, -0.20361997187137604, 0.2390104979276657], "agreement_weight": 0.0853896513581276, "orthogonal_weight": -0.177906796336174}, {"head": 2, "memory_to_active": 0.03526625409722328, "contribution_to_active": 0.3997594118118286, "mix_mean": 0.32691705226898193, "mix_low": 0.017808914184570312, "mix_high": 0.0003795623779296875, "read_cosine": 0.7250336408615112, "beta_mean": 0.0011475203791633248, "nonfinite": 0.0, "weighted_memory_to_active": 0.013812948018312454, "mix_std": 0.2577003538608551, "beta_std": 0.001581238815560937, "beta_low": 0.9957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.004811153281480074, "evidence_weights": [-0.22241590917110443, -0.23836342990398407, 0.284196138381958], "agreement_weight": 0.05226349085569382, "orthogonal_weight": -0.48241445422172546}, {"head": 3, "memory_to_active": 0.3196566104888916, "contribution_to_active": 0.3253752887248993, "mix_mean": 0.20087552070617676, "mix_low": 0.016923904418945312, "mix_high": 3.0517578125e-05, "read_cosine": 0.6681971549987793, "beta_mean": 0.03082461841404438, "nonfinite": 0.0, "weighted_memory_to_active": 0.1469111144542694, "mix_std": 0.18815232813358307, "beta_std": 0.04266693815588951, "beta_low": 0.24310302734375, "beta_high": 0.0, "orthogonal_to_active": 0.011368166655302048, "evidence_weights": [-0.1109088584780693, -0.18375705182552338, 0.2839386463165283], "agreement_weight": 0.08295608311891556, "orthogonal_weight": -0.11608558148145676}, {"head": 4, "memory_to_active": 0.21291272342205048, "contribution_to_active": 0.44016486406326294, "mix_mean": 0.3223903179168701, "mix_low": 0.015011787414550781, "mix_high": 0.00013828277587890625, "read_cosine": 0.037660807371139526, "beta_mean": 0.06887800991535187, "nonfinite": 0.0, "weighted_memory_to_active": 0.09396519511938095, "mix_std": 0.25951534509658813, "beta_std": 0.11088510602712631, "beta_low": 0.28619384765625, "beta_high": 0.0, "orthogonal_to_active": 0.05372360721230507, "evidence_weights": [-0.25523388385772705, -0.25101548433303833, 0.1968355029821396], "agreement_weight": 0.007514473050832748, "orthogonal_weight": -0.6219038367271423}, {"head": 5, "memory_to_active": 0.0019058387260884047, "contribution_to_active": 0.41901683807373047, "mix_mean": 0.43523791432380676, "mix_low": 0.007975578308105469, "mix_high": 0.003987312316894531, "read_cosine": 0.49322372674942017, "beta_mean": 3.7356097891461104e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007358381408266723, "mix_std": 0.27917617559432983, "beta_std": 8.308440010296181e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00022704988077748567, "evidence_weights": [-0.3316251337528229, -0.27771368622779846, 0.39445367455482483], "agreement_weight": 0.05878537520766258, "orthogonal_weight": 0.3333180248737335}]}]}, {"step": 6000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7506341230000544, "memory_to_active": 1.1501312255859375, "contribution_to_active": 0.3547149896621704, "mix_mean": 0.194292813539505, "mix_low": 0.2134014070034027, "mix_high": 0.0029500725213438272, "read_cosine": 0.40345025062561035, "beta_mean": 0.11330358684062958, "nonfinite": 0.0, "weighted_memory_to_active": 0.20554053783416748, "mix_std": 0.1714438945055008, "beta_std": 0.08242225646972656, "beta_low": 0.3285910487174988, "beta_high": 3.814697265625e-06, "orthogonal_to_active": 0.02856726013123989, "samples": 4, "scored_tokens": 899, "normal_loss": 3.3207895755767822, "normal_token_accuracy": 0.41713014245033264, "state_active": 0.0722007006406784, "state_memory": 0.03637147322297096, "state_peak": 54.07975387573242, "growth_active": 1.1313872337341309, "growth_memory": 1.560211420059204, "output_influence": 0.24782542884349823, "residual_influence": 0.0731925517320633, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.4118380546569824, "no_memory_loss_delta": 0.0835932046175003, "scrambled_loss_delta": 0.28088271617889404, "memory_only_loss_delta": 1.494743824005127, "no_evidence_loss_delta": 0.028793523088097572, "no_agreement_loss_delta": 0.008212493732571602, "no_orthogonal_loss_delta": -0.0018445495516061783}, {"index": 94, "tokens": 256, "normal_loss": 3.4965455532073975, "no_memory_loss_delta": 0.22627301514148712, "scrambled_loss_delta": 0.3494059145450592, "memory_only_loss_delta": 1.538194179534912, "no_evidence_loss_delta": 0.08557530492544174, "no_agreement_loss_delta": 0.019100693985819817, "no_orthogonal_loss_delta": 0.005935262888669968}, {"index": 188, "tokens": 256, "normal_loss": 3.702439308166504, "no_memory_loss_delta": 0.04812925308942795, "scrambled_loss_delta": 0.1127128079533577, "memory_only_loss_delta": 1.6777193546295166, "no_evidence_loss_delta": 0.008569978177547455, "no_agreement_loss_delta": -0.0019783033058047295, "no_orthogonal_loss_delta": -0.011623259633779526}, {"index": 283, "tokens": 131, "normal_loss": 2.0535824298858643, "no_memory_loss_delta": 0.20315206050872803, "scrambled_loss_delta": 0.4622412919998169, "memory_only_loss_delta": 2.1974997520446777, "no_evidence_loss_delta": 0.06442447006702423, "no_agreement_loss_delta": 0.008276917971670628, "no_orthogonal_loss_delta": 0.005026411265134811}], "agreement_weight_abs_mean": 0.05767476558685303, "agreement_weight_abs_max": 0.44119638204574585, "agreement_weight_mean": 0.033569637686014175, "orthogonal_weight_abs_mean": 0.2606733441352844, "orthogonal_weight_abs_max": 1.0089025497436523, "orthogonal_weight_mean": -0.0994080975651741, "no_memory_loss": 3.452335834503174, "no_memory_loss_delta": 0.13154588639736176, "no_memory_token_accuracy_gain": 0.0033370412420481443, "no_memory_token_win_fraction": 0.5817574858665466, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 12.326095618137856, "scrambled_loss": 3.599724292755127, "scrambled_loss_delta": 0.2789343297481537, "scrambled_token_accuracy_gain": 0.031145717948675156, "scrambled_token_win_fraction": 0.6162402629852295, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 24.341041298762082, "memory_only_loss": 4.982414245605469, "memory_only_loss_delta": 1.6616246700286865, "memory_only_token_accuracy_gain": 0.17018909752368927, "memory_only_token_win_fraction": 0.8520578145980835, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 81.01696821956497, "no_evidence_loss": 3.3651857376098633, "no_evidence_loss_delta": 0.04439593479037285, "no_evidence_token_accuracy_gain": 0.0011123470030725002, "no_evidence_token_win_fraction": 0.535038948059082, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 4.342485889326385, "no_agreement_loss": 3.3292102813720703, "no_agreement_loss_delta": 0.008420472964644432, "no_agreement_token_accuracy_gain": -0.006674082484096289, "no_agreement_token_win_fraction": 0.4961067736148834, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 0.8385120081094375, "no_orthogonal_loss": 3.3193769454956055, "no_orthogonal_loss_delta": -0.0014125383459031582, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5428253412246704, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": -0.14135364480896948, "short_no_memory_loss_delta": 0.08818614482879639, "long_context_memory_gain": 0.04335974156856537, "normal_context_loss_gain": 0.12775981426239014, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.2366416454315186, "contribution_to_active": 0.6753870248794556, "mix_mean": 0.0934707447886467, "mix_low": 0.2190047949552536, "mix_high": 0.0009169578552246094, "read_cosine": 0.5507488250732422, "beta_mean": 0.25005632638931274, "nonfinite": 0.0, "weighted_memory_to_active": 0.6346194744110107, "mix_std": 0.08361473679542542, "beta_std": 0.11551162600517273, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.059928905218839645, "output_influence": 0.17288213968276978, "residual_influence": 0.23188193142414093, "state_active": 0.12116610258817673, "state_memory": 0.061854131519794464, "state_peak": 38.007347106933594, "growth_active": 1.3437917232513428, "growth_memory": 1.4361814260482788, "heads": [{"head": 0, "memory_to_active": 2.359395980834961, "contribution_to_active": 0.6061916351318359, "mix_mean": 0.031916067004203796, "mix_low": 0.20360183715820312, "mix_high": 0.0, "read_cosine": 0.46822965145111084, "beta_mean": 0.2136680781841278, "nonfinite": 0.0, "weighted_memory_to_active": 0.6664578318595886, "mix_std": 0.05509202927350998, "beta_std": 0.07578960061073303, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02219833992421627, "evidence_weights": [-0.05387447029352188, -0.06563016772270203, 0.12800870835781097], "agreement_weight": -0.041886039078235626, "orthogonal_weight": 0.04334450885653496}, {"head": 1, "memory_to_active": 2.291670560836792, "contribution_to_active": 1.1892706155776978, "mix_mean": 0.028677063062787056, "mix_low": 0.18857192993164062, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5457912683486938, "beta_mean": 0.3205859065055847, "nonfinite": 0.0, "weighted_memory_to_active": 1.2572070360183716, "mix_std": 0.056059256196022034, "beta_std": 0.11377814412117004, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11439789831638336, "evidence_weights": [-0.02331184595823288, -0.0659303218126297, 0.0573381707072258], "agreement_weight": -0.010493743233382702, "orthogonal_weight": 0.1670086532831192}, {"head": 2, "memory_to_active": 1.778523325920105, "contribution_to_active": 0.4418678879737854, "mix_mean": 0.022665709257125854, "mix_low": 0.3254060745239258, "mix_high": 0.0, "read_cosine": 0.46489471197128296, "beta_mean": 0.288343220949173, "nonfinite": 0.0, "weighted_memory_to_active": 0.45501959323883057, "mix_std": 0.037399694323539734, "beta_std": 0.1689956933259964, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0071331895887851715, "evidence_weights": [-0.02136814221739769, -0.019713642075657845, -0.08457914739847183], "agreement_weight": -0.019362514838576317, "orthogonal_weight": 0.02378237061202526}, {"head": 3, "memory_to_active": 3.1409752368927, "contribution_to_active": 1.3460445404052734, "mix_mean": 0.08976820111274719, "mix_low": 0.01882648468017578, "mix_high": 0.0, "read_cosine": 0.5150220394134521, "beta_mean": 0.2819576859474182, "nonfinite": 0.0, "weighted_memory_to_active": 1.2615381479263306, "mix_std": 0.090823233127594, "beta_std": 0.12529730796813965, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.19014324247837067, "evidence_weights": [-0.15275219082832336, -0.17254571616649628, 0.2353181093931198], "agreement_weight": -0.048349060118198395, "orthogonal_weight": 0.19932590425014496}, {"head": 4, "memory_to_active": 3.69240140914917, "contribution_to_active": 0.11275778710842133, "mix_mean": 0.012636534869670868, "mix_low": 0.5689678192138672, "mix_high": 0.0, "read_cosine": 0.364654541015625, "beta_mean": 0.29800841212272644, "nonfinite": 0.0, "weighted_memory_to_active": 0.0959794670343399, "mix_std": 0.014484858140349388, "beta_std": 0.14884629845619202, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.021037692204117775, "evidence_weights": [0.034431688487529755, 0.010034625418484211, 0.02279222570359707], "agreement_weight": -0.04318227991461754, "orthogonal_weight": 0.24833981692790985}, {"head": 5, "memory_to_active": 0.1568835973739624, "contribution_to_active": 0.3561895489692688, "mix_mean": 0.3751608729362488, "mix_low": 0.008654594421386719, "mix_high": 0.00550079345703125, "read_cosine": 0.9459006786346436, "beta_mean": 0.09777466952800751, "nonfinite": 0.0, "weighted_memory_to_active": 0.07151499390602112, "mix_std": 0.2478293478488922, "beta_std": 0.06036271154880524, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.004663038998842239, "evidence_weights": [-0.3424986004829407, -0.3498111963272095, 0.4520404636859894], "agreement_weight": 0.12932290136814117, "orthogonal_weight": 0.1832417994737625}]}, {"layer": 1, "memory_to_active": 2.770366668701172, "contribution_to_active": 0.27714455127716064, "mix_mean": 0.04521407559514046, "mix_low": 0.47451943159103394, "mix_high": 1.1126201115985168e-06, "read_cosine": 0.35387998819351196, "beta_mean": 0.28045371174812317, "nonfinite": 0.0, "weighted_memory_to_active": 0.2805330157279968, "mix_std": 0.056799184530973434, "beta_std": 0.1498354971408844, "beta_low": 0.0416361503303051, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.030295461416244507, "output_influence": 0.12254787236452103, "residual_influence": 0.025861617177724838, "state_active": 0.05725318193435669, "state_memory": 0.07421684265136719, "state_peak": 23.74420166015625, "growth_active": 1.0838102102279663, "growth_memory": 1.615990161895752, "heads": [{"head": 0, "memory_to_active": 1.7208306789398193, "contribution_to_active": 0.0519748218357563, "mix_mean": 0.011064686812460423, "mix_low": 0.7501392364501953, "mix_high": 0.0, "read_cosine": 0.05793372169137001, "beta_mean": 0.03698529675602913, "nonfinite": 0.0, "weighted_memory_to_active": 0.04335259273648262, "mix_std": 0.026276344433426857, "beta_std": 0.03496773913502693, "beta_low": 0.2467041015625, "beta_high": 0.0, "orthogonal_to_active": 0.010561658069491386, "evidence_weights": [0.0781959742307663, 0.05177001655101776, -0.14310549199581146], "agreement_weight": -0.12152176350355148, "orthogonal_weight": -0.2514124810695648}, {"head": 1, "memory_to_active": 3.573502540588379, "contribution_to_active": 0.17100420594215393, "mix_mean": 0.028157493099570274, "mix_low": 0.31924915313720703, "mix_high": 0.0, "read_cosine": 0.4831202030181885, "beta_mean": 0.454517662525177, "nonfinite": 0.0, "weighted_memory_to_active": 0.17632782459259033, "mix_std": 0.042087018489837646, "beta_std": 0.20985187590122223, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.007323109544813633, "evidence_weights": [-0.0501699224114418, -0.04686467722058296, 0.1295870989561081], "agreement_weight": -0.03440103307366371, "orthogonal_weight": 0.0498538464307785}, {"head": 2, "memory_to_active": 0.9723639488220215, "contribution_to_active": 0.37560394406318665, "mix_mean": 0.17494866251945496, "mix_low": 0.01598834991455078, "mix_high": 3.814697265625e-06, "read_cosine": 0.7209835052490234, "beta_mean": 0.16975703835487366, "nonfinite": 0.0, "weighted_memory_to_active": 0.49513357877731323, "mix_std": 0.1669679880142212, "beta_std": 0.0733131542801857, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0529407300055027, "evidence_weights": [-0.21616490185260773, -0.27708032727241516, 0.2300182282924652], "agreement_weight": -0.019140753895044327, "orthogonal_weight": 0.17672321200370789}, {"head": 3, "memory_to_active": 2.910731554031372, "contribution_to_active": 0.05103876069188118, "mix_mean": 0.008081404492259026, "mix_low": 0.7911767959594727, "mix_high": 0.0, "read_cosine": 0.19167667627334595, "beta_mean": 0.4053592085838318, "nonfinite": 0.0, "weighted_memory_to_active": 0.054768770933151245, "mix_std": 0.016857683658599854, "beta_std": 0.25642457604408264, "beta_low": 0.00018310546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.00179305300116539, "evidence_weights": [0.015202266164124012, 0.014308680780231953, -0.005825293716043234], "agreement_weight": -0.0143094127997756, "orthogonal_weight": -0.036538224667310715}, {"head": 4, "memory_to_active": 2.9625117778778076, "contribution_to_active": 0.9467634558677673, "mix_mean": 0.04118894040584564, "mix_low": 0.18642425537109375, "mix_high": 0.0, "read_cosine": 0.4305122196674347, "beta_mean": 0.23478536307811737, "nonfinite": 0.0, "weighted_memory_to_active": 0.849793553352356, "mix_std": 0.06444713473320007, "beta_std": 0.08589249849319458, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10784278064966202, "evidence_weights": [-0.05532814934849739, -0.05493694916367531, 0.2038234919309616], "agreement_weight": -0.03860142081975937, "orthogonal_weight": 0.13309642672538757}, {"head": 5, "memory_to_active": 4.4822587966918945, "contribution_to_active": 0.0664822906255722, "mix_mean": 0.007843253202736378, "mix_low": 0.7841386795043945, "mix_high": 2.86102294921875e-06, "read_cosine": 0.2390536516904831, "beta_mean": 0.38131779432296753, "nonfinite": 0.0, "weighted_memory_to_active": 0.06382164359092712, "mix_std": 0.02415889874100685, "beta_std": 0.2385631799697876, "beta_low": 0.0029296875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0013114383909851313, "evidence_weights": [0.06537032127380371, 0.04830218851566315, 0.1255781203508377], "agreement_weight": -0.09142269194126129, "orthogonal_weight": 0.021582061424851418}]}, {"layer": 2, "memory_to_active": 1.4163740873336792, "contribution_to_active": 0.21829518675804138, "mix_mean": 0.10700887441635132, "mix_low": 0.3806440234184265, "mix_high": 0.0007985433330759406, "read_cosine": 0.3722829222679138, "beta_mean": 0.111027792096138, "nonfinite": 0.0, "weighted_memory_to_active": 0.10811435431241989, "mix_std": 0.11683011054992676, "beta_std": 0.08405545353889465, "beta_low": 0.3851827383041382, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.005417584907263517, "output_influence": 0.1865079700946808, "residual_influence": 0.03465355187654495, "state_active": 0.03852546960115433, "state_memory": 0.037147458642721176, "state_peak": 41.48923873901367, "growth_active": 0.9650534987449646, "growth_memory": 1.648471713066101, "heads": [{"head": 0, "memory_to_active": 0.959440290927887, "contribution_to_active": 0.01896522007882595, "mix_mean": 0.00989689864218235, "mix_low": 0.7720317840576172, "mix_high": 0.0, "read_cosine": 0.2439780980348587, "beta_mean": 0.03248815983533859, "nonfinite": 0.0, "weighted_memory_to_active": 0.019061999395489693, "mix_std": 0.023814190179109573, "beta_std": 0.026760283857584, "beta_low": 0.20404052734375, "beta_high": 0.0, "orthogonal_to_active": 0.00175095081795007, "evidence_weights": [0.07818509638309479, 0.06934673339128494, -0.05253545939922333], "agreement_weight": -0.07233334332704544, "orthogonal_weight": -0.09624585509300232}, {"head": 1, "memory_to_active": 3.1669223308563232, "contribution_to_active": 0.07226933538913727, "mix_mean": 0.008745769038796425, "mix_low": 0.7572288513183594, "mix_high": 9.5367431640625e-07, "read_cosine": 0.1688317060470581, "beta_mean": 0.23660293221473694, "nonfinite": 0.0, "weighted_memory_to_active": 0.07027298957109451, "mix_std": 0.013998948037624359, "beta_std": 0.12435674667358398, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005872722249478102, "evidence_weights": [0.0475316047668457, 0.05513983964920044, 0.004692096263170242], "agreement_weight": -0.006055598612874746, "orthogonal_weight": 0.09219103306531906}, {"head": 2, "memory_to_active": 0.26836061477661133, "contribution_to_active": 0.3031828999519348, "mix_mean": 0.1947222799062729, "mix_low": 0.025943756103515625, "mix_high": 0.00024127960205078125, "read_cosine": 0.4458705186843872, "beta_mean": 0.0007590302266180515, "nonfinite": 0.0, "weighted_memory_to_active": 0.07588315010070801, "mix_std": 0.2073519229888916, "beta_std": 0.0013569167349487543, "beta_low": 0.9967041015625, "beta_high": 0.0, "orthogonal_to_active": 0.004978217650204897, "evidence_weights": [-0.12382148206233978, -0.1288079172372818, 0.20689207315444946], "agreement_weight": 0.07600130140781403, "orthogonal_weight": -0.08649497479200363}, {"head": 3, "memory_to_active": 0.6099952459335327, "contribution_to_active": 0.2579275667667389, "mix_mean": 0.08621925860643387, "mix_low": 0.09833812713623047, "mix_high": 0.0022630691528320312, "read_cosine": 0.5745178461074829, "beta_mean": 0.09246942400932312, "nonfinite": 0.0, "weighted_memory_to_active": 0.13635215163230896, "mix_std": 0.1298169493675232, "beta_std": 0.12877804040908813, "beta_low": 0.09539794921875, "beta_high": 0.0, "orthogonal_to_active": 0.015659280121326447, "evidence_weights": [-0.026033319532871246, -0.08309862017631531, 0.2959787845611572], "agreement_weight": 0.08831367641687393, "orthogonal_weight": -0.18897543847560883}, {"head": 4, "memory_to_active": 3.444967031478882, "contribution_to_active": 0.2829951047897339, "mix_mean": 0.01585523411631584, "mix_low": 0.6241321563720703, "mix_high": 0.0001239776611328125, "read_cosine": 0.21489614248275757, "beta_mean": 0.30374789237976074, "nonfinite": 0.0, "weighted_memory_to_active": 0.3266620934009552, "mix_std": 0.05429333075881004, "beta_std": 0.22288770973682404, "beta_low": 0.01495361328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0033874374348670244, "evidence_weights": [0.028238313272595406, 0.021895064041018486, 0.13307137787342072], "agreement_weight": 0.019106747582554817, "orthogonal_weight": 0.013096990995109081}, {"head": 5, "memory_to_active": 0.048558883368968964, "contribution_to_active": 0.37443098425865173, "mix_mean": 0.32661378383636475, "mix_low": 0.0061893463134765625, "mix_high": 0.0021619796752929688, "read_cosine": 0.5856032967567444, "beta_mean": 9.929834777722135e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.020453713834285736, "mix_std": 0.2717053294181824, "beta_std": 0.00019301344582345337, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008568998891860247, "evidence_weights": [-0.20146240293979645, -0.17896215617656708, 0.291640967130661], "agreement_weight": 0.0069165960885584354, "orthogonal_weight": 0.08954572677612305}]}, {"layer": 3, "memory_to_active": 1.7402076721191406, "contribution_to_active": 0.22672536969184875, "mix_mean": 0.06816691160202026, "mix_low": 0.4043750762939453, "mix_high": 0.0009051959495991468, "read_cosine": 0.3192402720451355, "beta_mean": 0.16928334534168243, "nonfinite": 0.0, "weighted_memory_to_active": 0.14424726366996765, "mix_std": 0.10861030220985413, "beta_std": 0.10572082549333572, "beta_low": 0.3196920156478882, "beta_high": 0.0, "orthogonal_to_active": 0.016802342608571053, "output_influence": 0.16465531289577484, "residual_influence": 0.023856764659285545, "state_active": 0.04156648367643356, "state_memory": 0.044452689588069916, "state_peak": 29.065059661865234, "growth_active": 0.9794695377349854, "growth_memory": 1.6546969413757324, "heads": [{"head": 0, "memory_to_active": 1.3637651205062866, "contribution_to_active": 0.11151200532913208, "mix_mean": 0.03255269676446915, "mix_low": 0.5527868270874023, "mix_high": 0.0, "read_cosine": 0.28028416633605957, "beta_mean": 0.02463388442993164, "nonfinite": 0.0, "weighted_memory_to_active": 0.10372070968151093, "mix_std": 0.07675063610076904, "beta_std": 0.017439931631088257, "beta_low": 0.10101318359375, "beta_high": 0.0, "orthogonal_to_active": 0.0019902954809367657, "evidence_weights": [0.042793065309524536, 0.010657895356416702, -0.091889888048172], "agreement_weight": -0.05952383950352669, "orthogonal_weight": -0.01939569227397442}, {"head": 1, "memory_to_active": 0.58905029296875, "contribution_to_active": 0.1355496197938919, "mix_mean": 0.054963722825050354, "mix_low": 0.3249673843383789, "mix_high": 0.0, "read_cosine": 0.3110579252243042, "beta_mean": 0.006193719804286957, "nonfinite": 0.0, "weighted_memory_to_active": 0.0431939959526062, "mix_std": 0.10337220877408981, "beta_std": 0.008483730256557465, "beta_low": 0.808837890625, "beta_high": 0.0, "orthogonal_to_active": 0.0035507301799952984, "evidence_weights": [-0.026380201801657677, -0.021977422758936882, 0.0633126050233841], "agreement_weight": -0.027266614139080048, "orthogonal_weight": -0.09675091505050659}, {"head": 2, "memory_to_active": 3.2937121391296387, "contribution_to_active": 0.25848284363746643, "mix_mean": 0.028796441853046417, "mix_low": 0.4393939971923828, "mix_high": 9.5367431640625e-07, "read_cosine": 0.23904401063919067, "beta_mean": 0.2846478521823883, "nonfinite": 0.0, "weighted_memory_to_active": 0.3405454754829407, "mix_std": 0.06867197155952454, "beta_std": 0.18472996354103088, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.043598953634500504, "evidence_weights": [-0.03010784462094307, -0.013490824028849602, 0.11877500265836716], "agreement_weight": -0.022654663771390915, "orthogonal_weight": -0.15772904455661774}, {"head": 3, "memory_to_active": 0.0837380662560463, "contribution_to_active": 0.325946182012558, "mix_mean": 0.20599615573883057, "mix_low": 0.05024433135986328, "mix_high": 0.0015954971313476562, "read_cosine": 0.32225480675697327, "beta_mean": 0.00021667274995706975, "nonfinite": 0.0, "weighted_memory_to_active": 0.027414318174123764, "mix_std": 0.22730129957199097, "beta_std": 0.0008413144387304783, "beta_low": 0.99871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.0035494077019393444, "evidence_weights": [-0.13057705760002136, -0.14057113230228424, 0.26933911442756653], "agreement_weight": 0.1929924339056015, "orthogonal_weight": -0.18115009367465973}, {"head": 4, "memory_to_active": 4.416756629943848, "contribution_to_active": 0.05568408966064453, "mix_mean": 0.006798305548727512, "mix_low": 0.8476419448852539, "mix_high": 0.0, "read_cosine": 0.23345224559307098, "beta_mean": 0.4677114486694336, "nonfinite": 0.0, "weighted_memory_to_active": 0.06251301616430283, "mix_std": 0.013886438682675362, "beta_std": 0.21992547810077667, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.009217948652803898, "evidence_weights": [0.11078009754419327, 0.10045166313648224, -0.15673163533210754], "agreement_weight": -0.11316706240177155, "orthogonal_weight": -0.14944009482860565}, {"head": 5, "memory_to_active": 0.6942234039306641, "contribution_to_active": 0.47317737340927124, "mix_mean": 0.07989414781332016, "mix_low": 0.21121597290039062, "mix_high": 0.0038347244262695312, "read_cosine": 0.5293483734130859, "beta_mean": 0.2322964370250702, "nonfinite": 0.0, "weighted_memory_to_active": 0.28809604048728943, "mix_std": 0.1616792231798172, "beta_std": 0.20290447771549225, "beta_low": 0.00909423828125, "beta_high": 0.0, "orthogonal_to_active": 0.03890671208500862, "evidence_weights": [-0.016364285722374916, -0.10219070315361023, 0.035626668483018875], "agreement_weight": 0.14381490647792816, "orthogonal_weight": -0.1972060203552246}]}, {"layer": 4, "memory_to_active": 1.3765230178833008, "contribution_to_active": 0.25163447856903076, "mix_mean": 0.15563416481018066, "mix_low": 0.24043957889080048, "mix_high": 0.0003032684326171875, "read_cosine": 0.4115995168685913, "beta_mean": 0.14154186844825745, "nonfinite": 0.0, "weighted_memory_to_active": 0.14956581592559814, "mix_std": 0.124868243932724, "beta_std": 0.10123094916343689, "beta_low": 0.1939595639705658, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.023338086903095245, "output_influence": 0.16298462450504303, "residual_influence": 0.03059900738298893, "state_active": 0.049422770738601685, "state_memory": 0.04621089622378349, "state_peak": 39.44211196899414, "growth_active": 1.0633151531219482, "growth_memory": 1.7336769104003906, "heads": [{"head": 0, "memory_to_active": 0.45913460850715637, "contribution_to_active": 0.39792701601982117, "mix_mean": 0.2510339915752411, "mix_low": 0.0120697021484375, "mix_high": 0.00020122528076171875, "read_cosine": 0.5996608138084412, "beta_mean": 0.06023059040307999, "nonfinite": 0.0, "weighted_memory_to_active": 0.23341816663742065, "mix_std": 0.2171589732170105, "beta_std": 0.07692931592464447, "beta_low": 0.1021728515625, "beta_high": 0.0, "orthogonal_to_active": 0.001634742715395987, "evidence_weights": [-0.12494807690382004, -0.16769033670425415, 0.2622773349285126], "agreement_weight": 0.07995525747537613, "orthogonal_weight": -0.010106383822858334}, {"head": 1, "memory_to_active": 0.7610450387001038, "contribution_to_active": 0.17729924619197845, "mix_mean": 0.1607409566640854, "mix_low": 0.052893638610839844, "mix_high": 0.0, "read_cosine": 0.49407029151916504, "beta_mean": 0.07232687622308731, "nonfinite": 0.0, "weighted_memory_to_active": 0.17065896093845367, "mix_std": 0.1650618016719818, "beta_std": 0.057300079613924026, "beta_low": 0.0123291015625, "beta_high": 0.0, "orthogonal_to_active": 0.03859248384833336, "evidence_weights": [-0.11627297848463058, -0.12201093882322311, 0.3581944704055786], "agreement_weight": 0.057723626494407654, "orthogonal_weight": -0.27666378021240234}, {"head": 2, "memory_to_active": 2.973937511444092, "contribution_to_active": 0.07587744295597076, "mix_mean": 0.01021263562142849, "mix_low": 0.7097787857055664, "mix_high": 0.0, "read_cosine": 0.29436641931533813, "beta_mean": 0.31205418705940247, "nonfinite": 0.0, "weighted_memory_to_active": 0.08522670716047287, "mix_std": 0.01569543033838272, "beta_std": 0.16411662101745605, "beta_low": 0.0, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0033037445973604918, "evidence_weights": [0.028606628999114037, 0.018747882917523384, -0.04975377023220062], "agreement_weight": 0.023439019918441772, "orthogonal_weight": 0.04836767539381981}, {"head": 3, "memory_to_active": 2.0368402004241943, "contribution_to_active": 0.08055683970451355, "mix_mean": 0.028197189792990685, "mix_low": 0.3365030288696289, "mix_high": 0.0, "read_cosine": 0.21393978595733643, "beta_mean": 0.17908835411071777, "nonfinite": 0.0, "weighted_memory_to_active": 0.1032833680510521, "mix_std": 0.041229795664548874, "beta_std": 0.15245988965034485, "beta_low": 0.04925537109375, "beta_high": 0.0, "orthogonal_to_active": 0.016106894239783287, "evidence_weights": [-0.025639446452260017, -0.001891939784400165, -0.12119483202695847], "agreement_weight": 0.009511311538517475, "orthogonal_weight": -0.1941293329000473}, {"head": 4, "memory_to_active": 0.04572153091430664, "contribution_to_active": 0.4525368809700012, "mix_mean": 0.4542979598045349, "mix_low": 0.0030813217163085938, "mix_high": 0.0016069412231445312, "read_cosine": 0.3805317282676697, "beta_mean": 0.00012094102567061782, "nonfinite": 0.0, "weighted_memory_to_active": 0.025885136798024178, "mix_std": 0.2643004059791565, "beta_std": 0.00020059198141098022, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002964908257126808, "evidence_weights": [-0.23051127791404724, -0.2395683079957962, 0.37393423914909363], "agreement_weight": -0.0011093361536040902, "orthogonal_weight": -0.14242713153362274}, {"head": 5, "memory_to_active": 1.9824600219726562, "contribution_to_active": 0.32560938596725464, "mix_mean": 0.029322190210223198, "mix_low": 0.3283109664916992, "mix_high": 1.1444091796875e-05, "read_cosine": 0.4870281219482422, "beta_mean": 0.22543029487133026, "nonfinite": 0.0, "weighted_memory_to_active": 0.27892252802848816, "mix_std": 0.04576302319765091, "beta_std": 0.15637923777103424, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0774257481098175, "evidence_weights": [-0.02088509500026703, -0.052034635096788406, 0.11024226993322372], "agreement_weight": -0.04895888268947601, "orthogonal_weight": 0.4014066457748413}]}, {"layer": 5, "memory_to_active": 0.8261054754257202, "contribution_to_active": 0.35632163286209106, "mix_mean": 0.2847311496734619, "mix_low": 0.1201726645231247, "mix_high": 0.004526774398982525, "read_cosine": 0.2522101402282715, "beta_mean": 0.08643469214439392, "nonfinite": 0.0, "weighted_memory_to_active": 0.17337676882743835, "mix_std": 0.2182009220123291, "beta_std": 0.08532403409481049, "beta_low": 0.3821614682674408, "beta_high": 0.0, "orthogonal_to_active": 0.02507658675312996, "output_influence": 0.31000572443008423, "residual_influence": 0.05589722469449043, "state_active": 0.07816661149263382, "state_memory": 0.039750806987285614, "state_peak": 54.07975387573242, "growth_active": 1.1281688213348389, "growth_memory": 1.7387332916259766, "heads": [{"head": 0, "memory_to_active": 1.7770740985870361, "contribution_to_active": 0.1849958598613739, "mix_mean": 0.05316777154803276, "mix_low": 0.22843170166015625, "mix_high": 0.0, "read_cosine": -0.053413938730955124, "beta_mean": 0.18843936920166016, "nonfinite": 0.0, "weighted_memory_to_active": 0.17558220028877258, "mix_std": 0.08978933840990067, "beta_std": 0.1671445220708847, "beta_low": 0.002197265625, "beta_high": 0.0, "orthogonal_to_active": 0.017170395702123642, "evidence_weights": [-0.031303055584430695, -0.03975912183523178, -0.16362564265727997], "agreement_weight": -0.01451430656015873, "orthogonal_weight": -0.10188081115484238}, {"head": 1, "memory_to_active": 0.014111937955021858, "contribution_to_active": 0.3124781548976898, "mix_mean": 0.27720654010772705, "mix_low": 0.053946495056152344, "mix_high": 0.0017061233520507812, "read_cosine": 0.13726137578487396, "beta_mean": 3.501596802379936e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004181453492492437, "mix_std": 0.2805931568145752, "beta_std": 0.00015193919534794986, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0019081386271864176, "evidence_weights": [-0.14736543595790863, -0.13248154520988464, 0.13172844052314758], "agreement_weight": 0.11983712017536163, "orthogonal_weight": -0.4798779785633087}, {"head": 2, "memory_to_active": 0.386200875043869, "contribution_to_active": 0.396232545375824, "mix_mean": 0.397162526845932, "mix_low": 0.004042625427246094, "mix_high": 0.0009002685546875, "read_cosine": 0.4256857931613922, "beta_mean": 0.025347255170345306, "nonfinite": 0.0, "weighted_memory_to_active": 0.2144365906715393, "mix_std": 0.2597957253456116, "beta_std": 0.015132617205381393, "beta_low": 0.07501220703125, "beta_high": 0.0, "orthogonal_to_active": 0.01006928738206625, "evidence_weights": [-0.2725382447242737, -0.2720295488834381, 0.43498486280441284], "agreement_weight": 0.0615168996155262, "orthogonal_weight": -0.05762334540486336}, {"head": 3, "memory_to_active": 2.1252903938293457, "contribution_to_active": 0.1518488973379135, "mix_mean": 0.036358967423439026, "mix_low": 0.3867607116699219, "mix_high": 0.0, "read_cosine": 0.22629615664482117, "beta_mean": 0.07832461595535278, "nonfinite": 0.0, "weighted_memory_to_active": 0.21206878125667572, "mix_std": 0.06681978702545166, "beta_std": 0.09782970696687698, "beta_low": 0.099609375, "beta_high": 0.0, "orthogonal_to_active": 0.04954121634364128, "evidence_weights": [0.012662255205214024, -0.008444122970104218, 0.09328042715787888], "agreement_weight": 0.03865901380777359, "orthogonal_weight": -0.2839829921722412}, {"head": 4, "memory_to_active": 0.5995700359344482, "contribution_to_active": 0.5319624543190002, "mix_mean": 0.3535483479499817, "mix_low": 0.04476165771484375, "mix_high": 0.011965751647949219, "read_cosine": 0.6138018369674683, "beta_mean": 0.22628545761108398, "nonfinite": 0.0, "weighted_memory_to_active": 0.4049263000488281, "mix_std": 0.3237336575984955, "beta_std": 0.23137353360652924, "beta_low": 0.11627197265625, "beta_high": 0.0, "orthogonal_to_active": 0.06401468813419342, "evidence_weights": [-0.16694791615009308, -0.22776100039482117, 0.2633741796016693], "agreement_weight": 0.061290644109249115, "orthogonal_weight": 0.2411242574453354}, {"head": 5, "memory_to_active": 0.05438556149601936, "contribution_to_active": 0.5604120492935181, "mix_mean": 0.5909425616264343, "mix_low": 0.0030927658081054688, "mix_high": 0.0125885009765625, "read_cosine": 0.1636296659708023, "beta_mean": 0.00017637036216910928, "nonfinite": 0.0, "weighted_memory_to_active": 0.02906527742743492, "mix_std": 0.28847384452819824, "beta_std": 0.0003118498716503382, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00775578897446394, "evidence_weights": [-0.255942702293396, -0.25341901183128357, 0.4141770005226135], "agreement_weight": 0.1597120612859726, "orthogonal_weight": -0.3038930892944336}]}, {"layer": 6, "memory_to_active": 0.6320571899414062, "contribution_to_active": 0.4504561424255371, "mix_mean": 0.3011707663536072, "mix_low": 0.13781914114952087, "mix_high": 0.00961033534258604, "read_cosine": 0.35491228103637695, "beta_mean": 0.07554733008146286, "nonfinite": 0.0, "weighted_memory_to_active": 0.19038157165050507, "mix_std": 0.24859774112701416, "beta_std": 0.08466005325317383, "beta_low": 0.412689208984375, "beta_high": 0.0, "orthogonal_to_active": 0.03141693025827408, "output_influence": 0.3463566303253174, "residual_influence": 0.07804683595895767, "state_active": 0.08081213384866714, "state_memory": 0.023470833897590637, "state_peak": 30.11511993408203, "growth_active": 1.2889810800552368, "growth_memory": 1.5468008518218994, "heads": [{"head": 0, "memory_to_active": 0.353147953748703, "contribution_to_active": 0.4288002848625183, "mix_mean": 0.2220708578824997, "mix_low": 0.06615066528320312, "mix_high": 0.0010509490966796875, "read_cosine": 0.3783038258552551, "beta_mean": 0.04171483591198921, "nonfinite": 0.0, "weighted_memory_to_active": 0.2274661660194397, "mix_std": 0.22623586654663086, "beta_std": 0.08869695663452148, "beta_low": 0.19195556640625, "beta_high": 0.0, "orthogonal_to_active": 0.07792322337627411, "evidence_weights": [-0.1170324832201004, -0.1357291340827942, 0.16281433403491974], "agreement_weight": 0.10623464733362198, "orthogonal_weight": -0.5595391392707825}, {"head": 1, "memory_to_active": 0.08541178703308105, "contribution_to_active": 0.33100229501724243, "mix_mean": 0.24474409222602844, "mix_low": 0.059391021728515625, "mix_high": 0.006443977355957031, "read_cosine": 0.4655093252658844, "beta_mean": 0.0001878286711871624, "nonfinite": 0.0, "weighted_memory_to_active": 0.021858226507902145, "mix_std": 0.25862303376197815, "beta_std": 0.0003727775765582919, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0035819492768496275, "evidence_weights": [-0.16244931519031525, -0.14802855253219604, 0.1517077535390854], "agreement_weight": -0.005512636620551348, "orthogonal_weight": -0.1978043019771576}, {"head": 2, "memory_to_active": 2.3602852821350098, "contribution_to_active": 0.08428176492452621, "mix_mean": 0.018221141770482063, "mix_low": 0.627192497253418, "mix_high": 5.7220458984375e-06, "read_cosine": 0.235589861869812, "beta_mean": 0.16466043889522552, "nonfinite": 0.0, "weighted_memory_to_active": 0.10766662657260895, "mix_std": 0.05239458754658699, "beta_std": 0.12626013159751892, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.0238826684653759, "evidence_weights": [0.07532214373350143, 0.0656425952911377, -0.09126190841197968], "agreement_weight": 0.01702459529042244, "orthogonal_weight": -0.23416632413864136}, {"head": 3, "memory_to_active": 0.251880943775177, "contribution_to_active": 0.5657284259796143, "mix_mean": 0.4641353487968445, "mix_low": 0.011937141418457031, "mix_high": 0.021338462829589844, "read_cosine": 0.5769707560539246, "beta_mean": 0.1467849165201187, "nonfinite": 0.0, "weighted_memory_to_active": 0.15249519050121307, "mix_std": 0.3427245318889618, "beta_std": 0.17135031521320343, "beta_low": 0.15325927734375, "beta_high": 0.0, "orthogonal_to_active": 0.030658161267638206, "evidence_weights": [-0.11838822066783905, -0.23715756833553314, 0.30418625473976135], "agreement_weight": -0.013824418187141418, "orthogonal_weight": 0.25626543164253235}, {"head": 4, "memory_to_active": 0.6724341511726379, "contribution_to_active": 0.8934895396232605, "mix_mean": 0.5480506420135498, "mix_low": 0.004216194152832031, "mix_high": 0.028761863708496094, "read_cosine": 0.1885479986667633, "beta_mean": 0.09953291714191437, "nonfinite": 0.0, "weighted_memory_to_active": 0.6067042946815491, "mix_std": 0.32785382866859436, "beta_std": 0.12077648192644119, "beta_low": 0.13092041015625, "beta_high": 0.0, "orthogonal_to_active": 0.031084004789590836, "evidence_weights": [-0.2127297818660736, -0.3767490088939667, -0.12551464140415192], "agreement_weight": -0.009793700650334358, "orthogonal_weight": 0.06324542313814163}, {"head": 5, "memory_to_active": 0.06918324530124664, "contribution_to_active": 0.3994346559047699, "mix_mean": 0.3098025918006897, "mix_low": 0.05802726745605469, "mix_high": 6.103515625e-05, "read_cosine": 0.2845517098903656, "beta_mean": 0.0004030630807392299, "nonfinite": 0.0, "weighted_memory_to_active": 0.026098866015672684, "mix_std": 0.28375446796417236, "beta_std": 0.0005036871298216283, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.021371573209762573, "evidence_weights": [-0.13487155735492706, -0.17889618873596191, -0.17300564050674438], "agreement_weight": 0.10113127529621124, "orthogonal_weight": -0.8745915293693542}]}, {"layer": 7, "memory_to_active": 2.3702733516693115, "contribution_to_active": 0.1566561907529831, "mix_mean": 0.023272182792425156, "mix_low": 0.5814632177352905, "mix_high": 5.086263172415784e-06, "read_cosine": 0.24407486617565155, "beta_mean": 0.18950927257537842, "nonfinite": 0.0, "weighted_memory_to_active": 0.16398420929908752, "mix_std": 0.044580087065696716, "beta_std": 0.12087244540452957, "beta_low": 0.0022481284104287624, "beta_high": 0.0, "orthogonal_to_active": 0.0161355622112751, "output_influence": 0.15676875412464142, "residual_influence": 0.029576292261481285, "state_active": 0.022119872272014618, "state_memory": 0.044659994542598724, "state_peak": 16.911224365234375, "growth_active": 0.98597651720047, "growth_memory": 1.4991264343261719, "heads": [{"head": 0, "memory_to_active": 2.0443849563598633, "contribution_to_active": 0.30871254205703735, "mix_mean": 0.02920663356781006, "mix_low": 0.4679107666015625, "mix_high": 2.193450927734375e-05, "read_cosine": 0.3930314779281616, "beta_mean": 0.16681545972824097, "nonfinite": 0.0, "weighted_memory_to_active": 0.3015134036540985, "mix_std": 0.05964191257953644, "beta_std": 0.11568555235862732, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05847333371639252, "evidence_weights": [-0.03465116396546364, -0.06792773306369781, 0.12031113356351852], "agreement_weight": -0.0361512191593647, "orthogonal_weight": 0.28075265884399414}, {"head": 1, "memory_to_active": 3.024257183074951, "contribution_to_active": 0.22559967637062073, "mix_mean": 0.04274652153253555, "mix_low": 0.2889842987060547, "mix_high": 0.0, "read_cosine": 0.017225392162799835, "beta_mean": 0.2672964930534363, "nonfinite": 0.0, "weighted_memory_to_active": 0.2558809518814087, "mix_std": 0.05905436724424362, "beta_std": 0.13185979425907135, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03308044746518135, "evidence_weights": [-0.07459346204996109, -0.07287188619375229, 0.09279993921518326], "agreement_weight": -0.008824403397738934, "orthogonal_weight": -0.1330886334180832}, {"head": 2, "memory_to_active": 1.6349971294403076, "contribution_to_active": 0.23495042324066162, "mix_mean": 0.03902725130319595, "mix_low": 0.47932910919189453, "mix_high": 8.58306884765625e-06, "read_cosine": 0.2269911915063858, "beta_mean": 0.17477750778198242, "nonfinite": 0.0, "weighted_memory_to_active": 0.23679548501968384, "mix_std": 0.08946532756090164, "beta_std": 0.09514115750789642, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 1.14450149339973e-05, "evidence_weights": [-0.04778901860117912, -0.09132277220487595, 0.18166591227054596], "agreement_weight": 0.019979804754257202, "orthogonal_weight": 5.556422547670081e-05}, {"head": 3, "memory_to_active": 3.201007127761841, "contribution_to_active": 0.02817128412425518, "mix_mean": 0.006076945923268795, "mix_low": 0.83465576171875, "mix_high": 0.0, "read_cosine": 0.16741913557052612, "beta_mean": 0.21824952960014343, "nonfinite": 0.0, "weighted_memory_to_active": 0.03307610750198364, "mix_std": 0.010847728699445724, "beta_std": 0.14020831882953644, "beta_low": 0.00140380859375, "beta_high": 0.0, "orthogonal_to_active": 0.004873924423009157, "evidence_weights": [0.08430232852697372, 0.09003476798534393, -0.032177288085222244], "agreement_weight": -0.0025463271886110306, "orthogonal_weight": -0.14949417114257812}, {"head": 4, "memory_to_active": 1.6382195949554443, "contribution_to_active": 0.06147422641515732, "mix_mean": 0.012510593049228191, "mix_low": 0.6781883239746094, "mix_high": 0.0, "read_cosine": 0.3117321729660034, "beta_mean": 0.16891160607337952, "nonfinite": 0.0, "weighted_memory_to_active": 0.06699701398611069, "mix_std": 0.029517194256186485, "beta_std": 0.132720947265625, "beta_low": 0.0072021484375, "beta_high": 0.0, "orthogonal_to_active": 4.638941027224064e-05, "evidence_weights": [0.035096924751996994, 0.028739694505929947, 0.003818112425506115], "agreement_weight": 0.056800369173288345, "orthogonal_weight": 0.000992669491097331}, {"head": 5, "memory_to_active": 2.67877459526062, "contribution_to_active": 0.0810289978981018, "mix_mean": 0.010065145790576935, "mix_low": 0.739710807800293, "mix_high": 0.0, "read_cosine": 0.3480498790740967, "beta_mean": 0.14100497961044312, "nonfinite": 0.0, "weighted_memory_to_active": 0.08964226394891739, "mix_std": 0.018953990191221237, "beta_std": 0.1096188947558403, "beta_low": 0.0048828125, "beta_high": 0.0, "orthogonal_to_active": 0.00032782740890979767, "evidence_weights": [0.022915204986929893, 0.04745955392718315, -0.15943463146686554], "agreement_weight": -0.011139502748847008, "orthogonal_weight": -0.004518997389823198}]}, {"layer": 8, "memory_to_active": 1.125461220741272, "contribution_to_active": 0.3032422363758087, "mix_mean": 0.1999495029449463, "mix_low": 0.12559270858764648, "mix_high": 0.002270857570692897, "read_cosine": 0.5277831554412842, "beta_mean": 0.10695744305849075, "nonfinite": 0.0, "weighted_memory_to_active": 0.1979185789823532, "mix_std": 0.16421979665756226, "beta_std": 0.08196592330932617, "beta_low": 0.204071044921875, "beta_high": 0.0, "orthogonal_to_active": 0.0343388170003891, "output_influence": 0.25651100277900696, "residual_influence": 0.052973438054323196, "state_active": 0.06475768983364105, "state_memory": 0.03784827142953873, "state_peak": 22.43187713623047, "growth_active": 1.0032495260238647, "growth_memory": 1.5789966583251953, "heads": [{"head": 0, "memory_to_active": 0.09915582090616226, "contribution_to_active": 0.407777339220047, "mix_mean": 0.3760891258716583, "mix_low": 0.0011758804321289062, "mix_high": 1.1444091796875e-05, "read_cosine": 0.7341094017028809, "beta_mean": 0.003399931825697422, "nonfinite": 0.0, "weighted_memory_to_active": 0.04584239795804024, "mix_std": 0.21874982118606567, "beta_std": 0.004467296879738569, "beta_low": 0.94091796875, "beta_high": 0.0, "orthogonal_to_active": 0.010579632595181465, "evidence_weights": [-0.24883933365345, -0.24475757777690887, 0.34265366196632385], "agreement_weight": 0.06463245302438736, "orthogonal_weight": -0.3912643492221832}, {"head": 1, "memory_to_active": 0.897010862827301, "contribution_to_active": 0.30976542830467224, "mix_mean": 0.26724332571029663, "mix_low": 0.0065670013427734375, "mix_high": 6.67572021484375e-06, "read_cosine": 0.6640444993972778, "beta_mean": 0.12231746315956116, "nonfinite": 0.0, "weighted_memory_to_active": 0.3192821145057678, "mix_std": 0.17850393056869507, "beta_std": 0.07278072088956833, "beta_low": 0.00213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.0353272520005703, "evidence_weights": [-0.17850783467292786, -0.16569647192955017, 0.3109239935874939], "agreement_weight": -0.001517946133390069, "orthogonal_weight": 0.15907350182533264}, {"head": 2, "memory_to_active": 2.315487861633301, "contribution_to_active": 0.16780999302864075, "mix_mean": 0.029317066073417664, "mix_low": 0.3930320739746094, "mix_high": 0.0, "read_cosine": 0.2440115511417389, "beta_mean": 0.2277347445487976, "nonfinite": 0.0, "weighted_memory_to_active": 0.2469979077577591, "mix_std": 0.05168838053941727, "beta_std": 0.14611956477165222, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04230710119009018, "evidence_weights": [0.0016741304425522685, -0.0356341227889061, 0.0030116750858724117], "agreement_weight": 0.08386887609958649, "orthogonal_weight": -0.21003641188144684}, {"head": 3, "memory_to_active": 0.7993596792221069, "contribution_to_active": 0.18131154775619507, "mix_mean": 0.14404848217964172, "mix_low": 0.04300403594970703, "mix_high": 0.0, "read_cosine": 0.6355818510055542, "beta_mean": 0.05605658143758774, "nonfinite": 0.0, "weighted_memory_to_active": 0.1856909990310669, "mix_std": 0.13141319155693054, "beta_std": 0.04674804210662842, "beta_low": 0.04229736328125, "beta_high": 0.0, "orthogonal_to_active": 0.035152681171894073, "evidence_weights": [-0.10055024921894073, -0.11214210093021393, 0.42196083068847656], "agreement_weight": 0.021171757951378822, "orthogonal_weight": 0.2863180637359619}, {"head": 4, "memory_to_active": 2.419646739959717, "contribution_to_active": 0.24000021815299988, "mix_mean": 0.04233921319246292, "mix_low": 0.20932292938232422, "mix_high": 0.0, "read_cosine": 0.344626247882843, "beta_mean": 0.14371168613433838, "nonfinite": 0.0, "weighted_memory_to_active": 0.2579030990600586, "mix_std": 0.05637349560856819, "beta_std": 0.08881805837154388, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.0070879291743040085, "evidence_weights": [-0.019487174227833748, -0.04087986424565315, 0.029027778655290604], "agreement_weight": 0.0007338583818636835, "orthogonal_weight": 0.03185734897851944}, {"head": 5, "memory_to_active": 0.22210590541362762, "contribution_to_active": 0.5127887725830078, "mix_mean": 0.34065982699394226, "mix_low": 0.10045433044433594, "mix_high": 0.013607025146484375, "read_cosine": 0.5443251729011536, "beta_mean": 0.08852426707744598, "nonfinite": 0.0, "weighted_memory_to_active": 0.13179486989974976, "mix_std": 0.3485899567604065, "beta_std": 0.13286183774471283, "beta_low": 0.2388916015625, "beta_high": 0.0, "orthogonal_to_active": 0.07557829469442368, "evidence_weights": [-0.11803817749023438, -0.3041973114013672, -0.14961501955986023], "agreement_weight": 0.025226525962352753, "orthogonal_weight": 0.732049286365509}]}, {"layer": 9, "memory_to_active": 1.3689205646514893, "contribution_to_active": 0.4477904438972473, "mix_mean": 0.10639694333076477, "mix_low": 0.33398008346557617, "mix_high": 0.0026041667442768812, "read_cosine": 0.3591121435165405, "beta_mean": 0.11859237402677536, "nonfinite": 0.0, "weighted_memory_to_active": 0.37292370200157166, "mix_std": 0.17311224341392517, "beta_std": 0.06321458518505096, "beta_low": 0.3727925717830658, "beta_high": 0.0, "orthogonal_to_active": 0.010267013683915138, "output_influence": 0.2528792917728424, "residual_influence": 0.05143433436751366, "state_active": 0.033581845462322235, "state_memory": 0.038684844970703125, "state_peak": 35.17398452758789, "growth_active": 0.9986599683761597, "growth_memory": 1.5989104509353638, "heads": [{"head": 0, "memory_to_active": 0.24910177290439606, "contribution_to_active": 0.2298101782798767, "mix_mean": 0.18158726394176483, "mix_low": 0.18100357055664062, "mix_high": 0.00010013580322265625, "read_cosine": 0.2615835666656494, "beta_mean": 0.00117771839722991, "nonfinite": 0.0, "weighted_memory_to_active": 0.05398579314351082, "mix_std": 0.24058419466018677, "beta_std": 0.0025049515534192324, "beta_low": 0.9822998046875, "beta_high": 0.0, "orthogonal_to_active": 0.016977662220597267, "evidence_weights": [-0.08710811287164688, -0.10623639822006226, 0.45469242334365845], "agreement_weight": 0.03519998490810394, "orthogonal_weight": -0.3352236747741699}, {"head": 1, "memory_to_active": 0.11101384460926056, "contribution_to_active": 0.23107433319091797, "mix_mean": 0.08753259479999542, "mix_low": 0.4184846878051758, "mix_high": 5.340576171875e-05, "read_cosine": 0.3959181308746338, "beta_mean": 0.00045528364717029035, "nonfinite": 0.0, "weighted_memory_to_active": 0.022124316543340683, "mix_std": 0.16964121162891388, "beta_std": 0.0009145158692263067, "beta_low": 0.9986572265625, "beta_high": 0.0, "orthogonal_to_active": 0.003908669110387564, "evidence_weights": [-0.07554454356431961, -0.0645802766084671, 0.13463297486305237], "agreement_weight": 0.1406046748161316, "orthogonal_weight": -0.18553867936134338}, {"head": 2, "memory_to_active": 0.7315715551376343, "contribution_to_active": 0.35182225704193115, "mix_mean": 0.1334909349679947, "mix_low": 0.2507305145263672, "mix_high": 2.86102294921875e-06, "read_cosine": 0.24400757253170013, "beta_mean": 0.027964681386947632, "nonfinite": 0.0, "weighted_memory_to_active": 0.20269818603992462, "mix_std": 0.20484724640846252, "beta_std": 0.03287488967180252, "beta_low": 0.25518798828125, "beta_high": 0.0, "orthogonal_to_active": 0.0032389257103204727, "evidence_weights": [-0.07214994728565216, -0.06383531540632248, 0.09452223777770996], "agreement_weight": 0.07414484024047852, "orthogonal_weight": -0.01914934441447258}, {"head": 3, "memory_to_active": 2.19553279876709, "contribution_to_active": 0.27708980441093445, "mix_mean": 0.05015646666288376, "mix_low": 0.31673431396484375, "mix_high": 0.0, "read_cosine": 0.4356832504272461, "beta_mean": 0.18508419394493103, "nonfinite": 0.0, "weighted_memory_to_active": 0.2569764256477356, "mix_std": 0.08857251703739166, "beta_std": 0.11045487970113754, "beta_low": 0.0006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.005771719850599766, "evidence_weights": [0.020992213860154152, 0.014828230254352093, -0.012523514218628407], "agreement_weight": -0.041783034801483154, "orthogonal_weight": 0.02156364545226097}, {"head": 4, "memory_to_active": 3.1620030403137207, "contribution_to_active": 1.177128553390503, "mix_mean": 0.1357099711894989, "mix_low": 0.2587614059448242, "mix_high": 0.015448570251464844, "read_cosine": 0.38149648904800415, "beta_mean": 0.25681430101394653, "nonfinite": 0.0, "weighted_memory_to_active": 1.188117265701294, "mix_std": 0.21671462059020996, "beta_std": 0.10982190072536469, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017065065912902355, "evidence_weights": [-0.042077966034412384, -0.06987455487251282, 0.11180917173624039], "agreement_weight": -0.06312545388936996, "orthogonal_weight": -0.0014986599562689662}, {"head": 5, "memory_to_active": 1.7642998695373535, "contribution_to_active": 0.41981735825538635, "mix_mean": 0.0499044694006443, "mix_low": 0.5781660079956055, "mix_high": 2.002716064453125e-05, "read_cosine": 0.4359837472438812, "beta_mean": 0.2400580197572708, "nonfinite": 0.0, "weighted_memory_to_active": 0.513640284538269, "mix_std": 0.11831370741128922, "beta_std": 0.12271638214588165, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.029998600482940674, "evidence_weights": [0.02735447697341442, -0.01729171723127365, 0.024291981011629105], "agreement_weight": -0.02175571396946907, "orthogonal_weight": 0.07714686542749405}]}, {"layer": 10, "memory_to_active": 0.5866498947143555, "contribution_to_active": 0.36878877878189087, "mix_mean": 0.2033020555973053, "mix_low": 0.18539810180664062, "mix_high": 0.0032280287705361843, "read_cosine": 0.36847108602523804, "beta_mean": 0.05828491225838661, "nonfinite": 0.0, "weighted_memory_to_active": 0.1697668731212616, "mix_std": 0.2021503895521164, "beta_std": 0.065329909324646, "beta_low": 0.4986979365348816, "beta_high": 0.0, "orthogonal_to_active": 0.04018732160329819, "output_influence": 0.34003108739852905, "residual_influence": 0.08452311158180237, "state_active": 0.08607500046491623, "state_memory": 0.02452358603477478, "state_peak": 46.26109313964844, "growth_active": 1.1659324169158936, "growth_memory": 1.458715796470642, "heads": [{"head": 0, "memory_to_active": 0.7731789946556091, "contribution_to_active": 0.8980423212051392, "mix_mean": 0.5088049173355103, "mix_low": 0.026206016540527344, "mix_high": 0.017454147338867188, "read_cosine": 0.12978166341781616, "beta_mean": 0.1473011076450348, "nonfinite": 0.0, "weighted_memory_to_active": 0.4948388934135437, "mix_std": 0.3444842994213104, "beta_std": 0.17801521718502045, "beta_low": 0.1214599609375, "beta_high": 0.0, "orthogonal_to_active": 0.14418989419937134, "evidence_weights": [-0.13417954742908478, -0.27579376101493835, -0.05965283140540123], "agreement_weight": 0.017951877787709236, "orthogonal_weight": 0.31085866689682007}, {"head": 1, "memory_to_active": 0.34737133979797363, "contribution_to_active": 0.19388943910598755, "mix_mean": 0.10598304867744446, "mix_low": 0.20306682586669922, "mix_high": 0.00031948089599609375, "read_cosine": 0.19705894589424133, "beta_mean": 0.0030058177653700113, "nonfinite": 0.0, "weighted_memory_to_active": 0.04128788039088249, "mix_std": 0.16748648881912231, "beta_std": 0.005147979129105806, "beta_low": 0.941650390625, "beta_high": 0.0, "orthogonal_to_active": 0.012536962516605854, "evidence_weights": [-0.06688316911458969, -0.06101502850651741, -0.039522890001535416], "agreement_weight": 0.038474373519420624, "orthogonal_weight": -0.31991517543792725}, {"head": 2, "memory_to_active": 1.9633172750473022, "contribution_to_active": 0.2833035886287689, "mix_mean": 0.01794080063700676, "mix_low": 0.6762847900390625, "mix_high": 0.0, "read_cosine": 0.19686681032180786, "beta_mean": 0.11859679222106934, "nonfinite": 0.0, "weighted_memory_to_active": 0.32697996497154236, "mix_std": 0.048731449991464615, "beta_std": 0.09261701256036758, "beta_low": 0.0042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.03988206759095192, "evidence_weights": [0.04096674919128418, 0.00852709636092186, 0.01931823417544365], "agreement_weight": -0.028883911669254303, "orthogonal_weight": -0.12571533024311066}, {"head": 3, "memory_to_active": 0.15893372893333435, "contribution_to_active": 0.2915724515914917, "mix_mean": 0.2486918419599533, "mix_low": 0.09576225280761719, "mix_high": 0.0015430450439453125, "read_cosine": 0.7262445092201233, "beta_mean": 0.07139795273542404, "nonfinite": 0.0, "weighted_memory_to_active": 0.07083290070295334, "mix_std": 0.2897076904773712, "beta_std": 0.1027575135231018, "beta_low": 0.198974609375, "beta_high": 0.0, "orthogonal_to_active": 0.030119623988866806, "evidence_weights": [-0.12922953069210052, -0.1597435176372528, 0.25865957140922546], "agreement_weight": 0.019125914201140404, "orthogonal_weight": 0.5617778897285461}, {"head": 4, "memory_to_active": 0.10409782081842422, "contribution_to_active": 0.18425004184246063, "mix_mean": 0.12895075976848602, "mix_low": 0.09836196899414062, "mix_high": 1.71661376953125e-05, "read_cosine": 0.22250786423683167, "beta_mean": 0.000550873635802418, "nonfinite": 0.0, "weighted_memory_to_active": 0.01757797598838806, "mix_std": 0.1689959019422531, "beta_std": 0.0019580733496695757, "beta_low": 0.9962158203125, "beta_high": 0.0, "orthogonal_to_active": 0.004099885933101177, "evidence_weights": [-0.10103296488523483, -0.07270466536283493, -0.06521419435739517], "agreement_weight": 0.03401647135615349, "orthogonal_weight": -0.26468464732170105}, {"head": 5, "memory_to_active": 0.17300021648406982, "contribution_to_active": 0.36167457699775696, "mix_mean": 0.20944087207317352, "mix_low": 0.012706756591796875, "mix_high": 3.4332275390625e-05, "read_cosine": 0.7383666634559631, "beta_mean": 0.008856933563947678, "nonfinite": 0.0, "weighted_memory_to_active": 0.06708358228206635, "mix_std": 0.19349654018878937, "beta_std": 0.011483668349683285, "beta_low": 0.7296142578125, "beta_high": 0.0, "orthogonal_to_active": 0.010295495390892029, "evidence_weights": [-0.16462872922420502, -0.1318921446800232, 0.15112075209617615], "agreement_weight": 0.08607882261276245, "orthogonal_weight": -0.3127420246601105}]}, {"layer": 11, "memory_to_active": 0.35580453276634216, "contribution_to_active": 0.3874417245388031, "mix_mean": 0.2851267457008362, "mix_low": 0.05775706097483635, "mix_high": 0.017757734283804893, "read_cosine": 0.5261729955673218, "beta_mean": 0.04278787970542908, "nonfinite": 0.0, "weighted_memory_to_active": 0.0980553925037384, "mix_std": 0.2679203152656555, "beta_std": 0.06197482347488403, "beta_low": 0.4277649223804474, "beta_high": 0.0, "orthogonal_to_active": 0.022305866703391075, "output_influence": 0.4875495731830597, "residual_influence": 0.12100948393344879, "state_active": 0.1288890391588211, "state_memory": 0.017920855432748795, "state_peak": 22.646541595458984, "growth_active": 1.2115401029586792, "growth_memory": 1.438369870185852, "heads": [{"head": 0, "memory_to_active": 0.11392594873905182, "contribution_to_active": 0.42449137568473816, "mix_mean": 0.30943119525909424, "mix_low": 0.02225017547607422, "mix_high": 0.0004901885986328125, "read_cosine": 0.4389839768409729, "beta_mean": 0.0007494747987948358, "nonfinite": 0.0, "weighted_memory_to_active": 0.058371562510728836, "mix_std": 0.2578042149543762, "beta_std": 0.004544328898191452, "beta_low": 0.994384765625, "beta_high": 0.0, "orthogonal_to_active": 0.02137218602001667, "evidence_weights": [-0.13875927031040192, -0.17428505420684814, 0.2133067399263382], "agreement_weight": 0.13242970407009125, "orthogonal_weight": -0.5267652273178101}, {"head": 1, "memory_to_active": 0.6306602954864502, "contribution_to_active": 0.17880232632160187, "mix_mean": 0.1676413118839264, "mix_low": 0.049920082092285156, "mix_high": 3.814697265625e-05, "read_cosine": 0.5863109827041626, "beta_mean": 0.0428546667098999, "nonfinite": 0.0, "weighted_memory_to_active": 0.1540507972240448, "mix_std": 0.19121475517749786, "beta_std": 0.06925193220376968, "beta_low": 0.29345703125, "beta_high": 0.0, "orthogonal_to_active": 0.009745920076966286, "evidence_weights": [-0.11560743302106857, -0.08251597732305527, 0.06806623190641403], "agreement_weight": 0.09168346971273422, "orthogonal_weight": -0.08684154599905014}, {"head": 2, "memory_to_active": 0.07831236720085144, "contribution_to_active": 0.7866615653038025, "mix_mean": 0.44676876068115234, "mix_low": 0.056601524353027344, "mix_high": 0.06098365783691406, "read_cosine": 0.35289210081100464, "beta_mean": 0.07022064924240112, "nonfinite": 0.0, "weighted_memory_to_active": 0.06499464809894562, "mix_std": 0.3986266851425171, "beta_std": 0.10585559904575348, "beta_low": 0.43670654296875, "beta_high": 0.0, "orthogonal_to_active": 0.057214535772800446, "evidence_weights": [-0.2180609405040741, -0.3098386526107788, -0.1545143723487854], "agreement_weight": -0.025524022057652473, "orthogonal_weight": 1.002553939819336}, {"head": 3, "memory_to_active": 0.05200690031051636, "contribution_to_active": 0.41271311044692993, "mix_mean": 0.44345760345458984, "mix_low": 0.06313514709472656, "mix_high": 0.04497051239013672, "read_cosine": 0.5763708353042603, "beta_mean": 0.07788208872079849, "nonfinite": 0.0, "weighted_memory_to_active": 0.0282896775752306, "mix_std": 0.40196293592453003, "beta_std": 0.1140751987695694, "beta_low": 0.3826904296875, "beta_high": 0.0, "orthogonal_to_active": 0.023055873811244965, "evidence_weights": [-0.2845213711261749, -0.32159706950187683, 0.12869955599308014], "agreement_weight": 0.011953174136579037, "orthogonal_weight": 1.0089025497436523}, {"head": 4, "memory_to_active": 0.9586352109909058, "contribution_to_active": 0.18190471827983856, "mix_mean": 0.09320646524429321, "mix_low": 0.13364219665527344, "mix_high": 0.0, "read_cosine": 0.3160844147205353, "beta_mean": 0.037561073899269104, "nonfinite": 0.0, "weighted_memory_to_active": 0.148786723613739, "mix_std": 0.13274626433849335, "beta_std": 0.04311757907271385, "beta_low": 0.159423828125, "beta_high": 0.0, "orthogonal_to_active": 0.003545730374753475, "evidence_weights": [-0.029913175851106644, -0.017184162512421608, 0.33579686284065247], "agreement_weight": -0.021843984723091125, "orthogonal_weight": -0.030207352712750435}, {"head": 5, "memory_to_active": 0.30128639936447144, "contribution_to_active": 0.3400772213935852, "mix_mean": 0.2502550482749939, "mix_low": 0.02099323272705078, "mix_high": 6.389617919921875e-05, "read_cosine": 0.8863955736160278, "beta_mean": 0.02745930105447769, "nonfinite": 0.0, "weighted_memory_to_active": 0.133838951587677, "mix_std": 0.2251671403646469, "beta_std": 0.035004280507564545, "beta_low": 0.2999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.018900951370596886, "evidence_weights": [-0.14256970584392548, -0.17979291081428528, 0.22130706906318665], "agreement_weight": 0.052849989384412766, "orthogonal_weight": -0.402554988861084}]}, {"layer": 12, "memory_to_active": 0.7955211997032166, "contribution_to_active": 0.42026400566101074, "mix_mean": 0.2612898349761963, "mix_low": 0.09945488721132278, "mix_high": 0.00030867260647937655, "read_cosine": 0.31742507219314575, "beta_mean": 0.03555323928594589, "nonfinite": 0.0, "weighted_memory_to_active": 0.27998143434524536, "mix_std": 0.21022309362888336, "beta_std": 0.03901604562997818, "beta_low": 0.5353190302848816, "beta_high": 0.0, "orthogonal_to_active": 0.04644358903169632, "output_influence": 0.21976381540298462, "residual_influence": 0.05197994410991669, "state_active": 0.058979183435440063, "state_memory": 0.024932321161031723, "state_peak": 40.291011810302734, "growth_active": 1.2172231674194336, "growth_memory": 1.4934606552124023, "heads": [{"head": 0, "memory_to_active": 1.0569366216659546, "contribution_to_active": 0.28051650524139404, "mix_mean": 0.15737654268741608, "mix_low": 0.05867576599121094, "mix_high": 0.000255584716796875, "read_cosine": 0.4377240538597107, "beta_mean": 0.028248431161046028, "nonfinite": 0.0, "weighted_memory_to_active": 0.369680792093277, "mix_std": 0.1785091757774353, "beta_std": 0.04479861259460449, "beta_low": 0.3038330078125, "beta_high": 0.0, "orthogonal_to_active": 0.07137944549322128, "evidence_weights": [-0.07953684031963348, -0.11407506465911865, 0.2235596925020218], "agreement_weight": 0.08791659772396088, "orthogonal_weight": -0.27856212854385376}, {"head": 1, "memory_to_active": 0.3579441010951996, "contribution_to_active": 0.2852649390697479, "mix_mean": 0.23499538004398346, "mix_low": 0.018014907836914062, "mix_high": 5.245208740234375e-05, "read_cosine": 0.5833286046981812, "beta_mean": 0.004584386944770813, "nonfinite": 0.0, "weighted_memory_to_active": 0.12620317935943604, "mix_std": 0.22205457091331482, "beta_std": 0.004684616811573505, "beta_low": 0.90374755859375, "beta_high": 0.0, "orthogonal_to_active": 0.06200159341096878, "evidence_weights": [-0.1071566566824913, -0.1545780748128891, 0.24815581738948822], "agreement_weight": 0.09349951893091202, "orthogonal_weight": -0.7353267073631287}, {"head": 2, "memory_to_active": 0.5756027698516846, "contribution_to_active": 0.34208595752716064, "mix_mean": 0.2795521914958954, "mix_low": 0.01754474639892578, "mix_high": 0.0003910064697265625, "read_cosine": 0.2601085603237152, "beta_mean": 0.06651058793067932, "nonfinite": 0.0, "weighted_memory_to_active": 0.2507511377334595, "mix_std": 0.23864950239658356, "beta_std": 0.06985819339752197, "beta_low": 0.0540771484375, "beta_high": 0.0, "orthogonal_to_active": 0.016269739717245102, "evidence_weights": [-0.14607784152030945, -0.10011906176805496, 0.25163424015045166], "agreement_weight": 0.049728479236364365, "orthogonal_weight": 0.07548385858535767}, {"head": 3, "memory_to_active": 0.11734522879123688, "contribution_to_active": 0.5075845122337341, "mix_mean": 0.4749033749103546, "mix_low": 0.0038213729858398438, "mix_high": 8.487701416015625e-05, "read_cosine": 0.012600293383002281, "beta_mean": 0.0020708523225039244, "nonfinite": 0.0, "weighted_memory_to_active": 0.06568804383277893, "mix_std": 0.26345211267471313, "beta_std": 0.0024104383774101734, "beta_low": 0.9837646484375, "beta_high": 0.0, "orthogonal_to_active": 0.02730194851756096, "evidence_weights": [-0.23234693706035614, -0.22804950177669525, 0.12450951337814331], "agreement_weight": 0.04713567718863487, "orthogonal_weight": -0.42964300513267517}, {"head": 4, "memory_to_active": 2.4999947547912598, "contribution_to_active": 0.7564608454704285, "mix_mean": 0.036192331463098526, "mix_low": 0.49105262756347656, "mix_high": 0.000324249267578125, "read_cosine": 0.1438368260860443, "beta_mean": 0.10772229731082916, "nonfinite": 0.0, "weighted_memory_to_active": 0.8044050931930542, "mix_std": 0.08901771903038025, "beta_std": 0.10574911534786224, "beta_low": 0.077880859375, "beta_high": 0.0, "orthogonal_to_active": 0.058036595582962036, "evidence_weights": [0.054316308349370956, 0.001379125053063035, 0.09397141635417938], "agreement_weight": -0.016565417870879173, "orthogonal_weight": -0.07812642306089401}, {"head": 5, "memory_to_active": 0.1653033345937729, "contribution_to_active": 0.34967124462127686, "mix_mean": 0.38471922278404236, "mix_low": 0.0076198577880859375, "mix_high": 0.000743865966796875, "read_cosine": 0.4669519066810608, "beta_mean": 0.004182858392596245, "nonfinite": 0.0, "weighted_memory_to_active": 0.06316018104553223, "mix_std": 0.2696554362773895, "beta_std": 0.006595309358090162, "beta_low": 0.88861083984375, "beta_high": 0.0, "orthogonal_to_active": 0.043672189116477966, "evidence_weights": [-0.20734825730323792, -0.2085856944322586, 0.3020912706851959], "agreement_weight": 0.0411161333322525, "orthogonal_weight": -0.7945683002471924}]}, {"layer": 13, "memory_to_active": 0.31281763315200806, "contribution_to_active": 0.3288952708244324, "mix_mean": 0.3043816387653351, "mix_low": 0.016697406768798828, "mix_high": 0.0014882087707519531, "read_cosine": 0.34482550621032715, "beta_mean": 0.022009458392858505, "nonfinite": 0.0, "weighted_memory_to_active": 0.10994987934827805, "mix_std": 0.2255031019449234, "beta_std": 0.03179554641246796, "beta_low": 0.6229349970817566, "beta_high": 0.0, "orthogonal_to_active": 0.05208741873502731, "output_influence": 0.288890540599823, "residual_influence": 0.08614973723888397, "state_active": 0.09677213430404663, "state_memory": 0.026756826788187027, "state_peak": 44.504703521728516, "growth_active": 1.1091488599777222, "growth_memory": 1.3317099809646606, "heads": [{"head": 0, "memory_to_active": 0.4051617383956909, "contribution_to_active": 0.273958295583725, "mix_mean": 0.22419363260269165, "mix_low": 0.017276763916015625, "mix_high": 5.245208740234375e-05, "read_cosine": 0.2898690104484558, "beta_mean": 0.010319551452994347, "nonfinite": 0.0, "weighted_memory_to_active": 0.1316928267478943, "mix_std": 0.21611477434635162, "beta_std": 0.011763256974518299, "beta_low": 0.6474609375, "beta_high": 0.0, "orthogonal_to_active": 0.10367120057344437, "evidence_weights": [-0.1319139003753662, -0.1568150669336319, 0.19436602294445038], "agreement_weight": 0.05904931575059891, "orthogonal_weight": -0.853330135345459}, {"head": 1, "memory_to_active": 0.38806605339050293, "contribution_to_active": 0.3140375018119812, "mix_mean": 0.23094666004180908, "mix_low": 0.026361465454101562, "mix_high": 1.430511474609375e-05, "read_cosine": 0.2744884490966797, "beta_mean": 0.005553673021495342, "nonfinite": 0.0, "weighted_memory_to_active": 0.12718366086483002, "mix_std": 0.2229584902524948, "beta_std": 0.006111896596848965, "beta_low": 0.869873046875, "beta_high": 0.0, "orthogonal_to_active": 0.09449287503957748, "evidence_weights": [-0.12570558488368988, -0.1461246907711029, 0.23001578450202942], "agreement_weight": 0.0590066984295845, "orthogonal_weight": -0.8127137422561646}, {"head": 2, "memory_to_active": 0.30835211277008057, "contribution_to_active": 0.26570332050323486, "mix_mean": 0.215861976146698, "mix_low": 0.01451873779296875, "mix_high": 2.6702880859375e-05, "read_cosine": 0.6723485589027405, "beta_mean": 0.03175823763012886, "nonfinite": 0.0, "weighted_memory_to_active": 0.10918283462524414, "mix_std": 0.18115246295928955, "beta_std": 0.05856727808713913, "beta_low": 0.43084716796875, "beta_high": 0.0, "orthogonal_to_active": 0.01939494162797928, "evidence_weights": [-0.15256184339523315, -0.13711576163768768, 0.3454612195491791], "agreement_weight": 0.08120676130056381, "orthogonal_weight": -0.3099673390388489}, {"head": 3, "memory_to_active": 0.054579369723796844, "contribution_to_active": 0.4846062660217285, "mix_mean": 0.504644513130188, "mix_low": 0.0044612884521484375, "mix_high": 0.00026607513427734375, "read_cosine": -0.15716688334941864, "beta_mean": 0.0003206781402695924, "nonfinite": 0.0, "weighted_memory_to_active": 0.028831012547016144, "mix_std": 0.21590059995651245, "beta_std": 0.0011663142358884215, "beta_low": 0.99798583984375, "beta_high": 0.0, "orthogonal_to_active": 0.019516833126544952, "evidence_weights": [-0.31741708517074585, -0.2539375424385071, 0.21462208032608032], "agreement_weight": 0.44119638204574585, "orthogonal_weight": -0.677564799785614}, {"head": 4, "memory_to_active": 0.2701982259750366, "contribution_to_active": 0.3144916892051697, "mix_mean": 0.4540853500366211, "mix_low": 0.004596710205078125, "mix_high": 0.008549690246582031, "read_cosine": 0.7530087232589722, "beta_mean": 0.06937557458877563, "nonfinite": 0.0, "weighted_memory_to_active": 0.12727074325084686, "mix_std": 0.2987440526485443, "beta_std": 0.07990847527980804, "beta_low": 0.09368896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0310601107776165, "evidence_weights": [-0.25256896018981934, -0.2739117741584778, 0.33637240529060364], "agreement_weight": 0.008015261963009834, "orthogonal_weight": 0.34984520077705383}, {"head": 5, "memory_to_active": 0.4505482316017151, "contribution_to_active": 0.3205743730068207, "mix_mean": 0.1965576857328415, "mix_low": 0.03296947479248047, "mix_high": 2.002716064453125e-05, "read_cosine": 0.2364051640033722, "beta_mean": 0.014729024842381477, "nonfinite": 0.0, "weighted_memory_to_active": 0.13553817570209503, "mix_std": 0.21814824640750885, "beta_std": 0.03325603902339935, "beta_low": 0.69775390625, "beta_high": 0.0, "orthogonal_to_active": 0.044388554990291595, "evidence_weights": [-0.12684069573879242, -0.14196988940238953, -0.01709599234163761], "agreement_weight": 0.04688134416937828, "orthogonal_weight": -0.3662424087524414}]}, {"layer": 14, "memory_to_active": 0.335507869720459, "contribution_to_active": 0.3936804533004761, "mix_mean": 0.3394146263599396, "mix_low": 0.02613004297018051, "mix_high": 0.0007564227562397718, "read_cosine": 0.5668919086456299, "beta_mean": 0.08637136220932007, "nonfinite": 0.0, "weighted_memory_to_active": 0.143648162484169, "mix_std": 0.24491606652736664, "beta_std": 0.08135993033647537, "beta_low": 0.2569580376148224, "beta_high": 0.0, "orthogonal_to_active": 0.025882523506879807, "output_influence": 0.22339048981666565, "residual_influence": 0.07683953642845154, "state_active": 0.08929438889026642, "state_memory": 0.024916652590036392, "state_peak": 41.52083969116211, "growth_active": 1.1630409955978394, "growth_memory": 1.4989196062088013, "heads": [{"head": 0, "memory_to_active": 0.2870151996612549, "contribution_to_active": 0.33531564474105835, "mix_mean": 0.30753642320632935, "mix_low": 0.021185874938964844, "mix_high": 0.00038814544677734375, "read_cosine": 0.6442542672157288, "beta_mean": 0.06787087023258209, "nonfinite": 0.0, "weighted_memory_to_active": 0.12177921831607819, "mix_std": 0.2652917206287384, "beta_std": 0.0994463860988617, "beta_low": 0.21826171875, "beta_high": 0.0, "orthogonal_to_active": 0.002182667376473546, "evidence_weights": [-0.0956306979060173, -0.17622247338294983, 0.4705142676830292], "agreement_weight": 0.05337729677557945, "orthogonal_weight": 0.02473372220993042}, {"head": 1, "memory_to_active": 0.4499896168708801, "contribution_to_active": 0.4737721085548401, "mix_mean": 0.3261147439479828, "mix_low": 0.06446361541748047, "mix_high": 0.00069427490234375, "read_cosine": 0.44892168045043945, "beta_mean": 0.31661319732666016, "nonfinite": 0.0, "weighted_memory_to_active": 0.22494886815547943, "mix_std": 0.29405897855758667, "beta_std": 0.2676456570625305, "beta_low": 0.0732421875, "beta_high": 0.0, "orthogonal_to_active": 0.005794588942080736, "evidence_weights": [-0.06415961682796478, -0.22242148220539093, 0.2593967020511627], "agreement_weight": 0.058342210948467255, "orthogonal_weight": -0.03237779438495636}, {"head": 2, "memory_to_active": 0.4287478029727936, "contribution_to_active": 0.3471900224685669, "mix_mean": 0.31297165155410767, "mix_low": 0.009549140930175781, "mix_high": 0.0001735687255859375, "read_cosine": 0.461271196603775, "beta_mean": 0.03583689033985138, "nonfinite": 0.0, "weighted_memory_to_active": 0.17491890490055084, "mix_std": 0.2502787113189697, "beta_std": 0.0331864058971405, "beta_low": 0.0780029296875, "beta_high": 0.0, "orthogonal_to_active": 0.09327398240566254, "evidence_weights": [-0.1782751977443695, -0.18255667388439178, 0.1331118941307068], "agreement_weight": 0.10723651945590973, "orthogonal_weight": -0.6440349221229553}, {"head": 3, "memory_to_active": 0.08146342635154724, "contribution_to_active": 0.594407320022583, "mix_mean": 0.6097036600112915, "mix_low": 0.0017604827880859375, "mix_high": 0.0023403167724609375, "read_cosine": 0.5573743581771851, "beta_mean": 0.010725179687142372, "nonfinite": 0.0, "weighted_memory_to_active": 0.056194111704826355, "mix_std": 0.23891571164131165, "beta_std": 0.008823426440358162, "beta_low": 0.6007080078125, "beta_high": 0.0, "orthogonal_to_active": 0.014727732166647911, "evidence_weights": [-0.357567697763443, -0.3150138854980469, 0.355602502822876], "agreement_weight": 0.0640082135796547, "orthogonal_weight": -0.3768012821674347}, {"head": 4, "memory_to_active": 0.31500378251075745, "contribution_to_active": 0.41693076491355896, "mix_mean": 0.3174051344394684, "mix_low": 0.007950782775878906, "mix_high": 0.0009164810180664062, "read_cosine": 0.5882536172866821, "beta_mean": 0.072685107588768, "nonfinite": 0.0, "weighted_memory_to_active": 0.14494892954826355, "mix_std": 0.24055776000022888, "beta_std": 0.05943060293793678, "beta_low": 0.0089111328125, "beta_high": 0.0, "orthogonal_to_active": 0.007705220486968756, "evidence_weights": [-0.1699506640434265, -0.21195143461227417, 0.13527093827724457], "agreement_weight": 0.10784104466438293, "orthogonal_weight": -0.07621636241674423}, {"head": 5, "memory_to_active": 0.45082736015319824, "contribution_to_active": 0.1944667100906372, "mix_mean": 0.16275595128536224, "mix_low": 0.05187034606933594, "mix_high": 2.574920654296875e-05, "read_cosine": 0.701276421546936, "beta_mean": 0.014496905729174614, "nonfinite": 0.0, "weighted_memory_to_active": 0.1390988528728485, "mix_std": 0.18039347231388092, "beta_std": 0.019627127796411514, "beta_low": 0.5626220703125, "beta_high": 0.0, "orthogonal_to_active": 0.031610943377017975, "evidence_weights": [-0.15467959642410278, -0.11795812845230103, 0.030589058995246887], "agreement_weight": 0.10282667726278305, "orthogonal_weight": -0.38185474276542664}]}, {"layer": 15, "memory_to_active": 0.1528674066066742, "contribution_to_active": 0.4127161502838135, "mix_mean": 0.3301548957824707, "mix_low": 0.010974407196044922, "mix_high": 0.0017197928391397, "read_cosine": 0.5855732560157776, "beta_mean": 0.03844645991921425, "nonfinite": 0.0, "weighted_memory_to_active": 0.07158227264881134, "mix_std": 0.2529560923576355, "beta_std": 0.04688846319913864, "beta_low": 0.601348876953125, "beta_high": 0.0, "orthogonal_to_active": 0.017152175307273865, "output_influence": 0.2734820246696472, "residual_influence": 0.13579805195331573, "state_active": 0.10782928019762039, "state_memory": 0.01459652278572321, "state_peak": 30.837879180908203, "growth_active": 1.394834280014038, "growth_memory": 1.690622329711914, "heads": [{"head": 0, "memory_to_active": 0.14295528829097748, "contribution_to_active": 0.47632476687431335, "mix_mean": 0.3363037109375, "mix_low": 0.010852813720703125, "mix_high": 0.00125885009765625, "read_cosine": 0.7008273601531982, "beta_mean": 0.07014419138431549, "nonfinite": 0.0, "weighted_memory_to_active": 0.07392407953739166, "mix_std": 0.25573641061782837, "beta_std": 0.06233303248882294, "beta_low": 0.10455322265625, "beta_high": 0.0, "orthogonal_to_active": 0.010308363474905491, "evidence_weights": [-0.18578986823558807, -0.2213125377893448, 0.29823943972587585], "agreement_weight": 0.08070757240056992, "orthogonal_weight": -0.2506692111492157}, {"head": 1, "memory_to_active": 0.2367493212223053, "contribution_to_active": 0.3786723017692566, "mix_mean": 0.3184218406677246, "mix_low": 0.007195472717285156, "mix_high": 0.004353523254394531, "read_cosine": 0.8091049194335938, "beta_mean": 0.0027016778476536274, "nonfinite": 0.0, "weighted_memory_to_active": 0.10285662114620209, "mix_std": 0.25212422013282776, "beta_std": 0.004591749981045723, "beta_low": 0.965576171875, "beta_high": 0.0, "orthogonal_to_active": 0.009688378311693668, "evidence_weights": [-0.1937512755393982, -0.20561206340789795, 0.24512207508087158], "agreement_weight": 0.09052896499633789, "orthogonal_weight": -0.17526713013648987}, {"head": 2, "memory_to_active": 0.02081235498189926, "contribution_to_active": 0.41244977712631226, "mix_mean": 0.32720518112182617, "mix_low": 0.01683807373046875, "mix_high": 0.00024509429931640625, "read_cosine": 0.7260227203369141, "beta_mean": 0.0005373168969526887, "nonfinite": 0.0, "weighted_memory_to_active": 0.008169732987880707, "mix_std": 0.26000794768333435, "beta_std": 0.0009145674994215369, "beta_low": 0.99884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.002908483147621155, "evidence_weights": [-0.2201949656009674, -0.2437494844198227, 0.28900501132011414], "agreement_weight": 0.04881850257515907, "orthogonal_weight": -0.4950696527957916}, {"head": 3, "memory_to_active": 0.29745644330978394, "contribution_to_active": 0.36388206481933594, "mix_mean": 0.221561461687088, "mix_low": 0.012578010559082031, "mix_high": 5.340576171875e-05, "read_cosine": 0.64690762758255, "beta_mean": 0.028981339186429977, "nonfinite": 0.0, "weighted_memory_to_active": 0.13942942023277283, "mix_std": 0.20058545470237732, "beta_std": 0.04563988000154495, "beta_low": 0.35516357421875, "beta_high": 0.0, "orthogonal_to_active": 0.02019483968615532, "evidence_weights": [-0.1224576011300087, -0.1923627406358719, 0.3031027615070343], "agreement_weight": 0.08028244972229004, "orthogonal_weight": -0.20388847589492798}, {"head": 4, "memory_to_active": 0.21723493933677673, "contribution_to_active": 0.43632251024246216, "mix_mean": 0.3397216796875, "mix_low": 0.012235641479492188, "mix_high": 6.771087646484375e-05, "read_cosine": 0.09730568528175354, "beta_mean": 0.12827526032924652, "nonfinite": 0.0, "weighted_memory_to_active": 0.10437490046024323, "mix_std": 0.26914840936660767, "beta_std": 0.16775494813919067, "beta_low": 0.1839599609375, "beta_high": 0.0, "orthogonal_to_active": 0.05957944691181183, "evidence_weights": [-0.26148417592048645, -0.2506212592124939, 0.3161299526691437], "agreement_weight": 0.0064453682862222195, "orthogonal_weight": -0.6344205141067505}, {"head": 5, "memory_to_active": 0.0019960009958595037, "contribution_to_active": 0.4086454510688782, "mix_mean": 0.4377155005931854, "mix_low": 0.006146430969238281, "mix_high": 0.004340171813964844, "read_cosine": 0.5332709550857544, "beta_mean": 3.896077032550238e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007388472440652549, "mix_std": 0.2801339328289032, "beta_std": 9.657948976382613e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0002335351164219901, "evidence_weights": [-0.3363875448703766, -0.2777082026004791, 0.42653778195381165], "agreement_weight": 0.06513314694166183, "orthogonal_weight": 0.36522340774536133}]}]}, {"step": 7000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8059293990008882, "memory_to_active": 1.1629133224487305, "contribution_to_active": 0.37485742568969727, "mix_mean": 0.2110801339149475, "mix_low": 0.2082185000181198, "mix_high": 0.0036699874326586723, "read_cosine": 0.4010099172592163, "beta_mean": 0.11311047524213791, "nonfinite": 0.0, "weighted_memory_to_active": 0.21126209199428558, "mix_std": 0.17806348204612732, "beta_std": 0.08160851895809174, "beta_low": 0.347430557012558, "beta_high": 2.9881795853725635e-05, "orthogonal_to_active": 0.03068528324365616, "samples": 4, "scored_tokens": 899, "normal_loss": 3.3120803833007812, "normal_token_accuracy": 0.418242484331131, "state_active": 0.07051150500774384, "state_memory": 0.03449142724275589, "state_peak": 58.63242721557617, "growth_active": 1.1207106113433838, "growth_memory": 1.5171740055084229, "output_influence": 0.2579401731491089, "residual_influence": 0.07693706452846527, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.397812843322754, "no_memory_loss_delta": 0.0872388407588005, "scrambled_loss_delta": 0.3825315833091736, "memory_only_loss_delta": 1.5874109268188477, "no_evidence_loss_delta": 0.0265716090798378, "no_agreement_loss_delta": -0.005720571614801884, "no_orthogonal_loss_delta": 0.0002628189977258444}, {"index": 94, "tokens": 256, "normal_loss": 3.479130983352661, "no_memory_loss_delta": 0.20703120529651642, "scrambled_loss_delta": 0.4432215690612793, "memory_only_loss_delta": 1.5928477048873901, "no_evidence_loss_delta": 0.08544144034385681, "no_agreement_loss_delta": 0.024373812600970268, "no_orthogonal_loss_delta": -0.001869911327958107}, {"index": 188, "tokens": 256, "normal_loss": 3.7452964782714844, "no_memory_loss_delta": 0.05618227645754814, "scrambled_loss_delta": 0.10337722301483154, "memory_only_loss_delta": 1.737579345703125, "no_evidence_loss_delta": 0.01684175431728363, "no_agreement_loss_delta": -0.0050576478242874146, "no_orthogonal_loss_delta": -0.006281537935137749}, {"index": 283, "tokens": 131, "normal_loss": 1.9715021848678589, "no_memory_loss_delta": 0.18075168132781982, "scrambled_loss_delta": 0.3199009895324707, "memory_only_loss_delta": 2.458350419998169, "no_evidence_loss_delta": 0.05721969157457352, "no_agreement_loss_delta": 0.0031506086234003305, "no_orthogonal_loss_delta": 0.013494351878762245}], "agreement_weight_abs_mean": 0.059561837464571, "agreement_weight_abs_max": 0.5026719570159912, "agreement_weight_mean": 0.03527587279677391, "orthogonal_weight_abs_mean": 0.2901571989059448, "orthogonal_weight_abs_max": 1.1695808172225952, "orthogonal_weight_mean": -0.11770807951688766, "no_memory_loss": 3.4382145404815674, "no_memory_loss_delta": 0.1261337697505951, "no_memory_token_accuracy_gain": 0.01001112349331379, "no_memory_token_win_fraction": 0.5928809642791748, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 11.85030787271502, "scrambled_loss": 3.6232755184173584, "scrambled_loss_delta": 0.31119510531425476, "scrambled_token_accuracy_gain": 0.025583982467651367, "scrambled_token_win_fraction": 0.6195772886276245, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 26.742906655681093, "memory_only_loss": 5.070713520050049, "memory_only_loss_delta": 1.7586324214935303, "memory_only_token_accuracy_gain": 0.19132369756698608, "memory_only_token_win_fraction": 0.8509454727172852, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 82.77196903604131, "no_evidence_loss": 3.3571112155914307, "no_evidence_loss_delta": 0.045030705630779266, "no_evidence_token_accuracy_gain": 0.006674082484096289, "no_evidence_token_win_fraction": 0.5450500845909119, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 4.403187222193347, "no_agreement_loss": 3.316411256790161, "no_agreement_loss_delta": 0.004330590832978487, "no_agreement_token_accuracy_gain": 0.0055617354810237885, "no_agreement_token_win_fraction": 0.4872080087661743, "no_agreement_probe_win_fraction": 0.5, "no_agreement_ppl_reduction_pct": 0.4321227345850774, "no_orthogonal_loss": 3.311800241470337, "no_orthogonal_loss_delta": -0.0002800097572617233, "no_orthogonal_token_accuracy_gain": 0.006674082484096289, "no_orthogonal_token_win_fraction": 0.5194661021232605, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": -0.02800489636531095, "short_no_memory_loss_delta": 0.08646700531244278, "long_context_memory_gain": 0.03966676443815231, "normal_context_loss_gain": 0.11674199998378754, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.343085527420044, "contribution_to_active": 0.7070703506469727, "mix_mean": 0.09537898004055023, "mix_low": 0.28287842869758606, "mix_high": 0.0011501312255859375, "read_cosine": 0.5408830046653748, "beta_mean": 0.25290778279304504, "nonfinite": 0.0, "weighted_memory_to_active": 0.6820873022079468, "mix_std": 0.08474960923194885, "beta_std": 0.1289743185043335, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04563918709754944, "output_influence": 0.1638745367527008, "residual_influence": 0.23074497282505035, "state_active": 0.1058870181441307, "state_memory": 0.06086426228284836, "state_peak": 42.71744155883789, "growth_active": 1.2513751983642578, "growth_memory": 1.3871322870254517, "heads": [{"head": 0, "memory_to_active": 2.3894479274749756, "contribution_to_active": 0.6075945496559143, "mix_mean": 0.0252455435693264, "mix_low": 0.32732295989990234, "mix_high": 0.0, "read_cosine": 0.42273762822151184, "beta_mean": 0.22627250850200653, "nonfinite": 0.0, "weighted_memory_to_active": 0.6527153253555298, "mix_std": 0.0517406091094017, "beta_std": 0.09108985215425491, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01078638806939125, "evidence_weights": [-0.045446231961250305, -0.05958498269319534, 0.0968148335814476], "agreement_weight": -0.040858231484889984, "orthogonal_weight": 0.01955372840166092}, {"head": 1, "memory_to_active": 2.5941731929779053, "contribution_to_active": 1.247554898262024, "mix_mean": 0.025644877925515175, "mix_low": 0.2676582336425781, "mix_high": 0.0, "read_cosine": 0.5430390238761902, "beta_mean": 0.3462112247943878, "nonfinite": 0.0, "weighted_memory_to_active": 1.3682454824447632, "mix_std": 0.055532049387693405, "beta_std": 0.14805038273334503, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07219356298446655, "evidence_weights": [-0.009481532499194145, -0.055562276393175125, 0.057151034474372864], "agreement_weight": -0.005973346997052431, "orthogonal_weight": 0.10405886173248291}, {"head": 2, "memory_to_active": 2.0387933254241943, "contribution_to_active": 0.5251542329788208, "mix_mean": 0.022136740386486053, "mix_low": 0.3936576843261719, "mix_high": 0.0, "read_cosine": 0.4339055120944977, "beta_mean": 0.2868277430534363, "nonfinite": 0.0, "weighted_memory_to_active": 0.5441755056381226, "mix_std": 0.03800998255610466, "beta_std": 0.17491593956947327, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011500897817313671, "evidence_weights": [-0.015015565790235996, -0.011549522168934345, -0.09539025276899338], "agreement_weight": -0.01509186066687107, "orthogonal_weight": 0.0033272455912083387}, {"head": 3, "memory_to_active": 3.387439489364624, "contribution_to_active": 1.3967747688293457, "mix_mean": 0.08007385581731796, "mix_low": 0.03832054138183594, "mix_high": 0.0, "read_cosine": 0.5284767746925354, "beta_mean": 0.2953368127346039, "nonfinite": 0.0, "weighted_memory_to_active": 1.3508672714233398, "mix_std": 0.0925181657075882, "beta_std": 0.1617043912410736, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16761621832847595, "evidence_weights": [-0.14573346078395844, -0.16618435084819794, 0.2071218192577362], "agreement_weight": -0.03785402700304985, "orthogonal_weight": 0.16880349814891815}, {"head": 4, "memory_to_active": 3.4702885150909424, "contribution_to_active": 0.10672915726900101, "mix_mean": 0.011049949564039707, "mix_low": 0.6605224609375, "mix_high": 0.0, "read_cosine": 0.36047059297561646, "beta_mean": 0.2589764893054962, "nonfinite": 0.0, "weighted_memory_to_active": 0.09200329333543777, "mix_std": 0.01405805442482233, "beta_std": 0.12421093881130219, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01926894299685955, "evidence_weights": [0.04074063152074814, 0.013848993927240372, -0.016268638893961906], "agreement_weight": -0.03663928434252739, "orthogonal_weight": 0.24115371704101562}, {"head": 5, "memory_to_active": 0.17837068438529968, "contribution_to_active": 0.3586142063140869, "mix_mean": 0.40812286734580994, "mix_low": 0.00978851318359375, "mix_high": 0.006900787353515625, "read_cosine": 0.9566686153411865, "beta_mean": 0.10382181406021118, "nonfinite": 0.0, "weighted_memory_to_active": 0.0845167264342308, "mix_std": 0.256638765335083, "beta_std": 0.07387435436248779, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028199278749525547, "evidence_weights": [-0.338474303483963, -0.3450840711593628, 0.4443679749965668], "agreement_weight": 0.1372632086277008, "orthogonal_weight": 0.09905962646007538}]}, {"layer": 1, "memory_to_active": 2.9992828369140625, "contribution_to_active": 0.3442218601703644, "mix_mean": 0.04991637170314789, "mix_low": 0.46498093008995056, "mix_high": 0.00017484030104242265, "read_cosine": 0.3527843654155731, "beta_mean": 0.29142695665359497, "nonfinite": 0.0, "weighted_memory_to_active": 0.3456808626651764, "mix_std": 0.0666172131896019, "beta_std": 0.15607282519340515, "beta_low": 0.021881103515625, "beta_high": 0.0004679362173192203, "orthogonal_to_active": 0.038712065666913986, "output_influence": 0.12809383869171143, "residual_influence": 0.0274631530046463, "state_active": 0.054179854691028595, "state_memory": 0.0703597441315651, "state_peak": 24.283824920654297, "growth_active": 1.0845667123794556, "growth_memory": 1.582129955291748, "heads": [{"head": 0, "memory_to_active": 2.418522834777832, "contribution_to_active": 0.08911823481321335, "mix_mean": 0.014928318560123444, "mix_low": 0.6622600555419922, "mix_high": 0.0, "read_cosine": 0.08281814306974411, "beta_mean": 0.06247611343860626, "nonfinite": 0.0, "weighted_memory_to_active": 0.0810256078839302, "mix_std": 0.03441622480750084, "beta_std": 0.06113716959953308, "beta_low": 0.11944580078125, "beta_high": 0.0, "orthogonal_to_active": 0.023344390094280243, "evidence_weights": [0.06903833150863647, 0.03379887342453003, -0.11202259361743927], "agreement_weight": -0.15060381591320038, "orthogonal_weight": -0.2920684516429901}, {"head": 1, "memory_to_active": 3.383169651031494, "contribution_to_active": 0.1682920902967453, "mix_mean": 0.031220817938447, "mix_low": 0.31671905517578125, "mix_high": 9.5367431640625e-07, "read_cosine": 0.45772600173950195, "beta_mean": 0.39931410551071167, "nonfinite": 0.0, "weighted_memory_to_active": 0.17574100196361542, "mix_std": 0.05379526689648628, "beta_std": 0.2206687480211258, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005804964806884527, "evidence_weights": [-0.047614362090826035, -0.04111016169190407, 0.1205696165561676], "agreement_weight": -0.027139846235513687, "orthogonal_weight": 0.03923093527555466}, {"head": 2, "memory_to_active": 0.9716887474060059, "contribution_to_active": 0.38411062955856323, "mix_mean": 0.19385410845279694, "mix_low": 0.016935348510742188, "mix_high": 0.0009746551513671875, "read_cosine": 0.7676477432250977, "beta_mean": 0.14914508163928986, "nonfinite": 0.0, "weighted_memory_to_active": 0.5285611152648926, "mix_std": 0.18834362924098969, "beta_std": 0.07051829993724823, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04883594810962677, "evidence_weights": [-0.2143394500017166, -0.29126572608947754, 0.22661294043064117], "agreement_weight": -0.02173742838203907, "orthogonal_weight": 0.16742272675037384}, {"head": 3, "memory_to_active": 3.113863468170166, "contribution_to_active": 0.06033997982740402, "mix_mean": 0.008253439329564571, "mix_low": 0.777430534362793, "mix_high": 0.0, "read_cosine": 0.19427810609340668, "beta_mean": 0.538555383682251, "nonfinite": 0.0, "weighted_memory_to_active": 0.06551187485456467, "mix_std": 0.017619146034121513, "beta_std": 0.24838018417358398, "beta_low": 0.0, "beta_high": 0.00274658203125, "orthogonal_to_active": 0.0033172438852488995, "evidence_weights": [0.017235854640603065, 0.014517613686621189, 0.004985715728253126], "agreement_weight": -0.01030127052217722, "orthogonal_weight": -0.054111432284116745}, {"head": 4, "memory_to_active": 3.2736496925354004, "contribution_to_active": 1.294610619544983, "mix_mean": 0.042830921709537506, "mix_low": 0.23468685150146484, "mix_high": 0.0, "read_cosine": 0.412291944026947, "beta_mean": 0.24901998043060303, "nonfinite": 0.0, "weighted_memory_to_active": 1.1566612720489502, "mix_std": 0.07427582144737244, "beta_std": 0.10626798868179321, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14963874220848083, "evidence_weights": [-0.04749094322323799, -0.04524139314889908, 0.19108271598815918], "agreement_weight": -0.042383283376693726, "orthogonal_weight": 0.13542261719703674}, {"head": 5, "memory_to_active": 4.834800720214844, "contribution_to_active": 0.06885968893766403, "mix_mean": 0.008410586044192314, "mix_low": 0.7818536758422852, "mix_high": 7.343292236328125e-05, "read_cosine": 0.20194417238235474, "beta_mean": 0.35005101561546326, "nonfinite": 0.0, "weighted_memory_to_active": 0.06658408045768738, "mix_std": 0.03125318884849548, "beta_std": 0.22946451604366302, "beta_low": 0.0118408203125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0013311008224263787, "evidence_weights": [0.07010549306869507, 0.04639114439487457, 0.1528632640838623], "agreement_weight": -0.09556552767753601, "orthogonal_weight": 0.020648963749408722}]}, {"layer": 2, "memory_to_active": 1.4620206356048584, "contribution_to_active": 0.22812187671661377, "mix_mean": 0.12680649757385254, "mix_low": 0.3689035177230835, "mix_high": 0.0007468859548680484, "read_cosine": 0.38848060369491577, "beta_mean": 0.12583275139331818, "nonfinite": 0.0, "weighted_memory_to_active": 0.09298176318407059, "mix_std": 0.1264665722846985, "beta_std": 0.07958292961120605, "beta_low": 0.41644287109375, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.006351158022880554, "output_influence": 0.18840184807777405, "residual_influence": 0.03603691607713699, "state_active": 0.041736118495464325, "state_memory": 0.03522356227040291, "state_peak": 43.888465881347656, "growth_active": 1.0421684980392456, "growth_memory": 1.5665559768676758, "heads": [{"head": 0, "memory_to_active": 0.7816148996353149, "contribution_to_active": 0.02754795551300049, "mix_mean": 0.01768275909125805, "mix_low": 0.6216888427734375, "mix_high": 0.0, "read_cosine": 0.24488775432109833, "beta_mean": 0.020071957260370255, "nonfinite": 0.0, "weighted_memory_to_active": 0.025448983535170555, "mix_std": 0.039061423391103745, "beta_std": 0.018374569714069366, "beta_low": 0.32159423828125, "beta_high": 0.0, "orthogonal_to_active": 0.0023595988750457764, "evidence_weights": [0.05204114690423012, 0.04326309263706207, -0.004480181727558374], "agreement_weight": -0.0591595359146595, "orthogonal_weight": -0.09712274372577667}, {"head": 1, "memory_to_active": 3.833252429962158, "contribution_to_active": 0.06464226543903351, "mix_mean": 0.006719284690916538, "mix_low": 0.8466587066650391, "mix_high": 0.0, "read_cosine": 0.17849200963974, "beta_mean": 0.3956182301044464, "nonfinite": 0.0, "weighted_memory_to_active": 0.05939982086420059, "mix_std": 0.012253286316990852, "beta_std": 0.1602208912372589, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006230943836271763, "evidence_weights": [0.06684628874063492, 0.07666157931089401, 0.014666249044239521], "agreement_weight": -0.000509906851220876, "orthogonal_weight": 0.10809896886348724}, {"head": 2, "memory_to_active": 0.20447403192520142, "contribution_to_active": 0.3204556107521057, "mix_mean": 0.24362251162528992, "mix_low": 0.016859054565429688, "mix_high": 0.000331878662109375, "read_cosine": 0.4521107077598572, "beta_mean": 0.0005068128812126815, "nonfinite": 0.0, "weighted_memory_to_active": 0.0566171370446682, "mix_std": 0.2339753955602646, "beta_std": 0.0009364746510982513, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.005759702529758215, "evidence_weights": [-0.14030037820339203, -0.14869943261146545, 0.23286782205104828], "agreement_weight": 0.06393042951822281, "orthogonal_weight": -0.13325278460979462}, {"head": 3, "memory_to_active": 0.44541817903518677, "contribution_to_active": 0.31885164976119995, "mix_mean": 0.10212966799736023, "mix_low": 0.0784158706665039, "mix_high": 0.001567840576171875, "read_cosine": 0.6668393015861511, "beta_mean": 0.06059923395514488, "nonfinite": 0.0, "weighted_memory_to_active": 0.14241556823253632, "mix_std": 0.1474323272705078, "beta_std": 0.08563525974750519, "beta_low": 0.16009521484375, "beta_high": 0.0, "orthogonal_to_active": 0.0198727585375309, "evidence_weights": [-0.028597775846719742, -0.08831323683261871, 0.30022189021110535], "agreement_weight": 0.08434208482503891, "orthogonal_weight": -0.2814745306968689}, {"head": 4, "memory_to_active": 3.464414119720459, "contribution_to_active": 0.22531446814537048, "mix_mean": 0.015258044935762882, "mix_low": 0.6470718383789062, "mix_high": 9.822845458984375e-05, "read_cosine": 0.21733465790748596, "beta_mean": 0.2781106233596802, "nonfinite": 0.0, "weighted_memory_to_active": 0.25309598445892334, "mix_std": 0.05021527409553528, "beta_std": 0.21213513612747192, "beta_low": 0.0179443359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0028914303984493017, "evidence_weights": [0.03440346196293831, 0.029536258429288864, 0.14323164522647858], "agreement_weight": 0.027864009141921997, "orthogonal_weight": 0.013984443619847298}, {"head": 5, "memory_to_active": 0.04294944182038307, "contribution_to_active": 0.4119192361831665, "mix_mean": 0.3754267692565918, "mix_low": 0.0027265548706054688, "mix_high": 0.002483367919921875, "read_cosine": 0.5712190866470337, "beta_mean": 8.955768862506375e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.02091309055685997, "mix_std": 0.2758617699146271, "beta_std": 0.0001952320453710854, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0009925152407959104, "evidence_weights": [-0.2128709852695465, -0.19264040887355804, 0.34845393896102905], "agreement_weight": 0.009334726259112358, "orthogonal_weight": 0.10025323927402496}]}, {"layer": 3, "memory_to_active": 1.7773038148880005, "contribution_to_active": 0.2657700181007385, "mix_mean": 0.08415744453668594, "mix_low": 0.35274648666381836, "mix_high": 0.0009160041809082031, "read_cosine": 0.31915944814682007, "beta_mean": 0.1765529215335846, "nonfinite": 0.0, "weighted_memory_to_active": 0.16283628344535828, "mix_std": 0.12364881485700607, "beta_std": 0.1000523790717125, "beta_low": 0.34234619140625, "beta_high": 0.0, "orthogonal_to_active": 0.01909300871193409, "output_influence": 0.20061880350112915, "residual_influence": 0.02956981770694256, "state_active": 0.04196050763130188, "state_memory": 0.04128812626004219, "state_peak": 29.277332305908203, "growth_active": 1.0151705741882324, "growth_memory": 1.6558091640472412, "heads": [{"head": 0, "memory_to_active": 1.3589330911636353, "contribution_to_active": 0.17736592888832092, "mix_mean": 0.04578884690999985, "mix_low": 0.456756591796875, "mix_high": 1.33514404296875e-05, "read_cosine": 0.2905324399471283, "beta_mean": 0.02794136106967926, "nonfinite": 0.0, "weighted_memory_to_active": 0.1662582904100418, "mix_std": 0.10005698353052139, "beta_std": 0.020610535517334938, "beta_low": 0.07623291015625, "beta_high": 0.0, "orthogonal_to_active": 0.00485945725813508, "evidence_weights": [0.030871747061610222, -0.006966512184590101, -0.06814748048782349], "agreement_weight": -0.07126708328723907, "orthogonal_weight": -0.029563629999756813}, {"head": 1, "memory_to_active": 0.37463468313217163, "contribution_to_active": 0.15459397435188293, "mix_mean": 0.07330766320228577, "mix_low": 0.2485208511352539, "mix_high": 2.193450927734375e-05, "read_cosine": 0.30755940079689026, "beta_mean": 0.0018446772592142224, "nonfinite": 0.0, "weighted_memory_to_active": 0.031105952337384224, "mix_std": 0.1310107707977295, "beta_std": 0.0035942215472459793, "beta_low": 0.9677734375, "beta_high": 0.0, "orthogonal_to_active": 0.002621316583827138, "evidence_weights": [-0.03950100764632225, -0.0326632522046566, 0.0958537757396698], "agreement_weight": -0.04194828122854233, "orthogonal_weight": -0.09638174623250961}, {"head": 2, "memory_to_active": 3.240412473678589, "contribution_to_active": 0.3135550618171692, "mix_mean": 0.03354352340102196, "mix_low": 0.3861980438232422, "mix_high": 1.9073486328125e-06, "read_cosine": 0.2351427525281906, "beta_mean": 0.3322969675064087, "nonfinite": 0.0, "weighted_memory_to_active": 0.4105525314807892, "mix_std": 0.07951053977012634, "beta_std": 0.18765795230865479, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.05483955144882202, "evidence_weights": [-0.03275331109762192, -0.010010955855250359, 0.10623359680175781], "agreement_weight": -0.03039437346160412, "orthogonal_weight": -0.16102387011051178}, {"head": 3, "memory_to_active": 0.08484098315238953, "contribution_to_active": 0.36142006516456604, "mix_mean": 0.25424402952194214, "mix_low": 0.03790569305419922, "mix_high": 0.0013341903686523438, "read_cosine": 0.32547086477279663, "beta_mean": 0.0002077064709737897, "nonfinite": 0.0, "weighted_memory_to_active": 0.033591583371162415, "mix_std": 0.2478758692741394, "beta_std": 0.0006304214475676417, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.004502443131059408, "evidence_weights": [-0.1489577293395996, -0.16038745641708374, 0.3493768870830536], "agreement_weight": 0.23415526747703552, "orthogonal_weight": -0.19809475541114807}, {"head": 4, "memory_to_active": 4.987863540649414, "contribution_to_active": 0.05861895531415939, "mix_mean": 0.006569900084286928, "mix_low": 0.8538970947265625, "mix_high": 0.0, "read_cosine": 0.21866419911384583, "beta_mean": 0.4961530864238739, "nonfinite": 0.0, "weighted_memory_to_active": 0.06854325532913208, "mix_std": 0.013020521029829979, "beta_std": 0.21228240430355072, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.010738388635218143, "evidence_weights": [0.1228451207280159, 0.11147570610046387, -0.16468681395053864], "agreement_weight": -0.098875030875206, "orthogonal_weight": -0.15961387753486633}, {"head": 5, "memory_to_active": 0.6171375513076782, "contribution_to_active": 0.5290660858154297, "mix_mean": 0.09149068593978882, "mix_low": 0.13320064544677734, "mix_high": 0.004124641418457031, "read_cosine": 0.53758704662323, "beta_mean": 0.20087365806102753, "nonfinite": 0.0, "weighted_memory_to_active": 0.26696598529815674, "mix_std": 0.17041820287704468, "beta_std": 0.17553867399692535, "beta_low": 0.010498046875, "beta_high": 0.0, "orthogonal_to_active": 0.036996886134147644, "evidence_weights": [-0.022721273824572563, -0.11051337420940399, 0.017392918467521667], "agreement_weight": 0.1505139172077179, "orthogonal_weight": -0.22368106245994568}]}, {"layer": 4, "memory_to_active": 1.3453278541564941, "contribution_to_active": 0.24954938888549805, "mix_mean": 0.1705547720193863, "mix_low": 0.2577770948410034, "mix_high": 0.00046459835721179843, "read_cosine": 0.41570180654525757, "beta_mean": 0.13457688689231873, "nonfinite": 0.0, "weighted_memory_to_active": 0.14441199600696564, "mix_std": 0.13191744685173035, "beta_std": 0.09975678473711014, "beta_low": 0.2057393491268158, "beta_high": 0.0, "orthogonal_to_active": 0.02505474165081978, "output_influence": 0.16908930242061615, "residual_influence": 0.031981755048036575, "state_active": 0.048870302736759186, "state_memory": 0.044047679752111435, "state_peak": 42.24285125732422, "growth_active": 1.0646179914474487, "growth_memory": 1.7068920135498047, "heads": [{"head": 0, "memory_to_active": 0.4547881782054901, "contribution_to_active": 0.3866170048713684, "mix_mean": 0.27489253878593445, "mix_low": 0.013165473937988281, "mix_high": 0.0002651214599609375, "read_cosine": 0.6230463981628418, "beta_mean": 0.060045480728149414, "nonfinite": 0.0, "weighted_memory_to_active": 0.22859814763069153, "mix_std": 0.22970329225063324, "beta_std": 0.07596683502197266, "beta_low": 0.12371826171875, "beta_high": 0.0, "orthogonal_to_active": 0.0038154777139425278, "evidence_weights": [-0.143961101770401, -0.1880945861339569, 0.3076421618461609], "agreement_weight": 0.07695397734642029, "orthogonal_weight": -0.024050883948802948}, {"head": 1, "memory_to_active": 0.6413845419883728, "contribution_to_active": 0.20111879706382751, "mix_mean": 0.1839754730463028, "mix_low": 0.03659820556640625, "mix_high": 5.7220458984375e-06, "read_cosine": 0.5084125995635986, "beta_mean": 0.05100477486848831, "nonfinite": 0.0, "weighted_memory_to_active": 0.1702086478471756, "mix_std": 0.18925359845161438, "beta_std": 0.05140145868062973, "beta_low": 0.044677734375, "beta_high": 0.0, "orthogonal_to_active": 0.05067805200815201, "evidence_weights": [-0.125481978058815, -0.12876036763191223, 0.3989187479019165], "agreement_weight": 0.05294504389166832, "orthogonal_weight": -0.3541039526462555}, {"head": 2, "memory_to_active": 2.6739933490753174, "contribution_to_active": 0.06949719786643982, "mix_mean": 0.010994620621204376, "mix_low": 0.6991233825683594, "mix_high": 0.0, "read_cosine": 0.26816326379776, "beta_mean": 0.19593888521194458, "nonfinite": 0.0, "weighted_memory_to_active": 0.08002159744501114, "mix_std": 0.017959164455533028, "beta_std": 0.14061006903648376, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024561858735978603, "evidence_weights": [0.031188620254397392, 0.019566137343645096, -0.06016556918621063], "agreement_weight": 0.025881269946694374, "orthogonal_weight": 0.03833808749914169}, {"head": 3, "memory_to_active": 1.9209985733032227, "contribution_to_active": 0.07560624927282333, "mix_mean": 0.02583901956677437, "mix_low": 0.3877992630004883, "mix_high": 0.0, "read_cosine": 0.21056389808654785, "beta_mean": 0.18834732472896576, "nonfinite": 0.0, "weighted_memory_to_active": 0.10572529584169388, "mix_std": 0.03979611396789551, "beta_std": 0.1655169129371643, "beta_low": 0.06610107421875, "beta_high": 0.0, "orthogonal_to_active": 0.01578703708946705, "evidence_weights": [-0.02132691629230976, 0.010088897310197353, -0.13612796366214752], "agreement_weight": 0.007778370752930641, "orthogonal_weight": -0.20905037224292755}, {"head": 4, "memory_to_active": 0.05253881961107254, "contribution_to_active": 0.4735332429409027, "mix_mean": 0.503172755241394, "mix_low": 0.0017375946044921875, "mix_high": 0.0024881362915039062, "read_cosine": 0.41243061423301697, "beta_mean": 0.00012105301721021533, "nonfinite": 0.0, "weighted_memory_to_active": 0.02913273498415947, "mix_std": 0.271587073802948, "beta_std": 0.0002600011066533625, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0033219321630895138, "evidence_weights": [-0.2437307983636856, -0.25878092646598816, 0.43902984261512756], "agreement_weight": -0.004268568009138107, "orthogonal_weight": -0.14310210943222046}, {"head": 5, "memory_to_active": 2.3282628059387207, "contribution_to_active": 0.29092374444007874, "mix_mean": 0.024454185739159584, "mix_low": 0.40823841094970703, "mix_high": 2.86102294921875e-05, "read_cosine": 0.4715942442417145, "beta_mean": 0.3120037913322449, "nonfinite": 0.0, "weighted_memory_to_active": 0.25278547406196594, "mix_std": 0.043205395340919495, "beta_std": 0.16478534042835236, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07426974922418594, "evidence_weights": [-0.006370693910866976, -0.04418760910630226, 0.08216173201799393], "agreement_weight": -0.048413343727588654, "orthogonal_weight": 0.4343871772289276}]}, {"layer": 5, "memory_to_active": 0.7965085506439209, "contribution_to_active": 0.3857286274433136, "mix_mean": 0.3097529411315918, "mix_low": 0.12661585211753845, "mix_high": 0.005801836960017681, "read_cosine": 0.24433180689811707, "beta_mean": 0.08286450058221817, "nonfinite": 0.0, "weighted_memory_to_active": 0.1628618985414505, "mix_std": 0.22852866351604462, "beta_std": 0.08508576452732086, "beta_low": 0.4296264946460724, "beta_high": 0.0, "orthogonal_to_active": 0.029746495187282562, "output_influence": 0.3337211012840271, "residual_influence": 0.06238517165184021, "state_active": 0.07413014024496078, "state_memory": 0.03468434512615204, "state_peak": 58.63242721557617, "growth_active": 1.102941870689392, "growth_memory": 1.6410279273986816, "heads": [{"head": 0, "memory_to_active": 1.8263105154037476, "contribution_to_active": 0.23941953480243683, "mix_mean": 0.058846160769462585, "mix_low": 0.23682117462158203, "mix_high": 0.0, "read_cosine": -0.17580267786979675, "beta_mean": 0.15523217618465424, "nonfinite": 0.0, "weighted_memory_to_active": 0.2073090523481369, "mix_std": 0.0982484519481659, "beta_std": 0.1491093635559082, "beta_low": 0.00811767578125, "beta_high": 0.0, "orthogonal_to_active": 0.026141557842493057, "evidence_weights": [-0.024940671399235725, -0.03789008408784866, -0.20806536078453064], "agreement_weight": -0.022830618545413017, "orthogonal_weight": -0.132449209690094}, {"head": 1, "memory_to_active": 0.023994557559490204, "contribution_to_active": 0.3503754436969757, "mix_mean": 0.35429614782333374, "mix_low": 0.03107166290283203, "mix_high": 0.004947662353515625, "read_cosine": 0.1381213366985321, "beta_mean": 6.383581785485148e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008087236434221268, "mix_std": 0.30793890357017517, "beta_std": 0.00021116268180776387, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004480727948248386, "evidence_weights": [-0.17119993269443512, -0.15717482566833496, 0.21100372076034546], "agreement_weight": 0.15462005138397217, "orthogonal_weight": -0.573710024356842}, {"head": 2, "memory_to_active": 0.28866177797317505, "contribution_to_active": 0.4214552044868469, "mix_mean": 0.43458715081214905, "mix_low": 0.0022649765014648438, "mix_high": 0.0040340423583984375, "read_cosine": 0.442216157913208, "beta_mean": 0.016911305487155914, "nonfinite": 0.0, "weighted_memory_to_active": 0.17209187150001526, "mix_std": 0.2687858045101166, "beta_std": 0.012344169430434704, "beta_low": 0.31390380859375, "beta_high": 0.0, "orthogonal_to_active": 0.01580485887825489, "evidence_weights": [-0.266119122505188, -0.2646060287952423, 0.459330290555954], "agreement_weight": 0.05004354938864708, "orthogonal_weight": -0.11658311635255814}, {"head": 3, "memory_to_active": 2.0482583045959473, "contribution_to_active": 0.13937410712242126, "mix_mean": 0.0378514900803566, "mix_low": 0.41039466857910156, "mix_high": 9.5367431640625e-07, "read_cosine": 0.23680439591407776, "beta_mean": 0.09101002663373947, "nonfinite": 0.0, "weighted_memory_to_active": 0.19237597286701202, "mix_std": 0.07754035294055939, "beta_std": 0.10762279480695724, "beta_low": 0.12042236328125, "beta_high": 0.0, "orthogonal_to_active": 0.05926194414496422, "evidence_weights": [0.01442459411919117, -0.011983406729996204, 0.10205598175525665], "agreement_weight": 0.04854925721883774, "orthogonal_weight": -0.3413311243057251}, {"head": 4, "memory_to_active": 0.5461291074752808, "contribution_to_active": 0.5793232321739197, "mix_mean": 0.35030490159988403, "mix_low": 0.07592010498046875, "mix_high": 0.014423370361328125, "read_cosine": 0.6127514839172363, "beta_mean": 0.23382484912872314, "nonfinite": 0.0, "weighted_memory_to_active": 0.36868494749069214, "mix_std": 0.3341222405433655, "beta_std": 0.24091127514839172, "beta_low": 0.135498046875, "beta_high": 0.0, "orthogonal_to_active": 0.06567242741584778, "evidence_weights": [-0.170623317360878, -0.23861002922058105, 0.23987995088100433], "agreement_weight": 0.058196958154439926, "orthogonal_weight": 0.26550841331481934}, {"head": 5, "memory_to_active": 0.04569695144891739, "contribution_to_active": 0.5844240784645081, "mix_mean": 0.6226316690444946, "mix_low": 0.0032224655151367188, "mix_high": 0.011404991149902344, "read_cosine": 0.21190005540847778, "beta_mean": 0.000144780075061135, "nonfinite": 0.0, "weighted_memory_to_active": 0.02862231433391571, "mix_std": 0.2845361828804016, "beta_std": 0.00031584317912347615, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.007117457687854767, "evidence_weights": [-0.270081490278244, -0.26618313789367676, 0.48774319887161255], "agreement_weight": 0.18445290625095367, "orthogonal_weight": -0.2879132330417633}]}, {"layer": 6, "memory_to_active": 0.5885037183761597, "contribution_to_active": 0.4675418436527252, "mix_mean": 0.34128427505493164, "mix_low": 0.1268792301416397, "mix_high": 0.012523969635367393, "read_cosine": 0.35679975152015686, "beta_mean": 0.08625074476003647, "nonfinite": 0.0, "weighted_memory_to_active": 0.19414421916007996, "mix_std": 0.25473853945732117, "beta_std": 0.09909576177597046, "beta_low": 0.4229940176010132, "beta_high": 0.0, "orthogonal_to_active": 0.03395766019821167, "output_influence": 0.34599998593330383, "residual_influence": 0.08291774988174438, "state_active": 0.07245922833681107, "state_memory": 0.021372638642787933, "state_peak": 35.3886604309082, "growth_active": 1.227193832397461, "growth_memory": 1.4737368822097778, "heads": [{"head": 0, "memory_to_active": 0.2906246781349182, "contribution_to_active": 0.39205580949783325, "mix_mean": 0.2843902111053467, "mix_low": 0.03212165832519531, "mix_high": 0.00087738037109375, "read_cosine": 0.4305896461009979, "beta_mean": 0.03637992590665817, "nonfinite": 0.0, "weighted_memory_to_active": 0.1934954971075058, "mix_std": 0.25074106454849243, "beta_std": 0.09468144178390503, "beta_low": 0.26483154296875, "beta_high": 0.0, "orthogonal_to_active": 0.07785430550575256, "evidence_weights": [-0.14678654074668884, -0.160169780254364, 0.18100537359714508], "agreement_weight": 0.10016176104545593, "orthogonal_weight": -0.6081419587135315}, {"head": 1, "memory_to_active": 0.03105895221233368, "contribution_to_active": 0.3605177402496338, "mix_mean": 0.27657896280288696, "mix_low": 0.03114795684814453, "mix_high": 0.007681846618652344, "read_cosine": 0.4802825152873993, "beta_mean": 6.09318412898574e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00924313347786665, "mix_std": 0.26644688844680786, "beta_std": 0.0001209878537338227, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017128433100879192, "evidence_weights": [-0.18389837443828583, -0.1710597574710846, 0.19019363820552826], "agreement_weight": -0.003519532736390829, "orthogonal_weight": -0.23420366644859314}, {"head": 2, "memory_to_active": 2.1437301635742188, "contribution_to_active": 0.08453816175460815, "mix_mean": 0.016793377697467804, "mix_low": 0.6569738388061523, "mix_high": 0.0, "read_cosine": 0.20592018961906433, "beta_mean": 0.12559829652309418, "nonfinite": 0.0, "weighted_memory_to_active": 0.10139469802379608, "mix_std": 0.045934733003377914, "beta_std": 0.11663107573986053, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.023874938488006592, "evidence_weights": [0.07844666391611099, 0.06287012994289398, -0.04751981794834137], "agreement_weight": 0.018632857128977776, "orthogonal_weight": -0.24077071249485016}, {"head": 3, "memory_to_active": 0.2753429710865021, "contribution_to_active": 0.6073401570320129, "mix_mean": 0.5311672687530518, "mix_low": 0.008876800537109375, "mix_high": 0.0296478271484375, "read_cosine": 0.5849325060844421, "beta_mean": 0.23477645218372345, "nonfinite": 0.0, "weighted_memory_to_active": 0.17742004990577698, "mix_std": 0.3481020927429199, "beta_std": 0.24506717920303345, "beta_low": 0.1214599609375, "beta_high": 0.0, "orthogonal_to_active": 0.04028654098510742, "evidence_weights": [-0.1288636028766632, -0.25699567794799805, 0.2939590513706207], "agreement_weight": -0.012296821922063828, "orthogonal_weight": 0.2891972064971924}, {"head": 4, "memory_to_active": 0.7118051648139954, "contribution_to_active": 0.9443379640579224, "mix_mean": 0.5940016508102417, "mix_low": 0.003795623779296875, "mix_high": 0.036739349365234375, "read_cosine": 0.17603659629821777, "beta_mean": 0.12032497674226761, "nonfinite": 0.0, "weighted_memory_to_active": 0.650283694267273, "mix_std": 0.3285529315471649, "beta_std": 0.1375923454761505, "beta_low": 0.151611328125, "beta_high": 0.0, "orthogonal_to_active": 0.03043653815984726, "evidence_weights": [-0.21722373366355896, -0.40613988041877747, -0.10691338032484055], "agreement_weight": -0.013453260064125061, "orthogonal_weight": 0.05744915083050728}, {"head": 5, "memory_to_active": 0.07846026122570038, "contribution_to_active": 0.41646111011505127, "mix_mean": 0.3447742462158203, "mix_low": 0.028359413146972656, "mix_high": 0.00019741058349609375, "read_cosine": 0.2630370259284973, "beta_mean": 0.00036384997656568885, "nonfinite": 0.0, "weighted_memory_to_active": 0.0330282524228096, "mix_std": 0.28865349292755127, "beta_std": 0.00048160634469240904, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.029580796137452126, "evidence_weights": [-0.15262724459171295, -0.19503876566886902, -0.10425086319446564], "agreement_weight": 0.07044143229722977, "orthogonal_weight": -0.9515283107757568}]}, {"layer": 7, "memory_to_active": 2.3359460830688477, "contribution_to_active": 0.1844796985387802, "mix_mean": 0.02386273816227913, "mix_low": 0.5791952013969421, "mix_high": 6.0399379435693845e-06, "read_cosine": 0.21923542022705078, "beta_mean": 0.1762358695268631, "nonfinite": 0.0, "weighted_memory_to_active": 0.18592657148838043, "mix_std": 0.04539085179567337, "beta_std": 0.11126517504453659, "beta_low": 0.00482177734375, "beta_high": 0.0, "orthogonal_to_active": 0.022322535514831543, "output_influence": 0.19907213747501373, "residual_influence": 0.038669683039188385, "state_active": 0.02281545102596283, "state_memory": 0.04338077828288078, "state_peak": 17.208406448364258, "growth_active": 1.0032345056533813, "growth_memory": 1.5177443027496338, "heads": [{"head": 0, "memory_to_active": 2.245509147644043, "contribution_to_active": 0.4648239016532898, "mix_mean": 0.026676543056964874, "mix_low": 0.5021514892578125, "mix_high": 3.62396240234375e-05, "read_cosine": 0.3953428268432617, "beta_mean": 0.17108717560768127, "nonfinite": 0.0, "weighted_memory_to_active": 0.4356551170349121, "mix_std": 0.05437640845775604, "beta_std": 0.11084827035665512, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08198971301317215, "evidence_weights": [-0.02385859563946724, -0.06360415369272232, 0.11077450215816498], "agreement_weight": -0.03623737767338753, "orthogonal_weight": 0.2734377682209015}, {"head": 1, "memory_to_active": 3.076399326324463, "contribution_to_active": 0.2238803207874298, "mix_mean": 0.0422724112868309, "mix_low": 0.3143119812011719, "mix_high": 0.0, "read_cosine": -0.031811702996492386, "beta_mean": 0.3110259473323822, "nonfinite": 0.0, "weighted_memory_to_active": 0.25986945629119873, "mix_std": 0.060805078595876694, "beta_std": 0.14012840390205383, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04160046577453613, "evidence_weights": [-0.07565747946500778, -0.07293008267879486, 0.07318756729364395], "agreement_weight": -0.0002640542224980891, "orthogonal_weight": -0.15983879566192627}, {"head": 2, "memory_to_active": 1.594892144203186, "contribution_to_active": 0.26523357629776, "mix_mean": 0.04418970271945, "mix_low": 0.4384956359863281, "mix_high": 0.0, "read_cosine": 0.19619899988174438, "beta_mean": 0.17442676424980164, "nonfinite": 0.0, "weighted_memory_to_active": 0.2507938742637634, "mix_std": 0.09453018754720688, "beta_std": 0.09618130326271057, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.0012136056320741773, "evidence_weights": [-0.045655302703380585, -0.09537133574485779, 0.1839013695716858], "agreement_weight": 0.024305785074830055, "orthogonal_weight": -0.005259499419480562}, {"head": 3, "memory_to_active": 2.991110324859619, "contribution_to_active": 0.03269454091787338, "mix_mean": 0.006918884813785553, "mix_low": 0.8070306777954102, "mix_high": 0.0, "read_cosine": 0.1382739543914795, "beta_mean": 0.1456337571144104, "nonfinite": 0.0, "weighted_memory_to_active": 0.039241984486579895, "mix_std": 0.014225706458091736, "beta_std": 0.10302051156759262, "beta_low": 0.00494384765625, "beta_high": 0.0, "orthogonal_to_active": 0.00694043654948473, "evidence_weights": [0.09108474850654602, 0.09205012768507004, 0.009638339281082153], "agreement_weight": -0.019371390342712402, "orthogonal_weight": -0.1805407851934433}, {"head": 4, "memory_to_active": 1.6613572835922241, "contribution_to_active": 0.06700025498867035, "mix_mean": 0.01447639986872673, "mix_low": 0.6418266296386719, "mix_high": 0.0, "read_cosine": 0.2973507046699524, "beta_mean": 0.15071500837802887, "nonfinite": 0.0, "weighted_memory_to_active": 0.06759938597679138, "mix_std": 0.032141316682100296, "beta_std": 0.12671613693237305, "beta_low": 0.01739501953125, "beta_high": 0.0, "orthogonal_to_active": 0.0011064617428928614, "evidence_weights": [0.03695666044950485, 0.028914399445056915, -0.008330088108778], "agreement_weight": 0.04906149208545685, "orthogonal_weight": -0.02085142955183983}, {"head": 5, "memory_to_active": 2.4464077949523926, "contribution_to_active": 0.053245533257722855, "mix_mean": 0.008642485365271568, "mix_low": 0.7713546752929688, "mix_high": 0.0, "read_cosine": 0.3200576901435852, "beta_mean": 0.10452648997306824, "nonfinite": 0.0, "weighted_memory_to_active": 0.06239958852529526, "mix_std": 0.01626640558242798, "beta_std": 0.09069640934467316, "beta_low": 0.006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.0010845327051356435, "evidence_weights": [0.03605876490473747, 0.061076413840055466, -0.15795883536338806], "agreement_weight": -0.0032484896946698427, "orthogonal_weight": -0.022427015006542206}]}, {"layer": 8, "memory_to_active": 1.0896801948547363, "contribution_to_active": 0.30561256408691406, "mix_mean": 0.20668399333953857, "mix_low": 0.1345500946044922, "mix_high": 0.0020119349937886, "read_cosine": 0.5190144181251526, "beta_mean": 0.08871442079544067, "nonfinite": 0.0, "weighted_memory_to_active": 0.18750861287117004, "mix_std": 0.16515704989433289, "beta_std": 0.07150736451148987, "beta_low": 0.210479736328125, "beta_high": 0.0, "orthogonal_to_active": 0.03687272593379021, "output_influence": 0.25992926955223083, "residual_influence": 0.0552683025598526, "state_active": 0.05949859321117401, "state_memory": 0.03339178115129471, "state_peak": 21.336959838867188, "growth_active": 0.9618093967437744, "growth_memory": 1.5768535137176514, "heads": [{"head": 0, "memory_to_active": 0.08960124850273132, "contribution_to_active": 0.4252745509147644, "mix_mean": 0.42230212688446045, "mix_low": 0.00064849853515625, "mix_high": 5.626678466796875e-05, "read_cosine": 0.7230641841888428, "beta_mean": 0.003228742629289627, "nonfinite": 0.0, "weighted_memory_to_active": 0.04168200120329857, "mix_std": 0.2264156937599182, "beta_std": 0.004805090371519327, "beta_low": 0.93890380859375, "beta_high": 0.0, "orthogonal_to_active": 0.012482533231377602, "evidence_weights": [-0.25701403617858887, -0.26371490955352783, 0.40675127506256104], "agreement_weight": 0.06722932308912277, "orthogonal_weight": -0.48161107301712036}, {"head": 1, "memory_to_active": 0.805250883102417, "contribution_to_active": 0.3018374443054199, "mix_mean": 0.266862154006958, "mix_low": 0.009950637817382812, "mix_high": 2.09808349609375e-05, "read_cosine": 0.6237436532974243, "beta_mean": 0.07694722712039948, "nonfinite": 0.0, "weighted_memory_to_active": 0.2783571481704712, "mix_std": 0.1790333092212677, "beta_std": 0.051717422902584076, "beta_low": 0.015380859375, "beta_high": 0.0, "orthogonal_to_active": 0.032258838415145874, "evidence_weights": [-0.17477910220623016, -0.1665772795677185, 0.3012867271900177], "agreement_weight": 0.0013572834432125092, "orthogonal_weight": 0.15902632474899292}, {"head": 2, "memory_to_active": 1.991361141204834, "contribution_to_active": 0.11344276368618011, "mix_mean": 0.031210899353027344, "mix_low": 0.38002490997314453, "mix_high": 0.0, "read_cosine": 0.2233283370733261, "beta_mean": 0.15975421667099, "nonfinite": 0.0, "weighted_memory_to_active": 0.17407716810703278, "mix_std": 0.054885756224393845, "beta_std": 0.11681221425533295, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.04232874512672424, "evidence_weights": [0.005264580715447664, -0.04168839752674103, 0.01767323911190033], "agreement_weight": 0.07852975279092789, "orthogonal_weight": -0.275519996881485}, {"head": 3, "memory_to_active": 0.7592853307723999, "contribution_to_active": 0.1942378282546997, "mix_mean": 0.15094710886478424, "mix_low": 0.043282508850097656, "mix_high": 0.0, "read_cosine": 0.6653530597686768, "beta_mean": 0.06246978044509888, "nonfinite": 0.0, "weighted_memory_to_active": 0.20785421133041382, "mix_std": 0.13962024450302124, "beta_std": 0.048479512333869934, "beta_low": 0.040771484375, "beta_high": 0.0, "orthogonal_to_active": 0.037520632147789, "evidence_weights": [-0.09927589446306229, -0.12131597846746445, 0.4636186361312866], "agreement_weight": 0.0322701632976532, "orthogonal_weight": 0.31016597151756287}, {"head": 4, "memory_to_active": 2.645909309387207, "contribution_to_active": 0.2619679868221283, "mix_mean": 0.033696260303258896, "mix_low": 0.27839088439941406, "mix_high": 0.0, "read_cosine": 0.33757245540618896, "beta_mean": 0.15275637805461884, "nonfinite": 0.0, "weighted_memory_to_active": 0.2699054181575775, "mix_std": 0.04594588279724121, "beta_std": 0.09441671520471573, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.0065226806327700615, "evidence_weights": [-0.007185987662523985, -0.02856445498764515, 0.01333136111497879], "agreement_weight": 0.006201483309268951, "orthogonal_weight": 0.027029449120163918}, {"head": 5, "memory_to_active": 0.24667324125766754, "contribution_to_active": 0.5369148850440979, "mix_mean": 0.335085391998291, "mix_low": 0.09500312805175781, "mix_high": 0.011994361877441406, "read_cosine": 0.5410246849060059, "beta_mean": 0.07713018357753754, "nonfinite": 0.0, "weighted_memory_to_active": 0.15317565202713013, "mix_std": 0.3450413942337036, "beta_std": 0.11281322687864304, "beta_low": 0.26690673828125, "beta_high": 0.0, "orthogonal_to_active": 0.09012292325496674, "evidence_weights": [-0.11367017030715942, -0.3192014992237091, -0.1831587553024292], "agreement_weight": 0.031081683933734894, "orthogonal_weight": 0.7922377586364746}]}, {"layer": 9, "memory_to_active": 1.4294410943984985, "contribution_to_active": 0.46013373136520386, "mix_mean": 0.11948304623365402, "mix_low": 0.2946963310241699, "mix_high": 0.00272369384765625, "read_cosine": 0.33441659808158875, "beta_mean": 0.11735163629055023, "nonfinite": 0.0, "weighted_memory_to_active": 0.36352112889289856, "mix_std": 0.17929360270500183, "beta_std": 0.0583915039896965, "beta_low": 0.388214111328125, "beta_high": 0.0, "orthogonal_to_active": 0.011518796905875206, "output_influence": 0.27471446990966797, "residual_influence": 0.05712765082716942, "state_active": 0.03387323394417763, "state_memory": 0.037717532366514206, "state_peak": 35.029449462890625, "growth_active": 1.0269333124160767, "growth_memory": 1.5921701192855835, "heads": [{"head": 0, "memory_to_active": 0.049025289714336395, "contribution_to_active": 0.27344632148742676, "mix_mean": 0.2279641330242157, "mix_low": 0.11031818389892578, "mix_high": 0.00077056884765625, "read_cosine": 0.21688610315322876, "beta_mean": 0.00012268972932361066, "nonfinite": 0.0, "weighted_memory_to_active": 0.013635561801493168, "mix_std": 0.25636783242225647, "beta_std": 0.00030021544080227613, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005096866749227047, "evidence_weights": [-0.12491282820701599, -0.13230372965335846, 0.5718926191329956], "agreement_weight": 0.026808198541402817, "orthogonal_weight": -0.39476245641708374}, {"head": 1, "memory_to_active": 0.10047304630279541, "contribution_to_active": 0.2602863311767578, "mix_mean": 0.10889904201030731, "mix_low": 0.3231840133666992, "mix_high": 0.000110626220703125, "read_cosine": 0.39222288131713867, "beta_mean": 0.000418578099925071, "nonfinite": 0.0, "weighted_memory_to_active": 0.02200555056333542, "mix_std": 0.18143875896930695, "beta_std": 0.0007753927493467927, "beta_low": 0.9990234375, "beta_high": 0.0, "orthogonal_to_active": 0.0047391122207045555, "evidence_weights": [-0.0910806953907013, -0.08031752705574036, 0.1818416863679886], "agreement_weight": 0.16753244400024414, "orthogonal_weight": -0.23127391934394836}, {"head": 2, "memory_to_active": 0.901444673538208, "contribution_to_active": 0.35131707787513733, "mix_mean": 0.15879979729652405, "mix_low": 0.1863412857055664, "mix_high": 7.534027099609375e-05, "read_cosine": 0.17168062925338745, "beta_mean": 0.02306079864501953, "nonfinite": 0.0, "weighted_memory_to_active": 0.2074136584997177, "mix_std": 0.2222130447626114, "beta_std": 0.027763482183218002, "beta_low": 0.33001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.010739282704889774, "evidence_weights": [-0.08074867725372314, -0.07409725338220596, 0.09386958926916122], "agreement_weight": 0.08535052090883255, "orthogonal_weight": -0.058855392038822174}, {"head": 3, "memory_to_active": 2.2840218544006348, "contribution_to_active": 0.2411142736673355, "mix_mean": 0.04648905247449875, "mix_low": 0.31824398040771484, "mix_high": 0.0, "read_cosine": 0.4392653703689575, "beta_mean": 0.18749675154685974, "nonfinite": 0.0, "weighted_memory_to_active": 0.22568431496620178, "mix_std": 0.08822154998779297, "beta_std": 0.10813380777835846, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.009532853029668331, "evidence_weights": [0.022832946851849556, 0.015480099245905876, -0.029171759262681007], "agreement_weight": -0.044247761368751526, "orthogonal_weight": 0.042185716331005096}, {"head": 4, "memory_to_active": 3.3298633098602295, "contribution_to_active": 1.1815099716186523, "mix_mean": 0.12536302208900452, "mix_low": 0.28252410888671875, "mix_high": 0.015363693237304688, "read_cosine": 0.36455297470092773, "beta_mean": 0.27861061692237854, "nonfinite": 0.0, "weighted_memory_to_active": 1.1710703372955322, "mix_std": 0.21253204345703125, "beta_std": 0.1142323687672615, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025467772502452135, "evidence_weights": [-0.04083220288157463, -0.07473734021186829, 0.10464870184659958], "agreement_weight": -0.06140248849987984, "orthogonal_weight": 0.002243505325168371}, {"head": 5, "memory_to_active": 1.9118177890777588, "contribution_to_active": 0.4531283974647522, "mix_mean": 0.049383215606212616, "mix_low": 0.5475664138793945, "mix_high": 2.193450927734375e-05, "read_cosine": 0.42189162969589233, "beta_mean": 0.21440033614635468, "nonfinite": 0.0, "weighted_memory_to_active": 0.5413172245025635, "mix_std": 0.11498837172985077, "beta_std": 0.09914374351501465, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.036457885056734085, "evidence_weights": [0.030021866783499718, -0.019543638452887535, 0.03250005096197128], "agreement_weight": -0.02936619706451893, "orthogonal_weight": 0.08882421255111694}]}, {"layer": 10, "memory_to_active": 0.5342770218849182, "contribution_to_active": 0.37919268012046814, "mix_mean": 0.23499250411987305, "mix_low": 0.15534275770187378, "mix_high": 0.006557147018611431, "read_cosine": 0.3680327534675598, "beta_mean": 0.05797889083623886, "nonfinite": 0.0, "weighted_memory_to_active": 0.15191717445850372, "mix_std": 0.21652823686599731, "beta_std": 0.06211623176932335, "beta_low": 0.519256591796875, "beta_high": 0.0, "orthogonal_to_active": 0.03734644129872322, "output_influence": 0.3577861189842224, "residual_influence": 0.09009856730699539, "state_active": 0.08694783598184586, "state_memory": 0.024597465991973877, "state_peak": 50.81972885131836, "growth_active": 1.1812289953231812, "growth_memory": 1.4906952381134033, "heads": [{"head": 0, "memory_to_active": 0.5922552347183228, "contribution_to_active": 0.8303255438804626, "mix_mean": 0.566885769367218, "mix_low": 0.014065742492675781, "mix_high": 0.03693199157714844, "read_cosine": 0.13927136361598969, "beta_mean": 0.1484278440475464, "nonfinite": 0.0, "weighted_memory_to_active": 0.38390883803367615, "mix_std": 0.34495028853416443, "beta_std": 0.18220408260822296, "beta_low": 0.128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.11853411793708801, "evidence_weights": [-0.13889876008033752, -0.292144238948822, -0.09374804049730301], "agreement_weight": 0.01814378798007965, "orthogonal_weight": 0.3354818820953369}, {"head": 1, "memory_to_active": 0.15003149211406708, "contribution_to_active": 0.2530963718891144, "mix_mean": 0.1594817340373993, "mix_low": 0.10613250732421875, "mix_high": 0.00023937225341796875, "read_cosine": 0.1771334409713745, "beta_mean": 0.0010442864149808884, "nonfinite": 0.0, "weighted_memory_to_active": 0.024586215615272522, "mix_std": 0.20119039714336395, "beta_std": 0.0017949249595403671, "beta_low": 0.99310302734375, "beta_high": 0.0, "orthogonal_to_active": 0.008866382762789726, "evidence_weights": [-0.10911533981561661, -0.09159128367900848, 0.035661883652210236], "agreement_weight": 0.05017415061593056, "orthogonal_weight": -0.36915501952171326}, {"head": 2, "memory_to_active": 2.056495428085327, "contribution_to_active": 0.29092270135879517, "mix_mean": 0.02142651379108429, "mix_low": 0.6586265563964844, "mix_high": 0.0, "read_cosine": 0.2174316942691803, "beta_mean": 0.13581719994544983, "nonfinite": 0.0, "weighted_memory_to_active": 0.3433188199996948, "mix_std": 0.05975564941763878, "beta_std": 0.09303267300128937, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.046416010707616806, "evidence_weights": [0.04184360057115555, -0.0004845850926358253, 0.05528973042964935], "agreement_weight": -0.023305198177695274, "orthogonal_weight": -0.1403491348028183}, {"head": 3, "memory_to_active": 0.1690589040517807, "contribution_to_active": 0.2716512680053711, "mix_mean": 0.25251829624176025, "mix_low": 0.08858585357666016, "mix_high": 0.002109527587890625, "read_cosine": 0.7458951473236084, "beta_mean": 0.053596459329128265, "nonfinite": 0.0, "weighted_memory_to_active": 0.07187741994857788, "mix_std": 0.29242414236068726, "beta_std": 0.08245624601840973, "beta_low": 0.24871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.0332484245300293, "evidence_weights": [-0.11897652596235275, -0.15511366724967957, 0.2442537397146225], "agreement_weight": 0.02025274932384491, "orthogonal_weight": 0.622198224067688}, {"head": 4, "memory_to_active": 0.07148724049329758, "contribution_to_active": 0.23247313499450684, "mix_mean": 0.1687132716178894, "mix_low": 0.053307533264160156, "mix_high": 2.956390380859375e-05, "read_cosine": 0.1753169149160385, "beta_mean": 0.0003606481186579913, "nonfinite": 0.0, "weighted_memory_to_active": 0.01685887761414051, "mix_std": 0.19442103803157806, "beta_std": 0.001786577980965376, "beta_low": 0.99871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.004020776599645615, "evidence_weights": [-0.1199444979429245, -0.08898954838514328, -0.018878581002354622], "agreement_weight": 0.029852261766791344, "orthogonal_weight": -0.26348045468330383}, {"head": 5, "memory_to_active": 0.16633373498916626, "contribution_to_active": 0.3966871201992035, "mix_mean": 0.2409294694662094, "mix_low": 0.011338233947753906, "mix_high": 3.24249267578125e-05, "read_cosine": 0.7531478404998779, "beta_mean": 0.008626868948340416, "nonfinite": 0.0, "weighted_memory_to_active": 0.07095281779766083, "mix_std": 0.20642787218093872, "beta_std": 0.011422870680689812, "beta_low": 0.74664306640625, "beta_high": 0.0, "orthogonal_to_active": 0.012992925941944122, "evidence_weights": [-0.17748408019542694, -0.14303095638751984, 0.16012628376483917], "agreement_weight": 0.08559955656528473, "orthogonal_weight": -0.3743588328361511}]}, {"layer": 11, "memory_to_active": 0.35386407375335693, "contribution_to_active": 0.40302014350891113, "mix_mean": 0.30571573972702026, "mix_low": 0.057023368775844574, "mix_high": 0.02096700854599476, "read_cosine": 0.5393002033233643, "beta_mean": 0.042132578790187836, "nonfinite": 0.0, "weighted_memory_to_active": 0.10000066459178925, "mix_std": 0.2746849060058594, "beta_std": 0.05944560468196869, "beta_low": 0.4305623471736908, "beta_high": 0.0, "orthogonal_to_active": 0.02432643622159958, "output_influence": 0.4765450358390808, "residual_influence": 0.1195475235581398, "state_active": 0.12622010707855225, "state_memory": 0.01688627153635025, "state_peak": 22.90904426574707, "growth_active": 1.1909122467041016, "growth_memory": 1.3820250034332275, "heads": [{"head": 0, "memory_to_active": 0.09215841442346573, "contribution_to_active": 0.45391401648521423, "mix_mean": 0.3621255159378052, "mix_low": 0.012355804443359375, "mix_high": 0.0006313323974609375, "read_cosine": 0.4359712600708008, "beta_mean": 0.0006234533502720296, "nonfinite": 0.0, "weighted_memory_to_active": 0.04810519143939018, "mix_std": 0.27081727981567383, "beta_std": 0.004060089588165283, "beta_low": 0.99407958984375, "beta_high": 0.0, "orthogonal_to_active": 0.01983688212931156, "evidence_weights": [-0.15136685967445374, -0.2007799744606018, 0.2634086310863495], "agreement_weight": 0.12526337802410126, "orthogonal_weight": -0.5696622133255005}, {"head": 1, "memory_to_active": 0.598463773727417, "contribution_to_active": 0.18231628835201263, "mix_mean": 0.17612403631210327, "mix_low": 0.053946495056152344, "mix_high": 2.86102294921875e-05, "read_cosine": 0.6046022176742554, "beta_mean": 0.04123799130320549, "nonfinite": 0.0, "weighted_memory_to_active": 0.16140013933181763, "mix_std": 0.1966683268547058, "beta_std": 0.06900488585233688, "beta_low": 0.30291748046875, "beta_high": 0.0, "orthogonal_to_active": 0.012745242565870285, "evidence_weights": [-0.12916675209999084, -0.09270628541707993, 0.08518923819065094], "agreement_weight": 0.09019574522972107, "orthogonal_weight": -0.11093483865261078}, {"head": 2, "memory_to_active": 0.07235093414783478, "contribution_to_active": 0.8037339448928833, "mix_mean": 0.4592629671096802, "mix_low": 0.052863121032714844, "mix_high": 0.0720376968383789, "read_cosine": 0.3747732639312744, "beta_mean": 0.06806191056966782, "nonfinite": 0.0, "weighted_memory_to_active": 0.060946520417928696, "mix_std": 0.4009420871734619, "beta_std": 0.09890662878751755, "beta_low": 0.44842529296875, "beta_high": 0.0, "orthogonal_to_active": 0.061440009623765945, "evidence_weights": [-0.21554836630821228, -0.33574995398521423, -0.227172389626503], "agreement_weight": -0.008461934514343739, "orthogonal_weight": 1.1322970390319824}, {"head": 3, "memory_to_active": 0.04580506309866905, "contribution_to_active": 0.4246184229850769, "mix_mean": 0.4507150650024414, "mix_low": 0.05815410614013672, "mix_high": 0.052227020263671875, "read_cosine": 0.6200786828994751, "beta_mean": 0.06700673699378967, "nonfinite": 0.0, "weighted_memory_to_active": 0.024601714685559273, "mix_std": 0.40574750304222107, "beta_std": 0.09702956676483154, "beta_low": 0.42987060546875, "beta_high": 0.0, "orthogonal_to_active": 0.022412192076444626, "evidence_weights": [-0.27537357807159424, -0.3355523645877838, 0.07938015460968018], "agreement_weight": 0.015428661368787289, "orthogonal_weight": 1.1695808172225952}, {"head": 4, "memory_to_active": 1.0283321142196655, "contribution_to_active": 0.18170440196990967, "mix_mean": 0.09003470838069916, "mix_low": 0.15404415130615234, "mix_high": 0.0, "read_cosine": 0.30142420530319214, "beta_mean": 0.050137557089328766, "nonfinite": 0.0, "weighted_memory_to_active": 0.16097788512706757, "mix_std": 0.12921470403671265, "beta_std": 0.05397837609052658, "beta_low": 0.08184814453125, "beta_high": 0.0, "orthogonal_to_active": 0.006804639007896185, "evidence_weights": [-0.029779354110360146, -0.016854265704751015, 0.4100126326084137], "agreement_weight": -0.02581222727894783, "orthogonal_weight": -0.054485246539115906}, {"head": 5, "memory_to_active": 0.28607410192489624, "contribution_to_active": 0.3718336224555969, "mix_mean": 0.2960321605205536, "mix_low": 0.010776519775390625, "mix_high": 0.00087738037109375, "read_cosine": 0.8989514112472534, "beta_mean": 0.02572779357433319, "nonfinite": 0.0, "weighted_memory_to_active": 0.14397244155406952, "mix_std": 0.244719460606575, "beta_std": 0.03369408845901489, "beta_low": 0.32623291015625, "beta_high": 0.0, "orthogonal_to_active": 0.022719651460647583, "evidence_weights": [-0.1630103439092636, -0.1998574286699295, 0.2349647879600525], "agreement_weight": 0.05300144851207733, "orthogonal_weight": -0.45006296038627625}]}, {"layer": 12, "memory_to_active": 0.8060896992683411, "contribution_to_active": 0.44216495752334595, "mix_mean": 0.2858337461948395, "mix_low": 0.08602937310934067, "mix_high": 0.00036160153103992343, "read_cosine": 0.3090154230594635, "beta_mean": 0.033312857151031494, "nonfinite": 0.0, "weighted_memory_to_active": 0.29246506094932556, "mix_std": 0.21425655484199524, "beta_std": 0.03991062194108963, "beta_low": 0.5585530996322632, "beta_high": 0.0, "orthogonal_to_active": 0.0561756007373333, "output_influence": 0.24426260590553284, "residual_influence": 0.059164077043533325, "state_active": 0.056771501898765564, "state_memory": 0.024087466299533844, "state_peak": 41.11531066894531, "growth_active": 1.174268364906311, "growth_memory": 1.4385218620300293, "heads": [{"head": 0, "memory_to_active": 1.0782822370529175, "contribution_to_active": 0.2993665039539337, "mix_mean": 0.1845233142375946, "mix_low": 0.048272132873535156, "mix_high": 0.00030422210693359375, "read_cosine": 0.3983457684516907, "beta_mean": 0.0295572467148304, "nonfinite": 0.0, "weighted_memory_to_active": 0.4198181927204132, "mix_std": 0.19704443216323853, "beta_std": 0.05757667496800423, "beta_low": 0.41729736328125, "beta_high": 0.0, "orthogonal_to_active": 0.10046461969614029, "evidence_weights": [-0.09136573225259781, -0.1282108724117279, 0.24164827167987823], "agreement_weight": 0.09152857959270477, "orthogonal_weight": -0.32731327414512634}, {"head": 1, "memory_to_active": 0.3889699876308441, "contribution_to_active": 0.30868229269981384, "mix_mean": 0.26677119731903076, "mix_low": 0.008679389953613281, "mix_high": 0.0001163482666015625, "read_cosine": 0.5623351335525513, "beta_mean": 0.006520361639559269, "nonfinite": 0.0, "weighted_memory_to_active": 0.15016993880271912, "mix_std": 0.23189318180084229, "beta_std": 0.007722363341599703, "beta_low": 0.80718994140625, "beta_high": 0.0, "orthogonal_to_active": 0.07876825332641602, "evidence_weights": [-0.12321780622005463, -0.17065511643886566, 0.3028511703014374], "agreement_weight": 0.0825737789273262, "orthogonal_weight": -0.789767324924469}, {"head": 2, "memory_to_active": 0.6246684789657593, "contribution_to_active": 0.3458581864833832, "mix_mean": 0.2830312252044678, "mix_low": 0.019529342651367188, "mix_high": 0.00042724609375, "read_cosine": 0.29377683997154236, "beta_mean": 0.07918441295623779, "nonfinite": 0.0, "weighted_memory_to_active": 0.2737430930137634, "mix_std": 0.23504728078842163, "beta_std": 0.07567481696605682, "beta_low": 0.025390625, "beta_high": 0.0, "orthogonal_to_active": 0.031668975949287415, "evidence_weights": [-0.16423888504505157, -0.09869401156902313, 0.21055492758750916], "agreement_weight": 0.048509273678064346, "orthogonal_weight": 0.1429075002670288}, {"head": 3, "memory_to_active": 0.06288924813270569, "contribution_to_active": 0.5528475642204285, "mix_mean": 0.5212604403495789, "mix_low": 0.0021123886108398438, "mix_high": 0.00035953521728515625, "read_cosine": 0.004834556952118874, "beta_mean": 0.0007942946976982057, "nonfinite": 0.0, "weighted_memory_to_active": 0.03664540871977806, "mix_std": 0.2599088251590729, "beta_std": 0.0010586383286863565, "beta_low": 0.9990234375, "beta_high": 0.0, "orthogonal_to_active": 0.018228039145469666, "evidence_weights": [-0.2577815651893616, -0.2459176927804947, 0.12105941772460938], "agreement_weight": 0.05368566885590553, "orthogonal_weight": -0.5102058053016663}, {"head": 4, "memory_to_active": 2.5795726776123047, "contribution_to_active": 0.7733873724937439, "mix_mean": 0.043528445065021515, "mix_low": 0.43195152282714844, "mix_high": 0.0004253387451171875, "read_cosine": 0.13555562496185303, "beta_mean": 0.08192507922649384, "nonfinite": 0.0, "weighted_memory_to_active": 0.8358170986175537, "mix_std": 0.09462298452854156, "beta_std": 0.09321156144142151, "beta_low": 0.14556884765625, "beta_high": 0.0, "orthogonal_to_active": 0.08005543053150177, "evidence_weights": [0.05880768224596977, -0.011247335001826286, 0.06406039744615555], "agreement_weight": -0.022925090044736862, "orthogonal_weight": -0.10373467952013016}, {"head": 5, "memory_to_active": 0.10215521603822708, "contribution_to_active": 0.37284767627716064, "mix_mean": 0.4158877730369568, "mix_low": 0.005631446838378906, "mix_high": 0.0005369186401367188, "read_cosine": 0.45924463868141174, "beta_mean": 0.001895745168440044, "nonfinite": 0.0, "weighted_memory_to_active": 0.038596488535404205, "mix_std": 0.26702266931533813, "beta_std": 0.0042196703143417835, "beta_low": 0.95684814453125, "beta_high": 0.0, "orthogonal_to_active": 0.027868274599313736, "evidence_weights": [-0.2235412448644638, -0.2244792878627777, 0.34174537658691406], "agreement_weight": 0.03765353187918663, "orthogonal_weight": -0.8223552703857422}]}, {"layer": 13, "memory_to_active": 0.291645348072052, "contribution_to_active": 0.3477960228919983, "mix_mean": 0.3300842046737671, "mix_low": 0.009467920288443565, "mix_high": 0.0012981097679585218, "read_cosine": 0.35578715801239014, "beta_mean": 0.020497003570199013, "nonfinite": 0.0, "weighted_memory_to_active": 0.11269247531890869, "mix_std": 0.2335405796766281, "beta_std": 0.028859160840511322, "beta_low": 0.6785176992416382, "beta_high": 0.0, "orthogonal_to_active": 0.05823308601975441, "output_influence": 0.289764940738678, "residual_influence": 0.0900927409529686, "state_active": 0.09887517988681793, "state_memory": 0.028079040348529816, "state_peak": 54.00759506225586, "growth_active": 1.0409042835235596, "growth_memory": 1.2224828004837036, "heads": [{"head": 0, "memory_to_active": 0.35234296321868896, "contribution_to_active": 0.31151264905929565, "mix_mean": 0.2632831335067749, "mix_low": 0.010367393493652344, "mix_high": 0.000118255615234375, "read_cosine": 0.28000345826148987, "beta_mean": 0.007337168324738741, "nonfinite": 0.0, "weighted_memory_to_active": 0.12396883964538574, "mix_std": 0.22844882309436798, "beta_std": 0.009045297279953957, "beta_low": 0.7646484375, "beta_high": 0.0, "orthogonal_to_active": 0.10182278603315353, "evidence_weights": [-0.15030358731746674, -0.17268571257591248, 0.29649126529693604], "agreement_weight": 0.0553521029651165, "orthogonal_weight": -0.8785718083381653}, {"head": 1, "memory_to_active": 0.2809959053993225, "contribution_to_active": 0.3563540577888489, "mix_mean": 0.27243316173553467, "mix_low": 0.011416435241699219, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2758161127567291, "beta_mean": 0.0027673151344060898, "nonfinite": 0.0, "weighted_memory_to_active": 0.10389584302902222, "mix_std": 0.23384888470172882, "beta_std": 0.0033998393919318914, "beta_low": 0.962890625, "beta_high": 0.0, "orthogonal_to_active": 0.08100788295269012, "evidence_weights": [-0.14570027589797974, -0.16352997720241547, 0.33087900280952454], "agreement_weight": 0.05895037576556206, "orthogonal_weight": -0.8507711887359619}, {"head": 2, "memory_to_active": 0.31309983134269714, "contribution_to_active": 0.27939581871032715, "mix_mean": 0.23774224519729614, "mix_low": 0.008242607116699219, "mix_high": 2.193450927734375e-05, "read_cosine": 0.6772249341011047, "beta_mean": 0.027713526040315628, "nonfinite": 0.0, "weighted_memory_to_active": 0.11301185190677643, "mix_std": 0.19146472215652466, "beta_std": 0.052079569548368454, "beta_low": 0.4688720703125, "beta_high": 0.0, "orthogonal_to_active": 0.0255818422883749, "evidence_weights": [-0.1641210913658142, -0.14250339567661285, 0.36725571751594543], "agreement_weight": 0.0767827033996582, "orthogonal_weight": -0.3875858187675476}, {"head": 3, "memory_to_active": 0.10951430350542068, "contribution_to_active": 0.4617815315723419, "mix_mean": 0.49211859703063965, "mix_low": 0.004565238952636719, "mix_high": 0.0006656646728515625, "read_cosine": -0.11472105234861374, "beta_mean": 0.0005608776700682938, "nonfinite": 0.0, "weighted_memory_to_active": 0.061866648495197296, "mix_std": 0.22138531506061554, "beta_std": 0.0029656575061380863, "beta_low": 0.99603271484375, "beta_high": 0.0, "orthogonal_to_active": 0.05030394718050957, "evidence_weights": [-0.3235178291797638, -0.25235462188720703, 0.3717101216316223], "agreement_weight": 0.5026719570159912, "orthogonal_weight": -0.8127365708351135}, {"head": 4, "memory_to_active": 0.29474616050720215, "contribution_to_active": 0.3161731958389282, "mix_mean": 0.47500187158584595, "mix_low": 0.0028057098388671875, "mix_high": 0.006900787353515625, "read_cosine": 0.7931699752807617, "beta_mean": 0.07378693670034409, "nonfinite": 0.0, "weighted_memory_to_active": 0.1408611387014389, "mix_std": 0.28983449935913086, "beta_std": 0.07649609446525574, "beta_low": 0.061767578125, "beta_high": 0.0, "orthogonal_to_active": 0.03780757635831833, "evidence_weights": [-0.2576713562011719, -0.27762770652770996, 0.35141101479530334], "agreement_weight": 0.01451597735285759, "orthogonal_weight": 0.4035775065422058}, {"head": 5, "memory_to_active": 0.39917290210723877, "contribution_to_active": 0.361558735370636, "mix_mean": 0.23992592096328735, "mix_low": 0.019410133361816406, "mix_high": 7.62939453125e-05, "read_cosine": 0.22322942316532135, "beta_mean": 0.010816180147230625, "nonfinite": 0.0, "weighted_memory_to_active": 0.13255053758621216, "mix_std": 0.23626115918159485, "beta_std": 0.02916851080954075, "beta_low": 0.81689453125, "beta_high": 0.0, "orthogonal_to_active": 0.052874475717544556, "evidence_weights": [-0.14414376020431519, -0.16168087720870972, 0.018489273265004158], "agreement_weight": 0.05314869061112404, "orthogonal_weight": -0.445442259311676}]}, {"layer": 14, "memory_to_active": 0.32733139395713806, "contribution_to_active": 0.39686185121536255, "mix_mean": 0.34393805265426636, "mix_low": 0.025549888610839844, "mix_high": 0.0011563301086425781, "read_cosine": 0.5800296068191528, "beta_mean": 0.08240613341331482, "nonfinite": 0.0, "weighted_memory_to_active": 0.14271123707294464, "mix_std": 0.24513696134090424, "beta_std": 0.07717294991016388, "beta_low": 0.287384033203125, "beta_high": 0.0, "orthogonal_to_active": 0.029827455058693886, "output_influence": 0.21875175833702087, "residual_influence": 0.07759246230125427, "state_active": 0.09608200192451477, "state_memory": 0.023664232343435287, "state_peak": 38.32316970825195, "growth_active": 1.1914931535720825, "growth_memory": 1.4967901706695557, "heads": [{"head": 0, "memory_to_active": 0.30163347721099854, "contribution_to_active": 0.29035285115242004, "mix_mean": 0.2739778459072113, "mix_low": 0.022002220153808594, "mix_high": 0.0001010894775390625, "read_cosine": 0.649075984954834, "beta_mean": 0.07296086102724075, "nonfinite": 0.0, "weighted_memory_to_active": 0.11409758031368256, "mix_std": 0.2446625828742981, "beta_std": 0.09982334822416306, "beta_low": 0.1923828125, "beta_high": 0.0, "orthogonal_to_active": 0.005067015998065472, "evidence_weights": [-0.08535782247781754, -0.16970424354076385, 0.5236250758171082], "agreement_weight": 0.041488464921712875, "orthogonal_weight": 0.062079038470983505}, {"head": 1, "memory_to_active": 0.42645493149757385, "contribution_to_active": 0.46565353870391846, "mix_mean": 0.3264601230621338, "mix_low": 0.07008934020996094, "mix_high": 0.001651763916015625, "read_cosine": 0.4675697088241577, "beta_mean": 0.30414125323295593, "nonfinite": 0.0, "weighted_memory_to_active": 0.20936989784240723, "mix_std": 0.2995698153972626, "beta_std": 0.2617340385913849, "beta_low": 0.0706787109375, "beta_high": 0.0, "orthogonal_to_active": 0.0013027359964326024, "evidence_weights": [-0.054181646555662155, -0.2317521572113037, 0.25193092226982117], "agreement_weight": 0.05786428600549698, "orthogonal_weight": -0.007892774417996407}, {"head": 2, "memory_to_active": 0.4088774621486664, "contribution_to_active": 0.387951135635376, "mix_mean": 0.34197044372558594, "mix_low": 0.009184837341308594, "mix_high": 0.0001163482666015625, "read_cosine": 0.44254347681999207, "beta_mean": 0.03050551377236843, "nonfinite": 0.0, "weighted_memory_to_active": 0.18725502490997314, "mix_std": 0.25666967034339905, "beta_std": 0.029750000685453415, "beta_low": 0.16192626953125, "beta_high": 0.0, "orthogonal_to_active": 0.10890471935272217, "evidence_weights": [-0.18679077923297882, -0.19117006659507751, 0.20456168055534363], "agreement_weight": 0.10168039053678513, "orthogonal_weight": -0.6921785473823547}, {"head": 3, "memory_to_active": 0.08487015962600708, "contribution_to_active": 0.6131669282913208, "mix_mean": 0.6189172267913818, "mix_low": 0.0024051666259765625, "mix_high": 0.003887176513671875, "read_cosine": 0.594356894493103, "beta_mean": 0.01109282299876213, "nonfinite": 0.0, "weighted_memory_to_active": 0.06002424657344818, "mix_std": 0.24079862236976624, "beta_std": 0.008473735302686691, "beta_low": 0.5584716796875, "beta_high": 0.0, "orthogonal_to_active": 0.016717251390218735, "evidence_weights": [-0.3669910728931427, -0.3201793432235718, 0.36782407760620117], "agreement_weight": 0.06979576498270035, "orthogonal_weight": -0.41110119223594666}, {"head": 4, "memory_to_active": 0.30011069774627686, "contribution_to_active": 0.4161818325519562, "mix_mean": 0.32636165618896484, "mix_low": 0.007002830505371094, "mix_high": 0.001129150390625, "read_cosine": 0.6408035159111023, "beta_mean": 0.0670865923166275, "nonfinite": 0.0, "weighted_memory_to_active": 0.14308267831802368, "mix_std": 0.24124017357826233, "beta_std": 0.05042688548564911, "beta_low": 0.01025390625, "beta_high": 0.0, "orthogonal_to_active": 0.01192114781588316, "evidence_weights": [-0.1786491423845291, -0.2217288762331009, 0.14618003368377686], "agreement_weight": 0.1070353165268898, "orthogonal_weight": -0.12656573951244354}, {"head": 5, "memory_to_active": 0.4420414865016937, "contribution_to_active": 0.20786461234092712, "mix_mean": 0.17594093084335327, "mix_low": 0.04261493682861328, "mix_high": 5.245208740234375e-05, "read_cosine": 0.6858279705047607, "beta_mean": 0.00864977203309536, "nonfinite": 0.0, "weighted_memory_to_active": 0.14243799448013306, "mix_std": 0.18788085877895355, "beta_std": 0.01282970979809761, "beta_low": 0.7305908203125, "beta_high": 0.0, "orthogonal_to_active": 0.035051845014095306, "evidence_weights": [-0.1641474813222885, -0.12151671200990677, 0.04143770411610603], "agreement_weight": 0.10539726912975311, "orthogonal_weight": -0.41170716285705566}]}, {"layer": 15, "memory_to_active": 0.12630656361579895, "contribution_to_active": 0.4304533004760742, "mix_mean": 0.3488369286060333, "mix_low": 0.008859634399414062, "mix_high": 0.0018596649169921875, "read_cosine": 0.5731863975524902, "beta_mean": 0.0407257154583931, "nonfinite": 0.0, "weighted_memory_to_active": 0.05844615399837494, "mix_std": 0.2583599388599396, "beta_std": 0.048446886241436005, "beta_low": 0.6420695185661316, "beta_high": 0.0, "orthogonal_to_active": 0.015787135809659958, "output_influence": 0.27641671895980835, "residual_influence": 0.14233244955539703, "state_active": 0.10787709057331085, "state_memory": 0.012217949144542217, "state_peak": 29.9688720703125, "growth_active": 1.3725489377975464, "growth_memory": 1.5442174673080444, "heads": [{"head": 0, "memory_to_active": 0.12172319740056992, "contribution_to_active": 0.4659024775028229, "mix_mean": 0.33816325664520264, "mix_low": 0.010674476623535156, "mix_high": 0.0013446807861328125, "read_cosine": 0.670326828956604, "beta_mean": 0.06639508157968521, "nonfinite": 0.0, "weighted_memory_to_active": 0.05986778438091278, "mix_std": 0.2557697892189026, "beta_std": 0.058615267276763916, "beta_low": 0.1116943359375, "beta_high": 0.0, "orthogonal_to_active": 0.009778168052434921, "evidence_weights": [-0.1841251701116562, -0.21958203613758087, 0.3271280825138092], "agreement_weight": 0.0796026661992073, "orthogonal_weight": -0.29029783606529236}, {"head": 1, "memory_to_active": 0.2117958813905716, "contribution_to_active": 0.3863903284072876, "mix_mean": 0.3336575925350189, "mix_low": 0.0061798095703125, "mix_high": 0.004054069519042969, "read_cosine": 0.8255339860916138, "beta_mean": 0.002991183428093791, "nonfinite": 0.0, "weighted_memory_to_active": 0.08856344223022461, "mix_std": 0.2584823966026306, "beta_std": 0.007447604089975357, "beta_low": 0.9541015625, "beta_high": 0.0, "orthogonal_to_active": 0.008946829475462437, "evidence_weights": [-0.19774846732616425, -0.20856282114982605, 0.24950748682022095], "agreement_weight": 0.0911342054605484, "orthogonal_weight": -0.17969438433647156}, {"head": 2, "memory_to_active": 0.017368391156196594, "contribution_to_active": 0.41449421644210815, "mix_mean": 0.3345949649810791, "mix_low": 0.011832237243652344, "mix_high": 0.00022411346435546875, "read_cosine": 0.7084153890609741, "beta_mean": 0.00036015259684063494, "nonfinite": 0.0, "weighted_memory_to_active": 0.006851439364254475, "mix_std": 0.26165783405303955, "beta_std": 0.0006132422131486237, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.002684162463992834, "evidence_weights": [-0.21637478470802307, -0.2462623566389084, 0.29692259430885315], "agreement_weight": 0.04292340576648712, "orthogonal_weight": -0.515097975730896}, {"head": 3, "memory_to_active": 0.22286930680274963, "contribution_to_active": 0.4071851670742035, "mix_mean": 0.24707534909248352, "mix_low": 0.008749008178710938, "mix_high": 3.910064697265625e-05, "read_cosine": 0.6365821361541748, "beta_mean": 0.015598739497363567, "nonfinite": 0.0, "weighted_memory_to_active": 0.10051964223384857, "mix_std": 0.21207764744758606, "beta_std": 0.02724749967455864, "beta_low": 0.612060546875, "beta_high": 0.0, "orthogonal_to_active": 0.019247526302933693, "evidence_weights": [-0.1359683722257614, -0.19870343804359436, 0.3167082369327545], "agreement_weight": 0.07150455564260483, "orthogonal_weight": -0.27766817808151245}, {"head": 4, "memory_to_active": 0.18128164112567902, "contribution_to_active": 0.4673566222190857, "mix_mean": 0.36974912881851196, "mix_low": 0.01039886474609375, "mix_high": 0.00016689300537109375, "read_cosine": 0.054207075387239456, "beta_mean": 0.15895552933216095, "nonfinite": 0.0, "weighted_memory_to_active": 0.09377069771289825, "mix_std": 0.2808903753757477, "beta_std": 0.19661210477352142, "beta_low": 0.17498779296875, "beta_high": 0.0, "orthogonal_to_active": 0.053669124841690063, "evidence_weights": [-0.2650662362575531, -0.24964947998523712, 0.3899383246898651], "agreement_weight": 0.0015691478038206697, "orthogonal_weight": -0.630090594291687}, {"head": 5, "memory_to_active": 0.002800937741994858, "contribution_to_active": 0.44139084219932556, "mix_mean": 0.4697812795639038, "mix_low": 0.0053234100341796875, "mix_high": 0.005329132080078125, "read_cosine": 0.5440528392791748, "beta_mean": 5.3593292250297964e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0011039323871955276, "mix_std": 0.28128156065940857, "beta_std": 0.0001455616729799658, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003970015386585146, "evidence_weights": [-0.3440517783164978, -0.2834653854370117, 0.48024982213974], "agreement_weight": 0.09314113110303879, "orthogonal_weight": 0.4083874523639679}]}]}, {"step": 8000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7989546899989364, "memory_to_active": 1.1268351078033447, "contribution_to_active": 0.3855556547641754, "mix_mean": 0.2230638861656189, "mix_low": 0.2073521912097931, "mix_high": 0.003660400863736868, "read_cosine": 0.403695285320282, "beta_mean": 0.10882453620433807, "nonfinite": 0.0, "weighted_memory_to_active": 0.21213491261005402, "mix_std": 0.1823362112045288, "beta_std": 0.08056247234344482, "beta_low": 0.3678150177001953, "beta_high": 1.3987223610456567e-05, "orthogonal_to_active": 0.03262800723314285, "samples": 4, "scored_tokens": 899, "normal_loss": 3.2667129039764404, "normal_token_accuracy": 0.43159064650535583, "state_active": 0.06836161017417908, "state_memory": 0.03215467929840088, "state_peak": 64.03182983398438, "growth_active": 1.113512396812439, "growth_memory": 1.4897468090057373, "output_influence": 0.26185232400894165, "residual_influence": 0.07906179130077362, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.3922183513641357, "no_memory_loss_delta": 0.09815958887338638, "scrambled_loss_delta": 0.341178834438324, "memory_only_loss_delta": 1.8138442039489746, "no_evidence_loss_delta": 0.03725026920437813, "no_agreement_loss_delta": 0.006961916573345661, "no_orthogonal_loss_delta": -0.0016375835984945297}, {"index": 94, "tokens": 256, "normal_loss": 3.4042162895202637, "no_memory_loss_delta": 0.26112884283065796, "scrambled_loss_delta": 0.39307811856269836, "memory_only_loss_delta": 1.86850106716156, "no_evidence_loss_delta": 0.1279354691505432, "no_agreement_loss_delta": 0.03293680399656296, "no_orthogonal_loss_delta": 0.014093993231654167}, {"index": 188, "tokens": 256, "normal_loss": 3.6297011375427246, "no_memory_loss_delta": 0.16201743483543396, "scrambled_loss_delta": 0.1557379961013794, "memory_only_loss_delta": 2.0108516216278076, "no_evidence_loss_delta": 0.10025257617235184, "no_agreement_loss_delta": 0.014952536672353745, "no_orthogonal_loss_delta": -0.003242792561650276}, {"index": 283, "tokens": 131, "normal_loss": 2.0433907508850098, "no_memory_loss_delta": 0.247070774435997, "scrambled_loss_delta": 0.4058796465396881, "memory_only_loss_delta": 2.4153952598571777, "no_evidence_loss_delta": 0.08450482040643692, "no_agreement_loss_delta": 0.011498602107167244, "no_orthogonal_loss_delta": -0.00036541104782372713}], "agreement_weight_abs_mean": 0.06086628511548042, "agreement_weight_abs_max": 0.5798913836479187, "agreement_weight_mean": 0.03799024224281311, "orthogonal_weight_abs_mean": 0.3158251643180847, "orthogonal_weight_abs_max": 1.3506882190704346, "orthogonal_weight_mean": -0.13161642849445343, "no_memory_loss": 3.4511632919311523, "no_memory_loss_delta": 0.18445000052452087, "no_memory_token_accuracy_gain": 0.011123470962047577, "no_memory_token_win_fraction": 0.6162402629852295, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 16.84384835003551, "scrambled_loss": 3.579291820526123, "scrambled_loss_delta": 0.312579482793808, "scrambled_token_accuracy_gain": 0.04226918891072273, "scrambled_token_win_fraction": 0.6384872198104858, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 26.844251959660554, "memory_only_loss": 5.239878177642822, "memory_only_loss_delta": 1.9731647968292236, "memory_only_token_accuracy_gain": 0.20800890028476715, "memory_only_token_win_fraction": 0.8553948998451233, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 86.09837987418092, "no_evidence_loss": 3.3546130657196045, "no_evidence_loss_delta": 0.08790026605129242, "no_evidence_token_accuracy_gain": 0.008898776024580002, "no_evidence_token_win_fraction": 0.5739710927009583, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 8.414778627374256, "no_agreement_loss": 3.2840075492858887, "no_agreement_loss_delta": 0.01729503832757473, "no_agreement_token_accuracy_gain": 0.007786429487168789, "no_agreement_token_win_fraction": 0.5194661021232605, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 1.7146337647622296, "no_orthogonal_loss": 3.2692832946777344, "no_orthogonal_loss_delta": 0.002570431213825941, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5261401534080505, "no_orthogonal_probe_win_fraction": 0.25, "no_orthogonal_ppl_reduction_pct": 0.2567130484218588, "short_no_memory_loss_delta": 0.11880674213171005, "long_context_memory_gain": 0.06564325839281082, "normal_context_loss_gain": 0.14918465912342072, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.4781408309936523, "contribution_to_active": 0.7182819843292236, "mix_mean": 0.09387411177158356, "mix_low": 0.33772677183151245, "mix_high": 0.0013783773174509406, "read_cosine": 0.5254923105239868, "beta_mean": 0.2630108594894409, "nonfinite": 0.0, "weighted_memory_to_active": 0.6996937990188599, "mix_std": 0.0837150365114212, "beta_std": 0.13306593894958496, "beta_low": 1.0172526344831567e-05, "beta_high": 0.0, "orthogonal_to_active": 0.040549881756305695, "output_influence": 0.1490894854068756, "residual_influence": 0.22091129422187805, "state_active": 0.10885175317525864, "state_memory": 0.058879874646663666, "state_peak": 46.9335823059082, "growth_active": 1.3046618700027466, "growth_memory": 1.380687952041626, "heads": [{"head": 0, "memory_to_active": 2.428860664367676, "contribution_to_active": 0.5388909578323364, "mix_mean": 0.024240413680672646, "mix_low": 0.37979888916015625, "mix_high": 0.0, "read_cosine": 0.4096885025501251, "beta_mean": 0.2354450225830078, "nonfinite": 0.0, "weighted_memory_to_active": 0.5964099168777466, "mix_std": 0.05037911981344223, "beta_std": 0.10244270414113998, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006743516423739493, "evidence_weights": [-0.047414202243089676, -0.06407143175601959, 0.08194539695978165], "agreement_weight": -0.048805877566337585, "orthogonal_weight": 0.001380433328449726}, {"head": 1, "memory_to_active": 2.6720499992370605, "contribution_to_active": 1.2225267887115479, "mix_mean": 0.023978043347597122, "mix_low": 0.3326082229614258, "mix_high": 0.0, "read_cosine": 0.5088695287704468, "beta_mean": 0.3494655191898346, "nonfinite": 0.0, "weighted_memory_to_active": 1.3730981349945068, "mix_std": 0.054089877754449844, "beta_std": 0.13407397270202637, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04154112935066223, "evidence_weights": [-0.0038813345599919558, -0.05172324180603027, 0.07124855369329453], "agreement_weight": -0.003571741981431842, "orthogonal_weight": 0.06163559854030609}, {"head": 2, "memory_to_active": 2.087785243988037, "contribution_to_active": 0.4658849537372589, "mix_mean": 0.018373582512140274, "mix_low": 0.4948902130126953, "mix_high": 0.0, "read_cosine": 0.4355735778808594, "beta_mean": 0.29611217975616455, "nonfinite": 0.0, "weighted_memory_to_active": 0.4924952983856201, "mix_std": 0.03414705768227577, "beta_std": 0.19289971888065338, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.011067025363445282, "evidence_weights": [-0.0007328852079808712, 0.002581105101853609, -0.08678825199604034], "agreement_weight": -0.014749597758054733, "orthogonal_weight": -0.03522295504808426}, {"head": 3, "memory_to_active": 3.8068840503692627, "contribution_to_active": 1.6261920928955078, "mix_mean": 0.07435379922389984, "mix_low": 0.050858497619628906, "mix_high": 0.0, "read_cosine": 0.5017383098602295, "beta_mean": 0.3171675205230713, "nonfinite": 0.0, "weighted_memory_to_active": 1.5705071687698364, "mix_std": 0.09229503571987152, "beta_std": 0.14634381234645844, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1710469126701355, "evidence_weights": [-0.14066416025161743, -0.1629916876554489, 0.17260132730007172], "agreement_weight": -0.0342622734606266, "orthogonal_weight": 0.14442427456378937}, {"head": 4, "memory_to_active": 3.7135114669799805, "contribution_to_active": 0.1081712394952774, "mix_mean": 0.008638440631330013, "mix_low": 0.757685661315918, "mix_high": 0.0, "read_cosine": 0.3411797881126404, "beta_mean": 0.2902924120426178, "nonfinite": 0.0, "weighted_memory_to_active": 0.09292234480381012, "mix_std": 0.012406125664710999, "beta_std": 0.16121186316013336, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.018369633704423904, "evidence_weights": [0.052709802985191345, 0.0229320228099823, -0.04624788835644722], "agreement_weight": -0.03539326786994934, "orthogonal_weight": 0.22387179732322693}, {"head": 5, "memory_to_active": 0.1597525030374527, "contribution_to_active": 0.34802573919296265, "mix_mean": 0.4136604070663452, "mix_low": 0.010519027709960938, "mix_high": 0.008270263671875, "read_cosine": 0.9559040069580078, "beta_mean": 0.08958254754543304, "nonfinite": 0.0, "weighted_memory_to_active": 0.0727296844124794, "mix_std": 0.25897303223609924, "beta_std": 0.061423465609550476, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.0006002308218739927, "evidence_weights": [-0.3300393521785736, -0.33378663659095764, 0.42956337332725525], "agreement_weight": 0.13480718433856964, "orthogonal_weight": 0.022876285016536713}]}, {"layer": 1, "memory_to_active": 2.7098066806793213, "contribution_to_active": 0.31323036551475525, "mix_mean": 0.050771962851285934, "mix_low": 0.4743935465812683, "mix_high": 0.00019852320838253945, "read_cosine": 0.33927834033966064, "beta_mean": 0.26543116569519043, "nonfinite": 0.0, "weighted_memory_to_active": 0.3151267170906067, "mix_std": 0.07349798083305359, "beta_std": 0.14054957032203674, "beta_low": 0.0911051481962204, "beta_high": 0.00016276042151730508, "orthogonal_to_active": 0.03322996944189072, "output_influence": 0.12308087199926376, "residual_influence": 0.02703985571861267, "state_active": 0.05527563765645027, "state_memory": 0.06399812549352646, "state_peak": 23.80084991455078, "growth_active": 1.1388357877731323, "growth_memory": 1.5574543476104736, "heads": [{"head": 0, "memory_to_active": 0.9096853137016296, "contribution_to_active": 0.10012482851743698, "mix_mean": 0.027136322110891342, "mix_low": 0.5451803207397461, "mix_high": 0.0, "read_cosine": 0.03488513082265854, "beta_mean": 0.013353701680898666, "nonfinite": 0.0, "weighted_memory_to_active": 0.08013550937175751, "mix_std": 0.06697317212820053, "beta_std": 0.014890007674694061, "beta_low": 0.53558349609375, "beta_high": 0.0, "orthogonal_to_active": 0.02706936188042164, "evidence_weights": [0.04113529995083809, -0.0001626827724976465, -0.0789632573723793], "agreement_weight": -0.09605485200881958, "orthogonal_weight": -0.3458186686038971}, {"head": 1, "memory_to_active": 3.5160956382751465, "contribution_to_active": 0.20975607633590698, "mix_mean": 0.028630157932639122, "mix_low": 0.3720073699951172, "mix_high": 0.0, "read_cosine": 0.4608381390571594, "beta_mean": 0.3490666151046753, "nonfinite": 0.0, "weighted_memory_to_active": 0.22278901934623718, "mix_std": 0.05197138711810112, "beta_std": 0.18938933312892914, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.004072224721312523, "evidence_weights": [-0.0453171469271183, -0.037696029990911484, 0.11870990693569183], "agreement_weight": -0.023417750373482704, "orthogonal_weight": 0.023481808602809906}, {"head": 2, "memory_to_active": 0.8327391147613525, "contribution_to_active": 0.35966986417770386, "mix_mean": 0.19263041019439697, "mix_low": 0.020544052124023438, "mix_high": 0.001186370849609375, "read_cosine": 0.7451642751693726, "beta_mean": 0.1453227400779724, "nonfinite": 0.0, "weighted_memory_to_active": 0.4920063614845276, "mix_std": 0.1947735995054245, "beta_std": 0.06506367027759552, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.041376471519470215, "evidence_weights": [-0.20938634872436523, -0.3026461899280548, 0.2224663943052292], "agreement_weight": -0.018941009417176247, "orthogonal_weight": 0.14941200613975525}, {"head": 3, "memory_to_active": 3.2707295417785645, "contribution_to_active": 0.05940467491745949, "mix_mean": 0.007299978286027908, "mix_low": 0.8115549087524414, "mix_high": 0.0, "read_cosine": 0.18830594420433044, "beta_mean": 0.4882045388221741, "nonfinite": 0.0, "weighted_memory_to_active": 0.06372790038585663, "mix_std": 0.0158771350979805, "beta_std": 0.24509042501449585, "beta_low": 0.0, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.004786992445588112, "evidence_weights": [0.029834987595677376, 0.027346588671207428, 0.00770956464111805], "agreement_weight": -0.007987083867192268, "orthogonal_weight": -0.07857770472764969}, {"head": 4, "memory_to_active": 2.8294806480407715, "contribution_to_active": 1.07831871509552, "mix_mean": 0.04122784361243248, "mix_low": 0.28760719299316406, "mix_high": 0.0, "read_cosine": 0.41527900099754333, "beta_mean": 0.2405170202255249, "nonfinite": 0.0, "weighted_memory_to_active": 0.9634879231452942, "mix_std": 0.07761827111244202, "beta_std": 0.09663008898496628, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12034551054239273, "evidence_weights": [-0.04209260269999504, -0.04029553756117821, 0.19454926252365112], "agreement_weight": -0.04439614713191986, "orthogonal_weight": 0.12998664379119873}, {"head": 5, "memory_to_active": 4.90010929107666, "contribution_to_active": 0.07210790365934372, "mix_mean": 0.007707053795456886, "mix_low": 0.8094673156738281, "mix_high": 4.76837158203125e-06, "read_cosine": 0.19119757413864136, "beta_mean": 0.3561224043369293, "nonfinite": 0.0, "weighted_memory_to_active": 0.06861353665590286, "mix_std": 0.03377426788210869, "beta_std": 0.23223388195037842, "beta_low": 0.01080322265625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0017292744014412165, "evidence_weights": [0.07856268435716629, 0.049069538712501526, 0.16939899325370789], "agreement_weight": -0.10794977843761444, "orthogonal_weight": 0.02586476504802704}]}, {"layer": 2, "memory_to_active": 1.293015718460083, "contribution_to_active": 0.24126380681991577, "mix_mean": 0.14332175254821777, "mix_low": 0.34806346893310547, "mix_high": 0.0009932519169524312, "read_cosine": 0.3778040409088135, "beta_mean": 0.1286199986934662, "nonfinite": 0.0, "weighted_memory_to_active": 0.09518571197986603, "mix_std": 0.13689309358596802, "beta_std": 0.08116702735424042, "beta_low": 0.4999898672103882, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.007863061502575874, "output_influence": 0.1840786635875702, "residual_influence": 0.03597378730773926, "state_active": 0.04056500643491745, "state_memory": 0.03244515135884285, "state_peak": 44.97025680541992, "growth_active": 1.0322874784469604, "growth_memory": 1.5385866165161133, "heads": [{"head": 0, "memory_to_active": 0.37820160388946533, "contribution_to_active": 0.03685075789690018, "mix_mean": 0.02808128111064434, "mix_low": 0.5059747695922852, "mix_high": 0.0, "read_cosine": 0.2570536136627197, "beta_mean": 0.006079168524593115, "nonfinite": 0.0, "weighted_memory_to_active": 0.016893375664949417, "mix_std": 0.057735372334718704, "beta_std": 0.007115041371434927, "beta_low": 0.81982421875, "beta_high": 0.0, "orthogonal_to_active": 0.001725038979202509, "evidence_weights": [0.029303016141057014, 0.02101067639887333, 0.031991805881261826], "agreement_weight": -0.0511702224612236, "orthogonal_weight": -0.10809128731489182}, {"head": 1, "memory_to_active": 3.9482927322387695, "contribution_to_active": 0.06345196068286896, "mix_mean": 0.0063007185235619545, "mix_low": 0.8552265167236328, "mix_high": 1.9073486328125e-06, "read_cosine": 0.19699028134346008, "beta_mean": 0.4631023406982422, "nonfinite": 0.0, "weighted_memory_to_active": 0.06322120875120163, "mix_std": 0.011882508173584938, "beta_std": 0.18220847845077515, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007499326951801777, "evidence_weights": [0.07549948245286942, 0.0831397995352745, 0.04640476778149605], "agreement_weight": 0.00680549954995513, "orthogonal_weight": 0.14023210108280182}, {"head": 2, "memory_to_active": 0.16709202527999878, "contribution_to_active": 0.3490048944950104, "mix_mean": 0.276645690202713, "mix_low": 0.011399269104003906, "mix_high": 0.0004215240478515625, "read_cosine": 0.48108306527137756, "beta_mean": 0.0003702261601574719, "nonfinite": 0.0, "weighted_memory_to_active": 0.05837832763791084, "mix_std": 0.24837006628513336, "beta_std": 0.0007225876906886697, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.006665615364909172, "evidence_weights": [-0.15390659868717194, -0.16568638384342194, 0.2577657699584961], "agreement_weight": 0.05020013079047203, "orthogonal_weight": -0.16750489175319672}, {"head": 3, "memory_to_active": 0.4312185049057007, "contribution_to_active": 0.3403344750404358, "mix_mean": 0.13709896802902222, "mix_low": 0.051245689392089844, "mix_high": 0.0019321441650390625, "read_cosine": 0.6113578081130981, "beta_mean": 0.06748758256435394, "nonfinite": 0.0, "weighted_memory_to_active": 0.13241292536258698, "mix_std": 0.17450733482837677, "beta_std": 0.09857785701751709, "beta_low": 0.14959716796875, "beta_high": 0.0, "orthogonal_to_active": 0.030031878501176834, "evidence_weights": [-0.05080266296863556, -0.11028056591749191, 0.3416076898574829], "agreement_weight": 0.08597812801599503, "orthogonal_weight": -0.3625886142253876}, {"head": 4, "memory_to_active": 2.7986788749694824, "contribution_to_active": 0.1988261640071869, "mix_mean": 0.015249940566718578, "mix_low": 0.6621112823486328, "mix_high": 8.678436279296875e-05, "read_cosine": 0.20435506105422974, "beta_mean": 0.2346065789461136, "nonfinite": 0.0, "weighted_memory_to_active": 0.2817418873310089, "mix_std": 0.049196306616067886, "beta_std": 0.19817568361759186, "beta_low": 0.0313720703125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0003916790592484176, "evidence_weights": [0.039902810007333755, 0.032368917018175125, 0.14334234595298767], "agreement_weight": 0.02372446097433567, "orthogonal_weight": -0.0027928596828132868}, {"head": 5, "memory_to_active": 0.03461049869656563, "contribution_to_active": 0.45911455154418945, "mix_mean": 0.3965539336204529, "mix_low": 0.0024232864379882812, "mix_high": 0.00351715087890625, "read_cosine": 0.5159844160079956, "beta_mean": 7.414456194965169e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.018466493114829063, "mix_std": 0.2796669900417328, "beta_std": 0.00020251759269740433, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.000864827714394778, "evidence_weights": [-0.2264220267534256, -0.2076031118631363, 0.3873319923877716], "agreement_weight": 0.018062420189380646, "orthogonal_weight": 0.09248930960893631}]}, {"layer": 3, "memory_to_active": 1.696630835533142, "contribution_to_active": 0.26426559686660767, "mix_mean": 0.09390608966350555, "mix_low": 0.36459144949913025, "mix_high": 0.0008745194063521922, "read_cosine": 0.2983592748641968, "beta_mean": 0.13473138213157654, "nonfinite": 0.0, "weighted_memory_to_active": 0.15870200097560883, "mix_std": 0.13197410106658936, "beta_std": 0.09416724741458893, "beta_low": 0.3465983271598816, "beta_high": 0.0, "orthogonal_to_active": 0.020394226536154747, "output_influence": 0.18612653017044067, "residual_influence": 0.029012078419327736, "state_active": 0.04081476107239723, "state_memory": 0.035253074020147324, "state_peak": 31.905746459960938, "growth_active": 1.0210496187210083, "growth_memory": 1.7193506956100464, "heads": [{"head": 0, "memory_to_active": 1.223355770111084, "contribution_to_active": 0.17578360438346863, "mix_mean": 0.06322737038135529, "mix_low": 0.4062213897705078, "mix_high": 0.0, "read_cosine": 0.23480133712291718, "beta_mean": 0.02267991378903389, "nonfinite": 0.0, "weighted_memory_to_active": 0.1498430073261261, "mix_std": 0.12427576631307602, "beta_std": 0.016149356961250305, "beta_low": 0.12847900390625, "beta_high": 0.0, "orthogonal_to_active": 0.008225575089454651, "evidence_weights": [0.02234541065990925, -0.021733742207288742, -0.07493451982736588], "agreement_weight": -0.06673725694417953, "orthogonal_weight": -0.054430436342954636}, {"head": 1, "memory_to_active": 0.46661657094955444, "contribution_to_active": 0.16343748569488525, "mix_mean": 0.08766888082027435, "mix_low": 0.24056434631347656, "mix_high": 2.47955322265625e-05, "read_cosine": 0.2704865336418152, "beta_mean": 0.0028266292065382004, "nonfinite": 0.0, "weighted_memory_to_active": 0.03869267553091049, "mix_std": 0.14817309379577637, "beta_std": 0.0052001820877194405, "beta_low": 0.934814453125, "beta_high": 0.0, "orthogonal_to_active": 0.0032908227294683456, "evidence_weights": [-0.04967872425913811, -0.03774256259202957, 0.11789871007204056], "agreement_weight": -0.04354112967848778, "orthogonal_weight": -0.08974850922822952}, {"head": 2, "memory_to_active": 3.8092200756073, "contribution_to_active": 0.3047679662704468, "mix_mean": 0.030887307599186897, "mix_low": 0.46581077575683594, "mix_high": 0.0, "read_cosine": 0.20018109679222107, "beta_mean": 0.28645846247673035, "nonfinite": 0.0, "weighted_memory_to_active": 0.4008761942386627, "mix_std": 0.07802682369947433, "beta_std": 0.17612025141716003, "beta_low": 0.00146484375, "beta_high": 0.0, "orthogonal_to_active": 0.055949728935956955, "evidence_weights": [-0.028371840715408325, 0.0005958558758720756, 0.09536527097225189], "agreement_weight": -0.03079417161643505, "orthogonal_weight": -0.16667330265045166}, {"head": 3, "memory_to_active": 0.08102461695671082, "contribution_to_active": 0.39059680700302124, "mix_mean": 0.2847752273082733, "mix_low": 0.034905433654785156, "mix_high": 0.00255584716796875, "read_cosine": 0.2977980971336365, "beta_mean": 0.00018374646606389433, "nonfinite": 0.0, "weighted_memory_to_active": 0.03262032940983772, "mix_std": 0.26595646142959595, "beta_std": 0.000582669279538095, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.004644942469894886, "evidence_weights": [-0.16196563839912415, -0.17495551705360413, 0.4417126476764679], "agreement_weight": 0.2597874402999878, "orthogonal_weight": -0.20715779066085815}, {"head": 4, "memory_to_active": 4.051044940948486, "contribution_to_active": 0.04945917800068855, "mix_mean": 0.006129463203251362, "mix_low": 0.8690462112426758, "mix_high": 0.0, "read_cosine": 0.21134983003139496, "beta_mean": 0.3111816346645355, "nonfinite": 0.0, "weighted_memory_to_active": 0.05774997919797897, "mix_std": 0.013770928606390953, "beta_std": 0.18973807990550995, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.009725929237902164, "evidence_weights": [0.12980788946151733, 0.11663927882909775, -0.14577017724514008], "agreement_weight": -0.08715689927339554, "orthogonal_weight": -0.17215415835380554}, {"head": 5, "memory_to_active": 0.5485222339630127, "contribution_to_active": 0.5015484690666199, "mix_mean": 0.09074822813272476, "mix_low": 0.17100048065185547, "mix_high": 0.002666473388671875, "read_cosine": 0.5755387544631958, "beta_mean": 0.18505790829658508, "nonfinite": 0.0, "weighted_memory_to_active": 0.27242976427078247, "mix_std": 0.16164149343967438, "beta_std": 0.1772128790616989, "beta_low": 0.01507568359375, "beta_high": 0.0, "orthogonal_to_active": 0.040528349578380585, "evidence_weights": [-0.020417185500264168, -0.10977643728256226, 0.016152936965227127], "agreement_weight": 0.14834389090538025, "orthogonal_weight": -0.26950472593307495}]}, {"layer": 4, "memory_to_active": 1.300801396369934, "contribution_to_active": 0.29929012060165405, "mix_mean": 0.19524121284484863, "mix_low": 0.26979511976242065, "mix_high": 0.0009140968904830515, "read_cosine": 0.4245400130748749, "beta_mean": 0.1290980577468872, "nonfinite": 0.0, "weighted_memory_to_active": 0.18044811487197876, "mix_std": 0.1383790671825409, "beta_std": 0.09968142211437225, "beta_low": 0.2056986689567566, "beta_high": 0.0, "orthogonal_to_active": 0.03335679695010185, "output_influence": 0.1854451298713684, "residual_influence": 0.03602215647697449, "state_active": 0.04766583442687988, "state_memory": 0.042677830904722214, "state_peak": 43.83079147338867, "growth_active": 1.0745940208435059, "growth_memory": 1.7114408016204834, "heads": [{"head": 0, "memory_to_active": 0.4886864125728607, "contribution_to_active": 0.4526703357696533, "mix_mean": 0.32761168479919434, "mix_low": 0.008301734924316406, "mix_high": 0.0005626678466796875, "read_cosine": 0.6024535298347473, "beta_mean": 0.07430228590965271, "nonfinite": 0.0, "weighted_memory_to_active": 0.271521657705307, "mix_std": 0.24681144952774048, "beta_std": 0.09086931496858597, "beta_low": 0.097900390625, "beta_high": 0.0, "orthogonal_to_active": 0.008773140609264374, "evidence_weights": [-0.15994302928447723, -0.20287296175956726, 0.33395642042160034], "agreement_weight": 0.06757143139839172, "orthogonal_weight": -0.04442865774035454}, {"head": 1, "memory_to_active": 0.6243137121200562, "contribution_to_active": 0.22584959864616394, "mix_mean": 0.2334555983543396, "mix_low": 0.026224136352539062, "mix_high": 1.33514404296875e-05, "read_cosine": 0.529430627822876, "beta_mean": 0.04979574680328369, "nonfinite": 0.0, "weighted_memory_to_active": 0.18646453320980072, "mix_std": 0.21514065563678741, "beta_std": 0.050966981798410416, "beta_low": 0.08306884765625, "beta_high": 0.0, "orthogonal_to_active": 0.06248636543750763, "evidence_weights": [-0.1437753438949585, -0.14066371321678162, 0.45170077681541443], "agreement_weight": 0.05017159879207611, "orthogonal_weight": -0.40676289796829224}, {"head": 2, "memory_to_active": 2.422809600830078, "contribution_to_active": 0.06345253437757492, "mix_mean": 0.011246255598962307, "mix_low": 0.6765995025634766, "mix_high": 0.0, "read_cosine": 0.31831225752830505, "beta_mean": 0.1688234806060791, "nonfinite": 0.0, "weighted_memory_to_active": 0.07367567718029022, "mix_std": 0.016708068549633026, "beta_std": 0.12624141573905945, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014819905627518892, "evidence_weights": [0.0352294035255909, 0.021417828276753426, -0.05537544563412666], "agreement_weight": 0.025323903188109398, "orthogonal_weight": 0.025901976972818375}, {"head": 3, "memory_to_active": 1.7187700271606445, "contribution_to_active": 0.08500352501869202, "mix_mean": 0.027071626856923103, "mix_low": 0.3931608200073242, "mix_high": 0.0, "read_cosine": 0.25142544507980347, "beta_mean": 0.18039701879024506, "nonfinite": 0.0, "weighted_memory_to_active": 0.12770670652389526, "mix_std": 0.04202892258763313, "beta_std": 0.15130044519901276, "beta_low": 0.05322265625, "beta_high": 0.0, "orthogonal_to_active": 0.018481038510799408, "evidence_weights": [-0.020355960354208946, 0.01605566218495369, -0.12039820104837418], "agreement_weight": 0.009100345894694328, "orthogonal_weight": -0.2281475067138672}, {"head": 4, "memory_to_active": 0.03475181758403778, "contribution_to_active": 0.5215268135070801, "mix_mean": 0.5519402027130127, "mix_low": 0.0017805099487304688, "mix_high": 0.004876136779785156, "read_cosine": 0.3874829113483429, "beta_mean": 8.296234591398388e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.021408261731266975, "mix_std": 0.26794570684432983, "beta_std": 0.00019307561160530895, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002318386686965823, "evidence_weights": [-0.2551492154598236, -0.27545034885406494, 0.4897487759590149], "agreement_weight": 0.020491022616624832, "orthogonal_weight": -0.1348392367362976}, {"head": 5, "memory_to_active": 2.5154764652252197, "contribution_to_active": 0.44723787903785706, "mix_mean": 0.020121948793530464, "mix_low": 0.5127038955688477, "mix_high": 3.24249267578125e-05, "read_cosine": 0.4581352174282074, "beta_mean": 0.30118680000305176, "nonfinite": 0.0, "weighted_memory_to_active": 0.40191179513931274, "mix_std": 0.04163956642150879, "beta_std": 0.17851731181144714, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1065998524427414, "evidence_weights": [0.01430735643953085, -0.026431560516357422, 0.03163272142410278], "agreement_weight": -0.048170968890190125, "orthogonal_weight": 0.45061561465263367}]}, {"layer": 5, "memory_to_active": 0.801275908946991, "contribution_to_active": 0.3989158868789673, "mix_mean": 0.32237499952316284, "mix_low": 0.12477494031190872, "mix_high": 0.006888548843562603, "read_cosine": 0.24649694561958313, "beta_mean": 0.08310958743095398, "nonfinite": 0.0, "weighted_memory_to_active": 0.173939049243927, "mix_std": 0.2326432466506958, "beta_std": 0.08324785530567169, "beta_low": 0.4804687798023224, "beta_high": 0.0, "orthogonal_to_active": 0.03787373751401901, "output_influence": 0.35692349076271057, "residual_influence": 0.06780356913805008, "state_active": 0.06955762207508087, "state_memory": 0.030885392799973488, "state_peak": 60.39066696166992, "growth_active": 1.112038493156433, "growth_memory": 1.5528501272201538, "heads": [{"head": 0, "memory_to_active": 1.681532382965088, "contribution_to_active": 0.23039792478084564, "mix_mean": 0.06049659103155136, "mix_low": 0.23770713806152344, "mix_high": 0.0, "read_cosine": -0.1029745414853096, "beta_mean": 0.18018963932991028, "nonfinite": 0.0, "weighted_memory_to_active": 0.2079775333404541, "mix_std": 0.10088600218296051, "beta_std": 0.1538865864276886, "beta_low": 0.00323486328125, "beta_high": 0.0, "orthogonal_to_active": 0.035987645387649536, "evidence_weights": [-0.0193317998200655, -0.038590576499700546, -0.2526627480983734], "agreement_weight": -0.025876900181174278, "orthogonal_weight": -0.1823253333568573}, {"head": 1, "memory_to_active": 0.01089842151850462, "contribution_to_active": 0.36570560932159424, "mix_mean": 0.36633729934692383, "mix_low": 0.030294418334960938, "mix_high": 0.0054454803466796875, "read_cosine": 0.13458646833896637, "beta_mean": 2.73004625341855e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037346493918448687, "mix_std": 0.30979859828948975, "beta_std": 8.585718751419336e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023777373135089874, "evidence_weights": [-0.17687062919139862, -0.16682150959968567, 0.24054086208343506], "agreement_weight": 0.1527835726737976, "orthogonal_weight": -0.6567137241363525}, {"head": 2, "memory_to_active": 0.23162992298603058, "contribution_to_active": 0.4418434500694275, "mix_mean": 0.4643630385398865, "mix_low": 0.0019016265869140625, "mix_high": 0.004857063293457031, "read_cosine": 0.44135165214538574, "beta_mean": 0.011620434001088142, "nonfinite": 0.0, "weighted_memory_to_active": 0.1413106918334961, "mix_std": 0.27595531940460205, "beta_std": 0.008481102995574474, "beta_low": 0.5494384765625, "beta_high": 0.0, "orthogonal_to_active": 0.015949483960866928, "evidence_weights": [-0.26470693945884705, -0.2646901309490204, 0.4844130575656891], "agreement_weight": 0.0506543405354023, "orthogonal_weight": -0.1457366645336151}, {"head": 3, "memory_to_active": 2.279344081878662, "contribution_to_active": 0.17998003959655762, "mix_mean": 0.042972929775714874, "mix_low": 0.38735103607177734, "mix_high": 0.0, "read_cosine": 0.1949186623096466, "beta_mean": 0.06844958662986755, "nonfinite": 0.0, "weighted_memory_to_active": 0.28343671560287476, "mix_std": 0.08788767457008362, "beta_std": 0.09679457545280457, "beta_low": 0.18719482421875, "beta_high": 0.0, "orthogonal_to_active": 0.09095185250043869, "evidence_weights": [0.01173468865454197, -0.02042902074754238, 0.10326849669218063], "agreement_weight": 0.044881489127874374, "orthogonal_weight": -0.3702472746372223}, {"head": 4, "memory_to_active": 0.5620800256729126, "contribution_to_active": 0.5903451442718506, "mix_mean": 0.359430193901062, "mix_low": 0.08715438842773438, "mix_high": 0.01593494415283203, "read_cosine": 0.5936862826347351, "beta_mean": 0.23822692036628723, "nonfinite": 0.0, "weighted_memory_to_active": 0.381636381149292, "mix_std": 0.3380573093891144, "beta_std": 0.23994769155979156, "beta_low": 0.14306640625, "beta_high": 0.0, "orthogonal_to_active": 0.07527414709329605, "evidence_weights": [-0.17251881957054138, -0.24831326305866241, 0.2133352905511856], "agreement_weight": 0.05464013293385506, "orthogonal_weight": 0.28987258672714233}, {"head": 5, "memory_to_active": 0.042170386761426926, "contribution_to_active": 0.5852230787277222, "mix_mean": 0.6406498551368713, "mix_low": 0.004240989685058594, "mix_high": 0.015093803405761719, "read_cosine": 0.21741309762001038, "beta_mean": 0.0001436237944290042, "nonfinite": 0.0, "weighted_memory_to_active": 0.025538239628076553, "mix_std": 0.2832745909690857, "beta_std": 0.00029126598383300006, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.006701539736241102, "evidence_weights": [-0.2818376421928406, -0.2759213149547577, 0.556672990322113], "agreement_weight": 0.2139156609773636, "orthogonal_weight": -0.29522591829299927}]}, {"layer": 6, "memory_to_active": 0.5352591872215271, "contribution_to_active": 0.4841979742050171, "mix_mean": 0.37016814947128296, "mix_low": 0.12169886380434036, "mix_high": 0.014403979294002056, "read_cosine": 0.3597412705421448, "beta_mean": 0.08263437449932098, "nonfinite": 0.0, "weighted_memory_to_active": 0.1672656536102295, "mix_std": 0.2626209259033203, "beta_std": 0.09523659944534302, "beta_low": 0.4453532099723816, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.03316163644194603, "output_influence": 0.36177492141723633, "residual_influence": 0.0922849103808403, "state_active": 0.07209286093711853, "state_memory": 0.017826005816459656, "state_peak": 38.989723205566406, "growth_active": 1.2361609935760498, "growth_memory": 1.4240565299987793, "heads": [{"head": 0, "memory_to_active": 0.24500510096549988, "contribution_to_active": 0.4299015402793884, "mix_mean": 0.3660239577293396, "mix_low": 0.019127845764160156, "mix_high": 0.00041675567626953125, "read_cosine": 0.43557432293891907, "beta_mean": 0.03038894757628441, "nonfinite": 0.0, "weighted_memory_to_active": 0.16506020724773407, "mix_std": 0.27407902479171753, "beta_std": 0.08377063274383545, "beta_low": 0.3707275390625, "beta_high": 0.0, "orthogonal_to_active": 0.08353680372238159, "evidence_weights": [-0.17861047387123108, -0.18549728393554688, 0.20651382207870483], "agreement_weight": 0.09723358601331711, "orthogonal_weight": -0.6694492101669312}, {"head": 1, "memory_to_active": 0.0101631386205554, "contribution_to_active": 0.40011537075042725, "mix_mean": 0.3083565831184387, "mix_low": 0.021282196044921875, "mix_high": 0.008447647094726562, "read_cosine": 0.5284737348556519, "beta_mean": 1.9692579371621832e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031770518980920315, "mix_std": 0.27416473627090454, "beta_std": 5.294151196721941e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005841342499479651, "evidence_weights": [-0.19273166358470917, -0.1839858591556549, 0.24355283379554749], "agreement_weight": -0.009846667759120464, "orthogonal_weight": -0.23284925520420074}, {"head": 2, "memory_to_active": 2.0592164993286133, "contribution_to_active": 0.07457525283098221, "mix_mean": 0.01729072816669941, "mix_low": 0.6529264450073242, "mix_high": 0.0, "read_cosine": 0.19584807753562927, "beta_mean": 0.12085980176925659, "nonfinite": 0.0, "weighted_memory_to_active": 0.09297403693199158, "mix_std": 0.048462241888046265, "beta_std": 0.09926824271678925, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.022305786609649658, "evidence_weights": [0.08596449345350266, 0.06676828116178513, -0.043571144342422485], "agreement_weight": 0.03866896405816078, "orthogonal_weight": -0.2502288818359375}, {"head": 3, "memory_to_active": 0.23649147152900696, "contribution_to_active": 0.6110973358154297, "mix_mean": 0.5388603210449219, "mix_low": 0.013341903686523438, "mix_high": 0.03458118438720703, "read_cosine": 0.5762743949890137, "beta_mean": 0.2334471195936203, "nonfinite": 0.0, "weighted_memory_to_active": 0.1605108678340912, "mix_std": 0.3519449830055237, "beta_std": 0.2509768009185791, "beta_low": 0.12835693359375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.03928578644990921, "evidence_weights": [-0.1378115862607956, -0.2712911069393158, 0.2710345685482025], "agreement_weight": -0.019259914755821228, "orthogonal_weight": 0.3158356845378876}, {"head": 4, "memory_to_active": 0.6023203134536743, "contribution_to_active": 0.9441670179367065, "mix_mean": 0.605926513671875, "mix_low": 0.005185127258300781, "mix_high": 0.04257965087890625, "read_cosine": 0.1276472955942154, "beta_mean": 0.11073024570941925, "nonfinite": 0.0, "weighted_memory_to_active": 0.554519772529602, "mix_std": 0.3305715322494507, "beta_std": 0.13672715425491333, "beta_low": 0.17279052734375, "beta_high": 0.0, "orthogonal_to_active": 0.02810894325375557, "evidence_weights": [-0.22329078614711761, -0.4318220913410187, -0.0690770149230957], "agreement_weight": -0.01160665974020958, "orthogonal_weight": 0.06237398087978363}, {"head": 5, "memory_to_active": 0.058358412235975266, "contribution_to_active": 0.44533127546310425, "mix_mean": 0.38455066084861755, "mix_low": 0.018329620361328125, "mix_high": 0.0003986358642578125, "read_cosine": 0.29462987184524536, "beta_mean": 0.0003604140365496278, "nonfinite": 0.0, "weighted_memory_to_active": 0.027351979166269302, "mix_std": 0.2965031862258911, "beta_std": 0.000623812316916883, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.02514835074543953, "evidence_weights": [-0.169838085770607, -0.2092406004667282, -0.04170579835772514], "agreement_weight": 0.06770647317171097, "orthogonal_weight": -0.9864306449890137}]}, {"layer": 7, "memory_to_active": 2.4763238430023193, "contribution_to_active": 0.1798892617225647, "mix_mean": 0.024469580501317978, "mix_low": 0.574576735496521, "mix_high": 8.58306884765625e-06, "read_cosine": 0.21658194065093994, "beta_mean": 0.17778785526752472, "nonfinite": 0.0, "weighted_memory_to_active": 0.18215809762477875, "mix_std": 0.048074621707201004, "beta_std": 0.11796148121356964, "beta_low": 0.00618489645421505, "beta_high": 0.0, "orthogonal_to_active": 0.02336306869983673, "output_influence": 0.2003975659608841, "residual_influence": 0.038754597306251526, "state_active": 0.02209295704960823, "state_memory": 0.042437322437763214, "state_peak": 17.42469024658203, "growth_active": 0.9874304533004761, "growth_memory": 1.4748438596725464, "heads": [{"head": 0, "memory_to_active": 2.078664541244507, "contribution_to_active": 0.3763921856880188, "mix_mean": 0.02712305448949337, "mix_low": 0.5038251876831055, "mix_high": 5.14984130859375e-05, "read_cosine": 0.39543378353118896, "beta_mean": 0.14585137367248535, "nonfinite": 0.0, "weighted_memory_to_active": 0.35044217109680176, "mix_std": 0.0561961904168129, "beta_std": 0.10701823979616165, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.07041387259960175, "evidence_weights": [-0.019465750083327293, -0.06606899201869965, 0.11063152551651001], "agreement_weight": -0.04004983231425285, "orthogonal_weight": 0.2712947726249695}, {"head": 1, "memory_to_active": 3.2959227561950684, "contribution_to_active": 0.2472151219844818, "mix_mean": 0.04260313883423805, "mix_low": 0.33103370666503906, "mix_high": 0.0, "read_cosine": -0.04037044197320938, "beta_mean": 0.272973895072937, "nonfinite": 0.0, "weighted_memory_to_active": 0.2868940234184265, "mix_std": 0.06375771015882492, "beta_std": 0.13213418424129486, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05097701773047447, "evidence_weights": [-0.0808044821023941, -0.07635626196861267, 0.02925223670899868], "agreement_weight": -0.001982573652639985, "orthogonal_weight": -0.17527896165847778}, {"head": 2, "memory_to_active": 1.6289525032043457, "contribution_to_active": 0.29391416907310486, "mix_mean": 0.04464494809508324, "mix_low": 0.43790435791015625, "mix_high": 0.0, "read_cosine": 0.19730260968208313, "beta_mean": 0.1481783539056778, "nonfinite": 0.0, "weighted_memory_to_active": 0.27583280205726624, "mix_std": 0.09726013988256454, "beta_std": 0.08704650402069092, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.0038462248630821705, "evidence_weights": [-0.04218174144625664, -0.09696238487958908, 0.17062830924987793], "agreement_weight": 0.024647656828165054, "orthogonal_weight": -0.014856096357107162}, {"head": 3, "memory_to_active": 3.019118309020996, "contribution_to_active": 0.03839248791337013, "mix_mean": 0.007937375456094742, "mix_low": 0.7917442321777344, "mix_high": 0.0, "read_cosine": 0.125679150223732, "beta_mean": 0.16748832166194916, "nonfinite": 0.0, "weighted_memory_to_active": 0.048768483102321625, "mix_std": 0.016886135563254356, "beta_std": 0.12007638812065125, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.010845967568457127, "evidence_weights": [0.09277115762233734, 0.09176238626241684, 0.026841800659894943], "agreement_weight": 0.0017313065472990274, "orthogonal_weight": -0.2264135777950287}, {"head": 4, "memory_to_active": 1.8733022212982178, "contribution_to_active": 0.05493069067597389, "mix_mean": 0.013402746990323067, "mix_low": 0.6577520370483398, "mix_high": 0.0, "read_cosine": 0.3120635747909546, "beta_mean": 0.21033212542533875, "nonfinite": 0.0, "weighted_memory_to_active": 0.053788673132658005, "mix_std": 0.03186063840985298, "beta_std": 0.15676535665988922, "beta_low": 0.01092529296875, "beta_high": 0.0, "orthogonal_to_active": 0.0015846553724259138, "evidence_weights": [0.04784395918250084, 0.03220386058092117, 0.006210062187165022], "agreement_weight": 0.04497845098376274, "orthogonal_weight": -0.035407379269599915}, {"head": 5, "memory_to_active": 2.9619832038879395, "contribution_to_active": 0.06849093735218048, "mix_mean": 0.011106220073997974, "mix_low": 0.7252006530761719, "mix_high": 0.0, "read_cosine": 0.3093828558921814, "beta_mean": 0.12190306186676025, "nonfinite": 0.0, "weighted_memory_to_active": 0.0772223025560379, "mix_std": 0.022486921399831772, "beta_std": 0.10472818464040756, "beta_low": 0.02508544921875, "beta_high": 0.0, "orthogonal_to_active": 0.0025106645189225674, "evidence_weights": [0.0425446555018425, 0.06841305643320084, -0.17044074833393097], "agreement_weight": -0.002781205577775836, "orthogonal_weight": -0.0360838957130909}]}, {"layer": 8, "memory_to_active": 1.0495537519454956, "contribution_to_active": 0.3173799514770508, "mix_mean": 0.21680128574371338, "mix_low": 0.12028615176677704, "mix_high": 0.0022333464585244656, "read_cosine": 0.5176230669021606, "beta_mean": 0.08003748953342438, "nonfinite": 0.0, "weighted_memory_to_active": 0.18670745193958282, "mix_std": 0.167251855134964, "beta_std": 0.06876794993877411, "beta_low": 0.2092183530330658, "beta_high": 0.0, "orthogonal_to_active": 0.04063833877444267, "output_influence": 0.2648003399372101, "residual_influence": 0.05786333978176117, "state_active": 0.05971607565879822, "state_memory": 0.030960552394390106, "state_peak": 27.3240966796875, "growth_active": 0.9675316214561462, "growth_memory": 1.571501612663269, "heads": [{"head": 0, "memory_to_active": 0.09533032774925232, "contribution_to_active": 0.4478122591972351, "mix_mean": 0.45263439416885376, "mix_low": 0.0003204345703125, "mix_high": 3.814697265625e-05, "read_cosine": 0.7392836809158325, "beta_mean": 0.0036083091981709003, "nonfinite": 0.0, "weighted_memory_to_active": 0.04744090139865875, "mix_std": 0.22735348343849182, "beta_std": 0.00500548304989934, "beta_low": 0.93292236328125, "beta_high": 0.0, "orthogonal_to_active": 0.016094475984573364, "evidence_weights": [-0.2579611539840698, -0.27612966299057007, 0.46346989274024963], "agreement_weight": 0.06765484809875488, "orthogonal_weight": -0.5504462122917175}, {"head": 1, "memory_to_active": 0.7306339740753174, "contribution_to_active": 0.2930140793323517, "mix_mean": 0.2610711455345154, "mix_low": 0.010099411010742188, "mix_high": 2.47955322265625e-05, "read_cosine": 0.6047550439834595, "beta_mean": 0.0700647309422493, "nonfinite": 0.0, "weighted_memory_to_active": 0.23789983987808228, "mix_std": 0.17642562091350555, "beta_std": 0.05207360163331032, "beta_low": 0.03662109375, "beta_high": 0.0, "orthogonal_to_active": 0.02348947525024414, "evidence_weights": [-0.16883249580860138, -0.16567087173461914, 0.29273486137390137], "agreement_weight": 6.975352152949199e-05, "orthogonal_weight": 0.13045264780521393}, {"head": 2, "memory_to_active": 1.7896935939788818, "contribution_to_active": 0.11397813260555267, "mix_mean": 0.037713825702667236, "mix_low": 0.3346366882324219, "mix_high": 0.0, "read_cosine": 0.19137971103191376, "beta_mean": 0.13097089529037476, "nonfinite": 0.0, "weighted_memory_to_active": 0.1897692084312439, "mix_std": 0.06572311371564865, "beta_std": 0.10226866602897644, "beta_low": 0.00238037109375, "beta_high": 0.0, "orthogonal_to_active": 0.060058727860450745, "evidence_weights": [0.0041732387617230415, -0.05482330918312073, 0.043162278831005096], "agreement_weight": 0.0788768082857132, "orthogonal_weight": -0.35008540749549866}, {"head": 3, "memory_to_active": 0.6811270713806152, "contribution_to_active": 0.19163988530635834, "mix_mean": 0.153540700674057, "mix_low": 0.03519916534423828, "mix_high": 0.0, "read_cosine": 0.6604290008544922, "beta_mean": 0.05075491964817047, "nonfinite": 0.0, "weighted_memory_to_active": 0.20164719223976135, "mix_std": 0.13990654051303864, "beta_std": 0.042897697538137436, "beta_low": 0.05279541015625, "beta_high": 0.0, "orthogonal_to_active": 0.035755399614572525, "evidence_weights": [-0.08728580921888351, -0.12344151735305786, 0.4967370629310608], "agreement_weight": 0.04979147017002106, "orthogonal_weight": 0.30899104475975037}, {"head": 4, "memory_to_active": 2.769329309463501, "contribution_to_active": 0.2820790410041809, "mix_mean": 0.033720772713422775, "mix_low": 0.2815523147583008, "mix_high": 0.0, "read_cosine": 0.3605233430862427, "beta_mean": 0.14029160141944885, "nonfinite": 0.0, "weighted_memory_to_active": 0.292834997177124, "mix_std": 0.048834435641765594, "beta_std": 0.0920386016368866, "beta_low": 0.0008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.009734652005136013, "evidence_weights": [-0.007581961806863546, -0.030202511698007584, -0.005907632876187563], "agreement_weight": 0.006665173452347517, "orthogonal_weight": 0.03839981183409691}, {"head": 5, "memory_to_active": 0.23120781779289246, "contribution_to_active": 0.5757563710212708, "mix_mean": 0.362126886844635, "mix_low": 0.05990886688232422, "mix_high": 0.013337135314941406, "read_cosine": 0.5493673086166382, "beta_mean": 0.08453445136547089, "nonfinite": 0.0, "weighted_memory_to_active": 0.15065255761146545, "mix_std": 0.3452679216861725, "beta_std": 0.11832364648580551, "beta_low": 0.229736328125, "beta_high": 0.0, "orthogonal_to_active": 0.09869728982448578, "evidence_weights": [-0.10636524856090546, -0.33115890622138977, -0.2232895791530609], "agreement_weight": 0.0359339714050293, "orthogonal_weight": 0.8483889102935791}]}, {"layer": 9, "memory_to_active": 1.4430124759674072, "contribution_to_active": 0.5195689797401428, "mix_mean": 0.13354474306106567, "mix_low": 0.2641638219356537, "mix_high": 0.0024321875534951687, "read_cosine": 0.3391883671283722, "beta_mean": 0.11465059220790863, "nonfinite": 0.0, "weighted_memory_to_active": 0.4063759446144104, "mix_std": 0.18535837531089783, "beta_std": 0.05763257294893265, "beta_low": 0.4236246943473816, "beta_high": 0.0, "orthogonal_to_active": 0.015347779728472233, "output_influence": 0.2933000326156616, "residual_influence": 0.06322072446346283, "state_active": 0.03388422727584839, "state_memory": 0.03467418625950813, "state_peak": 34.76823425292969, "growth_active": 1.0130006074905396, "growth_memory": 1.5325883626937866, "heads": [{"head": 0, "memory_to_active": 0.02125677838921547, "contribution_to_active": 0.3025626540184021, "mix_mean": 0.2744542956352234, "mix_low": 0.060977935791015625, "mix_high": 0.00079345703125, "read_cosine": 0.2315472662448883, "beta_mean": 6.626571121159941e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007012281566858292, "mix_std": 0.2677145004272461, "beta_std": 0.00017789259436540306, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002944187493994832, "evidence_weights": [-0.16006653010845184, -0.15728473663330078, 0.6563904881477356], "agreement_weight": 0.04979011416435242, "orthogonal_weight": -0.43824663758277893}, {"head": 1, "memory_to_active": 0.07726279646158218, "contribution_to_active": 0.2854386270046234, "mix_mean": 0.1292053759098053, "mix_low": 0.24596881866455078, "mix_high": 0.0001220703125, "read_cosine": 0.4061316251754761, "beta_mean": 0.00029960283427499235, "nonfinite": 0.0, "weighted_memory_to_active": 0.018622681498527527, "mix_std": 0.19441571831703186, "beta_std": 0.0006187426624819636, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.0045992364175617695, "evidence_weights": [-0.10838895291090012, -0.09820151329040527, 0.24984495341777802], "agreement_weight": 0.18625859916210175, "orthogonal_weight": -0.26843246817588806}, {"head": 2, "memory_to_active": 0.7308168411254883, "contribution_to_active": 0.36730140447616577, "mix_mean": 0.1759365350008011, "mix_low": 0.15213871002197266, "mix_high": 0.000152587890625, "read_cosine": 0.23829281330108643, "beta_mean": 0.014320281334221363, "nonfinite": 0.0, "weighted_memory_to_active": 0.2116856724023819, "mix_std": 0.23298414051532745, "beta_std": 0.017347237095236778, "beta_low": 0.53887939453125, "beta_high": 0.0, "orthogonal_to_active": 0.01847025565803051, "evidence_weights": [-0.08581672608852386, -0.08103954792022705, 0.07971836626529694], "agreement_weight": 0.09364397823810577, "orthogonal_weight": -0.10571292787790298}, {"head": 3, "memory_to_active": 2.2997541427612305, "contribution_to_active": 0.2519564926624298, "mix_mean": 0.04747036099433899, "mix_low": 0.33249664306640625, "mix_high": 0.0, "read_cosine": 0.4343218207359314, "beta_mean": 0.15902504324913025, "nonfinite": 0.0, "weighted_memory_to_active": 0.22475093603134155, "mix_std": 0.09478482604026794, "beta_std": 0.10036806762218475, "beta_low": 0.0032958984375, "beta_high": 0.0, "orthogonal_to_active": 0.01637357473373413, "evidence_weights": [0.021260175853967667, 0.014119381085038185, -0.028847509995102882], "agreement_weight": -0.02287566475570202, "orthogonal_weight": 0.06964732706546783}, {"head": 4, "memory_to_active": 3.5148534774780273, "contribution_to_active": 1.3894999027252197, "mix_mean": 0.12742511928081512, "mix_low": 0.26312923431396484, "mix_high": 0.013498306274414062, "read_cosine": 0.303677499294281, "beta_mean": 0.28836292028427124, "nonfinite": 0.0, "weighted_memory_to_active": 1.3673380613327026, "mix_std": 0.21389830112457275, "beta_std": 0.11314442753791809, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0033221421763300896, "evidence_weights": [-0.04255572706460953, -0.08322914689779282, 0.10881643742322922], "agreement_weight": -0.05435361713171005, "orthogonal_weight": 0.002519045490771532}, {"head": 5, "memory_to_active": 2.0141303539276123, "contribution_to_active": 0.5206549167633057, "mix_mean": 0.04677668586373329, "mix_low": 0.5302715301513672, "mix_high": 2.6702880859375e-05, "read_cosine": 0.42115917801856995, "beta_mean": 0.22582942247390747, "nonfinite": 0.0, "weighted_memory_to_active": 0.608846127986908, "mix_std": 0.10835278034210205, "beta_std": 0.11413907259702682, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0463772788643837, "evidence_weights": [0.035116709768772125, -0.018275760114192963, 0.028186652809381485], "agreement_weight": -0.03054727427661419, "orthogonal_weight": 0.10090933740139008}]}, {"layer": 10, "memory_to_active": 0.4527980387210846, "contribution_to_active": 0.38085493445396423, "mix_mean": 0.24905845522880554, "mix_low": 0.1366969794034958, "mix_high": 0.004797141067683697, "read_cosine": 0.3708854913711548, "beta_mean": 0.05784086138010025, "nonfinite": 0.0, "weighted_memory_to_active": 0.14748068153858185, "mix_std": 0.21806925535202026, "beta_std": 0.06381328403949738, "beta_low": 0.48284912109375, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.04120732098817825, "output_influence": 0.34592685103416443, "residual_influence": 0.08849562704563141, "state_active": 0.07648216187953949, "state_memory": 0.02227517031133175, "state_peak": 42.976810455322266, "growth_active": 1.1024298667907715, "growth_memory": 1.3476094007492065, "heads": [{"head": 0, "memory_to_active": 0.6763194799423218, "contribution_to_active": 0.8995817303657532, "mix_mean": 0.5727033019065857, "mix_low": 0.016588211059570312, "mix_high": 0.028008460998535156, "read_cosine": 0.1260593682527542, "beta_mean": 0.19275128841400146, "nonfinite": 0.0, "weighted_memory_to_active": 0.46313929557800293, "mix_std": 0.34461450576782227, "beta_std": 0.21212708950042725, "beta_low": 0.09039306640625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.15157398581504822, "evidence_weights": [-0.1455174684524536, -0.30949607491493225, -0.12529371678829193], "agreement_weight": 0.01575692743062973, "orthogonal_weight": 0.347169429063797}, {"head": 1, "memory_to_active": 0.025753643363714218, "contribution_to_active": 0.2983086109161377, "mix_mean": 0.21299615502357483, "mix_low": 0.04630851745605469, "mix_high": 4.57763671875e-05, "read_cosine": 0.1793188601732254, "beta_mean": 0.0001228727342095226, "nonfinite": 0.0, "weighted_memory_to_active": 0.005001481622457504, "mix_std": 0.22311758995056152, "beta_std": 0.0002783106465358287, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001900569419376552, "evidence_weights": [-0.145613893866539, -0.11604209244251251, 0.0974109098315239], "agreement_weight": 0.03616733476519585, "orthogonal_weight": -0.4013453423976898}, {"head": 2, "memory_to_active": 1.5493942499160767, "contribution_to_active": 0.21165332198143005, "mix_mean": 0.02303503081202507, "mix_low": 0.6226425170898438, "mix_high": 0.0, "read_cosine": 0.22959718108177185, "beta_mean": 0.0761403739452362, "nonfinite": 0.0, "weighted_memory_to_active": 0.24998313188552856, "mix_std": 0.05932895094156265, "beta_std": 0.06360110640525818, "beta_low": 0.010009765625, "beta_high": 0.0, "orthogonal_to_active": 0.036737941205501556, "evidence_weights": [0.04362596943974495, -0.009201563894748688, 0.09214486926794052], "agreement_weight": -0.03496966138482094, "orthogonal_weight": -0.1508926898241043}, {"head": 3, "memory_to_active": 0.20786985754966736, "contribution_to_active": 0.21099920570850372, "mix_mean": 0.21696290373802185, "mix_low": 0.0962057113647461, "mix_high": 0.0007162094116210938, "read_cosine": 0.754170298576355, "beta_mean": 0.06797230243682861, "nonfinite": 0.0, "weighted_memory_to_active": 0.06964927166700363, "mix_std": 0.2599939703941345, "beta_std": 0.09220857918262482, "beta_low": 0.09942626953125, "beta_high": 0.0, "orthogonal_to_active": 0.036020971834659576, "evidence_weights": [-0.11548185348510742, -0.15155814588069916, 0.2361476570367813], "agreement_weight": 0.016071919351816177, "orthogonal_weight": 0.6856912970542908}, {"head": 4, "memory_to_active": 0.07614367455244064, "contribution_to_active": 0.2714420557022095, "mix_mean": 0.21216510236263275, "mix_low": 0.027976036071777344, "mix_high": 2.86102294921875e-06, "read_cosine": 0.17304639518260956, "beta_mean": 0.0003464353794697672, "nonfinite": 0.0, "weighted_memory_to_active": 0.019567370414733887, "mix_std": 0.2127596139907837, "beta_std": 0.0022377397399395704, "beta_low": 0.99859619140625, "beta_high": 0.0, "orthogonal_to_active": 0.004578443244099617, "evidence_weights": [-0.14066003262996674, -0.10971040278673172, 0.018677836284041405], "agreement_weight": 0.03600260987877846, "orthogonal_weight": -0.25784221291542053}, {"head": 5, "memory_to_active": 0.1813071370124817, "contribution_to_active": 0.3931444585323334, "mix_mean": 0.2564881443977356, "mix_low": 0.010460853576660156, "mix_high": 9.5367431640625e-06, "read_cosine": 0.7631207704544067, "beta_mean": 0.009711864404380322, "nonfinite": 0.0, "weighted_memory_to_active": 0.07754352688789368, "mix_std": 0.20860078930854797, "beta_std": 0.012426910921931267, "beta_low": 0.69866943359375, "beta_high": 0.0, "orthogonal_to_active": 0.016432005912065506, "evidence_weights": [-0.18698713183403015, -0.15351900458335876, 0.1716764271259308], "agreement_weight": 0.08317844569683075, "orthogonal_weight": -0.42296838760375977}]}, {"layer": 11, "memory_to_active": 0.32166606187820435, "contribution_to_active": 0.3998357951641083, "mix_mean": 0.31222862005233765, "mix_low": 0.058928173035383224, "mix_high": 0.018739065155386925, "read_cosine": 0.530491828918457, "beta_mean": 0.04191046208143234, "nonfinite": 0.0, "weighted_memory_to_active": 0.09439054131507874, "mix_std": 0.2763248085975647, "beta_std": 0.06011787801980972, "beta_low": 0.4436137080192566, "beta_high": 0.0, "orthogonal_to_active": 0.02801789715886116, "output_influence": 0.46641263365745544, "residual_influence": 0.11514265090227127, "state_active": 0.12560445070266724, "state_memory": 0.016270089894533157, "state_peak": 23.55205535888672, "growth_active": 1.1838879585266113, "growth_memory": 1.4005862474441528, "heads": [{"head": 0, "memory_to_active": 0.0861300528049469, "contribution_to_active": 0.46961450576782227, "mix_mean": 0.39100801944732666, "mix_low": 0.009171485900878906, "mix_high": 0.0022020339965820312, "read_cosine": 0.435669869184494, "beta_mean": 0.0006079774466343224, "nonfinite": 0.0, "weighted_memory_to_active": 0.04710294306278229, "mix_std": 0.27488115429878235, "beta_std": 0.00508532440289855, "beta_low": 0.99560546875, "beta_high": 0.0, "orthogonal_to_active": 0.021128617227077484, "evidence_weights": [-0.15532095730304718, -0.2212591916322708, 0.3197847604751587], "agreement_weight": 0.12856003642082214, "orthogonal_weight": -0.6016328930854797}, {"head": 1, "memory_to_active": 0.5456297397613525, "contribution_to_active": 0.17509914934635162, "mix_mean": 0.17109620571136475, "mix_low": 0.06558990478515625, "mix_high": 3.62396240234375e-05, "read_cosine": 0.6077661514282227, "beta_mean": 0.038903363049030304, "nonfinite": 0.0, "weighted_memory_to_active": 0.14744152128696442, "mix_std": 0.19638104736804962, "beta_std": 0.06444697827100754, "beta_low": 0.32000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.0129974614828825, "evidence_weights": [-0.13525275886058807, -0.09409444779157639, 0.10575306415557861], "agreement_weight": 0.09218032658100128, "orthogonal_weight": -0.1219906359910965}, {"head": 2, "memory_to_active": 0.07836213707923889, "contribution_to_active": 0.8002001047134399, "mix_mean": 0.4598701000213623, "mix_low": 0.039211273193359375, "mix_high": 0.06419086456298828, "read_cosine": 0.368111789226532, "beta_mean": 0.07117687165737152, "nonfinite": 0.0, "weighted_memory_to_active": 0.06451309472322464, "mix_std": 0.39689913392066956, "beta_std": 0.10258302092552185, "beta_low": 0.4188232421875, "beta_high": 0.0, "orthogonal_to_active": 0.07308124005794525, "evidence_weights": [-0.20675157010555267, -0.35244765877723694, -0.3084769546985626], "agreement_weight": -0.024320043623447418, "orthogonal_weight": 1.2710517644882202}, {"head": 3, "memory_to_active": 0.04302414879202843, "contribution_to_active": 0.3920341730117798, "mix_mean": 0.4470479488372803, "mix_low": 0.055141448974609375, "mix_high": 0.045416831970214844, "read_cosine": 0.5690038800239563, "beta_mean": 0.07412727177143097, "nonfinite": 0.0, "weighted_memory_to_active": 0.023101603612303734, "mix_std": 0.4045320451259613, "beta_std": 0.10779103636741638, "beta_low": 0.395263671875, "beta_high": 0.0, "orthogonal_to_active": 0.025609329342842102, "evidence_weights": [-0.2651807963848114, -0.34802499413490295, 0.03126734495162964], "agreement_weight": 0.014494281262159348, "orthogonal_weight": 1.3506882190704346}, {"head": 4, "memory_to_active": 0.9160338044166565, "contribution_to_active": 0.1831941306591034, "mix_mean": 0.09448293596506119, "mix_low": 0.17483997344970703, "mix_high": 0.0, "read_cosine": 0.3123096823692322, "beta_mean": 0.0452100932598114, "nonfinite": 0.0, "weighted_memory_to_active": 0.15450076758861542, "mix_std": 0.13775452971458435, "beta_std": 0.0535004585981369, "beta_low": 0.13616943359375, "beta_high": 0.0, "orthogonal_to_active": 0.01132146455347538, "evidence_weights": [-0.035381149500608444, -0.022327367216348648, 0.4783257246017456], "agreement_weight": -0.035405565053224564, "orthogonal_weight": -0.09174646437168121}, {"head": 5, "memory_to_active": 0.26081639528274536, "contribution_to_active": 0.37887245416641235, "mix_mean": 0.30986642837524414, "mix_low": 0.009614944458007812, "mix_high": 0.0005884170532226562, "read_cosine": 0.8900895118713379, "beta_mean": 0.021437186747789383, "nonfinite": 0.0, "weighted_memory_to_active": 0.12968327105045319, "mix_std": 0.24750085175037384, "beta_std": 0.02730042114853859, "beta_low": 0.39581298828125, "beta_high": 0.0, "orthogonal_to_active": 0.023969270288944244, "evidence_weights": [-0.17396286129951477, -0.21155408024787903, 0.24845613539218903], "agreement_weight": 0.047410812228918076, "orthogonal_weight": -0.4907282292842865}]}, {"layer": 12, "memory_to_active": 0.7497409582138062, "contribution_to_active": 0.4460659623146057, "mix_mean": 0.30334609746932983, "mix_low": 0.07959922403097153, "mix_high": 0.0005232493276707828, "read_cosine": 0.3366858959197998, "beta_mean": 0.03129354491829872, "nonfinite": 0.0, "weighted_memory_to_active": 0.27128496766090393, "mix_std": 0.22022700309753418, "beta_std": 0.037635523825883865, "beta_low": 0.6049296259880066, "beta_high": 0.0, "orthogonal_to_active": 0.055400557816028595, "output_influence": 0.25797921419143677, "residual_influence": 0.06478410959243774, "state_active": 0.05923470854759216, "state_memory": 0.023319482803344727, "state_peak": 45.211143493652344, "growth_active": 1.174943447113037, "growth_memory": 1.4036147594451904, "heads": [{"head": 0, "memory_to_active": 0.8299728035926819, "contribution_to_active": 0.23363766074180603, "mix_mean": 0.19370907545089722, "mix_low": 0.054619789123535156, "mix_high": 0.0001049041748046875, "read_cosine": 0.4584987461566925, "beta_mean": 0.01929667219519615, "nonfinite": 0.0, "weighted_memory_to_active": 0.26024317741394043, "mix_std": 0.20419740676879883, "beta_std": 0.04091665521264076, "beta_low": 0.5780029296875, "beta_high": 0.0, "orthogonal_to_active": 0.06561901420354843, "evidence_weights": [-0.09587566554546356, -0.13392360508441925, 0.23408842086791992], "agreement_weight": 0.09450199455022812, "orthogonal_weight": -0.3717687129974365}, {"head": 1, "memory_to_active": 0.30015045404434204, "contribution_to_active": 0.3552342653274536, "mix_mean": 0.30382367968559265, "mix_low": 0.005154609680175781, "mix_high": 0.00015926361083984375, "read_cosine": 0.6150097846984863, "beta_mean": 0.004327801521867514, "nonfinite": 0.0, "weighted_memory_to_active": 0.13549330830574036, "mix_std": 0.2403363585472107, "beta_std": 0.006523206830024719, "beta_low": 0.89813232421875, "beta_high": 0.0, "orthogonal_to_active": 0.06790615618228912, "evidence_weights": [-0.13725952804088593, -0.18385808169841766, 0.3723963499069214], "agreement_weight": 0.08575168997049332, "orthogonal_weight": -0.808020830154419}, {"head": 2, "memory_to_active": 0.6495199799537659, "contribution_to_active": 0.3605509102344513, "mix_mean": 0.2909339666366577, "mix_low": 0.023046493530273438, "mix_high": 0.00045490264892578125, "read_cosine": 0.3126683831214905, "beta_mean": 0.09001922607421875, "nonfinite": 0.0, "weighted_memory_to_active": 0.28492283821105957, "mix_std": 0.24324704706668854, "beta_std": 0.08144466578960419, "beta_low": 0.009033203125, "beta_high": 0.0, "orthogonal_to_active": 0.051309749484062195, "evidence_weights": [-0.17892871797084808, -0.09386810660362244, 0.17774930596351624], "agreement_weight": 0.04373062402009964, "orthogonal_weight": 0.22551143169403076}, {"head": 3, "memory_to_active": 0.07391677796840668, "contribution_to_active": 0.5571094155311584, "mix_mean": 0.5416327714920044, "mix_low": 0.001430511474609375, "mix_high": 0.0012416839599609375, "read_cosine": 0.060949891805648804, "beta_mean": 0.001110342564061284, "nonfinite": 0.0, "weighted_memory_to_active": 0.041407495737075806, "mix_std": 0.25827908515930176, "beta_std": 0.0014876984059810638, "beta_low": 0.99688720703125, "beta_high": 0.0, "orthogonal_to_active": 0.022732023149728775, "evidence_weights": [-0.27553287148475647, -0.26001831889152527, 0.1848689168691635], "agreement_weight": 0.0803268775343895, "orthogonal_weight": -0.5688351988792419}, {"head": 4, "memory_to_active": 2.5376038551330566, "contribution_to_active": 0.793514609336853, "mix_mean": 0.05342670530080795, "mix_low": 0.38704967498779297, "mix_high": 0.0006618499755859375, "read_cosine": 0.14645756781101227, "beta_mean": 0.07112261652946472, "nonfinite": 0.0, "weighted_memory_to_active": 0.8650370836257935, "mix_std": 0.1089492067694664, "beta_std": 0.08952877670526505, "beta_low": 0.19366455078125, "beta_high": 0.0, "orthogonal_to_active": 0.09513530880212784, "evidence_weights": [0.05928049981594086, -0.027000393718481064, 0.0686393454670906], "agreement_weight": -0.021074308082461357, "orthogonal_weight": -0.11854127794504166}, {"head": 5, "memory_to_active": 0.10728191584348679, "contribution_to_active": 0.3763488531112671, "mix_mean": 0.436550498008728, "mix_low": 0.00629425048828125, "mix_high": 0.0005168914794921875, "read_cosine": 0.4265309274196625, "beta_mean": 0.0018846186576411128, "nonfinite": 0.0, "weighted_memory_to_active": 0.04060584306716919, "mix_std": 0.26635292172431946, "beta_std": 0.005912122782319784, "beta_low": 0.953857421875, "beta_high": 0.0, "orthogonal_to_active": 0.029701080173254013, "evidence_weights": [-0.23343516886234283, -0.23379932343959808, 0.396200954914093], "agreement_weight": 0.03642672300338745, "orthogonal_weight": -0.8229044675827026}]}, {"layer": 13, "memory_to_active": 0.2709941267967224, "contribution_to_active": 0.36864763498306274, "mix_mean": 0.35344019532203674, "mix_low": 0.006850560661405325, "mix_high": 0.0011739730834960938, "read_cosine": 0.39461976289749146, "beta_mean": 0.02057068608701229, "nonfinite": 0.0, "weighted_memory_to_active": 0.11315901577472687, "mix_std": 0.24089442193508148, "beta_std": 0.026807859539985657, "beta_low": 0.699188232421875, "beta_high": 0.0, "orthogonal_to_active": 0.06298276782035828, "output_influence": 0.3162214159965515, "residual_influence": 0.09891573339700699, "state_active": 0.08817052096128464, "state_memory": 0.028276458382606506, "state_peak": 64.03182983398438, "growth_active": 0.9660132527351379, "growth_memory": 1.129448652267456, "heads": [{"head": 0, "memory_to_active": 0.3022215962409973, "contribution_to_active": 0.3410990238189697, "mix_mean": 0.30280473828315735, "mix_low": 0.006562232971191406, "mix_high": 0.000354766845703125, "read_cosine": 0.33853471279144287, "beta_mean": 0.005262451712042093, "nonfinite": 0.0, "weighted_memory_to_active": 0.12219174206256866, "mix_std": 0.23915432393550873, "beta_std": 0.006663797423243523, "beta_low": 0.85101318359375, "beta_high": 0.0, "orthogonal_to_active": 0.09984154999256134, "evidence_weights": [-0.16469217836856842, -0.18453964591026306, 0.36875900626182556], "agreement_weight": 0.05274110659956932, "orthogonal_weight": -0.8990532159805298}, {"head": 1, "memory_to_active": 0.21915289759635925, "contribution_to_active": 0.38507556915283203, "mix_mean": 0.3076609671115875, "mix_low": 0.006855964660644531, "mix_high": 9.5367431640625e-06, "read_cosine": 0.3056856691837311, "beta_mean": 0.0018990151584148407, "nonfinite": 0.0, "weighted_memory_to_active": 0.08487825095653534, "mix_std": 0.23933809995651245, "beta_std": 0.0032071764580905437, "beta_low": 0.9730224609375, "beta_high": 0.0, "orthogonal_to_active": 0.06866539269685745, "evidence_weights": [-0.16044054925441742, -0.17787788808345795, 0.4215027391910553], "agreement_weight": 0.06274684518575668, "orthogonal_weight": -0.874893844127655}, {"head": 2, "memory_to_active": 0.2991611361503601, "contribution_to_active": 0.2862016558647156, "mix_mean": 0.25371846556663513, "mix_low": 0.0056667327880859375, "mix_high": 6.67572021484375e-06, "read_cosine": 0.7120501399040222, "beta_mean": 0.023057466372847557, "nonfinite": 0.0, "weighted_memory_to_active": 0.11038197576999664, "mix_std": 0.19737830758094788, "beta_std": 0.04222777485847473, "beta_low": 0.506103515625, "beta_high": 0.0, "orthogonal_to_active": 0.028932593762874603, "evidence_weights": [-0.17308758199214935, -0.1449611335992813, 0.36496874690055847], "agreement_weight": 0.06680928170681, "orthogonal_weight": -0.45691344141960144}, {"head": 3, "memory_to_active": 0.1304919570684433, "contribution_to_active": 0.511785089969635, "mix_mean": 0.5270767211914062, "mix_low": 0.004425048828125, "mix_high": 0.0010309219360351562, "read_cosine": -0.05025738105177879, "beta_mean": 0.0007220315164886415, "nonfinite": 0.0, "weighted_memory_to_active": 0.08476680517196655, "mix_std": 0.22693675756454468, "beta_std": 0.0032883239910006523, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.07686174660921097, "evidence_weights": [-0.33272287249565125, -0.25378134846687317, 0.5480208396911621], "agreement_weight": 0.5798913836479187, "orthogonal_weight": -0.9209480285644531}, {"head": 4, "memory_to_active": 0.3321179151535034, "contribution_to_active": 0.29224127531051636, "mix_mean": 0.44335007667541504, "mix_low": 0.0046253204345703125, "mix_high": 0.005475044250488281, "read_cosine": 0.7984336614608765, "beta_mean": 0.08399688452482224, "nonfinite": 0.0, "weighted_memory_to_active": 0.1481408029794693, "mix_std": 0.2908080220222473, "beta_std": 0.08275974541902542, "beta_low": 0.027587890625, "beta_high": 0.0, "orthogonal_to_active": 0.04658937454223633, "evidence_weights": [-0.2598505914211273, -0.2776165306568146, 0.3686298727989197], "agreement_weight": 0.010425560176372528, "orthogonal_weight": 0.469288170337677}, {"head": 5, "memory_to_active": 0.3428192138671875, "contribution_to_active": 0.39548319578170776, "mix_mean": 0.28603002429008484, "mix_low": 0.012968063354492188, "mix_high": 0.00016689300537109375, "read_cosine": 0.26327162981033325, "beta_mean": 0.008486256003379822, "nonfinite": 0.0, "weighted_memory_to_active": 0.1285945028066635, "mix_std": 0.25175103545188904, "beta_std": 0.022700343281030655, "beta_low": 0.84442138671875, "beta_high": 0.0, "orthogonal_to_active": 0.05700591206550598, "evidence_weights": [-0.15984900295734406, -0.1816815882921219, 0.07368145138025284], "agreement_weight": 0.04840819537639618, "orthogonal_weight": -0.5048401951789856}]}, {"layer": 14, "memory_to_active": 0.31681525707244873, "contribution_to_active": 0.4030313193798065, "mix_mean": 0.3519805669784546, "mix_low": 0.028636615723371506, "mix_high": 0.0012466113548725843, "read_cosine": 0.581081748008728, "beta_mean": 0.08067421615123749, "nonfinite": 0.0, "weighted_memory_to_active": 0.13969910144805908, "mix_std": 0.24672247469425201, "beta_std": 0.07541963458061218, "beta_low": 0.3127950131893158, "beta_high": 0.0, "orthogonal_to_active": 0.032074980437755585, "output_influence": 0.21848449110984802, "residual_influence": 0.0818784162402153, "state_active": 0.09000780433416367, "state_memory": 0.022346146404743195, "state_peak": 37.28109359741211, "growth_active": 1.1557203531265259, "growth_memory": 1.4937070608139038, "heads": [{"head": 0, "memory_to_active": 0.2937912940979004, "contribution_to_active": 0.2944628894329071, "mix_mean": 0.26606544852256775, "mix_low": 0.026329994201660156, "mix_high": 0.00014495849609375, "read_cosine": 0.6393986344337463, "beta_mean": 0.06938808411359787, "nonfinite": 0.0, "weighted_memory_to_active": 0.11045821756124496, "mix_std": 0.24527348577976227, "beta_std": 0.09245139360427856, "beta_low": 0.17889404296875, "beta_high": 0.0, "orthogonal_to_active": 0.006202181801199913, "evidence_weights": [-0.07659973204135895, -0.15990759432315826, 0.5658782720565796], "agreement_weight": 0.05133044347167015, "orthogonal_weight": 0.07865069061517715}, {"head": 1, "memory_to_active": 0.4163248836994171, "contribution_to_active": 0.451844185590744, "mix_mean": 0.3036973476409912, "mix_low": 0.09027099609375, "mix_high": 0.0008115768432617188, "read_cosine": 0.4995819926261902, "beta_mean": 0.3144848942756653, "nonfinite": 0.0, "weighted_memory_to_active": 0.19812509417533875, "mix_std": 0.2872312068939209, "beta_std": 0.2669672667980194, "beta_low": 0.07562255859375, "beta_high": 0.0, "orthogonal_to_active": 0.0024087021593004465, "evidence_weights": [-0.045480456203222275, -0.24012447893619537, 0.2124243825674057], "agreement_weight": 0.05846146121621132, "orthogonal_weight": 0.015784552320837975}, {"head": 2, "memory_to_active": 0.3770855665206909, "contribution_to_active": 0.3959808945655823, "mix_mean": 0.37143653631210327, "mix_low": 0.00685882568359375, "mix_high": 0.00018405914306640625, "read_cosine": 0.4322878420352936, "beta_mean": 0.024223264306783676, "nonfinite": 0.0, "weighted_memory_to_active": 0.1812143325805664, "mix_std": 0.262006014585495, "beta_std": 0.026095999404788017, "beta_low": 0.25811767578125, "beta_high": 0.0, "orthogonal_to_active": 0.11093069612979889, "evidence_weights": [-0.19423797726631165, -0.19906136393547058, 0.27660658955574036], "agreement_weight": 0.09292235225439072, "orthogonal_weight": -0.7179175615310669}, {"head": 3, "memory_to_active": 0.07797972857952118, "contribution_to_active": 0.6143214702606201, "mix_mean": 0.625076174736023, "mix_low": 0.0021371841430664062, "mix_high": 0.00457000732421875, "read_cosine": 0.6094534397125244, "beta_mean": 0.011239958927035332, "nonfinite": 0.0, "weighted_memory_to_active": 0.05257481336593628, "mix_std": 0.23848026990890503, "beta_std": 0.008619502186775208, "beta_low": 0.55145263671875, "beta_high": 0.0, "orthogonal_to_active": 0.01617070846259594, "evidence_weights": [-0.37077486515045166, -0.3215166926383972, 0.38524895906448364], "agreement_weight": 0.06858009845018387, "orthogonal_weight": -0.43043166399002075}, {"head": 4, "memory_to_active": 0.3050394058227539, "contribution_to_active": 0.4405856430530548, "mix_mean": 0.3514123260974884, "mix_low": 0.006350517272949219, "mix_high": 0.001644134521484375, "read_cosine": 0.6231434345245361, "beta_mean": 0.05729031190276146, "nonfinite": 0.0, "weighted_memory_to_active": 0.14788827300071716, "mix_std": 0.24787558615207672, "beta_std": 0.04350602626800537, "beta_low": 0.01519775390625, "beta_high": 0.0, "orthogonal_to_active": 0.016374491155147552, "evidence_weights": [-0.19083301723003387, -0.23570220172405243, 0.16833409667015076], "agreement_weight": 0.10395023971796036, "orthogonal_weight": -0.1697612702846527}, {"head": 5, "memory_to_active": 0.43067044019699097, "contribution_to_active": 0.2209928333759308, "mix_mean": 0.1941956877708435, "mix_low": 0.039872169494628906, "mix_high": 0.00012493133544921875, "read_cosine": 0.6826250553131104, "beta_mean": 0.007418787106871605, "nonfinite": 0.0, "weighted_memory_to_active": 0.14793390035629272, "mix_std": 0.19946826994419098, "beta_std": 0.01487760990858078, "beta_low": 0.7974853515625, "beta_high": 0.0, "orthogonal_to_active": 0.04036308825016022, "evidence_weights": [-0.17260177433490753, -0.1260797083377838, 0.05502042919397354], "agreement_weight": 0.10250547528266907, "orthogonal_weight": -0.44043275713920593}]}, {"layer": 15, "memory_to_active": 0.13352665305137634, "contribution_to_active": 0.4341706931591034, "mix_mean": 0.3544943630695343, "mix_low": 0.006852626800537109, "mix_high": 0.0017609596252441406, "read_cosine": 0.6002541184425354, "beta_mean": 0.04979131743311882, "nonfinite": 0.0, "weighted_memory_to_active": 0.06254176050424576, "mix_std": 0.2547331750392914, "beta_std": 0.05372778698801994, "beta_low": 0.6334127187728882, "beta_high": 0.0, "orthogonal_to_active": 0.01658611372113228, "output_influence": 0.27959510684013367, "residual_influence": 0.1468857377767563, "state_active": 0.10376930236816406, "state_memory": 0.011950021609663963, "state_peak": 21.224105834960938, "growth_active": 1.3456125259399414, "growth_memory": 1.597622036933899, "heads": [{"head": 0, "memory_to_active": 0.13256418704986572, "contribution_to_active": 0.46258023381233215, "mix_mean": 0.3328273296356201, "mix_low": 0.008696556091308594, "mix_high": 0.0015459060668945312, "read_cosine": 0.7086928486824036, "beta_mean": 0.0679357647895813, "nonfinite": 0.0, "weighted_memory_to_active": 0.06661481410264969, "mix_std": 0.25116395950317383, "beta_std": 0.058700550347566605, "beta_low": 0.09063720703125, "beta_high": 0.0, "orthogonal_to_active": 0.012682669796049595, "evidence_weights": [-0.17937099933624268, -0.21507959067821503, 0.33458051085472107], "agreement_weight": 0.08027929067611694, "orthogonal_weight": -0.3449012041091919}, {"head": 1, "memory_to_active": 0.2529197335243225, "contribution_to_active": 0.3620920181274414, "mix_mean": 0.33362653851509094, "mix_low": 0.005166053771972656, "mix_high": 0.004080772399902344, "read_cosine": 0.8281153440475464, "beta_mean": 0.003046709345653653, "nonfinite": 0.0, "weighted_memory_to_active": 0.1031026765704155, "mix_std": 0.2521026134490967, "beta_std": 0.005722407251596451, "beta_low": 0.949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.010324929840862751, "evidence_weights": [-0.19774559140205383, -0.20685695111751556, 0.2530371844768524], "agreement_weight": 0.09272841364145279, "orthogonal_weight": -0.17818069458007812}, {"head": 2, "memory_to_active": 0.01568867266178131, "contribution_to_active": 0.4181939661502838, "mix_mean": 0.3402395248413086, "mix_low": 0.009222030639648438, "mix_high": 0.0001316070556640625, "read_cosine": 0.7265942692756653, "beta_mean": 0.0003286516584921628, "nonfinite": 0.0, "weighted_memory_to_active": 0.0060514844954013824, "mix_std": 0.2557026445865631, "beta_std": 0.0006355341756716371, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.0022331506479531527, "evidence_weights": [-0.21844060719013214, -0.2545941174030304, 0.30979618430137634], "agreement_weight": 0.04109155759215355, "orthogonal_weight": -0.5081976056098938}, {"head": 3, "memory_to_active": 0.19999493658542633, "contribution_to_active": 0.41432419419288635, "mix_mean": 0.25814011693000793, "mix_low": 0.0057239532470703125, "mix_high": 2.6702880859375e-05, "read_cosine": 0.7180996537208557, "beta_mean": 0.012862885370850563, "nonfinite": 0.0, "weighted_memory_to_active": 0.09174754470586777, "mix_std": 0.2085396945476532, "beta_std": 0.017708566039800644, "beta_low": 0.6134033203125, "beta_high": 0.0, "orthogonal_to_active": 0.016935791820287704, "evidence_weights": [-0.1478021740913391, -0.20172534883022308, 0.3351181745529175], "agreement_weight": 0.0607261061668396, "orthogonal_weight": -0.34199389815330505}, {"head": 4, "memory_to_active": 0.19604089856147766, "contribution_to_active": 0.5003631711006165, "mix_mean": 0.3859294056892395, "mix_low": 0.0077686309814453125, "mix_high": 0.00011920928955078125, "read_cosine": 0.09244859218597412, "beta_mean": 0.2145005762577057, "nonfinite": 0.0, "weighted_memory_to_active": 0.10618571937084198, "mix_std": 0.2848254442214966, "beta_std": 0.23944154381752014, "beta_low": 0.14715576171875, "beta_high": 0.0, "orthogonal_to_active": 0.056672245264053345, "evidence_weights": [-0.2733888030052185, -0.2527659833431244, 0.4323538541793823], "agreement_weight": 0.0031997777987271547, "orthogonal_weight": -0.594900369644165}, {"head": 5, "memory_to_active": 0.003951488062739372, "contribution_to_active": 0.44747036695480347, "mix_mean": 0.47620323300361633, "mix_low": 0.004538536071777344, "mix_high": 0.00466156005859375, "read_cosine": 0.5275738835334778, "beta_mean": 7.328467472689226e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0015483287861570716, "mix_std": 0.27606460452079773, "beta_std": 0.0001581309043103829, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006678883219137788, "evidence_weights": [-0.34489938616752625, -0.2816706895828247, 0.5114433765411377], "agreement_weight": 0.09686165302991867, "orthogonal_weight": 0.46732568740844727}]}]}, {"step": 9000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8090483010018943, "memory_to_active": 1.091460108757019, "contribution_to_active": 0.3884166479110718, "mix_mean": 0.23684866726398468, "mix_low": 0.19903644919395447, "mix_high": 0.003731012577190995, "read_cosine": 0.4013490676879883, "beta_mean": 0.1066780537366867, "nonfinite": 0.0, "weighted_memory_to_active": 0.20379751920700073, "mix_std": 0.18672792613506317, "beta_std": 0.07946412265300751, "beta_low": 0.3876476585865021, "beta_high": 5.086263172415784e-06, "orthogonal_to_active": 0.033202216029167175, "samples": 4, "scored_tokens": 899, "normal_loss": 3.25142765045166, "normal_token_accuracy": 0.4282536208629608, "state_active": 0.06817357242107391, "state_memory": 0.031017780303955078, "state_peak": 65.4473648071289, "growth_active": 1.120896339416504, "growth_memory": 1.4545842409133911, "output_influence": 0.26837095618247986, "residual_influence": 0.08175565302371979, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.3127236366271973, "no_memory_loss_delta": 0.10126302391290665, "scrambled_loss_delta": 0.2674194574356079, "memory_only_loss_delta": 1.8749648332595825, "no_evidence_loss_delta": 0.0326329730451107, "no_agreement_loss_delta": -0.0007610784377902746, "no_orthogonal_loss_delta": -0.00013627030421048403}, {"index": 94, "tokens": 256, "normal_loss": 3.390352725982666, "no_memory_loss_delta": 0.22566761076450348, "scrambled_loss_delta": 0.40103045105934143, "memory_only_loss_delta": 1.8542996644973755, "no_evidence_loss_delta": 0.09316915273666382, "no_agreement_loss_delta": 0.019468415528535843, "no_orthogonal_loss_delta": 0.01363234780728817}, {"index": 188, "tokens": 256, "normal_loss": 3.6505534648895264, "no_memory_loss_delta": 0.1251462697982788, "scrambled_loss_delta": 0.12740541994571686, "memory_only_loss_delta": 1.881847620010376, "no_evidence_loss_delta": 0.0704917460680008, "no_agreement_loss_delta": 0.001937046181410551, "no_orthogonal_loss_delta": -0.007130655460059643}, {"index": 283, "tokens": 131, "normal_loss": 2.0801849365234375, "no_memory_loss_delta": 0.19952832162380219, "scrambled_loss_delta": 0.3162108361721039, "memory_only_loss_delta": 2.5701863765716553, "no_evidence_loss_delta": 0.03963978588581085, "no_agreement_loss_delta": 0.006855667568743229, "no_orthogonal_loss_delta": -0.011732065118849277}], "agreement_weight_abs_mean": 0.06222148239612579, "agreement_weight_abs_max": 0.6639424562454224, "agreement_weight_mean": 0.042109619826078415, "orthogonal_weight_abs_mean": 0.34127628803253174, "orthogonal_weight_abs_max": 1.5253701210021973, "orthogonal_weight_mean": -0.141631081700325, "no_memory_loss": 3.409236192703247, "no_memory_loss_delta": 0.1578085571527481, "no_memory_token_accuracy_gain": 0.007786429487168789, "no_memory_token_win_fraction": 0.5917686223983765, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 14.598673895018171, "scrambled_loss": 3.5241334438323975, "scrambled_loss_delta": 0.2727058529853821, "scrambled_token_accuracy_gain": 0.032258063554763794, "scrambled_token_win_fraction": 0.6173526048660278, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 23.868330627069636, "memory_only_loss": 5.22377347946167, "memory_only_loss_delta": 1.9723460674285889, "memory_only_token_accuracy_gain": 0.20244716107845306, "memory_only_token_win_fraction": 0.86429363489151, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 86.0869935485437, "no_evidence_loss": 3.3131003379821777, "no_evidence_loss_delta": 0.06167301535606384, "no_evidence_token_accuracy_gain": -0.0011123470030725002, "no_evidence_token_win_fraction": 0.5428253412246704, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 5.9809735686705885, "no_agreement_loss": 3.258305072784424, "no_agreement_loss_delta": 0.006877702195197344, "no_agreement_token_accuracy_gain": -0.0055617354810237885, "no_agreement_token_win_fraction": 0.46718576550483704, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 0.6854104930765347, "no_orthogonal_loss": 3.251530647277832, "no_orthogonal_loss_delta": 0.00010305543401045725, "no_orthogonal_token_accuracy_gain": 0.0033370412420481443, "no_orthogonal_token_win_fraction": 0.510567307472229, "no_orthogonal_probe_win_fraction": 0.25, "no_orthogonal_ppl_reduction_pct": 0.010305012398162837, "short_no_memory_loss_delta": 0.13177794218063354, "long_context_memory_gain": 0.026030614972114563, "normal_context_loss_gain": 0.12506315112113953, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.5296638011932373, "contribution_to_active": 0.7074015736579895, "mix_mean": 0.09266416728496552, "mix_low": 0.3759691119194031, "mix_high": 0.0019307136535644531, "read_cosine": 0.5095013380050659, "beta_mean": 0.2688705623149872, "nonfinite": 0.0, "weighted_memory_to_active": 0.6993988752365112, "mix_std": 0.08392337709665298, "beta_std": 0.1285952925682068, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.030312802642583847, "output_influence": 0.14281132817268372, "residual_influence": 0.21495592594146729, "state_active": 0.10413762181997299, "state_memory": 0.057338375598192215, "state_peak": 50.69615936279297, "growth_active": 1.2884994745254517, "growth_memory": 1.3514418601989746, "heads": [{"head": 0, "memory_to_active": 2.4206490516662598, "contribution_to_active": 0.47948676347732544, "mix_mean": 0.020937427878379822, "mix_low": 0.4489269256591797, "mix_high": 0.0, "read_cosine": 0.3618823289871216, "beta_mean": 0.24417182803153992, "nonfinite": 0.0, "weighted_memory_to_active": 0.5443217754364014, "mix_std": 0.04957962781190872, "beta_std": 0.09408396482467651, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006720639765262604, "evidence_weights": [-0.044936154037714005, -0.0644964724779129, 0.061831362545490265], "agreement_weight": -0.05238860100507736, "orthogonal_weight": -0.014832878485321999}, {"head": 1, "memory_to_active": 2.842944622039795, "contribution_to_active": 1.2329051494598389, "mix_mean": 0.0229116752743721, "mix_low": 0.3763713836669922, "mix_high": 0.0, "read_cosine": 0.4848822355270386, "beta_mean": 0.3541613817214966, "nonfinite": 0.0, "weighted_memory_to_active": 1.4053215980529785, "mix_std": 0.0532982312142849, "beta_std": 0.13874413073062897, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.004230750724673271, "evidence_weights": [-0.0020370713900774717, -0.04911104589700699, 0.06579103320837021], "agreement_weight": -0.005073181353509426, "orthogonal_weight": 0.00613939855247736}, {"head": 2, "memory_to_active": 2.092841863632202, "contribution_to_active": 0.41749000549316406, "mix_mean": 0.01740843430161476, "mix_low": 0.5513191223144531, "mix_high": 0.0, "read_cosine": 0.4116222858428955, "beta_mean": 0.30480852723121643, "nonfinite": 0.0, "weighted_memory_to_active": 0.4541398882865906, "mix_std": 0.03643485903739929, "beta_std": 0.1799379587173462, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.019280094653367996, "evidence_weights": [0.008642975240945816, 0.01470467820763588, -0.06359146535396576], "agreement_weight": -0.01095502357929945, "orthogonal_weight": -0.0684208795428276}, {"head": 3, "memory_to_active": 3.9841179847717285, "contribution_to_active": 1.658155083656311, "mix_mean": 0.06506375968456268, "mix_low": 0.11431217193603516, "mix_high": 0.0, "read_cosine": 0.49818432331085205, "beta_mean": 0.32859212160110474, "nonfinite": 0.0, "weighted_memory_to_active": 1.6239169836044312, "mix_std": 0.09036490321159363, "beta_std": 0.15537691116333008, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.13053658604621887, "evidence_weights": [-0.1307811290025711, -0.15552611649036407, 0.13218529522418976], "agreement_weight": -0.03477148339152336, "orthogonal_weight": 0.10684902966022491}, {"head": 4, "memory_to_active": 3.6774260997772217, "contribution_to_active": 0.11553995311260223, "mix_mean": 0.008436057716608047, "mix_low": 0.7541561126708984, "mix_high": 0.0, "read_cosine": 0.33910179138183594, "beta_mean": 0.29604393243789673, "nonfinite": 0.0, "weighted_memory_to_active": 0.09917695820331573, "mix_std": 0.01307207066565752, "beta_std": 0.15271097421646118, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01994994655251503, "evidence_weights": [0.05554773285984993, 0.01924467273056507, -0.07631699740886688], "agreement_weight": -0.03554327413439751, "orthogonal_weight": 0.2315668761730194}, {"head": 5, "memory_to_active": 0.1600024700164795, "contribution_to_active": 0.3408324718475342, "mix_mean": 0.4212276339530945, "mix_low": 0.010728836059570312, "mix_high": 0.011584281921386719, "read_cosine": 0.9613348841667175, "beta_mean": 0.08544553071260452, "nonfinite": 0.0, "weighted_memory_to_active": 0.0695156678557396, "mix_std": 0.26079052686691284, "beta_std": 0.0507177971303463, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011587763438001275, "evidence_weights": [-0.3186676502227783, -0.31995171308517456, 0.40981680154800415], "agreement_weight": 0.1356627196073532, "orthogonal_weight": -0.04755247011780739}]}, {"layer": 1, "memory_to_active": 2.44288969039917, "contribution_to_active": 0.29951345920562744, "mix_mean": 0.060562457889318466, "mix_low": 0.43805474042892456, "mix_high": 0.00021568934607785195, "read_cosine": 0.3425483703613281, "beta_mean": 0.24827811121940613, "nonfinite": 0.0, "weighted_memory_to_active": 0.29044386744499207, "mix_std": 0.08250071853399277, "beta_std": 0.13526377081871033, "beta_low": 0.155914306640625, "beta_high": 4.069010537932627e-05, "orthogonal_to_active": 0.02990494668483734, "output_influence": 0.12431793659925461, "residual_influence": 0.027518218383193016, "state_active": 0.051150720566511154, "state_memory": 0.06139108166098595, "state_peak": 24.774343490600586, "growth_active": 1.1133217811584473, "growth_memory": 1.541576862335205, "heads": [{"head": 0, "memory_to_active": 0.399303674697876, "contribution_to_active": 0.13567319512367249, "mix_mean": 0.056401629000902176, "mix_low": 0.35236072540283203, "mix_high": 0.0, "read_cosine": -0.023312950506806374, "beta_mean": 0.003464705543592572, "nonfinite": 0.0, "weighted_memory_to_active": 0.061190925538539886, "mix_std": 0.11052680760622025, "beta_std": 0.005123582668602467, "beta_low": 0.91546630859375, "beta_high": 0.0, "orthogonal_to_active": 0.022792832925915718, "evidence_weights": [-0.0013767092023044825, -0.04099879786372185, -0.03521735593676567], "agreement_weight": -0.07590855658054352, "orthogonal_weight": -0.40044280886650085}, {"head": 1, "memory_to_active": 3.5135719776153564, "contribution_to_active": 0.18532738089561462, "mix_mean": 0.028873544186353683, "mix_low": 0.36680030822753906, "mix_high": 7.62939453125e-06, "read_cosine": 0.4677947461605072, "beta_mean": 0.3901810348033905, "nonfinite": 0.0, "weighted_memory_to_active": 0.19655972719192505, "mix_std": 0.05377494543790817, "beta_std": 0.20624184608459473, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.003867597784847021, "evidence_weights": [-0.03917199745774269, -0.029212187975645065, 0.12049858272075653], "agreement_weight": -0.018180521205067635, "orthogonal_weight": 0.024956809356808662}, {"head": 2, "memory_to_active": 0.8895902037620544, "contribution_to_active": 0.37307536602020264, "mix_mean": 0.22111201286315918, "mix_low": 0.018497467041015625, "mix_high": 0.0012083053588867188, "read_cosine": 0.7667970657348633, "beta_mean": 0.16461388766765594, "nonfinite": 0.0, "weighted_memory_to_active": 0.4787333905696869, "mix_std": 0.20807036757469177, "beta_std": 0.07386961579322815, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03701925650238991, "evidence_weights": [-0.21269196271896362, -0.32140788435935974, 0.2241591215133667], "agreement_weight": -0.0233724694699049, "orthogonal_weight": 0.13652892410755157}, {"head": 3, "memory_to_active": 3.283979654312134, "contribution_to_active": 0.05704234540462494, "mix_mean": 0.007615481968969107, "mix_low": 0.8042097091674805, "mix_high": 0.0, "read_cosine": 0.20646293461322784, "beta_mean": 0.5120685696601868, "nonfinite": 0.0, "weighted_memory_to_active": 0.061604224145412445, "mix_std": 0.016211101785302162, "beta_std": 0.26165518164634705, "beta_low": 6.103515625e-05, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.005859024822711945, "evidence_weights": [0.03001702018082142, 0.02717095986008644, -0.012807056307792664], "agreement_weight": -0.016612764447927475, "orthogonal_weight": -0.09807373583316803}, {"head": 4, "memory_to_active": 2.757678747177124, "contribution_to_active": 0.988632082939148, "mix_mean": 0.041634880006313324, "mix_low": 0.27065467834472656, "mix_high": 8.58306884765625e-06, "read_cosine": 0.41681423783302307, "beta_mean": 0.18702295422554016, "nonfinite": 0.0, "weighted_memory_to_active": 0.8887171745300293, "mix_std": 0.07708653062582016, "beta_std": 0.08215431123971939, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10864675045013428, "evidence_weights": [-0.036157168447971344, -0.03440285101532936, 0.1954091489315033], "agreement_weight": -0.0458148717880249, "orthogonal_weight": 0.12783870100975037}, {"head": 5, "memory_to_active": 3.813213348388672, "contribution_to_active": 0.05733034387230873, "mix_mean": 0.007737214211374521, "mix_low": 0.8158054351806641, "mix_high": 6.961822509765625e-05, "read_cosine": 0.22073425352573395, "beta_mean": 0.23231744766235352, "nonfinite": 0.0, "weighted_memory_to_active": 0.05585767701268196, "mix_std": 0.029334545135498047, "beta_std": 0.18253804743289948, "beta_low": 0.01995849609375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0012442052830010653, "evidence_weights": [0.08825583755970001, 0.05386991426348686, 0.19914835691452026], "agreement_weight": -0.09455401450395584, "orthogonal_weight": 0.02301843836903572}]}, {"layer": 2, "memory_to_active": 1.289151906967163, "contribution_to_active": 0.24465104937553406, "mix_mean": 0.16371487081050873, "mix_low": 0.3201216161251068, "mix_high": 0.0009603500366210938, "read_cosine": 0.37465205788612366, "beta_mean": 0.10439029335975647, "nonfinite": 0.0, "weighted_memory_to_active": 0.0854175016283989, "mix_std": 0.14766591787338257, "beta_std": 0.07447851449251175, "beta_low": 0.5407613515853882, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.00798703357577324, "output_influence": 0.1980857253074646, "residual_influence": 0.039068762212991714, "state_active": 0.04015161842107773, "state_memory": 0.031161781400442123, "state_peak": 46.0794792175293, "growth_active": 1.01194429397583, "growth_memory": 1.5784211158752441, "heads": [{"head": 0, "memory_to_active": 0.20868875086307526, "contribution_to_active": 0.047863710671663284, "mix_mean": 0.06146474555134773, "mix_low": 0.2963733673095703, "mix_high": 0.0, "read_cosine": 0.27680546045303345, "beta_mean": 0.0019671821501106024, "nonfinite": 0.0, "weighted_memory_to_active": 0.0157668124884367, "mix_std": 0.1069750040769577, "beta_std": 0.0027389349415898323, "beta_low": 0.9832763671875, "beta_high": 0.0, "orthogonal_to_active": 0.0017162672011181712, "evidence_weights": [-0.009858286008238792, -0.01242863293737173, 0.05200191214680672], "agreement_weight": -0.05466149374842644, "orthogonal_weight": -0.11752509325742722}, {"head": 1, "memory_to_active": 3.616830825805664, "contribution_to_active": 0.04038526862859726, "mix_mean": 0.005214127711951733, "mix_low": 0.8932123184204102, "mix_high": 0.0, "read_cosine": 0.19168326258659363, "beta_mean": 0.3206784725189209, "nonfinite": 0.0, "weighted_memory_to_active": 0.03557075560092926, "mix_std": 0.010058608837425709, "beta_std": 0.14255225658416748, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0050790742971003056, "evidence_weights": [0.09472748637199402, 0.10066413879394531, 0.059144191443920135], "agreement_weight": 0.00765590462833643, "orthogonal_weight": 0.15296252071857452}, {"head": 2, "memory_to_active": 0.11860617995262146, "contribution_to_active": 0.3563789129257202, "mix_mean": 0.3105652928352356, "mix_low": 0.007954597473144531, "mix_high": 0.0007038116455078125, "read_cosine": 0.41269683837890625, "beta_mean": 0.00023327316739596426, "nonfinite": 0.0, "weighted_memory_to_active": 0.04224574565887451, "mix_std": 0.25796568393707275, "beta_std": 0.0005198416183702648, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.005106198601424694, "evidence_weights": [-0.1640206128358841, -0.17759281396865845, 0.2860216796398163], "agreement_weight": 0.03103690966963768, "orthogonal_weight": -0.15604452788829803}, {"head": 3, "memory_to_active": 0.3619339168071747, "contribution_to_active": 0.3696613013744354, "mix_mean": 0.15771141648292542, "mix_low": 0.032769203186035156, "mix_high": 0.0017404556274414062, "read_cosine": 0.6424953937530518, "beta_mean": 0.05513859912753105, "nonfinite": 0.0, "weighted_memory_to_active": 0.11182086169719696, "mix_std": 0.186514213681221, "beta_std": 0.08930904418230057, "beta_low": 0.23101806640625, "beta_high": 0.0, "orthogonal_to_active": 0.03211171180009842, "evidence_weights": [-0.06480672955513, -0.12413950264453888, 0.3892579972743988], "agreement_weight": 0.08692696690559387, "orthogonal_weight": -0.4487500786781311}, {"head": 4, "memory_to_active": 3.3958747386932373, "contribution_to_active": 0.21732789278030396, "mix_mean": 0.01291811652481556, "mix_low": 0.6881504058837891, "mix_high": 3.62396240234375e-05, "read_cosine": 0.1800098866224289, "beta_mean": 0.24826276302337646, "nonfinite": 0.0, "weighted_memory_to_active": 0.2902070879936218, "mix_std": 0.04054774343967438, "beta_std": 0.21159236133098602, "beta_low": 0.030517578125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0031303968280553818, "evidence_weights": [0.03716815635561943, 0.03099750727415085, 0.14400748908519745], "agreement_weight": 0.025280268862843513, "orthogonal_weight": -0.017444996163249016}, {"head": 5, "memory_to_active": 0.03297632932662964, "contribution_to_active": 0.43628913164138794, "mix_mean": 0.43441545963287354, "mix_low": 0.002269744873046875, "mix_high": 0.0032815933227539062, "read_cosine": 0.5442217588424683, "beta_mean": 6.14274395047687e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.016893720254302025, "mix_std": 0.2839341461658478, "beta_std": 0.0001586204452905804, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000778552086558193, "evidence_weights": [-0.22931760549545288, -0.21528199315071106, 0.4365656077861786], "agreement_weight": 0.018880775198340416, "orthogonal_weight": 0.09056136012077332}]}, {"layer": 3, "memory_to_active": 1.6182336807250977, "contribution_to_active": 0.28726303577423096, "mix_mean": 0.11276276409626007, "mix_low": 0.3407057523727417, "mix_high": 0.0011479060631245375, "read_cosine": 0.30157914757728577, "beta_mean": 0.13603779673576355, "nonfinite": 0.0, "weighted_memory_to_active": 0.15253421664237976, "mix_std": 0.1483624130487442, "beta_std": 0.09570156782865524, "beta_low": 0.35467529296875, "beta_high": 0.0, "orthogonal_to_active": 0.02186502143740654, "output_influence": 0.2022615373134613, "residual_influence": 0.03332319110631943, "state_active": 0.03727380931377411, "state_memory": 0.03452785313129425, "state_peak": 32.79726791381836, "growth_active": 0.9956495761871338, "growth_memory": 1.6911324262619019, "heads": [{"head": 0, "memory_to_active": 1.1470906734466553, "contribution_to_active": 0.2030390501022339, "mix_mean": 0.07291653752326965, "mix_low": 0.39000511169433594, "mix_high": 1.430511474609375e-05, "read_cosine": 0.22470787167549133, "beta_mean": 0.021069832146167755, "nonfinite": 0.0, "weighted_memory_to_active": 0.16276273131370544, "mix_std": 0.14473089575767517, "beta_std": 0.014143413864076138, "beta_low": 0.1397705078125, "beta_high": 0.0, "orthogonal_to_active": 0.014203892089426517, "evidence_weights": [0.025793088600039482, -0.026250658556818962, -0.04735855385661125], "agreement_weight": -0.033467426896095276, "orthogonal_weight": -0.08774209022521973}, {"head": 1, "memory_to_active": 0.331548273563385, "contribution_to_active": 0.19673162698745728, "mix_mean": 0.12166714668273926, "mix_low": 0.18282794952392578, "mix_high": 3.719329833984375e-05, "read_cosine": 0.2820098400115967, "beta_mean": 0.0016979428473860025, "nonfinite": 0.0, "weighted_memory_to_active": 0.03395799174904823, "mix_std": 0.18424174189567566, "beta_std": 0.003596780588850379, "beta_low": 0.96844482421875, "beta_high": 0.0, "orthogonal_to_active": 0.002570763695985079, "evidence_weights": [-0.07505953311920166, -0.05846129730343819, 0.16451165080070496], "agreement_weight": -0.024221954867243767, "orthogonal_weight": -0.07464137673377991}, {"head": 2, "memory_to_active": 3.59002685546875, "contribution_to_active": 0.29356080293655396, "mix_mean": 0.03342444449663162, "mix_low": 0.45369720458984375, "mix_high": 2.574920654296875e-05, "read_cosine": 0.22365590929985046, "beta_mean": 0.2510615885257721, "nonfinite": 0.0, "weighted_memory_to_active": 0.38511043787002563, "mix_std": 0.08620933443307877, "beta_std": 0.1606311798095703, "beta_low": 0.0028076171875, "beta_high": 0.0, "orthogonal_to_active": 0.049775268882513046, "evidence_weights": [-0.028973694890737534, 0.004648992791771889, 0.11100883036851883], "agreement_weight": -0.019429704174399376, "orthogonal_weight": -0.15706679224967957}, {"head": 3, "memory_to_active": 0.03554421663284302, "contribution_to_active": 0.417663037776947, "mix_mean": 0.33126381039619446, "mix_low": 0.032494544982910156, "mix_high": 0.002471923828125, "read_cosine": 0.3086903989315033, "beta_mean": 9.649238199926913e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.016271453350782394, "mix_std": 0.27955904603004456, "beta_std": 0.00024910009233281016, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0027486346662044525, "evidence_weights": [-0.1686655431985855, -0.18312175571918488, 0.5499415397644043], "agreement_weight": 0.2997484505176544, "orthogonal_weight": -0.23977096378803253}, {"head": 4, "memory_to_active": 4.043840408325195, "contribution_to_active": 0.04502502828836441, "mix_mean": 0.006935521960258484, "mix_low": 0.8387842178344727, "mix_high": 0.0, "read_cosine": 0.21656545996665955, "beta_mean": 0.34996140003204346, "nonfinite": 0.0, "weighted_memory_to_active": 0.054616134613752365, "mix_std": 0.015410670079290867, "beta_std": 0.21442195773124695, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.010288883931934834, "evidence_weights": [0.13542701303958893, 0.11981762200593948, -0.1683236062526703], "agreement_weight": -0.09610819816589355, "orthogonal_weight": -0.19608615338802338}, {"head": 5, "memory_to_active": 0.5613515377044678, "contribution_to_active": 0.5675585865974426, "mix_mean": 0.11036911606788635, "mix_low": 0.1464252471923828, "mix_high": 0.004338264465332031, "read_cosine": 0.5538454055786133, "beta_mean": 0.1923394650220871, "nonfinite": 0.0, "weighted_memory_to_active": 0.2624864876270294, "mix_std": 0.180022731423378, "beta_std": 0.18116700649261475, "beta_low": 0.016845703125, "beta_high": 0.0, "orthogonal_to_active": 0.051602669060230255, "evidence_weights": [-0.03181122988462448, -0.11943153291940689, 0.022079266607761383], "agreement_weight": 0.14929506182670593, "orthogonal_weight": -0.28387126326560974}]}, {"layer": 4, "memory_to_active": 1.4333539009094238, "contribution_to_active": 0.2687860429286957, "mix_mean": 0.20279455184936523, "mix_low": 0.28423166275024414, "mix_high": 0.0010522207012400031, "read_cosine": 0.4142889380455017, "beta_mean": 0.1606076955795288, "nonfinite": 0.0, "weighted_memory_to_active": 0.14289434254169464, "mix_std": 0.13902989029884338, "beta_std": 0.11376235634088516, "beta_low": 0.2148844450712204, "beta_high": 0.0, "orthogonal_to_active": 0.028139103204011917, "output_influence": 0.1774141937494278, "residual_influence": 0.035184573382139206, "state_active": 0.04902321472764015, "state_memory": 0.042418356984853745, "state_peak": 43.82090377807617, "growth_active": 1.1064646244049072, "growth_memory": 1.6071029901504517, "heads": [{"head": 0, "memory_to_active": 0.43961164355278015, "contribution_to_active": 0.4466487467288971, "mix_mean": 0.33330297470092773, "mix_low": 0.010390281677246094, "mix_high": 6.198883056640625e-05, "read_cosine": 0.5926655530929565, "beta_mean": 0.08270750939846039, "nonfinite": 0.0, "weighted_memory_to_active": 0.2218799889087677, "mix_std": 0.24655207991600037, "beta_std": 0.1102956235408783, "beta_low": 0.146484375, "beta_high": 0.0, "orthogonal_to_active": 0.008150058798491955, "evidence_weights": [-0.17191021144390106, -0.21400360763072968, 0.3581817150115967], "agreement_weight": 0.07215891033411026, "orthogonal_weight": -0.048706118017435074}, {"head": 1, "memory_to_active": 0.6145987510681152, "contribution_to_active": 0.2620832920074463, "mix_mean": 0.26680874824523926, "mix_low": 0.013341903686523438, "mix_high": 3.719329833984375e-05, "read_cosine": 0.5356723070144653, "beta_mean": 0.05245822295546532, "nonfinite": 0.0, "weighted_memory_to_active": 0.2172480970621109, "mix_std": 0.23211637139320374, "beta_std": 0.057222411036491394, "beta_low": 0.08892822265625, "beta_high": 0.0, "orthogonal_to_active": 0.08188000321388245, "evidence_weights": [-0.15865115821361542, -0.15209393203258514, 0.5015231966972351], "agreement_weight": 0.05606828257441521, "orthogonal_weight": -0.45831653475761414}, {"head": 2, "memory_to_active": 2.9528512954711914, "contribution_to_active": 0.08962938189506531, "mix_mean": 0.010515015572309494, "mix_low": 0.6995782852172852, "mix_high": 0.0, "read_cosine": 0.30854272842407227, "beta_mean": 0.2538081705570221, "nonfinite": 0.0, "weighted_memory_to_active": 0.10750860720872879, "mix_std": 0.01637740433216095, "beta_std": 0.1599939614534378, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010883904760703444, "evidence_weights": [0.03596164286136627, 0.02124643325805664, -0.055269014090299606], "agreement_weight": 0.03160715103149414, "orthogonal_weight": 0.013832134194672108}, {"head": 3, "memory_to_active": 2.0242092609405518, "contribution_to_active": 0.06618718057870865, "mix_mean": 0.022459477186203003, "mix_low": 0.4343719482421875, "mix_high": 0.0, "read_cosine": 0.2258354127407074, "beta_mean": 0.20939646661281586, "nonfinite": 0.0, "weighted_memory_to_active": 0.09691636264324188, "mix_std": 0.03572603315114975, "beta_std": 0.1778402030467987, "beta_low": 0.053955078125, "beta_high": 0.0, "orthogonal_to_active": 0.016361353918910027, "evidence_weights": [-0.011312163434922695, 0.02833770215511322, -0.104497991502285], "agreement_weight": 0.016303952783346176, "orthogonal_weight": -0.22861821949481964}, {"head": 4, "memory_to_active": 0.04847987741231918, "contribution_to_active": 0.5321446061134338, "mix_mean": 0.5665059089660645, "mix_low": 0.001251220703125, "mix_high": 0.006198883056640625, "read_cosine": 0.36994749307632446, "beta_mean": 0.00010940936044789851, "nonfinite": 0.0, "weighted_memory_to_active": 0.030260302126407623, "mix_std": 0.26629719138145447, "beta_std": 0.0002870319876819849, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.002436789683997631, "evidence_weights": [-0.2622417211532593, -0.2868482768535614, 0.5694290995597839], "agreement_weight": 0.0023334426805377007, "orthogonal_weight": -0.10094523429870605}, {"head": 5, "memory_to_active": 2.5203723907470703, "contribution_to_active": 0.2160230427980423, "mix_mean": 0.01717511937022209, "mix_low": 0.5464563369750977, "mix_high": 1.52587890625e-05, "read_cosine": 0.45307010412216187, "beta_mean": 0.3651663064956665, "nonfinite": 0.0, "weighted_memory_to_active": 0.18355262279510498, "mix_std": 0.03711028769612312, "beta_std": 0.1769348829984665, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05891801416873932, "evidence_weights": [0.01754404976963997, -0.027719054371118546, -0.02271297201514244], "agreement_weight": -0.0447583831846714, "orthogonal_weight": 0.4812786281108856}]}, {"layer": 5, "memory_to_active": 0.7812092304229736, "contribution_to_active": 0.4028934836387634, "mix_mean": 0.33855071663856506, "mix_low": 0.12861332297325134, "mix_high": 0.007839679718017578, "read_cosine": 0.25895220041275024, "beta_mean": 0.08544199913740158, "nonfinite": 0.0, "weighted_memory_to_active": 0.16859707236289978, "mix_std": 0.23515591025352478, "beta_std": 0.0917050689458847, "beta_low": 0.4819132685661316, "beta_high": 0.0, "orthogonal_to_active": 0.037761151790618896, "output_influence": 0.36853256821632385, "residual_influence": 0.07381918281316757, "state_active": 0.06863827258348465, "state_memory": 0.029527438804507256, "state_peak": 55.1719856262207, "growth_active": 1.147549033164978, "growth_memory": 1.5075777769088745, "heads": [{"head": 0, "memory_to_active": 1.6392499208450317, "contribution_to_active": 0.23077528178691864, "mix_mean": 0.06762580573558807, "mix_low": 0.23058032989501953, "mix_high": 0.0, "read_cosine": -0.14815692603588104, "beta_mean": 0.1506066769361496, "nonfinite": 0.0, "weighted_memory_to_active": 0.21550175547599792, "mix_std": 0.10993686318397522, "beta_std": 0.15805155038833618, "beta_low": 0.0059814453125, "beta_high": 0.0, "orthogonal_to_active": 0.04117932543158531, "evidence_weights": [-0.021359169855713844, -0.04454842582345009, -0.2595342993736267], "agreement_weight": -0.031097661703824997, "orthogonal_weight": -0.19592979550361633}, {"head": 1, "memory_to_active": 0.049800362437963486, "contribution_to_active": 0.403097540140152, "mix_mean": 0.40751078724861145, "mix_low": 0.021790504455566406, "mix_high": 0.006937980651855469, "read_cosine": 0.14174041152000427, "beta_mean": 0.0001272039080504328, "nonfinite": 0.0, "weighted_memory_to_active": 0.01910371519625187, "mix_std": 0.3170523941516876, "beta_std": 0.0002763255615718663, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013357063755393028, "evidence_weights": [-0.18947307765483856, -0.18222571909427643, 0.32732170820236206], "agreement_weight": 0.18407601118087769, "orthogonal_weight": -0.7313302159309387}, {"head": 2, "memory_to_active": 0.2380034178495407, "contribution_to_active": 0.4615669250488281, "mix_mean": 0.501123309135437, "mix_low": 0.0021343231201171875, "mix_high": 0.007781982421875, "read_cosine": 0.5654884576797485, "beta_mean": 0.012220151722431183, "nonfinite": 0.0, "weighted_memory_to_active": 0.15230713784694672, "mix_std": 0.2786339223384857, "beta_std": 0.011807962320744991, "beta_low": 0.5849609375, "beta_high": 0.0, "orthogonal_to_active": 0.017998047173023224, "evidence_weights": [-0.26975828409194946, -0.26887258887290955, 0.49128544330596924], "agreement_weight": 0.049887411296367645, "orthogonal_weight": -0.16693271696567535}, {"head": 3, "memory_to_active": 2.135737180709839, "contribution_to_active": 0.12784436345100403, "mix_mean": 0.04265955835580826, "mix_low": 0.39534950256347656, "mix_high": 9.5367431640625e-07, "read_cosine": 0.18421293795108795, "beta_mean": 0.10582467168569565, "nonfinite": 0.0, "weighted_memory_to_active": 0.209661602973938, "mix_std": 0.08671821653842926, "beta_std": 0.12499978393316269, "beta_low": 0.13067626953125, "beta_high": 0.0, "orthogonal_to_active": 0.06622838973999023, "evidence_weights": [0.011900385841727257, -0.024708392098546028, 0.11096639931201935], "agreement_weight": 0.08167144656181335, "orthogonal_weight": -0.39458146691322327}, {"head": 4, "memory_to_active": 0.5891095399856567, "contribution_to_active": 0.5991253852844238, "mix_mean": 0.3449241518974304, "mix_low": 0.11736106872558594, "mix_high": 0.016357421875, "read_cosine": 0.5848554372787476, "beta_mean": 0.24375028908252716, "nonfinite": 0.0, "weighted_memory_to_active": 0.39342010021209717, "mix_std": 0.34196001291275024, "beta_std": 0.254823237657547, "beta_low": 0.16986083984375, "beta_high": 0.0, "orthogonal_to_active": 0.08154696971178055, "evidence_weights": [-0.16676320135593414, -0.2531965970993042, 0.1768789142370224], "agreement_weight": 0.04743918403983116, "orthogonal_weight": 0.2956768274307251}, {"head": 5, "memory_to_active": 0.03535465896129608, "contribution_to_active": 0.5949512124061584, "mix_mean": 0.6674606204032898, "mix_low": 0.004464149475097656, "mix_high": 0.015959739685058594, "read_cosine": 0.22557282447814941, "beta_mean": 0.0001229677436640486, "nonfinite": 0.0, "weighted_memory_to_active": 0.021588105708360672, "mix_std": 0.27663400769233704, "beta_std": 0.0002715258742682636, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006257100496441126, "evidence_weights": [-0.29138585925102234, -0.28384914994239807, 0.6210885047912598], "agreement_weight": 0.24174652993679047, "orthogonal_weight": -0.33179011940956116}]}, {"layer": 6, "memory_to_active": 0.5727242827415466, "contribution_to_active": 0.5052276849746704, "mix_mean": 0.3972455859184265, "mix_low": 0.10886923968791962, "mix_high": 0.013972759246826172, "read_cosine": 0.3350526988506317, "beta_mean": 0.08690601587295532, "nonfinite": 0.0, "weighted_memory_to_active": 0.19232341647148132, "mix_std": 0.2626601457595825, "beta_std": 0.09920674562454224, "beta_low": 0.4604695737361908, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04093772917985916, "output_influence": 0.3793279528617859, "residual_influence": 0.09856774657964706, "state_active": 0.07146351039409637, "state_memory": 0.017994266003370285, "state_peak": 43.50421905517578, "growth_active": 1.2221044301986694, "growth_memory": 1.3075792789459229, "heads": [{"head": 0, "memory_to_active": 0.3092173933982849, "contribution_to_active": 0.44079482555389404, "mix_mean": 0.40987348556518555, "mix_low": 0.007090568542480469, "mix_high": 0.0005006790161132812, "read_cosine": 0.3419000208377838, "beta_mean": 0.02550537884235382, "nonfinite": 0.0, "weighted_memory_to_active": 0.2247735857963562, "mix_std": 0.2750619351863861, "beta_std": 0.06683602929115295, "beta_low": 0.43902587890625, "beta_high": 0.0, "orthogonal_to_active": 0.11617088317871094, "evidence_weights": [-0.2044326514005661, -0.20530696213245392, 0.21504366397857666], "agreement_weight": 0.0999516025185585, "orthogonal_weight": -0.716914176940918}, {"head": 1, "memory_to_active": 0.013226918876171112, "contribution_to_active": 0.38632652163505554, "mix_mean": 0.34486034512519836, "mix_low": 0.008615493774414062, "mix_high": 0.007829666137695312, "read_cosine": 0.5454249978065491, "beta_mean": 2.5697103410493582e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004101535305380821, "mix_std": 0.2796376645565033, "beta_std": 6.12101357546635e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009294777410104871, "evidence_weights": [-0.20419985055923462, -0.1992594599723816, 0.28831204771995544], "agreement_weight": -0.016590844839811325, "orthogonal_weight": -0.2964257299900055}, {"head": 2, "memory_to_active": 2.125804901123047, "contribution_to_active": 0.07840438932180405, "mix_mean": 0.018136784434318542, "mix_low": 0.6122970581054688, "mix_high": 0.0, "read_cosine": 0.19212700426578522, "beta_mean": 0.1187601387500763, "nonfinite": 0.0, "weighted_memory_to_active": 0.09781412780284882, "mix_std": 0.04414071887731552, "beta_std": 0.09858620911836624, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.022389862686395645, "evidence_weights": [0.0817355364561081, 0.05847586318850517, -0.019452953711152077], "agreement_weight": 0.034457795321941376, "orthogonal_weight": -0.2414838969707489}, {"head": 3, "memory_to_active": 0.23514188826084137, "contribution_to_active": 0.6428881287574768, "mix_mean": 0.5617267489433289, "mix_low": 0.010931968688964844, "mix_high": 0.028606414794921875, "read_cosine": 0.5569869875907898, "beta_mean": 0.24310900270938873, "nonfinite": 0.0, "weighted_memory_to_active": 0.1629972606897354, "mix_std": 0.34939664602279663, "beta_std": 0.26380372047424316, "beta_low": 0.149169921875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.04287273436784744, "evidence_weights": [-0.14548057317733765, -0.2854003310203552, 0.24855639040470123], "agreement_weight": -0.014728124253451824, "orthogonal_weight": 0.3361053764820099}, {"head": 4, "memory_to_active": 0.6882691383361816, "contribution_to_active": 1.0103907585144043, "mix_mean": 0.6232984066009521, "mix_low": 0.005611419677734375, "mix_high": 0.04625701904296875, "read_cosine": 0.09931664168834686, "beta_mean": 0.1336420476436615, "nonfinite": 0.0, "weighted_memory_to_active": 0.6314212679862976, "mix_std": 0.33123794198036194, "beta_std": 0.16541679203510284, "beta_low": 0.17462158203125, "beta_high": 0.0, "orthogonal_to_active": 0.03199553117156029, "evidence_weights": [-0.22737833857536316, -0.45419543981552124, -0.05118748918175697], "agreement_weight": -0.015471859835088253, "orthogonal_weight": 0.06256815791130066}, {"head": 5, "memory_to_active": 0.0646854117512703, "contribution_to_active": 0.472561240196228, "mix_mean": 0.4255778193473816, "mix_low": 0.008668899536132812, "mix_high": 0.0006427764892578125, "read_cosine": 0.27456042170524597, "beta_mean": 0.00039384831325151026, "nonfinite": 0.0, "weighted_memory_to_active": 0.03283267095685005, "mix_std": 0.296485960483551, "beta_std": 0.0005364681128412485, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.03126787394285202, "evidence_weights": [-0.18296721577644348, -0.22124004364013672, 0.03924662619829178], "agreement_weight": 0.0706515684723854, "orthogonal_weight": -1.0163315534591675}]}, {"layer": 7, "memory_to_active": 2.382535696029663, "contribution_to_active": 0.16062991321086884, "mix_mean": 0.024425912648439407, "mix_low": 0.5799574851989746, "mix_high": 2.066294428004767e-06, "read_cosine": 0.21822971105575562, "beta_mean": 0.16435378789901733, "nonfinite": 0.0, "weighted_memory_to_active": 0.1605900079011917, "mix_std": 0.04645613953471184, "beta_std": 0.11202998459339142, "beta_low": 0.01173909567296505, "beta_high": 0.0, "orthogonal_to_active": 0.023132117465138435, "output_influence": 0.16998866200447083, "residual_influence": 0.03288809582591057, "state_active": 0.021783288568258286, "state_memory": 0.040173597633838654, "state_peak": 19.922061920166016, "growth_active": 0.9979227185249329, "growth_memory": 1.4828945398330688, "heads": [{"head": 0, "memory_to_active": 2.1784372329711914, "contribution_to_active": 0.29347437620162964, "mix_mean": 0.026555651798844337, "mix_low": 0.5248088836669922, "mix_high": 1.239776611328125e-05, "read_cosine": 0.3902512788772583, "beta_mean": 0.17042872309684753, "nonfinite": 0.0, "weighted_memory_to_active": 0.25838810205459595, "mix_std": 0.056797292083501816, "beta_std": 0.11883453279733658, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.05685608834028244, "evidence_weights": [-0.015092724934220314, -0.07028786838054657, 0.11118791252374649], "agreement_weight": -0.03846555948257446, "orthogonal_weight": 0.26828983426094055}, {"head": 1, "memory_to_active": 3.392956256866455, "contribution_to_active": 0.2594519853591919, "mix_mean": 0.044701166450977325, "mix_low": 0.33912086486816406, "mix_high": 0.0, "read_cosine": -0.023651301860809326, "beta_mean": 0.30545634031295776, "nonfinite": 0.0, "weighted_memory_to_active": 0.3094865083694458, "mix_std": 0.0674116387963295, "beta_std": 0.14259132742881775, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06031107157468796, "evidence_weights": [-0.08572504669427872, -0.07892389595508575, 0.03207787498831749], "agreement_weight": 0.0050670006312429905, "orthogonal_weight": -0.19049447774887085}, {"head": 2, "memory_to_active": 1.5025427341461182, "contribution_to_active": 0.23398065567016602, "mix_mean": 0.04411992430686951, "mix_low": 0.4238576889038086, "mix_high": 0.0, "read_cosine": 0.2268206775188446, "beta_mean": 0.14227095246315002, "nonfinite": 0.0, "weighted_memory_to_active": 0.21009984612464905, "mix_std": 0.09319444000720978, "beta_std": 0.08596602082252502, "beta_low": 0.00244140625, "beta_high": 0.0, "orthogonal_to_active": 0.0032460372895002365, "evidence_weights": [-0.04014875367283821, -0.09986036270856857, 0.17880666255950928], "agreement_weight": 0.026064617559313774, "orthogonal_weight": -0.016526097431778908}, {"head": 3, "memory_to_active": 2.6620993614196777, "contribution_to_active": 0.03454335033893585, "mix_mean": 0.009262235835194588, "mix_low": 0.72991943359375, "mix_high": 0.0, "read_cosine": 0.1179293543100357, "beta_mean": 0.10910052061080933, "nonfinite": 0.0, "weighted_memory_to_active": 0.04411439597606659, "mix_std": 0.015606893226504326, "beta_std": 0.09766943752765656, "beta_low": 0.01544189453125, "beta_high": 0.0, "orthogonal_to_active": 0.011390414088964462, "evidence_weights": [0.09242089837789536, 0.08478391915559769, 0.05757169425487518], "agreement_weight": 0.035158731043338776, "orthogonal_weight": -0.2704450190067291}, {"head": 4, "memory_to_active": 1.5633454322814941, "contribution_to_active": 0.058403532952070236, "mix_mean": 0.010696802288293839, "mix_low": 0.7292346954345703, "mix_high": 0.0, "read_cosine": 0.3126850426197052, "beta_mean": 0.12483524531126022, "nonfinite": 0.0, "weighted_memory_to_active": 0.04616253077983856, "mix_std": 0.024029549211263657, "beta_std": 0.11816680431365967, "beta_low": 0.040283203125, "beta_high": 0.0, "orthogonal_to_active": 0.0024355528876185417, "evidence_weights": [0.06174594163894653, 0.03885835409164429, 0.0016305341850966215], "agreement_weight": 0.05212603881955147, "orthogonal_weight": -0.06737737357616425}, {"head": 5, "memory_to_active": 2.9958317279815674, "contribution_to_active": 0.0839255303144455, "mix_mean": 0.011219678446650505, "mix_low": 0.7328033447265625, "mix_high": 0.0, "read_cosine": 0.2853432297706604, "beta_mean": 0.13403092324733734, "nonfinite": 0.0, "weighted_memory_to_active": 0.09528867155313492, "mix_std": 0.02169700898230076, "beta_std": 0.10895176231861115, "beta_low": 0.01214599609375, "beta_high": 0.0, "orthogonal_to_active": 0.004553527105599642, "evidence_weights": [0.050895512104034424, 0.07721513509750366, -0.17686240375041962], "agreement_weight": 0.0016268668696284294, "orthogonal_weight": -0.06000208482146263}]}, {"layer": 8, "memory_to_active": 0.9655055999755859, "contribution_to_active": 0.3218318223953247, "mix_mean": 0.21997025609016418, "mix_low": 0.10115163028240204, "mix_high": 0.0016121864318847656, "read_cosine": 0.5198097229003906, "beta_mean": 0.07028927654027939, "nonfinite": 0.0, "weighted_memory_to_active": 0.18200352787971497, "mix_std": 0.1730966866016388, "beta_std": 0.06360191851854324, "beta_low": 0.2114766538143158, "beta_high": 0.0, "orthogonal_to_active": 0.04251573234796524, "output_influence": 0.26528751850128174, "residual_influence": 0.058384645730257034, "state_active": 0.05897195264697075, "state_memory": 0.02794470265507698, "state_peak": 29.64914894104004, "growth_active": 0.9816485643386841, "growth_memory": 1.5665662288665771, "heads": [{"head": 0, "memory_to_active": 0.07650282979011536, "contribution_to_active": 0.4497568607330322, "mix_mean": 0.4637027084827423, "mix_low": 0.0001544952392578125, "mix_high": 7.724761962890625e-05, "read_cosine": 0.7236957550048828, "beta_mean": 0.004079268779605627, "nonfinite": 0.0, "weighted_memory_to_active": 0.03773270547389984, "mix_std": 0.2282441258430481, "beta_std": 0.006132349371910095, "beta_low": 0.90533447265625, "beta_high": 0.0, "orthogonal_to_active": 0.014650830067694187, "evidence_weights": [-0.2577763497829437, -0.28814148902893066, 0.5133444666862488], "agreement_weight": 0.06581899523735046, "orthogonal_weight": -0.6085165739059448}, {"head": 1, "memory_to_active": 0.7531904578208923, "contribution_to_active": 0.28573936223983765, "mix_mean": 0.253442645072937, "mix_low": 0.008645057678222656, "mix_high": 2.288818359375e-05, "read_cosine": 0.5814463496208191, "beta_mean": 0.07250205427408218, "nonfinite": 0.0, "weighted_memory_to_active": 0.23082777857780457, "mix_std": 0.18130141496658325, "beta_std": 0.054538507014513016, "beta_low": 0.0340576171875, "beta_high": 0.0, "orthogonal_to_active": 0.01768675446510315, "evidence_weights": [-0.1626981496810913, -0.16497395932674408, 0.29546213150024414], "agreement_weight": 0.0009285196429118514, "orthogonal_weight": 0.0974578708410263}, {"head": 2, "memory_to_active": 1.1967706680297852, "contribution_to_active": 0.12776055932044983, "mix_mean": 0.07091528922319412, "mix_low": 0.1816854476928711, "mix_high": 0.0, "read_cosine": 0.22435492277145386, "beta_mean": 0.06548741459846497, "nonfinite": 0.0, "weighted_memory_to_active": 0.19039709866046906, "mix_std": 0.10928304493427277, "beta_std": 0.06153193488717079, "beta_low": 0.04205322265625, "beta_high": 0.0, "orthogonal_to_active": 0.0729861930012703, "evidence_weights": [-0.02424461767077446, -0.09709300845861435, 0.08038751035928726], "agreement_weight": 0.08105325698852539, "orthogonal_weight": -0.4434391260147095}, {"head": 3, "memory_to_active": 0.6018224954605103, "contribution_to_active": 0.18097905814647675, "mix_mean": 0.15098944306373596, "mix_low": 0.032395362854003906, "mix_high": 0.0, "read_cosine": 0.6851304769515991, "beta_mean": 0.053581830114126205, "nonfinite": 0.0, "weighted_memory_to_active": 0.18664230406284332, "mix_std": 0.13579341769218445, "beta_std": 0.04723058268427849, "beta_low": 0.04827880859375, "beta_high": 0.0, "orthogonal_to_active": 0.029584083706140518, "evidence_weights": [-0.07767964154481888, -0.12863071262836456, 0.525998055934906], "agreement_weight": 0.05844992771744728, "orthogonal_weight": 0.30312085151672363}, {"head": 4, "memory_to_active": 2.9257731437683105, "contribution_to_active": 0.2859722375869751, "mix_mean": 0.030477287247776985, "mix_low": 0.3176116943359375, "mix_high": 0.0, "read_cosine": 0.34743309020996094, "beta_mean": 0.1438995599746704, "nonfinite": 0.0, "weighted_memory_to_active": 0.29127275943756104, "mix_std": 0.04360600560903549, "beta_std": 0.09830676019191742, "beta_low": 0.00164794921875, "beta_high": 0.0, "orthogonal_to_active": 0.010111097246408463, "evidence_weights": [0.005353471264243126, -0.019044842571020126, -0.025988807901740074], "agreement_weight": 0.005375571083277464, "orthogonal_weight": 0.039152421057224274}, {"head": 5, "memory_to_active": 0.2389742136001587, "contribution_to_active": 0.6007829308509827, "mix_mean": 0.3502941131591797, "mix_low": 0.06641769409179688, "mix_high": 0.009572982788085938, "read_cosine": 0.5567977428436279, "beta_mean": 0.0821855217218399, "nonfinite": 0.0, "weighted_memory_to_active": 0.15514850616455078, "mix_std": 0.3403520882129669, "beta_std": 0.11387137323617935, "beta_low": 0.23748779296875, "beta_high": 0.0, "orthogonal_to_active": 0.1100754365324974, "evidence_weights": [-0.09688093513250351, -0.33807137608528137, -0.2768596410751343], "agreement_weight": 0.025993963703513145, "orthogonal_weight": 0.8996007442474365}]}, {"layer": 9, "memory_to_active": 1.3724234104156494, "contribution_to_active": 0.5151330828666687, "mix_mean": 0.14516377449035645, "mix_low": 0.23574766516685486, "mix_high": 0.002189477439969778, "read_cosine": 0.35040152072906494, "beta_mean": 0.1100478544831276, "nonfinite": 0.0, "weighted_memory_to_active": 0.39293617010116577, "mix_std": 0.1882372796535492, "beta_std": 0.05340149253606796, "beta_low": 0.4305013418197632, "beta_high": 0.0, "orthogonal_to_active": 0.019211115315556526, "output_influence": 0.30215582251548767, "residual_influence": 0.06715689599514008, "state_active": 0.035905566066503525, "state_memory": 0.03336239606142044, "state_peak": 34.108421325683594, "growth_active": 1.0820740461349487, "growth_memory": 1.4751039743423462, "heads": [{"head": 0, "memory_to_active": 0.015767131000757217, "contribution_to_active": 0.3307943344116211, "mix_mean": 0.305151104927063, "mix_low": 0.0352783203125, "mix_high": 0.001018524169921875, "read_cosine": 0.24892684817314148, "beta_mean": 5.002322359359823e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005557439289987087, "mix_std": 0.2732817530632019, "beta_std": 0.00012486455671023577, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002503418829292059, "evidence_weights": [-0.1823795884847641, -0.17445731163024902, 0.7529761791229248], "agreement_weight": 0.044308681041002274, "orthogonal_weight": -0.4696935713291168}, {"head": 1, "memory_to_active": 0.06189649552106857, "contribution_to_active": 0.297134667634964, "mix_mean": 0.15369278192520142, "mix_low": 0.18048667907714844, "mix_high": 0.00027751922607421875, "read_cosine": 0.3945116400718689, "beta_mean": 0.00021172178094275296, "nonfinite": 0.0, "weighted_memory_to_active": 0.01714860461652279, "mix_std": 0.20757293701171875, "beta_std": 0.00042922713328152895, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.004914738237857819, "evidence_weights": [-0.12461406737565994, -0.11457514017820358, 0.31115421652793884], "agreement_weight": 0.21469126641750336, "orthogonal_weight": -0.3155664801597595}, {"head": 2, "memory_to_active": 0.5742373466491699, "contribution_to_active": 0.37507855892181396, "mix_mean": 0.19981977343559265, "mix_low": 0.10518741607666016, "mix_high": 4.57763671875e-05, "read_cosine": 0.31472882628440857, "beta_mean": 0.012848012149333954, "nonfinite": 0.0, "weighted_memory_to_active": 0.21600931882858276, "mix_std": 0.2413695603609085, "beta_std": 0.01591668650507927, "beta_low": 0.5787353515625, "beta_high": 0.0, "orthogonal_to_active": 0.02457982674241066, "evidence_weights": [-0.09759068489074707, -0.09333794564008713, 0.09447101503610611], "agreement_weight": 0.1066373735666275, "orthogonal_weight": -0.14593321084976196}, {"head": 3, "memory_to_active": 2.148061513900757, "contribution_to_active": 0.21265549957752228, "mix_mean": 0.043892912566661835, "mix_low": 0.34497833251953125, "mix_high": 0.0, "read_cosine": 0.4526994824409485, "beta_mean": 0.12680387496948242, "nonfinite": 0.0, "weighted_memory_to_active": 0.1897180676460266, "mix_std": 0.09220772981643677, "beta_std": 0.08074883371591568, "beta_low": 0.00439453125, "beta_high": 0.0, "orthogonal_to_active": 0.01689760386943817, "evidence_weights": [0.028872422873973846, 0.02045232616364956, -0.052724629640579224], "agreement_weight": -0.0006373556097969413, "orthogonal_weight": 0.08839350193738937}, {"head": 4, "memory_to_active": 3.4321389198303223, "contribution_to_active": 1.3029508590698242, "mix_mean": 0.11863835155963898, "mix_low": 0.24452590942382812, "mix_high": 0.011759757995605469, "read_cosine": 0.2820972204208374, "beta_mean": 0.25971490144729614, "nonfinite": 0.0, "weighted_memory_to_active": 1.2631843090057373, "mix_std": 0.20311464369297028, "beta_std": 0.10616950690746307, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008832639083266258, "evidence_weights": [-0.04066075012087822, -0.08802209049463272, 0.09399117529392242], "agreement_weight": -0.04922408610582352, "orthogonal_weight": 0.007262427359819412}, {"head": 5, "memory_to_active": 2.002438545227051, "contribution_to_active": 0.5721842646598816, "mix_mean": 0.04978763684630394, "mix_low": 0.5040292739868164, "mix_high": 3.528594970703125e-05, "read_cosine": 0.40944504737854004, "beta_mean": 0.2606585621833801, "nonfinite": 0.0, "weighted_memory_to_active": 0.6659995317459106, "mix_std": 0.11187706887722015, "beta_std": 0.11701982468366623, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.057538460940122604, "evidence_weights": [0.029768388718366623, -0.02838546223938465, 0.039884455502033234], "agreement_weight": -0.01968652568757534, "orthogonal_weight": 0.1162915900349617}]}, {"layer": 10, "memory_to_active": 0.40280288457870483, "contribution_to_active": 0.3802281618118286, "mix_mean": 0.2727756202220917, "mix_low": 0.1138555258512497, "mix_high": 0.0065425243228673935, "read_cosine": 0.3898864686489105, "beta_mean": 0.05088440328836441, "nonfinite": 0.0, "weighted_memory_to_active": 0.12784239649772644, "mix_std": 0.2194131314754486, "beta_std": 0.05757465958595276, "beta_low": 0.5060831904411316, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03849140927195549, "output_influence": 0.36678558588027954, "residual_influence": 0.09764787554740906, "state_active": 0.07469525933265686, "state_memory": 0.02063160389661789, "state_peak": 47.88023376464844, "growth_active": 1.095747709274292, "growth_memory": 1.3058867454528809, "heads": [{"head": 0, "memory_to_active": 0.6013548970222473, "contribution_to_active": 0.8588395714759827, "mix_mean": 0.6058804988861084, "mix_low": 0.01251983642578125, "mix_high": 0.038596153259277344, "read_cosine": 0.15503965318202972, "beta_mean": 0.17377494275569916, "nonfinite": 0.0, "weighted_memory_to_active": 0.4113398492336273, "mix_std": 0.341840535402298, "beta_std": 0.2057325392961502, "beta_low": 0.12432861328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.14467650651931763, "evidence_weights": [-0.1477329134941101, -0.3186647891998291, -0.1418682485818863], "agreement_weight": 0.021234769374132156, "orthogonal_weight": 0.3751372992992401}, {"head": 1, "memory_to_active": 0.011369447223842144, "contribution_to_active": 0.33266231417655945, "mix_mean": 0.25876909494400024, "mix_low": 0.019212722778320312, "mix_high": 5.817413330078125e-05, "read_cosine": 0.21739166975021362, "beta_mean": 6.856553955003619e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0032569644972682, "mix_std": 0.23223228752613068, "beta_std": 0.00011871687456732616, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001276660244911909, "evidence_weights": [-0.17394395172595978, -0.13366153836250305, 0.13635185360908508], "agreement_weight": 0.039061978459358215, "orthogonal_weight": -0.42980027198791504}, {"head": 2, "memory_to_active": 1.4266307353973389, "contribution_to_active": 0.18660515546798706, "mix_mean": 0.029169514775276184, "mix_low": 0.5673351287841797, "mix_high": 0.0, "read_cosine": 0.2348816692829132, "beta_mean": 0.06566975265741348, "nonfinite": 0.0, "weighted_memory_to_active": 0.21779441833496094, "mix_std": 0.06938114762306213, "beta_std": 0.05900072306394577, "beta_low": 0.03387451171875, "beta_high": 0.0, "orthogonal_to_active": 0.03286749869585037, "evidence_weights": [0.03936435654759407, -0.02294391579926014, 0.12223596125841141], "agreement_weight": -0.01903354935348034, "orthogonal_weight": -0.15545427799224854}, {"head": 3, "memory_to_active": 0.2148159146308899, "contribution_to_active": 0.1827440857887268, "mix_mean": 0.23476223647594452, "mix_low": 0.058066368103027344, "mix_high": 0.000576019287109375, "read_cosine": 0.7933269739151001, "beta_mean": 0.05858764052391052, "nonfinite": 0.0, "weighted_memory_to_active": 0.06417752057313919, "mix_std": 0.25443601608276367, "beta_std": 0.07007122784852982, "beta_low": 0.06866455078125, "beta_high": 0.0, "orthogonal_to_active": 0.037110887467861176, "evidence_weights": [-0.11647265404462814, -0.15104345977306366, 0.22369877994060516], "agreement_weight": 0.011319978162646294, "orthogonal_weight": 0.7616803646087646}, {"head": 4, "memory_to_active": 0.03359689936041832, "contribution_to_active": 0.28460267186164856, "mix_mean": 0.23466092348098755, "mix_low": 0.014639854431152344, "mix_high": 0.0, "read_cosine": 0.1426885724067688, "beta_mean": 0.00015359990356955677, "nonfinite": 0.0, "weighted_memory_to_active": 0.00966260302811861, "mix_std": 0.2131786048412323, "beta_std": 0.000850989599712193, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.0024116626009345055, "evidence_weights": [-0.150899738073349, -0.1240912452340126, 0.023316236212849617], "agreement_weight": 0.030199827626347542, "orthogonal_weight": -0.27635568380355835}, {"head": 5, "memory_to_active": 0.1290493607521057, "contribution_to_active": 0.43591490387916565, "mix_mean": 0.27341127395629883, "mix_low": 0.011359214782714844, "mix_high": 2.47955322265625e-05, "read_cosine": 0.7959900498390198, "beta_mean": 0.007051913999021053, "nonfinite": 0.0, "weighted_memory_to_active": 0.060823000967502594, "mix_std": 0.2054101526737213, "beta_std": 0.009673750028014183, "beta_low": 0.8099365234375, "beta_high": 0.0, "orthogonal_to_active": 0.012605234049260616, "evidence_weights": [-0.1916024088859558, -0.1590551733970642, 0.18273261189460754], "agreement_weight": 0.08183735609054565, "orthogonal_weight": -0.4709208607673645}]}, {"layer": 11, "memory_to_active": 0.3269909620285034, "contribution_to_active": 0.41959524154663086, "mix_mean": 0.32600486278533936, "mix_low": 0.05133787915110588, "mix_high": 0.015508970245718956, "read_cosine": 0.5184125900268555, "beta_mean": 0.043558862060308456, "nonfinite": 0.0, "weighted_memory_to_active": 0.09906072914600372, "mix_std": 0.2826521098613739, "beta_std": 0.06310271471738815, "beta_low": 0.459136962890625, "beta_high": 0.0, "orthogonal_to_active": 0.03023666702210903, "output_influence": 0.4633604884147644, "residual_influence": 0.11771510541439056, "state_active": 0.12110498547554016, "state_memory": 0.0154484324157238, "state_peak": 23.47430419921875, "growth_active": 1.1281664371490479, "growth_memory": 1.3838077783584595, "heads": [{"head": 0, "memory_to_active": 0.08857546746730804, "contribution_to_active": 0.47335001826286316, "mix_mean": 0.4144795536994934, "mix_low": 0.00711822509765625, "mix_high": 0.0021123886108398438, "read_cosine": 0.4255782961845398, "beta_mean": 0.0005305246450006962, "nonfinite": 0.0, "weighted_memory_to_active": 0.048685744404792786, "mix_std": 0.2749882638454437, "beta_std": 0.005007757805287838, "beta_low": 0.99639892578125, "beta_high": 0.0, "orthogonal_to_active": 0.022842619568109512, "evidence_weights": [-0.156728595495224, -0.2384212762117386, 0.3582075238227844], "agreement_weight": 0.12151959538459778, "orthogonal_weight": -0.6099245548248291}, {"head": 1, "memory_to_active": 0.6073112487792969, "contribution_to_active": 0.19182837009429932, "mix_mean": 0.20090755820274353, "mix_low": 0.03944969177246094, "mix_high": 5.91278076171875e-05, "read_cosine": 0.6110801696777344, "beta_mean": 0.042671751230955124, "nonfinite": 0.0, "weighted_memory_to_active": 0.1692829132080078, "mix_std": 0.2100280374288559, "beta_std": 0.07187947630882263, "beta_low": 0.287841796875, "beta_high": 0.0, "orthogonal_to_active": 0.015248998068273067, "evidence_weights": [-0.1428883671760559, -0.09929844737052917, 0.12173677235841751], "agreement_weight": 0.0805945172905922, "orthogonal_weight": -0.12652082741260529}, {"head": 2, "memory_to_active": 0.06925557553768158, "contribution_to_active": 0.8236652612686157, "mix_mean": 0.45506149530410767, "mix_low": 0.04534721374511719, "mix_high": 0.059586524963378906, "read_cosine": 0.3504321873188019, "beta_mean": 0.07937020063400269, "nonfinite": 0.0, "weighted_memory_to_active": 0.057589784264564514, "mix_std": 0.40256527066230774, "beta_std": 0.11382992565631866, "beta_low": 0.43170166015625, "beta_high": 0.0, "orthogonal_to_active": 0.07501788437366486, "evidence_weights": [-0.1950017809867859, -0.37301501631736755, -0.39649245142936707], "agreement_weight": -0.01949235610663891, "orthogonal_weight": 1.4276373386383057}, {"head": 3, "memory_to_active": 0.040323760360479355, "contribution_to_active": 0.42129209637641907, "mix_mean": 0.43246185779571533, "mix_low": 0.0758056640625, "mix_high": 0.030699729919433594, "read_cosine": 0.5571070909500122, "beta_mean": 0.07829998433589935, "nonfinite": 0.0, "weighted_memory_to_active": 0.021652592346072197, "mix_std": 0.4082145392894745, "beta_std": 0.11495266854763031, "beta_low": 0.42657470703125, "beta_high": 0.0, "orthogonal_to_active": 0.0271501075476408, "evidence_weights": [-0.24082930386066437, -0.350458025932312, -0.030868474394083023], "agreement_weight": 0.01546056754887104, "orthogonal_weight": 1.5253701210021973}, {"head": 4, "memory_to_active": 0.918030321598053, "contribution_to_active": 0.20957614481449127, "mix_mean": 0.1121739074587822, "mix_low": 0.1356964111328125, "mix_high": 0.0, "read_cosine": 0.2704010605812073, "beta_mean": 0.04175855219364166, "nonfinite": 0.0, "weighted_memory_to_active": 0.17397315800189972, "mix_std": 0.15117247402668, "beta_std": 0.04716484993696213, "beta_low": 0.13787841796875, "beta_high": 0.0, "orthogonal_to_active": 0.01710299216210842, "evidence_weights": [-0.04518258199095726, -0.030620014294981956, 0.5622797608375549], "agreement_weight": -0.04616871848702431, "orthogonal_weight": -0.11863167583942413}, {"head": 5, "memory_to_active": 0.23844951391220093, "contribution_to_active": 0.39785927534103394, "mix_mean": 0.340944766998291, "mix_low": 0.0046100616455078125, "mix_high": 0.0005960464477539062, "read_cosine": 0.8958768844604492, "beta_mean": 0.018722139298915863, "nonfinite": 0.0, "weighted_memory_to_active": 0.123180091381073, "mix_std": 0.2489439696073532, "beta_std": 0.025781594216823578, "beta_low": 0.47442626953125, "beta_high": 0.0, "orthogonal_to_active": 0.024057382717728615, "evidence_weights": [-0.18277603387832642, -0.2209315448999405, 0.26588940620422363], "agreement_weight": 0.04650025814771652, "orthogonal_weight": -0.5116608738899231}]}, {"layer": 12, "memory_to_active": 0.6614776849746704, "contribution_to_active": 0.4628202021121979, "mix_mean": 0.3358559310436249, "mix_low": 0.06924264132976532, "mix_high": 0.0009894371032714844, "read_cosine": 0.31636589765548706, "beta_mean": 0.024252336472272873, "nonfinite": 0.0, "weighted_memory_to_active": 0.2584286332130432, "mix_std": 0.22834011912345886, "beta_std": 0.028584526851773262, "beta_low": 0.6433614492416382, "beta_high": 0.0, "orthogonal_to_active": 0.060282692313194275, "output_influence": 0.29770663380622864, "residual_influence": 0.07399992644786835, "state_active": 0.060641150921583176, "state_memory": 0.020051946863532066, "state_peak": 36.00786209106445, "growth_active": 1.2073657512664795, "growth_memory": 1.3314831256866455, "heads": [{"head": 0, "memory_to_active": 0.7574863433837891, "contribution_to_active": 0.26505422592163086, "mix_mean": 0.2240796685218811, "mix_low": 0.04314422607421875, "mix_high": 7.724761962890625e-05, "read_cosine": 0.4816877543926239, "beta_mean": 0.015080925077199936, "nonfinite": 0.0, "weighted_memory_to_active": 0.29459649324417114, "mix_std": 0.21994274854660034, "beta_std": 0.03198038041591644, "beta_low": 0.64215087890625, "beta_high": 0.0, "orthogonal_to_active": 0.08247578144073486, "evidence_weights": [-0.10557813942432404, -0.14582668244838715, 0.24032042920589447], "agreement_weight": 0.09856781363487244, "orthogonal_weight": -0.3868546485900879}, {"head": 1, "memory_to_active": 0.28479576110839844, "contribution_to_active": 0.3667680323123932, "mix_mean": 0.33397042751312256, "mix_low": 0.00331878662109375, "mix_high": 0.00017642974853515625, "read_cosine": 0.5717175006866455, "beta_mean": 0.0038755498826503754, "nonfinite": 0.0, "weighted_memory_to_active": 0.11997686326503754, "mix_std": 0.24855859577655792, "beta_std": 0.00657226238399744, "beta_low": 0.91571044921875, "beta_high": 0.0, "orthogonal_to_active": 0.06930690258741379, "evidence_weights": [-0.14866101741790771, -0.19553647935390472, 0.42620131373405457], "agreement_weight": 0.08268506079912186, "orthogonal_weight": -0.8183494806289673}, {"head": 2, "memory_to_active": 0.6577005386352539, "contribution_to_active": 0.4038606286048889, "mix_mean": 0.33107632398605347, "mix_low": 0.018671035766601562, "mix_high": 0.0013971328735351562, "read_cosine": 0.2955220937728882, "beta_mean": 0.08449655771255493, "nonfinite": 0.0, "weighted_memory_to_active": 0.29578864574432373, "mix_std": 0.25024664402008057, "beta_std": 0.07319359481334686, "beta_low": 0.0057373046875, "beta_high": 0.0, "orthogonal_to_active": 0.07807602733373642, "evidence_weights": [-0.19252623617649078, -0.08761831372976303, 0.14235150814056396], "agreement_weight": 0.033483412116765976, "orthogonal_weight": 0.31534087657928467}, {"head": 3, "memory_to_active": 0.05670038238167763, "contribution_to_active": 0.6047377586364746, "mix_mean": 0.5854184627532959, "mix_low": 0.001201629638671875, "mix_high": 0.0027780532836914062, "read_cosine": -0.010254915803670883, "beta_mean": 0.0006274779443629086, "nonfinite": 0.0, "weighted_memory_to_active": 0.03316144645214081, "mix_std": 0.2599487900733948, "beta_std": 0.0009632577421143651, "beta_low": 0.99884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.020021630451083183, "evidence_weights": [-0.2879718542098999, -0.2686331570148468, 0.21806587278842926], "agreement_weight": 0.07741757482290268, "orthogonal_weight": -0.6192283034324646}, {"head": 4, "memory_to_active": 2.1434450149536133, "contribution_to_active": 0.7282747030258179, "mix_mean": 0.06477440148591995, "mix_low": 0.3433256149291992, "mix_high": 0.00081634521484375, "read_cosine": 0.10223996639251709, "beta_mean": 0.04058883339166641, "nonfinite": 0.0, "weighted_memory_to_active": 0.778403639793396, "mix_std": 0.12081454694271088, "beta_std": 0.05694197118282318, "beta_low": 0.30621337890625, "beta_high": 0.0, "orthogonal_to_active": 0.09121625870466232, "evidence_weights": [0.054552264511585236, -0.04395516589283943, 0.07480792701244354], "agreement_weight": -0.008951305411756039, "orthogonal_weight": -0.1284758448600769}, {"head": 5, "memory_to_active": 0.06873778998851776, "contribution_to_active": 0.4082259237766266, "mix_mean": 0.4758163392543793, "mix_low": 0.005794525146484375, "mix_high": 0.0006914138793945312, "read_cosine": 0.4572829604148865, "beta_mean": 0.0008446633582934737, "nonfinite": 0.0, "weighted_memory_to_active": 0.028644509613513947, "mix_std": 0.27052929997444153, "beta_std": 0.001855695154517889, "beta_low": 0.99151611328125, "beta_high": 0.0, "orthogonal_to_active": 0.02059955522418022, "evidence_weights": [-0.24661867320537567, -0.24801352620124817, 0.4464033842086792], "agreement_weight": 0.04369639232754707, "orthogonal_weight": -0.8237498998641968}]}, {"layer": 13, "memory_to_active": 0.2700416147708893, "contribution_to_active": 0.38546571135520935, "mix_mean": 0.37628939747810364, "mix_low": 0.005777836311608553, "mix_high": 0.0019488335819914937, "read_cosine": 0.37829241156578064, "beta_mean": 0.020937545225024223, "nonfinite": 0.0, "weighted_memory_to_active": 0.11777015775442123, "mix_std": 0.24464112520217896, "beta_std": 0.026100698858499527, "beta_low": 0.7088420391082764, "beta_high": 0.0, "orthogonal_to_active": 0.07104747742414474, "output_influence": 0.33397340774536133, "residual_influence": 0.10540133714675903, "state_active": 0.09024989604949951, "state_memory": 0.03131566569209099, "state_peak": 65.4473648071289, "growth_active": 0.981377363204956, "growth_memory": 1.0888997316360474, "heads": [{"head": 0, "memory_to_active": 0.28907012939453125, "contribution_to_active": 0.3622117042541504, "mix_mean": 0.33043158054351807, "mix_low": 0.005054473876953125, "mix_high": 0.0001544952392578125, "read_cosine": 0.3328365087509155, "beta_mean": 0.003303281031548977, "nonfinite": 0.0, "weighted_memory_to_active": 0.12117671966552734, "mix_std": 0.2433171570301056, "beta_std": 0.004750217776745558, "beta_low": 0.92962646484375, "beta_high": 0.0, "orthogonal_to_active": 0.10199616849422455, "evidence_weights": [-0.17617712914943695, -0.1946975737810135, 0.43553003668785095], "agreement_weight": 0.05229457467794418, "orthogonal_weight": -0.9208747744560242}, {"head": 1, "memory_to_active": 0.18369446694850922, "contribution_to_active": 0.4180119037628174, "mix_mean": 0.33899587392807007, "mix_low": 0.004694938659667969, "mix_high": 1.9073486328125e-05, "read_cosine": 0.28808289766311646, "beta_mean": 0.0012280130758881569, "nonfinite": 0.0, "weighted_memory_to_active": 0.07745042443275452, "mix_std": 0.24411612749099731, "beta_std": 0.0017557862447574735, "beta_low": 0.9931640625, "beta_high": 0.0, "orthogonal_to_active": 0.06498101353645325, "evidence_weights": [-0.1717381477355957, -0.1904166042804718, 0.4940524995326996], "agreement_weight": 0.07324756681919098, "orthogonal_weight": -0.90491783618927}, {"head": 2, "memory_to_active": 0.2870595157146454, "contribution_to_active": 0.29673394560813904, "mix_mean": 0.27493470907211304, "mix_low": 0.004820823669433594, "mix_high": 1.52587890625e-05, "read_cosine": 0.6756333708763123, "beta_mean": 0.029322374612092972, "nonfinite": 0.0, "weighted_memory_to_active": 0.1088453084230423, "mix_std": 0.2024545967578888, "beta_std": 0.05390692502260208, "beta_low": 0.439453125, "beta_high": 0.0, "orthogonal_to_active": 0.03305104374885559, "evidence_weights": [-0.1812216192483902, -0.14605869352817535, 0.3817625641822815], "agreement_weight": 0.06550101935863495, "orthogonal_weight": -0.5080729126930237}, {"head": 3, "memory_to_active": 0.15862631797790527, "contribution_to_active": 0.4893350601196289, "mix_mean": 0.5005975961685181, "mix_low": 0.005015373229980469, "mix_high": 0.0007352828979492188, "read_cosine": -0.05293475091457367, "beta_mean": 0.0007527695852331817, "nonfinite": 0.0, "weighted_memory_to_active": 0.10351236164569855, "mix_std": 0.23183703422546387, "beta_std": 0.004777389578521252, "beta_low": 0.994384765625, "beta_high": 0.0, "orthogonal_to_active": 0.10188589245080948, "evidence_weights": [-0.33491066098213196, -0.24770227074623108, 0.7455143928527832], "agreement_weight": 0.6639424562454224, "orthogonal_weight": -0.9907272458076477}, {"head": 4, "memory_to_active": 0.36638563871383667, "contribution_to_active": 0.31501221656799316, "mix_mean": 0.49649134278297424, "mix_low": 0.0037593841552734375, "mix_high": 0.010571479797363281, "read_cosine": 0.8015016317367554, "beta_mean": 0.08544164150953293, "nonfinite": 0.0, "weighted_memory_to_active": 0.17046807706356049, "mix_std": 0.2860846221446991, "beta_std": 0.07614404708147049, "beta_low": 0.0093994140625, "beta_high": 0.0, "orthogonal_to_active": 0.06501968204975128, "evidence_weights": [-0.26111283898353577, -0.27324825525283813, 0.38364556431770325], "agreement_weight": 0.0059945471584796906, "orthogonal_weight": 0.5553487539291382}, {"head": 5, "memory_to_active": 0.33541369438171387, "contribution_to_active": 0.43148910999298096, "mix_mean": 0.316285103559494, "mix_low": 0.011322021484375, "mix_high": 0.00019741058349609375, "read_cosine": 0.2246347814798355, "beta_mean": 0.005577188916504383, "nonfinite": 0.0, "weighted_memory_to_active": 0.12516802549362183, "mix_std": 0.26003721356391907, "beta_std": 0.015269805677235126, "beta_low": 0.88702392578125, "beta_high": 0.0, "orthogonal_to_active": 0.05935107544064522, "evidence_weights": [-0.17042706906795502, -0.19353383779525757, 0.1233607828617096], "agreement_weight": 0.05387825146317482, "orthogonal_weight": -0.5421881079673767}]}, {"layer": 14, "memory_to_active": 0.29936683177948, "contribution_to_active": 0.4138917624950409, "mix_mean": 0.36462080478668213, "mix_low": 0.024034500122070312, "mix_high": 0.0017544429283589125, "read_cosine": 0.5949094295501709, "beta_mean": 0.08303500711917877, "nonfinite": 0.0, "weighted_memory_to_active": 0.13839560747146606, "mix_std": 0.24763809144496918, "beta_std": 0.07568400353193283, "beta_low": 0.3439534604549408, "beta_high": 0.0, "orthogonal_to_active": 0.035020824521780014, "output_influence": 0.22846725583076477, "residual_influence": 0.0874108225107193, "state_active": 0.0951554924249649, "state_memory": 0.02163561061024666, "state_peak": 31.76708221435547, "growth_active": 1.1913793087005615, "growth_memory": 1.4423185586929321, "heads": [{"head": 0, "memory_to_active": 0.26955240964889526, "contribution_to_active": 0.2966946065425873, "mix_mean": 0.2766728401184082, "mix_low": 0.018613815307617188, "mix_high": 3.719329833984375e-05, "read_cosine": 0.6763582229614258, "beta_mean": 0.06999234110116959, "nonfinite": 0.0, "weighted_memory_to_active": 0.10258575528860092, "mix_std": 0.24255409836769104, "beta_std": 0.0930989533662796, "beta_low": 0.171875, "beta_high": 0.0, "orthogonal_to_active": 0.00858880765736103, "evidence_weights": [-0.07326394319534302, -0.15713870525360107, 0.610550045967102], "agreement_weight": 0.0429716482758522, "orthogonal_weight": 0.11888279020786285}, {"head": 1, "memory_to_active": 0.4273897409439087, "contribution_to_active": 0.4737357199192047, "mix_mean": 0.3136950433254242, "mix_low": 0.07447528839111328, "mix_high": 0.0011157989501953125, "read_cosine": 0.5203917026519775, "beta_mean": 0.3217836618423462, "nonfinite": 0.0, "weighted_memory_to_active": 0.21648406982421875, "mix_std": 0.28824660181999207, "beta_std": 0.27136802673339844, "beta_low": 0.07025146484375, "beta_high": 0.0, "orthogonal_to_active": 0.0060466681607067585, "evidence_weights": [-0.035047706216573715, -0.24706727266311646, 0.16840426623821259], "agreement_weight": 0.05493006110191345, "orthogonal_weight": 0.037958353757858276}, {"head": 2, "memory_to_active": 0.36613476276397705, "contribution_to_active": 0.41961905360221863, "mix_mean": 0.3965919017791748, "mix_low": 0.006928443908691406, "mix_high": 0.0003452301025390625, "read_cosine": 0.43481504917144775, "beta_mean": 0.020827464759349823, "nonfinite": 0.0, "weighted_memory_to_active": 0.18702363967895508, "mix_std": 0.26572537422180176, "beta_std": 0.01972518488764763, "beta_low": 0.31756591796875, "beta_high": 0.0, "orthogonal_to_active": 0.11946295201778412, "evidence_weights": [-0.19570156931877136, -0.20445767045021057, 0.3735700845718384], "agreement_weight": 0.08992745727300644, "orthogonal_weight": -0.7446712255477905}, {"head": 3, "memory_to_active": 0.07056871801614761, "contribution_to_active": 0.6134634017944336, "mix_mean": 0.6279196739196777, "mix_low": 0.0023126602172851562, "mix_high": 0.0060100555419921875, "read_cosine": 0.5830624103546143, "beta_mean": 0.010294689796864986, "nonfinite": 0.0, "weighted_memory_to_active": 0.04689886420965195, "mix_std": 0.23710066080093384, "beta_std": 0.008397270925343037, "beta_low": 0.6204833984375, "beta_high": 0.0, "orthogonal_to_active": 0.016160421073436737, "evidence_weights": [-0.3715447783470154, -0.3209399878978729, 0.3892771899700165], "agreement_weight": 0.0729222223162651, "orthogonal_weight": -0.4688507318496704}, {"head": 4, "memory_to_active": 0.27520400285720825, "contribution_to_active": 0.44253456592559814, "mix_mean": 0.3701102137565613, "mix_low": 0.0056476593017578125, "mix_high": 0.0028591156005859375, "read_cosine": 0.6601065397262573, "beta_mean": 0.0701802670955658, "nonfinite": 0.0, "weighted_memory_to_active": 0.1386115700006485, "mix_std": 0.2507023513317108, "beta_std": 0.052512556314468384, "beta_low": 0.00518798828125, "beta_high": 0.0, "orthogonal_to_active": 0.020063284784555435, "evidence_weights": [-0.19523772597312927, -0.24177996814250946, 0.18112747371196747], "agreement_weight": 0.10531798005104065, "orthogonal_weight": -0.21920688450336456}, {"head": 5, "memory_to_active": 0.387351393699646, "contribution_to_active": 0.23730309307575226, "mix_mean": 0.20273509621620178, "mix_low": 0.03622913360595703, "mix_high": 0.00015926361083984375, "read_cosine": 0.6947223544120789, "beta_mean": 0.005131623707711697, "nonfinite": 0.0, "weighted_memory_to_active": 0.13876976072788239, "mix_std": 0.20149940252304077, "beta_std": 0.009002020582556725, "beta_low": 0.87835693359375, "beta_high": 0.0, "orthogonal_to_active": 0.03980279713869095, "evidence_weights": [-0.17615635693073273, -0.12550058960914612, 0.06450717151165009], "agreement_weight": 0.10196796804666519, "orthogonal_weight": -0.46384352445602417}]}, {"layer": 15, "memory_to_active": 0.1149907261133194, "contribution_to_active": 0.43933385610580444, "mix_mean": 0.3561769723892212, "mix_low": 0.006912549491971731, "mix_high": 0.002028942108154297, "read_cosine": 0.5987021327018738, "beta_mean": 0.0489572137594223, "nonfinite": 0.0, "weighted_memory_to_active": 0.052123669534921646, "mix_std": 0.2578737735748291, "beta_std": 0.05263269320130348, "beta_low": 0.67864990234375, "beta_high": 0.0, "orthogonal_to_active": 0.014389670453965664, "output_influence": 0.27345871925354004, "residual_influence": 0.1450481414794922, "state_active": 0.11043069511651993, "state_memory": 0.011361360549926758, "state_peak": 26.961374282836914, "growth_active": 1.383124828338623, "growth_memory": 1.6115556955337524, "heads": [{"head": 0, "memory_to_active": 0.1306159645318985, "contribution_to_active": 0.44619956612586975, "mix_mean": 0.32929253578186035, "mix_low": 0.009379386901855469, "mix_high": 0.0017480850219726562, "read_cosine": 0.6814560890197754, "beta_mean": 0.04914582520723343, "nonfinite": 0.0, "weighted_memory_to_active": 0.05184473842382431, "mix_std": 0.25363868474960327, "beta_std": 0.04182714223861694, "beta_low": 0.12213134765625, "beta_high": 0.0, "orthogonal_to_active": 0.012174954637885094, "evidence_weights": [-0.17647576332092285, -0.21263062953948975, 0.34846606850624084], "agreement_weight": 0.08067741245031357, "orthogonal_weight": -0.3892189860343933}, {"head": 1, "memory_to_active": 0.2202579826116562, "contribution_to_active": 0.399417519569397, "mix_mean": 0.34485676884651184, "mix_low": 0.0061798095703125, "mix_high": 0.0055789947509765625, "read_cosine": 0.8494579792022705, "beta_mean": 0.002761955140158534, "nonfinite": 0.0, "weighted_memory_to_active": 0.09839118272066116, "mix_std": 0.26215860247612, "beta_std": 0.005353906657546759, "beta_low": 0.95989990234375, "beta_high": 0.0, "orthogonal_to_active": 0.008069203235208988, "evidence_weights": [-0.1966327279806137, -0.20467349886894226, 0.25747770071029663], "agreement_weight": 0.09370272606611252, "orthogonal_weight": -0.15987610816955566}, {"head": 2, "memory_to_active": 0.012838652357459068, "contribution_to_active": 0.406189888715744, "mix_mean": 0.32643377780914307, "mix_low": 0.009519577026367188, "mix_high": 7.343292236328125e-05, "read_cosine": 0.7438433766365051, "beta_mean": 0.0002711980778258294, "nonfinite": 0.0, "weighted_memory_to_active": 0.004714719019830227, "mix_std": 0.25397375226020813, "beta_std": 0.000542983878403902, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.0017583839362487197, "evidence_weights": [-0.2109016627073288, -0.253447562456131, 0.30774587392807007], "agreement_weight": 0.039809390902519226, "orthogonal_weight": -0.5094602108001709}, {"head": 3, "memory_to_active": 0.14336441457271576, "contribution_to_active": 0.41515910625457764, "mix_mean": 0.2640931010246277, "mix_low": 0.0040912628173828125, "mix_high": 2.574920654296875e-05, "read_cosine": 0.671524167060852, "beta_mean": 0.007152032572776079, "nonfinite": 0.0, "weighted_memory_to_active": 0.06003540754318237, "mix_std": 0.21188047528266907, "beta_std": 0.013955846428871155, "beta_low": 0.8460693359375, "beta_high": 0.0, "orthogonal_to_active": 0.014528686180710793, "evidence_weights": [-0.15357175469398499, -0.19940921664237976, 0.34032145142555237], "agreement_weight": 0.05274081230163574, "orthogonal_weight": -0.3908199071884155}, {"head": 4, "memory_to_active": 0.17816433310508728, "contribution_to_active": 0.5249338150024414, "mix_mean": 0.38873574137687683, "mix_low": 0.006945610046386719, "mix_high": 0.00020599365234375, "read_cosine": 0.11543994396924973, "beta_mean": 0.23431789875030518, "nonfinite": 0.0, "weighted_memory_to_active": 0.09587462246417999, "mix_std": 0.2873395085334778, "beta_std": 0.25393393635749817, "beta_low": 0.1441650390625, "beta_high": 0.0, "orthogonal_to_active": 0.04883778095245361, "evidence_weights": [-0.2755691409111023, -0.2519683837890625, 0.4148406386375427], "agreement_weight": 0.006682251580059528, "orthogonal_weight": -0.5663325190544128}, {"head": 5, "memory_to_active": 0.004702999256551266, "contribution_to_active": 0.4441031813621521, "mix_mean": 0.48364976048469543, "mix_low": 0.005359649658203125, "mix_high": 0.0045413970947265625, "read_cosine": 0.5304911732673645, "beta_mean": 9.43729464779608e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.001881332602351904, "mix_std": 0.278251588344574, "beta_std": 0.00018230524437967688, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0009690164006315172, "evidence_weights": [-0.3529052436351776, -0.28725501894950867, 0.5379238724708557], "agreement_weight": 0.10166428238153458, "orthogonal_weight": 0.5561618804931641}]}]}, {"step": 10000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8505584919985267, "memory_to_active": 1.0845630168914795, "contribution_to_active": 0.4052995443344116, "mix_mean": 0.24783194065093994, "mix_low": 0.1943199336528778, "mix_high": 0.003871023887768388, "read_cosine": 0.4017013609409332, "beta_mean": 0.10449419915676117, "nonfinite": 0.0, "weighted_memory_to_active": 0.2113797962665558, "mix_std": 0.19204682111740112, "beta_std": 0.079267218708992, "beta_low": 0.39674824476242065, "beta_high": 7.62939453125e-06, "orthogonal_to_active": 0.03574230521917343, "samples": 4, "scored_tokens": 899, "normal_loss": 3.2323834896087646, "normal_token_accuracy": 0.42380422353744507, "state_active": 0.06482356041669846, "state_memory": 0.029374361038208008, "state_peak": 67.2616958618164, "growth_active": 1.114297866821289, "growth_memory": 1.4691362380981445, "output_influence": 0.27803075313568115, "residual_influence": 0.08580619096755981, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.3035888671875, "no_memory_loss_delta": 0.15160441398620605, "scrambled_loss_delta": 0.3065685033798218, "memory_only_loss_delta": 1.8435444831848145, "no_evidence_loss_delta": 0.0525846928358078, "no_agreement_loss_delta": 0.0003803728614002466, "no_orthogonal_loss_delta": -0.007797246798872948}, {"index": 94, "tokens": 256, "normal_loss": 3.3581433296203613, "no_memory_loss_delta": 0.27084800601005554, "scrambled_loss_delta": 0.37578341364860535, "memory_only_loss_delta": 1.8956217765808105, "no_evidence_loss_delta": 0.1497633159160614, "no_agreement_loss_delta": 0.04020034521818161, "no_orthogonal_loss_delta": 0.012142583727836609}, {"index": 188, "tokens": 256, "normal_loss": 3.685835838317871, "no_memory_loss_delta": 0.16678069531917572, "scrambled_loss_delta": 0.14124713838100433, "memory_only_loss_delta": 2.018221378326416, "no_evidence_loss_delta": 0.0865170806646347, "no_agreement_loss_delta": 0.008842960931360722, "no_orthogonal_loss_delta": 0.0017426111735403538}, {"index": 283, "tokens": 131, "normal_loss": 1.9613395929336548, "no_memory_loss_delta": 0.2355954349040985, "scrambled_loss_delta": 0.3836505711078644, "memory_only_loss_delta": 2.474669933319092, "no_evidence_loss_delta": 0.040008362382650375, "no_agreement_loss_delta": -0.00966118648648262, "no_orthogonal_loss_delta": 0.011937296018004417}], "agreement_weight_abs_mean": 0.06555651128292084, "agreement_weight_abs_max": 0.7646854519844055, "agreement_weight_mean": 0.04541950300335884, "orthogonal_weight_abs_mean": 0.36661043763160706, "orthogonal_weight_abs_max": 1.702148675918579, "orthogonal_weight_mean": -0.150077685713768, "no_memory_loss": 3.434504508972168, "no_memory_loss_delta": 0.20212088525295258, "no_memory_token_accuracy_gain": 0.008898776024580002, "no_memory_token_win_fraction": 0.625139057636261, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 18.300384081485056, "scrambled_loss": 3.5228168964385986, "scrambled_loss_delta": 0.2904333770275116, "scrambled_token_accuracy_gain": 0.02892102301120758, "scrambled_token_win_fraction": 0.631813108921051, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 25.206064240449354, "memory_only_loss": 5.2324652671813965, "memory_only_loss_delta": 2.0000813007354736, "memory_only_token_accuracy_gain": 0.2091212421655655, "memory_only_token_win_fraction": 0.860956609249115, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 86.46757191741918, "no_evidence_loss": 3.3204710483551025, "no_evidence_loss_delta": 0.08808737993240356, "no_evidence_token_accuracy_gain": 0.006674082484096289, "no_evidence_token_win_fraction": 0.5884315967559814, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 8.43191389042493, "no_agreement_loss": 3.2450499534606934, "no_agreement_loss_delta": 0.012666125781834126, "no_agreement_token_accuracy_gain": -0.0033370412420481443, "no_agreement_token_win_fraction": 0.4660734236240387, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.2586248013627077, "no_orthogonal_loss": 3.235856771469116, "no_orthogonal_loss_delta": 0.003473081160336733, "no_orthogonal_token_accuracy_gain": -0.004449388012290001, "no_orthogonal_token_win_fraction": 0.5372636318206787, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.34670569901256254, "short_no_memory_loss_delta": 0.1437581330537796, "long_context_memory_gain": 0.058362752199172974, "normal_context_loss_gain": 0.1325027197599411, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.6334803104400635, "contribution_to_active": 0.7204315662384033, "mix_mean": 0.09267271310091019, "mix_low": 0.4039888381958008, "mix_high": 0.002435843227431178, "read_cosine": 0.504459798336029, "beta_mean": 0.2604246437549591, "nonfinite": 0.0, "weighted_memory_to_active": 0.7234601974487305, "mix_std": 0.08425040543079376, "beta_std": 0.1370527744293213, "beta_low": 1.0172526344831567e-05, "beta_high": 0.0, "orthogonal_to_active": 0.034817859530448914, "output_influence": 0.1457062065601349, "residual_influence": 0.22307038307189941, "state_active": 0.09669028967618942, "state_memory": 0.05569703131914139, "state_peak": 54.38564682006836, "growth_active": 1.2830246686935425, "growth_memory": 1.3451523780822754, "heads": [{"head": 0, "memory_to_active": 2.393406867980957, "contribution_to_active": 0.5575624704360962, "mix_mean": 0.0232708640396595, "mix_low": 0.4581327438354492, "mix_high": 0.0, "read_cosine": 0.3693208694458008, "beta_mean": 0.2194850742816925, "nonfinite": 0.0, "weighted_memory_to_active": 0.6330208778381348, "mix_std": 0.055768370628356934, "beta_std": 0.10084350407123566, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.015598944388329983, "evidence_weights": [-0.05223206430673599, -0.07469550520181656, 0.04483494907617569], "agreement_weight": -0.052737582474946976, "orthogonal_weight": -0.028892312198877335}, {"head": 1, "memory_to_active": 2.940669536590576, "contribution_to_active": 1.1380481719970703, "mix_mean": 0.019815858453512192, "mix_low": 0.43599510192871094, "mix_high": 0.0, "read_cosine": 0.5077853798866272, "beta_mean": 0.32785525918006897, "nonfinite": 0.0, "weighted_memory_to_active": 1.3468682765960693, "mix_std": 0.048521988093853, "beta_std": 0.1254035085439682, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.023126496002078056, "evidence_weights": [0.00275724777020514, -0.04250967875123024, 0.07200071960687637], "agreement_weight": 0.001722820452414453, "orthogonal_weight": -0.03700702637434006}, {"head": 2, "memory_to_active": 2.1586484909057617, "contribution_to_active": 0.4097909927368164, "mix_mean": 0.0160996001213789, "mix_low": 0.5852746963500977, "mix_high": 0.0, "read_cosine": 0.38502901792526245, "beta_mean": 0.2938231825828552, "nonfinite": 0.0, "weighted_memory_to_active": 0.45135581493377686, "mix_std": 0.034002259373664856, "beta_std": 0.18501918017864227, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.026636259630322456, "evidence_weights": [0.01885007694363594, 0.023697715252637863, -0.04340333491563797], "agreement_weight": -0.010232795029878616, "orthogonal_weight": -0.0956568568944931}, {"head": 3, "memory_to_active": 4.257792949676514, "contribution_to_active": 1.7826069593429565, "mix_mean": 0.06048860028386116, "mix_low": 0.16085147857666016, "mix_high": 0.0, "read_cosine": 0.47400057315826416, "beta_mean": 0.34240931272506714, "nonfinite": 0.0, "weighted_memory_to_active": 1.7540743350982666, "mix_std": 0.09117960184812546, "beta_std": 0.18423619866371155, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1250208020210266, "evidence_weights": [-0.12366282194852829, -0.14946144819259644, 0.09966649115085602], "agreement_weight": -0.03301440179347992, "orthogonal_weight": 0.09238655120134354}, {"head": 4, "memory_to_active": 3.894202470779419, "contribution_to_active": 0.09797979891300201, "mix_mean": 0.007692319806665182, "mix_low": 0.7730779647827148, "mix_high": 0.0, "read_cosine": 0.3299643397331238, "beta_mean": 0.30113929510116577, "nonfinite": 0.0, "weighted_memory_to_active": 0.08670886605978012, "mix_std": 0.011129656806588173, "beta_std": 0.17790216207504272, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.015403704717755318, "evidence_weights": [0.0620952844619751, 0.02153504639863968, -0.09349647164344788], "agreement_weight": -0.02551601640880108, "orthogonal_weight": 0.208037331700325}, {"head": 5, "memory_to_active": 0.1561606526374817, "contribution_to_active": 0.33660054206848145, "mix_mean": 0.4286690354347229, "mix_low": 0.010601043701171875, "mix_high": 0.014615058898925781, "read_cosine": 0.9606585502624512, "beta_mean": 0.07783568650484085, "nonfinite": 0.0, "weighted_memory_to_active": 0.06873289495706558, "mix_std": 0.2649005055427551, "beta_std": 0.048912033438682556, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.0031209425069391727, "evidence_weights": [-0.30830371379852295, -0.3071938753128052, 0.38691264390945435], "agreement_weight": 0.14054833352565765, "orthogonal_weight": -0.12152934819459915}]}, {"layer": 1, "memory_to_active": 2.534489154815674, "contribution_to_active": 0.3892054259777069, "mix_mean": 0.07078952342271805, "mix_low": 0.42359066009521484, "mix_high": 0.0003112157282885164, "read_cosine": 0.31632205843925476, "beta_mean": 0.2451697140932083, "nonfinite": 0.0, "weighted_memory_to_active": 0.3769766688346863, "mix_std": 0.09446574747562408, "beta_std": 0.1346898078918457, "beta_low": 0.168212890625, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.03735978156328201, "output_influence": 0.1434897482395172, "residual_influence": 0.03155628591775894, "state_active": 0.048146650195121765, "state_memory": 0.058095891028642654, "state_peak": 25.86094856262207, "growth_active": 1.126677393913269, "growth_memory": 1.5624935626983643, "heads": [{"head": 0, "memory_to_active": 0.21511749923229218, "contribution_to_active": 0.16889207065105438, "mix_mean": 0.09687203168869019, "mix_low": 0.22350311279296875, "mix_high": 3.814697265625e-06, "read_cosine": -0.07653114944696426, "beta_mean": 0.0010925156529992819, "nonfinite": 0.0, "weighted_memory_to_active": 0.053718678653240204, "mix_std": 0.15688316524028778, "beta_std": 0.0020875846967101097, "beta_low": 0.98992919921875, "beta_high": 0.0, "orthogonal_to_active": 0.022055458277463913, "evidence_weights": [-0.03647373616695404, -0.07454650849103928, -0.052368924021720886], "agreement_weight": -0.06039544939994812, "orthogonal_weight": -0.4598727226257324}, {"head": 1, "memory_to_active": 3.501182794570923, "contribution_to_active": 0.21271277964115143, "mix_mean": 0.028935682028532028, "mix_low": 0.3944072723388672, "mix_high": 1.9073486328125e-06, "read_cosine": 0.47283032536506653, "beta_mean": 0.40136393904685974, "nonfinite": 0.0, "weighted_memory_to_active": 0.22854475677013397, "mix_std": 0.054175201803445816, "beta_std": 0.20592810213565826, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0036498424597084522, "evidence_weights": [-0.039538443088531494, -0.028864454478025436, 0.10219133645296097], "agreement_weight": -0.016057072207331657, "orthogonal_weight": 0.021997729316353798}, {"head": 2, "memory_to_active": 0.9009394645690918, "contribution_to_active": 0.4393590986728668, "mix_mean": 0.24108776450157166, "mix_low": 0.018520355224609375, "mix_high": 0.0018558502197265625, "read_cosine": 0.7702916860580444, "beta_mean": 0.14192448556423187, "nonfinite": 0.0, "weighted_memory_to_active": 0.6013466119766235, "mix_std": 0.22438940405845642, "beta_std": 0.06525001674890518, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03958955407142639, "evidence_weights": [-0.2106533646583557, -0.33111560344696045, 0.21706609427928925], "agreement_weight": -0.015639223158359528, "orthogonal_weight": 0.12124088406562805}, {"head": 3, "memory_to_active": 3.3321893215179443, "contribution_to_active": 0.06928011775016785, "mix_mean": 0.00803470704704523, "mix_low": 0.8056974411010742, "mix_high": 0.0, "read_cosine": 0.1616973727941513, "beta_mean": 0.5030248165130615, "nonfinite": 0.0, "weighted_memory_to_active": 0.07863914966583252, "mix_std": 0.021334679797291756, "beta_std": 0.2529762387275696, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0090071065351367, "evidence_weights": [0.027314020320773125, 0.02276424877345562, -0.01360592246055603], "agreement_weight": -0.020027535036206245, "orthogonal_weight": -0.12099019438028336}, {"head": 4, "memory_to_active": 3.2871694564819336, "contribution_to_active": 1.3894870281219482, "mix_mean": 0.042174410074949265, "mix_low": 0.27669525146484375, "mix_high": 1.9073486328125e-06, "read_cosine": 0.41295361518859863, "beta_mean": 0.1755538135766983, "nonfinite": 0.0, "weighted_memory_to_active": 1.2466367483139038, "mix_std": 0.08126572519540787, "beta_std": 0.08869219571352005, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14906509220600128, "evidence_weights": [-0.03646527975797653, -0.03087162971496582, 0.1847696602344513], "agreement_weight": -0.0430055595934391, "orthogonal_weight": 0.13096146285533905}, {"head": 5, "memory_to_active": 3.9703359603881836, "contribution_to_active": 0.05550132691860199, "mix_mean": 0.00763250607997179, "mix_low": 0.8227205276489258, "mix_high": 3.814697265625e-06, "read_cosine": 0.15669035911560059, "beta_mean": 0.24805864691734314, "nonfinite": 0.0, "weighted_memory_to_active": 0.05297408252954483, "mix_std": 0.028746316209435463, "beta_std": 0.19320470094680786, "beta_low": 0.019287109375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.000791632104665041, "evidence_weights": [0.09917629510164261, 0.056852247565984726, 0.23479802906513214], "agreement_weight": -0.11015180498361588, "orthogonal_weight": 0.015530236065387726}]}, {"layer": 2, "memory_to_active": 1.3646711111068726, "contribution_to_active": 0.2650045156478882, "mix_mean": 0.18242147564888, "mix_low": 0.29921770095825195, "mix_high": 0.0010913213482126594, "read_cosine": 0.3646344244480133, "beta_mean": 0.1321059763431549, "nonfinite": 0.0, "weighted_memory_to_active": 0.09047026932239532, "mix_std": 0.16357094049453735, "beta_std": 0.08839510381221771, "beta_low": 0.5370890498161316, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.011572051793336868, "output_influence": 0.23087090253829956, "residual_influence": 0.0490950383245945, "state_active": 0.03949125111103058, "state_memory": 0.03055751509964466, "state_peak": 42.019798278808594, "growth_active": 0.9917044043540955, "growth_memory": 1.506781816482544, "heads": [{"head": 0, "memory_to_active": 0.1587163209915161, "contribution_to_active": 0.06983932852745056, "mix_mean": 0.10838115215301514, "mix_low": 0.19196224212646484, "mix_high": 0.0, "read_cosine": 0.2699897885322571, "beta_mean": 0.001642210758291185, "nonfinite": 0.0, "weighted_memory_to_active": 0.019980013370513916, "mix_std": 0.1626889407634735, "beta_std": 0.0024396555963903666, "beta_low": 0.98388671875, "beta_high": 0.0, "orthogonal_to_active": 0.0020546086598187685, "evidence_weights": [-0.04702916741371155, -0.03751879930496216, 0.06979084759950638], "agreement_weight": -0.06798243522644043, "orthogonal_weight": -0.11299536377191544}, {"head": 1, "memory_to_active": 3.9190001487731934, "contribution_to_active": 0.051022570580244064, "mix_mean": 0.005435734987258911, "mix_low": 0.8786869049072266, "mix_high": 9.5367431640625e-07, "read_cosine": 0.1579640954732895, "beta_mean": 0.41699135303497314, "nonfinite": 0.0, "weighted_memory_to_active": 0.04937734827399254, "mix_std": 0.010395478457212448, "beta_std": 0.1859334260225296, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008580032736063004, "evidence_weights": [0.08970089256763458, 0.09649886190891266, 0.09404444694519043], "agreement_weight": 0.008577113971114159, "orthogonal_weight": 0.20517492294311523}, {"head": 2, "memory_to_active": 0.09816423058509827, "contribution_to_active": 0.37826693058013916, "mix_mean": 0.3289877772331238, "mix_low": 0.006485939025878906, "mix_high": 0.0012845993041992188, "read_cosine": 0.40034472942352295, "beta_mean": 0.00017743578064255416, "nonfinite": 0.0, "weighted_memory_to_active": 0.03855960816144943, "mix_std": 0.26708513498306274, "beta_std": 0.0003831147914752364, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.005024583078920841, "evidence_weights": [-0.17041747272014618, -0.1874062567949295, 0.29161396622657776], "agreement_weight": 0.028173834085464478, "orthogonal_weight": -0.18198581039905548}, {"head": 3, "memory_to_active": 0.36107397079467773, "contribution_to_active": 0.37402641773223877, "mix_mean": 0.19549289345741272, "mix_low": 0.023305892944335938, "mix_high": 0.002368927001953125, "read_cosine": 0.6495325565338135, "beta_mean": 0.06222648173570633, "nonfinite": 0.0, "weighted_memory_to_active": 0.13129453361034393, "mix_std": 0.2129736691713333, "beta_std": 0.10186555236577988, "beta_low": 0.21502685546875, "beta_high": 0.0, "orthogonal_to_active": 0.04559754207730293, "evidence_weights": [-0.08042890578508377, -0.14162103831768036, 0.4259714186191559], "agreement_weight": 0.07921683043241501, "orthogonal_weight": -0.5431373715400696}, {"head": 4, "memory_to_active": 3.6103076934814453, "contribution_to_active": 0.2564789950847626, "mix_mean": 0.01316255982965231, "mix_low": 0.6921195983886719, "mix_high": 2.193450927734375e-05, "read_cosine": 0.19206556677818298, "beta_mean": 0.31151720881462097, "nonfinite": 0.0, "weighted_memory_to_active": 0.28217682242393494, "mix_std": 0.043940283358097076, "beta_std": 0.23958396911621094, "beta_low": 0.02374267578125, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.007045682519674301, "evidence_weights": [0.03918907046318054, 0.03222412243485451, 0.1358933448791504], "agreement_weight": 0.019413327798247337, "orthogonal_weight": -0.027836039662361145}, {"head": 5, "memory_to_active": 0.04076416790485382, "contribution_to_active": 0.46039289236068726, "mix_mean": 0.44306862354278564, "mix_low": 0.0027456283569335938, "mix_high": 0.0028715133666992188, "read_cosine": 0.5179097652435303, "beta_mean": 8.120488928398117e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.021433234214782715, "mix_std": 0.28434211015701294, "beta_std": 0.0001649022742640227, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011298657627776265, "evidence_weights": [-0.22948414087295532, -0.21888762712478638, 0.4593823254108429], "agreement_weight": 0.015339535661041737, "orthogonal_weight": 0.1018332913517952}]}, {"layer": 3, "memory_to_active": 1.4765647649765015, "contribution_to_active": 0.2902509272098541, "mix_mean": 0.11805389821529388, "mix_low": 0.32199037075042725, "mix_high": 0.0010375977726653218, "read_cosine": 0.29598087072372437, "beta_mean": 0.12607738375663757, "nonfinite": 0.0, "weighted_memory_to_active": 0.15395963191986084, "mix_std": 0.15421125292778015, "beta_std": 0.09310899674892426, "beta_low": 0.3680013120174408, "beta_high": 0.0, "orthogonal_to_active": 0.022438129410147667, "output_influence": 0.1981709748506546, "residual_influence": 0.033140234649181366, "state_active": 0.03680456429719925, "state_memory": 0.0309239961206913, "state_peak": 33.62323760986328, "growth_active": 0.9834862947463989, "growth_memory": 1.7250525951385498, "heads": [{"head": 0, "memory_to_active": 1.1335967779159546, "contribution_to_active": 0.22177107632160187, "mix_mean": 0.08858723938465118, "mix_low": 0.33042049407958984, "mix_high": 8.7738037109375e-05, "read_cosine": 0.1949530690908432, "beta_mean": 0.021019887179136276, "nonfinite": 0.0, "weighted_memory_to_active": 0.1629519760608673, "mix_std": 0.16457073390483856, "beta_std": 0.0168314091861248, "beta_low": 0.1995849609375, "beta_high": 0.0, "orthogonal_to_active": 0.019236896187067032, "evidence_weights": [0.014202135615050793, -0.04732953757047653, -0.037536732852458954], "agreement_weight": -0.04745697230100632, "orthogonal_weight": -0.11653243005275726}, {"head": 1, "memory_to_active": 0.28618836402893066, "contribution_to_active": 0.22107189893722534, "mix_mean": 0.1372171938419342, "mix_low": 0.15647411346435547, "mix_high": 0.0003757476806640625, "read_cosine": 0.268460750579834, "beta_mean": 0.0011857240460813046, "nonfinite": 0.0, "weighted_memory_to_active": 0.031060153618454933, "mix_std": 0.1979849487543106, "beta_std": 0.002217439003288746, "beta_low": 0.9893798828125, "beta_high": 0.0, "orthogonal_to_active": 0.002074160845950246, "evidence_weights": [-0.08536159247159958, -0.0639159232378006, 0.2002636045217514], "agreement_weight": -0.027103889733552933, "orthogonal_weight": -0.06647469848394394}, {"head": 2, "memory_to_active": 3.4870429039001465, "contribution_to_active": 0.2957583963871002, "mix_mean": 0.033149365335702896, "mix_low": 0.4299745559692383, "mix_high": 0.00012969970703125, "read_cosine": 0.2004806399345398, "beta_mean": 0.30508682131767273, "nonfinite": 0.0, "weighted_memory_to_active": 0.406129390001297, "mix_std": 0.08497029542922974, "beta_std": 0.18275289237499237, "beta_low": 0.00286865234375, "beta_high": 0.0, "orthogonal_to_active": 0.04654708504676819, "evidence_weights": [-0.031731873750686646, 0.006450667977333069, 0.11069268733263016], "agreement_weight": -0.023146264255046844, "orthogonal_weight": -0.14896260201931}, {"head": 3, "memory_to_active": 0.04602551460266113, "contribution_to_active": 0.4124033749103546, "mix_mean": 0.3318786025047302, "mix_low": 0.03733062744140625, "mix_high": 0.0021610260009765625, "read_cosine": 0.3204377293586731, "beta_mean": 0.0001489909045631066, "nonfinite": 0.0, "weighted_memory_to_active": 0.02104393020272255, "mix_std": 0.28168317675590515, "beta_std": 0.0003234358155168593, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004151457455009222, "evidence_weights": [-0.16841043531894684, -0.18436574935913086, 0.671156644821167], "agreement_weight": 0.3888434171676636, "orthogonal_weight": -0.2768303453922272}, {"head": 4, "memory_to_active": 3.363875389099121, "contribution_to_active": 0.04569161310791969, "mix_mean": 0.006788333412259817, "mix_low": 0.845271110534668, "mix_high": 0.0, "read_cosine": 0.24199584126472473, "beta_mean": 0.24871301651000977, "nonfinite": 0.0, "weighted_memory_to_active": 0.04951648414134979, "mix_std": 0.01660165563225746, "beta_std": 0.18258792161941528, "beta_low": 0.00128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.009540580213069916, "evidence_weights": [0.1329282820224762, 0.11665628850460052, -0.13529162108898163], "agreement_weight": -0.08875278383493423, "orthogonal_weight": -0.2102358341217041}, {"head": 5, "memory_to_active": 0.5426592826843262, "contribution_to_active": 0.5448092222213745, "mix_mean": 0.11070269346237183, "mix_low": 0.13247108459472656, "mix_high": 0.00347137451171875, "read_cosine": 0.5495572686195374, "beta_mean": 0.18030983209609985, "nonfinite": 0.0, "weighted_memory_to_active": 0.25305578112602234, "mix_std": 0.17945674061775208, "beta_std": 0.17394085228443146, "beta_low": 0.01495361328125, "beta_high": 0.0, "orthogonal_to_active": 0.05307859182357788, "evidence_weights": [-0.03286420926451683, -0.12066857516765594, 0.039713457226753235], "agreement_weight": 0.15194499492645264, "orthogonal_weight": -0.3117453455924988}]}, {"layer": 4, "memory_to_active": 1.467272162437439, "contribution_to_active": 0.308002233505249, "mix_mean": 0.21543662250041962, "mix_low": 0.29080474376678467, "mix_high": 0.0008390744915232062, "read_cosine": 0.4032590985298157, "beta_mean": 0.15331436693668365, "nonfinite": 0.0, "weighted_memory_to_active": 0.18440626561641693, "mix_std": 0.14588703215122223, "beta_std": 0.11613084375858307, "beta_low": 0.2137349545955658, "beta_high": 0.0, "orthogonal_to_active": 0.03630518540740013, "output_influence": 0.19420869648456573, "residual_influence": 0.040784694254398346, "state_active": 0.04768730327486992, "state_memory": 0.04053555428981781, "state_peak": 46.40519714355469, "growth_active": 1.09119713306427, "growth_memory": 1.6426799297332764, "heads": [{"head": 0, "memory_to_active": 0.4786617159843445, "contribution_to_active": 0.4672442376613617, "mix_mean": 0.3957066833972931, "mix_low": 0.004839897155761719, "mix_high": 0.0007390975952148438, "read_cosine": 0.626186728477478, "beta_mean": 0.12007841467857361, "nonfinite": 0.0, "weighted_memory_to_active": 0.2875998020172119, "mix_std": 0.2736387848854065, "beta_std": 0.14208170771598816, "beta_low": 0.10430908203125, "beta_high": 0.0, "orthogonal_to_active": 0.00547521049156785, "evidence_weights": [-0.18497268855571747, -0.22676266729831696, 0.37987974286079407], "agreement_weight": 0.06051330268383026, "orthogonal_weight": -0.02753414772450924}, {"head": 1, "memory_to_active": 0.5424532294273376, "contribution_to_active": 0.26236072182655334, "mix_mean": 0.2746496796607971, "mix_low": 0.017923355102539062, "mix_high": 0.0001201629638671875, "read_cosine": 0.5189269781112671, "beta_mean": 0.04040171578526497, "nonfinite": 0.0, "weighted_memory_to_active": 0.19464656710624695, "mix_std": 0.23539650440216064, "beta_std": 0.05174160748720169, "beta_low": 0.14111328125, "beta_high": 0.0, "orthogonal_to_active": 0.08283901959657669, "evidence_weights": [-0.16429521143436432, -0.15405841171741486, 0.5442070960998535], "agreement_weight": 0.04378927871584892, "orthogonal_weight": -0.5124992728233337}, {"head": 2, "memory_to_active": 2.8595762252807617, "contribution_to_active": 0.09382638335227966, "mix_mean": 0.01071627251803875, "mix_low": 0.7238063812255859, "mix_high": 0.0, "read_cosine": 0.29913532733917236, "beta_mean": 0.1947501301765442, "nonfinite": 0.0, "weighted_memory_to_active": 0.11642282456159592, "mix_std": 0.019098352640867233, "beta_std": 0.13258957862854004, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003988121170550585, "evidence_weights": [0.04003416746854782, 0.024464784190058708, -0.06165449321269989], "agreement_weight": 0.03698601946234703, "orthogonal_weight": -0.0050056325271725655}, {"head": 3, "memory_to_active": 2.1574344635009766, "contribution_to_active": 0.10164038836956024, "mix_mean": 0.025456951931118965, "mix_low": 0.41700267791748047, "mix_high": 0.0, "read_cosine": 0.21436992287635803, "beta_mean": 0.23824180662631989, "nonfinite": 0.0, "weighted_memory_to_active": 0.15812355279922485, "mix_std": 0.040975891053676605, "beta_std": 0.18529516458511353, "beta_low": 0.0369873046875, "beta_high": 0.0, "orthogonal_to_active": 0.02416028082370758, "evidence_weights": [-0.014440744183957577, 0.03146076574921608, -0.1189209446310997], "agreement_weight": 0.013432911597192287, "orthogonal_weight": -0.23015861213207245}, {"head": 4, "memory_to_active": 0.012783680111169815, "contribution_to_active": 0.5559942126274109, "mix_mean": 0.5691297054290771, "mix_low": 0.0014963150024414062, "mix_high": 0.0041561126708984375, "read_cosine": 0.3329155743122101, "beta_mean": 3.265698251198046e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008312800899147987, "mix_std": 0.26703527569770813, "beta_std": 9.285384294344112e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008688728557899594, "evidence_weights": [-0.26410922408103943, -0.29283130168914795, 0.6315476298332214], "agreement_weight": 0.018126318231225014, "orthogonal_weight": -0.12755735218524933}, {"head": 5, "memory_to_active": 2.75272274017334, "contribution_to_active": 0.36694738268852234, "mix_mean": 0.016960397362709045, "mix_low": 0.5797595977783203, "mix_high": 1.9073486328125e-05, "read_cosine": 0.4280198812484741, "beta_mean": 0.3263814449310303, "nonfinite": 0.0, "weighted_memory_to_active": 0.3413320481777191, "mix_std": 0.03917736932635307, "beta_std": 0.18498417735099792, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10408890247344971, "evidence_weights": [0.024541787803173065, -0.0267208069562912, -0.030637800693511963], "agreement_weight": -0.042066678404808044, "orthogonal_weight": 0.5137339234352112}]}, {"layer": 5, "memory_to_active": 0.7954073548316956, "contribution_to_active": 0.4208102226257324, "mix_mean": 0.3435325622558594, "mix_low": 0.12491099536418915, "mix_high": 0.007081508636474609, "read_cosine": 0.26397424936294556, "beta_mean": 0.0672919750213623, "nonfinite": 0.0, "weighted_memory_to_active": 0.18348751962184906, "mix_std": 0.23378613591194153, "beta_std": 0.07732832431793213, "beta_low": 0.5256754755973816, "beta_high": 0.0, "orthogonal_to_active": 0.043087683618068695, "output_influence": 0.39064502716064453, "residual_influence": 0.07692337781190872, "state_active": 0.06604740023612976, "state_memory": 0.02591157704591751, "state_peak": 58.32157516479492, "growth_active": 1.1011749505996704, "growth_memory": 1.4603543281555176, "heads": [{"head": 0, "memory_to_active": 1.8714373111724854, "contribution_to_active": 0.30814817547798157, "mix_mean": 0.0708320289850235, "mix_low": 0.20969772338867188, "mix_high": 0.0, "read_cosine": -0.1485399454832077, "beta_mean": 0.11542682349681854, "nonfinite": 0.0, "weighted_memory_to_active": 0.32998907566070557, "mix_std": 0.11105219274759293, "beta_std": 0.12282606214284897, "beta_low": 0.02020263671875, "beta_high": 0.0, "orthogonal_to_active": 0.06786409020423889, "evidence_weights": [-0.023274140432476997, -0.04936658963561058, -0.28513336181640625], "agreement_weight": -0.02655884623527527, "orthogonal_weight": -0.21081720292568207}, {"head": 1, "memory_to_active": 0.03659849986433983, "contribution_to_active": 0.4015466570854187, "mix_mean": 0.42414799332618713, "mix_low": 0.01624298095703125, "mix_high": 0.0068225860595703125, "read_cosine": 0.17103791236877441, "beta_mean": 9.245732508134097e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013723727315664291, "mix_std": 0.3149617314338684, "beta_std": 0.00018291370361112058, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010457206517457962, "evidence_weights": [-0.19432787597179413, -0.1903281807899475, 0.3812187612056732], "agreement_weight": 0.17578832805156708, "orthogonal_weight": -0.7900141477584839}, {"head": 2, "memory_to_active": 0.18391573429107666, "contribution_to_active": 0.4725559651851654, "mix_mean": 0.5165815353393555, "mix_low": 0.00145721435546875, "mix_high": 0.006312370300292969, "read_cosine": 0.5695538520812988, "beta_mean": 0.009656645357608795, "nonfinite": 0.0, "weighted_memory_to_active": 0.12078667432069778, "mix_std": 0.27454352378845215, "beta_std": 0.009334816597402096, "beta_low": 0.68621826171875, "beta_high": 0.0, "orthogonal_to_active": 0.014213604852557182, "evidence_weights": [-0.2710215151309967, -0.27269625663757324, 0.5517682433128357], "agreement_weight": 0.03235476836562157, "orthogonal_weight": -0.16941630840301514}, {"head": 3, "memory_to_active": 2.092318296432495, "contribution_to_active": 0.15397024154663086, "mix_mean": 0.04353933036327362, "mix_low": 0.37542247772216797, "mix_high": 0.0, "read_cosine": 0.1758253276348114, "beta_mean": 0.05904395878314972, "nonfinite": 0.0, "weighted_memory_to_active": 0.23775899410247803, "mix_std": 0.08997540175914764, "beta_std": 0.09602007269859314, "beta_low": 0.25347900390625, "beta_high": 0.0, "orthogonal_to_active": 0.0784296914935112, "evidence_weights": [0.010277689434587955, -0.03146509826183319, 0.09081348031759262], "agreement_weight": 0.0738958790898323, "orthogonal_weight": -0.40972474217414856}, {"head": 4, "memory_to_active": 0.5480539202690125, "contribution_to_active": 0.5880640745162964, "mix_mean": 0.3356700837612152, "mix_low": 0.1416492462158203, "mix_high": 0.015830039978027344, "read_cosine": 0.5920357704162598, "beta_mean": 0.2193758338689804, "nonfinite": 0.0, "weighted_memory_to_active": 0.3737580180168152, "mix_std": 0.34125781059265137, "beta_std": 0.2351638376712799, "beta_low": 0.1943359375, "beta_high": 0.0, "orthogonal_to_active": 0.07929610460996628, "evidence_weights": [-0.1614231914281845, -0.25635895133018494, 0.15636983513832092], "agreement_weight": 0.051928479224443436, "orthogonal_weight": 0.31512054800987244}, {"head": 5, "memory_to_active": 0.04012003168463707, "contribution_to_active": 0.6005762815475464, "mix_mean": 0.670424222946167, "mix_low": 0.004996299743652344, "mix_high": 0.013524055480957031, "read_cosine": 0.22393253445625305, "beta_mean": 0.00015608887770213187, "nonfinite": 0.0, "weighted_memory_to_active": 0.024908607825636864, "mix_std": 0.2709261476993561, "beta_std": 0.0004422407364472747, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.008265368640422821, "evidence_weights": [-0.296123743057251, -0.2873574197292328, 0.7234454154968262], "agreement_weight": 0.26044896245002747, "orthogonal_weight": -0.3638867437839508}]}, {"layer": 6, "memory_to_active": 0.5438645482063293, "contribution_to_active": 0.5033142566680908, "mix_mean": 0.42319491505622864, "mix_low": 0.10545079410076141, "mix_high": 0.01736052893102169, "read_cosine": 0.34400832653045654, "beta_mean": 0.084907166659832, "nonfinite": 0.0, "weighted_memory_to_active": 0.1787930279970169, "mix_std": 0.2666279375553131, "beta_std": 0.09862062335014343, "beta_low": 0.4432169795036316, "beta_high": 0.0, "orthogonal_to_active": 0.03602459654211998, "output_influence": 0.3900909423828125, "residual_influence": 0.10531814396381378, "state_active": 0.06819861382246017, "state_memory": 0.01658732444047928, "state_peak": 44.24820327758789, "growth_active": 1.243667483329773, "growth_memory": 1.3671655654907227, "heads": [{"head": 0, "memory_to_active": 0.2293994426727295, "contribution_to_active": 0.46091148257255554, "mix_mean": 0.48024433851242065, "mix_low": 0.00185394287109375, "mix_high": 0.0013895034790039062, "read_cosine": 0.3554134964942932, "beta_mean": 0.02991514280438423, "nonfinite": 0.0, "weighted_memory_to_active": 0.1664830893278122, "mix_std": 0.27881041169166565, "beta_std": 0.06349365413188934, "beta_low": 0.32012939453125, "beta_high": 0.0, "orthogonal_to_active": 0.09279756247997284, "evidence_weights": [-0.22826413810253143, -0.22459910809993744, 0.23492242395877838], "agreement_weight": 0.09399902075529099, "orthogonal_weight": -0.7514127492904663}, {"head": 1, "memory_to_active": 0.009454330429434776, "contribution_to_active": 0.3908007740974426, "mix_mean": 0.36530601978302, "mix_low": 0.0083160400390625, "mix_high": 0.007833480834960938, "read_cosine": 0.5339608192443848, "beta_mean": 2.1020312487962656e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002911399817094207, "mix_std": 0.2888607084751129, "beta_std": 5.494036304298788e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007067675469443202, "evidence_weights": [-0.20728738605976105, -0.2086348831653595, 0.3759305477142334], "agreement_weight": -0.011382995173335075, "orthogonal_weight": -0.3094935417175293}, {"head": 2, "memory_to_active": 2.0683348178863525, "contribution_to_active": 0.06621444970369339, "mix_mean": 0.018720030784606934, "mix_low": 0.5972614288330078, "mix_high": 0.0, "read_cosine": 0.19308264553546906, "beta_mean": 0.12176378071308136, "nonfinite": 0.0, "weighted_memory_to_active": 0.08764249086380005, "mix_std": 0.042300835251808167, "beta_std": 0.10731050372123718, "beta_low": 0.00128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.01977407932281494, "evidence_weights": [0.08593656867742538, 0.05824195221066475, 0.011601235717535019], "agreement_weight": 0.040794722735881805, "orthogonal_weight": -0.24746553599834442}, {"head": 3, "memory_to_active": 0.19969426095485687, "contribution_to_active": 0.6186725497245789, "mix_mean": 0.5842586755752563, "mix_low": 0.012338638305664062, "mix_high": 0.035170555114746094, "read_cosine": 0.5602113008499146, "beta_mean": 0.22069431841373444, "nonfinite": 0.0, "weighted_memory_to_active": 0.13063599169254303, "mix_std": 0.35380375385284424, "beta_std": 0.2521553337574005, "beta_low": 0.167724609375, "beta_high": 0.0, "orthogonal_to_active": 0.040148746222257614, "evidence_weights": [-0.15417589247226715, -0.29927897453308105, 0.2271367609500885], "agreement_weight": -0.025154367089271545, "orthogonal_weight": 0.3759281635284424}, {"head": 4, "memory_to_active": 0.6925168037414551, "contribution_to_active": 0.9786746501922607, "mix_mean": 0.6320472955703735, "mix_low": 0.005574226379394531, "mix_high": 0.059173583984375, "read_cosine": 0.15058670938014984, "beta_mean": 0.13660326600074768, "nonfinite": 0.0, "weighted_memory_to_active": 0.6498339176177979, "mix_std": 0.33353573083877563, "beta_std": 0.1680314540863037, "beta_low": 0.17047119140625, "beta_high": 0.0, "orthogonal_to_active": 0.028669757768511772, "evidence_weights": [-0.23075957596302032, -0.47639551758766174, -0.0594005323946476], "agreement_weight": -0.009759709239006042, "orthogonal_weight": 0.05459935590624809}, {"head": 5, "memory_to_active": 0.06378740072250366, "contribution_to_active": 0.5046112537384033, "mix_mean": 0.45859313011169434, "mix_low": 0.0073604583740234375, "mix_high": 0.0005960464477539062, "read_cosine": 0.27079498767852783, "beta_mean": 0.0004454952431842685, "nonfinite": 0.0, "weighted_memory_to_active": 0.03525122255086899, "mix_std": 0.30245622992515564, "beta_std": 0.0006778484093956649, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.034050650894641876, "evidence_weights": [-0.19255907833576202, -0.22904467582702637, 0.09957236796617508], "agreement_weight": 0.07636500149965286, "orthogonal_weight": -1.0314393043518066}]}, {"layer": 7, "memory_to_active": 2.370539665222168, "contribution_to_active": 0.16441704332828522, "mix_mean": 0.02373526245355606, "mix_low": 0.5699254870414734, "mix_high": 4.291534423828125e-06, "read_cosine": 0.21191319823265076, "beta_mean": 0.16265764832496643, "nonfinite": 0.0, "weighted_memory_to_active": 0.1661645621061325, "mix_std": 0.0435091108083725, "beta_std": 0.1171794980764389, "beta_low": 0.013763428665697575, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.02560879848897457, "output_influence": 0.18923403322696686, "residual_influence": 0.03882695734500885, "state_active": 0.02044949121773243, "state_memory": 0.039003290235996246, "state_peak": 17.92642593383789, "growth_active": 1.007502794265747, "growth_memory": 1.494694709777832, "heads": [{"head": 0, "memory_to_active": 2.1330559253692627, "contribution_to_active": 0.33727890253067017, "mix_mean": 0.02566113881766796, "mix_low": 0.5095281600952148, "mix_high": 2.574920654296875e-05, "read_cosine": 0.37181034684181213, "beta_mean": 0.1509815901517868, "nonfinite": 0.0, "weighted_memory_to_active": 0.2957857549190521, "mix_std": 0.05206982046365738, "beta_std": 0.11641257256269455, "beta_low": 0.00128173828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.06038864701986313, "evidence_weights": [-0.014772027730941772, -0.0733766108751297, 0.10624068230390549], "agreement_weight": -0.03375556319952011, "orthogonal_weight": 0.26639828085899353}, {"head": 1, "memory_to_active": 3.199878454208374, "contribution_to_active": 0.24916419386863708, "mix_mean": 0.04511214792728424, "mix_low": 0.32932376861572266, "mix_high": 0.0, "read_cosine": -0.014825010672211647, "beta_mean": 0.27650126814842224, "nonfinite": 0.0, "weighted_memory_to_active": 0.3079535961151123, "mix_std": 0.06637662649154663, "beta_std": 0.13577860593795776, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06332391500473022, "evidence_weights": [-0.09011903405189514, -0.08026690036058426, 0.004285193979740143], "agreement_weight": 0.005924962926656008, "orthogonal_weight": -0.2007976770401001}, {"head": 2, "memory_to_active": 1.6471751928329468, "contribution_to_active": 0.2528703510761261, "mix_mean": 0.04130513593554497, "mix_low": 0.38939762115478516, "mix_high": 0.0, "read_cosine": 0.1743287742137909, "beta_mean": 0.18121203780174255, "nonfinite": 0.0, "weighted_memory_to_active": 0.22828726470470428, "mix_std": 0.08429966866970062, "beta_std": 0.106532983481884, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.005499267019331455, "evidence_weights": [-0.037379588931798935, -0.10132629424333572, 0.18804201483726501], "agreement_weight": 0.032377034425735474, "orthogonal_weight": -0.025159412994980812}, {"head": 3, "memory_to_active": 2.9404428005218506, "contribution_to_active": 0.04387880861759186, "mix_mean": 0.009513220749795437, "mix_low": 0.7213544845581055, "mix_high": 0.0, "read_cosine": 0.14517046511173248, "beta_mean": 0.12907806038856506, "nonfinite": 0.0, "weighted_memory_to_active": 0.060293909162282944, "mix_std": 0.017078306525945663, "beta_std": 0.11237642168998718, "beta_low": 0.01129150390625, "beta_high": 0.0, "orthogonal_to_active": 0.018129704520106316, "evidence_weights": [0.09691857546567917, 0.08303700387477875, 0.08249005675315857], "agreement_weight": 0.04034854471683502, "orthogonal_weight": -0.31312474608421326}, {"head": 4, "memory_to_active": 1.5967711210250854, "contribution_to_active": 0.04306575655937195, "mix_mean": 0.010217370465397835, "mix_low": 0.7411298751831055, "mix_high": 0.0, "read_cosine": 0.3176421523094177, "beta_mean": 0.1331043839454651, "nonfinite": 0.0, "weighted_memory_to_active": 0.037749189883470535, "mix_std": 0.022545073181390762, "beta_std": 0.12530715763568878, "beta_low": 0.043701171875, "beta_high": 0.0, "orthogonal_to_active": 0.0019820760935544968, "evidence_weights": [0.07140316814184189, 0.04091455787420273, 0.008748042397201061], "agreement_weight": 0.05384877324104309, "orthogonal_weight": -0.06161937117576599}, {"head": 5, "memory_to_active": 2.7059130668640137, "contribution_to_active": 0.06024421751499176, "mix_mean": 0.010602553375065327, "mix_low": 0.7288188934326172, "mix_high": 0.0, "read_cosine": 0.2773524522781372, "beta_mean": 0.10506856441497803, "nonfinite": 0.0, "weighted_memory_to_active": 0.0669175535440445, "mix_std": 0.01868515834212303, "beta_std": 0.10666923224925995, "beta_low": 0.026123046875, "beta_high": 0.0, "orthogonal_to_active": 0.004329173360019922, "evidence_weights": [0.056004926562309265, 0.08031018823385239, -0.166482612490654], "agreement_weight": -0.012290306389331818, "orthogonal_weight": -0.07770641893148422}]}, {"layer": 8, "memory_to_active": 0.8760302662849426, "contribution_to_active": 0.3603169322013855, "mix_mean": 0.23658691346645355, "mix_low": 0.09297371655702591, "mix_high": 0.0012470881920307875, "read_cosine": 0.5226038694381714, "beta_mean": 0.06591790169477463, "nonfinite": 0.0, "weighted_memory_to_active": 0.19372758269309998, "mix_std": 0.18657426536083221, "beta_std": 0.05859141796827316, "beta_low": 0.2322896420955658, "beta_high": 0.0, "orthogonal_to_active": 0.04688810557126999, "output_influence": 0.27209022641181946, "residual_influence": 0.06275954842567444, "state_active": 0.05573752522468567, "state_memory": 0.025274701416492462, "state_peak": 29.266048431396484, "growth_active": 0.9810962677001953, "growth_memory": 1.5754172801971436, "heads": [{"head": 0, "memory_to_active": 0.09272205829620361, "contribution_to_active": 0.4675581455230713, "mix_mean": 0.4773862361907959, "mix_low": 0.00022792816162109375, "mix_high": 0.0001049041748046875, "read_cosine": 0.7378143072128296, "beta_mean": 0.0054219383746385574, "nonfinite": 0.0, "weighted_memory_to_active": 0.04807073250412941, "mix_std": 0.23323015868663788, "beta_std": 0.008744004182517529, "beta_low": 0.86077880859375, "beta_high": 0.0, "orthogonal_to_active": 0.01956600695848465, "evidence_weights": [-0.25131213665008545, -0.29448962211608887, 0.5566729307174683], "agreement_weight": 0.06902271509170532, "orthogonal_weight": -0.6480547189712524}, {"head": 1, "memory_to_active": 0.7103850841522217, "contribution_to_active": 0.31045377254486084, "mix_mean": 0.2779086232185364, "mix_low": 0.008016586303710938, "mix_high": 3.147125244140625e-05, "read_cosine": 0.5800607204437256, "beta_mean": 0.06685326993465424, "nonfinite": 0.0, "weighted_memory_to_active": 0.23936055600643158, "mix_std": 0.19429069757461548, "beta_std": 0.05113194137811661, "beta_low": 0.04852294921875, "beta_high": 0.0, "orthogonal_to_active": 0.011571143753826618, "evidence_weights": [-0.16568481922149658, -0.1732170134782791, 0.30198103189468384], "agreement_weight": 0.032178498804569244, "orthogonal_weight": 0.06265472620725632}, {"head": 2, "memory_to_active": 0.6807228326797485, "contribution_to_active": 0.18521633744239807, "mix_mean": 0.1396564543247223, "mix_low": 0.07799148559570312, "mix_high": 0.0, "read_cosine": 0.2831282317638397, "beta_mean": 0.03174186497926712, "nonfinite": 0.0, "weighted_memory_to_active": 0.15725374221801758, "mix_std": 0.16597560048103333, "beta_std": 0.030141500756144524, "beta_low": 0.16375732421875, "beta_high": 0.0, "orthogonal_to_active": 0.07949614524841309, "evidence_weights": [-0.06988929957151413, -0.1535482108592987, 0.13135378062725067], "agreement_weight": 0.06304177641868591, "orthogonal_weight": -0.5609098672866821}, {"head": 3, "memory_to_active": 0.5441854000091553, "contribution_to_active": 0.20107702910900116, "mix_mean": 0.16367940604686737, "mix_low": 0.02956390380859375, "mix_high": 0.0, "read_cosine": 0.6798041462898254, "beta_mean": 0.04267210513353348, "nonfinite": 0.0, "weighted_memory_to_active": 0.1902245730161667, "mix_std": 0.14683666825294495, "beta_std": 0.03687840327620506, "beta_low": 0.0670166015625, "beta_high": 0.0, "orthogonal_to_active": 0.026596225798130035, "evidence_weights": [-0.07642360031604767, -0.14053955674171448, 0.5607756972312927], "agreement_weight": 0.06800813227891922, "orthogonal_weight": 0.278179407119751}, {"head": 4, "memory_to_active": 2.981825351715088, "contribution_to_active": 0.37148046493530273, "mix_mean": 0.02835506945848465, "mix_low": 0.35677528381347656, "mix_high": 0.0, "read_cosine": 0.3313407301902771, "beta_mean": 0.1715961992740631, "nonfinite": 0.0, "weighted_memory_to_active": 0.36287161707878113, "mix_std": 0.042004406452178955, "beta_std": 0.11593229323625565, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.01801353693008423, "evidence_weights": [0.0050508612766861916, -0.017791438847780228, -0.03700099140405655], "agreement_weight": 0.016851086169481277, "orthogonal_weight": 0.0537501722574234}, {"head": 5, "memory_to_active": 0.246340811252594, "contribution_to_active": 0.6261158585548401, "mix_mean": 0.33253562450408936, "mix_low": 0.0852670669555664, "mix_high": 0.007346153259277344, "read_cosine": 0.5234749913215637, "beta_mean": 0.07722197473049164, "nonfinite": 0.0, "weighted_memory_to_active": 0.16458415985107422, "mix_std": 0.3371080756187439, "beta_std": 0.1087203323841095, "beta_low": 0.2535400390625, "beta_high": 0.0, "orthogonal_to_active": 0.1260855793952942, "evidence_weights": [-0.08927619457244873, -0.34484922885894775, -0.311940997838974], "agreement_weight": 0.02186586707830429, "orthogonal_weight": 0.9480069875717163}]}, {"layer": 9, "memory_to_active": 1.3258885145187378, "contribution_to_active": 0.5094789862632751, "mix_mean": 0.16052524745464325, "mix_low": 0.22266101837158203, "mix_high": 0.001561164972372353, "read_cosine": 0.361508309841156, "beta_mean": 0.11128196865320206, "nonfinite": 0.0, "weighted_memory_to_active": 0.36601942777633667, "mix_std": 0.19363021850585938, "beta_std": 0.05625240504741669, "beta_low": 0.4688008725643158, "beta_high": 0.0, "orthogonal_to_active": 0.018695034086704254, "output_influence": 0.30835843086242676, "residual_influence": 0.06901378184556961, "state_active": 0.03484360873699188, "state_memory": 0.033141057938337326, "state_peak": 32.43378448486328, "growth_active": 1.080100417137146, "growth_memory": 1.5034393072128296, "heads": [{"head": 0, "memory_to_active": 0.012248293496668339, "contribution_to_active": 0.3516292870044708, "mix_mean": 0.3394860625267029, "mix_low": 0.021562576293945312, "mix_high": 0.0019922256469726562, "read_cosine": 0.23525077104568481, "beta_mean": 3.365790325915441e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004808301106095314, "mix_std": 0.27929943799972534, "beta_std": 0.00010601208487059921, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023345202207565308, "evidence_weights": [-0.20209713280200958, -0.18980424106121063, 0.8335164785385132], "agreement_weight": 0.04814071208238602, "orthogonal_weight": -0.5111724734306335}, {"head": 1, "memory_to_active": 0.06511851400136948, "contribution_to_active": 0.34650444984436035, "mix_mean": 0.18575604259967804, "mix_low": 0.11858081817626953, "mix_high": 0.00035381317138671875, "read_cosine": 0.3906305432319641, "beta_mean": 0.00021646247478201985, "nonfinite": 0.0, "weighted_memory_to_active": 0.021113064140081406, "mix_std": 0.2266838550567627, "beta_std": 0.00042350252624601126, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0062740882858633995, "evidence_weights": [-0.14310431480407715, -0.13474811613559723, 0.4051857888698578], "agreement_weight": 0.22862830758094788, "orthogonal_weight": -0.3277316093444824}, {"head": 2, "memory_to_active": 0.4140324294567108, "contribution_to_active": 0.40091973543167114, "mix_mean": 0.22978822886943817, "mix_low": 0.073577880859375, "mix_high": 9.5367431640625e-05, "read_cosine": 0.3565860986709595, "beta_mean": 0.006444290746003389, "nonfinite": 0.0, "weighted_memory_to_active": 0.1758333146572113, "mix_std": 0.2566555142402649, "beta_std": 0.00842928234487772, "beta_low": 0.81243896484375, "beta_high": 0.0, "orthogonal_to_active": 0.02482084184885025, "evidence_weights": [-0.10640580207109451, -0.1033240258693695, 0.12546826899051666], "agreement_weight": 0.11608270555734634, "orthogonal_weight": -0.18106424808502197}, {"head": 3, "memory_to_active": 2.1320900917053223, "contribution_to_active": 0.18587665259838104, "mix_mean": 0.037818681448698044, "mix_low": 0.3964710235595703, "mix_high": 0.0, "read_cosine": 0.4542083144187927, "beta_mean": 0.1475715935230255, "nonfinite": 0.0, "weighted_memory_to_active": 0.16704440116882324, "mix_std": 0.08219069242477417, "beta_std": 0.08924580365419388, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.01653251238167286, "evidence_weights": [0.03532511740922928, 0.026053287088871002, -0.06824313849210739], "agreement_weight": 0.0021379650570452213, "orthogonal_weight": 0.10070624202489853}, {"head": 4, "memory_to_active": 3.527831792831421, "contribution_to_active": 1.3267489671707153, "mix_mean": 0.11935003846883774, "mix_low": 0.24484634399414062, "mix_high": 0.006867408752441406, "read_cosine": 0.2912425398826599, "beta_mean": 0.26627928018569946, "nonfinite": 0.0, "weighted_memory_to_active": 1.2634916305541992, "mix_std": 0.20421938598155975, "beta_std": 0.11976876109838486, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013315265066921711, "evidence_weights": [-0.04101647436618805, -0.09513606876134872, 0.08460274338722229], "agreement_weight": -0.050154346972703934, "orthogonal_weight": 0.010691419243812561}, {"head": 5, "memory_to_active": 1.8040096759796143, "contribution_to_active": 0.445194810628891, "mix_mean": 0.05095231905579567, "mix_low": 0.4809274673461914, "mix_high": 5.817413330078125e-05, "read_cosine": 0.44113147258758545, "beta_mean": 0.24714654684066772, "nonfinite": 0.0, "weighted_memory_to_active": 0.563825786113739, "mix_std": 0.11273235082626343, "beta_std": 0.11954107135534286, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.04889298230409622, "evidence_weights": [0.030610233545303345, -0.031136689707636833, 0.05080821365118027], "agreement_weight": -0.01691988855600357, "orthogonal_weight": 0.12188122421503067}]}, {"layer": 10, "memory_to_active": 0.3480265140533447, "contribution_to_active": 0.3880782425403595, "mix_mean": 0.29367637634277344, "mix_low": 0.0886794775724411, "mix_high": 0.006290118210017681, "read_cosine": 0.3890485167503357, "beta_mean": 0.050074558705091476, "nonfinite": 0.0, "weighted_memory_to_active": 0.1283262073993683, "mix_std": 0.2308845967054367, "beta_std": 0.05460502207279205, "beta_low": 0.5131937861442566, "beta_high": 0.0, "orthogonal_to_active": 0.04228974133729935, "output_influence": 0.35807934403419495, "residual_influence": 0.09567119181156158, "state_active": 0.07177187502384186, "state_memory": 0.018599867820739746, "state_peak": 58.406578063964844, "growth_active": 1.1258227825164795, "growth_memory": 1.3050962686538696, "heads": [{"head": 0, "memory_to_active": 0.6296104788780212, "contribution_to_active": 0.8791261315345764, "mix_mean": 0.5906674265861511, "mix_low": 0.015905380249023438, "mix_high": 0.0365142822265625, "read_cosine": 0.1616487056016922, "beta_mean": 0.1933595985174179, "nonfinite": 0.0, "weighted_memory_to_active": 0.43216511607170105, "mix_std": 0.34717780351638794, "beta_std": 0.20976895093917847, "beta_low": 0.099365234375, "beta_high": 0.0, "orthogonal_to_active": 0.16115087270736694, "evidence_weights": [-0.15446582436561584, -0.32785236835479736, -0.1749274730682373], "agreement_weight": 0.0219333004206419, "orthogonal_weight": 0.4032362699508667}, {"head": 1, "memory_to_active": 0.00276288203895092, "contribution_to_active": 0.36347508430480957, "mix_mean": 0.3092195391654968, "mix_low": 0.010975837707519531, "mix_high": 0.000179290771484375, "read_cosine": 0.23283420503139496, "beta_mean": 1.6074798622867092e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0009351276094093919, "mix_std": 0.2435082346200943, "beta_std": 3.1903669878374785e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004138186923228204, "evidence_weights": [-0.19694533944129944, -0.14690525829792023, 0.2072732150554657], "agreement_weight": 0.07380592077970505, "orthogonal_weight": -0.4881138801574707}, {"head": 2, "memory_to_active": 1.045330286026001, "contribution_to_active": 0.17003703117370605, "mix_mean": 0.04618256539106369, "mix_low": 0.44321250915527344, "mix_high": 0.0, "read_cosine": 0.21132253110408783, "beta_mean": 0.03705593943595886, "nonfinite": 0.0, "weighted_memory_to_active": 0.18860220909118652, "mix_std": 0.09603815525770187, "beta_std": 0.04040955752134323, "beta_low": 0.18646240234375, "beta_high": 0.0, "orthogonal_to_active": 0.031210236251354218, "evidence_weights": [0.026030760258436203, -0.04481828957796097, 0.10029516369104385], "agreement_weight": -0.017400309443473816, "orthogonal_weight": -0.17047752439975739}, {"head": 3, "memory_to_active": 0.22359348833560944, "contribution_to_active": 0.17160703241825104, "mix_mean": 0.25141870975494385, "mix_low": 0.043994903564453125, "mix_high": 0.001003265380859375, "read_cosine": 0.8094426393508911, "beta_mean": 0.06181637570261955, "nonfinite": 0.0, "weighted_memory_to_active": 0.06330496072769165, "mix_std": 0.2603374123573303, "beta_std": 0.06568832695484161, "beta_low": 0.028564453125, "beta_high": 0.0, "orthogonal_to_active": 0.04145779460668564, "evidence_weights": [-0.12152079492807388, -0.1508871167898178, 0.22955642640590668], "agreement_weight": 0.009485123679041862, "orthogonal_weight": 0.8385864496231079}, {"head": 4, "memory_to_active": 0.04176068678498268, "contribution_to_active": 0.31898605823516846, "mix_mean": 0.2749573290348053, "mix_low": 0.009105682373046875, "mix_high": 0.0, "read_cosine": 0.12360969930887222, "beta_mean": 0.00016213425260502845, "nonfinite": 0.0, "weighted_memory_to_active": 0.013056397438049316, "mix_std": 0.22508761286735535, "beta_std": 0.0013460557674989104, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.003306227969005704, "evidence_weights": [-0.15919917821884155, -0.13559174537658691, 0.09499990940093994], "agreement_weight": 0.02065923623740673, "orthogonal_weight": -0.2744251787662506}, {"head": 5, "memory_to_active": 0.1451011300086975, "contribution_to_active": 0.4252379536628723, "mix_mean": 0.28961265087127686, "mix_low": 0.008882522583007812, "mix_high": 4.38690185546875e-05, "read_cosine": 0.7954332828521729, "beta_mean": 0.008037214167416096, "nonfinite": 0.0, "weighted_memory_to_active": 0.07189343869686127, "mix_std": 0.21315830945968628, "beta_std": 0.010385347530245781, "beta_low": 0.76507568359375, "beta_high": 0.0, "orthogonal_to_active": 0.01619948446750641, "evidence_weights": [-0.1940414309501648, -0.1645546853542328, 0.20244577527046204], "agreement_weight": 0.08491247892379761, "orthogonal_weight": -0.4968123733997345}]}, {"layer": 11, "memory_to_active": 0.30921587347984314, "contribution_to_active": 0.42492422461509705, "mix_mean": 0.3320838212966919, "mix_low": 0.05953121557831764, "mix_high": 0.016188621520996094, "read_cosine": 0.5040795207023621, "beta_mean": 0.04240080714225769, "nonfinite": 0.0, "weighted_memory_to_active": 0.09006726741790771, "mix_std": 0.28538110852241516, "beta_std": 0.06048426032066345, "beta_low": 0.4464111328125, "beta_high": 0.0, "orthogonal_to_active": 0.03132235258817673, "output_influence": 0.46737101674079895, "residual_influence": 0.11890207231044769, "state_active": 0.12465548515319824, "state_memory": 0.01483853254467249, "state_peak": 30.3319149017334, "growth_active": 1.186374306678772, "growth_memory": 1.4730948209762573, "heads": [{"head": 0, "memory_to_active": 0.07282502949237823, "contribution_to_active": 0.4911021590232849, "mix_mean": 0.43669724464416504, "mix_low": 0.0068225860595703125, "mix_high": 0.002407073974609375, "read_cosine": 0.4166303873062134, "beta_mean": 0.0003636375768110156, "nonfinite": 0.0, "weighted_memory_to_active": 0.041513655334711075, "mix_std": 0.2772607207298279, "beta_std": 0.005272991023957729, "beta_low": 0.99749755859375, "beta_high": 0.0, "orthogonal_to_active": 0.02031298726797104, "evidence_weights": [-0.1578243374824524, -0.25592294335365295, 0.40027835965156555], "agreement_weight": 0.12178140878677368, "orthogonal_weight": -0.6228932738304138}, {"head": 1, "memory_to_active": 0.5592312812805176, "contribution_to_active": 0.1934574693441391, "mix_mean": 0.19260165095329285, "mix_low": 0.05248546600341797, "mix_high": 1.52587890625e-05, "read_cosine": 0.604019045829773, "beta_mean": 0.04426073282957077, "nonfinite": 0.0, "weighted_memory_to_active": 0.1558915674686432, "mix_std": 0.21020746231079102, "beta_std": 0.07983370125293732, "beta_low": 0.308349609375, "beta_high": 0.0, "orthogonal_to_active": 0.016098763793706894, "evidence_weights": [-0.14307907223701477, -0.09686391800642014, 0.12671445310115814], "agreement_weight": 0.08616552501916885, "orthogonal_weight": -0.1413780301809311}, {"head": 2, "memory_to_active": 0.0661168247461319, "contribution_to_active": 0.83897465467453, "mix_mean": 0.46447306871414185, "mix_low": 0.05183982849121094, "mix_high": 0.06018829345703125, "read_cosine": 0.35596710443496704, "beta_mean": 0.07302240282297134, "nonfinite": 0.0, "weighted_memory_to_active": 0.056311361491680145, "mix_std": 0.4052717387676239, "beta_std": 0.09919898211956024, "beta_low": 0.421875, "beta_high": 0.0, "orthogonal_to_active": 0.0801515281200409, "evidence_weights": [-0.18659241497516632, -0.38723865151405334, -0.4706966280937195], "agreement_weight": -0.013191336765885353, "orthogonal_weight": 1.5736947059631348}, {"head": 3, "memory_to_active": 0.03383661061525345, "contribution_to_active": 0.4258022904396057, "mix_mean": 0.4375831186771393, "mix_low": 0.08480262756347656, "mix_high": 0.03369712829589844, "read_cosine": 0.4810830354690552, "beta_mean": 0.06898771226406097, "nonfinite": 0.0, "weighted_memory_to_active": 0.019221492111682892, "mix_std": 0.41146740317344666, "beta_std": 0.09666708111763, "beta_low": 0.42291259765625, "beta_high": 0.0, "orthogonal_to_active": 0.026716336607933044, "evidence_weights": [-0.22730499505996704, -0.35769546031951904, -0.0768078863620758], "agreement_weight": 0.017290377989411354, "orthogonal_weight": 1.702148675918579}, {"head": 4, "memory_to_active": 0.909712553024292, "contribution_to_active": 0.21091589331626892, "mix_mean": 0.10924700647592545, "mix_low": 0.15683937072753906, "mix_high": 9.5367431640625e-07, "read_cosine": 0.26837050914764404, "beta_mean": 0.044637102633714676, "nonfinite": 0.0, "weighted_memory_to_active": 0.1617974042892456, "mix_std": 0.15306274592876434, "beta_std": 0.05057525262236595, "beta_low": 0.1212158203125, "beta_high": 0.0, "orthogonal_to_active": 0.022832999005913734, "evidence_weights": [-0.05143316835165024, -0.03684292361140251, 0.625900149345398], "agreement_weight": -0.049819886684417725, "orthogonal_weight": -0.1638576090335846}, {"head": 5, "memory_to_active": 0.21357285976409912, "contribution_to_active": 0.38929277658462524, "mix_mean": 0.3519008755683899, "mix_low": 0.004397392272949219, "mix_high": 0.0008230209350585938, "read_cosine": 0.8984068632125854, "beta_mean": 0.02313324436545372, "nonfinite": 0.0, "weighted_memory_to_active": 0.10566815733909607, "mix_std": 0.2550165057182312, "beta_std": 0.031357541680336, "beta_low": 0.4066162109375, "beta_high": 0.0, "orthogonal_to_active": 0.021821504458785057, "evidence_weights": [-0.18479260802268982, -0.22453859448432922, 0.28509223461151123], "agreement_weight": 0.04607805609703064, "orthogonal_weight": -0.5304039716720581}]}, {"layer": 12, "memory_to_active": 0.6384748220443726, "contribution_to_active": 0.46035829186439514, "mix_mean": 0.3394815921783447, "mix_low": 0.06577746570110321, "mix_high": 0.0008230209932662547, "read_cosine": 0.3353300094604492, "beta_mean": 0.025995571166276932, "nonfinite": 0.0, "weighted_memory_to_active": 0.24162010848522186, "mix_std": 0.22927075624465942, "beta_std": 0.028580032289028168, "beta_low": 0.6697489619255066, "beta_high": 0.0, "orthogonal_to_active": 0.061719369143247604, "output_influence": 0.2948266565799713, "residual_influence": 0.0788140594959259, "state_active": 0.054247550666332245, "state_memory": 0.01951512321829796, "state_peak": 35.7872428894043, "growth_active": 1.1393041610717773, "growth_memory": 1.3866333961486816, "heads": [{"head": 0, "memory_to_active": 0.6010560393333435, "contribution_to_active": 0.2691030502319336, "mix_mean": 0.23960427939891815, "mix_low": 0.03367137908935547, "mix_high": 5.340576171875e-05, "read_cosine": 0.4576680362224579, "beta_mean": 0.013371883891522884, "nonfinite": 0.0, "weighted_memory_to_active": 0.2180580198764801, "mix_std": 0.22602561116218567, "beta_std": 0.029944531619548798, "beta_low": 0.70513916015625, "beta_high": 0.0, "orthogonal_to_active": 0.07235705852508545, "evidence_weights": [-0.11624322831630707, -0.16060470044612885, 0.25157755613327026], "agreement_weight": 0.09379281103610992, "orthogonal_weight": -0.424044668674469}, {"head": 1, "memory_to_active": 0.23006847500801086, "contribution_to_active": 0.37069034576416016, "mix_mean": 0.35404473543167114, "mix_low": 0.0027418136596679688, "mix_high": 0.0002231597900390625, "read_cosine": 0.6013747453689575, "beta_mean": 0.002653048373758793, "nonfinite": 0.0, "weighted_memory_to_active": 0.09396655857563019, "mix_std": 0.24935971200466156, "beta_std": 0.005927454214543104, "beta_low": 0.95562744140625, "beta_high": 0.0, "orthogonal_to_active": 0.057787977159023285, "evidence_weights": [-0.1564938873052597, -0.2035294622182846, 0.47955116629600525], "agreement_weight": 0.08437486737966537, "orthogonal_weight": -0.8326700925827026}, {"head": 2, "memory_to_active": 0.7071245908737183, "contribution_to_active": 0.3722451329231262, "mix_mean": 0.29547515511512756, "mix_low": 0.025755882263183594, "mix_high": 0.000286102294921875, "read_cosine": 0.37246161699295044, "beta_mean": 0.10413900762796402, "nonfinite": 0.0, "weighted_memory_to_active": 0.29501569271087646, "mix_std": 0.24552848935127258, "beta_std": 0.08195552229881287, "beta_low": 0.0018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.0927339643239975, "evidence_weights": [-0.20064964890480042, -0.08067917823791504, 0.08813376724720001], "agreement_weight": 0.04632830619812012, "orthogonal_weight": 0.3958747982978821}, {"head": 3, "memory_to_active": 0.034942664206027985, "contribution_to_active": 0.6149014234542847, "mix_mean": 0.5936011672019958, "mix_low": 0.0008993148803710938, "mix_high": 0.0028858184814453125, "read_cosine": 0.10535453259944916, "beta_mean": 0.0003334439534228295, "nonfinite": 0.0, "weighted_memory_to_active": 0.020527195185422897, "mix_std": 0.25757330656051636, "beta_std": 0.0008124637533910573, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.013496003113687038, "evidence_weights": [-0.29548805952072144, -0.27732086181640625, 0.2911897897720337], "agreement_weight": 0.10181368887424469, "orthogonal_weight": -0.6767973899841309}, {"head": 4, "memory_to_active": 2.182262420654297, "contribution_to_active": 0.72234708070755, "mix_mean": 0.06954263895750046, "mix_low": 0.3258075714111328, "mix_high": 0.000804901123046875, "read_cosine": 0.10152380168437958, "beta_mean": 0.03432196378707886, "nonfinite": 0.0, "weighted_memory_to_active": 0.7890380620956421, "mix_std": 0.12690186500549316, "beta_std": 0.049488380551338196, "beta_low": 0.3780517578125, "beta_high": 0.0, "orthogonal_to_active": 0.10855451971292496, "evidence_weights": [0.05281570181250572, -0.056155014783144, 0.058810390532016754], "agreement_weight": -0.016902055591344833, "orthogonal_weight": -0.1493609994649887}, {"head": 5, "memory_to_active": 0.0753941759467125, "contribution_to_active": 0.41286253929138184, "mix_mean": 0.48462143540382385, "mix_low": 0.0057888031005859375, "mix_high": 0.0006847381591796875, "read_cosine": 0.37359726428985596, "beta_mean": 0.0011540802661329508, "nonfinite": 0.0, "weighted_memory_to_active": 0.033115118741989136, "mix_std": 0.27023541927337646, "beta_std": 0.0033518336713314056, "beta_low": 0.978515625, "beta_high": 0.0, "orthogonal_to_active": 0.025386691093444824, "evidence_weights": [-0.24992793798446655, -0.25342637300491333, 0.4964051842689514], "agreement_weight": 0.054331887513399124, "orthogonal_weight": -0.8268195986747742}]}, {"layer": 13, "memory_to_active": 0.25014057755470276, "contribution_to_active": 0.4047825038433075, "mix_mean": 0.39696764945983887, "mix_low": 0.0044870381243526936, "mix_high": 0.0011987686157226562, "read_cosine": 0.38641369342803955, "beta_mean": 0.018009934574365616, "nonfinite": 0.0, "weighted_memory_to_active": 0.11187173426151276, "mix_std": 0.24894681572914124, "beta_std": 0.021067488938570023, "beta_low": 0.738494873046875, "beta_high": 0.0, "orthogonal_to_active": 0.07252702116966248, "output_influence": 0.34467610716819763, "residual_influence": 0.11138725280761719, "state_active": 0.08214075863361359, "state_memory": 0.029264871031045914, "state_peak": 67.2616958618164, "growth_active": 0.9669926166534424, "growth_memory": 1.1146223545074463, "heads": [{"head": 0, "memory_to_active": 0.26678499579429626, "contribution_to_active": 0.3852822184562683, "mix_mean": 0.3567694425582886, "mix_low": 0.003963470458984375, "mix_high": 0.0006923675537109375, "read_cosine": 0.3624991774559021, "beta_mean": 0.0034322498831897974, "nonfinite": 0.0, "weighted_memory_to_active": 0.11564358323812485, "mix_std": 0.2493157684803009, "beta_std": 0.00483374297618866, "beta_low": 0.923583984375, "beta_high": 0.0, "orthogonal_to_active": 0.09819403290748596, "evidence_weights": [-0.18347176909446716, -0.20145171880722046, 0.494157612323761], "agreement_weight": 0.046496544033288956, "orthogonal_weight": -0.9348737597465515}, {"head": 1, "memory_to_active": 0.1497100591659546, "contribution_to_active": 0.4378962218761444, "mix_mean": 0.36177998781204224, "mix_low": 0.0036678314208984375, "mix_high": 4.1961669921875e-05, "read_cosine": 0.2994534373283386, "beta_mean": 0.0010343169560655951, "nonfinite": 0.0, "weighted_memory_to_active": 0.06446173787117004, "mix_std": 0.2491200566291809, "beta_std": 0.0024323943071067333, "beta_low": 0.98724365234375, "beta_high": 0.0, "orthogonal_to_active": 0.056087225675582886, "evidence_weights": [-0.17779524624347687, -0.20082391798496246, 0.5535352826118469], "agreement_weight": 0.07145432382822037, "orthogonal_weight": -0.9299149513244629}, {"head": 2, "memory_to_active": 0.2745407223701477, "contribution_to_active": 0.3182363212108612, "mix_mean": 0.27590417861938477, "mix_low": 0.004384040832519531, "mix_high": 1.430511474609375e-05, "read_cosine": 0.6373587846755981, "beta_mean": 0.019851192831993103, "nonfinite": 0.0, "weighted_memory_to_active": 0.10374005138874054, "mix_std": 0.20509329438209534, "beta_std": 0.041817739605903625, "beta_low": 0.6070556640625, "beta_high": 0.0, "orthogonal_to_active": 0.037663783878088, "evidence_weights": [-0.1893320083618164, -0.14660723507404327, 0.4013442099094391], "agreement_weight": 0.06306768208742142, "orthogonal_weight": -0.5539616346359253}, {"head": 3, "memory_to_active": 0.15546441078186035, "contribution_to_active": 0.49355262517929077, "mix_mean": 0.5224210023880005, "mix_low": 0.0047760009765625, "mix_high": 0.0009250640869140625, "read_cosine": -0.04733879119157791, "beta_mean": 0.0007564770057797432, "nonfinite": 0.0, "weighted_memory_to_active": 0.10554949194192886, "mix_std": 0.24003422260284424, "beta_std": 0.004789707250893116, "beta_low": 0.99517822265625, "beta_high": 0.0, "orthogonal_to_active": 0.10532714426517487, "evidence_weights": [-0.33913567662239075, -0.2435372769832611, 0.8837440013885498], "agreement_weight": 0.7646854519844055, "orthogonal_weight": -1.0058566331863403}, {"head": 4, "memory_to_active": 0.390252023935318, "contribution_to_active": 0.32335540652275085, "mix_mean": 0.4953513741493225, "mix_low": 0.0034046173095703125, "mix_high": 0.005089759826660156, "read_cosine": 0.802431583404541, "beta_mean": 0.07886959612369537, "nonfinite": 0.0, "weighted_memory_to_active": 0.17630428075790405, "mix_std": 0.27977263927459717, "beta_std": 0.061215467751026154, "beta_low": 0.00439453125, "beta_high": 0.0, "orthogonal_to_active": 0.08294842392206192, "evidence_weights": [-0.2642592787742615, -0.2705625891685486, 0.4086652994155884], "agreement_weight": 0.0022537927143275738, "orthogonal_weight": 0.6674478650093079}, {"head": 5, "memory_to_active": 0.2640911936759949, "contribution_to_active": 0.4703723192214966, "mix_mean": 0.3695797324180603, "mix_low": 0.006726264953613281, "mix_high": 0.0004291534423828125, "read_cosine": 0.2640778422355652, "beta_mean": 0.004115771502256393, "nonfinite": 0.0, "weighted_memory_to_active": 0.10553120821714401, "mix_std": 0.2703448235988617, "beta_std": 0.011315880343317986, "beta_low": 0.91351318359375, "beta_high": 0.0, "orthogonal_to_active": 0.05494149774312973, "evidence_weights": [-0.18363630771636963, -0.20897908508777618, 0.1891278624534607], "agreement_weight": 0.05506548285484314, "orthogonal_weight": -0.5713319778442383}]}, {"layer": 14, "memory_to_active": 0.30400359630584717, "contribution_to_active": 0.41454365849494934, "mix_mean": 0.3663003444671631, "mix_low": 0.028068384155631065, "mix_high": 0.002076308010146022, "read_cosine": 0.594128429889679, "beta_mean": 0.07613834738731384, "nonfinite": 0.0, "weighted_memory_to_active": 0.1385284960269928, "mix_std": 0.247719869017601, "beta_std": 0.07234086841344833, "beta_low": 0.3462626338005066, "beta_high": 0.0, "orthogonal_to_active": 0.036691877990961075, "output_influence": 0.231534942984581, "residual_influence": 0.08921587467193604, "state_active": 0.08440248668193817, "state_memory": 0.020927930250763893, "state_peak": 28.951894760131836, "growth_active": 1.1533207893371582, "growth_memory": 1.4898710250854492, "heads": [{"head": 0, "memory_to_active": 0.30040138959884644, "contribution_to_active": 0.273345947265625, "mix_mean": 0.2537977695465088, "mix_low": 0.027358055114746094, "mix_high": 3.337860107421875e-05, "read_cosine": 0.632282555103302, "beta_mean": 0.07689248770475388, "nonfinite": 0.0, "weighted_memory_to_active": 0.10715103149414062, "mix_std": 0.23817571997642517, "beta_std": 0.09870975464582443, "beta_low": 0.1494140625, "beta_high": 0.0, "orthogonal_to_active": 0.01180740911513567, "evidence_weights": [-0.06311240792274475, -0.1475815623998642, 0.6401010751724243], "agreement_weight": 0.0431898832321167, "orthogonal_weight": 0.14605309069156647}, {"head": 1, "memory_to_active": 0.43197453022003174, "contribution_to_active": 0.4722375273704529, "mix_mean": 0.2951105833053589, "mix_low": 0.0960540771484375, "mix_high": 0.0008544921875, "read_cosine": 0.519752025604248, "beta_mean": 0.2863534092903137, "nonfinite": 0.0, "weighted_memory_to_active": 0.21378293633460999, "mix_std": 0.2852051258087158, "beta_std": 0.2512208819389343, "beta_low": 0.083251953125, "beta_high": 0.0, "orthogonal_to_active": 0.010272642597556114, "evidence_weights": [-0.028665054589509964, -0.2522777318954468, 0.12152163684368134], "agreement_weight": 0.058410532772541046, "orthogonal_weight": 0.06500647962093353}, {"head": 2, "memory_to_active": 0.33129623532295227, "contribution_to_active": 0.42407169938087463, "mix_mean": 0.4141504764556885, "mix_low": 0.005523681640625, "mix_high": 0.0006666183471679688, "read_cosine": 0.43623965978622437, "beta_mean": 0.018675053492188454, "nonfinite": 0.0, "weighted_memory_to_active": 0.1694212704896927, "mix_std": 0.2650156617164612, "beta_std": 0.019652802497148514, "beta_low": 0.38946533203125, "beta_high": 0.0, "orthogonal_to_active": 0.11019261181354523, "evidence_weights": [-0.19861838221549988, -0.21055161952972412, 0.449020117521286], "agreement_weight": 0.0874873474240303, "orthogonal_weight": -0.7599167823791504}, {"head": 3, "memory_to_active": 0.07766313850879669, "contribution_to_active": 0.6338738203048706, "mix_mean": 0.6373826265335083, "mix_low": 0.0021715164184570312, "mix_high": 0.008084297180175781, "read_cosine": 0.5685825347900391, "beta_mean": 0.00975540280342102, "nonfinite": 0.0, "weighted_memory_to_active": 0.05268745496869087, "mix_std": 0.23842568695545197, "beta_std": 0.007773794233798981, "beta_low": 0.632568359375, "beta_high": 0.0, "orthogonal_to_active": 0.019690826535224915, "evidence_weights": [-0.37415704131126404, -0.3227137327194214, 0.39278796315193176], "agreement_weight": 0.06926889717578888, "orthogonal_weight": -0.5032437443733215}, {"head": 4, "memory_to_active": 0.2603091597557068, "contribution_to_active": 0.4503035843372345, "mix_mean": 0.383358359336853, "mix_low": 0.006222724914550781, "mix_high": 0.0027494430541992188, "read_cosine": 0.716931939125061, "beta_mean": 0.058413002640008926, "nonfinite": 0.0, "weighted_memory_to_active": 0.1351119428873062, "mix_std": 0.25245121121406555, "beta_std": 0.04356493055820465, "beta_low": 0.0120849609375, "beta_high": 0.0, "orthogonal_to_active": 0.02176102250814438, "evidence_weights": [-0.20592860877513885, -0.2526928782463074, 0.20394986867904663], "agreement_weight": 0.10705025494098663, "orthogonal_weight": -0.25608694553375244}, {"head": 5, "memory_to_active": 0.4223768711090088, "contribution_to_active": 0.23342931270599365, "mix_mean": 0.21400228142738342, "mix_low": 0.031080245971679688, "mix_high": 6.961822509765625e-05, "read_cosine": 0.6909817457199097, "beta_mean": 0.006740699987858534, "nonfinite": 0.0, "weighted_memory_to_active": 0.15301629900932312, "mix_std": 0.20704567432403564, "beta_std": 0.013122998178005219, "beta_low": 0.810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.046426739543676376, "evidence_weights": [-0.18013371527194977, -0.12821175158023834, 0.07346756011247635], "agreement_weight": 0.09899485111236572, "orthogonal_weight": -0.48607349395751953}]}, {"layer": 15, "memory_to_active": 0.11493880301713943, "contribution_to_active": 0.46087372303009033, "mix_mean": 0.3698522746562958, "mix_low": 0.00706084631383419, "mix_high": 0.0023899078369140625, "read_cosine": 0.6295574903488159, "beta_mean": 0.050139158964157104, "nonfinite": 0.0, "weighted_memory_to_active": 0.05419785901904106, "mix_std": 0.2640328109264374, "beta_std": 0.053847990930080414, "beta_low": 0.6630656719207764, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.014529294334352016, "output_influence": 0.2891387343406677, "residual_influence": 0.14842009544372559, "state_active": 0.10586211830377579, "state_memory": 0.011115476489067078, "state_peak": 30.145675659179688, "growth_active": 1.367318868637085, "growth_memory": 1.5536309480667114, "heads": [{"head": 0, "memory_to_active": 0.11862355470657349, "contribution_to_active": 0.4844804108142853, "mix_mean": 0.3373032212257385, "mix_low": 0.008152008056640625, "mix_high": 0.0022640228271484375, "read_cosine": 0.6628066301345825, "beta_mean": 0.05488109216094017, "nonfinite": 0.0, "weighted_memory_to_active": 0.057620804756879807, "mix_std": 0.2566274404525757, "beta_std": 0.04085258021950722, "beta_low": 0.09051513671875, "beta_high": 0.0, "orthogonal_to_active": 0.013983349315822124, "evidence_weights": [-0.17334362864494324, -0.20997893810272217, 0.35459011793136597], "agreement_weight": 0.08199942111968994, "orthogonal_weight": -0.4163874089717865}, {"head": 1, "memory_to_active": 0.23557470738887787, "contribution_to_active": 0.38283488154411316, "mix_mean": 0.3521033227443695, "mix_low": 0.005718231201171875, "mix_high": 0.005293846130371094, "read_cosine": 0.8356885313987732, "beta_mean": 0.003375713713467121, "nonfinite": 0.0, "weighted_memory_to_active": 0.09323400259017944, "mix_std": 0.26234301924705505, "beta_std": 0.00818687118589878, "beta_low": 0.9378662109375, "beta_high": 0.0, "orthogonal_to_active": 0.008768945932388306, "evidence_weights": [-0.19908443093299866, -0.20662164688110352, 0.2647375762462616], "agreement_weight": 0.09380020201206207, "orthogonal_weight": -0.15607817471027374}, {"head": 2, "memory_to_active": 0.01587914116680622, "contribution_to_active": 0.43596380949020386, "mix_mean": 0.3477781116962433, "mix_low": 0.010470390319824219, "mix_high": 0.00026798248291015625, "read_cosine": 0.7448035478591919, "beta_mean": 0.0003195283643435687, "nonfinite": 0.0, "weighted_memory_to_active": 0.006420735735446215, "mix_std": 0.26408880949020386, "beta_std": 0.0006344272987917066, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.0021823174320161343, "evidence_weights": [-0.21159811317920685, -0.2603450119495392, 0.31298065185546875], "agreement_weight": 0.04294706881046295, "orthogonal_weight": -0.4870329201221466}, {"head": 3, "memory_to_active": 0.14097121357917786, "contribution_to_active": 0.4713081419467926, "mix_mean": 0.28531473875045776, "mix_low": 0.0030488967895507812, "mix_high": 1.811981201171875e-05, "read_cosine": 0.7249035239219666, "beta_mean": 0.007455829530954361, "nonfinite": 0.0, "weighted_memory_to_active": 0.0714898407459259, "mix_std": 0.22441239655017853, "beta_std": 0.011358486488461494, "beta_low": 0.80340576171875, "beta_high": 0.0, "orthogonal_to_active": 0.017859572544693947, "evidence_weights": [-0.16428332030773163, -0.20540310442447662, 0.35110583901405334], "agreement_weight": 0.04483262822031975, "orthogonal_weight": -0.47652941942214966}, {"head": 4, "memory_to_active": 0.1716519445180893, "contribution_to_active": 0.531165599822998, "mix_mean": 0.39340370893478394, "mix_low": 0.010578155517578125, "mix_high": 0.00046443939208984375, "read_cosine": 0.2073391228914261, "beta_mean": 0.2346799373626709, "nonfinite": 0.0, "weighted_memory_to_active": 0.09363000094890594, "mix_std": 0.29608386754989624, "beta_std": 0.26181092858314514, "beta_low": 0.14715576171875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.04292190074920654, "evidence_weights": [-0.2689204812049866, -0.24429106712341309, 0.39218848943710327], "agreement_weight": 0.005283628590404987, "orthogonal_weight": -0.5112385153770447}, {"head": 5, "memory_to_active": 0.006932226940989494, "contribution_to_active": 0.4594893753528595, "mix_mean": 0.5032104253768921, "mix_low": 0.004397392272949219, "mix_high": 0.006031036376953125, "read_cosine": 0.6018033623695374, "beta_mean": 0.00012283239630050957, "nonfinite": 0.0, "weighted_memory_to_active": 0.0027917439583688974, "mix_std": 0.28064125776290894, "beta_std": 0.00024463568115606904, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.001459676306694746, "evidence_weights": [-0.35521233081817627, -0.2912631034851074, 0.5385733842849731], "agreement_weight": 0.11714736372232437, "orthogonal_weight": 0.602712094783783}]}]}, {"step": 11000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 36.41197752499966, "memory_to_active": 1.0485005378723145, "contribution_to_active": 0.4134589731693268, "mix_mean": 0.2602071464061737, "mix_low": 0.18874074518680573, "mix_high": 0.004055897239595652, "read_cosine": 0.4035429358482361, "beta_mean": 0.10039722174406052, "nonfinite": 0.0, "weighted_memory_to_active": 0.21057656407356262, "mix_std": 0.19683358073234558, "beta_std": 0.07801292836666107, "beta_low": 0.41526922583580017, "beta_high": 1.0808309525600635e-05, "orthogonal_to_active": 0.03592746704816818, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1898305416107178, "normal_token_accuracy": 0.4371523857116699, "state_active": 0.06400705873966217, "state_memory": 0.027939079329371452, "state_peak": 73.07311248779297, "growth_active": 1.1199233531951904, "growth_memory": 1.4280600547790527, "output_influence": 0.28257641196250916, "residual_influence": 0.08760468661785126, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.279017686843872, "no_memory_loss_delta": 0.15585437417030334, "scrambled_loss_delta": 0.23991388082504272, "memory_only_loss_delta": 1.9986764192581177, "no_evidence_loss_delta": 0.048071879893541336, "no_agreement_loss_delta": -7.256795652210712e-05, "no_orthogonal_loss_delta": 0.00040219409856945276}, {"index": 94, "tokens": 256, "normal_loss": 3.319948434829712, "no_memory_loss_delta": 0.3082221746444702, "scrambled_loss_delta": 0.4095023572444916, "memory_only_loss_delta": 2.096781015396118, "no_evidence_loss_delta": 0.15399004518985748, "no_agreement_loss_delta": 0.034622929990291595, "no_orthogonal_loss_delta": 0.02368168532848358}, {"index": 188, "tokens": 256, "normal_loss": 3.590721368789673, "no_memory_loss_delta": 0.119568832218647, "scrambled_loss_delta": 0.1619497537612915, "memory_only_loss_delta": 2.1716322898864746, "no_evidence_loss_delta": 0.0646415501832962, "no_agreement_loss_delta": -0.013364608399569988, "no_orthogonal_loss_delta": -0.00034948717802762985}, {"index": 283, "tokens": 131, "normal_loss": 1.9778428077697754, "no_memory_loss_delta": 0.16951850056648254, "scrambled_loss_delta": 0.3368415832519531, "memory_only_loss_delta": 2.6603832244873047, "no_evidence_loss_delta": 0.07825770229101181, "no_agreement_loss_delta": 0.010270862840116024, "no_orthogonal_loss_delta": 0.015508291311562061}], "agreement_weight_abs_mean": 0.06618581712245941, "agreement_weight_abs_max": 0.8801455497741699, "agreement_weight_mean": 0.05026627704501152, "orthogonal_weight_abs_mean": 0.3885062038898468, "orthogonal_weight_abs_max": 1.8871761560440063, "orthogonal_weight_mean": -0.15377114713191986, "no_memory_loss": 3.3807318210601807, "no_memory_loss_delta": 0.19090116024017334, "no_memory_token_accuracy_gain": 0.01890989951789379, "no_memory_token_win_fraction": 0.600667417049408, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 17.378575306670758, "scrambled_loss": 3.469959020614624, "scrambled_loss_delta": 0.28012895584106445, "scrambled_token_accuracy_gain": 0.030033370479941368, "scrambled_token_win_fraction": 0.6140155792236328, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 24.431371498839287, "memory_only_loss": 5.36211633682251, "memory_only_loss_delta": 2.17228627204895, "memory_only_token_accuracy_gain": 0.22358176112174988, "memory_only_token_win_fraction": 0.878754198551178, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 88.6083126005126, "no_evidence_loss": 3.2771804332733154, "no_evidence_loss_delta": 0.08735021948814392, "no_evidence_token_accuracy_gain": 0.0, "no_evidence_token_win_fraction": 0.5406006574630737, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 8.364388633972169, "no_agreement_loss": 3.197360038757324, "no_agreement_loss_delta": 0.0075295171700417995, "no_agreement_token_accuracy_gain": -0.007786429487168789, "no_agreement_token_win_fraction": 0.47608453035354614, "no_agreement_probe_win_fraction": 0.5, "no_agreement_ppl_reduction_pct": 0.7501241367854529, "no_orthogonal_loss": 3.1988489627838135, "no_orthogonal_loss_delta": 0.00901845470070839, "no_orthogonal_token_accuracy_gain": 0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5205784440040588, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.8977910411935285, "short_no_memory_loss_delta": 0.13572166860103607, "long_context_memory_gain": 0.05517949163913727, "normal_context_loss_gain": 0.14194753766059875, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.729919672012329, "contribution_to_active": 0.7731468081474304, "mix_mean": 0.09191577136516571, "mix_low": 0.425332248210907, "mix_high": 0.0025348663330078125, "read_cosine": 0.49553269147872925, "beta_mean": 0.25889503955841064, "nonfinite": 0.0, "weighted_memory_to_active": 0.7927316427230835, "mix_std": 0.0847894549369812, "beta_std": 0.14877687394618988, "beta_low": 2.0345052689663135e-05, "beta_high": 0.0, "orthogonal_to_active": 0.03668064624071121, "output_influence": 0.1417098343372345, "residual_influence": 0.2161266803741455, "state_active": 0.09412990510463715, "state_memory": 0.05477346479892731, "state_peak": 53.48720169067383, "growth_active": 1.2646570205688477, "growth_memory": 1.3415753841400146, "heads": [{"head": 0, "memory_to_active": 2.2578048706054688, "contribution_to_active": 0.4849289655685425, "mix_mean": 0.021183960139751434, "mix_low": 0.4996147155761719, "mix_high": 0.0, "read_cosine": 0.3726498782634735, "beta_mean": 0.2173512876033783, "nonfinite": 0.0, "weighted_memory_to_active": 0.5714988112449646, "mix_std": 0.05251052975654602, "beta_std": 0.1025637611746788, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0190434493124485, "evidence_weights": [-0.048195213079452515, -0.07233990728855133, 0.0217004232108593], "agreement_weight": -0.053426358848810196, "orthogonal_weight": -0.03986605256795883}, {"head": 1, "memory_to_active": 3.304241418838501, "contribution_to_active": 1.249183177947998, "mix_mean": 0.0184140894562006, "mix_low": 0.5006856918334961, "mix_high": 0.0, "read_cosine": 0.4849397838115692, "beta_mean": 0.3218799829483032, "nonfinite": 0.0, "weighted_memory_to_active": 1.5028403997421265, "mix_std": 0.047681570053100586, "beta_std": 0.15674851834774017, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04983055219054222, "evidence_weights": [0.01250498741865158, -0.03283447399735451, 0.0879419595003128], "agreement_weight": 0.008661833591759205, "orthogonal_weight": -0.07467605918645859}, {"head": 2, "memory_to_active": 2.2118024826049805, "contribution_to_active": 0.47701990604400635, "mix_mean": 0.01783483475446701, "mix_low": 0.5704584121704102, "mix_high": 0.0, "read_cosine": 0.36530083417892456, "beta_mean": 0.29537227749824524, "nonfinite": 0.0, "weighted_memory_to_active": 0.5215068459510803, "mix_std": 0.037366833537817, "beta_std": 0.20296373963356018, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.033524420112371445, "evidence_weights": [0.020427918061614037, 0.027641044929623604, 0.0010095944162458181], "agreement_weight": -0.014482955448329449, "orthogonal_weight": -0.10071160644292831}, {"head": 3, "memory_to_active": 4.704499244689941, "contribution_to_active": 1.9958672523498535, "mix_mean": 0.056498657912015915, "mix_low": 0.18585872650146484, "mix_high": 0.0, "read_cosine": 0.4540126323699951, "beta_mean": 0.32544517517089844, "nonfinite": 0.0, "weighted_memory_to_active": 2.006863832473755, "mix_std": 0.0905567929148674, "beta_std": 0.1879483461380005, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0978139117360115, "evidence_weights": [-0.11802738904953003, -0.143539696931839, 0.06449814885854721], "agreement_weight": -0.021872758865356445, "orthogonal_weight": 0.06519413739442825}, {"head": 4, "memory_to_active": 3.746155261993408, "contribution_to_active": 0.09936611354351044, "mix_mean": 0.007760399486869574, "mix_low": 0.783604621887207, "mix_high": 0.0, "read_cosine": 0.3384731411933899, "beta_mean": 0.31556764245033264, "nonfinite": 0.0, "weighted_memory_to_active": 0.0882355198264122, "mix_std": 0.012116366997361183, "beta_std": 0.18458876013755798, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.015600345097482204, "evidence_weights": [0.06296777725219727, 0.0209038183093071, -0.13363902270793915], "agreement_weight": -0.02671552263200283, "orthogonal_weight": 0.20395486056804657}, {"head": 5, "memory_to_active": 0.15501466393470764, "contribution_to_active": 0.3325151205062866, "mix_mean": 0.42980265617370605, "mix_low": 0.011771202087402344, "mix_high": 0.015209197998046875, "read_cosine": 0.9578197002410889, "beta_mean": 0.07775375992059708, "nonfinite": 0.0, "weighted_memory_to_active": 0.06544412672519684, "mix_std": 0.26850464940071106, "beta_std": 0.057848140597343445, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.004271205514669418, "evidence_weights": [-0.3002861440181732, -0.29762232303619385, 0.36718514561653137], "agreement_weight": 0.14274649322032928, "orthogonal_weight": -0.17158083617687225}]}, {"layer": 1, "memory_to_active": 2.3880996704101562, "contribution_to_active": 0.3360196352005005, "mix_mean": 0.08064008504152298, "mix_low": 0.41097864508628845, "mix_high": 0.0005337397451512516, "read_cosine": 0.3081865608692169, "beta_mean": 0.23483812808990479, "nonfinite": 0.0, "weighted_memory_to_active": 0.3209134340286255, "mix_std": 0.1017562747001648, "beta_std": 0.13040529191493988, "beta_low": 0.1716308742761612, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.03039362281560898, "output_influence": 0.12719111144542694, "residual_influence": 0.027991067618131638, "state_active": 0.04668795317411423, "state_memory": 0.05446895584464073, "state_peak": 24.20518684387207, "growth_active": 1.1112409830093384, "growth_memory": 1.5317692756652832, "heads": [{"head": 0, "memory_to_active": 0.18190211057662964, "contribution_to_active": 0.21980462968349457, "mix_mean": 0.14224590361118317, "mix_low": 0.14551067352294922, "mix_high": 3.910064697265625e-05, "read_cosine": -0.12849822640419006, "beta_mean": 0.0008156389812938869, "nonfinite": 0.0, "weighted_memory_to_active": 0.052848491817712784, "mix_std": 0.1944570690393448, "beta_std": 0.0012567144585773349, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.023825783282518387, "evidence_weights": [-0.06993794441223145, -0.1050962284207344, -0.05282283201813698], "agreement_weight": -0.00776864355430007, "orthogonal_weight": -0.5095226168632507}, {"head": 1, "memory_to_active": 3.613980531692505, "contribution_to_active": 0.1833426058292389, "mix_mean": 0.026576155796647072, "mix_low": 0.41452789306640625, "mix_high": 0.0001239776611328125, "read_cosine": 0.4455904960632324, "beta_mean": 0.35682374238967896, "nonfinite": 0.0, "weighted_memory_to_active": 0.1978238970041275, "mix_std": 0.05431142449378967, "beta_std": 0.1991991102695465, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0031057200394570827, "evidence_weights": [-0.03066166676580906, -0.0207352414727211, 0.08041980862617493], "agreement_weight": -0.013383317738771439, "orthogonal_weight": 0.02232537418603897}, {"head": 2, "memory_to_active": 0.7894189357757568, "contribution_to_active": 0.37943100929260254, "mix_mean": 0.2591065764427185, "mix_low": 0.01637744903564453, "mix_high": 0.0028867721557617188, "read_cosine": 0.7941429615020752, "beta_mean": 0.13881492614746094, "nonfinite": 0.0, "weighted_memory_to_active": 0.5473739504814148, "mix_std": 0.23194193840026855, "beta_std": 0.06440717726945877, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.027645882219076157, "evidence_weights": [-0.2141972929239273, -0.34608909487724304, 0.22148284316062927], "agreement_weight": -0.007911152206361294, "orthogonal_weight": 0.10242483764886856}, {"head": 3, "memory_to_active": 3.2694714069366455, "contribution_to_active": 0.0573948435485363, "mix_mean": 0.007542212028056383, "mix_low": 0.8085851669311523, "mix_high": 0.0, "read_cosine": 0.17690669000148773, "beta_mean": 0.504983127117157, "nonfinite": 0.0, "weighted_memory_to_active": 0.06514640152454376, "mix_std": 0.020519597455859184, "beta_std": 0.24612011015415192, "beta_low": 0.0, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.009804864414036274, "evidence_weights": [0.03591197356581688, 0.028330350294709206, 0.031320273876190186], "agreement_weight": 0.0027954275719821453, "orthogonal_weight": -0.1539260298013687}, {"head": 4, "memory_to_active": 2.84356689453125, "contribution_to_active": 1.1085973978042603, "mix_mean": 0.040187906473875046, "mix_low": 0.2702455520629883, "mix_high": 8.58306884765625e-05, "read_cosine": 0.4123801589012146, "beta_mean": 0.1784115880727768, "nonfinite": 0.0, "weighted_memory_to_active": 0.9991037845611572, "mix_std": 0.07796086370944977, "beta_std": 0.08706203103065491, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11734665930271149, "evidence_weights": [-0.03152168169617653, -0.028186967596411705, 0.16282035410404205], "agreement_weight": -0.031789250671863556, "orthogonal_weight": 0.1292003095149994}, {"head": 5, "memory_to_active": 3.630258560180664, "contribution_to_active": 0.06754737347364426, "mix_mean": 0.008181758224964142, "mix_low": 0.8106250762939453, "mix_high": 6.67572021484375e-05, "read_cosine": 0.14859721064567566, "beta_mean": 0.2291797548532486, "nonfinite": 0.0, "weighted_memory_to_active": 0.06318403780460358, "mix_std": 0.03134671971201897, "beta_std": 0.1843865066766739, "beta_low": 0.0302734375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0006328063900582492, "evidence_weights": [0.09822399914264679, 0.05088537186384201, 0.2558877766132355], "agreement_weight": -0.12142955511808395, "orthogonal_weight": 0.010188126936554909}]}, {"layer": 2, "memory_to_active": 1.2330577373504639, "contribution_to_active": 0.28637704253196716, "mix_mean": 0.19168993830680847, "mix_low": 0.2893216013908386, "mix_high": 0.00128714251331985, "read_cosine": 0.36815184354782104, "beta_mean": 0.11982844769954681, "nonfinite": 0.0, "weighted_memory_to_active": 0.10503879189491272, "mix_std": 0.17039498686790466, "beta_std": 0.0793837159872055, "beta_low": 0.545654296875, "beta_high": 8.138021075865254e-05, "orthogonal_to_active": 0.013078015297651291, "output_influence": 0.21548323333263397, "residual_influence": 0.046153053641319275, "state_active": 0.03707364574074745, "state_memory": 0.029190758243203163, "state_peak": 48.310890197753906, "growth_active": 0.986801266670227, "growth_memory": 1.5125283002853394, "heads": [{"head": 0, "memory_to_active": 0.0637097880244255, "contribution_to_active": 0.09011698514223099, "mix_mean": 0.14280307292938232, "mix_low": 0.13658809661865234, "mix_high": 0.0001163482666015625, "read_cosine": 0.2718227505683899, "beta_mean": 0.00039882774581201375, "nonfinite": 0.0, "weighted_memory_to_active": 0.009969651699066162, "mix_std": 0.19738443195819855, "beta_std": 0.0006741898250766098, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0006539159221574664, "evidence_weights": [-0.07688338309526443, -0.05764761194586754, 0.11078672111034393], "agreement_weight": -0.013002212159335613, "orthogonal_weight": -0.07413981109857559}, {"head": 1, "memory_to_active": 3.6137983798980713, "contribution_to_active": 0.0907927006483078, "mix_mean": 0.005204996559768915, "mix_low": 0.8853168487548828, "mix_high": 0.0, "read_cosine": 0.16791623830795288, "beta_mean": 0.3508296310901642, "nonfinite": 0.0, "weighted_memory_to_active": 0.07849480211734772, "mix_std": 0.010748371481895447, "beta_std": 0.16192498803138733, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016028838232159615, "evidence_weights": [0.0938902497291565, 0.09994887560606003, 0.10761628299951553], "agreement_weight": 0.00492801982909441, "orthogonal_weight": 0.22162193059921265}, {"head": 2, "memory_to_active": 0.06807919591665268, "contribution_to_active": 0.3875068426132202, "mix_mean": 0.33361974358558655, "mix_low": 0.006771087646484375, "mix_high": 0.0007877349853515625, "read_cosine": 0.3668333888053894, "beta_mean": 0.00012029560457449406, "nonfinite": 0.0, "weighted_memory_to_active": 0.02590174600481987, "mix_std": 0.2680763602256775, "beta_std": 0.0002577962295617908, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0037524886429309845, "evidence_weights": [-0.17492496967315674, -0.19375362992286682, 0.2969200313091278], "agreement_weight": 0.024312054738402367, "orthogonal_weight": -0.18823763728141785}, {"head": 3, "memory_to_active": 0.31853607296943665, "contribution_to_active": 0.3950704038143158, "mix_mean": 0.20364335179328918, "mix_low": 0.016042709350585938, "mix_high": 0.0028533935546875, "read_cosine": 0.6575832366943359, "beta_mean": 0.04557656869292259, "nonfinite": 0.0, "weighted_memory_to_active": 0.11844244599342346, "mix_std": 0.21231110394001007, "beta_std": 0.07177571207284927, "beta_low": 0.25384521484375, "beta_high": 0.0, "orthogonal_to_active": 0.04674746096134186, "evidence_weights": [-0.08461761474609375, -0.14925619959831238, 0.4591098725795746], "agreement_weight": 0.07936498522758484, "orthogonal_weight": -0.6172934174537659}, {"head": 4, "memory_to_active": 3.2825160026550293, "contribution_to_active": 0.292920857667923, "mix_mean": 0.014349572360515594, "mix_low": 0.6884965896606445, "mix_high": 0.00010204315185546875, "read_cosine": 0.20921415090560913, "beta_mean": 0.3219425678253174, "nonfinite": 0.0, "weighted_memory_to_active": 0.3692437410354614, "mix_std": 0.048474401235580444, "beta_std": 0.2414635717868805, "beta_low": 0.020263671875, "beta_high": 0.00048828125, "orthogonal_to_active": 0.009621622040867805, "evidence_weights": [0.0420696996152401, 0.035032860934734344, 0.13614852726459503], "agreement_weight": 0.030605200678110123, "orthogonal_weight": -0.039617571979761124}, {"head": 5, "memory_to_active": 0.05170691758394241, "contribution_to_active": 0.4618544578552246, "mix_mean": 0.45051881670951843, "mix_low": 0.0027141571044921875, "mix_high": 0.0038633346557617188, "read_cosine": 0.535541296005249, "beta_mean": 0.00010280685091856867, "nonfinite": 0.0, "weighted_memory_to_active": 0.028180353343486786, "mix_std": 0.28537508845329285, "beta_std": 0.00020601031428668648, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0016637661028653383, "evidence_weights": [-0.2339208573102951, -0.22670674324035645, 0.5242455005645752], "agreement_weight": 0.02446710877120495, "orthogonal_weight": 0.11372481286525726}]}, {"layer": 3, "memory_to_active": 1.6083120107650757, "contribution_to_active": 0.29209020733833313, "mix_mean": 0.13131792843341827, "mix_low": 0.3139861524105072, "mix_high": 0.0012523334007710218, "read_cosine": 0.29840368032455444, "beta_mean": 0.14672428369522095, "nonfinite": 0.0, "weighted_memory_to_active": 0.15398955345153809, "mix_std": 0.16221734881401062, "beta_std": 0.09353644400835037, "beta_low": 0.375885009765625, "beta_high": 0.0, "orthogonal_to_active": 0.022878441959619522, "output_influence": 0.21758568286895752, "residual_influence": 0.037220075726509094, "state_active": 0.03529400750994682, "state_memory": 0.02919854037463665, "state_peak": 30.984769821166992, "growth_active": 1.0088050365447998, "growth_memory": 1.6658047437667847, "heads": [{"head": 0, "memory_to_active": 0.9388440847396851, "contribution_to_active": 0.24516922235488892, "mix_mean": 0.10981623828411102, "mix_low": 0.2971811294555664, "mix_high": 0.00013256072998046875, "read_cosine": 0.25573962926864624, "beta_mean": 0.02128526382148266, "nonfinite": 0.0, "weighted_memory_to_active": 0.16901376843452454, "mix_std": 0.19271206855773926, "beta_std": 0.017697075381875038, "beta_low": 0.22344970703125, "beta_high": 0.0, "orthogonal_to_active": 0.024685099720954895, "evidence_weights": [-0.0017957594245672226, -0.06827693432569504, -0.06441520154476166], "agreement_weight": -0.02020280435681343, "orthogonal_weight": -0.1468227207660675}, {"head": 1, "memory_to_active": 0.1380135416984558, "contribution_to_active": 0.2166331559419632, "mix_mean": 0.1496407687664032, "mix_low": 0.13714599609375, "mix_high": 0.0005588531494140625, "read_cosine": 0.23893380165100098, "beta_mean": 0.0006049040239304304, "nonfinite": 0.0, "weighted_memory_to_active": 0.019228126853704453, "mix_std": 0.20531553030014038, "beta_std": 0.0010321240406483412, "beta_low": 0.998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.0010895865270867944, "evidence_weights": [-0.09809118509292603, -0.07281044125556946, 0.23910528421401978], "agreement_weight": -0.019058993086218834, "orthogonal_weight": -0.05807076394557953}, {"head": 2, "memory_to_active": 3.9123587608337402, "contribution_to_active": 0.29489776492118835, "mix_mean": 0.033495787531137466, "mix_low": 0.43500709533691406, "mix_high": 6.67572021484375e-06, "read_cosine": 0.20313343405723572, "beta_mean": 0.30312055349349976, "nonfinite": 0.0, "weighted_memory_to_active": 0.39691370725631714, "mix_std": 0.08111830800771713, "beta_std": 0.18044790625572205, "beta_low": 0.00201416015625, "beta_high": 0.0, "orthogonal_to_active": 0.04275874048471451, "evidence_weights": [-0.034514110535383224, 0.00803079642355442, 0.1141471415758133], "agreement_weight": -0.017302656546235085, "orthogonal_weight": -0.13866186141967773}, {"head": 3, "memory_to_active": 0.04513116553425789, "contribution_to_active": 0.43994903564453125, "mix_mean": 0.3650614321231842, "mix_low": 0.03221702575683594, "mix_high": 0.0030994415283203125, "read_cosine": 0.31986087560653687, "beta_mean": 0.00015168165555223823, "nonfinite": 0.0, "weighted_memory_to_active": 0.022626379504799843, "mix_std": 0.29397451877593994, "beta_std": 0.0003866598126478493, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.004778169561177492, "evidence_weights": [-0.17518745362758636, -0.19325067102909088, 0.781898021697998], "agreement_weight": 0.4447866380214691, "orthogonal_weight": -0.3009125590324402}, {"head": 4, "memory_to_active": 4.08585262298584, "contribution_to_active": 0.031166033819317818, "mix_mean": 0.005857967305928469, "mix_low": 0.8597202301025391, "mix_high": 0.0, "read_cosine": 0.23739726841449738, "beta_mean": 0.4232722520828247, "nonfinite": 0.0, "weighted_memory_to_active": 0.03872235119342804, "mix_std": 0.012651648372411728, "beta_std": 0.22469758987426758, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008807765319943428, "evidence_weights": [0.14967231452465057, 0.13060250878334045, -0.12938344478607178], "agreement_weight": -0.07319598644971848, "orthogonal_weight": -0.2361316829919815}, {"head": 5, "memory_to_active": 0.5296710133552551, "contribution_to_active": 0.5247259140014648, "mix_mean": 0.12403535097837448, "mix_low": 0.12264537811279297, "mix_high": 0.0037164688110351562, "read_cosine": 0.5353570580482483, "beta_mean": 0.1319110095500946, "nonfinite": 0.0, "weighted_memory_to_active": 0.27743300795555115, "mix_std": 0.18753193318843842, "beta_std": 0.13695725798606873, "beta_low": 0.03173828125, "beta_high": 0.0, "orthogonal_to_active": 0.05515130236744881, "evidence_weights": [-0.04283381253480911, -0.12842029333114624, 0.07159304618835449], "agreement_weight": 0.1602642983198166, "orthogonal_weight": -0.3235095143318176}]}, {"layer": 4, "memory_to_active": 1.374694585800171, "contribution_to_active": 0.3049790561199188, "mix_mean": 0.23041918873786926, "mix_low": 0.2742190361022949, "mix_high": 0.0011402766685932875, "read_cosine": 0.40795639157295227, "beta_mean": 0.13754597306251526, "nonfinite": 0.0, "weighted_memory_to_active": 0.1676240861415863, "mix_std": 0.149148091673851, "beta_std": 0.11228100955486298, "beta_low": 0.2449137568473816, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03176873177289963, "output_influence": 0.2104163020849228, "residual_influence": 0.045504044741392136, "state_active": 0.0453268326818943, "state_memory": 0.03888573870062828, "state_peak": 49.33306121826172, "growth_active": 1.0697561502456665, "growth_memory": 1.6456769704818726, "heads": [{"head": 0, "memory_to_active": 0.4501078128814697, "contribution_to_active": 0.49351295828819275, "mix_mean": 0.4015148878097534, "mix_low": 0.010210990905761719, "mix_high": 0.0006809234619140625, "read_cosine": 0.6217465996742249, "beta_mean": 0.12348051369190216, "nonfinite": 0.0, "weighted_memory_to_active": 0.2785780727863312, "mix_std": 0.2794855237007141, "beta_std": 0.16012223064899445, "beta_low": 0.15106201171875, "beta_high": 0.0, "orthogonal_to_active": 0.0014459093799814582, "evidence_weights": [-0.19312499463558197, -0.23595337569713593, 0.4161500334739685], "agreement_weight": 0.0694211795926094, "orthogonal_weight": 0.007215018384158611}, {"head": 1, "memory_to_active": 0.49005410075187683, "contribution_to_active": 0.2873859107494354, "mix_mean": 0.31634828448295593, "mix_low": 0.00904083251953125, "mix_high": 0.0006761550903320312, "read_cosine": 0.5399987101554871, "beta_mean": 0.0288271214812994, "nonfinite": 0.0, "weighted_memory_to_active": 0.1869850754737854, "mix_std": 0.24952252209186554, "beta_std": 0.03424197435379028, "beta_low": 0.25799560546875, "beta_high": 0.0, "orthogonal_to_active": 0.08470983058214188, "evidence_weights": [-0.17634661495685577, -0.1618681102991104, 0.5814938545227051], "agreement_weight": 0.04589599370956421, "orthogonal_weight": -0.5461279153823853}, {"head": 2, "memory_to_active": 2.829873561859131, "contribution_to_active": 0.10569217801094055, "mix_mean": 0.013540491461753845, "mix_low": 0.6637887954711914, "mix_high": 0.0, "read_cosine": 0.2993035912513733, "beta_mean": 0.21364665031433105, "nonfinite": 0.0, "weighted_memory_to_active": 0.13169898092746735, "mix_std": 0.023475078865885735, "beta_std": 0.14086252450942993, "beta_low": 0.0001220703125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0006820824928581715, "evidence_weights": [0.0348261259496212, 0.01867063343524933, -0.06347418576478958], "agreement_weight": 0.03311645984649658, "orthogonal_weight": -0.007509906310588121}, {"head": 3, "memory_to_active": 1.9026200771331787, "contribution_to_active": 0.09893642365932465, "mix_mean": 0.026687294244766235, "mix_low": 0.4005146026611328, "mix_high": 0.0, "read_cosine": 0.22287046909332275, "beta_mean": 0.17547357082366943, "nonfinite": 0.0, "weighted_memory_to_active": 0.16507375240325928, "mix_std": 0.04411524534225464, "beta_std": 0.16333851218223572, "beta_low": 0.06036376953125, "beta_high": 0.0, "orthogonal_to_active": 0.022496985271573067, "evidence_weights": [-0.018163520842790604, 0.03221527114510536, -0.1339264214038849], "agreement_weight": 0.01975618302822113, "orthogonal_weight": -0.2239806205034256}, {"head": 4, "memory_to_active": 0.027998263016343117, "contribution_to_active": 0.5781425833702087, "mix_mean": 0.6072714328765869, "mix_low": 0.0009641647338867188, "mix_high": 0.005457878112792969, "read_cosine": 0.33495399355888367, "beta_mean": 6.50852161925286e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.018801745027303696, "mix_std": 0.261148601770401, "beta_std": 0.00014238250150810927, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0022019739262759686, "evidence_weights": [-0.2686033546924591, -0.30187398195266724, 0.6683249473571777], "agreement_weight": 0.02862209640443325, "orthogonal_weight": -0.14412225782871246}, {"head": 5, "memory_to_active": 2.5475125312805176, "contribution_to_active": 0.26620420813560486, "mix_mean": 0.017152689397335052, "mix_low": 0.5607948303222656, "mix_high": 2.6702880859375e-05, "read_cosine": 0.42886483669281006, "beta_mean": 0.28378286957740784, "nonfinite": 0.0, "weighted_memory_to_active": 0.22460685670375824, "mix_std": 0.0371415950357914, "beta_std": 0.17497842013835907, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07907559722661972, "evidence_weights": [0.025217631831765175, -0.029598379507660866, -0.07262086868286133], "agreement_weight": -0.04677175357937813, "orthogonal_weight": 0.5466839671134949}]}, {"layer": 5, "memory_to_active": 0.7439674139022827, "contribution_to_active": 0.40739065408706665, "mix_mean": 0.3570871949195862, "mix_low": 0.11975876986980438, "mix_high": 0.00760253332555294, "read_cosine": 0.25720852613449097, "beta_mean": 0.07772929966449738, "nonfinite": 0.0, "weighted_memory_to_active": 0.16645628213882446, "mix_std": 0.23442500829696655, "beta_std": 0.08803260326385498, "beta_low": 0.521697998046875, "beta_high": 0.0, "orthogonal_to_active": 0.041741497814655304, "output_influence": 0.3845612406730652, "residual_influence": 0.08107510209083557, "state_active": 0.0638481080532074, "state_memory": 0.024978311732411385, "state_peak": 53.60300827026367, "growth_active": 1.1143715381622314, "growth_memory": 1.4699630737304688, "heads": [{"head": 0, "memory_to_active": 1.4380123615264893, "contribution_to_active": 0.1988772451877594, "mix_mean": 0.07300703227519989, "mix_low": 0.21410655975341797, "mix_high": 0.0, "read_cosine": -0.11924004554748535, "beta_mean": 0.16573664546012878, "nonfinite": 0.0, "weighted_memory_to_active": 0.18559718132019043, "mix_std": 0.11095702648162842, "beta_std": 0.1727244108915329, "beta_low": 0.0128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.040232785046100616, "evidence_weights": [-0.01822366937994957, -0.04567137733101845, -0.30874180793762207], "agreement_weight": -0.0300957839936018, "orthogonal_weight": -0.2192324995994568}, {"head": 1, "memory_to_active": 0.042387865483760834, "contribution_to_active": 0.41837278008461, "mix_mean": 0.4530976116657257, "mix_low": 0.014733314514160156, "mix_high": 0.007077217102050781, "read_cosine": 0.10167192667722702, "beta_mean": 0.00012069071817677468, "nonfinite": 0.0, "weighted_memory_to_active": 0.017090538516640663, "mix_std": 0.3155122399330139, "beta_std": 0.00021359595120884478, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013866104185581207, "evidence_weights": [-0.20535601675510406, -0.2043861597776413, 0.43514081835746765], "agreement_weight": 0.1881302148103714, "orthogonal_weight": -0.8342380523681641}, {"head": 2, "memory_to_active": 0.1724555492401123, "contribution_to_active": 0.4805728793144226, "mix_mean": 0.5371389389038086, "mix_low": 0.001377105712890625, "mix_high": 0.0069122314453125, "read_cosine": 0.5733824968338013, "beta_mean": 0.008083772845566273, "nonfinite": 0.0, "weighted_memory_to_active": 0.1123766228556633, "mix_std": 0.27308911085128784, "beta_std": 0.008838125504553318, "beta_low": 0.745849609375, "beta_high": 0.0, "orthogonal_to_active": 0.013108933344483376, "evidence_weights": [-0.27559369802474976, -0.27968549728393555, 0.5854634642601013], "agreement_weight": 0.027971144765615463, "orthogonal_weight": -0.16875790059566498}, {"head": 3, "memory_to_active": 2.2018814086914062, "contribution_to_active": 0.1618524193763733, "mix_mean": 0.047913748770952225, "mix_low": 0.36104488372802734, "mix_high": 0.0, "read_cosine": 0.1452273279428482, "beta_mean": 0.08875314891338348, "nonfinite": 0.0, "weighted_memory_to_active": 0.2689254879951477, "mix_std": 0.0938531830906868, "beta_std": 0.11927605420351028, "beta_low": 0.17999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.08693380653858185, "evidence_weights": [0.008599093183875084, -0.03448116406798363, 0.09526658058166504], "agreement_weight": 0.06498709321022034, "orthogonal_weight": -0.43459126353263855}, {"head": 4, "memory_to_active": 0.5660231113433838, "contribution_to_active": 0.5884569883346558, "mix_mean": 0.3492802381515503, "mix_low": 0.12313079833984375, "mix_high": 0.0162200927734375, "read_cosine": 0.5803649425506592, "beta_mean": 0.20350214838981628, "nonfinite": 0.0, "weighted_memory_to_active": 0.38820767402648926, "mix_std": 0.3429013788700104, "beta_std": 0.2265501171350479, "beta_low": 0.1922607421875, "beta_high": 0.0, "orthogonal_to_active": 0.08640246838331223, "evidence_weights": [-0.15802784264087677, -0.2622761130332947, 0.13048280775547028], "agreement_weight": 0.05391952395439148, "orthogonal_weight": 0.32952627539634705}, {"head": 5, "memory_to_active": 0.04304388165473938, "contribution_to_active": 0.5962114930152893, "mix_mean": 0.6820855736732483, "mix_low": 0.0041599273681640625, "mix_high": 0.015405654907226562, "read_cosine": 0.26184457540512085, "beta_mean": 0.00017936769290827215, "nonfinite": 0.0, "weighted_memory_to_active": 0.026540104299783707, "mix_std": 0.2702370584011078, "beta_std": 0.0005933011416345835, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.00990491546690464, "evidence_weights": [-0.3044275939464569, -0.29347237944602966, 0.8689466714859009], "agreement_weight": 0.28774333000183105, "orthogonal_weight": -0.41828155517578125}]}, {"layer": 6, "memory_to_active": 0.4861380159854889, "contribution_to_active": 0.5315276980400085, "mix_mean": 0.4342121183872223, "mix_low": 0.1027485579252243, "mix_high": 0.01694456674158573, "read_cosine": 0.3406667113304138, "beta_mean": 0.08097276091575623, "nonfinite": 0.0, "weighted_memory_to_active": 0.18874697387218475, "mix_std": 0.26451897621154785, "beta_std": 0.09798754751682281, "beta_low": 0.4402669370174408, "beta_high": 0.0, "orthogonal_to_active": 0.038945481181144714, "output_influence": 0.3980899751186371, "residual_influence": 0.11205166578292847, "state_active": 0.06469059735536575, "state_memory": 0.01596478372812271, "state_peak": 53.94631576538086, "growth_active": 1.2525386810302734, "growth_memory": 1.330634355545044, "heads": [{"head": 0, "memory_to_active": 0.24160116910934448, "contribution_to_active": 0.4901849329471588, "mix_mean": 0.5098605155944824, "mix_low": 0.00138092041015625, "mix_high": 0.0015420913696289062, "read_cosine": 0.38311073184013367, "beta_mean": 0.034192025661468506, "nonfinite": 0.0, "weighted_memory_to_active": 0.17702609300613403, "mix_std": 0.2738232910633087, "beta_std": 0.07994260638952255, "beta_low": 0.29107666015625, "beta_high": 0.0, "orthogonal_to_active": 0.10341446101665497, "evidence_weights": [-0.24680964648723602, -0.241988405585289, 0.2619363069534302], "agreement_weight": 0.09763934463262558, "orthogonal_weight": -0.7967899441719055}, {"head": 1, "memory_to_active": 0.007369927130639553, "contribution_to_active": 0.4020460247993469, "mix_mean": 0.3803858160972595, "mix_low": 0.008981704711914062, "mix_high": 0.008678436279296875, "read_cosine": 0.5471434593200684, "beta_mean": 1.7628181012696587e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0026043360121548176, "mix_std": 0.2881326377391815, "beta_std": 5.5386593885486946e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005116202519275248, "evidence_weights": [-0.21147553622722626, -0.21960791945457458, 0.41897934675216675], "agreement_weight": -0.020248299464583397, "orthogonal_weight": -0.26912161707878113}, {"head": 2, "memory_to_active": 1.652076005935669, "contribution_to_active": 0.06655362993478775, "mix_mean": 0.020046107470989227, "mix_low": 0.5841131210327148, "mix_high": 0.0, "read_cosine": 0.20852433145046234, "beta_mean": 0.09302487969398499, "nonfinite": 0.0, "weighted_memory_to_active": 0.07865124940872192, "mix_std": 0.048202645033597946, "beta_std": 0.08736521750688553, "beta_low": 0.00604248046875, "beta_high": 0.0, "orthogonal_to_active": 0.01681169681251049, "evidence_weights": [0.08632880449295044, 0.05272192507982254, 0.03507698327302933], "agreement_weight": 0.027949245646595955, "orthogonal_weight": -0.2371341437101364}, {"head": 3, "memory_to_active": 0.19198183715343475, "contribution_to_active": 0.6422411799430847, "mix_mean": 0.5767714977264404, "mix_low": 0.013054847717285156, "mix_high": 0.028212547302246094, "read_cosine": 0.5455023646354675, "beta_mean": 0.22789543867111206, "nonfinite": 0.0, "weighted_memory_to_active": 0.1311911940574646, "mix_std": 0.3513268530368805, "beta_std": 0.259130597114563, "beta_low": 0.16070556640625, "beta_high": 0.0, "orthogonal_to_active": 0.04339039698243141, "evidence_weights": [-0.1603311449289322, -0.30897796154022217, 0.21136675775051117], "agreement_weight": -0.012571066617965698, "orthogonal_weight": 0.407186895608902}, {"head": 4, "memory_to_active": 0.7639464139938354, "contribution_to_active": 1.0842949151992798, "mix_mean": 0.648289680480957, "mix_low": 0.0038003921508789062, "mix_high": 0.06283950805664062, "read_cosine": 0.0694657415151596, "beta_mean": 0.13023746013641357, "nonfinite": 0.0, "weighted_memory_to_active": 0.709568977355957, "mix_std": 0.32626545429229736, "beta_std": 0.1606919765472412, "beta_low": 0.18408203125, "beta_high": 0.0, "orthogonal_to_active": 0.03723843768239021, "evidence_weights": [-0.23291216790676117, -0.49036484956741333, -0.021393731236457825], "agreement_weight": -0.010912816040217876, "orthogonal_weight": 0.0644146129488945}, {"head": 5, "memory_to_active": 0.05985249578952789, "contribution_to_active": 0.5038455128669739, "mix_mean": 0.46991902589797974, "mix_low": 0.005160331726074219, "mix_high": 0.0003948211669921875, "read_cosine": 0.2902534008026123, "beta_mean": 0.0004691363428719342, "nonfinite": 0.0, "weighted_memory_to_active": 0.033440008759498596, "mix_std": 0.29936301708221436, "beta_std": 0.0007395079010166228, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.032306261360645294, "evidence_weights": [-0.20016874372959137, -0.23821011185646057, 0.15750828385353088], "agreement_weight": 0.07270115613937378, "orthogonal_weight": -1.0427665710449219}]}, {"layer": 7, "memory_to_active": 2.282804489135742, "contribution_to_active": 0.17408928275108337, "mix_mean": 0.025434214621782303, "mix_low": 0.5775274038314819, "mix_high": 1.4781952813791577e-05, "read_cosine": 0.22680772840976715, "beta_mean": 0.14659324288368225, "nonfinite": 0.0, "weighted_memory_to_active": 0.1789664775133133, "mix_std": 0.0494484007358551, "beta_std": 0.11011529713869095, "beta_low": 0.01642862893640995, "beta_high": 0.0, "orthogonal_to_active": 0.02949451096355915, "output_influence": 0.18551990389823914, "residual_influence": 0.03907638043165207, "state_active": 0.019312813878059387, "state_memory": 0.037306539714336395, "state_peak": 20.536008834838867, "growth_active": 1.005031704902649, "growth_memory": 1.4564543962478638, "heads": [{"head": 0, "memory_to_active": 2.1198923587799072, "contribution_to_active": 0.4157342314720154, "mix_mean": 0.02645087242126465, "mix_low": 0.5509366989135742, "mix_high": 8.7738037109375e-05, "read_cosine": 0.4187381863594055, "beta_mean": 0.13641811907291412, "nonfinite": 0.0, "weighted_memory_to_active": 0.3966834247112274, "mix_std": 0.05842979997396469, "beta_std": 0.11649753153324127, "beta_low": 0.0032958984375, "beta_high": 0.0, "orthogonal_to_active": 0.07453849166631699, "evidence_weights": [-0.012851838953793049, -0.07446489483118057, 0.0909416526556015], "agreement_weight": -0.034737229347229004, "orthogonal_weight": 0.26472756266593933}, {"head": 1, "memory_to_active": 2.9197707176208496, "contribution_to_active": 0.26525968313217163, "mix_mean": 0.05089682713150978, "mix_low": 0.33580970764160156, "mix_high": 0.0, "read_cosine": -0.011965606361627579, "beta_mean": 0.24877434968948364, "nonfinite": 0.0, "weighted_memory_to_active": 0.3196735084056854, "mix_std": 0.07870634645223618, "beta_std": 0.12478644400835037, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.069733165204525, "evidence_weights": [-0.09526318311691284, -0.08305119723081589, -0.020258724689483643], "agreement_weight": 0.005702564958482981, "orthogonal_weight": -0.21565163135528564}, {"head": 2, "memory_to_active": 1.4580483436584473, "contribution_to_active": 0.214972123503685, "mix_mean": 0.043707702308893204, "mix_low": 0.38230228424072266, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19183705747127533, "beta_mean": 0.1529785543680191, "nonfinite": 0.0, "weighted_memory_to_active": 0.18338654935359955, "mix_std": 0.09197686612606049, "beta_std": 0.09060508012771606, "beta_low": 0.001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.005193564109504223, "evidence_weights": [-0.03738970682024956, -0.10539829730987549, 0.19604092836380005], "agreement_weight": 0.02495530992746353, "orthogonal_weight": -0.029651373624801636}, {"head": 3, "memory_to_active": 3.076380491256714, "contribution_to_active": 0.040213849395513535, "mix_mean": 0.009153308346867561, "mix_low": 0.7353105545043945, "mix_high": 0.0, "read_cosine": 0.1452416479587555, "beta_mean": 0.1327448934316635, "nonfinite": 0.0, "weighted_memory_to_active": 0.05857965350151062, "mix_std": 0.01737889088690281, "beta_std": 0.11951714009046555, "beta_low": 0.01214599609375, "beta_high": 0.0, "orthogonal_to_active": 0.01980840414762497, "evidence_weights": [0.1037384420633316, 0.08481691032648087, 0.1197841614484787], "agreement_weight": 0.053984954953193665, "orthogonal_weight": -0.36689621210098267}, {"head": 4, "memory_to_active": 1.5434565544128418, "contribution_to_active": 0.05193730443716049, "mix_mean": 0.010916619561612606, "mix_low": 0.7464361190795898, "mix_high": 0.0, "read_cosine": 0.30606621503829956, "beta_mean": 0.11340058594942093, "nonfinite": 0.0, "weighted_memory_to_active": 0.04688979685306549, "mix_std": 0.02747265249490738, "beta_std": 0.11376455426216125, "beta_low": 0.0452880859375, "beta_high": 0.0, "orthogonal_to_active": 0.0024123135954141617, "evidence_weights": [0.06578369438648224, 0.03527975082397461, 0.003714190563187003], "agreement_weight": 0.04900149628520012, "orthogonal_weight": -0.057824548333883286}, {"head": 5, "memory_to_active": 2.579277276992798, "contribution_to_active": 0.056418515741825104, "mix_mean": 0.011479957029223442, "mix_low": 0.7143688201904297, "mix_high": 0.0, "read_cosine": 0.3109287917613983, "beta_mean": 0.09524288773536682, "nonfinite": 0.0, "weighted_memory_to_active": 0.06858585774898529, "mix_std": 0.022725846618413925, "beta_std": 0.09552101045846939, "beta_low": 0.03662109375, "beta_high": 0.0, "orthogonal_to_active": 0.005281124729663134, "evidence_weights": [0.057658713310956955, 0.07725713402032852, -0.1311357468366623], "agreement_weight": -0.0022820685990154743, "orthogonal_weight": -0.08866170793771744}]}, {"layer": 8, "memory_to_active": 0.8338619470596313, "contribution_to_active": 0.38391879200935364, "mix_mean": 0.2613529860973358, "mix_low": 0.08821837604045868, "mix_high": 0.0008962949505075812, "read_cosine": 0.5259733200073242, "beta_mean": 0.05807624012231827, "nonfinite": 0.0, "weighted_memory_to_active": 0.17969325184822083, "mix_std": 0.2000052034854889, "beta_std": 0.05352241173386574, "beta_low": 0.297637939453125, "beta_high": 0.0, "orthogonal_to_active": 0.04543287307024002, "output_influence": 0.2756692171096802, "residual_influence": 0.06417161971330643, "state_active": 0.05584900826215744, "state_memory": 0.021989652886986732, "state_peak": 32.536678314208984, "growth_active": 0.9982632994651794, "growth_memory": 1.508475422859192, "heads": [{"head": 0, "memory_to_active": 0.07038899511098862, "contribution_to_active": 0.47470712661743164, "mix_mean": 0.49338436126708984, "mix_low": 0.0001735687255859375, "mix_high": 0.00015354156494140625, "read_cosine": 0.7409289479255676, "beta_mean": 0.0036375359632074833, "nonfinite": 0.0, "weighted_memory_to_active": 0.03690599650144577, "mix_std": 0.2381172627210617, "beta_std": 0.005876832641661167, "beta_low": 0.91925048828125, "beta_high": 0.0, "orthogonal_to_active": 0.015826137736439705, "evidence_weights": [-0.2469329535961151, -0.3026256561279297, 0.5910132527351379], "agreement_weight": 0.06925290822982788, "orthogonal_weight": -0.6838591694831848}, {"head": 1, "memory_to_active": 0.6284210681915283, "contribution_to_active": 0.32271331548690796, "mix_mean": 0.29728949069976807, "mix_low": 0.008589744567871094, "mix_high": 0.000152587890625, "read_cosine": 0.5546888113021851, "beta_mean": 0.05519440397620201, "nonfinite": 0.0, "weighted_memory_to_active": 0.2233046144247055, "mix_std": 0.20766711235046387, "beta_std": 0.04133689031004906, "beta_low": 0.05224609375, "beta_high": 0.0, "orthogonal_to_active": 0.003264214377850294, "evidence_weights": [-0.16505737602710724, -0.17608733475208282, 0.3106444180011749], "agreement_weight": 0.03422078862786293, "orthogonal_weight": 0.018228307366371155}, {"head": 2, "memory_to_active": 0.5067971348762512, "contribution_to_active": 0.2624457776546478, "mix_mean": 0.218084454536438, "mix_low": 0.034470558166503906, "mix_high": 2.86102294921875e-06, "read_cosine": 0.29475346207618713, "beta_mean": 0.018086515367031097, "nonfinite": 0.0, "weighted_memory_to_active": 0.1492847353219986, "mix_std": 0.21084854006767273, "beta_std": 0.02000582590699196, "beta_low": 0.3994140625, "beta_high": 0.0, "orthogonal_to_active": 0.08808546513319016, "evidence_weights": [-0.10905598849058151, -0.19734393060207367, 0.19130021333694458], "agreement_weight": 0.0529339574277401, "orthogonal_weight": -0.6471678614616394}, {"head": 3, "memory_to_active": 0.4366929233074188, "contribution_to_active": 0.241933211684227, "mix_mean": 0.195243239402771, "mix_low": 0.02235698699951172, "mix_high": 0.0, "read_cosine": 0.6901686191558838, "beta_mean": 0.027104010805487633, "nonfinite": 0.0, "weighted_memory_to_active": 0.15419161319732666, "mix_std": 0.16659125685691833, "beta_std": 0.026458067819476128, "beta_low": 0.16278076171875, "beta_high": 0.0, "orthogonal_to_active": 0.021724944934248924, "evidence_weights": [-0.07499485462903976, -0.15204168856143951, 0.5840653777122498], "agreement_weight": 0.06541960686445236, "orthogonal_weight": 0.24601882696151733}, {"head": 4, "memory_to_active": 3.124600887298584, "contribution_to_active": 0.3678124248981476, "mix_mean": 0.0276702418923378, "mix_low": 0.3840293884277344, "mix_high": 0.0, "read_cosine": 0.33760181069374084, "beta_mean": 0.16761332750320435, "nonfinite": 0.0, "weighted_memory_to_active": 0.35579317808151245, "mix_std": 0.04456200450658798, "beta_std": 0.11782637983560562, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.017696231603622437, "evidence_weights": [0.012349443510174751, -0.011257331818342209, -0.045162323862314224], "agreement_weight": 0.006293713115155697, "orthogonal_weight": 0.05225902795791626}, {"head": 5, "memory_to_active": 0.23627054691314697, "contribution_to_active": 0.63390052318573, "mix_mean": 0.3364461064338684, "mix_low": 0.07968997955322266, "mix_high": 0.005068778991699219, "read_cosine": 0.5376980304718018, "beta_mean": 0.07682166248559952, "nonfinite": 0.0, "weighted_memory_to_active": 0.15867938101291656, "mix_std": 0.33224499225616455, "beta_std": 0.10963047295808792, "beta_low": 0.2520751953125, "beta_high": 0.0, "orthogonal_to_active": 0.12600024044513702, "evidence_weights": [-0.08195925503969193, -0.3452657163143158, -0.3455042243003845], "agreement_weight": 0.026455454528331757, "orthogonal_weight": 0.9860184788703918}]}, {"layer": 9, "memory_to_active": 1.288118600845337, "contribution_to_active": 0.5496324300765991, "mix_mean": 0.17688581347465515, "mix_low": 0.20717477798461914, "mix_high": 0.0013812383404001594, "read_cosine": 0.3589150905609131, "beta_mean": 0.09421907365322113, "nonfinite": 0.0, "weighted_memory_to_active": 0.3794994056224823, "mix_std": 0.20267128944396973, "beta_std": 0.05119703710079193, "beta_low": 0.4878947138786316, "beta_high": 0.0, "orthogonal_to_active": 0.02016211673617363, "output_influence": 0.3258599638938904, "residual_influence": 0.07455772161483765, "state_active": 0.03743109107017517, "state_memory": 0.030223090201616287, "state_peak": 39.525001525878906, "growth_active": 1.1203616857528687, "growth_memory": 1.4590725898742676, "heads": [{"head": 0, "memory_to_active": 0.009477948769927025, "contribution_to_active": 0.3753083050251007, "mix_mean": 0.369465708732605, "mix_low": 0.01633453369140625, "mix_high": 0.0024709701538085938, "read_cosine": 0.23976559937000275, "beta_mean": 2.3916829377412796e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004208697471767664, "mix_std": 0.2872338891029358, "beta_std": 6.379927799571306e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002229098929092288, "evidence_weights": [-0.2163686603307724, -0.20220616459846497, 0.8661410212516785], "agreement_weight": 0.047819435596466064, "orthogonal_weight": -0.555213987827301}, {"head": 1, "memory_to_active": 0.02566477097570896, "contribution_to_active": 0.3597443103790283, "mix_mean": 0.2038845717906952, "mix_low": 0.08319854736328125, "mix_high": 0.0005712509155273438, "read_cosine": 0.4215432405471802, "beta_mean": 7.027722313068807e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008168242871761322, "mix_std": 0.23159988224506378, "beta_std": 0.00014537917741108686, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0027708904817700386, "evidence_weights": [-0.1520889550447464, -0.145717054605484, 0.49645382165908813], "agreement_weight": 0.2478480190038681, "orthogonal_weight": -0.37037649750709534}, {"head": 2, "memory_to_active": 0.328307569026947, "contribution_to_active": 0.4351189136505127, "mix_mean": 0.26821014285087585, "mix_low": 0.043747901916503906, "mix_high": 0.000202178955078125, "read_cosine": 0.3881503641605377, "beta_mean": 0.0037285368889570236, "nonfinite": 0.0, "weighted_memory_to_active": 0.15612459182739258, "mix_std": 0.27057117223739624, "beta_std": 0.006540228612720966, "beta_low": 0.9248046875, "beta_high": 0.0, "orthogonal_to_active": 0.02529107965528965, "evidence_weights": [-0.11717883497476578, -0.11709976196289062, 0.17769312858581543], "agreement_weight": 0.12643015384674072, "orthogonal_weight": -0.20609891414642334}, {"head": 3, "memory_to_active": 1.9336695671081543, "contribution_to_active": 0.20920532941818237, "mix_mean": 0.04450958967208862, "mix_low": 0.36577606201171875, "mix_high": 0.0, "read_cosine": 0.437911719083786, "beta_mean": 0.10659704357385635, "nonfinite": 0.0, "weighted_memory_to_active": 0.16924317181110382, "mix_std": 0.09813894331455231, "beta_std": 0.06841598451137543, "beta_low": 0.00250244140625, "beta_high": 0.0, "orthogonal_to_active": 0.020809350535273552, "evidence_weights": [0.030767254531383514, 0.021253015846014023, -0.07529932260513306], "agreement_weight": 0.008246355690062046, "orthogonal_weight": 0.1185423955321312}, {"head": 4, "memory_to_active": 3.5223300457000732, "contribution_to_active": 1.3860812187194824, "mix_mean": 0.12160792946815491, "mix_low": 0.24927330017089844, "mix_high": 0.004986763000488281, "read_cosine": 0.2370436191558838, "beta_mean": 0.24054671823978424, "nonfinite": 0.0, "weighted_memory_to_active": 1.3094568252563477, "mix_std": 0.20843058824539185, "beta_std": 0.11470931768417358, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006895981263369322, "evidence_weights": [-0.04107149317860603, -0.10157796740531921, 0.09312693774700165], "agreement_weight": -0.04731365293264389, "orthogonal_weight": 0.00533044571056962}, {"head": 5, "memory_to_active": 1.9092621803283691, "contribution_to_active": 0.5323362946510315, "mix_mean": 0.05363689363002777, "mix_low": 0.48471832275390625, "mix_high": 5.626678466796875e-05, "read_cosine": 0.42907559871673584, "beta_mean": 0.2143479436635971, "nonfinite": 0.0, "weighted_memory_to_active": 0.6297948956489563, "mix_std": 0.12005329877138138, "beta_std": 0.11730751395225525, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.06297630071640015, "evidence_weights": [0.030779706314206123, -0.035191960632801056, 0.055469855666160583], "agreement_weight": -0.014909042045474052, "orthogonal_weight": 0.13597926497459412}]}, {"layer": 10, "memory_to_active": 0.2584301233291626, "contribution_to_active": 0.37826183438301086, "mix_mean": 0.3232157826423645, "mix_low": 0.06158463656902313, "mix_high": 0.00878000259399414, "read_cosine": 0.4006023406982422, "beta_mean": 0.047447994351387024, "nonfinite": 0.0, "weighted_memory_to_active": 0.10068757086992264, "mix_std": 0.24176600575447083, "beta_std": 0.05379544943571091, "beta_low": 0.5769450068473816, "beta_high": 0.0, "orthogonal_to_active": 0.03373746946454048, "output_influence": 0.3714807629585266, "residual_influence": 0.10462071746587753, "state_active": 0.07401445508003235, "state_memory": 0.017405778169631958, "state_peak": 69.7154312133789, "growth_active": 1.130307674407959, "growth_memory": 1.1828533411026, "heads": [{"head": 0, "memory_to_active": 0.40387699007987976, "contribution_to_active": 0.7349022626876831, "mix_mean": 0.6194712519645691, "mix_low": 0.014039039611816406, "mix_high": 0.051769256591796875, "read_cosine": 0.21900326013565063, "beta_mean": 0.19819419085979462, "nonfinite": 0.0, "weighted_memory_to_active": 0.29109129309654236, "mix_std": 0.3481140434741974, "beta_std": 0.21626894176006317, "beta_low": 0.1048583984375, "beta_high": 0.0, "orthogonal_to_active": 0.11145022511482239, "evidence_weights": [-0.15609723329544067, -0.3302714228630066, -0.17436349391937256], "agreement_weight": 0.02733519859611988, "orthogonal_weight": 0.43020305037498474}, {"head": 1, "memory_to_active": 0.002174952533096075, "contribution_to_active": 0.39824870228767395, "mix_mean": 0.35243868827819824, "mix_low": 0.007824897766113281, "mix_high": 0.00022220611572265625, "read_cosine": 0.22848713397979736, "beta_mean": 1.264064667338971e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.000814981060102582, "mix_std": 0.25224193930625916, "beta_std": 2.6106707082362846e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00041445397073403, "evidence_weights": [-0.21530845761299133, -0.15776734054088593, 0.19068531692028046], "agreement_weight": 0.07123256474733353, "orthogonal_weight": -0.55863356590271}, {"head": 2, "memory_to_active": 0.7473719716072083, "contribution_to_active": 0.18853607773780823, "mix_mean": 0.08249767869710922, "mix_low": 0.30169010162353516, "mix_high": 4.76837158203125e-06, "read_cosine": 0.20794442296028137, "beta_mean": 0.02048128843307495, "nonfinite": 0.0, "weighted_memory_to_active": 0.17093028128147125, "mix_std": 0.14721016585826874, "beta_std": 0.030855875462293625, "beta_low": 0.49365234375, "beta_high": 0.0, "orthogonal_to_active": 0.030194085091352463, "evidence_weights": [0.005051091779023409, -0.07217280566692352, 0.15095938742160797], "agreement_weight": -0.01286034844815731, "orthogonal_weight": -0.18041256070137024}, {"head": 3, "memory_to_active": 0.22161367535591125, "contribution_to_active": 0.16032148897647858, "mix_mean": 0.2740270495414734, "mix_low": 0.03202342987060547, "mix_high": 0.0006542205810546875, "read_cosine": 0.8252696990966797, "beta_mean": 0.059575408697128296, "nonfinite": 0.0, "weighted_memory_to_active": 0.06065630540251732, "mix_std": 0.2559305429458618, "beta_std": 0.06434529274702072, "beta_low": 0.02642822265625, "beta_high": 0.0, "orthogonal_to_active": 0.04145078733563423, "evidence_weights": [-0.12199436128139496, -0.14874573051929474, 0.23294316232204437], "agreement_weight": 0.007306077983230352, "orthogonal_weight": 0.9289229512214661}, {"head": 4, "memory_to_active": 0.04272942245006561, "contribution_to_active": 0.3419684171676636, "mix_mean": 0.3024890422821045, "mix_low": 0.0069370269775390625, "mix_high": 2.86102294921875e-06, "read_cosine": 0.12070108950138092, "beta_mean": 0.00012455778778530657, "nonfinite": 0.0, "weighted_memory_to_active": 0.013698859140276909, "mix_std": 0.23203101754188538, "beta_std": 0.002644619671627879, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0034896661527454853, "evidence_weights": [-0.1642906367778778, -0.1462712436914444, 0.12681014835834503], "agreement_weight": 0.0178532637655735, "orthogonal_weight": -0.26597413420677185}, {"head": 5, "memory_to_active": 0.13281360268592834, "contribution_to_active": 0.445593923330307, "mix_mean": 0.30837082862854004, "mix_low": 0.006993293762207031, "mix_high": 2.6702880859375e-05, "read_cosine": 0.8022083640098572, "beta_mean": 0.006299883127212524, "nonfinite": 0.0, "weighted_memory_to_active": 0.06693365424871445, "mix_std": 0.2150682806968689, "beta_std": 0.008631848730146885, "beta_low": 0.83685302734375, "beta_high": 0.0, "orthogonal_to_active": 0.015425576828420162, "evidence_weights": [-0.19400165975093842, -0.16804175078868866, 0.21981455385684967], "agreement_weight": 0.08477970957756042, "orthogonal_weight": -0.5152267217636108}]}, {"layer": 11, "memory_to_active": 0.29047298431396484, "contribution_to_active": 0.4281182885169983, "mix_mean": 0.3413695693016052, "mix_low": 0.05688905715942383, "mix_high": 0.015715917572379112, "read_cosine": 0.5143329501152039, "beta_mean": 0.03930877894163132, "nonfinite": 0.0, "weighted_memory_to_active": 0.09081745892763138, "mix_std": 0.28724342584609985, "beta_std": 0.058132778853178024, "beta_low": 0.4870504140853882, "beta_high": 0.0, "orthogonal_to_active": 0.03539995849132538, "output_influence": 0.4741458296775818, "residual_influence": 0.12323682010173798, "state_active": 0.11371084302663803, "state_memory": 0.01375902071595192, "state_peak": 24.080041885375977, "growth_active": 1.1453516483306885, "growth_memory": 1.3993428945541382, "heads": [{"head": 0, "memory_to_active": 0.07276193797588348, "contribution_to_active": 0.5108318328857422, "mix_mean": 0.4623962342739105, "mix_low": 0.0055217742919921875, "mix_high": 0.004368782043457031, "read_cosine": 0.4173295497894287, "beta_mean": 0.00029529602034017444, "nonfinite": 0.0, "weighted_memory_to_active": 0.0440729483962059, "mix_std": 0.2797909379005432, "beta_std": 0.005491884890943766, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.021834488958120346, "evidence_weights": [-0.16091866791248322, -0.2731991708278656, 0.43079400062561035], "agreement_weight": 0.11942517012357712, "orthogonal_weight": -0.6218727231025696}, {"head": 1, "memory_to_active": 0.533208966255188, "contribution_to_active": 0.20363357663154602, "mix_mean": 0.19679822027683258, "mix_low": 0.0517425537109375, "mix_high": 3.528594970703125e-05, "read_cosine": 0.6066516637802124, "beta_mean": 0.04439094662666321, "nonfinite": 0.0, "weighted_memory_to_active": 0.1527041792869568, "mix_std": 0.21082369983196259, "beta_std": 0.08130788058042526, "beta_low": 0.3260498046875, "beta_high": 0.0, "orthogonal_to_active": 0.017595019191503525, "evidence_weights": [-0.14738158881664276, -0.1005169004201889, 0.1272023767232895], "agreement_weight": 0.08845669031143188, "orthogonal_weight": -0.15749819576740265}, {"head": 2, "memory_to_active": 0.06709171831607819, "contribution_to_active": 0.8169810771942139, "mix_mean": 0.46866536140441895, "mix_low": 0.04591941833496094, "mix_high": 0.0538330078125, "read_cosine": 0.35590964555740356, "beta_mean": 0.07347558438777924, "nonfinite": 0.0, "weighted_memory_to_active": 0.05625955015420914, "mix_std": 0.4023556113243103, "beta_std": 0.10149290412664413, "beta_low": 0.41522216796875, "beta_high": 0.0, "orthogonal_to_active": 0.08931565284729004, "evidence_weights": [-0.16971717774868011, -0.39606261253356934, -0.553714394569397], "agreement_weight": -0.013247351162135601, "orthogonal_weight": 1.744277000427246}, {"head": 3, "memory_to_active": 0.034343793988227844, "contribution_to_active": 0.412232905626297, "mix_mean": 0.4407763183116913, "mix_low": 0.0867013931274414, "mix_high": 0.035536766052246094, "read_cosine": 0.535997748374939, "beta_mean": 0.06502392888069153, "nonfinite": 0.0, "weighted_memory_to_active": 0.01912696845829487, "mix_std": 0.4123520255088806, "beta_std": 0.09568370133638382, "beta_low": 0.4432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.03017161786556244, "evidence_weights": [-0.20750118792057037, -0.36279332637786865, -0.1270604282617569], "agreement_weight": 0.018484603613615036, "orthogonal_weight": 1.8871761560440063}, {"head": 4, "memory_to_active": 0.8279245495796204, "contribution_to_active": 0.2281443178653717, "mix_mean": 0.12151685357093811, "mix_low": 0.14779949188232422, "mix_high": 1.9073486328125e-06, "read_cosine": 0.27127063274383545, "beta_mean": 0.03773647919297218, "nonfinite": 0.0, "weighted_memory_to_active": 0.16920331120491028, "mix_std": 0.16250653564929962, "beta_std": 0.04402615502476692, "beta_low": 0.16925048828125, "beta_high": 0.0, "orthogonal_to_active": 0.030385931953787804, "evidence_weights": [-0.059702079743146896, -0.044130440801382065, 0.7077650427818298], "agreement_weight": -0.06665139645338058, "orthogonal_weight": -0.20780645310878754}, {"head": 5, "memory_to_active": 0.20750689506530762, "contribution_to_active": 0.3968859314918518, "mix_mean": 0.3580641746520996, "mix_low": 0.0036497116088867188, "mix_high": 0.0005197525024414062, "read_cosine": 0.8988381624221802, "beta_mean": 0.01493043452501297, "nonfinite": 0.0, "weighted_memory_to_active": 0.10353776067495346, "mix_std": 0.25563162565231323, "beta_std": 0.020794153213500977, "beta_low": 0.57012939453125, "beta_high": 0.0, "orthogonal_to_active": 0.023097027093172073, "evidence_weights": [-0.18112753331661224, -0.2230536788702011, 0.3001684844493866], "agreement_weight": 0.048730555921792984, "orthogonal_weight": -0.5465631484985352}]}, {"layer": 12, "memory_to_active": 0.6256111264228821, "contribution_to_active": 0.4920217990875244, "mix_mean": 0.35615018010139465, "mix_low": 0.05667368695139885, "mix_high": 0.0011432965984568, "read_cosine": 0.32952117919921875, "beta_mean": 0.0238173920661211, "nonfinite": 0.0, "weighted_memory_to_active": 0.2555598020553589, "mix_std": 0.23563547432422638, "beta_std": 0.028003431856632233, "beta_low": 0.6837158799171448, "beta_high": 0.0, "orthogonal_to_active": 0.06994616240262985, "output_influence": 0.32548582553863525, "residual_influence": 0.08584080636501312, "state_active": 0.05544780194759369, "state_memory": 0.01747169904410839, "state_peak": 40.589054107666016, "growth_active": 1.1971235275268555, "growth_memory": 1.3006327152252197, "heads": [{"head": 0, "memory_to_active": 0.5125247836112976, "contribution_to_active": 0.28629881143569946, "mix_mean": 0.2650102376937866, "mix_low": 0.03021526336669922, "mix_high": 6.29425048828125e-05, "read_cosine": 0.4408011734485626, "beta_mean": 0.010209914296865463, "nonfinite": 0.0, "weighted_memory_to_active": 0.18183664977550507, "mix_std": 0.23872758448123932, "beta_std": 0.025777433067560196, "beta_low": 0.763671875, "beta_high": 0.0, "orthogonal_to_active": 0.0640755444765091, "evidence_weights": [-0.12351370602846146, -0.16949400305747986, 0.24724754691123962], "agreement_weight": 0.09612923860549927, "orthogonal_weight": -0.44442662596702576}, {"head": 1, "memory_to_active": 0.22932685911655426, "contribution_to_active": 0.391594260931015, "mix_mean": 0.3757307529449463, "mix_low": 0.0019865036010742188, "mix_high": 0.00026702880859375, "read_cosine": 0.603425145149231, "beta_mean": 0.002183778677135706, "nonfinite": 0.0, "weighted_memory_to_active": 0.10185708850622177, "mix_std": 0.2545217275619507, "beta_std": 0.005935008637607098, "beta_low": 0.968505859375, "beta_high": 0.0, "orthogonal_to_active": 0.0630483329296112, "evidence_weights": [-0.16244837641716003, -0.20983657240867615, 0.525806725025177], "agreement_weight": 0.08496491611003876, "orthogonal_weight": -0.8440589904785156}, {"head": 2, "memory_to_active": 0.6633979678153992, "contribution_to_active": 0.4199202358722687, "mix_mean": 0.3147035241127014, "mix_low": 0.021073341369628906, "mix_high": 0.0001850128173828125, "read_cosine": 0.3749241232872009, "beta_mean": 0.08948209881782532, "nonfinite": 0.0, "weighted_memory_to_active": 0.3002212345600128, "mix_std": 0.24801868200302124, "beta_std": 0.07300297170877457, "beta_low": 0.00299072265625, "beta_high": 0.0, "orthogonal_to_active": 0.11559154838323593, "evidence_weights": [-0.2080097198486328, -0.07282647490501404, 0.05230039730668068], "agreement_weight": 0.03809002414345741, "orthogonal_weight": 0.4798751473426819}, {"head": 3, "memory_to_active": 0.04420555382966995, "contribution_to_active": 0.6443303227424622, "mix_mean": 0.6041560173034668, "mix_low": 0.0007162094116210938, "mix_high": 0.0047321319580078125, "read_cosine": 0.02907405234873295, "beta_mean": 0.000363772502169013, "nonfinite": 0.0, "weighted_memory_to_active": 0.027928650379180908, "mix_std": 0.2566683888435364, "beta_std": 0.0007750029908493161, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.019469670951366425, "evidence_weights": [-0.30001190304756165, -0.282611221075058, 0.3499472737312317], "agreement_weight": 0.12756246328353882, "orthogonal_weight": -0.7175971269607544}, {"head": 4, "memory_to_active": 2.2471072673797607, "contribution_to_active": 0.8049339056015015, "mix_mean": 0.08232046663761139, "mix_low": 0.28018665313720703, "mix_high": 0.0010499954223632812, "read_cosine": 0.07724297046661377, "beta_mean": 0.03992593288421631, "nonfinite": 0.0, "weighted_memory_to_active": 0.8963379263877869, "mix_std": 0.1426740139722824, "beta_std": 0.06066793575882912, "beta_low": 0.37579345703125, "beta_high": 0.0, "orthogonal_to_active": 0.13859090209007263, "evidence_weights": [0.048326559364795685, -0.0692552700638771, 0.03142064809799194], "agreement_weight": -0.009994789958000183, "orthogonal_weight": -0.1682339310646057}, {"head": 5, "memory_to_active": 0.057104334235191345, "contribution_to_active": 0.4050532579421997, "mix_mean": 0.49498000741004944, "mix_low": 0.005864143371582031, "mix_high": 0.0005626678466796875, "read_cosine": 0.4516594409942627, "beta_mean": 0.0007388442172668874, "nonfinite": 0.0, "weighted_memory_to_active": 0.02517719566822052, "mix_std": 0.27320244908332825, "beta_std": 0.0018622393254190683, "beta_low": 0.99212646484375, "beta_high": 0.0, "orthogonal_to_active": 0.01890094392001629, "evidence_weights": [-0.2526554465293884, -0.25980499386787415, 0.5331742763519287], "agreement_weight": 0.050668228417634964, "orthogonal_weight": -0.8279300928115845}]}, {"layer": 13, "memory_to_active": 0.23760391771793365, "contribution_to_active": 0.4184997081756592, "mix_mean": 0.41317346692085266, "mix_low": 0.004254341125488281, "mix_high": 0.001002152799628675, "read_cosine": 0.37241896986961365, "beta_mean": 0.01774071529507637, "nonfinite": 0.0, "weighted_memory_to_active": 0.10813825577497482, "mix_std": 0.25105032324790955, "beta_std": 0.019810238853096962, "beta_low": 0.7461955547332764, "beta_high": 0.0, "orthogonal_to_active": 0.07531805336475372, "output_influence": 0.35437920689582825, "residual_influence": 0.11216354370117188, "state_active": 0.07995159178972244, "state_memory": 0.030828196555376053, "state_peak": 73.07311248779297, "growth_active": 0.9673869013786316, "growth_memory": 1.0791656970977783, "heads": [{"head": 0, "memory_to_active": 0.2328130453824997, "contribution_to_active": 0.40965813398361206, "mix_mean": 0.3790254294872284, "mix_low": 0.0031919479370117188, "mix_high": 0.0008039474487304688, "read_cosine": 0.3156653046607971, "beta_mean": 0.003006014507263899, "nonfinite": 0.0, "weighted_memory_to_active": 0.10233017802238464, "mix_std": 0.2540563642978668, "beta_std": 0.004985945299267769, "beta_low": 0.93841552734375, "beta_high": 0.0, "orthogonal_to_active": 0.09067981690168381, "evidence_weights": [-0.18832792341709137, -0.20639550685882568, 0.5388928651809692], "agreement_weight": 0.04338103532791138, "orthogonal_weight": -0.9603040814399719}, {"head": 1, "memory_to_active": 0.11599518358707428, "contribution_to_active": 0.4596792459487915, "mix_mean": 0.38277125358581543, "mix_low": 0.0030832290649414062, "mix_high": 2.47955322265625e-05, "read_cosine": 0.23586073517799377, "beta_mean": 0.0007166328141465783, "nonfinite": 0.0, "weighted_memory_to_active": 0.05341501533985138, "mix_std": 0.25307101011276245, "beta_std": 0.0016043793875724077, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.04732370376586914, "evidence_weights": [-0.18192477524280548, -0.2085588425397873, 0.5985597372055054], "agreement_weight": 0.07668428868055344, "orthogonal_weight": -0.9356489777565002}, {"head": 2, "memory_to_active": 0.2353079468011856, "contribution_to_active": 0.3144671618938446, "mix_mean": 0.29441240429878235, "mix_low": 0.0034246444702148438, "mix_high": 1.049041748046875e-05, "read_cosine": 0.6478434205055237, "beta_mean": 0.018258698284626007, "nonfinite": 0.0, "weighted_memory_to_active": 0.08841896057128906, "mix_std": 0.20905375480651855, "beta_std": 0.03803712874650955, "beta_low": 0.62286376953125, "beta_high": 0.0, "orthogonal_to_active": 0.034043580293655396, "evidence_weights": [-0.19749674201011658, -0.14894238114356995, 0.41120487451553345], "agreement_weight": 0.05415588989853859, "orthogonal_weight": -0.5850393176078796}, {"head": 3, "memory_to_active": 0.1741056740283966, "contribution_to_active": 0.5189557671546936, "mix_mean": 0.531903862953186, "mix_low": 0.004313468933105469, "mix_high": 0.0023069381713867188, "read_cosine": -0.05180627852678299, "beta_mean": 0.0009075733833014965, "nonfinite": 0.0, "weighted_memory_to_active": 0.12092110514640808, "mix_std": 0.24633435904979706, "beta_std": 0.0053468323312699795, "beta_low": 0.9935302734375, "beta_high": 0.0, "orthogonal_to_active": 0.12209124118089676, "evidence_weights": [-0.3446817696094513, -0.24102382361888885, 1.0146617889404297], "agreement_weight": 0.8801455497741699, "orthogonal_weight": -1.0208526849746704}, {"head": 4, "memory_to_active": 0.4142136573791504, "contribution_to_active": 0.31966719031333923, "mix_mean": 0.4975815415382385, "mix_low": 0.003711700439453125, "mix_high": 0.0023717880249023438, "read_cosine": 0.8335779309272766, "beta_mean": 0.08032919466495514, "nonfinite": 0.0, "weighted_memory_to_active": 0.17298860847949982, "mix_std": 0.26991015672683716, "beta_std": 0.05960993096232414, "beta_low": 0.00079345703125, "beta_high": 0.0, "orthogonal_to_active": 0.09781163930892944, "evidence_weights": [-0.26427173614501953, -0.26650187373161316, 0.4333709180355072], "agreement_weight": 0.004732748027890921, "orthogonal_weight": 0.8242227435112}, {"head": 5, "memory_to_active": 0.25318798422813416, "contribution_to_active": 0.4885706603527069, "mix_mean": 0.39334627985954285, "mix_low": 0.007801055908203125, "mix_high": 0.0004949569702148438, "read_cosine": 0.2533726990222931, "beta_mean": 0.0032261726446449757, "nonfinite": 0.0, "weighted_memory_to_active": 0.11075557768344879, "mix_std": 0.2738761305809021, "beta_std": 0.009277207776904106, "beta_low": 0.9261474609375, "beta_high": 0.0, "orthogonal_to_active": 0.05995834246277809, "evidence_weights": [-0.18947505950927734, -0.21656754612922668, 0.22796788811683655], "agreement_weight": 0.05267667770385742, "orthogonal_weight": -0.5978072881698608}]}, {"layer": 14, "memory_to_active": 0.2918195426464081, "contribution_to_active": 0.40723657608032227, "mix_mean": 0.3744323253631592, "mix_low": 0.024675052613019943, "mix_high": 0.0022753081284463406, "read_cosine": 0.610713005065918, "beta_mean": 0.07162937521934509, "nonfinite": 0.0, "weighted_memory_to_active": 0.1337818205356598, "mix_std": 0.2506188452243805, "beta_std": 0.07003943622112274, "beta_low": 0.3749491572380066, "beta_high": 0.0, "orthogonal_to_active": 0.037939682602882385, "output_influence": 0.2366093546152115, "residual_influence": 0.09058796614408493, "state_active": 0.0905969887971878, "state_memory": 0.020190346986055374, "state_peak": 23.065019607543945, "growth_active": 1.1825647354125977, "growth_memory": 1.4428832530975342, "heads": [{"head": 0, "memory_to_active": 0.28106024861335754, "contribution_to_active": 0.2611503005027771, "mix_mean": 0.2694705128669739, "mix_low": 0.021701812744140625, "mix_high": 4.1961669921875e-05, "read_cosine": 0.676791250705719, "beta_mean": 0.07005847990512848, "nonfinite": 0.0, "weighted_memory_to_active": 0.1026524007320404, "mix_std": 0.2448045015335083, "beta_std": 0.08958626538515091, "beta_low": 0.149169921875, "beta_high": 0.0, "orthogonal_to_active": 0.01281784102320671, "evidence_weights": [-0.06078881025314331, -0.13977277278900146, 0.6651860475540161], "agreement_weight": 0.0420716367661953, "orthogonal_weight": 0.17730417847633362}, {"head": 1, "memory_to_active": 0.45611074566841125, "contribution_to_active": 0.44865870475769043, "mix_mean": 0.295818954706192, "mix_low": 0.08736133575439453, "mix_high": 0.0010929107666015625, "read_cosine": 0.6011534333229065, "beta_mean": 0.27730804681777954, "nonfinite": 0.0, "weighted_memory_to_active": 0.22619903087615967, "mix_std": 0.28301483392715454, "beta_std": 0.248507559299469, "beta_low": 0.08428955078125, "beta_high": 0.0, "orthogonal_to_active": 0.013645518571138382, "evidence_weights": [-0.023607855662703514, -0.25922691822052, 0.08088788390159607], "agreement_weight": 0.054108381271362305, "orthogonal_weight": 0.0888066291809082}, {"head": 2, "memory_to_active": 0.3331555426120758, "contribution_to_active": 0.43311402201652527, "mix_mean": 0.43054091930389404, "mix_low": 0.0054473876953125, "mix_high": 0.0008687973022460938, "read_cosine": 0.4228402078151703, "beta_mean": 0.014859815128147602, "nonfinite": 0.0, "weighted_memory_to_active": 0.17311161756515503, "mix_std": 0.2683599293231964, "beta_std": 0.018174298107624054, "beta_low": 0.4923095703125, "beta_high": 0.0, "orthogonal_to_active": 0.11634945869445801, "evidence_weights": [-0.19974473118782043, -0.2172139286994934, 0.5061087608337402], "agreement_weight": 0.08208232372999191, "orthogonal_weight": -0.7728752493858337}, {"head": 3, "memory_to_active": 0.07247202098369598, "contribution_to_active": 0.6277051568031311, "mix_mean": 0.6411888599395752, "mix_low": 0.0022439956665039062, "mix_high": 0.008710861206054688, "read_cosine": 0.5649144649505615, "beta_mean": 0.010228952392935753, "nonfinite": 0.0, "weighted_memory_to_active": 0.04830531030893326, "mix_std": 0.2403210699558258, "beta_std": 0.008388791233301163, "beta_low": 0.612060546875, "beta_high": 0.0, "orthogonal_to_active": 0.019119281321763992, "evidence_weights": [-0.37491562962532043, -0.3233773112297058, 0.395407110452652], "agreement_weight": 0.07353516668081284, "orthogonal_weight": -0.5355721116065979}, {"head": 4, "memory_to_active": 0.22323697805404663, "contribution_to_active": 0.4278876781463623, "mix_mean": 0.3832336664199829, "mix_low": 0.006412506103515625, "mix_high": 0.0028066635131835938, "read_cosine": 0.7028230428695679, "beta_mean": 0.05208992585539818, "nonfinite": 0.0, "weighted_memory_to_active": 0.10924698412418365, "mix_std": 0.253077894449234, "beta_std": 0.042162470519542694, "beta_low": 0.04473876953125, "beta_high": 0.0, "orthogonal_to_active": 0.020531782880425453, "evidence_weights": [-0.206033393740654, -0.253690242767334, 0.22194534540176392], "agreement_weight": 0.10472431778907776, "orthogonal_weight": -0.2898210287094116}, {"head": 5, "memory_to_active": 0.3848816752433777, "contribution_to_active": 0.24490350484848022, "mix_mean": 0.22634102404117584, "mix_low": 0.024883270263671875, "mix_high": 0.00013065338134765625, "read_cosine": 0.6957554817199707, "beta_mean": 0.005231004673987627, "nonfinite": 0.0, "weighted_memory_to_active": 0.14317555725574493, "mix_std": 0.2141348123550415, "beta_std": 0.013417206704616547, "beta_low": 0.86712646484375, "beta_high": 0.0, "orthogonal_to_active": 0.04517420381307602, "evidence_weights": [-0.18017005920410156, -0.12783442437648773, 0.0865151658654213], "agreement_weight": 0.09856931120157242, "orthogonal_weight": -0.5037488341331482}]}, {"layer": 15, "memory_to_active": 0.10309629142284393, "contribution_to_active": 0.45203372836112976, "mix_mean": 0.37401777505874634, "mix_low": 0.0065096220932900906, "mix_high": 0.0023899078369140625, "read_cosine": 0.6412959098815918, "beta_mean": 0.05098879337310791, "nonfinite": 0.0, "weighted_memory_to_active": 0.04658005014061928, "mix_std": 0.26364821195602417, "beta_std": 0.05318731069564819, "beta_low": 0.6734212636947632, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.011922174133360386, "output_influence": 0.277035117149353, "residual_influence": 0.14129771292209625, "state_active": 0.11074741929769516, "state_memory": 0.010390396229922771, "state_peak": 25.601295471191406, "growth_active": 1.3642120361328125, "growth_memory": 1.5221292972564697, "heads": [{"head": 0, "memory_to_active": 0.12177962809801102, "contribution_to_active": 0.46497654914855957, "mix_mean": 0.3333989083766937, "mix_low": 0.006123542785644531, "mix_high": 0.0019674301147460938, "read_cosine": 0.695625901222229, "beta_mean": 0.05527514964342117, "nonfinite": 0.0, "weighted_memory_to_active": 0.055466458201408386, "mix_std": 0.25336819887161255, "beta_std": 0.0409352071583271, "beta_low": 0.06646728515625, "beta_high": 0.0, "orthogonal_to_active": 0.013748161494731903, "evidence_weights": [-0.17004439234733582, -0.20699630677700043, 0.3519597053527832], "agreement_weight": 0.08155103027820587, "orthogonal_weight": -0.4414899945259094}, {"head": 1, "memory_to_active": 0.22556787729263306, "contribution_to_active": 0.39337289333343506, "mix_mean": 0.3642966151237488, "mix_low": 0.005995750427246094, "mix_high": 0.006016731262207031, "read_cosine": 0.8172585368156433, "beta_mean": 0.002804303076118231, "nonfinite": 0.0, "weighted_memory_to_active": 0.0886620506644249, "mix_std": 0.26723814010620117, "beta_std": 0.00631342688575387, "beta_low": 0.95391845703125, "beta_high": 0.0, "orthogonal_to_active": 0.008065227419137955, "evidence_weights": [-0.19816839694976807, -0.20488151907920837, 0.2704441249370575], "agreement_weight": 0.09591612964868546, "orthogonal_weight": -0.1395459920167923}, {"head": 2, "memory_to_active": 0.011307064443826675, "contribution_to_active": 0.4345211386680603, "mix_mean": 0.3504909873008728, "mix_low": 0.010066032409667969, "mix_high": 0.00018405914306640625, "read_cosine": 0.7653032541275024, "beta_mean": 0.0002207897196058184, "nonfinite": 0.0, "weighted_memory_to_active": 0.004403267987072468, "mix_std": 0.26262664794921875, "beta_std": 0.0004478218615986407, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0013347570784389973, "evidence_weights": [-0.21262331306934357, -0.2628386616706848, 0.28533485531806946], "agreement_weight": 0.047193918377161026, "orthogonal_weight": -0.4540504813194275}, {"head": 3, "memory_to_active": 0.11710664629936218, "contribution_to_active": 0.4267573058605194, "mix_mean": 0.2839961349964142, "mix_low": 0.00389862060546875, "mix_high": 1.1444091796875e-05, "read_cosine": 0.7177708148956299, "beta_mean": 0.004542968701571226, "nonfinite": 0.0, "weighted_memory_to_active": 0.05164256691932678, "mix_std": 0.2232762724161148, "beta_std": 0.007768061012029648, "beta_low": 0.879150390625, "beta_high": 0.0, "orthogonal_to_active": 0.017282864078879356, "evidence_weights": [-0.16194958984851837, -0.20032405853271484, 0.3374711573123932], "agreement_weight": 0.028634360060095787, "orthogonal_weight": -0.5504304766654968}, {"head": 4, "memory_to_active": 0.13582788407802582, "contribution_to_active": 0.550415575504303, "mix_mean": 0.4097713530063629, "mix_low": 0.008623123168945312, "mix_high": 0.0005331039428710938, "read_cosine": 0.2515622675418854, "beta_mean": 0.242939293384552, "nonfinite": 0.0, "weighted_memory_to_active": 0.0765242874622345, "mix_std": 0.29918980598449707, "beta_std": 0.2634059488773346, "beta_low": 0.14117431640625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.029548201709985733, "evidence_weights": [-0.2711772620677948, -0.24754126369953156, 0.3707520365715027], "agreement_weight": 0.00835530087351799, "orthogonal_weight": -0.44815605878829956}, {"head": 5, "memory_to_active": 0.0069886427372694016, "contribution_to_active": 0.44215893745422363, "mix_mean": 0.5021526217460632, "mix_low": 0.0043506622314453125, "mix_high": 0.005626678466796875, "read_cosine": 0.600254476070404, "beta_mean": 0.00015024664753582329, "nonfinite": 0.0, "weighted_memory_to_active": 0.002781641436740756, "mix_std": 0.27619025111198425, "beta_std": 0.0002533713704906404, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015538337174803019, "evidence_weights": [-0.3542443513870239, -0.2923121750354767, 0.5259170532226562], "agreement_weight": 0.12081410735845566, "orthogonal_weight": 0.655730128288269}]}]}, {"step": 12000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.893472479000593, "memory_to_active": 1.0362099409103394, "contribution_to_active": 0.42122411727905273, "mix_mean": 0.2710605561733246, "mix_low": 0.18258824944496155, "mix_high": 0.004476636648178101, "read_cosine": 0.39677026867866516, "beta_mean": 0.09854765236377716, "nonfinite": 0.0, "weighted_memory_to_active": 0.20646291971206665, "mix_std": 0.19983284175395966, "beta_std": 0.07673893868923187, "beta_low": 0.4237232506275177, "beta_high": 7.62939453125e-06, "orthogonal_to_active": 0.0379684641957283, "samples": 4, "scored_tokens": 899, "normal_loss": 3.2054219245910645, "normal_token_accuracy": 0.42602890729904175, "state_active": 0.06445342302322388, "state_memory": 0.0266267042607069, "state_peak": 83.66018676757812, "growth_active": 1.134669303894043, "growth_memory": 1.4232856035232544, "output_influence": 0.28903743624687195, "residual_influence": 0.09023706614971161, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2172818183898926, "no_memory_loss_delta": 0.14242608845233917, "scrambled_loss_delta": 0.16736425459384918, "memory_only_loss_delta": 2.0130503177642822, "no_evidence_loss_delta": 0.07035490870475769, "no_agreement_loss_delta": 0.007175896316766739, "no_orthogonal_loss_delta": 0.0023091903422027826}, {"index": 94, "tokens": 256, "normal_loss": 3.292728900909424, "no_memory_loss_delta": 0.24604925513267517, "scrambled_loss_delta": 0.3948986828327179, "memory_only_loss_delta": 2.0427086353302, "no_evidence_loss_delta": 0.11271563917398453, "no_agreement_loss_delta": 0.017223749309778214, "no_orthogonal_loss_delta": 0.010614996775984764}, {"index": 188, "tokens": 256, "normal_loss": 3.639209032058716, "no_memory_loss_delta": 0.11109726876020432, "scrambled_loss_delta": 0.06669370830059052, "memory_only_loss_delta": 2.030714988708496, "no_evidence_loss_delta": 0.05381190404295921, "no_agreement_loss_delta": -0.006352518685162067, "no_orthogonal_loss_delta": 0.0014451167080551386}, {"index": 283, "tokens": 131, "normal_loss": 2.1639256477355957, "no_memory_loss_delta": 0.11630011349916458, "scrambled_loss_delta": 0.2074742466211319, "memory_only_loss_delta": 2.5415215492248535, "no_evidence_loss_delta": 0.009583774954080582, "no_agreement_loss_delta": -0.006964977364987135, "no_orthogonal_loss_delta": -0.029854441061615944}], "agreement_weight_abs_mean": 0.0689752846956253, "agreement_weight_abs_max": 0.9957216382026672, "agreement_weight_mean": 0.053871702402830124, "orthogonal_weight_abs_mean": 0.4099007844924927, "orthogonal_weight_abs_max": 2.0620317459106445, "orthogonal_weight_mean": -0.15873785316944122, "no_memory_loss": 3.3646275997161865, "no_memory_loss_delta": 0.15920567512512207, "no_memory_token_accuracy_gain": 0.011123470962047577, "no_memory_token_win_fraction": 0.5717464089393616, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 14.717906312312934, "scrambled_loss": 3.414757013320923, "scrambled_loss_delta": 0.2093348354101181, "scrambled_token_accuracy_gain": 0.01223581749945879, "scrambled_token_win_fraction": 0.600667417049408, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.887640273347156, "memory_only_loss": 5.308956146240234, "memory_only_loss_delta": 2.1035335063934326, "memory_only_token_accuracy_gain": 0.221357062458992, "memory_only_token_win_fraction": 0.8665183782577515, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 87.79755087436789, "no_evidence_loss": 3.274273157119751, "no_evidence_loss_delta": 0.06885137408971786, "no_evidence_token_accuracy_gain": 0.0011123470030725002, "no_evidence_token_win_fraction": 0.5194661021232605, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 6.6534593085435025, "no_agreement_loss": 3.2095465660095215, "no_agreement_loss_delta": 0.00412419717758894, "no_agreement_token_accuracy_gain": 0.006674082484096289, "no_agreement_token_win_fraction": 0.48275861144065857, "no_agreement_probe_win_fraction": 0.5, "no_agreement_ppl_reduction_pct": 0.411570435577789, "no_orthogonal_loss": 3.2051639556884766, "no_orthogonal_loss_delta": -0.00025849719531834126, "no_orthogonal_token_accuracy_gain": -0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.513904333114624, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": -0.025853060859735263, "short_no_memory_loss_delta": 0.14266085624694824, "long_context_memory_gain": 0.016544818878173828, "normal_context_loss_gain": 0.1075926423072815, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.7692341804504395, "contribution_to_active": 0.7658833265304565, "mix_mean": 0.09301096200942993, "mix_low": 0.4328036308288574, "mix_high": 0.0027322769165039062, "read_cosine": 0.49461692571640015, "beta_mean": 0.263094961643219, "nonfinite": 0.0, "weighted_memory_to_active": 0.7927439212799072, "mix_std": 0.08392144739627838, "beta_std": 0.1538640707731247, "beta_low": 5.086263263365254e-05, "beta_high": 0.0, "orthogonal_to_active": 0.042301375418901443, "output_influence": 0.13377495110034943, "residual_influence": 0.20579269528388977, "state_active": 0.09390886127948761, "state_memory": 0.053142283111810684, "state_peak": 48.7291259765625, "growth_active": 1.30962073802948, "growth_memory": 1.319414734840393, "heads": [{"head": 0, "memory_to_active": 2.312347888946533, "contribution_to_active": 0.39766043424606323, "mix_mean": 0.021519735455513, "mix_low": 0.5040998458862305, "mix_high": 0.0, "read_cosine": 0.3697064220905304, "beta_mean": 0.23474495112895966, "nonfinite": 0.0, "weighted_memory_to_active": 0.48254281282424927, "mix_std": 0.051717691123485565, "beta_std": 0.10824704170227051, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02117568626999855, "evidence_weights": [-0.04361916705965996, -0.07028037309646606, 0.007776512298732996], "agreement_weight": -0.06159939989447594, "orthogonal_weight": -0.053820788860321045}, {"head": 1, "memory_to_active": 3.4366748332977295, "contribution_to_active": 1.315832257270813, "mix_mean": 0.020634211599826813, "mix_low": 0.4867267608642578, "mix_high": 0.0, "read_cosine": 0.4775487780570984, "beta_mean": 0.32442706823349, "nonfinite": 0.0, "weighted_memory_to_active": 1.6075133085250854, "mix_std": 0.049536339938640594, "beta_std": 0.151835635304451, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08706051111221313, "evidence_weights": [0.003854628885164857, -0.04145801439881325, 0.11774276196956635], "agreement_weight": 0.006931456271559, "orthogonal_weight": -0.12473607808351517}, {"head": 2, "memory_to_active": 2.4674265384674072, "contribution_to_active": 0.5558021664619446, "mix_mean": 0.015757782384753227, "mix_low": 0.5754976272583008, "mix_high": 0.0, "read_cosine": 0.3794493079185486, "beta_mean": 0.32394230365753174, "nonfinite": 0.0, "weighted_memory_to_active": 0.6232224702835083, "mix_std": 0.032742939889431, "beta_std": 0.2374165952205658, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04609731212258339, "evidence_weights": [0.02728516049683094, 0.03491244465112686, -0.005983459763228893], "agreement_weight": -0.006385923363268375, "orthogonal_weight": -0.12618114054203033}, {"head": 3, "memory_to_active": 4.658048629760742, "contribution_to_active": 1.893991470336914, "mix_mean": 0.05512509495019913, "mix_low": 0.21198558807373047, "mix_high": 0.0, "read_cosine": 0.4457027316093445, "beta_mean": 0.3373631238937378, "nonfinite": 0.0, "weighted_memory_to_active": 1.9071099758148193, "mix_std": 0.08846166729927063, "beta_std": 0.20868661999702454, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08143053948879242, "evidence_weights": [-0.11386459320783615, -0.13985079526901245, 0.04016635939478874], "agreement_weight": -0.0201825350522995, "orthogonal_weight": 0.05748564377427101}, {"head": 4, "memory_to_active": 3.602074384689331, "contribution_to_active": 0.08953742682933807, "mix_mean": 0.00697185518220067, "mix_low": 0.8067636489868164, "mix_high": 0.0, "read_cosine": 0.3423253893852234, "beta_mean": 0.2845713496208191, "nonfinite": 0.0, "weighted_memory_to_active": 0.07946498692035675, "mix_std": 0.011455928906798363, "beta_std": 0.15900656580924988, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012804250232875347, "evidence_weights": [0.0697709247469902, 0.026511328294873238, -0.17845742404460907], "agreement_weight": -0.030094459652900696, "orthogonal_weight": 0.18327918648719788}, {"head": 5, "memory_to_active": 0.13883119821548462, "contribution_to_active": 0.34247636795043945, "mix_mean": 0.4380570650100708, "mix_low": 0.011748313903808594, "mix_high": 0.016393661499023438, "read_cosine": 0.9529688358306885, "beta_mean": 0.07352109253406525, "nonfinite": 0.0, "weighted_memory_to_active": 0.05660973861813545, "mix_std": 0.2696141004562378, "beta_std": 0.05799199268221855, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.005239948630332947, "evidence_weights": [-0.29659533500671387, -0.2907254695892334, 0.3475532829761505], "agreement_weight": 0.14751775562763214, "orthogonal_weight": -0.22137190401554108}]}, {"layer": 1, "memory_to_active": 2.5337131023406982, "contribution_to_active": 0.34330737590789795, "mix_mean": 0.08548948168754578, "mix_low": 0.4092276990413666, "mix_high": 0.0003032684326171875, "read_cosine": 0.3084064722061157, "beta_mean": 0.2472388744354248, "nonfinite": 0.0, "weighted_memory_to_active": 0.308114230632782, "mix_std": 0.10120755434036255, "beta_std": 0.1331603229045868, "beta_low": 0.1687723994255066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.029716292396187782, "output_influence": 0.12443186342716217, "residual_influence": 0.027807019650936127, "state_active": 0.047568537294864655, "state_memory": 0.052488259971141815, "state_peak": 25.73882293701172, "growth_active": 1.1360836029052734, "growth_memory": 1.5107839107513428, "heads": [{"head": 0, "memory_to_active": 0.07568468153476715, "contribution_to_active": 0.23099584877490997, "mix_mean": 0.16309738159179688, "mix_low": 0.10645580291748047, "mix_high": 1.811981201171875e-05, "read_cosine": -0.12938475608825684, "beta_mean": 0.00034595560282468796, "nonfinite": 0.0, "weighted_memory_to_active": 0.025097494944930077, "mix_std": 0.20753706991672516, "beta_std": 0.000540752662345767, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.011875640600919724, "evidence_weights": [-0.08009525388479233, -0.11413499712944031, -0.053592171519994736], "agreement_weight": -0.00028380664298310876, "orthogonal_weight": -0.540134847164154}, {"head": 1, "memory_to_active": 3.7885851860046387, "contribution_to_active": 0.18051408231258392, "mix_mean": 0.02347441017627716, "mix_low": 0.45505619049072266, "mix_high": 1.9073486328125e-06, "read_cosine": 0.4657371938228607, "beta_mean": 0.3573269248008728, "nonfinite": 0.0, "weighted_memory_to_active": 0.19214633107185364, "mix_std": 0.04747557267546654, "beta_std": 0.20790626108646393, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0038152653723955154, "evidence_weights": [-0.02866275981068611, -0.013185760006308556, 0.057331107556819916], "agreement_weight": -0.01400415226817131, "orthogonal_weight": 0.027522612363100052}, {"head": 2, "memory_to_active": 0.7226527333259583, "contribution_to_active": 0.3588079810142517, "mix_mean": 0.26988691091537476, "mix_low": 0.010406494140625, "mix_high": 0.00179290771484375, "read_cosine": 0.7896386981010437, "beta_mean": 0.13475479185581207, "nonfinite": 0.0, "weighted_memory_to_active": 0.463930606842041, "mix_std": 0.22769244015216827, "beta_std": 0.06204322725534439, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.019596951082348824, "evidence_weights": [-0.2127535343170166, -0.353855699300766, 0.215999573469162], "agreement_weight": -0.010252553038299084, "orthogonal_weight": 0.08215131610631943}, {"head": 3, "memory_to_active": 3.284259080886841, "contribution_to_active": 0.05811474472284317, "mix_mean": 0.007799983490258455, "mix_low": 0.8183555603027344, "mix_high": 0.0, "read_cosine": 0.1572953313589096, "beta_mean": 0.5085848569869995, "nonfinite": 0.0, "weighted_memory_to_active": 0.06766316294670105, "mix_std": 0.022425169125199318, "beta_std": 0.2404014766216278, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.010781930759549141, "evidence_weights": [0.03651752322912216, 0.025129737332463264, 0.03140481561422348], "agreement_weight": -0.03282665088772774, "orthogonal_weight": -0.16521601378917694}, {"head": 4, "memory_to_active": 2.915484666824341, "contribution_to_active": 1.1561756134033203, "mix_mean": 0.040213290601968765, "mix_low": 0.2672901153564453, "mix_high": 1.9073486328125e-06, "read_cosine": 0.4018034040927887, "beta_mean": 0.1683690845966339, "nonfinite": 0.0, "weighted_memory_to_active": 1.0285767316818237, "mix_std": 0.0730619803071022, "beta_std": 0.07689949125051498, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1314104199409485, "evidence_weights": [-0.034199368208646774, -0.02797207608819008, 0.14652588963508606], "agreement_weight": -0.03575146570801735, "orthogonal_weight": 0.13860468566417694}, {"head": 5, "memory_to_active": 4.4156107902526855, "contribution_to_active": 0.0752362310886383, "mix_mean": 0.008464928716421127, "mix_low": 0.7978019714355469, "mix_high": 4.76837158203125e-06, "read_cosine": 0.1653488427400589, "beta_mean": 0.31405162811279297, "nonfinite": 0.0, "weighted_memory_to_active": 0.07127095758914948, "mix_std": 0.029053080826997757, "beta_std": 0.21117068827152252, "beta_low": 0.01300048828125, "beta_high": 0.0, "orthogonal_to_active": 0.0008175219409167767, "evidence_weights": [0.10340148210525513, 0.05187468230724335, 0.254578173160553], "agreement_weight": -0.117920882999897, "orthogonal_weight": 0.011350159533321857}]}, {"layer": 2, "memory_to_active": 1.2291924953460693, "contribution_to_active": 0.2822306156158447, "mix_mean": 0.2105003297328949, "mix_low": 0.2787725329399109, "mix_high": 0.0015306472778320312, "read_cosine": 0.36277490854263306, "beta_mean": 0.11431549489498138, "nonfinite": 0.0, "weighted_memory_to_active": 0.09040382504463196, "mix_std": 0.179578959941864, "beta_std": 0.07972106337547302, "beta_low": 0.528594970703125, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.014799229800701141, "output_influence": 0.23176322877407074, "residual_influence": 0.0526808500289917, "state_active": 0.041977409273386, "state_memory": 0.029045285657048225, "state_peak": 52.28556442260742, "growth_active": 1.039522409439087, "growth_memory": 1.5329581499099731, "heads": [{"head": 0, "memory_to_active": 0.0350586473941803, "contribution_to_active": 0.11291545629501343, "mix_mean": 0.19240808486938477, "mix_low": 0.08012676239013672, "mix_high": 0.00028514862060546875, "read_cosine": 0.27422434091567993, "beta_mean": 0.00018741382518783212, "nonfinite": 0.0, "weighted_memory_to_active": 0.007658853195607662, "mix_std": 0.2288724184036255, "beta_std": 0.00030619578319601715, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00042820992530323565, "evidence_weights": [-0.10987583547830582, -0.07995148748159409, 0.15767797827720642], "agreement_weight": -0.002631422132253647, "orthogonal_weight": -0.06769613921642303}, {"head": 1, "memory_to_active": 3.3494911193847656, "contribution_to_active": 0.06473720073699951, "mix_mean": 0.005826747044920921, "mix_low": 0.8677291870117188, "mix_high": 0.0, "read_cosine": 0.16546155512332916, "beta_mean": 0.30562111735343933, "nonfinite": 0.0, "weighted_memory_to_active": 0.05719848349690437, "mix_std": 0.013599948957562447, "beta_std": 0.1540163904428482, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01335046999156475, "evidence_weights": [0.09193176031112671, 0.09853960573673248, 0.1347992718219757], "agreement_weight": 0.01949026994407177, "orthogonal_weight": 0.2594418525695801}, {"head": 2, "memory_to_active": 0.06518346816301346, "contribution_to_active": 0.39432764053344727, "mix_mean": 0.3384319543838501, "mix_low": 0.00705718994140625, "mix_high": 0.0009908676147460938, "read_cosine": 0.35961878299713135, "beta_mean": 0.00010852303239516914, "nonfinite": 0.0, "weighted_memory_to_active": 0.025193797424435616, "mix_std": 0.2704334855079651, "beta_std": 0.00022647937294095755, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003242210252210498, "evidence_weights": [-0.17497576773166656, -0.19758398830890656, 0.34370630979537964], "agreement_weight": 0.025960488244891167, "orthogonal_weight": -0.1713601052761078}, {"head": 3, "memory_to_active": 0.3385319113731384, "contribution_to_active": 0.3886309564113617, "mix_mean": 0.23768676817417145, "mix_low": 0.011115074157714844, "mix_high": 0.0042934417724609375, "read_cosine": 0.660321831703186, "beta_mean": 0.0637226551771164, "nonfinite": 0.0, "weighted_memory_to_active": 0.13499881327152252, "mix_std": 0.23189687728881836, "beta_std": 0.0889257863163948, "beta_low": 0.156494140625, "beta_high": 0.0, "orthogonal_to_active": 0.05859299749135971, "evidence_weights": [-0.09619206935167313, -0.15974339842796326, 0.5053058862686157], "agreement_weight": 0.07059984654188156, "orthogonal_weight": -0.6562904119491577}, {"head": 4, "memory_to_active": 3.559276819229126, "contribution_to_active": 0.26497316360473633, "mix_mean": 0.014055054634809494, "mix_low": 0.7042598724365234, "mix_high": 0.00016307830810546875, "read_cosine": 0.19047850370407104, "beta_mean": 0.31619948148727417, "nonfinite": 0.0, "weighted_memory_to_active": 0.30262336134910583, "mix_std": 0.0477169007062912, "beta_std": 0.2347518652677536, "beta_low": 0.01507568359375, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.012438100762665272, "evidence_weights": [0.04717924818396568, 0.03989635407924652, 0.14953772723674774], "agreement_weight": 0.029205242171883583, "orthogonal_weight": -0.047568920999765396}, {"head": 5, "memory_to_active": 0.02761228382587433, "contribution_to_active": 0.46779924631118774, "mix_mean": 0.47459328174591064, "mix_low": 0.0023469924926757812, "mix_high": 0.0034513473510742188, "read_cosine": 0.5265442728996277, "beta_mean": 5.3723622841062024e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014749638736248016, "mix_std": 0.28495416045188904, "beta_std": 9.965876961359754e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007433831924572587, "evidence_weights": [-0.23661865293979645, -0.23224426805973053, 0.5412841439247131], "agreement_weight": 0.04561576619744301, "orthogonal_weight": 0.09207366406917572}]}, {"layer": 3, "memory_to_active": 1.4247204065322876, "contribution_to_active": 0.312702476978302, "mix_mean": 0.14931745827198029, "mix_low": 0.28688177466392517, "mix_high": 0.0013879140606150031, "read_cosine": 0.2805085778236389, "beta_mean": 0.12696781754493713, "nonfinite": 0.0, "weighted_memory_to_active": 0.16279387474060059, "mix_std": 0.17385178804397583, "beta_std": 0.09021526575088501, "beta_low": 0.39007568359375, "beta_high": 0.0, "orthogonal_to_active": 0.024180646985769272, "output_influence": 0.217040553689003, "residual_influence": 0.03709299489855766, "state_active": 0.03610888868570328, "state_memory": 0.02616501972079277, "state_peak": 31.667940139770508, "growth_active": 1.0549466609954834, "growth_memory": 1.7305866479873657, "heads": [{"head": 0, "memory_to_active": 0.9173633456230164, "contribution_to_active": 0.2604824900627136, "mix_mean": 0.13131260871887207, "mix_low": 0.23262596130371094, "mix_high": 0.00028228759765625, "read_cosine": 0.2140958160161972, "beta_mean": 0.017838304862380028, "nonfinite": 0.0, "weighted_memory_to_active": 0.17452235519886017, "mix_std": 0.21096843481063843, "beta_std": 0.014346301555633545, "beta_low": 0.31549072265625, "beta_high": 0.0, "orthogonal_to_active": 0.03276027366518974, "evidence_weights": [-0.016677049919962883, -0.08615510165691376, -0.07398953288793564], "agreement_weight": -0.005819373298436403, "orthogonal_weight": -0.19207848608493805}, {"head": 1, "memory_to_active": 0.08564190566539764, "contribution_to_active": 0.23878876864910126, "mix_mean": 0.1863788217306137, "mix_low": 0.10681438446044922, "mix_high": 0.0011425018310546875, "read_cosine": 0.21947215497493744, "beta_mean": 0.00030390190659090877, "nonfinite": 0.0, "weighted_memory_to_active": 0.014176919125020504, "mix_std": 0.22913850843906403, "beta_std": 0.0005392264574766159, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0007916015456430614, "evidence_weights": [-0.11297784745693207, -0.08504712581634521, 0.27015015482902527], "agreement_weight": 0.0040193903259932995, "orthogonal_weight": -0.0588073655962944}, {"head": 2, "memory_to_active": 3.2840590476989746, "contribution_to_active": 0.3063865005970001, "mix_mean": 0.035978157073259354, "mix_low": 0.4108467102050781, "mix_high": 1.430511474609375e-05, "read_cosine": 0.20361822843551636, "beta_mean": 0.2660989761352539, "nonfinite": 0.0, "weighted_memory_to_active": 0.41808250546455383, "mix_std": 0.08933741599321365, "beta_std": 0.1672382652759552, "beta_low": 0.00299072265625, "beta_high": 0.0, "orthogonal_to_active": 0.0375852957367897, "evidence_weights": [-0.03636632114648819, 0.011564360931515694, 0.09055128693580627], "agreement_weight": -0.02034521847963333, "orthogonal_weight": -0.12110035866498947}, {"head": 3, "memory_to_active": 0.08608009666204453, "contribution_to_active": 0.43767696619033813, "mix_mean": 0.3959774374961853, "mix_low": 0.03227043151855469, "mix_high": 0.0040569305419921875, "read_cosine": 0.31717628240585327, "beta_mean": 0.00025163483223877847, "nonfinite": 0.0, "weighted_memory_to_active": 0.04519759118556976, "mix_std": 0.300112783908844, "beta_std": 0.000642536673694849, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.009050894528627396, "evidence_weights": [-0.17767709493637085, -0.19663719832897186, 0.8643991947174072], "agreement_weight": 0.4839957356452942, "orthogonal_weight": -0.30384203791618347}, {"head": 4, "memory_to_active": 3.718930244445801, "contribution_to_active": 0.04842281714081764, "mix_mean": 0.007101766765117645, "mix_low": 0.840937614440918, "mix_high": 0.0, "read_cosine": 0.2069278508424759, "beta_mean": 0.33226996660232544, "nonfinite": 0.0, "weighted_memory_to_active": 0.05957905203104019, "mix_std": 0.017483744770288467, "beta_std": 0.208096444606781, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.014717419631779194, "evidence_weights": [0.14726868271827698, 0.126650869846344, -0.13347437977790833], "agreement_weight": -0.07108138501644135, "orthogonal_weight": -0.2597637474536896}, {"head": 5, "memory_to_active": 0.45624712109565735, "contribution_to_active": 0.584457278251648, "mix_mean": 0.13915592432022095, "mix_low": 0.09779548645019531, "mix_high": 0.0028314590454101562, "read_cosine": 0.5217610597610474, "beta_mean": 0.14504410326480865, "nonfinite": 0.0, "weighted_memory_to_active": 0.2652047276496887, "mix_std": 0.1960698664188385, "beta_std": 0.15042878687381744, "beta_low": 0.02178955078125, "beta_high": 0.0, "orthogonal_to_active": 0.050178397446870804, "evidence_weights": [-0.0490448884665966, -0.1328902691602707, 0.09996827691793442], "agreement_weight": 0.15796810388565063, "orthogonal_weight": -0.3659425377845764}]}, {"layer": 4, "memory_to_active": 1.4583380222320557, "contribution_to_active": 0.34750989079475403, "mix_mean": 0.24201299250125885, "mix_low": 0.27870720624923706, "mix_high": 0.0013178189983591437, "read_cosine": 0.4043053984642029, "beta_mean": 0.13958770036697388, "nonfinite": 0.0, "weighted_memory_to_active": 0.1850845217704773, "mix_std": 0.1523180454969406, "beta_std": 0.1082259863615036, "beta_low": 0.2510782778263092, "beta_high": 0.0, "orthogonal_to_active": 0.040696099400520325, "output_influence": 0.2272748500108719, "residual_influence": 0.04869420826435089, "state_active": 0.045200612396001816, "state_memory": 0.037199877202510834, "state_peak": 53.221214294433594, "growth_active": 1.075792670249939, "growth_memory": 1.6759073734283447, "heads": [{"head": 0, "memory_to_active": 0.40734124183654785, "contribution_to_active": 0.5100427865982056, "mix_mean": 0.4284381866455078, "mix_low": 0.006229400634765625, "mix_high": 0.00039958953857421875, "read_cosine": 0.6060785055160522, "beta_mean": 0.1184409037232399, "nonfinite": 0.0, "weighted_memory_to_active": 0.24666285514831543, "mix_std": 0.28285396099090576, "beta_std": 0.15017491579055786, "beta_low": 0.18377685546875, "beta_high": 0.0, "orthogonal_to_active": 0.0059401122853159904, "evidence_weights": [-0.20008696615695953, -0.24283835291862488, 0.4290849566459656], "agreement_weight": 0.05426725000143051, "orthogonal_weight": 0.03189338371157646}, {"head": 1, "memory_to_active": 0.44509178400039673, "contribution_to_active": 0.3177185356616974, "mix_mean": 0.34861689805984497, "mix_low": 0.007526397705078125, "mix_high": 0.0004949569702148438, "read_cosine": 0.5555391907691956, "beta_mean": 0.028371399268507957, "nonfinite": 0.0, "weighted_memory_to_active": 0.1844162940979004, "mix_std": 0.2590285539627075, "beta_std": 0.03452511131763458, "beta_low": 0.26446533203125, "beta_high": 0.0, "orthogonal_to_active": 0.08634398877620697, "evidence_weights": [-0.1897764503955841, -0.17081999778747559, 0.6103691458702087], "agreement_weight": 0.041195016354322433, "orthogonal_weight": -0.5849549174308777}, {"head": 2, "memory_to_active": 3.2251954078674316, "contribution_to_active": 0.11229973286390305, "mix_mean": 0.013162783347070217, "mix_low": 0.6768388748168945, "mix_high": 0.0, "read_cosine": 0.26634618639945984, "beta_mean": 0.241493821144104, "nonfinite": 0.0, "weighted_memory_to_active": 0.13345448672771454, "mix_std": 0.023042554035782814, "beta_std": 0.15364474058151245, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.002004044596105814, "evidence_weights": [0.033511098474264145, 0.016560038551688194, -0.06039976328611374], "agreement_weight": 0.03403976559638977, "orthogonal_weight": -0.01965862326323986}, {"head": 3, "memory_to_active": 1.7875151634216309, "contribution_to_active": 0.09480883926153183, "mix_mean": 0.028663532808423042, "mix_low": 0.39555835723876953, "mix_high": 0.0, "read_cosine": 0.20577871799468994, "beta_mean": 0.15761473774909973, "nonfinite": 0.0, "weighted_memory_to_active": 0.13882015645503998, "mix_std": 0.04888301342725754, "beta_std": 0.14279057085514069, "beta_low": 0.05810546875, "beta_high": 0.0, "orthogonal_to_active": 0.023146718740463257, "evidence_weights": [-0.019706537947058678, 0.03473972529172897, -0.13441596925258636], "agreement_weight": 0.02011384256184101, "orthogonal_weight": -0.22773341834545135}, {"head": 4, "memory_to_active": 0.02092393860220909, "contribution_to_active": 0.600383996963501, "mix_mean": 0.6164361238479614, "mix_low": 0.0010738372802734375, "mix_high": 0.0069789886474609375, "read_cosine": 0.37152960896492004, "beta_mean": 4.622796404873952e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014990948140621185, "mix_std": 0.2608928680419922, "beta_std": 9.415392560185865e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016927231336012483, "evidence_weights": [-0.2717644274234772, -0.3099350929260254, 0.6908130645751953], "agreement_weight": 0.0386776477098465, "orthogonal_weight": -0.14291246235370636}, {"head": 5, "memory_to_active": 2.8639605045318604, "contribution_to_active": 0.44980520009994507, "mix_mean": 0.01676037721335888, "mix_low": 0.5850162506103516, "mix_high": 3.337860107421875e-05, "read_cosine": 0.42055994272232056, "beta_mean": 0.29155898094177246, "nonfinite": 0.0, "weighted_memory_to_active": 0.3921624720096588, "mix_std": 0.03920730948448181, "beta_std": 0.1681264042854309, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12504900991916656, "evidence_weights": [0.026781992986798286, -0.02986368164420128, -0.08327679336071014], "agreement_weight": -0.03460167720913887, "orthogonal_weight": 0.5766364336013794}]}, {"layer": 5, "memory_to_active": 0.7076756954193115, "contribution_to_active": 0.41780877113342285, "mix_mean": 0.3664016127586365, "mix_low": 0.11466217041015625, "mix_high": 0.007246176712214947, "read_cosine": 0.2467932254076004, "beta_mean": 0.0644523873925209, "nonfinite": 0.0, "weighted_memory_to_active": 0.1603991985321045, "mix_std": 0.23675882816314697, "beta_std": 0.07436663657426834, "beta_low": 0.5482381582260132, "beta_high": 0.0, "orthogonal_to_active": 0.042009223252534866, "output_influence": 0.39322373270988464, "residual_influence": 0.08206810057163239, "state_active": 0.06408258527517319, "state_memory": 0.021747510880231857, "state_peak": 61.378501892089844, "growth_active": 1.1266429424285889, "growth_memory": 1.4298112392425537, "heads": [{"head": 0, "memory_to_active": 1.5371466875076294, "contribution_to_active": 0.21939633786678314, "mix_mean": 0.08235856890678406, "mix_low": 0.1892232894897461, "mix_high": 0.0, "read_cosine": -0.197615385055542, "beta_mean": 0.10837230831384659, "nonfinite": 0.0, "weighted_memory_to_active": 0.1956038773059845, "mix_std": 0.12108392268419266, "beta_std": 0.11759568750858307, "beta_low": 0.0328369140625, "beta_high": 0.0, "orthogonal_to_active": 0.04029233008623123, "evidence_weights": [-0.02559049427509308, -0.05533687397837639, -0.312766969203949], "agreement_weight": -0.022742707282304764, "orthogonal_weight": -0.20703141391277313}, {"head": 1, "memory_to_active": 0.0341155119240284, "contribution_to_active": 0.44633054733276367, "mix_mean": 0.48859167098999023, "mix_low": 0.013590812683105469, "mix_high": 0.0076618194580078125, "read_cosine": 0.11105411499738693, "beta_mean": 8.882170368451625e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014243335463106632, "mix_std": 0.318924218416214, "beta_std": 0.0001734290417516604, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011666685342788696, "evidence_weights": [-0.21196907758712769, -0.2143813669681549, 0.4913524091243744], "agreement_weight": 0.20087923109531403, "orthogonal_weight": -0.851064145565033}, {"head": 2, "memory_to_active": 0.13691356778144836, "contribution_to_active": 0.47658273577690125, "mix_mean": 0.5435450673103333, "mix_low": 0.0018796920776367188, "mix_high": 0.007068634033203125, "read_cosine": 0.5072131752967834, "beta_mean": 0.006037458777427673, "nonfinite": 0.0, "weighted_memory_to_active": 0.09202940762042999, "mix_std": 0.27700504660606384, "beta_std": 0.005817625671625137, "beta_low": 0.84100341796875, "beta_high": 0.0, "orthogonal_to_active": 0.011942155659198761, "evidence_weights": [-0.2713651657104492, -0.28048738837242126, 0.5970922708511353], "agreement_weight": 0.02673063799738884, "orthogonal_weight": -0.17678092420101166}, {"head": 3, "memory_to_active": 1.9978718757629395, "contribution_to_active": 0.18724603950977325, "mix_mean": 0.05199957638978958, "mix_low": 0.3373241424560547, "mix_high": 0.0, "read_cosine": 0.1823873668909073, "beta_mean": 0.061154309660196304, "nonfinite": 0.0, "weighted_memory_to_active": 0.30211204290390015, "mix_std": 0.10017883777618408, "beta_std": 0.09276139736175537, "beta_low": 0.23046875, "beta_high": 0.0, "orthogonal_to_active": 0.10352765023708344, "evidence_weights": [0.0030193254351615906, -0.0403452068567276, 0.08050504326820374], "agreement_weight": 0.07149970531463623, "orthogonal_weight": -0.4361266791820526}, {"head": 4, "memory_to_active": 0.4924179017543793, "contribution_to_active": 0.581985354423523, "mix_mean": 0.33046954870224, "mix_low": 0.14221668243408203, "mix_high": 0.011425018310546875, "read_cosine": 0.5979111790657043, "beta_mean": 0.21088245511054993, "nonfinite": 0.0, "weighted_memory_to_active": 0.3286263942718506, "mix_std": 0.33793461322784424, "beta_std": 0.22916612029075623, "beta_low": 0.186279296875, "beta_high": 0.0, "orthogonal_to_active": 0.07219246029853821, "evidence_weights": [-0.14700452983379364, -0.261804461479187, 0.10306768864393234], "agreement_weight": 0.04761386662721634, "orthogonal_weight": 0.33184555172920227}, {"head": 5, "memory_to_active": 0.04758843034505844, "contribution_to_active": 0.5953115820884705, "mix_mean": 0.7014452815055847, "mix_low": 0.0037384033203125, "mix_high": 0.01732158660888672, "read_cosine": 0.2798089385032654, "beta_mean": 0.00017895048949867487, "nonfinite": 0.0, "weighted_memory_to_active": 0.02978002466261387, "mix_std": 0.265426367521286, "beta_std": 0.0006855224492028356, "beta_low": 0.99884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.01243404671549797, "evidence_weights": [-0.307798832654953, -0.2964130640029907, 1.0245065689086914], "agreement_weight": 0.3057539761066437, "orthogonal_weight": -0.4595690071582794}]}, {"layer": 6, "memory_to_active": 0.49403607845306396, "contribution_to_active": 0.53728187084198, "mix_mean": 0.4539649486541748, "mix_low": 0.09814755618572235, "mix_high": 0.019368648529052734, "read_cosine": 0.3209332823753357, "beta_mean": 0.07963724434375763, "nonfinite": 0.0, "weighted_memory_to_active": 0.1829339861869812, "mix_std": 0.2613394260406494, "beta_std": 0.0924828052520752, "beta_low": 0.46429443359375, "beta_high": 0.0, "orthogonal_to_active": 0.040562041103839874, "output_influence": 0.4061110019683838, "residual_influence": 0.11286550015211105, "state_active": 0.06227757781744003, "state_memory": 0.014430057257413864, "state_peak": 52.57890319824219, "growth_active": 1.1713950634002686, "growth_memory": 1.32570481300354, "heads": [{"head": 0, "memory_to_active": 0.2130393385887146, "contribution_to_active": 0.5167727470397949, "mix_mean": 0.5608634948730469, "mix_low": 0.0007781982421875, "mix_high": 0.0020303726196289062, "read_cosine": 0.259998083114624, "beta_mean": 0.025518562644720078, "nonfinite": 0.0, "weighted_memory_to_active": 0.15382716059684753, "mix_std": 0.2646293342113495, "beta_std": 0.05806450545787811, "beta_low": 0.3944091796875, "beta_high": 0.0, "orthogonal_to_active": 0.10261596739292145, "evidence_weights": [-0.2625066936016083, -0.256428986787796, 0.29161542654037476], "agreement_weight": 0.09262079000473022, "orthogonal_weight": -0.806327223777771}, {"head": 1, "memory_to_active": 0.01208517886698246, "contribution_to_active": 0.4133010506629944, "mix_mean": 0.3983243405818939, "mix_low": 0.007702827453613281, "mix_high": 0.008819580078125, "read_cosine": 0.5391783714294434, "beta_mean": 2.6888617867371067e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003986522555351257, "mix_std": 0.2867082357406616, "beta_std": 5.9530091675696895e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009503221954219043, "evidence_weights": [-0.20974323153495789, -0.22479145228862762, 0.4743368327617645], "agreement_weight": -0.013925200328230858, "orthogonal_weight": -0.3078289330005646}, {"head": 2, "memory_to_active": 1.7676458358764648, "contribution_to_active": 0.0759713426232338, "mix_mean": 0.020501181483268738, "mix_low": 0.5607137680053711, "mix_high": 0.0, "read_cosine": 0.2094792127609253, "beta_mean": 0.11574128270149231, "nonfinite": 0.0, "weighted_memory_to_active": 0.09877775609493256, "mix_std": 0.04422399029135704, "beta_std": 0.08607891201972961, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.020867500454187393, "evidence_weights": [0.08289318531751633, 0.04752056673169136, 0.0676400363445282], "agreement_weight": 0.04142172262072563, "orthogonal_weight": -0.2285567969083786}, {"head": 3, "memory_to_active": 0.17025890946388245, "contribution_to_active": 0.6456862688064575, "mix_mean": 0.585293173789978, "mix_low": 0.013734817504882812, "mix_high": 0.02561664581298828, "read_cosine": 0.525969386100769, "beta_mean": 0.20384371280670166, "nonfinite": 0.0, "weighted_memory_to_active": 0.12009817361831665, "mix_std": 0.3508985638618469, "beta_std": 0.24519294500350952, "beta_low": 0.204345703125, "beta_high": 0.0, "orthogonal_to_active": 0.042219121009111404, "evidence_weights": [-0.16285322606563568, -0.3161456882953644, 0.19906000792980194], "agreement_weight": -0.0179641991853714, "orthogonal_weight": 0.42776668071746826}, {"head": 4, "memory_to_active": 0.7283543348312378, "contribution_to_active": 1.0437910556793213, "mix_mean": 0.6605019569396973, "mix_low": 0.00270843505859375, "mix_high": 0.07920360565185547, "read_cosine": 0.11346802115440369, "beta_mean": 0.13213276863098145, "nonfinite": 0.0, "weighted_memory_to_active": 0.6789008378982544, "mix_std": 0.32718175649642944, "beta_std": 0.16450591385364532, "beta_low": 0.18756103515625, "beta_high": 0.0, "orthogonal_to_active": 0.03541349992156029, "evidence_weights": [-0.2351221889257431, -0.5043789744377136, -0.02429775334894657], "agreement_weight": -0.014915677718818188, "orthogonal_weight": 0.06416352093219757}, {"head": 5, "memory_to_active": 0.07283274084329605, "contribution_to_active": 0.5281686186790466, "mix_mean": 0.49830567836761475, "mix_low": 0.0032472610473632812, "mix_high": 0.00054168701171875, "read_cosine": 0.27750664949417114, "beta_mean": 0.0005602903547696769, "nonfinite": 0.0, "weighted_memory_to_active": 0.04201347753405571, "mix_std": 0.2943946123123169, "beta_std": 0.0009949787054210901, "beta_low": 0.99847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.04130582511425018, "evidence_weights": [-0.20557044446468353, -0.2436477690935135, 0.19868876039981842], "agreement_weight": 0.07320968061685562, "orthogonal_weight": -1.0538374185562134}]}, {"layer": 7, "memory_to_active": 2.24550724029541, "contribution_to_active": 0.17190876603126526, "mix_mean": 0.027298573404550552, "mix_low": 0.5620760917663574, "mix_high": 1.3510387361748144e-05, "read_cosine": 0.22768104076385498, "beta_mean": 0.14399144053459167, "nonfinite": 0.0, "weighted_memory_to_active": 0.17846544086933136, "mix_std": 0.05062252655625343, "beta_std": 0.1118544489145279, "beta_low": 0.020477294921875, "beta_high": 0.0, "orthogonal_to_active": 0.028644055128097534, "output_influence": 0.20299306511878967, "residual_influence": 0.04316307604312897, "state_active": 0.020295485854148865, "state_memory": 0.03642125800251961, "state_peak": 24.837799072265625, "growth_active": 1.0343928337097168, "growth_memory": 1.4553812742233276, "heads": [{"head": 0, "memory_to_active": 2.233625650405884, "contribution_to_active": 0.3779813051223755, "mix_mean": 0.02521340548992157, "mix_low": 0.5636415481567383, "mix_high": 8.106231689453125e-05, "read_cosine": 0.37454164028167725, "beta_mean": 0.1518096625804901, "nonfinite": 0.0, "weighted_memory_to_active": 0.33389952778816223, "mix_std": 0.056494999676942825, "beta_std": 0.11567870527505875, "beta_low": 0.00506591796875, "beta_high": 0.0, "orthogonal_to_active": 0.055021315813064575, "evidence_weights": [-0.007237057201564312, -0.07395550608634949, 0.09649106860160828], "agreement_weight": -0.0407605841755867, "orthogonal_weight": 0.24859319627285004}, {"head": 1, "memory_to_active": 2.677189588546753, "contribution_to_active": 0.26312652230262756, "mix_mean": 0.05585925281047821, "mix_low": 0.34166908264160156, "mix_high": 0.0, "read_cosine": 0.04505109786987305, "beta_mean": 0.21229055523872375, "nonfinite": 0.0, "weighted_memory_to_active": 0.3437827229499817, "mix_std": 0.08521775901317596, "beta_std": 0.11857764422893524, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07520638406276703, "evidence_weights": [-0.10062141716480255, -0.08590663969516754, -0.04829876869916916], "agreement_weight": 0.007467145100235939, "orthogonal_weight": -0.22209109365940094}, {"head": 2, "memory_to_active": 1.3671884536743164, "contribution_to_active": 0.24540561437606812, "mix_mean": 0.050867959856987, "mix_low": 0.32430267333984375, "mix_high": 0.0, "read_cosine": 0.21059851348400116, "beta_mean": 0.12564918398857117, "nonfinite": 0.0, "weighted_memory_to_active": 0.2202596813440323, "mix_std": 0.10010486841201782, "beta_std": 0.07278665900230408, "beta_low": 0.0048828125, "beta_high": 0.0, "orthogonal_to_active": 0.008149969391524792, "evidence_weights": [-0.038374077528715134, -0.10853210091590881, 0.18421603739261627], "agreement_weight": 0.025630084797739983, "orthogonal_weight": -0.039453670382499695}, {"head": 3, "memory_to_active": 3.181082010269165, "contribution_to_active": 0.040694642812013626, "mix_mean": 0.009830079972743988, "mix_low": 0.7145719528198242, "mix_high": 0.0, "read_cosine": 0.1133524477481842, "beta_mean": 0.14805448055267334, "nonfinite": 0.0, "weighted_memory_to_active": 0.06158759817481041, "mix_std": 0.019084054976701736, "beta_std": 0.13384559750556946, "beta_low": 0.0155029296875, "beta_high": 0.0, "orthogonal_to_active": 0.02467375621199608, "evidence_weights": [0.10661903768777847, 0.08430580049753189, 0.15364888310432434], "agreement_weight": 0.06405500322580338, "orthogonal_weight": -0.42381930351257324}, {"head": 4, "memory_to_active": 1.6624754667282104, "contribution_to_active": 0.04471734166145325, "mix_mean": 0.010304799303412437, "mix_low": 0.7382488250732422, "mix_high": 0.0, "read_cosine": 0.3315856456756592, "beta_mean": 0.14593765139579773, "nonfinite": 0.0, "weighted_memory_to_active": 0.04015908017754555, "mix_std": 0.022090286016464233, "beta_std": 0.135588139295578, "beta_low": 0.03118896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0018585971556603909, "evidence_weights": [0.07152353227138519, 0.03751564770936966, 0.01106064859777689], "agreement_weight": 0.03990655764937401, "orthogonal_weight": -0.05273045226931572}, {"head": 5, "memory_to_active": 2.35148024559021, "contribution_to_active": 0.05952712520956993, "mix_mean": 0.011715933680534363, "mix_low": 0.6900224685668945, "mix_high": 0.0, "read_cosine": 0.2909567356109619, "beta_mean": 0.08020703494548798, "nonfinite": 0.0, "weighted_memory_to_active": 0.07110396772623062, "mix_std": 0.02074316330254078, "beta_std": 0.09464992582798004, "beta_low": 0.06622314453125, "beta_high": 0.0, "orthogonal_to_active": 0.006954302079975605, "evidence_weights": [0.06304939836263657, 0.07955080270767212, -0.11033467948436737], "agreement_weight": -0.001040458446368575, "orthogonal_weight": -0.11329445242881775}]}, {"layer": 8, "memory_to_active": 0.7748219966888428, "contribution_to_active": 0.400678813457489, "mix_mean": 0.27812454104423523, "mix_low": 0.08732128143310547, "mix_high": 0.0011285146465525031, "read_cosine": 0.512869656085968, "beta_mean": 0.050309523940086365, "nonfinite": 0.0, "weighted_memory_to_active": 0.1729344129562378, "mix_std": 0.2065093070268631, "beta_std": 0.052043817937374115, "beta_low": 0.3390706479549408, "beta_high": 0.0, "orthogonal_to_active": 0.0479138158261776, "output_influence": 0.28496426343917847, "residual_influence": 0.07015934586524963, "state_active": 0.05534493178129196, "state_memory": 0.02017766609787941, "state_peak": 32.60336685180664, "growth_active": 0.9913486242294312, "growth_memory": 1.4471850395202637, "heads": [{"head": 0, "memory_to_active": 0.06733273714780807, "contribution_to_active": 0.4824078679084778, "mix_mean": 0.5043617486953735, "mix_low": 0.00018215179443359375, "mix_high": 0.00017261505126953125, "read_cosine": 0.7259517312049866, "beta_mean": 0.003345630131661892, "nonfinite": 0.0, "weighted_memory_to_active": 0.03588440641760826, "mix_std": 0.23826712369918823, "beta_std": 0.005900313146412373, "beta_low": 0.92449951171875, "beta_high": 0.0, "orthogonal_to_active": 0.01638171263039112, "evidence_weights": [-0.24237988889217377, -0.311170369386673, 0.6227428317070007], "agreement_weight": 0.07027916610240936, "orthogonal_weight": -0.7056200504302979}, {"head": 1, "memory_to_active": 0.6181862950325012, "contribution_to_active": 0.3097577393054962, "mix_mean": 0.2962891459465027, "mix_low": 0.010189056396484375, "mix_high": 0.0003299713134765625, "read_cosine": 0.53052818775177, "beta_mean": 0.04902492091059685, "nonfinite": 0.0, "weighted_memory_to_active": 0.20712122321128845, "mix_std": 0.20919732749462128, "beta_std": 0.03821106255054474, "beta_low": 0.05615234375, "beta_high": 0.0, "orthogonal_to_active": 0.0065595791675150394, "evidence_weights": [-0.1612127423286438, -0.17557360231876373, 0.3180539608001709], "agreement_weight": 0.03346997871994972, "orthogonal_weight": -0.0387611910700798}, {"head": 2, "memory_to_active": 0.3477616608142853, "contribution_to_active": 0.3557603359222412, "mix_mean": 0.30227184295654297, "mix_low": 0.011416435241699219, "mix_high": 0.00014495849609375, "read_cosine": 0.30212292075157166, "beta_mean": 0.010123961605131626, "nonfinite": 0.0, "weighted_memory_to_active": 0.13825678825378418, "mix_std": 0.23997271060943604, "beta_std": 0.011857190169394016, "beta_low": 0.67333984375, "beta_high": 0.0, "orthogonal_to_active": 0.09122321009635925, "evidence_weights": [-0.14372952282428741, -0.2307223379611969, 0.26188427209854126], "agreement_weight": 0.03538331761956215, "orthogonal_weight": -0.7185538411140442}, {"head": 3, "memory_to_active": 0.4002833366394043, "contribution_to_active": 0.25580665469169617, "mix_mean": 0.21132740378379822, "mix_low": 0.02276134490966797, "mix_high": 0.0, "read_cosine": 0.6578764915466309, "beta_mean": 0.03538360074162483, "nonfinite": 0.0, "weighted_memory_to_active": 0.14783364534378052, "mix_std": 0.1774863600730896, "beta_std": 0.03451976180076599, "beta_low": 0.11517333984375, "beta_high": 0.0, "orthogonal_to_active": 0.017594989389181137, "evidence_weights": [-0.0762147530913353, -0.161817729473114, 0.6032438278198242], "agreement_weight": 0.07005520164966583, "orthogonal_weight": 0.19647708535194397}, {"head": 4, "memory_to_active": 2.9713404178619385, "contribution_to_active": 0.3480868339538574, "mix_mean": 0.02717391960322857, "mix_low": 0.39632701873779297, "mix_high": 0.0, "read_cosine": 0.3295074701309204, "beta_mean": 0.12033891677856445, "nonfinite": 0.0, "weighted_memory_to_active": 0.3410801887512207, "mix_std": 0.04248325526714325, "beta_std": 0.09833980351686478, "beta_low": 0.00341796875, "beta_high": 0.0, "orthogonal_to_active": 0.01597217284142971, "evidence_weights": [0.01668514870107174, -0.008693039417266846, -0.060995470732450485], "agreement_weight": 0.011239870451390743, "orthogonal_weight": 0.050799302756786346}, {"head": 5, "memory_to_active": 0.24402785301208496, "contribution_to_active": 0.652253270149231, "mix_mean": 0.32732313871383667, "mix_low": 0.08305168151855469, "mix_high": 0.006123542785644531, "read_cosine": 0.531231164932251, "beta_mean": 0.08364010602235794, "nonfinite": 0.0, "weighted_memory_to_active": 0.16743014752864838, "mix_std": 0.3316490054130554, "beta_std": 0.12343476712703705, "beta_low": 0.2618408203125, "beta_high": 0.0, "orthogonal_to_active": 0.13975122570991516, "evidence_weights": [-0.07653184980154037, -0.34231165051460266, -0.3774124085903168], "agreement_weight": 0.023440934717655182, "orthogonal_weight": 1.030187964439392}]}, {"layer": 9, "memory_to_active": 1.2269178628921509, "contribution_to_active": 0.5431865453720093, "mix_mean": 0.19323398172855377, "mix_low": 0.19630226492881775, "mix_high": 0.0010094642639160156, "read_cosine": 0.36719805002212524, "beta_mean": 0.0987202599644661, "nonfinite": 0.0, "weighted_memory_to_active": 0.36296582221984863, "mix_std": 0.20563118159770966, "beta_std": 0.054264068603515625, "beta_low": 0.4925333857536316, "beta_high": 0.0, "orthogonal_to_active": 0.02225957065820694, "output_influence": 0.3311879634857178, "residual_influence": 0.07928841561079025, "state_active": 0.040428660809993744, "state_memory": 0.028852388262748718, "state_peak": 31.681941986083984, "growth_active": 1.1711535453796387, "growth_memory": 1.474177360534668, "heads": [{"head": 0, "memory_to_active": 0.014148728922009468, "contribution_to_active": 0.40649133920669556, "mix_mean": 0.4082503914833069, "mix_low": 0.010656356811523438, "mix_high": 0.0029897689819335938, "read_cosine": 0.21752429008483887, "beta_mean": 2.937031240435317e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006418305449187756, "mix_std": 0.2920180559158325, "beta_std": 0.00011290951806586236, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035817669704556465, "evidence_weights": [-0.2285371869802475, -0.21592503786087036, 0.9224507212638855], "agreement_weight": 0.050164517015218735, "orthogonal_weight": -0.5832101702690125}, {"head": 1, "memory_to_active": 0.03514521196484566, "contribution_to_active": 0.39134681224823, "mix_mean": 0.24238194525241852, "mix_low": 0.05453968048095703, "mix_high": 0.0004749298095703125, "read_cosine": 0.40156883001327515, "beta_mean": 0.00010955813922919333, "nonfinite": 0.0, "weighted_memory_to_active": 0.013005703687667847, "mix_std": 0.2511221766471863, "beta_std": 0.0002039418468484655, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004680791404098272, "evidence_weights": [-0.1634242832660675, -0.1591004878282547, 0.5608140230178833], "agreement_weight": 0.2653690278530121, "orthogonal_weight": -0.40189722180366516}, {"head": 2, "memory_to_active": 0.3114244341850281, "contribution_to_active": 0.44461506605148315, "mix_mean": 0.29415613412857056, "mix_low": 0.034285545349121094, "mix_high": 0.000553131103515625, "read_cosine": 0.423933207988739, "beta_mean": 0.002842884510755539, "nonfinite": 0.0, "weighted_memory_to_active": 0.15364767611026764, "mix_std": 0.2772369384765625, "beta_std": 0.006353885401040316, "beta_low": 0.95306396484375, "beta_high": 0.0, "orthogonal_to_active": 0.027110308408737183, "evidence_weights": [-0.12395370006561279, -0.12673024833202362, 0.2517927587032318], "agreement_weight": 0.1318589597940445, "orthogonal_weight": -0.2278638631105423}, {"head": 3, "memory_to_active": 1.9697165489196777, "contribution_to_active": 0.2060507833957672, "mix_mean": 0.04079912230372429, "mix_low": 0.39369869232177734, "mix_high": 0.0, "read_cosine": 0.4788542091846466, "beta_mean": 0.1156555563211441, "nonfinite": 0.0, "weighted_memory_to_active": 0.17408913373947144, "mix_std": 0.0932307094335556, "beta_std": 0.07790914177894592, "beta_low": 0.0020751953125, "beta_high": 0.0, "orthogonal_to_active": 0.0219222754240036, "evidence_weights": [0.03434077650308609, 0.024567537009716034, -0.09459195286035538], "agreement_weight": 0.02154356613755226, "orthogonal_weight": 0.12401287257671356}, {"head": 4, "memory_to_active": 3.208209753036499, "contribution_to_active": 1.2446577548980713, "mix_mean": 0.12015280872583389, "mix_low": 0.22408676147460938, "mix_high": 0.0019426345825195312, "read_cosine": 0.26611900329589844, "beta_mean": 0.26152172684669495, "nonfinite": 0.0, "weighted_memory_to_active": 1.160063624382019, "mix_std": 0.20001792907714844, "beta_std": 0.128074511885643, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008263681083917618, "evidence_weights": [-0.04227060079574585, -0.10799989104270935, 0.08452598750591278], "agreement_weight": -0.04918723925948143, "orthogonal_weight": 0.007108732126653194}, {"head": 5, "memory_to_active": 1.822861909866333, "contribution_to_active": 0.5659574270248413, "mix_mean": 0.05366351455450058, "mix_low": 0.46054649353027344, "mix_high": 9.632110595703125e-05, "read_cosine": 0.4151887893676758, "beta_mean": 0.21216244995594025, "nonfinite": 0.0, "weighted_memory_to_active": 0.6705702543258667, "mix_std": 0.12016122043132782, "beta_std": 0.11292998492717743, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.06799858808517456, "evidence_weights": [0.03406265005469322, -0.03620273247361183, 0.047627370804548264], "agreement_weight": -0.005756050813943148, "orthogonal_weight": 0.13981102406978607}]}, {"layer": 10, "memory_to_active": 0.2355530709028244, "contribution_to_active": 0.39320528507232666, "mix_mean": 0.33870750665664673, "mix_low": 0.04507716745138168, "mix_high": 0.00987386703491211, "read_cosine": 0.38829681277275085, "beta_mean": 0.04326637461781502, "nonfinite": 0.0, "weighted_memory_to_active": 0.09736409783363342, "mix_std": 0.2493671476840973, "beta_std": 0.05145525932312012, "beta_low": 0.6063232421875, "beta_high": 0.0, "orthogonal_to_active": 0.03672533854842186, "output_influence": 0.36933261156082153, "residual_influence": 0.10727815330028534, "state_active": 0.06635908037424088, "state_memory": 0.016580753028392792, "state_peak": 83.66018676757812, "growth_active": 1.1091409921646118, "growth_memory": 1.1212377548217773, "heads": [{"head": 0, "memory_to_active": 0.42746981978416443, "contribution_to_active": 0.7631828784942627, "mix_mean": 0.6251261830329895, "mix_low": 0.015619277954101562, "mix_high": 0.05801677703857422, "read_cosine": 0.21430069208145142, "beta_mean": 0.1742316037416458, "nonfinite": 0.0, "weighted_memory_to_active": 0.29612332582473755, "mix_std": 0.3534344434738159, "beta_std": 0.2003883421421051, "beta_low": 0.1195068359375, "beta_high": 0.0, "orthogonal_to_active": 0.1215534582734108, "evidence_weights": [-0.1629500687122345, -0.33506903052330017, -0.20255908370018005], "agreement_weight": 0.019164783880114555, "orthogonal_weight": 0.4532598555088043}, {"head": 1, "memory_to_active": 0.0016345740295946598, "contribution_to_active": 0.43391263484954834, "mix_mean": 0.38749223947525024, "mix_low": 0.006861686706542969, "mix_high": 0.0004978179931640625, "read_cosine": 0.19651226699352264, "beta_mean": 9.600759767636191e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0006708919536322355, "mix_std": 0.25959551334381104, "beta_std": 2.0180403225822374e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003934805281460285, "evidence_weights": [-0.22881482541561127, -0.1661246120929718, 0.18763524293899536], "agreement_weight": 0.06854837387800217, "orthogonal_weight": -0.633895218372345}, {"head": 2, "memory_to_active": 0.5494805574417114, "contribution_to_active": 0.19344264268875122, "mix_mean": 0.11984476447105408, "mix_low": 0.20055389404296875, "mix_high": 1.1444091796875e-05, "read_cosine": 0.16700932383537292, "beta_mean": 0.011858662590384483, "nonfinite": 0.0, "weighted_memory_to_active": 0.14195454120635986, "mix_std": 0.17821133136749268, "beta_std": 0.02289723791182041, "beta_low": 0.69390869140625, "beta_high": 0.0, "orthogonal_to_active": 0.02774818241596222, "evidence_weights": [-0.016742466017603874, -0.09624475985765457, 0.1881643384695053], "agreement_weight": 0.005524080712348223, "orthogonal_weight": -0.20247110724449158}, {"head": 3, "memory_to_active": 0.2720736861228943, "contribution_to_active": 0.1618453860282898, "mix_mean": 0.26337361335754395, "mix_low": 0.03685569763183594, "mix_high": 0.0006952285766601562, "read_cosine": 0.8392622470855713, "beta_mean": 0.06635169684886932, "nonfinite": 0.0, "weighted_memory_to_active": 0.06840893626213074, "mix_std": 0.25076788663864136, "beta_std": 0.07362917810678482, "beta_low": 0.01995849609375, "beta_high": 0.0, "orthogonal_to_active": 0.052708905190229416, "evidence_weights": [-0.12295389920473099, -0.14686354994773865, 0.23067310452461243], "agreement_weight": 0.005298812408000231, "orthogonal_weight": 1.0216481685638428}, {"head": 4, "memory_to_active": 0.02292376570403576, "contribution_to_active": 0.3647364675998688, "mix_mean": 0.3248830735683441, "mix_low": 0.004352569580078125, "mix_high": 8.58306884765625e-06, "read_cosine": 0.09221655875444412, "beta_mean": 6.808467878727242e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007162740919739008, "mix_std": 0.239775151014328, "beta_std": 0.0013251429190859199, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0017649184446781874, "evidence_weights": [-0.16936534643173218, -0.15752790868282318, 0.1571437418460846], "agreement_weight": 0.016028568148612976, "orthogonal_weight": -0.25797057151794434}, {"head": 5, "memory_to_active": 0.13973601162433624, "contribution_to_active": 0.4421115517616272, "mix_mean": 0.3115250766277313, "mix_low": 0.0062198638916015625, "mix_high": 1.33514404296875e-05, "read_cosine": 0.820479691028595, "beta_mean": 0.007078573107719421, "nonfinite": 0.0, "weighted_memory_to_active": 0.06986412405967712, "mix_std": 0.21441853046417236, "beta_std": 0.01047144178301096, "beta_low": 0.8046875, "beta_high": 0.0, "orthogonal_to_active": 0.01618310436606407, "evidence_weights": [-0.1891125589609146, -0.16671517491340637, 0.23567481338977814], "agreement_weight": 0.08631991595029831, "orthogonal_weight": -0.53018718957901}]}, {"layer": 11, "memory_to_active": 0.316389262676239, "contribution_to_active": 0.4450690746307373, "mix_mean": 0.35993003845214844, "mix_low": 0.04713853448629379, "mix_high": 0.018530210480093956, "read_cosine": 0.5139023065567017, "beta_mean": 0.0422341451048851, "nonfinite": 0.0, "weighted_memory_to_active": 0.1000361442565918, "mix_std": 0.29517701268196106, "beta_std": 0.06062929332256317, "beta_low": 0.45263671875, "beta_high": 0.0, "orthogonal_to_active": 0.03860132396221161, "output_influence": 0.4746619760990143, "residual_influence": 0.12278890609741211, "state_active": 0.12015672028064728, "state_memory": 0.014053162187337875, "state_peak": 26.477962493896484, "growth_active": 1.1686172485351562, "growth_memory": 1.4137896299362183, "heads": [{"head": 0, "memory_to_active": 0.05018775910139084, "contribution_to_active": 0.5258493423461914, "mix_mean": 0.48109880089759827, "mix_low": 0.0053348541259765625, "mix_high": 0.003940582275390625, "read_cosine": 0.43470802903175354, "beta_mean": 0.00022666167933493853, "nonfinite": 0.0, "weighted_memory_to_active": 0.031121838837862015, "mix_std": 0.28120583295822144, "beta_std": 0.0048059276305139065, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.015751589089632034, "evidence_weights": [-0.15981099009513855, -0.2845589816570282, 0.45994314551353455], "agreement_weight": 0.12898105382919312, "orthogonal_weight": -0.6457921266555786}, {"head": 1, "memory_to_active": 0.5798986554145813, "contribution_to_active": 0.20970240235328674, "mix_mean": 0.21673353016376495, "mix_low": 0.045022010803222656, "mix_high": 0.000141143798828125, "read_cosine": 0.5970421433448792, "beta_mean": 0.04087185859680176, "nonfinite": 0.0, "weighted_memory_to_active": 0.1758895069360733, "mix_std": 0.22577683627605438, "beta_std": 0.07337448745965958, "beta_low": 0.3489990234375, "beta_high": 0.0, "orthogonal_to_active": 0.020577039569616318, "evidence_weights": [-0.15133775770664215, -0.10393878072500229, 0.12967190146446228], "agreement_weight": 0.0902545154094696, "orthogonal_weight": -0.15988530218601227}, {"head": 2, "memory_to_active": 0.061108946800231934, "contribution_to_active": 0.839129626750946, "mix_mean": 0.4813616871833801, "mix_low": 0.036238670349121094, "mix_high": 0.0629281997680664, "read_cosine": 0.3526967763900757, "beta_mean": 0.07035241276025772, "nonfinite": 0.0, "weighted_memory_to_active": 0.05129549652338028, "mix_std": 0.40110450983047485, "beta_std": 0.09797777235507965, "beta_low": 0.4395751953125, "beta_high": 0.0, "orthogonal_to_active": 0.08897162973880768, "evidence_weights": [-0.16480019688606262, -0.40725547075271606, -0.5997713804244995], "agreement_weight": -0.014145603403449059, "orthogonal_weight": 1.904919981956482}, {"head": 3, "memory_to_active": 0.0341864675283432, "contribution_to_active": 0.4574870765209198, "mix_mean": 0.44167381525039673, "mix_low": 0.0853586196899414, "mix_high": 0.04355430603027344, "read_cosine": 0.5463719367980957, "beta_mean": 0.06957001984119415, "nonfinite": 0.0, "weighted_memory_to_active": 0.019371256232261658, "mix_std": 0.4137197732925415, "beta_std": 0.1014445424079895, "beta_low": 0.433837890625, "beta_high": 0.0, "orthogonal_to_active": 0.03250391408801079, "evidence_weights": [-0.18564186990261078, -0.36414384841918945, -0.17786863446235657], "agreement_weight": 0.021147292107343674, "orthogonal_weight": 2.0620317459106445}, {"head": 4, "memory_to_active": 0.9631786346435547, "contribution_to_active": 0.25463587045669556, "mix_mean": 0.15343183279037476, "mix_low": 0.10839557647705078, "mix_high": 1.9073486328125e-06, "read_cosine": 0.26831817626953125, "beta_mean": 0.04692676290869713, "nonfinite": 0.0, "weighted_memory_to_active": 0.2212902009487152, "mix_std": 0.1890251487493515, "beta_std": 0.050435684621334076, "beta_low": 0.1014404296875, "beta_high": 0.0, "orthogonal_to_active": 0.0484500490128994, "evidence_weights": [-0.06942996382713318, -0.0518367737531662, 0.7829835414886475], "agreement_weight": -0.06463156640529633, "orthogonal_weight": -0.2484213262796402}, {"head": 5, "memory_to_active": 0.20977498590946198, "contribution_to_active": 0.38360995054244995, "mix_mean": 0.38528046011924744, "mix_low": 0.0024814605712890625, "mix_high": 0.0006151199340820312, "read_cosine": 0.8842763900756836, "beta_mean": 0.025457171723246574, "nonfinite": 0.0, "weighted_memory_to_active": 0.10124853998422623, "mix_std": 0.2602298855781555, "beta_std": 0.03573733568191528, "beta_low": 0.39288330078125, "beta_high": 0.0, "orthogonal_to_active": 0.0253536868840456, "evidence_weights": [-0.18711914122104645, -0.22865170240402222, 0.32177969813346863], "agreement_weight": 0.046088576316833496, "orthogonal_weight": -0.541972815990448}]}, {"layer": 12, "memory_to_active": 0.5385172367095947, "contribution_to_active": 0.473471075296402, "mix_mean": 0.3636092245578766, "mix_low": 0.04943815991282463, "mix_high": 0.0012485187035053968, "read_cosine": 0.2984948456287384, "beta_mean": 0.02062845602631569, "nonfinite": 0.0, "weighted_memory_to_active": 0.21578876674175262, "mix_std": 0.2334018051624298, "beta_std": 0.02426038309931755, "beta_low": 0.7143962383270264, "beta_high": 0.0, "orthogonal_to_active": 0.06346773356199265, "output_influence": 0.33003586530685425, "residual_influence": 0.08794916421175003, "state_active": 0.05449068173766136, "state_memory": 0.015843404456973076, "state_peak": 35.33341979980469, "growth_active": 1.172692060470581, "growth_memory": 1.2865798473358154, "heads": [{"head": 0, "memory_to_active": 0.43414556980133057, "contribution_to_active": 0.2964349389076233, "mix_mean": 0.2716330885887146, "mix_low": 0.023319244384765625, "mix_high": 3.0517578125e-05, "read_cosine": 0.4015345573425293, "beta_mean": 0.007292215246707201, "nonfinite": 0.0, "weighted_memory_to_active": 0.15690773725509644, "mix_std": 0.23597830533981323, "beta_std": 0.019411172717809677, "beta_low": 0.83935546875, "beta_high": 0.0, "orthogonal_to_active": 0.0577886700630188, "evidence_weights": [-0.12847162783145905, -0.17909148335456848, 0.2427120804786682], "agreement_weight": 0.09911226481199265, "orthogonal_weight": -0.4582537114620209}, {"head": 1, "memory_to_active": 0.2162395715713501, "contribution_to_active": 0.40603506565093994, "mix_mean": 0.3892767131328583, "mix_low": 0.0014200210571289062, "mix_high": 0.0001583099365234375, "read_cosine": 0.5987426042556763, "beta_mean": 0.00295356591232121, "nonfinite": 0.0, "weighted_memory_to_active": 0.0970120057463646, "mix_std": 0.2533113956451416, "beta_std": 0.0074797277338802814, "beta_low": 0.94232177734375, "beta_high": 0.0, "orthogonal_to_active": 0.06132034957408905, "evidence_weights": [-0.16850879788398743, -0.21669408679008484, 0.568812370300293], "agreement_weight": 0.0906585231423378, "orthogonal_weight": -0.8597265481948853}, {"head": 2, "memory_to_active": 0.5879426598548889, "contribution_to_active": 0.3602662682533264, "mix_mean": 0.2937573790550232, "mix_low": 0.022832870483398438, "mix_high": 6.866455078125e-05, "read_cosine": 0.34931954741477966, "beta_mean": 0.08899278938770294, "nonfinite": 0.0, "weighted_memory_to_active": 0.23073768615722656, "mix_std": 0.2381952702999115, "beta_std": 0.0732700452208519, "beta_low": 0.00384521484375, "beta_high": 0.0, "orthogonal_to_active": 0.1065230444073677, "evidence_weights": [-0.21478626132011414, -0.06675455719232559, 0.02566840685904026], "agreement_weight": 0.03857536241412163, "orthogonal_weight": 0.5616387724876404}, {"head": 3, "memory_to_active": 0.03230755403637886, "contribution_to_active": 0.6498070955276489, "mix_mean": 0.6234139204025269, "mix_low": 0.0005893707275390625, "mix_high": 0.005748748779296875, "read_cosine": -0.00583435595035553, "beta_mean": 0.00024896906688809395, "nonfinite": 0.0, "weighted_memory_to_active": 0.020343661308288574, "mix_std": 0.2536586821079254, "beta_std": 0.0005817886558361351, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.015186524949967861, "evidence_weights": [-0.30507516860961914, -0.29094257950782776, 0.4249301552772522], "agreement_weight": 0.14559023082256317, "orthogonal_weight": -0.7682160139083862}, {"head": 4, "memory_to_active": 1.9024581909179688, "contribution_to_active": 0.7132723331451416, "mix_mean": 0.09330695122480392, "mix_low": 0.2431621551513672, "mix_high": 0.0009679794311523438, "read_cosine": 0.0507037416100502, "beta_mean": 0.023514073342084885, "nonfinite": 0.0, "weighted_memory_to_active": 0.7627715468406677, "mix_std": 0.14951585233211517, "beta_std": 0.042156465351581573, "beta_low": 0.512939453125, "beta_high": 0.0, "orthogonal_to_active": 0.1196354553103447, "evidence_weights": [0.04233362525701523, -0.08549876511096954, 0.008578479290008545], "agreement_weight": -0.01151199173182249, "orthogonal_weight": -0.17230677604675293}, {"head": 5, "memory_to_active": 0.058009758591651917, "contribution_to_active": 0.4150104820728302, "mix_mean": 0.5102671384811401, "mix_low": 0.005305290222167969, "mix_high": 0.0005168914794921875, "read_cosine": 0.39650288224220276, "beta_mean": 0.0007691240753047168, "nonfinite": 0.0, "weighted_memory_to_active": 0.026959877461194992, "mix_std": 0.2697512209415436, "beta_std": 0.002663105260580778, "beta_low": 0.9884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.020352356135845184, "evidence_weights": [-0.25708556175231934, -0.2674928307533264, 0.5740352272987366], "agreement_weight": 0.05526082217693329, "orthogonal_weight": -0.8308846354484558}]}, {"layer": 13, "memory_to_active": 0.24063901603221893, "contribution_to_active": 0.42289111018180847, "mix_mean": 0.4181188941001892, "mix_low": 0.00449117086827755, "mix_high": 0.0006268819561228156, "read_cosine": 0.3935798406600952, "beta_mean": 0.017997892573475838, "nonfinite": 0.0, "weighted_memory_to_active": 0.11089091747999191, "mix_std": 0.25071191787719727, "beta_std": 0.020013054832816124, "beta_low": 0.7485352158546448, "beta_high": 0.0, "orthogonal_to_active": 0.08364668488502502, "output_influence": 0.3681013286113739, "residual_influence": 0.12000132352113724, "state_active": 0.08243905007839203, "state_memory": 0.03173181414604187, "state_peak": 72.94173431396484, "growth_active": 0.9942735433578491, "growth_memory": 1.0687795877456665, "heads": [{"head": 0, "memory_to_active": 0.2197011113166809, "contribution_to_active": 0.41927698254585266, "mix_mean": 0.3932030200958252, "mix_low": 0.00266265869140625, "mix_high": 0.0005130767822265625, "read_cosine": 0.36084824800491333, "beta_mean": 0.002547377720475197, "nonfinite": 0.0, "weighted_memory_to_active": 0.10317552089691162, "mix_std": 0.2554190456867218, "beta_std": 0.004082805477082729, "beta_low": 0.951904296875, "beta_high": 0.0, "orthogonal_to_active": 0.09045934677124023, "evidence_weights": [-0.19112156331539154, -0.2097269743680954, 0.5813456773757935], "agreement_weight": 0.045922331511974335, "orthogonal_weight": -0.9728839993476868}, {"head": 1, "memory_to_active": 0.12294892966747284, "contribution_to_active": 0.4770331084728241, "mix_mean": 0.4065965712070465, "mix_low": 0.0028705596923828125, "mix_high": 4.1961669921875e-05, "read_cosine": 0.2707885205745697, "beta_mean": 0.0006932196556590497, "nonfinite": 0.0, "weighted_memory_to_active": 0.05941876396536827, "mix_std": 0.2571812570095062, "beta_std": 0.0015137020964175463, "beta_low": 0.996826171875, "beta_high": 0.0, "orthogonal_to_active": 0.05240800231695175, "evidence_weights": [-0.18573525547981262, -0.21747328341007233, 0.6583309173583984], "agreement_weight": 0.08247692137956619, "orthogonal_weight": -0.9417675137519836}, {"head": 2, "memory_to_active": 0.2227834165096283, "contribution_to_active": 0.30700844526290894, "mix_mean": 0.29465001821517944, "mix_low": 0.0031862258911132812, "mix_high": 8.58306884765625e-06, "read_cosine": 0.6502264738082886, "beta_mean": 0.01853526011109352, "nonfinite": 0.0, "weighted_memory_to_active": 0.07831685990095139, "mix_std": 0.20614731311798096, "beta_std": 0.039125777781009674, "beta_low": 0.61480712890625, "beta_high": 0.0, "orthogonal_to_active": 0.03113379143178463, "evidence_weights": [-0.20007485151290894, -0.14791008830070496, 0.4270102381706238], "agreement_weight": 0.05240476876497269, "orthogonal_weight": -0.6181351542472839}, {"head": 3, "memory_to_active": 0.22090235352516174, "contribution_to_active": 0.5007551312446594, "mix_mean": 0.524739146232605, "mix_low": 0.005372047424316406, "mix_high": 0.0018463134765625, "read_cosine": -0.032409049570560455, "beta_mean": 0.001079125446267426, "nonfinite": 0.0, "weighted_memory_to_active": 0.1531599462032318, "mix_std": 0.2499639242887497, "beta_std": 0.007340774871408939, "beta_low": 0.99005126953125, "beta_high": 0.0, "orthogonal_to_active": 0.15565653145313263, "evidence_weights": [-0.34675318002700806, -0.23722054064273834, 1.1585569381713867], "agreement_weight": 0.9957216382026672, "orthogonal_weight": -1.027735710144043}, {"head": 4, "memory_to_active": 0.4314846396446228, "contribution_to_active": 0.307573527097702, "mix_mean": 0.45864057540893555, "mix_low": 0.006045341491699219, "mix_high": 0.0007343292236328125, "read_cosine": 0.831655740737915, "beta_mean": 0.08247922360897064, "nonfinite": 0.0, "weighted_memory_to_active": 0.16321983933448792, "mix_std": 0.2602313458919525, "beta_std": 0.05993980914354324, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.11155760288238525, "evidence_weights": [-0.26058995723724365, -0.2602649927139282, 0.44943055510520935], "agreement_weight": -0.004609964322298765, "orthogonal_weight": 0.9753790497779846}, {"head": 5, "memory_to_active": 0.22601357102394104, "contribution_to_active": 0.5256994962692261, "mix_mean": 0.43088382482528687, "mix_low": 0.006810188293457031, "mix_high": 0.0006170272827148438, "read_cosine": 0.2803688645362854, "beta_mean": 0.0026531401090323925, "nonfinite": 0.0, "weighted_memory_to_active": 0.10805452615022659, "mix_std": 0.2753283977508545, "beta_std": 0.00807545892894268, "beta_low": 0.93707275390625, "beta_high": 0.0, "orthogonal_to_active": 0.06066479533910751, "evidence_weights": [-0.20061533153057098, -0.2294333428144455, 0.28643113374710083], "agreement_weight": 0.06324750930070877, "orthogonal_weight": -0.6401005983352661}]}, {"layer": 14, "memory_to_active": 0.28968545794487, "contribution_to_active": 0.428724467754364, "mix_mean": 0.38795632123947144, "mix_low": 0.022617023438215256, "mix_high": 0.002834161277860403, "read_cosine": 0.6077836751937866, "beta_mean": 0.07394395768642426, "nonfinite": 0.0, "weighted_memory_to_active": 0.13776595890522003, "mix_std": 0.25134968757629395, "beta_std": 0.06923305243253708, "beta_low": 0.3611348867416382, "beta_high": 0.0, "orthogonal_to_active": 0.04074746370315552, "output_influence": 0.24546217918395996, "residual_influence": 0.09431873261928558, "state_active": 0.086952805519104, "state_memory": 0.018459344282746315, "state_peak": 18.48443603515625, "growth_active": 1.1948226690292358, "growth_memory": 1.4123528003692627, "heads": [{"head": 0, "memory_to_active": 0.29152148962020874, "contribution_to_active": 0.2717097997665405, "mix_mean": 0.2647685110569, "mix_low": 0.01681232452392578, "mix_high": 0.00015735626220703125, "read_cosine": 0.6861928105354309, "beta_mean": 0.06740079820156097, "nonfinite": 0.0, "weighted_memory_to_active": 0.10438662767410278, "mix_std": 0.23282113671302795, "beta_std": 0.08634394407272339, "beta_low": 0.1435546875, "beta_high": 0.0, "orthogonal_to_active": 0.0146973617374897, "evidence_weights": [-0.05861607939004898, -0.1331142783164978, 0.7030805945396423], "agreement_weight": 0.04130670800805092, "orthogonal_weight": 0.20429672300815582}, {"head": 1, "memory_to_active": 0.4544258117675781, "contribution_to_active": 0.4789222478866577, "mix_mean": 0.3027622103691101, "mix_low": 0.0823678970336914, "mix_high": 0.0008296966552734375, "read_cosine": 0.5738338232040405, "beta_mean": 0.3023749589920044, "nonfinite": 0.0, "weighted_memory_to_active": 0.2347213178873062, "mix_std": 0.2874755859375, "beta_std": 0.2622235119342804, "beta_low": 0.07684326171875, "beta_high": 0.0, "orthogonal_to_active": 0.0164671428501606, "evidence_weights": [-0.017943454906344414, -0.2630336880683899, 0.043875135481357574], "agreement_weight": 0.056155309081077576, "orthogonal_weight": 0.10171056538820267}, {"head": 2, "memory_to_active": 0.330864280462265, "contribution_to_active": 0.4562768340110779, "mix_mean": 0.4483966827392578, "mix_low": 0.005228996276855469, "mix_high": 0.0013647079467773438, "read_cosine": 0.445588618516922, "beta_mean": 0.01624983176589012, "nonfinite": 0.0, "weighted_memory_to_active": 0.18229015171527863, "mix_std": 0.27062076330184937, "beta_std": 0.0169222392141819, "beta_low": 0.43524169921875, "beta_high": 0.0, "orthogonal_to_active": 0.12120518088340759, "evidence_weights": [-0.19821596145629883, -0.2215576320886612, 0.560112714767456], "agreement_weight": 0.08670352399349213, "orthogonal_weight": -0.7834689021110535}, {"head": 3, "memory_to_active": 0.06852942705154419, "contribution_to_active": 0.6468180418014526, "mix_mean": 0.6478878855705261, "mix_low": 0.0022306442260742188, "mix_high": 0.009461402893066406, "read_cosine": 0.5236907005310059, "beta_mean": 0.01127622090280056, "nonfinite": 0.0, "weighted_memory_to_active": 0.046797581017017365, "mix_std": 0.23938024044036865, "beta_std": 0.009380370378494263, "beta_low": 0.55377197265625, "beta_high": 0.0, "orthogonal_to_active": 0.020594073459506035, "evidence_weights": [-0.3763482868671417, -0.3242053985595703, 0.40443289279937744], "agreement_weight": 0.07265191525220871, "orthogonal_weight": -0.5725939869880676}, {"head": 4, "memory_to_active": 0.22489286959171295, "contribution_to_active": 0.46365299820899963, "mix_mean": 0.42596569657325745, "mix_low": 0.0044422149658203125, "mix_high": 0.00501251220703125, "read_cosine": 0.7318969964981079, "beta_mean": 0.04231617599725723, "nonfinite": 0.0, "weighted_memory_to_active": 0.11863282322883606, "mix_std": 0.2568846642971039, "beta_std": 0.030448557808995247, "beta_low": 0.0423583984375, "beta_high": 0.0, "orthogonal_to_active": 0.0251797903329134, "evidence_weights": [-0.21334071457386017, -0.2612401247024536, 0.2453196495771408], "agreement_weight": 0.10601364821195602, "orthogonal_weight": -0.3390069007873535}, {"head": 5, "memory_to_active": 0.3678787648677826, "contribution_to_active": 0.25496697425842285, "mix_mean": 0.23795703053474426, "mix_low": 0.02462005615234375, "mix_high": 0.000179290771484375, "read_cosine": 0.6854991912841797, "beta_mean": 0.0040457360446453094, "nonfinite": 0.0, "weighted_memory_to_active": 0.13976721465587616, "mix_std": 0.22091570496559143, "beta_std": 0.010079668834805489, "beta_low": 0.9150390625, "beta_high": 0.0, "orthogonal_to_active": 0.04634121060371399, "evidence_weights": [-0.18160907924175262, -0.1291753500699997, 0.09950993955135345], "agreement_weight": 0.09762919694185257, "orthogonal_weight": -0.518488883972168}]}, {"layer": 15, "memory_to_active": 0.09441917389631271, "contribution_to_active": 0.4537261128425598, "mix_mean": 0.3692920207977295, "mix_low": 0.007747491355985403, "mix_high": 0.0024743080139160156, "read_cosine": 0.6201790571212769, "beta_mean": 0.050375863909721375, "nonfinite": 0.0, "weighted_memory_to_active": 0.04472148418426514, "mix_std": 0.26557886600494385, "beta_std": 0.05203348398208618, "beta_low": 0.693359375, "beta_high": 0.0, "orthogonal_to_active": 0.011224472895264626, "output_influence": 0.2842395305633545, "residual_influence": 0.15184451639652252, "state_active": 0.11366284638643265, "state_memory": 0.009689181111752987, "state_peak": 24.672269821166992, "growth_active": 1.4042633771896362, "growth_memory": 1.567919135093689, "heads": [{"head": 0, "memory_to_active": 0.11895838379859924, "contribution_to_active": 0.4567459225654602, "mix_mean": 0.32658901810646057, "mix_low": 0.008101463317871094, "mix_high": 0.0025129318237304688, "read_cosine": 0.6685153841972351, "beta_mean": 0.0513087660074234, "nonfinite": 0.0, "weighted_memory_to_active": 0.055682457983493805, "mix_std": 0.25048011541366577, "beta_std": 0.04049541428685188, "beta_low": 0.1097412109375, "beta_high": 0.0, "orthogonal_to_active": 0.01386190950870514, "evidence_weights": [-0.16924446821212769, -0.20515240728855133, 0.3495469093322754], "agreement_weight": 0.0815163180232048, "orthogonal_weight": -0.4610550105571747}, {"head": 1, "memory_to_active": 0.19584141671657562, "contribution_to_active": 0.3963810205459595, "mix_mean": 0.3690684735774994, "mix_low": 0.005980491638183594, "mix_high": 0.006360054016113281, "read_cosine": 0.8426610827445984, "beta_mean": 0.0018965702038258314, "nonfinite": 0.0, "weighted_memory_to_active": 0.07924249768257141, "mix_std": 0.2669941782951355, "beta_std": 0.004749315325170755, "beta_low": 0.97613525390625, "beta_high": 0.0, "orthogonal_to_active": 0.006670136470347643, "evidence_weights": [-0.19631683826446533, -0.20167577266693115, 0.272899329662323], "agreement_weight": 0.09495674073696136, "orthogonal_weight": -0.1402021199464798}, {"head": 2, "memory_to_active": 0.010687315836548805, "contribution_to_active": 0.4318259656429291, "mix_mean": 0.3402405083179474, "mix_low": 0.012734413146972656, "mix_high": 0.0001583099365234375, "read_cosine": 0.7450140714645386, "beta_mean": 0.00024140092136804014, "nonfinite": 0.0, "weighted_memory_to_active": 0.004191912245005369, "mix_std": 0.26625096797943115, "beta_std": 0.0005177073180675507, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.0011760932393372059, "evidence_weights": [-0.21498897671699524, -0.26942145824432373, 0.31160208582878113], "agreement_weight": 0.049162913113832474, "orthogonal_weight": -0.4249357581138611}, {"head": 3, "memory_to_active": 0.08827939629554749, "contribution_to_active": 0.4666762948036194, "mix_mean": 0.29020175337791443, "mix_low": 0.0033512115478515625, "mix_high": 4.76837158203125e-06, "read_cosine": 0.6798082590103149, "beta_mean": 0.0029050668235868216, "nonfinite": 0.0, "weighted_memory_to_active": 0.04326634109020233, "mix_std": 0.23061150312423706, "beta_std": 0.0053043123334646225, "beta_low": 0.93316650390625, "beta_high": 0.0, "orthogonal_to_active": 0.01596037670969963, "evidence_weights": [-0.1703001707792282, -0.21013863384723663, 0.37195760011672974], "agreement_weight": 0.027731725946068764, "orthogonal_weight": -0.6390266418457031}, {"head": 4, "memory_to_active": 0.14782242476940155, "contribution_to_active": 0.5307297706604004, "mix_mean": 0.3966487646102905, "mix_low": 0.010694503784179688, "mix_high": 0.00048732757568359375, "read_cosine": 0.23702166974544525, "beta_mean": 0.24578247964382172, "nonfinite": 0.0, "weighted_memory_to_active": 0.08400152623653412, "mix_std": 0.30038413405418396, "beta_std": 0.2609315514564514, "beta_low": 0.14166259765625, "beta_high": 0.0, "orthogonal_to_active": 0.02852926403284073, "evidence_weights": [-0.26863619685173035, -0.2469465434551239, 0.3308473527431488], "agreement_weight": 0.006032309029251337, "orthogonal_weight": -0.39968428015708923}, {"head": 5, "memory_to_active": 0.0049260808154940605, "contribution_to_active": 0.4399975836277008, "mix_mean": 0.49300360679626465, "mix_low": 0.00562286376953125, "mix_high": 0.005322456359863281, "read_cosine": 0.548053503036499, "beta_mean": 0.00012089314986951649, "nonfinite": 0.0, "weighted_memory_to_active": 0.001944170449860394, "mix_std": 0.2787521481513977, "beta_std": 0.00020258974109310657, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011490585748106241, "evidence_weights": [-0.3555287718772888, -0.292741060256958, 0.49521365761756897], "agreement_weight": 0.1312781572341919, "orthogonal_weight": 0.6897318959236145}]}]}, {"step": 13000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7627667020005902, "memory_to_active": 1.0305142402648926, "contribution_to_active": 0.4348134696483612, "mix_mean": 0.2791191637516022, "mix_low": 0.1802869588136673, "mix_high": 0.004194001667201519, "read_cosine": 0.39539995789527893, "beta_mean": 0.09679011255502701, "nonfinite": 0.0, "weighted_memory_to_active": 0.21345826983451843, "mix_std": 0.20343291759490967, "beta_std": 0.07479671388864517, "beta_low": 0.43691253662109375, "beta_high": 7.629394076502649e-06, "orthogonal_to_active": 0.03916757181286812, "samples": 4, "scored_tokens": 899, "normal_loss": 3.230473756790161, "normal_token_accuracy": 0.41156840324401855, "state_active": 0.06292718648910522, "state_memory": 0.025916634127497673, "state_peak": 91.08601379394531, "growth_active": 1.1470134258270264, "growth_memory": 1.4075583219528198, "output_influence": 0.29321974515914917, "residual_influence": 0.09162619709968567, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2829666137695312, "no_memory_loss_delta": 0.07967385649681091, "scrambled_loss_delta": 0.1601368635892868, "memory_only_loss_delta": 2.071195602416992, "no_evidence_loss_delta": 0.014152870513498783, "no_agreement_loss_delta": -0.014872778207063675, "no_orthogonal_loss_delta": -0.001240838784724474}, {"index": 94, "tokens": 256, "normal_loss": 3.2720565795898438, "no_memory_loss_delta": 0.28918159008026123, "scrambled_loss_delta": 0.4324544370174408, "memory_only_loss_delta": 2.143310785293579, "no_evidence_loss_delta": 0.13485394418239594, "no_agreement_loss_delta": 0.012947472743690014, "no_orthogonal_loss_delta": 0.013236917555332184}, {"index": 188, "tokens": 256, "normal_loss": 3.731489896774292, "no_memory_loss_delta": 0.12048386037349701, "scrambled_loss_delta": 0.09063000977039337, "memory_only_loss_delta": 1.9392237663269043, "no_evidence_loss_delta": 0.061466168612241745, "no_agreement_loss_delta": -0.02971978485584259, "no_orthogonal_loss_delta": -0.00040572346188127995}, {"index": 283, "tokens": 131, "normal_loss": 2.0675477981567383, "no_memory_loss_delta": 0.20011746883392334, "scrambled_loss_delta": 0.45300570130348206, "memory_only_loss_delta": 2.593904495239258, "no_evidence_loss_delta": 0.05368952825665474, "no_agreement_loss_delta": -0.007389368023723364, "no_orthogonal_loss_delta": -0.001682206871919334}], "agreement_weight_abs_mean": 0.0736495703458786, "agreement_weight_abs_max": 1.1163008213043213, "agreement_weight_mean": 0.0587499625980854, "orthogonal_weight_abs_mean": 0.429353803396225, "orthogonal_weight_abs_max": 2.227269411087036, "orthogonal_weight_mean": -0.1620134562253952, "no_memory_loss": 3.3989789485931396, "no_memory_loss_delta": 0.16850529611110687, "no_memory_token_accuracy_gain": 0.006674082484096289, "no_memory_token_win_fraction": 0.5717464089393616, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 15.50732114190627, "scrambled_loss": 3.491039514541626, "scrambled_loss_delta": 0.26056551933288574, "scrambled_token_accuracy_gain": 0.027808675542473793, "scrambled_token_win_fraction": 0.5962180495262146, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 22.938433550227707, "memory_only_loss": 5.360792636871338, "memory_only_loss_delta": 2.1303186416625977, "memory_only_token_accuracy_gain": 0.2091212421655655, "memory_only_token_win_fraction": 0.8687430620193481, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 88.12005666281986, "no_evidence_loss": 3.298232316970825, "no_evidence_loss_delta": 0.06775796413421631, "no_evidence_token_accuracy_gain": 0.0011123470030725002, "no_evidence_token_win_fraction": 0.510567307472229, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 6.551337451286408, "no_agreement_loss": 3.220386028289795, "no_agreement_loss_delta": -0.010088043287396431, "no_agreement_token_accuracy_gain": -0.0033370412420481443, "no_agreement_token_win_fraction": 0.46162402629852295, "no_agreement_probe_win_fraction": 0.25, "no_agreement_ppl_reduction_pct": -1.0139099136191123, "no_orthogonal_loss": 3.2335293292999268, "no_orthogonal_loss_delta": 0.0030553529504686594, "no_orthogonal_token_accuracy_gain": -0.004449388012290001, "no_orthogonal_token_win_fraction": 0.5061178803443909, "no_orthogonal_probe_win_fraction": 0.25, "no_orthogonal_ppl_reduction_pct": 0.30506901097263084, "short_no_memory_loss_delta": 0.14154566824436188, "long_context_memory_gain": 0.026959627866744995, "normal_context_loss_gain": 0.12917110323905945, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.908600330352783, "contribution_to_active": 0.783827006816864, "mix_mean": 0.09348778426647186, "mix_low": 0.4401583671569824, "mix_high": 0.0028160414658486843, "read_cosine": 0.4794747233390808, "beta_mean": 0.26266199350357056, "nonfinite": 0.0, "weighted_memory_to_active": 0.8114291429519653, "mix_std": 0.08283105492591858, "beta_std": 0.1482447385787964, "beta_low": 8.138020348269492e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04600889980792999, "output_influence": 0.1311045140028, "residual_influence": 0.20074597001075745, "state_active": 0.08566154539585114, "state_memory": 0.05272841453552246, "state_peak": 49.24934768676758, "growth_active": 1.2513830661773682, "growth_memory": 1.3194386959075928, "heads": [{"head": 0, "memory_to_active": 2.2596492767333984, "contribution_to_active": 0.38190802931785583, "mix_mean": 0.020694773644208908, "mix_low": 0.48956966400146484, "mix_high": 0.0, "read_cosine": 0.3394448161125183, "beta_mean": 0.22108827531337738, "nonfinite": 0.0, "weighted_memory_to_active": 0.4650835394859314, "mix_std": 0.0506739467382431, "beta_std": 0.09713360667228699, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.024866163730621338, "evidence_weights": [-0.04469495266675949, -0.07274551689624786, -0.009780720807611942], "agreement_weight": -0.07222587615251541, "orthogonal_weight": -0.06333056092262268}, {"head": 1, "memory_to_active": 3.759612560272217, "contribution_to_active": 1.3633933067321777, "mix_mean": 0.01914788782596588, "mix_low": 0.5088157653808594, "mix_high": 0.0, "read_cosine": 0.4659835398197174, "beta_mean": 0.32785534858703613, "nonfinite": 0.0, "weighted_memory_to_active": 1.658449649810791, "mix_std": 0.04575624316930771, "beta_std": 0.15631863474845886, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11121534556150436, "evidence_weights": [0.00771098118275404, -0.03551805391907692, 0.1273873746395111], "agreement_weight": 0.005667139310389757, "orthogonal_weight": -0.15601880848407745}, {"head": 2, "memory_to_active": 2.450629711151123, "contribution_to_active": 0.5126519203186035, "mix_mean": 0.01514497585594654, "mix_low": 0.6065959930419922, "mix_high": 0.0, "read_cosine": 0.35483479499816895, "beta_mean": 0.30825501680374146, "nonfinite": 0.0, "weighted_memory_to_active": 0.5741594433784485, "mix_std": 0.03281371295452118, "beta_std": 0.18859270215034485, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04966495558619499, "evidence_weights": [0.033547937870025635, 0.042142804712057114, 0.009979375638067722], "agreement_weight": -0.009828461334109306, "orthogonal_weight": -0.13983488082885742}, {"head": 3, "memory_to_active": 5.134435653686523, "contribution_to_active": 2.0157530307769775, "mix_mean": 0.05308190733194351, "mix_low": 0.2217884063720703, "mix_high": 0.0, "read_cosine": 0.41888830065727234, "beta_mean": 0.33913880586624146, "nonfinite": 0.0, "weighted_memory_to_active": 2.031893253326416, "mix_std": 0.08526715636253357, "beta_std": 0.21665805578231812, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07254589349031448, "evidence_weights": [-0.10966107249259949, -0.13576605916023254, 0.016268832609057426], "agreement_weight": -0.02063373290002346, "orthogonal_weight": 0.0466906800866127}, {"head": 4, "memory_to_active": 3.696061611175537, "contribution_to_active": 0.08323107659816742, "mix_mean": 0.006918707396835089, "mix_low": 0.7995309829711914, "mix_high": 0.0, "read_cosine": 0.34171056747436523, "beta_mean": 0.30949366092681885, "nonfinite": 0.0, "weighted_memory_to_active": 0.07568091154098511, "mix_std": 0.011091792955994606, "beta_std": 0.17087028920650482, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.011351952329277992, "evidence_weights": [0.06645078957080841, 0.02420380339026451, -0.19396156072616577], "agreement_weight": -0.026580048725008965, "orthogonal_weight": 0.17155392467975616}, {"head": 5, "memory_to_active": 0.15121130645275116, "contribution_to_active": 0.3460247218608856, "mix_mean": 0.4459384083747864, "mix_low": 0.014649391174316406, "mix_high": 0.01689624786376953, "read_cosine": 0.9559861421585083, "beta_mean": 0.07014069706201553, "nonfinite": 0.0, "weighted_memory_to_active": 0.06330807507038116, "mix_std": 0.27138346433639526, "beta_std": 0.05989517271518707, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.006409092806279659, "evidence_weights": [-0.2937038540840149, -0.28649210929870605, 0.3286532759666443], "agreement_weight": 0.15429455041885376, "orthogonal_weight": -0.24848386645317078}]}, {"layer": 1, "memory_to_active": 2.4834206104278564, "contribution_to_active": 0.37724459171295166, "mix_mean": 0.09595538675785065, "mix_low": 0.40078625082969666, "mix_high": 0.0004221598501317203, "read_cosine": 0.31111663579940796, "beta_mean": 0.2486254870891571, "nonfinite": 0.0, "weighted_memory_to_active": 0.33676931262016296, "mix_std": 0.10888671875, "beta_std": 0.140207439661026, "beta_low": 0.1698913723230362, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03281589224934578, "output_influence": 0.13133713603019714, "residual_influence": 0.02899588644504547, "state_active": 0.04738455265760422, "state_memory": 0.051500722765922546, "state_peak": 35.10873031616211, "growth_active": 1.1925849914550781, "growth_memory": 1.520297884941101, "heads": [{"head": 0, "memory_to_active": 0.11940807104110718, "contribution_to_active": 0.2807311415672302, "mix_mean": 0.20583003759384155, "mix_low": 0.06673431396484375, "mix_high": 0.00011348724365234375, "read_cosine": -0.15946565568447113, "beta_mean": 0.0004965988919138908, "nonfinite": 0.0, "weighted_memory_to_active": 0.043330855667591095, "mix_std": 0.23756733536720276, "beta_std": 0.0008626141352578998, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.021608181297779083, "evidence_weights": [-0.10523366928100586, -0.13607068359851837, -0.04015989974141121], "agreement_weight": 0.03714938461780548, "orthogonal_weight": -0.5646526217460632}, {"head": 1, "memory_to_active": 4.013688564300537, "contribution_to_active": 0.18498149514198303, "mix_mean": 0.023928191512823105, "mix_low": 0.4615049362182617, "mix_high": 9.5367431640625e-06, "read_cosine": 0.4602412283420563, "beta_mean": 0.43596482276916504, "nonfinite": 0.0, "weighted_memory_to_active": 0.1914909929037094, "mix_std": 0.050671570003032684, "beta_std": 0.22978238761425018, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005157209932804108, "evidence_weights": [-0.026295384392142296, -0.009726564399898052, 0.04597371444106102], "agreement_weight": -0.01880025863647461, "orthogonal_weight": 0.037319958209991455}, {"head": 2, "memory_to_active": 0.7355421781539917, "contribution_to_active": 0.3654751479625702, "mix_mean": 0.28887152671813965, "mix_low": 0.010186195373535156, "mix_high": 0.0023965835571289062, "read_cosine": 0.8081858158111572, "beta_mean": 0.09943710267543793, "nonfinite": 0.0, "weighted_memory_to_active": 0.4833899140357971, "mix_std": 0.23610487580299377, "beta_std": 0.0498478040099144, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.015048560686409473, "evidence_weights": [-0.20742788910865784, -0.35418713092803955, 0.2133389413356781], "agreement_weight": -0.0036045436281710863, "orthogonal_weight": 0.06345336884260178}, {"head": 3, "memory_to_active": 3.0098514556884766, "contribution_to_active": 0.06734864413738251, "mix_mean": 0.009387676604092121, "mix_low": 0.7897434234619141, "mix_high": 0.0, "read_cosine": 0.1964893639087677, "beta_mean": 0.4398609697818756, "nonfinite": 0.0, "weighted_memory_to_active": 0.07957780361175537, "mix_std": 0.02660186216235161, "beta_std": 0.2455030083656311, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.013791585341095924, "evidence_weights": [0.03365766257047653, 0.020230645313858986, 0.026529347524046898], "agreement_weight": -0.02186744101345539, "orthogonal_weight": -0.18082648515701294}, {"head": 4, "memory_to_active": 3.055284261703491, "contribution_to_active": 1.293045163154602, "mix_mean": 0.03857427462935448, "mix_low": 0.2887153625488281, "mix_high": 8.58306884765625e-06, "read_cosine": 0.405041366815567, "beta_mean": 0.20040813088417053, "nonfinite": 0.0, "weighted_memory_to_active": 1.1546854972839355, "mix_std": 0.07515250891447067, "beta_std": 0.08685806393623352, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14014950394630432, "evidence_weights": [-0.027691664174199104, -0.021708834916353226, 0.12311592698097229], "agreement_weight": -0.03619913384318352, "orthogonal_weight": 0.13391244411468506}, {"head": 5, "memory_to_active": 3.966749906539917, "contribution_to_active": 0.07188598811626434, "mix_mean": 0.009140582755208015, "mix_low": 0.7878332138061523, "mix_high": 4.76837158203125e-06, "read_cosine": 0.1562076061964035, "beta_mean": 0.31558531522750854, "nonfinite": 0.0, "weighted_memory_to_active": 0.06814062595367432, "mix_std": 0.027222156524658203, "beta_std": 0.22839069366455078, "beta_low": 0.0203857421875, "beta_high": 0.0, "orthogonal_to_active": 0.0011403013486415148, "evidence_weights": [0.10738345235586166, 0.05430949479341507, 0.248214989900589], "agreement_weight": -0.1450594812631607, "orthogonal_weight": 0.017088469117879868}]}, {"layer": 2, "memory_to_active": 1.1668411493301392, "contribution_to_active": 0.28530392050743103, "mix_mean": 0.21784135699272156, "mix_low": 0.2784225344657898, "mix_high": 0.0015962918987497687, "read_cosine": 0.3381827771663666, "beta_mean": 0.11922676861286163, "nonfinite": 0.0, "weighted_memory_to_active": 0.08218525350093842, "mix_std": 0.18007715046405792, "beta_std": 0.07451514899730682, "beta_low": 0.5480855703353882, "beta_high": 7.120767986634746e-05, "orthogonal_to_active": 0.014322136528789997, "output_influence": 0.23764045536518097, "residual_influence": 0.053382016718387604, "state_active": 0.04096253961324692, "state_memory": 0.0278993621468544, "state_peak": 45.4571418762207, "growth_active": 1.0313854217529297, "growth_memory": 1.498992919921875, "heads": [{"head": 0, "memory_to_active": 0.035288937389850616, "contribution_to_active": 0.11947901546955109, "mix_mean": 0.21753893792629242, "mix_low": 0.05932903289794922, "mix_high": 0.00020503997802734375, "read_cosine": 0.21870964765548706, "beta_mean": 0.0002503461146261543, "nonfinite": 0.0, "weighted_memory_to_active": 0.00932567659765482, "mix_std": 0.2374272346496582, "beta_std": 0.00042509130435064435, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000338874408043921, "evidence_weights": [-0.12970632314682007, -0.08897361159324646, 0.17434239387512207], "agreement_weight": 0.011511304415762424, "orthogonal_weight": -0.04229754954576492}, {"head": 1, "memory_to_active": 3.350637197494507, "contribution_to_active": 0.0776825100183487, "mix_mean": 0.005386377684772015, "mix_low": 0.878657341003418, "mix_high": 1.9073486328125e-06, "read_cosine": 0.16543257236480713, "beta_mean": 0.39998263120651245, "nonfinite": 0.0, "weighted_memory_to_active": 0.07089591771364212, "mix_std": 0.01239435188472271, "beta_std": 0.16365385055541992, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016431303694844246, "evidence_weights": [0.09518226981163025, 0.1033327728509903, 0.1639004498720169], "agreement_weight": 0.04230867698788643, "orthogonal_weight": 0.278154194355011}, {"head": 2, "memory_to_active": 0.041124191135168076, "contribution_to_active": 0.4050077497959137, "mix_mean": 0.34974533319473267, "mix_low": 0.005927085876464844, "mix_high": 0.0010881423950195312, "read_cosine": 0.32081139087677, "beta_mean": 6.98463263688609e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015526849776506424, "mix_std": 0.26853933930397034, "beta_std": 0.000160347786732018, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002475101500749588, "evidence_weights": [-0.18015973269939423, -0.2056620717048645, 0.3733104169368744], "agreement_weight": 0.0331156887114048, "orthogonal_weight": -0.20056743919849396}, {"head": 3, "memory_to_active": 0.28375694155693054, "contribution_to_active": 0.3973187804222107, "mix_mean": 0.249986469745636, "mix_low": 0.009984970092773438, "mix_high": 0.0047512054443359375, "read_cosine": 0.6638780236244202, "beta_mean": 0.0399172306060791, "nonfinite": 0.0, "weighted_memory_to_active": 0.11538399755954742, "mix_std": 0.2349529266357422, "beta_std": 0.05826132744550705, "beta_low": 0.26318359375, "beta_high": 0.0, "orthogonal_to_active": 0.05483699589967728, "evidence_weights": [-0.10518057644367218, -0.16620701551437378, 0.5354160070419312], "agreement_weight": 0.0696699470281601, "orthogonal_weight": -0.706916332244873}, {"head": 4, "memory_to_active": 3.2710671424865723, "contribution_to_active": 0.22604283690452576, "mix_mean": 0.012713504955172539, "mix_low": 0.7138404846191406, "mix_high": 2.193450927734375e-05, "read_cosine": 0.17636138200759888, "beta_mean": 0.275098979473114, "nonfinite": 0.0, "weighted_memory_to_active": 0.27154603600502014, "mix_std": 0.04173796623945236, "beta_std": 0.22451916337013245, "beta_low": 0.02532958984375, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.011200438253581524, "evidence_weights": [0.045090414583683014, 0.038449984043836594, 0.15061502158641815], "agreement_weight": 0.03041091375052929, "orthogonal_weight": -0.04977007955312729}, {"head": 5, "memory_to_active": 0.01917256787419319, "contribution_to_active": 0.48629260063171387, "mix_mean": 0.4716774523258209, "mix_low": 0.002796173095703125, "mix_high": 0.003509521484375, "read_cosine": 0.4839034676551819, "beta_mean": 4.1527913708705455e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01043301448225975, "mix_std": 0.2854110598564148, "beta_std": 7.109613943612203e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006501038442365825, "evidence_weights": [-0.23545542359352112, -0.23670607805252075, 0.5958220362663269], "agreement_weight": 0.025444701313972473, "orthogonal_weight": 0.11500413715839386}]}, {"layer": 3, "memory_to_active": 1.5251197814941406, "contribution_to_active": 0.331135630607605, "mix_mean": 0.1653481423854828, "mix_low": 0.281202495098114, "mix_high": 0.002028306480497122, "read_cosine": 0.2865889370441437, "beta_mean": 0.11860422790050507, "nonfinite": 0.0, "weighted_memory_to_active": 0.1664365530014038, "mix_std": 0.18782976269721985, "beta_std": 0.08507779240608215, "beta_low": 0.4025065302848816, "beta_high": 0.0, "orthogonal_to_active": 0.029175974428653717, "output_influence": 0.22577887773513794, "residual_influence": 0.04022683948278427, "state_active": 0.036062367260456085, "state_memory": 0.026067152619361877, "state_peak": 29.419925689697266, "growth_active": 1.0822408199310303, "growth_memory": 1.7422850131988525, "heads": [{"head": 0, "memory_to_active": 0.860109269618988, "contribution_to_active": 0.2833089828491211, "mix_mean": 0.1472722887992859, "mix_low": 0.22058582305908203, "mix_high": 0.00033473968505859375, "read_cosine": 0.2011469006538391, "beta_mean": 0.014990883879363537, "nonfinite": 0.0, "weighted_memory_to_active": 0.1797463595867157, "mix_std": 0.22294467687606812, "beta_std": 0.011555423960089684, "beta_low": 0.39813232421875, "beta_high": 0.0, "orthogonal_to_active": 0.044184502214193344, "evidence_weights": [-0.02232930064201355, -0.09384512901306152, -0.07584445923566818], "agreement_weight": 0.022095439955592155, "orthogonal_weight": -0.2520415782928467}, {"head": 1, "memory_to_active": 0.08877940475940704, "contribution_to_active": 0.2828456163406372, "mix_mean": 0.22304292023181915, "mix_low": 0.09592151641845703, "mix_high": 0.000980377197265625, "read_cosine": 0.2330903857946396, "beta_mean": 0.00037277047522366047, "nonfinite": 0.0, "weighted_memory_to_active": 0.016407903283834457, "mix_std": 0.2562314569950104, "beta_std": 0.0007723374292254448, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.000823012727778405, "evidence_weights": [-0.12497261166572571, -0.0964813083410263, 0.2994818389415741], "agreement_weight": 0.018992403522133827, "orthogonal_weight": -0.052728842943906784}, {"head": 2, "memory_to_active": 3.770885467529297, "contribution_to_active": 0.335182249546051, "mix_mean": 0.039138443768024445, "mix_low": 0.39778900146484375, "mix_high": 0.000152587890625, "read_cosine": 0.19973544776439667, "beta_mean": 0.25267452001571655, "nonfinite": 0.0, "weighted_memory_to_active": 0.4493192434310913, "mix_std": 0.09645648300647736, "beta_std": 0.15303589403629303, "beta_low": 0.00341796875, "beta_high": 0.0, "orthogonal_to_active": 0.04144880548119545, "evidence_weights": [-0.03784002736210823, 0.01199669111520052, 0.08749058097600937], "agreement_weight": -0.01932857558131218, "orthogonal_weight": -0.12024489790201187}, {"head": 3, "memory_to_active": 0.08287347853183746, "contribution_to_active": 0.4502195417881012, "mix_mean": 0.41777974367141724, "mix_low": 0.032660484313964844, "mix_high": 0.007033348083496094, "read_cosine": 0.3027302622795105, "beta_mean": 0.00026174282538704574, "nonfinite": 0.0, "weighted_memory_to_active": 0.04646800830960274, "mix_std": 0.3144467771053314, "beta_std": 0.0007403089548461139, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.01095274742692709, "evidence_weights": [-0.17743375897407532, -0.19784283638000488, 0.9375021457672119], "agreement_weight": 0.5283224582672119, "orthogonal_weight": -0.329708069562912}, {"head": 4, "memory_to_active": 3.8859288692474365, "contribution_to_active": 0.039033595472574234, "mix_mean": 0.007127688731998205, "mix_low": 0.8390026092529297, "mix_high": 0.0, "read_cosine": 0.20179277658462524, "beta_mean": 0.2898287773132324, "nonfinite": 0.0, "weighted_memory_to_active": 0.053124263882637024, "mix_std": 0.020206525921821594, "beta_std": 0.19127126038074493, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.015612482093274593, "evidence_weights": [0.15219758450984955, 0.12966398894786835, -0.12778936326503754], "agreement_weight": -0.061218954622745514, "orthogonal_weight": -0.3032611310482025}, {"head": 5, "memory_to_active": 0.46214181184768677, "contribution_to_active": 0.596223771572113, "mix_mean": 0.15772780776023865, "mix_low": 0.10125541687011719, "mix_high": 0.0036687850952148438, "read_cosine": 0.5810376405715942, "beta_mean": 0.15349659323692322, "nonfinite": 0.0, "weighted_memory_to_active": 0.2535533607006073, "mix_std": 0.21669264137744904, "beta_std": 0.15309149026870728, "beta_low": 0.014404296875, "beta_high": 0.0, "orthogonal_to_active": 0.062034301459789276, "evidence_weights": [-0.05575202777981758, -0.1381017416715622, 0.1318357139825821], "agreement_weight": 0.16357138752937317, "orthogonal_weight": -0.39613157510757446}]}, {"layer": 4, "memory_to_active": 1.3377389907836914, "contribution_to_active": 0.3178008198738098, "mix_mean": 0.2497768998146057, "mix_low": 0.27846559882164, "mix_high": 0.0013875962467864156, "read_cosine": 0.403643399477005, "beta_mean": 0.1311229020357132, "nonfinite": 0.0, "weighted_memory_to_active": 0.160912424325943, "mix_std": 0.1539021134376526, "beta_std": 0.10472245514392853, "beta_low": 0.2516275942325592, "beta_high": 0.0, "orthogonal_to_active": 0.034625887870788574, "output_influence": 0.22665813565254211, "residual_influence": 0.051145151257514954, "state_active": 0.04692922532558441, "state_memory": 0.03522813692688942, "state_peak": 56.05318832397461, "growth_active": 1.1204779148101807, "growth_memory": 1.6157935857772827, "heads": [{"head": 0, "memory_to_active": 0.414142370223999, "contribution_to_active": 0.5379530191421509, "mix_mean": 0.46281546354293823, "mix_low": 0.006503105163574219, "mix_high": 0.00171661376953125, "read_cosine": 0.5805426836013794, "beta_mean": 0.14173397421836853, "nonfinite": 0.0, "weighted_memory_to_active": 0.2718827724456787, "mix_std": 0.29670053720474243, "beta_std": 0.1686829775571823, "beta_low": 0.14068603515625, "beta_high": 0.0, "orthogonal_to_active": 0.015402372926473618, "evidence_weights": [-0.21067890524864197, -0.25588083267211914, 0.4420238435268402], "agreement_weight": 0.055709995329380035, "orthogonal_weight": 0.07445300370454788}, {"head": 1, "memory_to_active": 0.4442441165447235, "contribution_to_active": 0.32004186511039734, "mix_mean": 0.36299410462379456, "mix_low": 0.005665779113769531, "mix_high": 0.0005617141723632812, "read_cosine": 0.569901704788208, "beta_mean": 0.027936384081840515, "nonfinite": 0.0, "weighted_memory_to_active": 0.18548792600631714, "mix_std": 0.2576008439064026, "beta_std": 0.035326167941093445, "beta_low": 0.296875, "beta_high": 0.0, "orthogonal_to_active": 0.08811604976654053, "evidence_weights": [-0.20228658616542816, -0.18013060092926025, 0.6293179392814636], "agreement_weight": 0.044365037232637405, "orthogonal_weight": -0.5992820262908936}, {"head": 2, "memory_to_active": 2.7768962383270264, "contribution_to_active": 0.11485523730516434, "mix_mean": 0.012387041002511978, "mix_low": 0.6710386276245117, "mix_high": 0.0, "read_cosine": 0.3004169166088104, "beta_mean": 0.16805706918239594, "nonfinite": 0.0, "weighted_memory_to_active": 0.144456148147583, "mix_std": 0.02112702839076519, "beta_std": 0.13313308358192444, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.003771934425458312, "evidence_weights": [0.04036029428243637, 0.021860161796212196, -0.054625529795885086], "agreement_weight": 0.04299495369195938, "orthogonal_weight": -0.03819112852215767}, {"head": 3, "memory_to_active": 1.598146677017212, "contribution_to_active": 0.09966610372066498, "mix_mean": 0.029405079782009125, "mix_low": 0.3703298568725586, "mix_high": 0.0, "read_cosine": 0.22761590778827667, "beta_mean": 0.12075334042310715, "nonfinite": 0.0, "weighted_memory_to_active": 0.1553979516029358, "mix_std": 0.04677094891667366, "beta_std": 0.11704479157924652, "beta_low": 0.07177734375, "beta_high": 0.0, "orthogonal_to_active": 0.020540693774819374, "evidence_weights": [-0.023387901484966278, 0.03498275205492973, -0.14066794514656067], "agreement_weight": 0.024715963751077652, "orthogonal_weight": -0.21747305989265442}, {"head": 4, "memory_to_active": 0.02202039398252964, "contribution_to_active": 0.5820537805557251, "mix_mean": 0.6157474517822266, "mix_low": 0.0010995864868164062, "mix_high": 0.005908012390136719, "read_cosine": 0.3354532718658447, "beta_mean": 4.5185861381469294e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015376253053545952, "mix_std": 0.2600542902946472, "beta_std": 0.00011266693763900548, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001878342474810779, "evidence_weights": [-0.27414122223854065, -0.31303709745407104, 0.7018054127693176], "agreement_weight": 0.03532685711979866, "orthogonal_weight": -0.14869815111160278}, {"head": 5, "memory_to_active": 2.770984172821045, "contribution_to_active": 0.2522348165512085, "mix_mean": 0.015312226489186287, "mix_low": 0.6161565780639648, "mix_high": 0.0001392364501953125, "read_cosine": 0.4079297184944153, "beta_mean": 0.3282114267349243, "nonfinite": 0.0, "weighted_memory_to_active": 0.1928735077381134, "mix_std": 0.041158974170684814, "beta_std": 0.1740349978208542, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07804591953754425, "evidence_weights": [0.03247075527906418, -0.026868820190429688, -0.1076328381896019], "agreement_weight": -0.03394123166799545, "orthogonal_weight": 0.5841408371925354}]}, {"layer": 5, "memory_to_active": 0.7725697159767151, "contribution_to_active": 0.4353516101837158, "mix_mean": 0.36085107922554016, "mix_low": 0.11572154611349106, "mix_high": 0.007811388000845909, "read_cosine": 0.25178980827331543, "beta_mean": 0.06279521435499191, "nonfinite": 0.0, "weighted_memory_to_active": 0.1986202746629715, "mix_std": 0.23943059146404266, "beta_std": 0.07323674112558365, "beta_low": 0.55694580078125, "beta_high": 0.0, "orthogonal_to_active": 0.05252896994352341, "output_influence": 0.3976837992668152, "residual_influence": 0.08465151488780975, "state_active": 0.06177930533885956, "state_memory": 0.02162526734173298, "state_peak": 63.01397705078125, "growth_active": 1.133103370666504, "growth_memory": 1.4018388986587524, "heads": [{"head": 0, "memory_to_active": 1.6605607271194458, "contribution_to_active": 0.26393765211105347, "mix_mean": 0.08997738361358643, "mix_low": 0.18419742584228516, "mix_high": 0.0, "read_cosine": -0.15426544845104218, "beta_mean": 0.10260213911533356, "nonfinite": 0.0, "weighted_memory_to_active": 0.2482982575893402, "mix_std": 0.13252094388008118, "beta_std": 0.11215105652809143, "beta_low": 0.0421142578125, "beta_high": 0.0, "orthogonal_to_active": 0.05122996121644974, "evidence_weights": [-0.028309186920523643, -0.05824229121208191, -0.314175546169281], "agreement_weight": -0.02168419025838375, "orthogonal_weight": -0.2086806744337082}, {"head": 1, "memory_to_active": 0.026751138269901276, "contribution_to_active": 0.4359598159790039, "mix_mean": 0.4698559641838074, "mix_low": 0.014544486999511719, "mix_high": 0.00809478759765625, "read_cosine": 0.15845617651939392, "beta_mean": 7.083261152729392e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011174933053553104, "mix_std": 0.31719595193862915, "beta_std": 0.0001751959789544344, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009554792195558548, "evidence_weights": [-0.2098250836133957, -0.21463480591773987, 0.5485220551490784], "agreement_weight": 0.2149558812379837, "orthogonal_weight": -0.8783303499221802}, {"head": 2, "memory_to_active": 0.1284426897764206, "contribution_to_active": 0.4885788559913635, "mix_mean": 0.5326712131500244, "mix_low": 0.0015573501586914062, "mix_high": 0.007912635803222656, "read_cosine": 0.4619227349758148, "beta_mean": 0.004340190440416336, "nonfinite": 0.0, "weighted_memory_to_active": 0.08501105010509491, "mix_std": 0.2791637182235718, "beta_std": 0.004031550604850054, "beta_low": 0.92034912109375, "beta_high": 0.0, "orthogonal_to_active": 0.010196910239756107, "evidence_weights": [-0.2668830156326294, -0.2827410101890564, 0.6176367998123169], "agreement_weight": 0.02439221553504467, "orthogonal_weight": -0.16885633766651154}, {"head": 3, "memory_to_active": 2.203704595565796, "contribution_to_active": 0.2464708685874939, "mix_mean": 0.05833557993173599, "mix_low": 0.31142330169677734, "mix_high": 0.0, "read_cosine": 0.18357710540294647, "beta_mean": 0.08800676465034485, "nonfinite": 0.0, "weighted_memory_to_active": 0.42470136284828186, "mix_std": 0.11018691956996918, "beta_std": 0.1173074021935463, "beta_low": 0.16607666015625, "beta_high": 0.0, "orthogonal_to_active": 0.13551226258277893, "evidence_weights": [0.0004313834651838988, -0.0433632954955101, 0.0941532626748085], "agreement_weight": 0.05415121093392372, "orthogonal_weight": -0.4428007900714874}, {"head": 4, "memory_to_active": 0.5341752767562866, "contribution_to_active": 0.5780549049377441, "mix_mean": 0.308962881565094, "mix_low": 0.17886924743652344, "mix_high": 0.010341644287109375, "read_cosine": 0.6030237078666687, "beta_mean": 0.1814742535352707, "nonfinite": 0.0, "weighted_memory_to_active": 0.37032216787338257, "mix_std": 0.3312380611896515, "beta_std": 0.20447123050689697, "beta_low": 0.21710205078125, "beta_high": 0.0, "orthogonal_to_active": 0.08416084200143814, "evidence_weights": [-0.1449676752090454, -0.26754260063171387, 0.08746213465929031], "agreement_weight": 0.047649264335632324, "orthogonal_weight": 0.35943612456321716}, {"head": 5, "memory_to_active": 0.08178406953811646, "contribution_to_active": 0.5991073846817017, "mix_mean": 0.705303430557251, "mix_low": 0.0037374496459960938, "mix_high": 0.020519256591796875, "read_cosine": 0.2580244541168213, "beta_mean": 0.00027707789558917284, "nonfinite": 0.0, "weighted_memory_to_active": 0.05221376568078995, "mix_std": 0.266277939081192, "beta_std": 0.0012840060517191887, "beta_low": 0.99603271484375, "beta_high": 0.0, "orthogonal_to_active": 0.02451905980706215, "evidence_weights": [-0.31197404861450195, -0.30067217350006104, 1.1683635711669922], "agreement_weight": 0.3172959089279175, "orthogonal_weight": -0.5121470093727112}]}, {"layer": 6, "memory_to_active": 0.4819711446762085, "contribution_to_active": 0.5358806252479553, "mix_mean": 0.4659818410873413, "mix_low": 0.09111913293600082, "mix_high": 0.019735177978873253, "read_cosine": 0.30127638578414917, "beta_mean": 0.0820881798863411, "nonfinite": 0.0, "weighted_memory_to_active": 0.17455047369003296, "mix_std": 0.26215672492980957, "beta_std": 0.09189540147781372, "beta_low": 0.4990641474723816, "beta_high": 0.0, "orthogonal_to_active": 0.03885253891348839, "output_influence": 0.4031817615032196, "residual_influence": 0.11684135347604752, "state_active": 0.0621219128370285, "state_memory": 0.01372356154024601, "state_peak": 57.87089157104492, "growth_active": 1.2599643468856812, "growth_memory": 1.3552294969558716, "heads": [{"head": 0, "memory_to_active": 0.18244250118732452, "contribution_to_active": 0.4921339452266693, "mix_mean": 0.575790286064148, "mix_low": 0.0008516311645507812, "mix_high": 0.0031499862670898438, "read_cosine": 0.13309592008590698, "beta_mean": 0.016147678717970848, "nonfinite": 0.0, "weighted_memory_to_active": 0.13129624724388123, "mix_std": 0.2672826945781708, "beta_std": 0.04052327573299408, "beta_low": 0.614501953125, "beta_high": 0.0, "orthogonal_to_active": 0.09337454289197922, "evidence_weights": [-0.272325336933136, -0.2651052176952362, 0.3201141953468323], "agreement_weight": 0.09324663132429123, "orthogonal_weight": -0.8235398530960083}, {"head": 1, "memory_to_active": 0.007230093702673912, "contribution_to_active": 0.45799556374549866, "mix_mean": 0.42120227217674255, "mix_low": 0.005125999450683594, "mix_high": 0.010866165161132812, "read_cosine": 0.5319513082504272, "beta_mean": 1.792298280633986e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002616924699395895, "mix_std": 0.2934768795967102, "beta_std": 5.139179847901687e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006395492237061262, "evidence_weights": [-0.21270303428173065, -0.23263436555862427, 0.5487921833992004], "agreement_weight": -0.013266957364976406, "orthogonal_weight": -0.32040438055992126}, {"head": 2, "memory_to_active": 1.7504382133483887, "contribution_to_active": 0.07298173755407333, "mix_mean": 0.02390415035188198, "mix_low": 0.5213136672973633, "mix_high": 0.0, "read_cosine": 0.20821553468704224, "beta_mean": 0.12304183840751648, "nonfinite": 0.0, "weighted_memory_to_active": 0.08916590362787247, "mix_std": 0.050668902695178986, "beta_std": 0.09354663640260696, "beta_low": 0.00177001953125, "beta_high": 0.0, "orthogonal_to_active": 0.019552240148186684, "evidence_weights": [0.0767347440123558, 0.036815349012613297, 0.09737346321344376], "agreement_weight": 0.04540393874049187, "orthogonal_weight": -0.23640213906764984}, {"head": 3, "memory_to_active": 0.16702641546726227, "contribution_to_active": 0.6325937509536743, "mix_mean": 0.6033141613006592, "mix_low": 0.014391899108886719, "mix_high": 0.02917766571044922, "read_cosine": 0.5318487286567688, "beta_mean": 0.22658386826515198, "nonfinite": 0.0, "weighted_memory_to_active": 0.11840729415416718, "mix_std": 0.34821829199790955, "beta_std": 0.26012903451919556, "beta_low": 0.17535400390625, "beta_high": 0.0, "orthogonal_to_active": 0.04380063712596893, "evidence_weights": [-0.16577085852622986, -0.3219406306743622, 0.18234562873840332], "agreement_weight": -0.006663171108812094, "orthogonal_weight": 0.4540175497531891}, {"head": 4, "memory_to_active": 0.7063539028167725, "contribution_to_active": 1.0448925495147705, "mix_mean": 0.6683821082115173, "mix_low": 0.0028972625732421875, "mix_high": 0.07494831085205078, "read_cosine": 0.12072719633579254, "beta_mean": 0.12603846192359924, "nonfinite": 0.0, "weighted_memory_to_active": 0.6604265570640564, "mix_std": 0.3215697109699249, "beta_std": 0.15572302043437958, "beta_low": 0.20733642578125, "beta_high": 0.0, "orthogonal_to_active": 0.03162287548184395, "evidence_weights": [-0.2401728630065918, -0.5188779830932617, -0.02641811966896057], "agreement_weight": -0.016431378200650215, "orthogonal_weight": 0.05874231457710266}, {"head": 5, "memory_to_active": 0.0783357247710228, "contribution_to_active": 0.5146860480308533, "mix_mean": 0.5032980442047119, "mix_low": 0.0021343231201171875, "mix_high": 0.0002689361572265625, "read_cosine": 0.2818196713924408, "beta_mean": 0.0006993025308474898, "nonfinite": 0.0, "weighted_memory_to_active": 0.04538998007774353, "mix_std": 0.29172375798225403, "beta_std": 0.001399058848619461, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.04412538558244705, "evidence_weights": [-0.2105761617422104, -0.25016364455223083, 0.2344769686460495], "agreement_weight": 0.0728439912199974, "orthogonal_weight": -1.0478132963180542}]}, {"layer": 7, "memory_to_active": 2.2149906158447266, "contribution_to_active": 0.2128007858991623, "mix_mean": 0.029126325622200966, "mix_low": 0.5530874729156494, "mix_high": 1.1444091796875e-05, "read_cosine": 0.22212719917297363, "beta_mean": 0.1371767818927765, "nonfinite": 0.0, "weighted_memory_to_active": 0.22628113627433777, "mix_std": 0.055907826870679855, "beta_std": 0.10933233797550201, "beta_low": 0.01991780661046505, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.035028621554374695, "output_influence": 0.19875945150852203, "residual_influence": 0.042698293924331665, "state_active": 0.0187324658036232, "state_memory": 0.03502018004655838, "state_peak": 33.129150390625, "growth_active": 1.0277317762374878, "growth_memory": 1.446845293045044, "heads": [{"head": 0, "memory_to_active": 2.3887641429901123, "contribution_to_active": 0.5988854169845581, "mix_mean": 0.026508096605539322, "mix_low": 0.5317192077636719, "mix_high": 6.771087646484375e-05, "read_cosine": 0.37138205766677856, "beta_mean": 0.1613023281097412, "nonfinite": 0.0, "weighted_memory_to_active": 0.578930139541626, "mix_std": 0.05681139975786209, "beta_std": 0.12824495136737823, "beta_low": 0.0064697265625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.07538138329982758, "evidence_weights": [-0.006694138050079346, -0.07607638835906982, 0.09202354401350021], "agreement_weight": -0.04244789853692055, "orthogonal_weight": 0.2307354360818863}, {"head": 1, "memory_to_active": 2.6731958389282227, "contribution_to_active": 0.2718762159347534, "mix_mean": 0.060660600662231445, "mix_low": 0.33657073974609375, "mix_high": 0.0, "read_cosine": 0.057559337466955185, "beta_mean": 0.19842000305652618, "nonfinite": 0.0, "weighted_memory_to_active": 0.3594023585319519, "mix_std": 0.09342335164546967, "beta_std": 0.11319650709629059, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07936292141675949, "evidence_weights": [-0.10421756654977798, -0.08687718212604523, -0.0626157894730568], "agreement_weight": 0.01739501953125, "orthogonal_weight": -0.2269786298274994}, {"head": 2, "memory_to_active": 1.3573486804962158, "contribution_to_active": 0.2470303326845169, "mix_mean": 0.052049122750759125, "mix_low": 0.33557796478271484, "mix_high": 9.5367431640625e-07, "read_cosine": 0.14962342381477356, "beta_mean": 0.13071218132972717, "nonfinite": 0.0, "weighted_memory_to_active": 0.2185828983783722, "mix_std": 0.10426343977451324, "beta_std": 0.08328905701637268, "beta_low": 0.00830078125, "beta_high": 0.0, "orthogonal_to_active": 0.010413836687803268, "evidence_weights": [-0.03579635173082352, -0.10793804377317429, 0.19960735738277435], "agreement_weight": 0.02453942596912384, "orthogonal_weight": -0.05032985657453537}, {"head": 3, "memory_to_active": 2.9676032066345215, "contribution_to_active": 0.04367285594344139, "mix_mean": 0.011832760646939278, "mix_low": 0.6851806640625, "mix_high": 0.0, "read_cosine": 0.12036388367414474, "beta_mean": 0.12316235899925232, "nonfinite": 0.0, "weighted_memory_to_active": 0.07645811140537262, "mix_std": 0.030349303036928177, "beta_std": 0.12980400025844574, "beta_low": 0.04742431640625, "beta_high": 0.0, "orthogonal_to_active": 0.03479805961251259, "evidence_weights": [0.11027389019727707, 0.08254565298557281, 0.19911810755729675], "agreement_weight": 0.07035790383815765, "orthogonal_weight": -0.4669453501701355}, {"head": 4, "memory_to_active": 1.586588978767395, "contribution_to_active": 0.05519772320985794, "mix_mean": 0.012327988632023335, "mix_low": 0.7200069427490234, "mix_high": 0.0, "read_cosine": 0.3263136148452759, "beta_mean": 0.12830998003482819, "nonfinite": 0.0, "weighted_memory_to_active": 0.053345173597335815, "mix_std": 0.028822384774684906, "beta_std": 0.12067317217588425, "beta_low": 0.02850341796875, "beta_high": 0.0, "orthogonal_to_active": 0.0023614540696144104, "evidence_weights": [0.07079443335533142, 0.03249691426753998, 0.018303213641047478], "agreement_weight": 0.05209814012050629, "orthogonal_weight": -0.05374116823077202}, {"head": 5, "memory_to_active": 2.3164427280426025, "contribution_to_active": 0.06014218553900719, "mix_mean": 0.011379377916455269, "mix_low": 0.7094688415527344, "mix_high": 0.0, "read_cosine": 0.3075207769870758, "beta_mean": 0.08115377277135849, "nonfinite": 0.0, "weighted_memory_to_active": 0.07096792757511139, "mix_std": 0.021777069196105003, "beta_std": 0.08078628778457642, "beta_low": 0.02880859375, "beta_high": 0.0, "orthogonal_to_active": 0.00785406120121479, "evidence_weights": [0.0717446580529213, 0.08582508563995361, -0.09271620959043503], "agreement_weight": -0.009686174802482128, "orthogonal_weight": -0.12308423221111298}]}, {"layer": 8, "memory_to_active": 0.7107303142547607, "contribution_to_active": 0.40691107511520386, "mix_mean": 0.2968369722366333, "mix_low": 0.0829528197646141, "mix_high": 0.0010007222881540656, "read_cosine": 0.521460771560669, "beta_mean": 0.04776624217629433, "nonfinite": 0.0, "weighted_memory_to_active": 0.16172900795936584, "mix_std": 0.21325582265853882, "beta_std": 0.04860260337591171, "beta_low": 0.3676961362361908, "beta_high": 0.0, "orthogonal_to_active": 0.046620581299066544, "output_influence": 0.29346662759780884, "residual_influence": 0.07396382093429565, "state_active": 0.05557049438357353, "state_memory": 0.01869628205895424, "state_peak": 32.603355407714844, "growth_active": 1.0170561075210571, "growth_memory": 1.41396164894104, "heads": [{"head": 0, "memory_to_active": 0.07370620220899582, "contribution_to_active": 0.49119511246681213, "mix_mean": 0.5179463028907776, "mix_low": 0.00022220611572265625, "mix_high": 0.00017547607421875, "read_cosine": 0.7418187856674194, "beta_mean": 0.0041537354700267315, "nonfinite": 0.0, "weighted_memory_to_active": 0.03966815397143364, "mix_std": 0.23573800921440125, "beta_std": 0.008045812137424946, "beta_low": 0.89910888671875, "beta_high": 0.0, "orthogonal_to_active": 0.018298860639333725, "evidence_weights": [-0.23890933394432068, -0.32136479020118713, 0.6466737985610962], "agreement_weight": 0.07027515769004822, "orthogonal_weight": -0.7315878868103027}, {"head": 1, "memory_to_active": 0.6237790584564209, "contribution_to_active": 0.3178348243236542, "mix_mean": 0.29890793561935425, "mix_low": 0.008334159851074219, "mix_high": 0.000335693359375, "read_cosine": 0.516477108001709, "beta_mean": 0.04874388128519058, "nonfinite": 0.0, "weighted_memory_to_active": 0.21904395520687103, "mix_std": 0.2162737101316452, "beta_std": 0.03878965228796005, "beta_low": 0.06878662109375, "beta_high": 0.0, "orthogonal_to_active": 0.01637342758476734, "evidence_weights": [-0.16052274405956268, -0.17738986015319824, 0.3306260406970978], "agreement_weight": 0.051991574466228485, "orthogonal_weight": -0.0920838862657547}, {"head": 2, "memory_to_active": 0.23594346642494202, "contribution_to_active": 0.3942423462867737, "mix_mean": 0.35432228446006775, "mix_low": 0.006499290466308594, "mix_high": 0.000202178955078125, "read_cosine": 0.36177581548690796, "beta_mean": 0.005897062830626965, "nonfinite": 0.0, "weighted_memory_to_active": 0.10113844275474548, "mix_std": 0.2504501938819885, "beta_std": 0.008102020248770714, "beta_low": 0.8377685546875, "beta_high": 0.0, "orthogonal_to_active": 0.06820008903741837, "evidence_weights": [-0.16239391267299652, -0.2505286931991577, 0.34644797444343567], "agreement_weight": 0.04004065319895744, "orthogonal_weight": -0.7497323155403137}, {"head": 3, "memory_to_active": 0.3930306136608124, "contribution_to_active": 0.2691700756549835, "mix_mean": 0.23971089720726013, "mix_low": 0.01854228973388672, "mix_high": 3.814697265625e-06, "read_cosine": 0.6980181932449341, "beta_mean": 0.0275768730789423, "nonfinite": 0.0, "weighted_memory_to_active": 0.15611909329891205, "mix_std": 0.19428974390029907, "beta_std": 0.028335312381386757, "beta_low": 0.16265869140625, "beta_high": 0.0, "orthogonal_to_active": 0.00969101581722498, "evidence_weights": [-0.07602240890264511, -0.1715977042913437, 0.6319268941879272], "agreement_weight": 0.07054922729730606, "orthogonal_weight": 0.10547754168510437}, {"head": 4, "memory_to_active": 2.6923961639404297, "contribution_to_active": 0.29202279448509216, "mix_mean": 0.028393233194947243, "mix_low": 0.3780221939086914, "mix_high": 0.0, "read_cosine": 0.28265735507011414, "beta_mean": 0.11602986603975296, "nonfinite": 0.0, "weighted_memory_to_active": 0.2805533707141876, "mix_std": 0.04525832086801529, "beta_std": 0.08879347145557404, "beta_low": 0.00311279296875, "beta_high": 0.0, "orthogonal_to_active": 0.015686597675085068, "evidence_weights": [0.020460711792111397, -0.005843176972121, -0.06851224601268768], "agreement_weight": 0.019125210121273994, "orthogonal_weight": 0.05899226292967796}, {"head": 5, "memory_to_active": 0.24552619457244873, "contribution_to_active": 0.6770012378692627, "mix_mean": 0.34174102544784546, "mix_low": 0.08609676361083984, "mix_high": 0.00528717041015625, "read_cosine": 0.5280170440673828, "beta_mean": 0.0841960459947586, "nonfinite": 0.0, "weighted_memory_to_active": 0.1738509237766266, "mix_std": 0.3375248908996582, "beta_std": 0.11954934149980545, "beta_low": 0.2347412109375, "beta_high": 0.0, "orthogonal_to_active": 0.1514734923839569, "evidence_weights": [-0.07527182996273041, -0.34073132276535034, -0.38956180214881897], "agreement_weight": 0.025630725547671318, "orthogonal_weight": 1.0720107555389404}]}, {"layer": 9, "memory_to_active": 1.2582297325134277, "contribution_to_active": 0.6010467410087585, "mix_mean": 0.2033657282590866, "mix_low": 0.1982024610042572, "mix_high": 0.0010949770221486688, "read_cosine": 0.32970938086509705, "beta_mean": 0.10250312089920044, "nonfinite": 0.0, "weighted_memory_to_active": 0.3983226716518402, "mix_std": 0.20760996639728546, "beta_std": 0.05480159819126129, "beta_low": 0.4942830502986908, "beta_high": 0.0, "orthogonal_to_active": 0.02429145574569702, "output_influence": 0.3550291359424591, "residual_influence": 0.08500970900058746, "state_active": 0.04118267446756363, "state_memory": 0.028800368309020996, "state_peak": 30.18904685974121, "growth_active": 1.2075129747390747, "growth_memory": 1.4366155862808228, "heads": [{"head": 0, "memory_to_active": 0.007903169840574265, "contribution_to_active": 0.41377586126327515, "mix_mean": 0.4284290671348572, "mix_low": 0.007898330688476562, "mix_high": 0.0034847259521484375, "read_cosine": 0.20820045471191406, "beta_mean": 1.7419548385078087e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003710776334628463, "mix_std": 0.29619163274765015, "beta_std": 6.343881250359118e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002234409563243389, "evidence_weights": [-0.2388053685426712, -0.22658753395080566, 0.9590663313865662], "agreement_weight": 0.05390413478016853, "orthogonal_weight": -0.6254291534423828}, {"head": 1, "memory_to_active": 0.05550651624798775, "contribution_to_active": 0.41471439599990845, "mix_mean": 0.2607390284538269, "mix_low": 0.043674468994140625, "mix_high": 0.0005159378051757812, "read_cosine": 0.4001339077949524, "beta_mean": 0.0001698642736300826, "nonfinite": 0.0, "weighted_memory_to_active": 0.021560726687312126, "mix_std": 0.25933918356895447, "beta_std": 0.0003086005453951657, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008340469561517239, "evidence_weights": [-0.17031052708625793, -0.16910265386104584, 0.6267691850662231], "agreement_weight": 0.2821776270866394, "orthogonal_weight": -0.4371800422668457}, {"head": 2, "memory_to_active": 0.30067524313926697, "contribution_to_active": 0.4541444480419159, "mix_mean": 0.31850385665893555, "mix_low": 0.02466869354248047, "mix_high": 0.0016145706176757812, "read_cosine": 0.38678663969039917, "beta_mean": 0.002327706664800644, "nonfinite": 0.0, "weighted_memory_to_active": 0.14466284215450287, "mix_std": 0.28010058403015137, "beta_std": 0.00607873871922493, "beta_low": 0.96514892578125, "beta_high": 0.0, "orthogonal_to_active": 0.02973530814051628, "evidence_weights": [-0.1307191550731659, -0.13611190021038055, 0.27050521969795227], "agreement_weight": 0.14219246804714203, "orthogonal_weight": -0.25337591767311096}, {"head": 3, "memory_to_active": 2.0580556392669678, "contribution_to_active": 0.19411373138427734, "mix_mean": 0.03714713454246521, "mix_low": 0.4199857711791992, "mix_high": 0.0, "read_cosine": 0.4663585126399994, "beta_mean": 0.139711394906044, "nonfinite": 0.0, "weighted_memory_to_active": 0.16796453297138214, "mix_std": 0.08800946176052094, "beta_std": 0.085821732878685, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.022696353495121002, "evidence_weights": [0.04225761443376541, 0.03216766193509102, -0.12768517434597015], "agreement_weight": 0.03209298476576805, "orthogonal_weight": 0.13762469589710236}, {"head": 4, "memory_to_active": 3.167301654815674, "contribution_to_active": 1.5036687850952148, "mix_mean": 0.12196868658065796, "mix_low": 0.2271108627319336, "mix_high": 0.0008974075317382812, "read_cosine": 0.13492757081985474, "beta_mean": 0.2410230189561844, "nonfinite": 0.0, "weighted_memory_to_active": 1.3521242141723633, "mix_std": 0.20198485255241394, "beta_std": 0.12034648656845093, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015779463574290276, "evidence_weights": [-0.04387977719306946, -0.11384432762861252, 0.06943324953317642], "agreement_weight": -0.04712720587849617, "orthogonal_weight": -0.0011997406836599112}, {"head": 5, "memory_to_active": 1.959936261177063, "contribution_to_active": 0.6258631944656372, "mix_mean": 0.05340656638145447, "mix_low": 0.46587657928466797, "mix_high": 5.7220458984375e-05, "read_cosine": 0.3818491995334625, "beta_mean": 0.23176929354667664, "nonfinite": 0.0, "weighted_memory_to_active": 0.6999130249023438, "mix_std": 0.12003408372402191, "beta_std": 0.1161905974149704, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08116423338651657, "evidence_weights": [0.038209788501262665, -0.03691400587558746, 0.06122574955224991], "agreement_weight": -0.00474023912101984, "orthogonal_weight": 0.15167614817619324}]}, {"layer": 10, "memory_to_active": 0.20905616879463196, "contribution_to_active": 0.4027640223503113, "mix_mean": 0.35933470726013184, "mix_low": 0.028896015137434006, "mix_high": 0.0060321493074297905, "read_cosine": 0.39848512411117554, "beta_mean": 0.04374252259731293, "nonfinite": 0.0, "weighted_memory_to_active": 0.09290759265422821, "mix_std": 0.2561320662498474, "beta_std": 0.050666213035583496, "beta_low": 0.6172384023666382, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.03660785034298897, "output_influence": 0.37309178709983826, "residual_influence": 0.10974365472793579, "state_active": 0.06587976217269897, "state_memory": 0.015561990439891815, "state_peak": 91.08601379394531, "growth_active": 1.1128302812576294, "growth_memory": 1.1176389455795288, "heads": [{"head": 0, "memory_to_active": 0.40441808104515076, "contribution_to_active": 0.7611488699913025, "mix_mean": 0.6112858653068542, "mix_low": 0.015432357788085938, "mix_high": 0.034793853759765625, "read_cosine": 0.23170599341392517, "beta_mean": 0.184958353638649, "nonfinite": 0.0, "weighted_memory_to_active": 0.29125869274139404, "mix_std": 0.34860938787460327, "beta_std": 0.20990850031375885, "beta_low": 0.1248779296875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.1252826750278473, "evidence_weights": [-0.1655571311712265, -0.33956173062324524, -0.22551462054252625], "agreement_weight": 0.023039134219288826, "orthogonal_weight": 0.48007485270500183}, {"head": 1, "memory_to_active": 0.002913249656558037, "contribution_to_active": 0.4309658408164978, "mix_mean": 0.4084319472312927, "mix_low": 0.006191253662109375, "mix_high": 0.0010013580322265625, "read_cosine": 0.23175938427448273, "beta_mean": 1.7526577721582726e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0012095242273062468, "mix_std": 0.2613319456577301, "beta_std": 3.971042679040693e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007613649358972907, "evidence_weights": [-0.23846857249736786, -0.17176638543605804, 0.21279336512088776], "agreement_weight": 0.08826156705617905, "orthogonal_weight": -0.6769006252288818}, {"head": 2, "memory_to_active": 0.4405694603919983, "contribution_to_active": 0.23432940244674683, "mix_mean": 0.16750627756118774, "mix_low": 0.1078500747680664, "mix_high": 1.52587890625e-05, "read_cosine": 0.15233761072158813, "beta_mean": 0.009839746169745922, "nonfinite": 0.0, "weighted_memory_to_active": 0.13324135541915894, "mix_std": 0.20915763080120087, "beta_std": 0.02109498530626297, "beta_low": 0.74462890625, "beta_high": 0.0, "orthogonal_to_active": 0.029271278530359268, "evidence_weights": [-0.04424477741122246, -0.12229721993207932, 0.21994052827358246], "agreement_weight": 0.030111858621239662, "orthogonal_weight": -0.2262541651725769}, {"head": 3, "memory_to_active": 0.2558740973472595, "contribution_to_active": 0.15889467298984528, "mix_mean": 0.2740391194820404, "mix_low": 0.03372478485107422, "mix_high": 0.00034332275390625, "read_cosine": 0.8584392070770264, "beta_mean": 0.06105724722146988, "nonfinite": 0.0, "weighted_memory_to_active": 0.06072738394141197, "mix_std": 0.24853789806365967, "beta_std": 0.06199729070067406, "beta_low": 0.01019287109375, "beta_high": 0.0, "orthogonal_to_active": 0.04673339053988457, "evidence_weights": [-0.1230611801147461, -0.14606446027755737, 0.23651044070720673], "agreement_weight": 0.004048376809805632, "orthogonal_weight": 1.091648817062378}, {"head": 4, "memory_to_active": 0.026467155665159225, "contribution_to_active": 0.39187514781951904, "mix_mean": 0.3624460697174072, "mix_low": 0.0038909912109375, "mix_high": 2.002716064453125e-05, "read_cosine": 0.09755135327577591, "beta_mean": 8.225331839639693e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00861862301826477, "mix_std": 0.24795721471309662, "beta_std": 0.0014564123703166842, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.002094740979373455, "evidence_weights": [-0.17508624494075775, -0.1698400378227234, 0.20200228691101074], "agreement_weight": 0.011524725705385208, "orthogonal_weight": -0.2533927857875824}, {"head": 5, "memory_to_active": 0.1240948960185051, "contribution_to_active": 0.43937015533447266, "mix_mean": 0.3322989344596863, "mix_low": 0.00628662109375, "mix_high": 1.9073486328125e-05, "read_cosine": 0.81911700963974, "beta_mean": 0.006500003859400749, "nonfinite": 0.0, "weighted_memory_to_active": 0.062389954924583435, "mix_std": 0.22119829058647156, "beta_std": 0.009500384330749512, "beta_low": 0.8238525390625, "beta_high": 0.0, "orthogonal_to_active": 0.015503646805882454, "evidence_weights": [-0.18829454481601715, -0.16965489089488983, 0.25105929374694824], "agreement_weight": 0.08816615492105484, "orthogonal_weight": -0.5438472628593445}]}, {"layer": 11, "memory_to_active": 0.27136141061782837, "contribution_to_active": 0.44340020418167114, "mix_mean": 0.36397796869277954, "mix_low": 0.052045345306396484, "mix_high": 0.015180589631199837, "read_cosine": 0.5172146558761597, "beta_mean": 0.038368191570043564, "nonfinite": 0.0, "weighted_memory_to_active": 0.09136562794446945, "mix_std": 0.29728952050209045, "beta_std": 0.0544697605073452, "beta_low": 0.495758056640625, "beta_high": 0.0, "orthogonal_to_active": 0.03794140741229057, "output_influence": 0.45721447467803955, "residual_influence": 0.11906564235687256, "state_active": 0.11551493406295776, "state_memory": 0.012959066778421402, "state_peak": 32.501861572265625, "growth_active": 1.181420922279358, "growth_memory": 1.3996142148971558, "heads": [{"head": 0, "memory_to_active": 0.03216275945305824, "contribution_to_active": 0.5248088836669922, "mix_mean": 0.5000135898590088, "mix_low": 0.003383636474609375, "mix_high": 0.0039520263671875, "read_cosine": 0.41896140575408936, "beta_mean": 0.00017576667596586049, "nonfinite": 0.0, "weighted_memory_to_active": 0.01990441232919693, "mix_std": 0.28087639808654785, "beta_std": 0.0036478627007454634, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.010183146223425865, "evidence_weights": [-0.16165286302566528, -0.2965400516986847, 0.48765164613723755], "agreement_weight": 0.12571221590042114, "orthogonal_weight": -0.6412088871002197}, {"head": 1, "memory_to_active": 0.542042076587677, "contribution_to_active": 0.22067669034004211, "mix_mean": 0.2291603684425354, "mix_low": 0.038023948669433594, "mix_high": 4.863739013671875e-05, "read_cosine": 0.5828040838241577, "beta_mean": 0.03955743834376335, "nonfinite": 0.0, "weighted_memory_to_active": 0.17389781773090363, "mix_std": 0.22856059670448303, "beta_std": 0.06963333487510681, "beta_low": 0.36407470703125, "beta_high": 0.0, "orthogonal_to_active": 0.021252280101180077, "evidence_weights": [-0.15256249904632568, -0.10510536283254623, 0.12964990735054016], "agreement_weight": 0.08294510841369629, "orthogonal_weight": -0.16384142637252808}, {"head": 2, "memory_to_active": 0.061985280364751816, "contribution_to_active": 0.8447863459587097, "mix_mean": 0.467252254486084, "mix_low": 0.053490638732910156, "mix_high": 0.05189037322998047, "read_cosine": 0.34563225507736206, "beta_mean": 0.07314714789390564, "nonfinite": 0.0, "weighted_memory_to_active": 0.05168720334768295, "mix_std": 0.4059189260005951, "beta_std": 0.0989752784371376, "beta_low": 0.4100341796875, "beta_high": 0.0, "orthogonal_to_active": 0.09740975499153137, "evidence_weights": [-0.1500493586063385, -0.4103211462497711, -0.6420700550079346], "agreement_weight": -0.0018801153637468815, "orthogonal_weight": 2.0642788410186768}, {"head": 3, "memory_to_active": 0.03186698257923126, "contribution_to_active": 0.4168086647987366, "mix_mean": 0.4298292398452759, "mix_low": 0.12120819091796875, "mix_high": 0.034542083740234375, "read_cosine": 0.5713083744049072, "beta_mean": 0.0650428906083107, "nonfinite": 0.0, "weighted_memory_to_active": 0.01714111678302288, "mix_std": 0.41599589586257935, "beta_std": 0.092710480093956, "beta_low": 0.42767333984375, "beta_high": 0.0, "orthogonal_to_active": 0.030665302649140358, "evidence_weights": [-0.16759273409843445, -0.3654591143131256, -0.19886279106140137], "agreement_weight": 0.019407549872994423, "orthogonal_weight": 2.227269411087036}, {"head": 4, "memory_to_active": 0.780759871006012, "contribution_to_active": 0.2580791711807251, "mix_mean": 0.1619180142879486, "mix_low": 0.0942850112915039, "mix_high": 2.002716064453125e-05, "read_cosine": 0.26450011134147644, "beta_mean": 0.040588073432445526, "nonfinite": 0.0, "weighted_memory_to_active": 0.19665518403053284, "mix_std": 0.19343571364879608, "beta_std": 0.045153647661209106, "beta_low": 0.1324462890625, "beta_high": 0.0, "orthogonal_to_active": 0.0496976263821125, "evidence_weights": [-0.0774465873837471, -0.059163276106119156, 0.8530014753341675], "agreement_weight": -0.0741613581776619, "orthogonal_weight": -0.2860758304595947}, {"head": 5, "memory_to_active": 0.17935135960578918, "contribution_to_active": 0.39524149894714355, "mix_mean": 0.3956944942474365, "mix_low": 0.001880645751953125, "mix_high": 0.0006303787231445312, "read_cosine": 0.920081615447998, "beta_mean": 0.011697819456458092, "nonfinite": 0.0, "weighted_memory_to_active": 0.08890798687934875, "mix_std": 0.25894954800605774, "beta_std": 0.016697945073246956, "beta_low": 0.6412353515625, "beta_high": 0.0, "orthogonal_to_active": 0.018440326675772667, "evidence_weights": [-0.19049443304538727, -0.23241037130355835, 0.3455829620361328], "agreement_weight": 0.04364465922117233, "orthogonal_weight": -0.5500022768974304}]}, {"layer": 12, "memory_to_active": 0.537498950958252, "contribution_to_active": 0.4857497215270996, "mix_mean": 0.37182366847991943, "mix_low": 0.04652675241231918, "mix_high": 0.0015883446903899312, "read_cosine": 0.2929885983467102, "beta_mean": 0.01794585771858692, "nonfinite": 0.0, "weighted_memory_to_active": 0.2306918501853943, "mix_std": 0.235845685005188, "beta_std": 0.022075166925787926, "beta_low": 0.7340800762176514, "beta_high": 0.0, "orthogonal_to_active": 0.06660661846399307, "output_influence": 0.3302437663078308, "residual_influence": 0.0886211097240448, "state_active": 0.05196203291416168, "state_memory": 0.015225797891616821, "state_peak": 43.779136657714844, "growth_active": 1.1764837503433228, "growth_memory": 1.2939375638961792, "heads": [{"head": 0, "memory_to_active": 0.4955343008041382, "contribution_to_active": 0.3150887191295624, "mix_mean": 0.2774812579154968, "mix_low": 0.024110794067382812, "mix_high": 0.00011920928955078125, "read_cosine": 0.3442016839981079, "beta_mean": 0.007386002689599991, "nonfinite": 0.0, "weighted_memory_to_active": 0.2027202695608139, "mix_std": 0.2374911606311798, "beta_std": 0.02191569097340107, "beta_low": 0.83404541015625, "beta_high": 0.0, "orthogonal_to_active": 0.06721285730600357, "evidence_weights": [-0.13299670815467834, -0.18639874458312988, 0.2470744401216507], "agreement_weight": 0.10040713101625443, "orthogonal_weight": -0.4818314015865326}, {"head": 1, "memory_to_active": 0.20086082816123962, "contribution_to_active": 0.40666741132736206, "mix_mean": 0.3983761966228485, "mix_low": 0.0012826919555664062, "mix_high": 0.00014781951904296875, "read_cosine": 0.5966124534606934, "beta_mean": 0.0022939317859709263, "nonfinite": 0.0, "weighted_memory_to_active": 0.08940482884645462, "mix_std": 0.2555773854255676, "beta_std": 0.006669518072158098, "beta_low": 0.96038818359375, "beta_high": 0.0, "orthogonal_to_active": 0.05722435563802719, "evidence_weights": [-0.17358551919460297, -0.22067061066627502, 0.6135823130607605], "agreement_weight": 0.08573736250400543, "orthogonal_weight": -0.8584229350090027}, {"head": 2, "memory_to_active": 0.5953633785247803, "contribution_to_active": 0.36939558386802673, "mix_mean": 0.30745750665664673, "mix_low": 0.02347564697265625, "mix_high": 0.000225067138671875, "read_cosine": 0.38710248470306396, "beta_mean": 0.08089327812194824, "nonfinite": 0.0, "weighted_memory_to_active": 0.2339622676372528, "mix_std": 0.2403830587863922, "beta_std": 0.06550595909357071, "beta_low": 0.0050048828125, "beta_high": 0.0, "orthogonal_to_active": 0.1182083860039711, "evidence_weights": [-0.22090481221675873, -0.06487755477428436, 0.005312521010637283], "agreement_weight": 0.039445824921131134, "orthogonal_weight": 0.620527446269989}, {"head": 3, "memory_to_active": 0.02852533385157585, "contribution_to_active": 0.6563835144042969, "mix_mean": 0.6321964859962463, "mix_low": 0.0005807876586914062, "mix_high": 0.007236480712890625, "read_cosine": 0.03125552088022232, "beta_mean": 0.00023179550771601498, "nonfinite": 0.0, "weighted_memory_to_active": 0.018397551029920578, "mix_std": 0.2528611123561859, "beta_std": 0.0008447830914519727, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.014589530415832996, "evidence_weights": [-0.3065232038497925, -0.2972995638847351, 0.5102857351303101], "agreement_weight": 0.16002972424030304, "orthogonal_weight": -0.8140704035758972}, {"head": 4, "memory_to_active": 1.8526381254196167, "contribution_to_active": 0.753963828086853, "mix_mean": 0.10406288504600525, "mix_low": 0.22312355041503906, "mix_high": 0.00101470947265625, "read_cosine": 0.053282663226127625, "beta_mean": 0.016375401988625526, "nonfinite": 0.0, "weighted_memory_to_active": 0.8161163330078125, "mix_std": 0.15725916624069214, "beta_std": 0.034503381699323654, "beta_low": 0.613525390625, "beta_high": 0.0, "orthogonal_to_active": 0.12448207288980484, "evidence_weights": [0.03545856103301048, -0.09844083338975906, -0.002853310201317072], "agreement_weight": -0.0078050801530480385, "orthogonal_weight": -0.16560864448547363}, {"head": 5, "memory_to_active": 0.05207161605358124, "contribution_to_active": 0.41299912333488464, "mix_mean": 0.5113677382469177, "mix_low": 0.006587028503417969, "mix_high": 0.0007867813110351562, "read_cosine": 0.34547680616378784, "beta_mean": 0.0004947416018694639, "nonfinite": 0.0, "weighted_memory_to_active": 0.0235498808324337, "mix_std": 0.27150219678878784, "beta_std": 0.003011662745848298, "beta_low": 0.992431640625, "beta_high": 0.0, "orthogonal_to_active": 0.017922507598996162, "evidence_weights": [-0.2600342035293579, -0.27278852462768555, 0.6128494143486023], "agreement_weight": 0.07143353670835495, "orthogonal_weight": -0.8281762003898621}]}, {"layer": 13, "memory_to_active": 0.22944919764995575, "contribution_to_active": 0.43690723180770874, "mix_mean": 0.43451446294784546, "mix_low": 0.003936767578125, "mix_high": 0.0008227031212300062, "read_cosine": 0.38357144594192505, "beta_mean": 0.017803480848670006, "nonfinite": 0.0, "weighted_memory_to_active": 0.10465502738952637, "mix_std": 0.2531723976135254, "beta_std": 0.019989944994449615, "beta_low": 0.7503154277801514, "beta_high": 0.0, "orthogonal_to_active": 0.08071964979171753, "output_influence": 0.380113810300827, "residual_influence": 0.12120893597602844, "state_active": 0.08178331702947617, "state_memory": 0.03302709385752678, "state_peak": 76.62675476074219, "growth_active": 1.0044968128204346, "growth_memory": 1.0445237159729004, "heads": [{"head": 0, "memory_to_active": 0.1810683161020279, "contribution_to_active": 0.43983545899391174, "mix_mean": 0.4156993627548218, "mix_low": 0.0021409988403320312, "mix_high": 0.0012454986572265625, "read_cosine": 0.3359902799129486, "beta_mean": 0.0015792781487107277, "nonfinite": 0.0, "weighted_memory_to_active": 0.08528847992420197, "mix_std": 0.25972914695739746, "beta_std": 0.0028403426986187696, "beta_low": 0.97833251953125, "beta_high": 0.0, "orthogonal_to_active": 0.07796784490346909, "evidence_weights": [-0.19636596739292145, -0.2162678986787796, 0.6318750381469727], "agreement_weight": 0.04528597742319107, "orthogonal_weight": -0.9975093603134155}, {"head": 1, "memory_to_active": 0.09740074723958969, "contribution_to_active": 0.4774182438850403, "mix_mean": 0.4054401218891144, "mix_low": 0.0025262832641601562, "mix_high": 2.002716064453125e-05, "read_cosine": 0.23348605632781982, "beta_mean": 0.000501743983477354, "nonfinite": 0.0, "weighted_memory_to_active": 0.04666121304035187, "mix_std": 0.256089985370636, "beta_std": 0.0011142876464873552, "beta_low": 0.998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.042381204664707184, "evidence_weights": [-0.18537478148937225, -0.22179146111011505, 0.680122435092926], "agreement_weight": 0.09093907475471497, "orthogonal_weight": -0.9589376449584961}, {"head": 2, "memory_to_active": 0.24607327580451965, "contribution_to_active": 0.32640397548675537, "mix_mean": 0.3137778043746948, "mix_low": 0.0021877288818359375, "mix_high": 1.71661376953125e-05, "read_cosine": 0.688071608543396, "beta_mean": 0.021295640617609024, "nonfinite": 0.0, "weighted_memory_to_active": 0.09643377363681793, "mix_std": 0.2129007875919342, "beta_std": 0.04606477543711662, "beta_low": 0.60589599609375, "beta_high": 0.0, "orthogonal_to_active": 0.04048244655132294, "evidence_weights": [-0.20729266107082367, -0.15143992006778717, 0.4523311257362366], "agreement_weight": 0.05039598047733307, "orthogonal_weight": -0.6448382139205933}, {"head": 3, "memory_to_active": 0.18301977217197418, "contribution_to_active": 0.5312012434005737, "mix_mean": 0.5542559623718262, "mix_low": 0.0047283172607421875, "mix_high": 0.0015249252319335938, "read_cosine": -0.05212990194559097, "beta_mean": 0.0009004650637507439, "nonfinite": 0.0, "weighted_memory_to_active": 0.12774395942687988, "mix_std": 0.25625714659690857, "beta_std": 0.0053649903275072575, "beta_low": 0.99163818359375, "beta_high": 0.0, "orthogonal_to_active": 0.1305813044309616, "evidence_weights": [-0.3484780788421631, -0.23532842099666595, 1.2811212539672852], "agreement_weight": 1.1163008213043213, "orthogonal_weight": -1.0305410623550415}, {"head": 4, "memory_to_active": 0.44629424810409546, "contribution_to_active": 0.31547072529792786, "mix_mean": 0.4774797260761261, "mix_low": 0.0059070587158203125, "mix_high": 0.0011434555053710938, "read_cosine": 0.8426474332809448, "beta_mean": 0.07958240807056427, "nonfinite": 0.0, "weighted_memory_to_active": 0.1610790491104126, "mix_std": 0.2535947263240814, "beta_std": 0.05654297396540642, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.12718847393989563, "evidence_weights": [-0.2551402747631073, -0.2546047866344452, 0.4627664387226105], "agreement_weight": 0.0006656592013314366, "orthogonal_weight": 1.098575234413147}, {"head": 5, "memory_to_active": 0.22283880412578583, "contribution_to_active": 0.5311136245727539, "mix_mean": 0.4404338002204895, "mix_low": 0.006130218505859375, "mix_high": 0.0009851455688476562, "read_cosine": 0.2533630132675171, "beta_mean": 0.0029613494407385588, "nonfinite": 0.0, "weighted_memory_to_active": 0.11072368919849396, "mix_std": 0.280462384223938, "beta_std": 0.008012293837964535, "beta_low": 0.92742919921875, "beta_high": 0.0, "orthogonal_to_active": 0.06571657955646515, "evidence_weights": [-0.20136095583438873, -0.23401488363742828, 0.3212498724460602], "agreement_weight": 0.06112169101834297, "orthogonal_weight": -0.6674200296401978}]}, {"layer": 14, "memory_to_active": 0.2877137362957001, "contribution_to_active": 0.4260704517364502, "mix_mean": 0.38249507546424866, "mix_low": 0.025253932923078537, "mix_high": 0.003017743583768606, "read_cosine": 0.6174441576004028, "beta_mean": 0.0671878233551979, "nonfinite": 0.0, "weighted_memory_to_active": 0.13534829020500183, "mix_std": 0.2508273422718048, "beta_std": 0.06630425900220871, "beta_low": 0.3974100947380066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04054548963904381, "output_influence": 0.25200149416923523, "residual_influence": 0.09656106680631638, "state_active": 0.08494909107685089, "state_memory": 0.017592594027519226, "state_peak": 27.543657302856445, "growth_active": 1.1990370750427246, "growth_memory": 1.383594274520874, "heads": [{"head": 0, "memory_to_active": 0.2797645330429077, "contribution_to_active": 0.27782922983169556, "mix_mean": 0.25136440992355347, "mix_low": 0.017675399780273438, "mix_high": 4.863739013671875e-05, "read_cosine": 0.7112205028533936, "beta_mean": 0.060059450566768646, "nonfinite": 0.0, "weighted_memory_to_active": 0.09607140719890594, "mix_std": 0.22847753763198853, "beta_std": 0.079947829246521, "beta_low": 0.17340087890625, "beta_high": 0.0, "orthogonal_to_active": 0.015486064366996288, "evidence_weights": [-0.05797477811574936, -0.1270960569381714, 0.7140052914619446], "agreement_weight": 0.04590148851275444, "orthogonal_weight": 0.24377143383026123}, {"head": 1, "memory_to_active": 0.46298283338546753, "contribution_to_active": 0.458902508020401, "mix_mean": 0.28382813930511475, "mix_low": 0.09905052185058594, "mix_high": 0.000823974609375, "read_cosine": 0.5804345607757568, "beta_mean": 0.2694597840309143, "nonfinite": 0.0, "weighted_memory_to_active": 0.23956815898418427, "mix_std": 0.2849496901035309, "beta_std": 0.24842748045921326, "beta_low": 0.098388671875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.01784788817167282, "evidence_weights": [-0.01281043142080307, -0.26644909381866455, 0.011191673576831818], "agreement_weight": 0.05971495807170868, "orthogonal_weight": 0.11331605911254883}, {"head": 2, "memory_to_active": 0.307375967502594, "contribution_to_active": 0.4639168977737427, "mix_mean": 0.45202386379241943, "mix_low": 0.006488800048828125, "mix_high": 0.0015821456909179688, "read_cosine": 0.412240207195282, "beta_mean": 0.014489805325865746, "nonfinite": 0.0, "weighted_memory_to_active": 0.16641905903816223, "mix_std": 0.26914775371551514, "beta_std": 0.016112906858325005, "beta_low": 0.5064697265625, "beta_high": 0.0, "orthogonal_to_active": 0.11572227627038956, "evidence_weights": [-0.19644036889076233, -0.22656404972076416, 0.5921292304992676], "agreement_weight": 0.08227984607219696, "orthogonal_weight": -0.7981690168380737}, {"head": 3, "memory_to_active": 0.0662466436624527, "contribution_to_active": 0.6393975019454956, "mix_mean": 0.6440293192863464, "mix_low": 0.0023641586303710938, "mix_high": 0.010128974914550781, "read_cosine": 0.5334072113037109, "beta_mean": 0.009575551375746727, "nonfinite": 0.0, "weighted_memory_to_active": 0.04545385390520096, "mix_std": 0.2406024932861328, "beta_std": 0.007929708808660507, "beta_low": 0.64422607421875, "beta_high": 0.0, "orthogonal_to_active": 0.01992272213101387, "evidence_weights": [-0.3764547109603882, -0.3249383270740509, 0.4089670777320862], "agreement_weight": 0.07330460846424103, "orthogonal_weight": -0.5884348750114441}, {"head": 4, "memory_to_active": 0.22176599502563477, "contribution_to_active": 0.4629213809967041, "mix_mean": 0.4181874990463257, "mix_low": 0.004553794860839844, "mix_high": 0.005288124084472656, "read_cosine": 0.7816880941390991, "beta_mean": 0.045383334159851074, "nonfinite": 0.0, "weighted_memory_to_active": 0.11518782377243042, "mix_std": 0.2572604715824127, "beta_std": 0.032850250601768494, "beta_low": 0.04901123046875, "beta_high": 0.0, "orthogonal_to_active": 0.024405330419540405, "evidence_weights": [-0.21377211809158325, -0.2617092728614807, 0.26789939403533936], "agreement_weight": 0.10691558569669724, "orthogonal_weight": -0.3667975664138794}, {"head": 5, "memory_to_active": 0.3881463408470154, "contribution_to_active": 0.2534552216529846, "mix_mean": 0.24553710222244263, "mix_low": 0.021390914916992188, "mix_high": 0.0002346038818359375, "read_cosine": 0.6856739521026611, "beta_mean": 0.004159005358815193, "nonfinite": 0.0, "weighted_memory_to_active": 0.14938940107822418, "mix_std": 0.2245260775089264, "beta_std": 0.01255738828331232, "beta_low": 0.9129638671875, "beta_high": 0.0, "orthogonal_to_active": 0.04988865554332733, "evidence_weights": [-0.18604616820812225, -0.134404256939888, 0.11552838236093521], "agreement_weight": 0.09486011415719986, "orthogonal_weight": -0.5358967185020447}]}, {"layer": 15, "memory_to_active": 0.09293673932552338, "contribution_to_active": 0.47482120990753174, "mix_mean": 0.37518882751464844, "mix_low": 0.00781393051147461, "mix_high": 0.002558390609920025, "read_cosine": 0.6713253259658813, "beta_mean": 0.05102306604385376, "nonfinite": 0.0, "weighted_memory_to_active": 0.04312793165445328, "mix_std": 0.269771933555603, "beta_std": 0.05260588601231575, "beta_low": 0.685699462890625, "beta_high": 0.0, "orthogonal_to_active": 0.009989173151552677, "output_influence": 0.2982109487056732, "residual_influence": 0.15315820276737213, "state_active": 0.1103588342666626, "state_memory": 0.009010166861116886, "state_peak": 26.378005981445312, "growth_active": 1.3545076847076416, "growth_memory": 1.5303248167037964, "heads": [{"head": 0, "memory_to_active": 0.09989623725414276, "contribution_to_active": 0.48029929399490356, "mix_mean": 0.3257458806037903, "mix_low": 0.007297515869140625, "mix_high": 0.0022974014282226562, "read_cosine": 0.7455642819404602, "beta_mean": 0.044900424778461456, "nonfinite": 0.0, "weighted_memory_to_active": 0.05051982402801514, "mix_std": 0.2509491443634033, "beta_std": 0.03313959389925003, "beta_low": 0.08984375, "beta_high": 0.0, "orthogonal_to_active": 0.011849869973957539, "evidence_weights": [-0.16338138282299042, -0.19817949831485748, 0.34458011388778687], "agreement_weight": 0.08291556686162949, "orthogonal_weight": -0.4731869399547577}, {"head": 1, "memory_to_active": 0.22333499789237976, "contribution_to_active": 0.39868903160095215, "mix_mean": 0.37547317147254944, "mix_low": 0.0069179534912109375, "mix_high": 0.0050945281982421875, "read_cosine": 0.8222699761390686, "beta_mean": 0.002059424761682749, "nonfinite": 0.0, "weighted_memory_to_active": 0.08937882632017136, "mix_std": 0.2697334587574005, "beta_std": 0.005244758911430836, "beta_low": 0.96783447265625, "beta_high": 0.0, "orthogonal_to_active": 0.00794467143714428, "evidence_weights": [-0.19754838943481445, -0.20253825187683105, 0.2796456813812256], "agreement_weight": 0.09569612145423889, "orthogonal_weight": -0.1348491907119751}, {"head": 2, "memory_to_active": 0.010181795805692673, "contribution_to_active": 0.4408400356769562, "mix_mean": 0.34513914585113525, "mix_low": 0.011904716491699219, "mix_high": 0.00018787384033203125, "read_cosine": 0.7656610608100891, "beta_mean": 0.00019813288236036897, "nonfinite": 0.0, "weighted_memory_to_active": 0.004152779467403889, "mix_std": 0.2681431770324707, "beta_std": 0.0003630642022471875, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.001025617471896112, "evidence_weights": [-0.2143748253583908, -0.27112600207328796, 0.3025430738925934], "agreement_weight": 0.0455394983291626, "orthogonal_weight": -0.3782327473163605}, {"head": 3, "memory_to_active": 0.09813854843378067, "contribution_to_active": 0.4819816052913666, "mix_mean": 0.3015463650226593, "mix_low": 0.003326416015625, "mix_high": 1.71661376953125e-05, "read_cosine": 0.7321130037307739, "beta_mean": 0.0034843666944652796, "nonfinite": 0.0, "weighted_memory_to_active": 0.04681408032774925, "mix_std": 0.23911724984645844, "beta_std": 0.005738899111747742, "beta_low": 0.9063720703125, "beta_high": 0.0, "orthogonal_to_active": 0.019136589020490646, "evidence_weights": [-0.17660020291805267, -0.21588842570781708, 0.38250038027763367], "agreement_weight": 0.02529306150972843, "orthogonal_weight": -0.7129577398300171}, {"head": 4, "memory_to_active": 0.12047570943832397, "contribution_to_active": 0.5743545889854431, "mix_mean": 0.39819973707199097, "mix_low": 0.012833595275878906, "mix_high": 0.0006437301635742188, "read_cosine": 0.41590967774391174, "beta_mean": 0.2553618252277374, "nonfinite": 0.0, "weighted_memory_to_active": 0.06549277156591415, "mix_std": 0.3067377209663391, "beta_std": 0.2709138095378876, "beta_low": 0.1502685546875, "beta_high": 0.0, "orthogonal_to_active": 0.018508149310946465, "evidence_weights": [-0.2701507806777954, -0.25084465742111206, 0.2994043827056885], "agreement_weight": 0.007117589935660362, "orthogonal_weight": -0.34712180495262146}, {"head": 5, "memory_to_active": 0.005593094974756241, "contribution_to_active": 0.4727623760700226, "mix_mean": 0.5050285458564758, "mix_low": 0.004603385925292969, "mix_high": 0.007109642028808594, "read_cosine": 0.5464340448379517, "beta_mean": 0.0001342120231129229, "nonfinite": 0.0, "weighted_memory_to_active": 0.0024093110114336014, "mix_std": 0.2839507460594177, "beta_std": 0.0002351629373151809, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001470139715820551, "evidence_weights": [-0.36125102639198303, -0.2990337014198303, 0.4329814910888672], "agreement_weight": 0.1290389448404312, "orthogonal_weight": 0.7423900365829468}]}]}, {"step": 14000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8883696690008946, "memory_to_active": 1.018092393875122, "contribution_to_active": 0.44004473090171814, "mix_mean": 0.28657782077789307, "mix_low": 0.17790135741233826, "mix_high": 0.004529486410319805, "read_cosine": 0.3955494165420532, "beta_mean": 0.09643839299678802, "nonfinite": 0.0, "weighted_memory_to_active": 0.20985233783721924, "mix_std": 0.20551598072052002, "beta_std": 0.07739656418561935, "beta_low": 0.44440653920173645, "beta_high": 1.2079875887138769e-05, "orthogonal_to_active": 0.03936898708343506, "samples": 4, "scored_tokens": 899, "normal_loss": 3.196765422821045, "normal_token_accuracy": 0.44716352224349976, "state_active": 0.062290169298648834, "state_memory": 0.0250236839056015, "state_peak": 98.47048950195312, "growth_active": 1.1406707763671875, "growth_memory": 1.4010703563690186, "output_influence": 0.2981286942958832, "residual_influence": 0.09382827579975128, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2893896102905273, "no_memory_loss_delta": 0.13610722124576569, "scrambled_loss_delta": 0.1836346983909607, "memory_only_loss_delta": 2.2230443954467773, "no_evidence_loss_delta": 0.07680827379226685, "no_agreement_loss_delta": -0.0023088455200195312, "no_orthogonal_loss_delta": -0.002288646064698696}, {"index": 94, "tokens": 256, "normal_loss": 3.316455841064453, "no_memory_loss_delta": 0.2858782708644867, "scrambled_loss_delta": 0.4008210301399231, "memory_only_loss_delta": 2.0556161403656006, "no_evidence_loss_delta": 0.14591674506664276, "no_agreement_loss_delta": 0.029067983850836754, "no_orthogonal_loss_delta": 0.007363416254520416}, {"index": 188, "tokens": 256, "normal_loss": 3.5699076652526855, "no_memory_loss_delta": 0.18440763652324677, "scrambled_loss_delta": 0.08157035708427429, "memory_only_loss_delta": 2.0142760276794434, "no_evidence_loss_delta": 0.09560758620500565, "no_agreement_loss_delta": -0.007740227039903402, "no_orthogonal_loss_delta": 0.0011305972002446651}, {"index": 283, "tokens": 131, "normal_loss": 2.0526654720306396, "no_memory_loss_delta": 0.18939734995365143, "scrambled_loss_delta": 0.43716156482696533, "memory_only_loss_delta": 2.7248785495758057, "no_evidence_loss_delta": 0.030747810378670692, "no_agreement_loss_delta": -0.010026508010923862, "no_orthogonal_loss_delta": -0.011902619153261185}], "agreement_weight_abs_mean": 0.07720313966274261, "agreement_weight_abs_max": 1.2487428188323975, "agreement_weight_mean": 0.06243530288338661, "orthogonal_weight_abs_mean": 0.4464204013347626, "orthogonal_weight_abs_max": 2.4204604625701904, "orthogonal_weight_mean": -0.1640300154685974, "no_memory_loss": 3.397040605545044, "no_memory_loss_delta": 0.20027552545070648, "no_memory_token_accuracy_gain": 0.02892102301120758, "no_memory_token_win_fraction": 0.5728587508201599, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 18.14947970079458, "scrambled_loss": 3.4501256942749023, "scrambled_loss_delta": 0.25336024165153503, "scrambled_token_accuracy_gain": 0.032258063554763794, "scrambled_token_win_fraction": 0.5995550751686096, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 22.38117838717338, "memory_only_loss": 5.385808944702148, "memory_only_loss_delta": 2.189044237136841, "memory_only_token_accuracy_gain": 0.25806450843811035, "memory_only_token_win_fraction": 0.8676307201385498, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 88.79762343772839, "no_evidence_loss": 3.2918944358825684, "no_evidence_loss_delta": 0.09512915462255478, "no_evidence_token_accuracy_gain": 0.004449388012290001, "no_evidence_token_win_fraction": 0.5183537006378174, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 9.074450766562087, "no_agreement_loss": 3.2007200717926025, "no_agreement_loss_delta": 0.00395480589941144, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.44938820600509644, "no_agreement_probe_win_fraction": 0.25, "no_agreement_ppl_reduction_pct": 0.39469959535592725, "no_orthogonal_loss": 3.1967976093292236, "no_orthogonal_loss_delta": 3.262595055275597e-05, "no_orthogonal_token_accuracy_gain": -0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5183537006378174, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 0.0032625418332219314, "short_no_memory_loss_delta": 0.17825569212436676, "long_context_memory_gain": 0.02201983332633972, "normal_context_loss_gain": 0.10415790230035782, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.013996124267578, "contribution_to_active": 0.816778302192688, "mix_mean": 0.09498904645442963, "mix_low": 0.45084303617477417, "mix_high": 0.003012498375028372, "read_cosine": 0.48530930280685425, "beta_mean": 0.27303346991539, "nonfinite": 0.0, "weighted_memory_to_active": 0.8584356307983398, "mix_std": 0.084509938955307, "beta_std": 0.1622018665075302, "beta_low": 9.1552734375e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04440322518348694, "output_influence": 0.13220643997192383, "residual_influence": 0.1997523307800293, "state_active": 0.08347031474113464, "state_memory": 0.05190446227788925, "state_peak": 47.736331939697266, "growth_active": 1.2754255533218384, "growth_memory": 1.30098557472229, "heads": [{"head": 0, "memory_to_active": 2.2936933040618896, "contribution_to_active": 0.37929996848106384, "mix_mean": 0.02118125557899475, "mix_low": 0.5124406814575195, "mix_high": 0.0, "read_cosine": 0.3364836573600769, "beta_mean": 0.2018914371728897, "nonfinite": 0.0, "weighted_memory_to_active": 0.46808958053588867, "mix_std": 0.05279851704835892, "beta_std": 0.08835506439208984, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03080040216445923, "evidence_weights": [-0.044365573674440384, -0.07374253869056702, -0.030375469475984573], "agreement_weight": -0.07573792338371277, "orthogonal_weight": -0.07870355993509293}, {"head": 1, "memory_to_active": 4.00512170791626, "contribution_to_active": 1.4872983694076538, "mix_mean": 0.019041135907173157, "mix_low": 0.5100879669189453, "mix_high": 0.0, "read_cosine": 0.45433592796325684, "beta_mean": 0.3659602403640747, "nonfinite": 0.0, "weighted_memory_to_active": 1.822865605354309, "mix_std": 0.04706287011504173, "beta_std": 0.1625223159790039, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12305977940559387, "evidence_weights": [0.008627636358141899, -0.03658781200647354, 0.14538249373435974], "agreement_weight": 0.011427781544625759, "orthogonal_weight": -0.16223831474781036}, {"head": 2, "memory_to_active": 2.6677470207214355, "contribution_to_active": 0.4954480230808258, "mix_mean": 0.014952640980482101, "mix_low": 0.6281766891479492, "mix_high": 0.0, "read_cosine": 0.3828209936618805, "beta_mean": 0.3289177119731903, "nonfinite": 0.0, "weighted_memory_to_active": 0.5624977946281433, "mix_std": 0.03195364028215408, "beta_std": 0.23795254528522491, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05246495455503464, "evidence_weights": [0.03880021721124649, 0.046450383961200714, -0.016023846343159676], "agreement_weight": -0.008432717993855476, "orthogonal_weight": -0.15424518287181854}, {"head": 3, "memory_to_active": 5.198660373687744, "contribution_to_active": 2.109196901321411, "mix_mean": 0.05165145918726921, "mix_low": 0.2393341064453125, "mix_high": 0.0, "read_cosine": 0.4397800862789154, "beta_mean": 0.33314424753189087, "nonfinite": 0.0, "weighted_memory_to_active": 2.1618502140045166, "mix_std": 0.08805625885725021, "beta_std": 0.20451313257217407, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04171230271458626, "evidence_weights": [-0.1048656776547432, -0.13144102692604065, -0.0033198227174580097], "agreement_weight": -0.014960423111915588, "orthogonal_weight": 0.025895459577441216}, {"head": 4, "memory_to_active": 3.7704272270202637, "contribution_to_active": 0.08255675435066223, "mix_mean": 0.006951932795345783, "mix_low": 0.7951526641845703, "mix_high": 0.0, "read_cosine": 0.3403022885322571, "beta_mean": 0.3367772698402405, "nonfinite": 0.0, "weighted_memory_to_active": 0.07499068975448608, "mix_std": 0.011434532701969147, "beta_std": 0.21149984002113342, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.011369043029844761, "evidence_weights": [0.06935197114944458, 0.02443591319024563, -0.23217198252677917], "agreement_weight": -0.028773609548807144, "orthogonal_weight": 0.17468710243701935}, {"head": 5, "memory_to_active": 0.14832746982574463, "contribution_to_active": 0.34686970710754395, "mix_mean": 0.45615583658218384, "mix_low": 0.019865989685058594, "mix_high": 0.018074989318847656, "read_cosine": 0.9581328630447388, "beta_mean": 0.07150989770889282, "nonfinite": 0.0, "weighted_memory_to_active": 0.060319799929857254, "mix_std": 0.27575385570526123, "beta_std": 0.06836824119091034, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.007012859918177128, "evidence_weights": [-0.2913104295730591, -0.2826893627643585, 0.3163999319076538], "agreement_weight": 0.1606009155511856, "orthogonal_weight": -0.28904402256011963}]}, {"layer": 1, "memory_to_active": 2.3308727741241455, "contribution_to_active": 0.40849316120147705, "mix_mean": 0.10249432176351547, "mix_low": 0.40447568893432617, "mix_high": 0.00045092901564203203, "read_cosine": 0.3111281991004944, "beta_mean": 0.2077106237411499, "nonfinite": 0.0, "weighted_memory_to_active": 0.3569895625114441, "mix_std": 0.11318399757146835, "beta_std": 0.13361434638500214, "beta_low": 0.1698099821805954, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.03528270125389099, "output_influence": 0.14289258420467377, "residual_influence": 0.03236928954720497, "state_active": 0.043961070477962494, "state_memory": 0.04751009866595268, "state_peak": 23.66153907775879, "growth_active": 1.1045513153076172, "growth_memory": 1.5448919534683228, "heads": [{"head": 0, "memory_to_active": 0.05079656094312668, "contribution_to_active": 0.2814653515815735, "mix_mean": 0.21902768313884735, "mix_low": 0.05796527862548828, "mix_high": 5.054473876953125e-05, "read_cosine": -0.0977184921503067, "beta_mean": 0.00020348309772089124, "nonfinite": 0.0, "weighted_memory_to_active": 0.019129524007439613, "mix_std": 0.24225632846355438, "beta_std": 0.0004493952728807926, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.009514708071947098, "evidence_weights": [-0.11387091130018234, -0.14595191180706024, -0.0063934470526874065], "agreement_weight": 0.05129507556557655, "orthogonal_weight": -0.5639888644218445}, {"head": 1, "memory_to_active": 3.536393165588379, "contribution_to_active": 0.19135740399360657, "mix_mean": 0.023391399532556534, "mix_low": 0.4855060577392578, "mix_high": 2.956390380859375e-05, "read_cosine": 0.4408448338508606, "beta_mean": 0.3244668245315552, "nonfinite": 0.0, "weighted_memory_to_active": 0.20088054239749908, "mix_std": 0.05051511153578758, "beta_std": 0.19980360567569733, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.0026158185210078955, "evidence_weights": [-0.02207200601696968, -0.0037561431527137756, 0.043024249374866486], "agreement_weight": -0.01543055847287178, "orthogonal_weight": 0.019999317824840546}, {"head": 2, "memory_to_active": 0.7027410268783569, "contribution_to_active": 0.41919928789138794, "mix_mean": 0.3152616620063782, "mix_low": 0.009507179260253906, "mix_high": 0.0025501251220703125, "read_cosine": 0.7455484867095947, "beta_mean": 0.10464043915271759, "nonfinite": 0.0, "weighted_memory_to_active": 0.5232144594192505, "mix_std": 0.24618661403656006, "beta_std": 0.05309291556477547, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.00961326714605093, "evidence_weights": [-0.2131223976612091, -0.36617153882980347, 0.2154499888420105], "agreement_weight": 0.004076529759913683, "orthogonal_weight": 0.03465879335999489}, {"head": 3, "memory_to_active": 2.79740047454834, "contribution_to_active": 0.0740521103143692, "mix_mean": 0.010019589215517044, "mix_low": 0.7817058563232422, "mix_high": 0.0, "read_cosine": 0.1993466317653656, "beta_mean": 0.35323411226272583, "nonfinite": 0.0, "weighted_memory_to_active": 0.09000717103481293, "mix_std": 0.03081512078642845, "beta_std": 0.23074032366275787, "beta_low": 0.0008544921875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.016991017386317253, "evidence_weights": [0.032721538096666336, 0.01625925302505493, 0.03121657483279705], "agreement_weight": -0.022090580314397812, "orthogonal_weight": -0.19805902242660522}, {"head": 4, "memory_to_active": 3.0964572429656982, "contribution_to_active": 1.4132088422775269, "mix_mean": 0.03766224533319473, "mix_low": 0.30549049377441406, "mix_high": 4.863739013671875e-05, "read_cosine": 0.3917675018310547, "beta_mean": 0.18221434950828552, "nonfinite": 0.0, "weighted_memory_to_active": 1.2413941621780396, "mix_std": 0.07592673599720001, "beta_std": 0.09948400408029556, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17154847085475922, "evidence_weights": [-0.025633448734879494, -0.017702773213386536, 0.11030855774879456], "agreement_weight": -0.03789220005273819, "orthogonal_weight": 0.15455740690231323}, {"head": 5, "memory_to_active": 3.801448106765747, "contribution_to_active": 0.07167600095272064, "mix_mean": 0.00960334949195385, "mix_low": 0.7866792678833008, "mix_high": 2.6702880859375e-05, "read_cosine": 0.1869802474975586, "beta_mean": 0.28150445222854614, "nonfinite": 0.0, "weighted_memory_to_active": 0.06731162965297699, "mix_std": 0.03340405970811844, "beta_std": 0.21811579167842865, "beta_low": 0.0184326171875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0014129300834611058, "evidence_weights": [0.11540226638317108, 0.055241432040929794, 0.30157962441444397], "agreement_weight": -0.13693346083164215, "orthogonal_weight": 0.021465210244059563}]}, {"layer": 2, "memory_to_active": 1.1834462881088257, "contribution_to_active": 0.29348084330558777, "mix_mean": 0.24074850976467133, "mix_low": 0.2671593129634857, "mix_high": 0.0019963583908975124, "read_cosine": 0.34675073623657227, "beta_mean": 0.11600334942340851, "nonfinite": 0.0, "weighted_memory_to_active": 0.08056530356407166, "mix_std": 0.18698811531066895, "beta_std": 0.07569972425699234, "beta_low": 0.5502828359603882, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.015677165240049362, "output_influence": 0.2420480102300644, "residual_influence": 0.0547996386885643, "state_active": 0.04115641117095947, "state_memory": 0.02713761478662491, "state_peak": 53.6021614074707, "growth_active": 1.0348305702209473, "growth_memory": 1.5488542318344116, "heads": [{"head": 0, "memory_to_active": 0.02194075472652912, "contribution_to_active": 0.14324650168418884, "mix_mean": 0.26551079750061035, "mix_low": 0.033844947814941406, "mix_high": 0.000576019287109375, "read_cosine": 0.24118252098560333, "beta_mean": 0.00013798626605421305, "nonfinite": 0.0, "weighted_memory_to_active": 0.007354610599577427, "mix_std": 0.262076735496521, "beta_std": 0.0002403774269623682, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00024349481100216508, "evidence_weights": [-0.15391138195991516, -0.1047963798046112, 0.20666521787643433], "agreement_weight": 0.044756222516298294, "orthogonal_weight": -0.0388067364692688}, {"head": 1, "memory_to_active": 3.1738240718841553, "contribution_to_active": 0.05746549367904663, "mix_mean": 0.005278776399791241, "mix_low": 0.8882894515991211, "mix_high": 0.0, "read_cosine": 0.18079213798046112, "beta_mean": 0.33492422103881836, "nonfinite": 0.0, "weighted_memory_to_active": 0.05173150449991226, "mix_std": 0.011880271136760712, "beta_std": 0.15112200379371643, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012324906885623932, "evidence_weights": [0.09377448260784149, 0.10157132893800735, 0.19146868586540222], "agreement_weight": 0.02603587508201599, "orthogonal_weight": 0.2803841531276703}, {"head": 2, "memory_to_active": 0.04519783705472946, "contribution_to_active": 0.42636966705322266, "mix_mean": 0.3713296353816986, "mix_low": 0.005131721496582031, "mix_high": 0.0019054412841796875, "read_cosine": 0.30912885069847107, "beta_mean": 7.647894381079823e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.019026879221200943, "mix_std": 0.2741739749908447, "beta_std": 0.00014891443424858153, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00321625592187047, "evidence_weights": [-0.1860121339559555, -0.21564334630966187, 0.3739297389984131], "agreement_weight": 0.024229589849710464, "orthogonal_weight": -0.22919440269470215}, {"head": 3, "memory_to_active": 0.3129110038280487, "contribution_to_active": 0.43198299407958984, "mix_mean": 0.289547324180603, "mix_low": 0.0061321258544921875, "mix_high": 0.004923820495605469, "read_cosine": 0.6686944365501404, "beta_mean": 0.043629858642816544, "nonfinite": 0.0, "weighted_memory_to_active": 0.13643069565296173, "mix_std": 0.24669454991817474, "beta_std": 0.06468474864959717, "beta_low": 0.27825927734375, "beta_high": 0.0, "orthogonal_to_active": 0.06641385704278946, "evidence_weights": [-0.11898969113826752, -0.1819160282611847, 0.5780386328697205], "agreement_weight": 0.07453038543462753, "orthogonal_weight": -0.7365610003471375}, {"head": 4, "memory_to_active": 3.5286946296691895, "contribution_to_active": 0.2198159098625183, "mix_mean": 0.014362627640366554, "mix_low": 0.6671819686889648, "mix_high": 4.76837158203125e-06, "read_cosine": 0.18657681345939636, "beta_mean": 0.317211389541626, "nonfinite": 0.0, "weighted_memory_to_active": 0.25895237922668457, "mix_std": 0.043565068393945694, "beta_std": 0.23793618381023407, "beta_low": 0.0234375, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.011211777105927467, "evidence_weights": [0.03974292427301407, 0.032537780702114105, 0.1419808566570282], "agreement_weight": 0.03005744144320488, "orthogonal_weight": -0.0543711856007576}, {"head": 5, "memory_to_active": 0.018108954653143883, "contribution_to_active": 0.4820045232772827, "mix_mean": 0.49846190214157104, "mix_low": 0.0023756027221679688, "mix_high": 0.0045680999755859375, "read_cosine": 0.49412965774536133, "beta_mean": 4.010360862594098e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009895741008222103, "mix_std": 0.2835380733013153, "beta_std": 6.608425610465929e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00065269396873191, "evidence_weights": [-0.23989476263523102, -0.24408507347106934, 0.6174029111862183], "agreement_weight": 0.04127403721213341, "orthogonal_weight": 0.11724277585744858}]}, {"layer": 3, "memory_to_active": 1.3836050033569336, "contribution_to_active": 0.3272373080253601, "mix_mean": 0.17128895223140717, "mix_low": 0.2820208966732025, "mix_high": 0.001710573909804225, "read_cosine": 0.269681841135025, "beta_mean": 0.09563788771629333, "nonfinite": 0.0, "weighted_memory_to_active": 0.13846659660339355, "mix_std": 0.18820908665657043, "beta_std": 0.07717554271221161, "beta_low": 0.4156494140625, "beta_high": 0.0, "orthogonal_to_active": 0.02652757242321968, "output_influence": 0.2291124016046524, "residual_influence": 0.04156911373138428, "state_active": 0.03351714462041855, "state_memory": 0.0228201225399971, "state_peak": 28.04247283935547, "growth_active": 1.0704560279846191, "growth_memory": 1.7965034246444702, "heads": [{"head": 0, "memory_to_active": 0.7597230672836304, "contribution_to_active": 0.3117145001888275, "mix_mean": 0.1813962757587433, "mix_low": 0.1830158233642578, "mix_high": 0.0008258819580078125, "read_cosine": 0.24536240100860596, "beta_mean": 0.014901693910360336, "nonfinite": 0.0, "weighted_memory_to_active": 0.1652289181947708, "mix_std": 0.25067561864852905, "beta_std": 0.012975549325346947, "beta_low": 0.43536376953125, "beta_high": 0.0, "orthogonal_to_active": 0.05030040815472603, "evidence_weights": [-0.03866470977663994, -0.11178509145975113, -0.07012888044118881], "agreement_weight": 0.06296918541193008, "orthogonal_weight": -0.3088908791542053}, {"head": 1, "memory_to_active": 0.06428202241659164, "contribution_to_active": 0.2542942762374878, "mix_mean": 0.22669783234596252, "mix_low": 0.08720016479492188, "mix_high": 0.0012388229370117188, "read_cosine": 0.22141996026039124, "beta_mean": 0.00028209557058289647, "nonfinite": 0.0, "weighted_memory_to_active": 0.013793615624308586, "mix_std": 0.2598450779914856, "beta_std": 0.0005443313275463879, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0005107576726004481, "evidence_weights": [-0.13102079927921295, -0.1009313240647316, 0.3332529664039612], "agreement_weight": 0.04483994469046593, "orthogonal_weight": -0.03974970802664757}, {"head": 2, "memory_to_active": 4.004674434661865, "contribution_to_active": 0.28024888038635254, "mix_mean": 0.03240588307380676, "mix_low": 0.4570608139038086, "mix_high": 9.5367431640625e-06, "read_cosine": 0.19624406099319458, "beta_mean": 0.2530010938644409, "nonfinite": 0.0, "weighted_memory_to_active": 0.37229394912719727, "mix_std": 0.07862935960292816, "beta_std": 0.16130372881889343, "beta_low": 0.00323486328125, "beta_high": 0.0, "orthogonal_to_active": 0.027712704613804817, "evidence_weights": [-0.04144929349422455, 0.011637805961072445, 0.08017485588788986], "agreement_weight": -0.010972222313284874, "orthogonal_weight": -0.09748123586177826}, {"head": 3, "memory_to_active": 0.04345245659351349, "contribution_to_active": 0.437914103269577, "mix_mean": 0.41460758447647095, "mix_low": 0.034142494201660156, "mix_high": 0.0052032470703125, "read_cosine": 0.29000380635261536, "beta_mean": 0.0001429111580364406, "nonfinite": 0.0, "weighted_memory_to_active": 0.022386780008673668, "mix_std": 0.3116900324821472, "beta_std": 0.000368179491488263, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005834875628352165, "evidence_weights": [-0.17947065830230713, -0.20033982396125793, 1.0151383876800537], "agreement_weight": 0.5556267499923706, "orthogonal_weight": -0.3565746247768402}, {"head": 4, "memory_to_active": 3.022033214569092, "contribution_to_active": 0.03598862886428833, "mix_mean": 0.006883702240884304, "mix_low": 0.8390798568725586, "mix_high": 0.0, "read_cosine": 0.2158225178718567, "beta_mean": 0.19757036864757538, "nonfinite": 0.0, "weighted_memory_to_active": 0.048017099499702454, "mix_std": 0.017383404076099396, "beta_std": 0.1623169183731079, "beta_low": 0.0107421875, "beta_high": 0.0, "orthogonal_to_active": 0.014865175820887089, "evidence_weights": [0.14894750714302063, 0.12351910024881363, -0.0900331512093544], "agreement_weight": -0.0514669232070446, "orthogonal_weight": -0.3250933289527893}, {"head": 5, "memory_to_active": 0.40746501088142395, "contribution_to_active": 0.6432633399963379, "mix_mean": 0.165742427110672, "mix_low": 0.09162616729736328, "mix_high": 0.0029859542846679688, "read_cosine": 0.44923824071884155, "beta_mean": 0.10792919248342514, "nonfinite": 0.0, "weighted_memory_to_active": 0.20907925069332123, "mix_std": 0.2110309898853302, "beta_std": 0.12554453313350677, "beta_low": 0.0447998046875, "beta_high": 0.0, "orthogonal_to_active": 0.05994151532649994, "evidence_weights": [-0.07081423699855804, -0.15100088715553284, 0.16895362734794617], "agreement_weight": 0.16031867265701294, "orthogonal_weight": -0.40785470604896545}]}, {"layer": 4, "memory_to_active": 1.4092426300048828, "contribution_to_active": 0.3475743532180786, "mix_mean": 0.2533407211303711, "mix_low": 0.2844465672969818, "mix_high": 0.0017854373436421156, "read_cosine": 0.3954426050186157, "beta_mean": 0.14040279388427734, "nonfinite": 0.0, "weighted_memory_to_active": 0.17728090286254883, "mix_std": 0.1565476804971695, "beta_std": 0.11784419417381287, "beta_low": 0.26251220703125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.0410853773355484, "output_influence": 0.242152139544487, "residual_influence": 0.055440500378608704, "state_active": 0.04629027098417282, "state_memory": 0.03482057526707649, "state_peak": 52.637447357177734, "growth_active": 1.145719051361084, "growth_memory": 1.6442614793777466, "heads": [{"head": 0, "memory_to_active": 0.4305766820907593, "contribution_to_active": 0.5433309674263, "mix_mean": 0.45681214332580566, "mix_low": 0.013517379760742188, "mix_high": 0.0020322799682617188, "read_cosine": 0.5909304618835449, "beta_mean": 0.19180917739868164, "nonfinite": 0.0, "weighted_memory_to_active": 0.277967631816864, "mix_std": 0.3071700930595398, "beta_std": 0.2131754606962204, "beta_low": 0.15093994140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.02035510167479515, "evidence_weights": [-0.21300144493579865, -0.2609950304031372, 0.451947420835495], "agreement_weight": 0.04247310757637024, "orthogonal_weight": 0.09769715368747711}, {"head": 1, "memory_to_active": 0.40296241641044617, "contribution_to_active": 0.3364998698234558, "mix_mean": 0.3799746036529541, "mix_low": 0.006163597106933594, "mix_high": 0.0012369155883789062, "read_cosine": 0.5649172067642212, "beta_mean": 0.022667594254016876, "nonfinite": 0.0, "weighted_memory_to_active": 0.17379431426525116, "mix_std": 0.26735517382621765, "beta_std": 0.03664545342326164, "beta_low": 0.3739013671875, "beta_high": 0.0, "orthogonal_to_active": 0.08548204600811005, "evidence_weights": [-0.20779713988304138, -0.184967041015625, 0.6370954513549805], "agreement_weight": 0.037785034626722336, "orthogonal_weight": -0.621696949005127}, {"head": 2, "memory_to_active": 2.993358612060547, "contribution_to_active": 0.1119966134428978, "mix_mean": 0.012237218208611012, "mix_low": 0.6759500503540039, "mix_high": 0.0, "read_cosine": 0.29612985253334045, "beta_mean": 0.18075218796730042, "nonfinite": 0.0, "weighted_memory_to_active": 0.14050662517547607, "mix_std": 0.021003568544983864, "beta_std": 0.13325554132461548, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.004257353954017162, "evidence_weights": [0.04870946332812309, 0.027500810101628304, -0.044623665511608124], "agreement_weight": 0.05546696484088898, "orthogonal_weight": -0.04233621805906296}, {"head": 3, "memory_to_active": 1.7560052871704102, "contribution_to_active": 0.1024467796087265, "mix_mean": 0.025893960148096085, "mix_low": 0.41405487060546875, "mix_high": 0.0, "read_cosine": 0.20295314490795135, "beta_mean": 0.1750383824110031, "nonfinite": 0.0, "weighted_memory_to_active": 0.16345445811748505, "mix_std": 0.04291166365146637, "beta_std": 0.15401658415794373, "beta_low": 0.0499267578125, "beta_high": 0.0, "orthogonal_to_active": 0.021392066031694412, "evidence_weights": [-0.014343241229653358, 0.04698239639401436, -0.16372278332710266], "agreement_weight": 0.03006703034043312, "orthogonal_weight": -0.21085646748542786}, {"head": 4, "memory_to_active": 0.028183791786432266, "contribution_to_active": 0.6005158424377441, "mix_mean": 0.6285949945449829, "mix_low": 0.0013151168823242188, "mix_high": 0.007289886474609375, "read_cosine": 0.3386976420879364, "beta_mean": 5.420045636128634e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.020174497738480568, "mix_std": 0.2582225203514099, "beta_std": 9.9141092505306e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0022800387814641, "evidence_weights": [-0.2774242162704468, -0.3171098232269287, 0.7220981121063232], "agreement_weight": 0.041147131472826004, "orthogonal_weight": -0.1391608864068985}, {"head": 5, "memory_to_active": 2.8443682193756104, "contribution_to_active": 0.3906559348106384, "mix_mean": 0.016531284898519516, "mix_low": 0.5956783294677734, "mix_high": 0.00015354156494140625, "read_cosine": 0.3790273070335388, "beta_mean": 0.27209511399269104, "nonfinite": 0.0, "weighted_memory_to_active": 0.2877878248691559, "mix_std": 0.04262297973036766, "beta_std": 0.1698729395866394, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11274563521146774, "evidence_weights": [0.028963377699255943, -0.03308477997779846, -0.12042071670293808], "agreement_weight": -0.05110601708292961, "orthogonal_weight": 0.609469473361969}]}, {"layer": 5, "memory_to_active": 0.7531718015670776, "contribution_to_active": 0.4262324273586273, "mix_mean": 0.37358295917510986, "mix_low": 0.11115710437297821, "mix_high": 0.009054979309439659, "read_cosine": 0.24900704622268677, "beta_mean": 0.07304799556732178, "nonfinite": 0.0, "weighted_memory_to_active": 0.17612650990486145, "mix_std": 0.24517589807510376, "beta_std": 0.0854468047618866, "beta_low": 0.5604146718978882, "beta_high": 0.0, "orthogonal_to_active": 0.05225415527820587, "output_influence": 0.41374820470809937, "residual_influence": 0.09066978096961975, "state_active": 0.060057658702135086, "state_memory": 0.019729500636458397, "state_peak": 59.25501251220703, "growth_active": 1.1118096113204956, "growth_memory": 1.3597337007522583, "heads": [{"head": 0, "memory_to_active": 1.4295194149017334, "contribution_to_active": 0.22056439518928528, "mix_mean": 0.09465935826301575, "mix_low": 0.18515777587890625, "mix_high": 1.049041748046875e-05, "read_cosine": -0.08267799019813538, "beta_mean": 0.12309738993644714, "nonfinite": 0.0, "weighted_memory_to_active": 0.21389003098011017, "mix_std": 0.1386530101299286, "beta_std": 0.13547281920909882, "beta_low": 0.0269775390625, "beta_high": 0.0, "orthogonal_to_active": 0.046252187341451645, "evidence_weights": [-0.03050030581653118, -0.06352075934410095, -0.32493671774864197], "agreement_weight": -0.019489329308271408, "orthogonal_weight": -0.21838009357452393}, {"head": 1, "memory_to_active": 0.05280746519565582, "contribution_to_active": 0.44602781534194946, "mix_mean": 0.5016381144523621, "mix_low": 0.011651992797851562, "mix_high": 0.009669303894042969, "read_cosine": 0.1301066130399704, "beta_mean": 0.00015603062638547271, "nonfinite": 0.0, "weighted_memory_to_active": 0.023707380518317223, "mix_std": 0.31726163625717163, "beta_std": 0.00037016766145825386, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.020946355536580086, "evidence_weights": [-0.2151842564344406, -0.2241973578929901, 0.6007358431816101], "agreement_weight": 0.2277912050485611, "orthogonal_weight": -0.9044404029846191}, {"head": 2, "memory_to_active": 0.11499512940645218, "contribution_to_active": 0.49360084533691406, "mix_mean": 0.5463681221008301, "mix_low": 0.0025968551635742188, "mix_high": 0.010660171508789062, "read_cosine": 0.45262277126312256, "beta_mean": 0.003670090576633811, "nonfinite": 0.0, "weighted_memory_to_active": 0.0784212127327919, "mix_std": 0.2827889919281006, "beta_std": 0.004212621133774519, "beta_low": 0.931640625, "beta_high": 0.0, "orthogonal_to_active": 0.01017518900334835, "evidence_weights": [-0.2720019519329071, -0.29568925499916077, 0.6358277201652527], "agreement_weight": 0.02749740704894066, "orthogonal_weight": -0.18021446466445923}, {"head": 3, "memory_to_active": 2.315805196762085, "contribution_to_active": 0.185170978307724, "mix_mean": 0.06065456569194794, "mix_low": 0.30693531036376953, "mix_high": 0.0, "read_cosine": 0.1752796173095703, "beta_mean": 0.08253030478954315, "nonfinite": 0.0, "weighted_memory_to_active": 0.3151821494102478, "mix_std": 0.11036346852779388, "beta_std": 0.12441948056221008, "beta_low": 0.19842529296875, "beta_high": 0.0, "orthogonal_to_active": 0.11604055017232895, "evidence_weights": [-0.0004304764443077147, -0.0462920106947422, 0.06191137433052063], "agreement_weight": 0.04492887482047081, "orthogonal_weight": -0.45943140983581543}, {"head": 4, "memory_to_active": 0.5230883359909058, "contribution_to_active": 0.6177794933319092, "mix_mean": 0.34310388565063477, "mix_low": 0.1566610336303711, "mix_high": 0.016910552978515625, "read_cosine": 0.5700870752334595, "beta_mean": 0.22854721546173096, "nonfinite": 0.0, "weighted_memory_to_active": 0.37294983863830566, "mix_std": 0.3502700626850128, "beta_std": 0.24697183072566986, "beta_low": 0.208984375, "beta_high": 0.0, "orthogonal_to_active": 0.09356586635112762, "evidence_weights": [-0.14075200259685516, -0.2718392014503479, 0.06352653354406357], "agreement_weight": 0.04602959379553795, "orthogonal_weight": 0.37461116909980774}, {"head": 5, "memory_to_active": 0.08281517028808594, "contribution_to_active": 0.5942509174346924, "mix_mean": 0.6950736045837402, "mix_low": 0.003939628601074219, "mix_high": 0.01707935333251953, "read_cosine": 0.24862413108348846, "beta_mean": 0.00028688000747933984, "nonfinite": 0.0, "weighted_memory_to_active": 0.05260854959487915, "mix_std": 0.2717181444168091, "beta_std": 0.0012338671367615461, "beta_low": 0.99652099609375, "beta_high": 0.0, "orthogonal_to_active": 0.02654477208852768, "evidence_weights": [-0.31330248713493347, -0.30130234360694885, 1.2656521797180176], "agreement_weight": 0.30801060795783997, "orthogonal_weight": -0.5508790612220764}]}, {"layer": 6, "memory_to_active": 0.447450190782547, "contribution_to_active": 0.5414636135101318, "mix_mean": 0.47161316871643066, "mix_low": 0.09544102847576141, "mix_high": 0.018210729584097862, "read_cosine": 0.30765995383262634, "beta_mean": 0.08271025121212006, "nonfinite": 0.0, "weighted_memory_to_active": 0.1714552938938141, "mix_std": 0.259150892496109, "beta_std": 0.09573160111904144, "beta_low": 0.4782715141773224, "beta_high": 0.0, "orthogonal_to_active": 0.037203848361968994, "output_influence": 0.4123138189315796, "residual_influence": 0.11949098855257034, "state_active": 0.05983613431453705, "state_memory": 0.0130417849868536, "state_peak": 62.30400466918945, "growth_active": 1.210086464881897, "growth_memory": 1.3214671611785889, "heads": [{"head": 0, "memory_to_active": 0.18238519132137299, "contribution_to_active": 0.5454922914505005, "mix_mean": 0.6084123849868774, "mix_low": 0.0003299713134765625, "mix_high": 0.0039806365966796875, "read_cosine": 0.11124984920024872, "beta_mean": 0.01810571551322937, "nonfinite": 0.0, "weighted_memory_to_active": 0.13696736097335815, "mix_std": 0.25824224948883057, "beta_std": 0.03660355135798454, "beta_low": 0.48626708984375, "beta_high": 0.0, "orthogonal_to_active": 0.09783320128917694, "evidence_weights": [-0.28325143456459045, -0.2785246670246124, 0.3634854555130005], "agreement_weight": 0.09697706252336502, "orthogonal_weight": -0.8371520042419434}, {"head": 1, "memory_to_active": 0.008741945028305054, "contribution_to_active": 0.4427497684955597, "mix_mean": 0.41911381483078003, "mix_low": 0.004113197326660156, "mix_high": 0.009654045104980469, "read_cosine": 0.6022681593894958, "beta_mean": 2.160502117476426e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029675003606826067, "mix_std": 0.29334568977355957, "beta_std": 5.824673280585557e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008525129524059594, "evidence_weights": [-0.21127349138259888, -0.2376597374677658, 0.5685657858848572], "agreement_weight": -0.022688530385494232, "orthogonal_weight": -0.3661606013774872}, {"head": 2, "memory_to_active": 1.5495295524597168, "contribution_to_active": 0.06684692203998566, "mix_mean": 0.02108139544725418, "mix_low": 0.5471735000610352, "mix_high": 0.0, "read_cosine": 0.1860044300556183, "beta_mean": 0.09661193192005157, "nonfinite": 0.0, "weighted_memory_to_active": 0.07521162927150726, "mix_std": 0.045563630759716034, "beta_std": 0.09392320364713669, "beta_low": 0.00665283203125, "beta_high": 0.0, "orthogonal_to_active": 0.0170437041670084, "evidence_weights": [0.08472856879234314, 0.04092105105519295, 0.12935948371887207], "agreement_weight": 0.05514123663306236, "orthogonal_weight": -0.2415032982826233}, {"head": 3, "memory_to_active": 0.17504140734672546, "contribution_to_active": 0.6210836172103882, "mix_mean": 0.5886831879615784, "mix_low": 0.016465187072753906, "mix_high": 0.016689300537109375, "read_cosine": 0.5437273383140564, "beta_mean": 0.2505475878715515, "nonfinite": 0.0, "weighted_memory_to_active": 0.11813727766275406, "mix_std": 0.3491179049015045, "beta_std": 0.2799433469772339, "beta_low": 0.161865234375, "beta_high": 0.0, "orthogonal_to_active": 0.04492092505097389, "evidence_weights": [-0.1719522327184677, -0.3302406072616577, 0.15491630136966705], "agreement_weight": -0.016193808987736702, "orthogonal_weight": 0.45908865332603455}, {"head": 4, "memory_to_active": 0.7201756238937378, "contribution_to_active": 1.037192702293396, "mix_mean": 0.6796201467514038, "mix_low": 0.0025882720947265625, "mix_high": 0.07806682586669922, "read_cosine": 0.14021117985248566, "beta_mean": 0.13059529662132263, "nonfinite": 0.0, "weighted_memory_to_active": 0.6659975051879883, "mix_std": 0.31788158416748047, "beta_std": 0.16316363215446472, "beta_low": 0.21539306640625, "beta_high": 0.0, "orthogonal_to_active": 0.03358851745724678, "evidence_weights": [-0.2438465654850006, -0.5316105484962463, -0.020557280629873276], "agreement_weight": -0.015724603086709976, "orthogonal_weight": 0.06077323481440544}, {"head": 5, "memory_to_active": 0.04882733151316643, "contribution_to_active": 0.535416305065155, "mix_mean": 0.5127677321434021, "mix_low": 0.00197601318359375, "mix_high": 0.000873565673828125, "read_cosine": 0.262498676776886, "beta_mean": 0.00037933149724267423, "nonfinite": 0.0, "weighted_memory_to_active": 0.029450472444295883, "mix_std": 0.2907542288303375, "beta_std": 0.0006976341246627271, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.028984200209379196, "evidence_weights": [-0.21483246982097626, -0.25568974018096924, 0.2667267322540283], "agreement_weight": 0.07140496373176575, "orthogonal_weight": -1.0515979528427124}]}, {"layer": 7, "memory_to_active": 2.2462472915649414, "contribution_to_active": 0.15736547112464905, "mix_mean": 0.032453350722789764, "mix_low": 0.5302761793136597, "mix_high": 4.609426014212659e-06, "read_cosine": 0.21723750233650208, "beta_mean": 0.14511807262897491, "nonfinite": 0.0, "weighted_memory_to_active": 0.1668393313884735, "mix_std": 0.05917450785636902, "beta_std": 0.1142241582274437, "beta_low": 0.01739501953125, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.03018338792026043, "output_influence": 0.1733189821243286, "residual_influence": 0.03696503862738609, "state_active": 0.01832096092402935, "state_memory": 0.033806998282670975, "state_peak": 15.307629585266113, "growth_active": 1.0437214374542236, "growth_memory": 1.4389138221740723, "heads": [{"head": 0, "memory_to_active": 2.090752601623535, "contribution_to_active": 0.26247459650039673, "mix_mean": 0.02899470180273056, "mix_low": 0.5250158309936523, "mix_high": 1.52587890625e-05, "read_cosine": 0.3890688419342041, "beta_mean": 0.1512073278427124, "nonfinite": 0.0, "weighted_memory_to_active": 0.23510269820690155, "mix_std": 0.062397658824920654, "beta_std": 0.11619707196950912, "beta_low": 0.00799560546875, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.04297765716910362, "evidence_weights": [-0.006759078241884708, -0.07975058257579803, 0.08851758390665054], "agreement_weight": -0.03973989188671112, "orthogonal_weight": 0.22523222863674164}, {"head": 1, "memory_to_active": 2.44339919090271, "contribution_to_active": 0.2657977342605591, "mix_mean": 0.07271881401538849, "mix_low": 0.3042783737182617, "mix_high": 0.0, "read_cosine": 0.06137324124574661, "beta_mean": 0.20397740602493286, "nonfinite": 0.0, "weighted_memory_to_active": 0.34159308671951294, "mix_std": 0.10740546882152557, "beta_std": 0.11410810053348541, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07357808947563171, "evidence_weights": [-0.11091826111078262, -0.09143121540546417, -0.055411312729120255], "agreement_weight": 0.01896228827536106, "orthogonal_weight": -0.222071573138237}, {"head": 2, "memory_to_active": 1.4207202196121216, "contribution_to_active": 0.25107261538505554, "mix_mean": 0.05616237595677376, "mix_low": 0.2983379364013672, "mix_high": 1.239776611328125e-05, "read_cosine": 0.13926354050636292, "beta_mean": 0.1377491056919098, "nonfinite": 0.0, "weighted_memory_to_active": 0.21290715038776398, "mix_std": 0.10664627701044083, "beta_std": 0.08276399970054626, "beta_low": 0.0047607421875, "beta_high": 0.0, "orthogonal_to_active": 0.012523546814918518, "evidence_weights": [-0.03626910224556923, -0.1102280244231224, 0.2022559940814972], "agreement_weight": 0.024565601721405983, "orthogonal_weight": -0.06174558401107788}, {"head": 3, "memory_to_active": 3.2368130683898926, "contribution_to_active": 0.045899856835603714, "mix_mean": 0.012766864150762558, "mix_low": 0.6598882675170898, "mix_high": 0.0, "read_cosine": 0.10916117578744888, "beta_mean": 0.1442558616399765, "nonfinite": 0.0, "weighted_memory_to_active": 0.08058550208806992, "mix_std": 0.030582252889871597, "beta_std": 0.14547614753246307, "beta_low": 0.02581787109375, "beta_high": 0.0, "orthogonal_to_active": 0.03935939073562622, "evidence_weights": [0.11801885068416595, 0.08347859233617783, 0.23716887831687927], "agreement_weight": 0.06815807521343231, "orthogonal_weight": -0.511629045009613}, {"head": 4, "memory_to_active": 1.8375427722930908, "contribution_to_active": 0.05526047945022583, "mix_mean": 0.012599479407072067, "mix_low": 0.6946706771850586, "mix_high": 0.0, "read_cosine": 0.317758709192276, "beta_mean": 0.15161371231079102, "nonfinite": 0.0, "weighted_memory_to_active": 0.05207178369164467, "mix_std": 0.026705436408519745, "beta_std": 0.1373523324728012, "beta_low": 0.02471923828125, "beta_high": 0.0, "orthogonal_to_active": 0.0031618664506822824, "evidence_weights": [0.07755202800035477, 0.03693544119596481, 0.023963017389178276], "agreement_weight": 0.04367959126830101, "orthogonal_weight": -0.06725872308015823}, {"head": 5, "memory_to_active": 2.448256492614746, "contribution_to_active": 0.06368743628263474, "mix_mean": 0.011477870866656303, "mix_low": 0.6994657516479492, "mix_high": 0.0, "read_cosine": 0.28679946064949036, "beta_mean": 0.08190494775772095, "nonfinite": 0.0, "weighted_memory_to_active": 0.0787757858633995, "mix_std": 0.02130996435880661, "beta_std": 0.0894472748041153, "beta_low": 0.04107666015625, "beta_high": 0.0, "orthogonal_to_active": 0.00949976034462452, "evidence_weights": [0.07373176515102386, 0.08455075323581696, -0.0473501943051815], "agreement_weight": 0.007990441285073757, "orthogonal_weight": -0.13797739148139954}]}, {"layer": 8, "memory_to_active": 0.730076253414154, "contribution_to_active": 0.42510631680488586, "mix_mean": 0.3109469413757324, "mix_low": 0.0859777182340622, "mix_high": 0.0007643700228072703, "read_cosine": 0.532396674156189, "beta_mean": 0.04925127327442169, "nonfinite": 0.0, "weighted_memory_to_active": 0.1651545763015747, "mix_std": 0.21621747314929962, "beta_std": 0.04776446521282196, "beta_low": 0.4007365107536316, "beta_high": 0.0, "orthogonal_to_active": 0.04567178711295128, "output_influence": 0.30255556106567383, "residual_influence": 0.07550733536481857, "state_active": 0.055502623319625854, "state_memory": 0.016976473852992058, "state_peak": 27.739585876464844, "growth_active": 1.0211704969406128, "growth_memory": 1.4104562997817993, "heads": [{"head": 0, "memory_to_active": 0.05930737033486366, "contribution_to_active": 0.49116918444633484, "mix_mean": 0.5207287669181824, "mix_low": 0.00014209747314453125, "mix_high": 0.0002994537353515625, "read_cosine": 0.737205982208252, "beta_mean": 0.002992564346641302, "nonfinite": 0.0, "weighted_memory_to_active": 0.032328978180885315, "mix_std": 0.23927336931228638, "beta_std": 0.005690212361514568, "beta_low": 0.94097900390625, "beta_high": 0.0, "orthogonal_to_active": 0.01546840276569128, "evidence_weights": [-0.2280357927083969, -0.32534706592559814, 0.6678473949432373], "agreement_weight": 0.0653064101934433, "orthogonal_weight": -0.7406826615333557}, {"head": 1, "memory_to_active": 0.5932122468948364, "contribution_to_active": 0.3355664908885956, "mix_mean": 0.3330886960029602, "mix_low": 0.006262779235839844, "mix_high": 0.0002231597900390625, "read_cosine": 0.5511478185653687, "beta_mean": 0.03773486614227295, "nonfinite": 0.0, "weighted_memory_to_active": 0.22840359807014465, "mix_std": 0.224510058760643, "beta_std": 0.03372729569673538, "beta_low": 0.103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.02573852799832821, "evidence_weights": [-0.16740885376930237, -0.18645209074020386, 0.35447585582733154], "agreement_weight": 0.05282726511359215, "orthogonal_weight": -0.14099691808223724}, {"head": 2, "memory_to_active": 0.21088334918022156, "contribution_to_active": 0.44474443793296814, "mix_mean": 0.3950875401496887, "mix_low": 0.0026378631591796875, "mix_high": 0.0003528594970703125, "read_cosine": 0.36518144607543945, "beta_mean": 0.005240381229668856, "nonfinite": 0.0, "weighted_memory_to_active": 0.10547398775815964, "mix_std": 0.25606387853622437, "beta_std": 0.007838701829314232, "beta_low": 0.86663818359375, "beta_high": 0.0, "orthogonal_to_active": 0.0732707530260086, "evidence_weights": [-0.17464834451675415, -0.2659238278865814, 0.40945035219192505], "agreement_weight": 0.04911193996667862, "orthogonal_weight": -0.7827528715133667}, {"head": 3, "memory_to_active": 0.3548508286476135, "contribution_to_active": 0.28853195905685425, "mix_mean": 0.26913711428642273, "mix_low": 0.0137786865234375, "mix_high": 1.9073486328125e-06, "read_cosine": 0.6942798495292664, "beta_mean": 0.02260986901819706, "nonfinite": 0.0, "weighted_memory_to_active": 0.15134648978710175, "mix_std": 0.20666025578975677, "beta_std": 0.02426483854651451, "beta_low": 0.253173828125, "beta_high": 0.0, "orthogonal_to_active": 0.0019720764830708504, "evidence_weights": [-0.08286821842193604, -0.1841239482164383, 0.6600626707077026], "agreement_weight": 0.07502444088459015, "orthogonal_weight": 0.02127796784043312}, {"head": 4, "memory_to_active": 2.9399735927581787, "contribution_to_active": 0.3294405937194824, "mix_mean": 0.026995522901415825, "mix_low": 0.4020814895629883, "mix_high": 0.0, "read_cosine": 0.3050925135612488, "beta_mean": 0.14533373713493347, "nonfinite": 0.0, "weighted_memory_to_active": 0.3189004957675934, "mix_std": 0.04284512996673584, "beta_std": 0.10051397234201431, "beta_low": 0.001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.018800342455506325, "evidence_weights": [0.023517882451415062, -0.004417248535901308, -0.10824453085660934], "agreement_weight": 0.015356375835835934, "orthogonal_weight": 0.06321971863508224}, {"head": 5, "memory_to_active": 0.22222957015037537, "contribution_to_active": 0.6611852645874023, "mix_mean": 0.32064390182495117, "mix_low": 0.09096336364746094, "mix_high": 0.0037088394165039062, "read_cosine": 0.5414719581604004, "beta_mean": 0.08159621059894562, "nonfinite": 0.0, "weighted_memory_to_active": 0.1544739156961441, "mix_std": 0.327952116727829, "beta_std": 0.11455173045396805, "beta_low": 0.2381591796875, "beta_high": 0.0, "orthogonal_to_active": 0.1387806087732315, "evidence_weights": [-0.07272479683160782, -0.33509498834609985, -0.41516464948654175], "agreement_weight": 0.018326779827475548, "orthogonal_weight": 1.113347053527832}]}, {"layer": 9, "memory_to_active": 1.2499618530273438, "contribution_to_active": 0.6012539863586426, "mix_mean": 0.21485644578933716, "mix_low": 0.1849726140499115, "mix_high": 0.0009897550335153937, "read_cosine": 0.35246890783309937, "beta_mean": 0.11560778319835663, "nonfinite": 0.0, "weighted_memory_to_active": 0.3958122730255127, "mix_std": 0.20977292954921722, "beta_std": 0.0580543577671051, "beta_low": 0.4966227412223816, "beta_high": 0.0, "orthogonal_to_active": 0.026273682713508606, "output_influence": 0.36245647072792053, "residual_influence": 0.08784731477499008, "state_active": 0.03888579085469246, "state_memory": 0.02776779979467392, "state_peak": 31.64754295349121, "growth_active": 1.1863436698913574, "growth_memory": 1.3918801546096802, "heads": [{"head": 0, "memory_to_active": 0.006374030373990536, "contribution_to_active": 0.4330783784389496, "mix_mean": 0.44927412271499634, "mix_low": 0.005779266357421875, "mix_high": 0.00313568115234375, "read_cosine": 0.22410428524017334, "beta_mean": 1.4363292393682059e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031680602114647627, "mix_std": 0.29469889402389526, "beta_std": 5.301389319356531e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002049965551123023, "evidence_weights": [-0.2437179833650589, -0.23356112837791443, 0.9417462944984436], "agreement_weight": 0.040434714406728745, "orthogonal_weight": -0.6763278841972351}, {"head": 1, "memory_to_active": 0.031165901571512222, "contribution_to_active": 0.42654353380203247, "mix_mean": 0.2780191898345947, "mix_low": 0.03535652160644531, "mix_high": 0.00087738037109375, "read_cosine": 0.4112674593925476, "beta_mean": 9.011197835206985e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01261834055185318, "mix_std": 0.2625768780708313, "beta_std": 0.00015934441762510687, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005277744494378567, "evidence_weights": [-0.173735573887825, -0.17589347064495087, 0.6604356169700623], "agreement_weight": 0.2946557104587555, "orthogonal_weight": -0.47661134600639343}, {"head": 2, "memory_to_active": 0.3034432828426361, "contribution_to_active": 0.4836914837360382, "mix_mean": 0.3429622948169708, "mix_low": 0.017980575561523438, "mix_high": 0.0011758804321289062, "read_cosine": 0.39806631207466125, "beta_mean": 0.0019262591376900673, "nonfinite": 0.0, "weighted_memory_to_active": 0.15553782880306244, "mix_std": 0.28317537903785706, "beta_std": 0.005383976269513369, "beta_low": 0.9791259765625, "beta_high": 0.0, "orthogonal_to_active": 0.03362478315830231, "evidence_weights": [-0.1351948231458664, -0.143766388297081, 0.3271016478538513], "agreement_weight": 0.14845609664916992, "orthogonal_weight": -0.27102288603782654}, {"head": 3, "memory_to_active": 2.0205211639404297, "contribution_to_active": 0.20449255406856537, "mix_mean": 0.04063282907009125, "mix_low": 0.3941326141357422, "mix_high": 0.0, "read_cosine": 0.4588642716407776, "beta_mean": 0.1417592614889145, "nonfinite": 0.0, "weighted_memory_to_active": 0.1724201738834381, "mix_std": 0.09545840322971344, "beta_std": 0.08896809816360474, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.026688488200306892, "evidence_weights": [0.037625715136528015, 0.02637888304889202, -0.13476453721523285], "agreement_weight": 0.044716160744428635, "orthogonal_weight": 0.15635764598846436}, {"head": 4, "memory_to_active": 3.165452003479004, "contribution_to_active": 1.4284669160842896, "mix_mean": 0.12266018241643906, "mix_low": 0.22716331481933594, "mix_high": 0.0007257461547851562, "read_cosine": 0.1943742036819458, "beta_mean": 0.2914559245109558, "nonfinite": 0.0, "weighted_memory_to_active": 1.3064429759979248, "mix_std": 0.20397436618804932, "beta_std": 0.12953823804855347, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0061040823347866535, "evidence_weights": [-0.04447653144598007, -0.11962569504976273, 0.06632824242115021], "agreement_weight": -0.04638336971402168, "orthogonal_weight": -0.004729578737169504}, {"head": 5, "memory_to_active": 1.9728138446807861, "contribution_to_active": 0.6312510967254639, "mix_mean": 0.05558987334370613, "mix_low": 0.42942333221435547, "mix_high": 2.384185791015625e-05, "read_cosine": 0.4281368851661682, "beta_mean": 0.25840073823928833, "nonfinite": 0.0, "weighted_memory_to_active": 0.7246861457824707, "mix_std": 0.11875360459089279, "beta_std": 0.12422347068786621, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.08389704674482346, "evidence_weights": [0.03885554522275925, -0.03985919430851936, 0.07163624465465546], "agreement_weight": 0.00030526265618391335, "orthogonal_weight": 0.15911434590816498}]}, {"layer": 10, "memory_to_active": 0.21333156526088715, "contribution_to_active": 0.4232412874698639, "mix_mean": 0.3736182153224945, "mix_low": 0.023395221680402756, "mix_high": 0.00991074275225401, "read_cosine": 0.38583120703697205, "beta_mean": 0.045721616595983505, "nonfinite": 0.0, "weighted_memory_to_active": 0.09583443403244019, "mix_std": 0.2619395852088928, "beta_std": 0.05148673802614212, "beta_low": 0.636322021484375, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.041627898812294006, "output_influence": 0.3767069876194, "residual_influence": 0.11216641962528229, "state_active": 0.06139282137155533, "state_memory": 0.015189450234174728, "state_peak": 98.47048950195312, "growth_active": 1.108066439628601, "growth_memory": 1.097609043121338, "heads": [{"head": 0, "memory_to_active": 0.4696463346481323, "contribution_to_active": 0.8249425888061523, "mix_mean": 0.6227494478225708, "mix_low": 0.018316268920898438, "mix_high": 0.05806541442871094, "read_cosine": 0.2055758535861969, "beta_mean": 0.19785892963409424, "nonfinite": 0.0, "weighted_memory_to_active": 0.3306218683719635, "mix_std": 0.3571312725543976, "beta_std": 0.21737723052501678, "beta_low": 0.12335205078125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.15304146707057953, "evidence_weights": [-0.16985560953617096, -0.3426066040992737, -0.24405598640441895], "agreement_weight": 0.026608968153595924, "orthogonal_weight": 0.5009440183639526}, {"head": 1, "memory_to_active": 0.002115224953740835, "contribution_to_active": 0.4682812988758087, "mix_mean": 0.44531115889549255, "mix_low": 0.0035963058471679688, "mix_high": 0.0010509490966796875, "read_cosine": 0.2190287560224533, "beta_mean": 1.2311926184338517e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0009940159507095814, "mix_std": 0.26523810625076294, "beta_std": 2.8700005714199506e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006553098792210221, "evidence_weights": [-0.24935416877269745, -0.17886726558208466, 0.2394941747188568], "agreement_weight": 0.09171617776155472, "orthogonal_weight": -0.7200819253921509}, {"head": 2, "memory_to_active": 0.32140281796455383, "contribution_to_active": 0.27283820509910583, "mix_mean": 0.20954661071300507, "mix_low": 0.06319332122802734, "mix_high": 9.72747802734375e-05, "read_cosine": 0.11126499623060226, "beta_mean": 0.0043456051498651505, "nonfinite": 0.0, "weighted_memory_to_active": 0.09777101874351501, "mix_std": 0.23359955847263336, "beta_std": 0.013483029790222645, "beta_low": 0.90582275390625, "beta_high": 0.0, "orthogonal_to_active": 0.02449888549745083, "evidence_weights": [-0.06925520300865173, -0.14300504326820374, 0.2811776101589203], "agreement_weight": 0.03150712326169014, "orthogonal_weight": -0.25432831048965454}, {"head": 3, "memory_to_active": 0.2980301082134247, "contribution_to_active": 0.13239501416683197, "mix_mean": 0.2558317184448242, "mix_low": 0.045833587646484375, "mix_high": 0.000209808349609375, "read_cosine": 0.8629076480865479, "beta_mean": 0.06456774473190308, "nonfinite": 0.0, "weighted_memory_to_active": 0.057014673948287964, "mix_std": 0.24472731351852417, "beta_std": 0.06478224694728851, "beta_low": 0.0103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.04994574561715126, "evidence_weights": [-0.12324152886867523, -0.14437322318553925, 0.23731327056884766], "agreement_weight": -0.003445639507845044, "orthogonal_weight": 1.1536262035369873}, {"head": 4, "memory_to_active": 0.03475422412157059, "contribution_to_active": 0.404490202665329, "mix_mean": 0.3719750642776489, "mix_low": 0.0032815933227539062, "mix_high": 2.86102294921875e-05, "read_cosine": 0.0947650820016861, "beta_mean": 8.8056709500961e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00912334956228733, "mix_std": 0.24968937039375305, "beta_std": 0.0027012070640921593, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0023247951176017523, "evidence_weights": [-0.17525027692317963, -0.17591457068920135, 0.20576345920562744], "agreement_weight": 0.012777122668921947, "orthogonal_weight": -0.26344379782676697}, {"head": 5, "memory_to_active": 0.1540406495332718, "contribution_to_active": 0.4365004301071167, "mix_mean": 0.33629536628723145, "mix_low": 0.006150245666503906, "mix_high": 1.239776611328125e-05, "read_cosine": 0.8214448690414429, "beta_mean": 0.0074570393189787865, "nonfinite": 0.0, "weighted_memory_to_active": 0.07948161661624908, "mix_std": 0.22125199437141418, "beta_std": 0.010548016987740993, "beta_low": 0.778564453125, "beta_high": 0.0, "orthogonal_to_active": 0.01930117979645729, "evidence_weights": [-0.18256035447120667, -0.16801223158836365, 0.25930672883987427], "agreement_weight": 0.09187275171279907, "orthogonal_weight": -0.5452929139137268}]}, {"layer": 11, "memory_to_active": 0.2559329867362976, "contribution_to_active": 0.4421255886554718, "mix_mean": 0.3701077103614807, "mix_low": 0.0487416610121727, "mix_high": 0.017167888581752777, "read_cosine": 0.5174676179885864, "beta_mean": 0.04080778360366821, "nonfinite": 0.0, "weighted_memory_to_active": 0.08603641390800476, "mix_std": 0.29555743932724, "beta_std": 0.058732569217681885, "beta_low": 0.4879862666130066, "beta_high": 0.0, "orthogonal_to_active": 0.03925830125808716, "output_influence": 0.4600302577018738, "residual_influence": 0.1231805831193924, "state_active": 0.11945371329784393, "state_memory": 0.012377905659377575, "state_peak": 28.703657150268555, "growth_active": 1.1879905462265015, "growth_memory": 1.3940156698226929, "heads": [{"head": 0, "memory_to_active": 0.027882393449544907, "contribution_to_active": 0.5239346027374268, "mix_mean": 0.5072965025901794, "mix_low": 0.0031347274780273438, "mix_high": 0.0048160552978515625, "read_cosine": 0.4219439923763275, "beta_mean": 0.00015580114268232137, "nonfinite": 0.0, "weighted_memory_to_active": 0.017234040424227715, "mix_std": 0.27842622995376587, "beta_std": 0.004269979428499937, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.00909773912280798, "evidence_weights": [-0.16216391324996948, -0.3087652325630188, 0.508064329624176], "agreement_weight": 0.1289113312959671, "orthogonal_weight": -0.6529434323310852}, {"head": 1, "memory_to_active": 0.5085689425468445, "contribution_to_active": 0.22692611813545227, "mix_mean": 0.22842547297477722, "mix_low": 0.034752845764160156, "mix_high": 6.67572021484375e-05, "read_cosine": 0.6176661252975464, "beta_mean": 0.041598279029130936, "nonfinite": 0.0, "weighted_memory_to_active": 0.16794034838676453, "mix_std": 0.22584375739097595, "beta_std": 0.0727432519197464, "beta_low": 0.345703125, "beta_high": 0.0, "orthogonal_to_active": 0.02042800560593605, "evidence_weights": [-0.15322378277778625, -0.10747363418340683, 0.13794933259487152], "agreement_weight": 0.08900196105241776, "orthogonal_weight": -0.17213602364063263}, {"head": 2, "memory_to_active": 0.05795089900493622, "contribution_to_active": 0.8454983234405518, "mix_mean": 0.47843313217163086, "mix_low": 0.04586601257324219, "mix_high": 0.058078765869140625, "read_cosine": 0.3294125199317932, "beta_mean": 0.07965932786464691, "nonfinite": 0.0, "weighted_memory_to_active": 0.04828869551420212, "mix_std": 0.40267616510391235, "beta_std": 0.10827073454856873, "beta_low": 0.4326171875, "beta_high": 0.0, "orthogonal_to_active": 0.09936549514532089, "evidence_weights": [-0.14027416706085205, -0.4211680293083191, -0.6938512921333313], "agreement_weight": -0.007335647940635681, "orthogonal_weight": 2.224207878112793}, {"head": 3, "memory_to_active": 0.030139977112412453, "contribution_to_active": 0.44439464807510376, "mix_mean": 0.4353821277618408, "mix_low": 0.12086009979248047, "mix_high": 0.039508819580078125, "read_cosine": 0.5572899580001831, "beta_mean": 0.07049810886383057, "nonfinite": 0.0, "weighted_memory_to_active": 0.01673370972275734, "mix_std": 0.41561296582221985, "beta_std": 0.10115986317396164, "beta_low": 0.44635009765625, "beta_high": 0.0, "orthogonal_to_active": 0.03292320668697357, "evidence_weights": [-0.1502292901277542, -0.3679918050765991, -0.22441576421260834], "agreement_weight": 0.020053155720233917, "orthogonal_weight": 2.4204604625701904}, {"head": 4, "memory_to_active": 0.7225123047828674, "contribution_to_active": 0.2464604526758194, "mix_mean": 0.16746430099010468, "mix_low": 0.08682823181152344, "mix_high": 0.0, "read_cosine": 0.2715538740158081, "beta_mean": 0.036797814071178436, "nonfinite": 0.0, "weighted_memory_to_active": 0.1780048906803131, "mix_std": 0.1935240924358368, "beta_std": 0.04160679876804352, "beta_low": 0.15789794921875, "beta_high": 0.0, "orthogonal_to_active": 0.05294756963849068, "evidence_weights": [-0.07951091229915619, -0.06239490583539009, 0.9176601767539978], "agreement_weight": -0.07874701172113419, "orthogonal_weight": -0.32958319783210754}, {"head": 5, "memory_to_active": 0.1885434091091156, "contribution_to_active": 0.3655393421649933, "mix_mean": 0.40364453196525574, "mix_low": 0.0010080337524414062, "mix_high": 0.0005369186401367188, "read_cosine": 0.9069390296936035, "beta_mean": 0.016137342900037766, "nonfinite": 0.0, "weighted_memory_to_active": 0.0880168154835701, "mix_std": 0.25726136565208435, "beta_std": 0.0243447907269001, "beta_low": 0.54608154296875, "beta_high": 0.0, "orthogonal_to_active": 0.020787788555026054, "evidence_weights": [-0.19107457995414734, -0.23137368261814117, 0.36823490262031555], "agreement_weight": 0.04185526445508003, "orthogonal_weight": -0.5590891242027283}]}, {"layer": 12, "memory_to_active": 0.4878060519695282, "contribution_to_active": 0.49536019563674927, "mix_mean": 0.3788709044456482, "mix_low": 0.04263639450073242, "mix_high": 0.0014200210571289062, "read_cosine": 0.2893008589744568, "beta_mean": 0.020197134464979172, "nonfinite": 0.0, "weighted_memory_to_active": 0.21779516339302063, "mix_std": 0.23711101710796356, "beta_std": 0.022899959236383438, "beta_low": 0.7554627060890198, "beta_high": 0.0, "orthogonal_to_active": 0.06565335392951965, "output_influence": 0.3431144058704376, "residual_influence": 0.09210344403982162, "state_active": 0.04680834338068962, "state_memory": 0.014345760457217693, "state_peak": 32.15244674682617, "growth_active": 1.1122063398361206, "growth_memory": 1.2077159881591797, "heads": [{"head": 0, "memory_to_active": 0.3249261975288391, "contribution_to_active": 0.3218684196472168, "mix_mean": 0.293967068195343, "mix_low": 0.017577171325683594, "mix_high": 5.626678466796875e-05, "read_cosine": 0.34229326248168945, "beta_mean": 0.005036074668169022, "nonfinite": 0.0, "weighted_memory_to_active": 0.12698958814144135, "mix_std": 0.23887278139591217, "beta_std": 0.0170248094946146, "beta_low": 0.8900146484375, "beta_high": 0.0, "orthogonal_to_active": 0.04810824245214462, "evidence_weights": [-0.13717453181743622, -0.19304010272026062, 0.24321956932544708], "agreement_weight": 0.10120163857936859, "orthogonal_weight": -0.4845522344112396}, {"head": 1, "memory_to_active": 0.1636539101600647, "contribution_to_active": 0.41745564341545105, "mix_mean": 0.4073217511177063, "mix_low": 0.0008335113525390625, "mix_high": 0.00010776519775390625, "read_cosine": 0.5813779830932617, "beta_mean": 0.0017730314284563065, "nonfinite": 0.0, "weighted_memory_to_active": 0.07360392808914185, "mix_std": 0.2527778744697571, "beta_std": 0.006242981180548668, "beta_low": 0.97393798828125, "beta_high": 0.0, "orthogonal_to_active": 0.04938998073339462, "evidence_weights": [-0.1803647130727768, -0.22819602489471436, 0.6445836424827576], "agreement_weight": 0.08832941949367523, "orthogonal_weight": -0.8642474412918091}, {"head": 2, "memory_to_active": 0.6630274057388306, "contribution_to_active": 0.40631863474845886, "mix_mean": 0.30491816997528076, "mix_low": 0.02462482452392578, "mix_high": 0.0003452301025390625, "read_cosine": 0.38801851868629456, "beta_mean": 0.10049004852771759, "nonfinite": 0.0, "weighted_memory_to_active": 0.26873618364334106, "mix_std": 0.24389013648033142, "beta_std": 0.07960167527198792, "beta_low": 0.0020751953125, "beta_high": 0.0, "orthogonal_to_active": 0.14655154943466187, "evidence_weights": [-0.22499017417430878, -0.060205910354852676, -0.0032373906578868628], "agreement_weight": 0.040481213480234146, "orthogonal_weight": 0.6808385848999023}, {"head": 3, "memory_to_active": 0.02584179863333702, "contribution_to_active": 0.6526749730110168, "mix_mean": 0.6304729580879211, "mix_low": 0.0006504058837890625, "mix_high": 0.006396293640136719, "read_cosine": 0.045298971235752106, "beta_mean": 0.00018168953829444945, "nonfinite": 0.0, "weighted_memory_to_active": 0.016922147944569588, "mix_std": 0.2524072527885437, "beta_std": 0.00045041291741654277, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.014043549075722694, "evidence_weights": [-0.30416423082351685, -0.30019497871398926, 0.6088347434997559], "agreement_weight": 0.17530424892902374, "orthogonal_weight": -0.8523346781730652}, {"head": 4, "memory_to_active": 1.6985197067260742, "contribution_to_active": 0.7598016858100891, "mix_mean": 0.1096327155828476, "mix_low": 0.2062978744506836, "mix_high": 0.0008573532104492188, "read_cosine": 0.016815874725580215, "beta_mean": 0.013192763552069664, "nonfinite": 0.0, "weighted_memory_to_active": 0.7969158887863159, "mix_std": 0.16316130757331848, "beta_std": 0.03142480552196503, "beta_low": 0.6748046875, "beta_high": 0.0, "orthogonal_to_active": 0.11778557300567627, "evidence_weights": [0.031797342002391815, -0.10890591889619827, 0.009827119298279285], "agreement_weight": -0.005311590153723955, "orthogonal_weight": -0.16199137270450592}, {"head": 5, "memory_to_active": 0.050867222249507904, "contribution_to_active": 0.41404181718826294, "mix_mean": 0.5269126892089844, "mix_low": 0.0058345794677734375, "mix_high": 0.0007572174072265625, "read_cosine": 0.3620004653930664, "beta_mean": 0.0005091953789815307, "nonfinite": 0.0, "weighted_memory_to_active": 0.02360331267118454, "mix_std": 0.27155664563179016, "beta_std": 0.0026550658512860537, "beta_low": 0.99224853515625, "beta_high": 0.0, "orthogonal_to_active": 0.018041178584098816, "evidence_weights": [-0.26118898391723633, -0.27657273411750793, 0.6612393856048584], "agreement_weight": 0.07052012532949448, "orthogonal_weight": -0.8257520794868469}]}, {"layer": 13, "memory_to_active": 0.22515356540679932, "contribution_to_active": 0.4421493113040924, "mix_mean": 0.43636322021484375, "mix_low": 0.004670619964599609, "mix_high": 0.0007762909517623484, "read_cosine": 0.3870072662830353, "beta_mean": 0.019964318722486496, "nonfinite": 0.0, "weighted_memory_to_active": 0.10199515521526337, "mix_std": 0.2555275857448578, "beta_std": 0.02112565189599991, "beta_low": 0.7499593496322632, "beta_high": 0.0, "orthogonal_to_active": 0.0812661200761795, "output_influence": 0.39093390107154846, "residual_influence": 0.12580335140228271, "state_active": 0.08348627388477325, "state_memory": 0.03639547526836395, "state_peak": 81.13774871826172, "growth_active": 0.9969364404678345, "growth_memory": 1.0582321882247925, "heads": [{"head": 0, "memory_to_active": 0.17087385058403015, "contribution_to_active": 0.44068941473960876, "mix_mean": 0.4237739145755768, "mix_low": 0.0021390914916992188, "mix_high": 0.0007486343383789062, "read_cosine": 0.32430389523506165, "beta_mean": 0.0016649243189021945, "nonfinite": 0.0, "weighted_memory_to_active": 0.07977220416069031, "mix_std": 0.26012012362480164, "beta_std": 0.003649048274382949, "beta_low": 0.96917724609375, "beta_high": 0.0, "orthogonal_to_active": 0.07325872033834457, "evidence_weights": [-0.19908198714256287, -0.22055527567863464, 0.671430766582489], "agreement_weight": 0.044898174703121185, "orthogonal_weight": -1.0055944919586182}, {"head": 1, "memory_to_active": 0.09252499043941498, "contribution_to_active": 0.4932188391685486, "mix_mean": 0.4224311411380768, "mix_low": 0.0031681060791015625, "mix_high": 5.340576171875e-05, "read_cosine": 0.25392115116119385, "beta_mean": 0.0004666040767915547, "nonfinite": 0.0, "weighted_memory_to_active": 0.04595326632261276, "mix_std": 0.2595876157283783, "beta_std": 0.0009421154391020536, "beta_low": 0.99884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.04139280319213867, "evidence_weights": [-0.18820440769195557, -0.2293107658624649, 0.7213698029518127], "agreement_weight": 0.09780319035053253, "orthogonal_weight": -0.9545263051986694}, {"head": 2, "memory_to_active": 0.24105031788349152, "contribution_to_active": 0.3062538802623749, "mix_mean": 0.318736732006073, "mix_low": 0.0026712417602539062, "mix_high": 1.621246337890625e-05, "read_cosine": 0.7002169489860535, "beta_mean": 0.022111253812909126, "nonfinite": 0.0, "weighted_memory_to_active": 0.08824850618839264, "mix_std": 0.21306560933589935, "beta_std": 0.045334309339523315, "beta_low": 0.58404541015625, "beta_high": 0.0, "orthogonal_to_active": 0.03806975483894348, "evidence_weights": [-0.2102535367012024, -0.1500772088766098, 0.4647315442562103], "agreement_weight": 0.04904231056571007, "orthogonal_weight": -0.6666833758354187}, {"head": 3, "memory_to_active": 0.16967740654945374, "contribution_to_active": 0.5214991569519043, "mix_mean": 0.535783052444458, "mix_low": 0.005753517150878906, "mix_high": 0.0016813278198242188, "read_cosine": -0.06075979769229889, "beta_mean": 0.0009412539075128734, "nonfinite": 0.0, "weighted_memory_to_active": 0.11585155874490738, "mix_std": 0.2593079209327698, "beta_std": 0.004559469874948263, "beta_low": 0.99066162109375, "beta_high": 0.0, "orthogonal_to_active": 0.11892338842153549, "evidence_weights": [-0.34674423933029175, -0.22987492382526398, 1.4195369482040405], "agreement_weight": 1.2487428188323975, "orthogonal_weight": -1.0376205444335938}, {"head": 4, "memory_to_active": 0.4768422842025757, "contribution_to_active": 0.3313576281070709, "mix_mean": 0.4469209909439087, "mix_low": 0.007841110229492188, "mix_high": 0.0009641647338867188, "read_cosine": 0.8337834477424622, "beta_mean": 0.0924495980143547, "nonfinite": 0.0, "weighted_memory_to_active": 0.17462997138500214, "mix_std": 0.25981244444847107, "beta_std": 0.06544946134090424, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.15048952400684357, "evidence_weights": [-0.2477179914712906, -0.24972066283226013, 0.47184887528419495], "agreement_weight": 0.002329308306798339, "orthogonal_weight": 1.2164018154144287}, {"head": 5, "memory_to_active": 0.19995248317718506, "contribution_to_active": 0.5598766803741455, "mix_mean": 0.47053343057632446, "mix_low": 0.006450653076171875, "mix_high": 0.001194000244140625, "read_cosine": 0.2705778181552887, "beta_mean": 0.0021522792521864176, "nonfinite": 0.0, "weighted_memory_to_active": 0.10751538723707199, "mix_std": 0.28127169609069824, "beta_std": 0.006819506175816059, "beta_low": 0.9569091796875, "beta_high": 0.0, "orthogonal_to_active": 0.06546253710985184, "evidence_weights": [-0.20931603014469147, -0.24457688629627228, 0.37504488229751587], "agreement_weight": 0.06538040935993195, "orthogonal_weight": -0.6857102513313293}]}, {"layer": 14, "memory_to_active": 0.27707192301750183, "contribution_to_active": 0.4257623553276062, "mix_mean": 0.3872920870780945, "mix_low": 0.022620998322963715, "mix_high": 0.0028826396446675062, "read_cosine": 0.6233025193214417, "beta_mean": 0.07469961047172546, "nonfinite": 0.0, "weighted_memory_to_active": 0.13132700324058533, "mix_std": 0.24907729029655457, "beta_std": 0.06965426355600357, "beta_low": 0.41693115234375, "beta_high": 0.0, "orthogonal_to_active": 0.03959466516971588, "output_influence": 0.25871583819389343, "residual_influence": 0.10034263134002686, "state_active": 0.08828791975975037, "state_memory": 0.01801268756389618, "state_peak": 22.87795066833496, "growth_active": 1.2505477666854858, "growth_memory": 1.3230500221252441, "heads": [{"head": 0, "memory_to_active": 0.3064783215522766, "contribution_to_active": 0.25499558448791504, "mix_mean": 0.2507617473602295, "mix_low": 0.01816082000732422, "mix_high": 5.14984130859375e-05, "read_cosine": 0.7013014554977417, "beta_mean": 0.07043401896953583, "nonfinite": 0.0, "weighted_memory_to_active": 0.10289451479911804, "mix_std": 0.22623133659362793, "beta_std": 0.09007776528596878, "beta_low": 0.12030029296875, "beta_high": 0.0, "orthogonal_to_active": 0.020332399755716324, "evidence_weights": [-0.05948305130004883, -0.1202303096652031, 0.7290782332420349], "agreement_weight": 0.0467870868742466, "orthogonal_weight": 0.28483107686042786}, {"head": 1, "memory_to_active": 0.4674850404262543, "contribution_to_active": 0.4511190950870514, "mix_mean": 0.2874763011932373, "mix_low": 0.08799171447753906, "mix_high": 0.0005502700805664062, "read_cosine": 0.6062027215957642, "beta_mean": 0.31171005964279175, "nonfinite": 0.0, "weighted_memory_to_active": 0.2373766005039215, "mix_std": 0.2788796126842499, "beta_std": 0.2663520574569702, "beta_low": 0.07177734375, "beta_high": 0.0, "orthogonal_to_active": 0.019644463434815407, "evidence_weights": [-0.005807607900351286, -0.2680850028991699, -0.009531697258353233], "agreement_weight": 0.05379477143287659, "orthogonal_weight": 0.12537451088428497}, {"head": 2, "memory_to_active": 0.28041872382164, "contribution_to_active": 0.4610847532749176, "mix_mean": 0.46653565764427185, "mix_low": 0.004776954650878906, "mix_high": 0.0012273788452148438, "read_cosine": 0.42803138494491577, "beta_mean": 0.009248621761798859, "nonfinite": 0.0, "weighted_memory_to_active": 0.1542423963546753, "mix_std": 0.26879164576530457, "beta_std": 0.011108105070888996, "beta_low": 0.70623779296875, "beta_high": 0.0, "orthogonal_to_active": 0.10713169723749161, "evidence_weights": [-0.19484412670135498, -0.2325417846441269, 0.6353434324264526], "agreement_weight": 0.08339473605155945, "orthogonal_weight": -0.7980812788009644}, {"head": 3, "memory_to_active": 0.06251125782728195, "contribution_to_active": 0.652390718460083, "mix_mean": 0.6387501955032349, "mix_low": 0.0023746490478515625, "mix_high": 0.009830474853515625, "read_cosine": 0.49933522939682007, "beta_mean": 0.009658736176788807, "nonfinite": 0.0, "weighted_memory_to_active": 0.042707111686468124, "mix_std": 0.24200548231601715, "beta_std": 0.008700273931026459, "beta_low": 0.65203857421875, "beta_high": 0.0, "orthogonal_to_active": 0.019627559930086136, "evidence_weights": [-0.37887778878211975, -0.32654663920402527, 0.41804811358451843], "agreement_weight": 0.07486697286367416, "orthogonal_weight": -0.6022868752479553}, {"head": 4, "memory_to_active": 0.19245073199272156, "contribution_to_active": 0.4692856967449188, "mix_mean": 0.429487943649292, "mix_low": 0.003940582275390625, "mix_high": 0.005356788635253906, "read_cosine": 0.8022371530532837, "beta_mean": 0.043241843581199646, "nonfinite": 0.0, "weighted_memory_to_active": 0.1067107692360878, "mix_std": 0.2554941177368164, "beta_std": 0.02947741001844406, "beta_low": 0.038330078125, "beta_high": 0.0, "orthogonal_to_active": 0.023300569504499435, "evidence_weights": [-0.2159949541091919, -0.26506924629211426, 0.2908779978752136], "agreement_weight": 0.10749261826276779, "orthogonal_weight": -0.3934323191642761}, {"head": 5, "memory_to_active": 0.35308748483657837, "contribution_to_active": 0.2656981348991394, "mix_mean": 0.2507407069206238, "mix_low": 0.01848125457763672, "mix_high": 0.00027942657470703125, "read_cosine": 0.7027069330215454, "beta_mean": 0.0039044152945280075, "nonfinite": 0.0, "weighted_memory_to_active": 0.14403057098388672, "mix_std": 0.22306153178215027, "beta_std": 0.012209922075271606, "beta_low": 0.91290283203125, "beta_high": 0.0, "orthogonal_to_active": 0.04753126576542854, "evidence_weights": [-0.18557745218276978, -0.1349588930606842, 0.1282041072845459], "agreement_weight": 0.09551060944795609, "orthogonal_weight": -0.5441259741783142}]}, {"layer": 15, "memory_to_active": 0.0821119099855423, "contribution_to_active": 0.46709144115448, "mix_mean": 0.3726785182952881, "mix_low": 0.007586638443171978, "mix_high": 0.002333959098905325, "read_cosine": 0.658798098564148, "beta_mean": 0.04310036823153496, "nonfinite": 0.0, "weighted_memory_to_active": 0.03752341866493225, "mix_std": 0.270112007856369, "beta_std": 0.04668877646327019, "beta_low": 0.7120565176010132, "beta_high": 0.0, "orthogonal_to_active": 0.007940519601106644, "output_influence": 0.287753164768219, "residual_influence": 0.15324464440345764, "state_active": 0.11621522158384323, "state_memory": 0.00854222197085619, "state_peak": 31.878551483154297, "growth_active": 1.3908699750900269, "growth_memory": 1.5785555839538574, "heads": [{"head": 0, "memory_to_active": 0.1033555269241333, "contribution_to_active": 0.4759150743484497, "mix_mean": 0.33724820613861084, "mix_low": 0.006749153137207031, "mix_high": 0.0022325515747070312, "read_cosine": 0.7276596426963806, "beta_mean": 0.033146634697914124, "nonfinite": 0.0, "weighted_memory_to_active": 0.04416083171963692, "mix_std": 0.2548421621322632, "beta_std": 0.026703305542469025, "beta_low": 0.17034912109375, "beta_high": 0.0, "orthogonal_to_active": 0.012478677555918694, "evidence_weights": [-0.16543126106262207, -0.20061592757701874, 0.3507968783378601], "agreement_weight": 0.08206132054328918, "orthogonal_weight": -0.4867820739746094}, {"head": 1, "memory_to_active": 0.2076173573732376, "contribution_to_active": 0.4057214856147766, "mix_mean": 0.37981313467025757, "mix_low": 0.006775856018066406, "mix_high": 0.005465507507324219, "read_cosine": 0.8506262302398682, "beta_mean": 0.001939177280291915, "nonfinite": 0.0, "weighted_memory_to_active": 0.08637959510087967, "mix_std": 0.2726747691631317, "beta_std": 0.004917497280985117, "beta_low": 0.9716796875, "beta_high": 0.0, "orthogonal_to_active": 0.006166214123368263, "evidence_weights": [-0.19619996845722198, -0.20012471079826355, 0.28365758061408997], "agreement_weight": 0.09568712115287781, "orthogonal_weight": -0.11771872639656067}, {"head": 2, "memory_to_active": 0.0076390719041228294, "contribution_to_active": 0.42664507031440735, "mix_mean": 0.3282998502254486, "mix_low": 0.013031959533691406, "mix_high": 0.00016117095947265625, "read_cosine": 0.75232994556427, "beta_mean": 0.00014641402231063694, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029164371080696583, "mix_std": 0.26456117630004883, "beta_std": 0.0003106754447799176, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0006535325665026903, "evidence_weights": [-0.21113687753677368, -0.27062660455703735, 0.2946699857711792], "agreement_weight": 0.037540797144174576, "orthogonal_weight": -0.33744025230407715}, {"head": 3, "memory_to_active": 0.06292160600423813, "contribution_to_active": 0.45766681432724, "mix_mean": 0.29438066482543945, "mix_low": 0.0038595199584960938, "mix_high": 3.0517578125e-05, "read_cosine": 0.6450201272964478, "beta_mean": 0.0018100730376318097, "nonfinite": 0.0, "weighted_memory_to_active": 0.030252903699874878, "mix_std": 0.23822727799415588, "beta_std": 0.004335435107350349, "beta_low": 0.96905517578125, "beta_high": 0.0, "orthogonal_to_active": 0.015059198252856731, "evidence_weights": [-0.17874972522258759, -0.22182638943195343, 0.40780535340309143], "agreement_weight": 0.013314318843185902, "orthogonal_weight": -0.7590140104293823}, {"head": 4, "memory_to_active": 0.10553500056266785, "contribution_to_active": 0.5810051560401917, "mix_mean": 0.4033159017562866, "mix_low": 0.010748863220214844, "mix_high": 0.00091552734375, "read_cosine": 0.4312583804130554, "beta_mean": 0.22141580283641815, "nonfinite": 0.0, "weighted_memory_to_active": 0.059097371995449066, "mix_std": 0.30894553661346436, "beta_std": 0.24361078441143036, "beta_low": 0.161376953125, "beta_high": 0.0, "orthogonal_to_active": 0.011755358427762985, "evidence_weights": [-0.26955172419548035, -0.2536270320415497, 0.2797279953956604], "agreement_weight": 0.008038666099309921, "orthogonal_weight": -0.2683545649051666}, {"head": 5, "memory_to_active": 0.005602877587080002, "contribution_to_active": 0.4555950164794922, "mix_mean": 0.49301332235336304, "mix_low": 0.0043544769287109375, "mix_high": 0.005198478698730469, "read_cosine": 0.5458940267562866, "beta_mean": 0.00014409393770620227, "nonfinite": 0.0, "weighted_memory_to_active": 0.0023333781864494085, "mix_std": 0.28142112493515015, "beta_std": 0.00025496893795207143, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015301341190934181, "evidence_weights": [-0.361427903175354, -0.297874391078949, 0.4059310555458069], "agreement_weight": 0.1411837339401245, "orthogonal_weight": 0.7789737582206726}]}]}, {"step": 15000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9034390020024148, "memory_to_active": 0.9821349382400513, "contribution_to_active": 0.442621648311615, "mix_mean": 0.29593363404273987, "mix_low": 0.17612376809120178, "mix_high": 0.004868318792432547, "read_cosine": 0.39752626419067383, "beta_mean": 0.09486111998558044, "nonfinite": 0.0, "weighted_memory_to_active": 0.20267556607723236, "mix_std": 0.20685747265815735, "beta_std": 0.07495409995317459, "beta_low": 0.45554351806640625, "beta_high": 1.2715658158413135e-05, "orthogonal_to_active": 0.03906784951686859, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1863861083984375, "normal_token_accuracy": 0.42046719789505005, "state_active": 0.06052876263856888, "state_memory": 0.024000713601708412, "state_peak": 102.85000610351562, "growth_active": 1.1183770895004272, "growth_memory": 1.3909839391708374, "output_influence": 0.3013461232185364, "residual_influence": 0.09470972418785095, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2466495037078857, "no_memory_loss_delta": 0.16267816722393036, "scrambled_loss_delta": 0.1019967794418335, "memory_only_loss_delta": 2.2136294841766357, "no_evidence_loss_delta": 0.07516062259674072, "no_agreement_loss_delta": 0.004675771575421095, "no_orthogonal_loss_delta": 0.0030066235922276974}, {"index": 94, "tokens": 256, "normal_loss": 3.304658889770508, "no_memory_loss_delta": 0.33164289593696594, "scrambled_loss_delta": 0.49041080474853516, "memory_only_loss_delta": 2.1957480907440186, "no_evidence_loss_delta": 0.15051332116127014, "no_agreement_loss_delta": 0.028501320630311966, "no_orthogonal_loss_delta": 0.02519887685775757}, {"index": 188, "tokens": 256, "normal_loss": 3.605459451675415, "no_memory_loss_delta": 0.17126567661762238, "scrambled_loss_delta": 0.1306571364402771, "memory_only_loss_delta": 2.0657525062561035, "no_evidence_loss_delta": 0.10493510961532593, "no_agreement_loss_delta": -0.013192553073167801, "no_orthogonal_loss_delta": 0.00681539298966527}, {"index": 283, "tokens": 131, "normal_loss": 2.0185391902923584, "no_memory_loss_delta": 0.19477252662181854, "scrambled_loss_delta": 0.39635804295539856, "memory_only_loss_delta": 2.608499526977539, "no_evidence_loss_delta": 0.05748285725712776, "no_agreement_loss_delta": 0.011671965010464191, "no_orthogonal_loss_delta": -0.0011816684855148196}], "agreement_weight_abs_mean": 0.08233169466257095, "agreement_weight_abs_max": 1.3838229179382324, "agreement_weight_mean": 0.06656656414270401, "orthogonal_weight_abs_mean": 0.4646191895008087, "orthogonal_weight_abs_max": 2.609370470046997, "orthogonal_weight_mean": -0.16511106491088867, "no_memory_loss": 3.4043006896972656, "no_memory_loss_delta": 0.21791478991508484, "no_memory_token_accuracy_gain": 0.01446051150560379, "no_memory_token_win_fraction": 0.5973303914070129, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 19.580603581691022, "scrambled_loss": 3.450042963027954, "scrambled_loss_delta": 0.2636568248271942, "scrambled_token_accuracy_gain": 0.027808675542473793, "scrambled_token_win_fraction": 0.6229143738746643, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 23.1762865664124, "memory_only_loss": 5.41035270690918, "memory_only_loss_delta": 2.2239673137664795, "memory_only_token_accuracy_gain": 0.21913236379623413, "memory_only_token_win_fraction": 0.8654060363769531, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 89.18209239860583, "no_evidence_loss": 3.2889068126678467, "no_evidence_loss_delta": 0.10252077132463455, "no_evidence_token_accuracy_gain": -0.0022246940061450005, "no_evidence_token_win_fraction": 0.5417129993438721, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 9.744059779612462, "no_agreement_loss": 3.1937777996063232, "no_agreement_loss_delta": 0.00739162415266037, "no_agreement_token_accuracy_gain": -0.0022246940061450005, "no_agreement_token_win_fraction": 0.46384871006011963, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 0.7364373282919685, "no_orthogonal_loss": 3.1961863040924072, "no_orthogonal_loss_delta": 0.009800390340387821, "no_orthogonal_token_accuracy_gain": -0.006674082484096289, "no_orthogonal_token_win_fraction": 0.5317018628120422, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.975252301542495, "short_no_memory_loss_delta": 0.17792236804962158, "long_context_memory_gain": 0.03999242186546326, "normal_context_loss_gain": 0.15003904700279236, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.91593074798584, "contribution_to_active": 0.7842239141464233, "mix_mean": 0.09412477910518646, "mix_low": 0.4576490819454193, "mix_high": 0.0031104087829589844, "read_cosine": 0.48040708899497986, "beta_mean": 0.2676735520362854, "nonfinite": 0.0, "weighted_memory_to_active": 0.8240435123443604, "mix_std": 0.08296357840299606, "beta_std": 0.15547749400138855, "beta_low": 0.0028279623948037624, "beta_high": 0.0, "orthogonal_to_active": 0.04009998217225075, "output_influence": 0.12639686465263367, "residual_influence": 0.18820001184940338, "state_active": 0.08451150357723236, "state_memory": 0.051970213651657104, "state_peak": 52.37035369873047, "growth_active": 1.2489186525344849, "growth_memory": 1.2818084955215454, "heads": [{"head": 0, "memory_to_active": 2.1878347396850586, "contribution_to_active": 0.31716272234916687, "mix_mean": 0.019406797364354134, "mix_low": 0.533782958984375, "mix_high": 0.0, "read_cosine": 0.33402353525161743, "beta_mean": 0.21618713438510895, "nonfinite": 0.0, "weighted_memory_to_active": 0.3911658227443695, "mix_std": 0.04913979023694992, "beta_std": 0.08390745520591736, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02972651645541191, "evidence_weights": [-0.04132472723722458, -0.07212600857019424, -0.059388041496276855], "agreement_weight": -0.07903219014406204, "orthogonal_weight": -0.0885770320892334}, {"head": 1, "memory_to_active": 3.8942689895629883, "contribution_to_active": 1.4216034412384033, "mix_mean": 0.01799796149134636, "mix_low": 0.5287532806396484, "mix_high": 0.0, "read_cosine": 0.4703868627548218, "beta_mean": 0.3422814607620239, "nonfinite": 0.0, "weighted_memory_to_active": 1.7504596710205078, "mix_std": 0.04615241289138794, "beta_std": 0.15079495310783386, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12104819715023041, "evidence_weights": [0.01135597936809063, -0.03558990731835365, 0.17162838578224182], "agreement_weight": 0.010546497069299221, "orthogonal_weight": -0.16817142069339752}, {"head": 2, "memory_to_active": 2.5518674850463867, "contribution_to_active": 0.4723401665687561, "mix_mean": 0.015401346608996391, "mix_low": 0.5957651138305664, "mix_high": 0.0, "read_cosine": 0.3634531497955322, "beta_mean": 0.3403623104095459, "nonfinite": 0.0, "weighted_memory_to_active": 0.5351135730743408, "mix_std": 0.030801326036453247, "beta_std": 0.23623906075954437, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.052345406264066696, "evidence_weights": [0.03945053741335869, 0.04695752263069153, -0.0030472613871097565], "agreement_weight": -0.0105478810146451, "orthogonal_weight": -0.15794669091701508}, {"head": 3, "memory_to_active": 5.158472061157227, "contribution_to_active": 2.077181577682495, "mix_mean": 0.0499565452337265, "mix_low": 0.2573099136352539, "mix_high": 0.0, "read_cosine": 0.4229682385921478, "beta_mean": 0.32949841022491455, "nonfinite": 0.0, "weighted_memory_to_active": 2.141037940979004, "mix_std": 0.08625750243663788, "beta_std": 0.22451959550380707, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.019464751705527306, "evidence_weights": [-0.10276619344949722, -0.12986604869365692, -0.02153322659432888], "agreement_weight": -0.019650759175419807, "orthogonal_weight": 0.01196244265884161}, {"head": 4, "memory_to_active": 3.5552139282226562, "contribution_to_active": 0.07357901334762573, "mix_mean": 0.006762599106878042, "mix_low": 0.807337760925293, "mix_high": 0.0, "read_cosine": 0.3356226682662964, "beta_mean": 0.31605565547943115, "nonfinite": 0.0, "weighted_memory_to_active": 0.06671052426099777, "mix_std": 0.01099892146885395, "beta_std": 0.18884578347206116, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.010020444169640541, "evidence_weights": [0.0743177980184555, 0.028486937284469604, -0.26041457056999207], "agreement_weight": -0.032813332974910736, "orthogonal_weight": 0.1702529340982437}, {"head": 5, "memory_to_active": 0.14792826771736145, "contribution_to_active": 0.3434765338897705, "mix_mean": 0.4552234411239624, "mix_low": 0.022945404052734375, "mix_high": 0.018662452697753906, "read_cosine": 0.9559880495071411, "beta_mean": 0.061656080186367035, "nonfinite": 0.0, "weighted_memory_to_active": 0.059773318469524384, "mix_std": 0.2744315266609192, "beta_std": 0.04855813458561897, "beta_low": 0.0169677734375, "beta_high": 0.0, "orthogonal_to_active": 0.007994577288627625, "evidence_weights": [-0.2914630174636841, -0.27879124879837036, 0.2983330488204956], "agreement_weight": 0.16434141993522644, "orthogonal_weight": -0.3190065920352936}]}, {"layer": 1, "memory_to_active": 2.360034465789795, "contribution_to_active": 0.4389309585094452, "mix_mean": 0.11172555387020111, "mix_low": 0.3998246192932129, "mix_high": 0.0006693204632028937, "read_cosine": 0.321191668510437, "beta_mean": 0.24967625737190247, "nonfinite": 0.0, "weighted_memory_to_active": 0.364199161529541, "mix_std": 0.1155361607670784, "beta_std": 0.14908307790756226, "beta_low": 0.1689860075712204, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.036993756890296936, "output_influence": 0.13779933750629425, "residual_influence": 0.031557075679302216, "state_active": 0.047449275851249695, "state_memory": 0.04777195677161217, "state_peak": 26.530364990234375, "growth_active": 1.2069292068481445, "growth_memory": 1.5047152042388916, "heads": [{"head": 0, "memory_to_active": 0.0343470424413681, "contribution_to_active": 0.3339575529098511, "mix_mean": 0.25994136929512024, "mix_low": 0.043860435485839844, "mix_high": 0.000152587890625, "read_cosine": -0.09396035969257355, "beta_mean": 0.00017728752573020756, "nonfinite": 0.0, "weighted_memory_to_active": 0.014675362966954708, "mix_std": 0.2627805173397064, "beta_std": 0.00047995889326557517, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.007486698217689991, "evidence_weights": [-0.12925824522972107, -0.16178560256958008, 0.018109997734427452], "agreement_weight": 0.0864303931593895, "orthogonal_weight": -0.588482141494751}, {"head": 1, "memory_to_active": 3.502793073654175, "contribution_to_active": 0.1743597835302353, "mix_mean": 0.021711330860853195, "mix_low": 0.48826026916503906, "mix_high": 8.58306884765625e-06, "read_cosine": 0.47002217173576355, "beta_mean": 0.4261058568954468, "nonfinite": 0.0, "weighted_memory_to_active": 0.1933882236480713, "mix_std": 0.0459342822432518, "beta_std": 0.2507786154747009, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.005667003802955151, "evidence_weights": [-0.015575050376355648, 0.00028639784432016313, 0.02482900768518448], "agreement_weight": -0.009163836017251015, "orthogonal_weight": 0.049824368208646774}, {"head": 2, "memory_to_active": 0.5692858695983887, "contribution_to_active": 0.4019445776939392, "mix_mean": 0.3307720422744751, "mix_low": 0.009905815124511719, "mix_high": 0.0038299560546875, "read_cosine": 0.7852993607521057, "beta_mean": 0.10401603579521179, "nonfinite": 0.0, "weighted_memory_to_active": 0.42665064334869385, "mix_std": 0.25442108511924744, "beta_std": 0.055212654173374176, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.00173329992685467, "evidence_weights": [-0.21846148371696472, -0.3721076250076294, 0.20906303822994232], "agreement_weight": 0.007127142045646906, "orthogonal_weight": 0.008165429346263409}, {"head": 3, "memory_to_active": 3.101693630218506, "contribution_to_active": 0.07115369290113449, "mix_mean": 0.010277952998876572, "mix_low": 0.7765178680419922, "mix_high": 0.0, "read_cosine": 0.20499297976493835, "beta_mean": 0.4501456022262573, "nonfinite": 0.0, "weighted_memory_to_active": 0.08903109282255173, "mix_std": 0.02982698753476143, "beta_std": 0.24712993204593658, "beta_low": 0.000732421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.017927497625350952, "evidence_weights": [0.0367533378303051, 0.016672685742378235, 0.01482434757053852], "agreement_weight": -0.01778494380414486, "orthogonal_weight": -0.2105095237493515}, {"head": 4, "memory_to_active": 3.348355770111084, "contribution_to_active": 1.5905449390411377, "mix_mean": 0.038920383900403976, "mix_low": 0.2843589782714844, "mix_high": 2.47955322265625e-05, "read_cosine": 0.39732012152671814, "beta_mean": 0.23006582260131836, "nonfinite": 0.0, "weighted_memory_to_active": 1.4042584896087646, "mix_std": 0.07427886128425598, "beta_std": 0.1212337464094162, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.18792203068733215, "evidence_weights": [-0.02663460560142994, -0.01539870910346508, 0.10054980218410492], "agreement_weight": -0.03307902440428734, "orthogonal_weight": 0.1523640900850296}, {"head": 5, "memory_to_active": 3.603729724884033, "contribution_to_active": 0.061625342816114426, "mix_mean": 0.008730195462703705, "mix_low": 0.7960443496704102, "mix_high": 0.0, "read_cosine": 0.16347581148147583, "beta_mean": 0.2875468134880066, "nonfinite": 0.0, "weighted_memory_to_active": 0.0571911558508873, "mix_std": 0.025975212454795837, "beta_std": 0.21966353058815002, "beta_low": 0.013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.0012260116636753082, "evidence_weights": [0.1203773021697998, 0.05516441538929939, 0.3206370770931244], "agreement_weight": -0.15081411600112915, "orthogonal_weight": 0.021596619859337807}]}, {"layer": 2, "memory_to_active": 1.080885887145996, "contribution_to_active": 0.2859398126602173, "mix_mean": 0.24332650005817413, "mix_low": 0.27373600006103516, "mix_high": 0.0018693606834858656, "read_cosine": 0.35839200019836426, "beta_mean": 0.10610844194889069, "nonfinite": 0.0, "weighted_memory_to_active": 0.06760838627815247, "mix_std": 0.18758882582187653, "beta_std": 0.0732654556632042, "beta_low": 0.5511068105697632, "beta_high": 0.00010172526526730508, "orthogonal_to_active": 0.013750064186751842, "output_influence": 0.23491138219833374, "residual_influence": 0.05454214662313461, "state_active": 0.04233885183930397, "state_memory": 0.02503850683569908, "state_peak": 56.686973571777344, "growth_active": 1.0390113592147827, "growth_memory": 1.5461755990982056, "heads": [{"head": 0, "memory_to_active": 0.02643211930990219, "contribution_to_active": 0.14640259742736816, "mix_mean": 0.28477489948272705, "mix_low": 0.02674579620361328, "mix_high": 0.0009069442749023438, "read_cosine": 0.23625726997852325, "beta_mean": 0.00017095424118451774, "nonfinite": 0.0, "weighted_memory_to_active": 0.009554190561175346, "mix_std": 0.2687532305717468, "beta_std": 0.00029392511351034045, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 9.601649799151346e-05, "evidence_weights": [-0.16949613392353058, -0.11422517150640488, 0.23998118937015533], "agreement_weight": 0.07734552770853043, "orthogonal_weight": -0.011810853146016598}, {"head": 1, "memory_to_active": 3.0758767127990723, "contribution_to_active": 0.04970146715641022, "mix_mean": 0.004693853668868542, "mix_low": 0.9049100875854492, "mix_high": 9.5367431640625e-07, "read_cosine": 0.18853791058063507, "beta_mean": 0.35095369815826416, "nonfinite": 0.0, "weighted_memory_to_active": 0.040388450026512146, "mix_std": 0.01048076432198286, "beta_std": 0.15611712634563446, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.010756407864391804, "evidence_weights": [0.09992828220129013, 0.10721983760595322, 0.21452394127845764], "agreement_weight": 0.048173245042562485, "orthogonal_weight": 0.2885707914829254}, {"head": 2, "memory_to_active": 0.03722192719578743, "contribution_to_active": 0.4252038598060608, "mix_mean": 0.3595312237739563, "mix_low": 0.0048122406005859375, "mix_high": 0.0012836456298828125, "read_cosine": 0.32542526721954346, "beta_mean": 6.601525819860399e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015424486249685287, "mix_std": 0.27119866013526917, "beta_std": 0.0001481901272200048, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028562741354107857, "evidence_weights": [-0.18532076478004456, -0.21860143542289734, 0.44279149174690247], "agreement_weight": 0.030835991725325584, "orthogonal_weight": -0.23335355520248413}, {"head": 3, "memory_to_active": 0.28378012776374817, "contribution_to_active": 0.42301812767982483, "mix_mean": 0.30229252576828003, "mix_low": 0.005179405212402344, "mix_high": 0.004532814025878906, "read_cosine": 0.7124683856964111, "beta_mean": 0.04129994660615921, "nonfinite": 0.0, "weighted_memory_to_active": 0.12755073606967926, "mix_std": 0.24916672706604004, "beta_std": 0.05999954417347908, "beta_low": 0.25634765625, "beta_high": 0.0, "orthogonal_to_active": 0.05869346857070923, "evidence_weights": [-0.12388408184051514, -0.1877020001411438, 0.6284081935882568], "agreement_weight": 0.07273833453655243, "orthogonal_weight": -0.7589936852455139}, {"head": 4, "memory_to_active": 3.0402655601501465, "contribution_to_active": 0.17276036739349365, "mix_mean": 0.013720025308430195, "mix_low": 0.6982908248901367, "mix_high": 1.9073486328125e-06, "read_cosine": 0.19224832952022552, "beta_mean": 0.24410763382911682, "nonfinite": 0.0, "weighted_memory_to_active": 0.19995316863059998, "mix_std": 0.043782200664281845, "beta_std": 0.22295649349689484, "beta_low": 0.05035400390625, "beta_high": 0.0006103515625, "orthogonal_to_active": 0.009271509945392609, "evidence_weights": [0.04324133321642876, 0.03771757706999779, 0.14318645000457764], "agreement_weight": 0.029287151992321014, "orthogonal_weight": -0.05610676854848862}, {"head": 5, "memory_to_active": 0.02173837646842003, "contribution_to_active": 0.4985523223876953, "mix_mean": 0.4949464201927185, "mix_low": 0.0024776458740234375, "mix_high": 0.004489898681640625, "read_cosine": 0.49541476368904114, "beta_mean": 5.2366045565577224e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012779300101101398, "mix_std": 0.28215140104293823, "beta_std": 7.74360669311136e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008267060038633645, "evidence_weights": [-0.24064265191555023, -0.2480015754699707, 0.6360189914703369], "agreement_weight": 0.017464980483055115, "orthogonal_weight": 0.12423016130924225}]}, {"layer": 3, "memory_to_active": 1.4385707378387451, "contribution_to_active": 0.33320313692092896, "mix_mean": 0.18043789267539978, "mix_low": 0.2706443667411804, "mix_high": 0.0019459724426269531, "read_cosine": 0.27229082584381104, "beta_mean": 0.09890278428792953, "nonfinite": 0.0, "weighted_memory_to_active": 0.1485704481601715, "mix_std": 0.192548468708992, "beta_std": 0.07621464133262634, "beta_low": 0.4063924551010132, "beta_high": 0.0, "orthogonal_to_active": 0.03039298579096794, "output_influence": 0.24232102930545807, "residual_influence": 0.04174201190471649, "state_active": 0.03373531252145767, "state_memory": 0.022199155762791634, "state_peak": 28.771499633789062, "growth_active": 1.0458495616912842, "growth_memory": 1.7447601556777954, "heads": [{"head": 0, "memory_to_active": 0.7759469151496887, "contribution_to_active": 0.3353568911552429, "mix_mean": 0.199203759431839, "mix_low": 0.16272354125976562, "mix_high": 0.001438140869140625, "read_cosine": 0.20536358654499054, "beta_mean": 0.015732482075691223, "nonfinite": 0.0, "weighted_memory_to_active": 0.1995820701122284, "mix_std": 0.261420875787735, "beta_std": 0.013307265006005764, "beta_low": 0.3868408203125, "beta_high": 0.0, "orthogonal_to_active": 0.06950672715902328, "evidence_weights": [-0.05115218833088875, -0.12505118548870087, -0.04641250893473625], "agreement_weight": 0.060976963490247726, "orthogonal_weight": -0.35832682251930237}, {"head": 1, "memory_to_active": 0.059062689542770386, "contribution_to_active": 0.23728713393211365, "mix_mean": 0.2294325828552246, "mix_low": 0.08257579803466797, "mix_high": 0.001003265380859375, "read_cosine": 0.203448086977005, "beta_mean": 0.00024827365996316075, "nonfinite": 0.0, "weighted_memory_to_active": 0.013949013315141201, "mix_std": 0.2575099766254425, "beta_std": 0.00047414429718628526, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.00046172074507921934, "evidence_weights": [-0.1319541335105896, -0.10013815015554428, 0.32973265647888184], "agreement_weight": 0.08890771120786667, "orthogonal_weight": -0.0360158309340477}, {"head": 2, "memory_to_active": 4.012614727020264, "contribution_to_active": 0.3052675724029541, "mix_mean": 0.032761260867118835, "mix_low": 0.45268917083740234, "mix_high": 1.9073486328125e-06, "read_cosine": 0.1723644882440567, "beta_mean": 0.24445568025112152, "nonfinite": 0.0, "weighted_memory_to_active": 0.38938724994659424, "mix_std": 0.0817144513130188, "beta_std": 0.15623536705970764, "beta_low": 0.0035400390625, "beta_high": 0.0, "orthogonal_to_active": 0.027896735817193985, "evidence_weights": [-0.04395154491066933, 0.011941875331103802, 0.062203146517276764], "agreement_weight": -0.02072395756840706, "orthogonal_weight": -0.09057405591011047}, {"head": 3, "memory_to_active": 0.03324161842465401, "contribution_to_active": 0.45951300859451294, "mix_mean": 0.43451735377311707, "mix_low": 0.02788829803466797, "mix_high": 0.005184173583984375, "read_cosine": 0.3011828660964966, "beta_mean": 0.0001088084391085431, "nonfinite": 0.0, "weighted_memory_to_active": 0.018381880596280098, "mix_std": 0.31271296739578247, "beta_std": 0.00025302698486484587, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004714474081993103, "evidence_weights": [-0.1802719086408615, -0.20349851250648499, 1.0989484786987305], "agreement_weight": 0.6138101816177368, "orthogonal_weight": -0.3550969064235687}, {"head": 4, "memory_to_active": 3.341158151626587, "contribution_to_active": 0.03787461668252945, "mix_mean": 0.007276701740920544, "mix_low": 0.8174839019775391, "mix_high": 0.0, "read_cosine": 0.2219027280807495, "beta_mean": 0.23465251922607422, "nonfinite": 0.0, "weighted_memory_to_active": 0.05361104756593704, "mix_std": 0.017203999683260918, "beta_std": 0.17779335379600525, "beta_low": 0.00390625, "beta_high": 0.0, "orthogonal_to_active": 0.018093043938279152, "evidence_weights": [0.1491127759218216, 0.12087752670049667, -0.09842421859502792], "agreement_weight": -0.06130062788724899, "orthogonal_weight": -0.34734049439430237}, {"head": 5, "memory_to_active": 0.4093998074531555, "contribution_to_active": 0.6239194869995117, "mix_mean": 0.17943567037582397, "mix_low": 0.08050537109375, "mix_high": 0.004048347473144531, "read_cosine": 0.5294830799102783, "beta_mean": 0.0982188954949379, "nonfinite": 0.0, "weighted_memory_to_active": 0.2165113389492035, "mix_std": 0.22472849488258362, "beta_std": 0.10922466218471527, "beta_low": 0.0443115234375, "beta_high": 0.0, "orthogonal_to_active": 0.06168519705533981, "evidence_weights": [-0.07676263153553009, -0.15473723411560059, 0.19109290838241577], "agreement_weight": 0.16922527551651, "orthogonal_weight": -0.44983768463134766}]}, {"layer": 4, "memory_to_active": 1.2626603841781616, "contribution_to_active": 0.3605164885520935, "mix_mean": 0.26265770196914673, "mix_low": 0.28308647871017456, "mix_high": 0.0016144118271768093, "read_cosine": 0.3973836898803711, "beta_mean": 0.12844353914260864, "nonfinite": 0.0, "weighted_memory_to_active": 0.16732385754585266, "mix_std": 0.15766403079032898, "beta_std": 0.10602036863565445, "beta_low": 0.3005879819393158, "beta_high": 0.0, "orthogonal_to_active": 0.041125111281871796, "output_influence": 0.2508346140384674, "residual_influence": 0.05639123544096947, "state_active": 0.04519284889101982, "state_memory": 0.031427137553691864, "state_peak": 52.661739349365234, "growth_active": 1.1097463369369507, "growth_memory": 1.6008737087249756, "heads": [{"head": 0, "memory_to_active": 0.38808661699295044, "contribution_to_active": 0.56460040807724, "mix_mean": 0.47567665576934814, "mix_low": 0.013545989990234375, "mix_high": 0.0009908676147460938, "read_cosine": 0.5605579614639282, "beta_mean": 0.17565783858299255, "nonfinite": 0.0, "weighted_memory_to_active": 0.2646212577819824, "mix_std": 0.30909955501556396, "beta_std": 0.20074285566806793, "beta_low": 0.16729736328125, "beta_high": 0.0, "orthogonal_to_active": 0.024518907070159912, "evidence_weights": [-0.22156541049480438, -0.2739068865776062, 0.4416867196559906], "agreement_weight": 0.0487939678132534, "orthogonal_weight": 0.12126054614782333}, {"head": 1, "memory_to_active": 0.3070136308670044, "contribution_to_active": 0.38744717836380005, "mix_mean": 0.4128996431827545, "mix_low": 0.0033082962036132812, "mix_high": 0.0014352798461914062, "read_cosine": 0.5715911984443665, "beta_mean": 0.014095917344093323, "nonfinite": 0.0, "weighted_memory_to_active": 0.14267116785049438, "mix_std": 0.2681855857372284, "beta_std": 0.02335522323846817, "beta_low": 0.57513427734375, "beta_high": 0.0, "orthogonal_to_active": 0.07397500425577164, "evidence_weights": [-0.218186616897583, -0.1959237903356552, 0.6487285494804382], "agreement_weight": 0.03830382227897644, "orthogonal_weight": -0.6468577980995178}, {"head": 2, "memory_to_active": 2.5349197387695312, "contribution_to_active": 0.09693581610918045, "mix_mean": 0.013346817344427109, "mix_low": 0.6631937026977539, "mix_high": 0.0, "read_cosine": 0.2666768431663513, "beta_mean": 0.14320893585681915, "nonfinite": 0.0, "weighted_memory_to_active": 0.12510865926742554, "mix_std": 0.023926960304379463, "beta_std": 0.1086396798491478, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.0035057514905929565, "evidence_weights": [0.048593100160360336, 0.027370991185307503, -0.0485382080078125], "agreement_weight": 0.05205784738063812, "orthogonal_weight": -0.04102007672190666}, {"head": 3, "memory_to_active": 1.5407836437225342, "contribution_to_active": 0.10230757296085358, "mix_mean": 0.0275752991437912, "mix_low": 0.40968990325927734, "mix_high": 0.0, "read_cosine": 0.2691473960876465, "beta_mean": 0.14903996884822845, "nonfinite": 0.0, "weighted_memory_to_active": 0.18232029676437378, "mix_std": 0.047829918563365936, "beta_std": 0.1359897404909134, "beta_low": 0.06011962890625, "beta_high": 0.0, "orthogonal_to_active": 0.017542103305459023, "evidence_weights": [-0.01847001351416111, 0.04523608088493347, -0.174329936504364], "agreement_weight": 0.02860666997730732, "orthogonal_weight": -0.18824541568756104}, {"head": 4, "memory_to_active": 0.016920652240514755, "contribution_to_active": 0.6117811799049377, "mix_mean": 0.630357027053833, "mix_low": 0.0008802413940429688, "mix_high": 0.0072307586669921875, "read_cosine": 0.3450801968574524, "beta_mean": 3.4048276575049385e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012485010549426079, "mix_std": 0.25584423542022705, "beta_std": 6.500177551060915e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013664065627381206, "evidence_weights": [-0.2779724895954132, -0.31894832849502563, 0.7333217263221741], "agreement_weight": 0.04359053447842598, "orthogonal_weight": -0.1354503184556961}, {"head": 5, "memory_to_active": 2.7882378101348877, "contribution_to_active": 0.40002647042274475, "mix_mean": 0.016090700402855873, "mix_low": 0.6079006195068359, "mix_high": 2.956390380859375e-05, "read_cosine": 0.37124815583229065, "beta_mean": 0.28862449526786804, "nonfinite": 0.0, "weighted_memory_to_active": 0.2767367362976074, "mix_std": 0.04109795019030571, "beta_std": 0.16732966899871826, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12584248185157776, "evidence_weights": [0.0304808858782053, -0.033561497926712036, -0.12623190879821777], "agreement_weight": -0.04309802129864693, "orthogonal_weight": 0.6343700885772705}]}, {"layer": 5, "memory_to_active": 0.6482037901878357, "contribution_to_active": 0.42670750617980957, "mix_mean": 0.37492549419403076, "mix_low": 0.10621325671672821, "mix_high": 0.007901668548583984, "read_cosine": 0.22142545878887177, "beta_mean": 0.057286735624074936, "nonfinite": 0.0, "weighted_memory_to_active": 0.15729153156280518, "mix_std": 0.2424437552690506, "beta_std": 0.06948043406009674, "beta_low": 0.5876871943473816, "beta_high": 0.0, "orthogonal_to_active": 0.04768161103129387, "output_influence": 0.39216211438179016, "residual_influence": 0.08702500909566879, "state_active": 0.057129424065351486, "state_memory": 0.0174521766602993, "state_peak": 64.24068450927734, "growth_active": 1.1143332719802856, "growth_memory": 1.3312594890594482, "heads": [{"head": 0, "memory_to_active": 1.4866840839385986, "contribution_to_active": 0.2813463807106018, "mix_mean": 0.10334759950637817, "mix_low": 0.16995620727539062, "mix_high": 0.0, "read_cosine": -0.2655462324619293, "beta_mean": 0.08608108013868332, "nonfinite": 0.0, "weighted_memory_to_active": 0.19905073940753937, "mix_std": 0.1450936645269394, "beta_std": 0.10384826362133026, "beta_low": 0.05279541015625, "beta_high": 0.0, "orthogonal_to_active": 0.04363596439361572, "evidence_weights": [-0.03556535392999649, -0.07056759297847748, -0.34476158022880554], "agreement_weight": -0.022635700181126595, "orthogonal_weight": -0.2133580446243286}, {"head": 1, "memory_to_active": 0.042295537889003754, "contribution_to_active": 0.43747666478157043, "mix_mean": 0.4959107041358948, "mix_low": 0.0099334716796875, "mix_high": 0.009163856506347656, "read_cosine": 0.16071872413158417, "beta_mean": 0.00012749494635500014, "nonfinite": 0.0, "weighted_memory_to_active": 0.0180550217628479, "mix_std": 0.3121888041496277, "beta_std": 0.0003001798177137971, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01616394892334938, "evidence_weights": [-0.21629424393177032, -0.22660522162914276, 0.62378990650177], "agreement_weight": 0.24003368616104126, "orthogonal_weight": -0.9242637753486633}, {"head": 2, "memory_to_active": 0.09670282900333405, "contribution_to_active": 0.4962473213672638, "mix_mean": 0.5503432750701904, "mix_low": 0.0020647048950195312, "mix_high": 0.009116172790527344, "read_cosine": 0.4695925712585449, "beta_mean": 0.00255537498742342, "nonfinite": 0.0, "weighted_memory_to_active": 0.06426519900560379, "mix_std": 0.2773666977882385, "beta_std": 0.0029041199013590813, "beta_low": 0.97222900390625, "beta_high": 0.0, "orthogonal_to_active": 0.008496381342411041, "evidence_weights": [-0.26848235726356506, -0.2986261248588562, 0.649515688419342], "agreement_weight": 0.03275594487786293, "orthogonal_weight": -0.17821437120437622}, {"head": 3, "memory_to_active": 1.6547973155975342, "contribution_to_active": 0.15634450316429138, "mix_mean": 0.06620557606220245, "mix_low": 0.28316497802734375, "mix_high": 0.0, "read_cosine": 0.14010107517242432, "beta_mean": 0.05293110013008118, "nonfinite": 0.0, "weighted_memory_to_active": 0.2552630603313446, "mix_std": 0.1199701577425003, "beta_std": 0.08376942574977875, "beta_low": 0.28216552734375, "beta_high": 0.0, "orthogonal_to_active": 0.09859487414360046, "evidence_weights": [-0.005802359897643328, -0.05351020768284798, 0.05438590049743652], "agreement_weight": 0.0584820993244648, "orthogonal_weight": -0.45515763759613037}, {"head": 4, "memory_to_active": 0.5106105804443359, "contribution_to_active": 0.5764188170433044, "mix_mean": 0.320568710565567, "mix_low": 0.16894149780273438, "mix_high": 0.011556625366210938, "read_cosine": 0.5760080218315125, "beta_mean": 0.20165657997131348, "nonfinite": 0.0, "weighted_memory_to_active": 0.3425956964492798, "mix_std": 0.33766886591911316, "beta_std": 0.22449524700641632, "beta_low": 0.22430419921875, "beta_high": 0.0, "orthogonal_to_active": 0.08492880314588547, "evidence_weights": [-0.13604925572872162, -0.274207204580307, 0.03989192843437195], "agreement_weight": 0.0435812771320343, "orthogonal_weight": 0.37830793857574463}, {"head": 5, "memory_to_active": 0.09813213348388672, "contribution_to_active": 0.6124112010002136, "mix_mean": 0.7131769061088562, "mix_low": 0.0032186508178710938, "mix_high": 0.01757335662841797, "read_cosine": 0.24767860770225525, "beta_mean": 0.00036878566606901586, "nonfinite": 0.0, "weighted_memory_to_active": 0.06451945751905441, "mix_std": 0.26237428188323975, "beta_std": 0.0015653516165912151, "beta_low": 0.99468994140625, "beta_high": 0.0, "orthogonal_to_active": 0.03426966816186905, "evidence_weights": [-0.31931036710739136, -0.3065033555030823, 1.3967708349227905], "agreement_weight": 0.3083038032054901, "orthogonal_weight": -0.5772882699966431}]}, {"layer": 6, "memory_to_active": 0.46371105313301086, "contribution_to_active": 0.5493018627166748, "mix_mean": 0.4882998466491699, "mix_low": 0.09222333133220673, "mix_high": 0.02482461929321289, "read_cosine": 0.30746912956237793, "beta_mean": 0.07996857911348343, "nonfinite": 0.0, "weighted_memory_to_active": 0.173418790102005, "mix_std": 0.25583741068840027, "beta_std": 0.09396485239267349, "beta_low": 0.5089823603630066, "beta_high": 0.0, "orthogonal_to_active": 0.03812023997306824, "output_influence": 0.4312022030353546, "residual_influence": 0.1304735541343689, "state_active": 0.05462218075990677, "state_memory": 0.011921335011720657, "state_peak": 59.39422607421875, "growth_active": 1.1313762664794922, "growth_memory": 1.2514102458953857, "heads": [{"head": 0, "memory_to_active": 0.18780148029327393, "contribution_to_active": 0.554217517375946, "mix_mean": 0.623152494430542, "mix_low": 0.000392913818359375, "mix_high": 0.003907203674316406, "read_cosine": 0.18097326159477234, "beta_mean": 0.014064176008105278, "nonfinite": 0.0, "weighted_memory_to_active": 0.14774960279464722, "mix_std": 0.25535914301872253, "beta_std": 0.0367993526160717, "beta_low": 0.6334228515625, "beta_high": 0.0, "orthogonal_to_active": 0.0988001599907875, "evidence_weights": [-0.2870989739894867, -0.28276878595352173, 0.386330783367157], "agreement_weight": 0.09792120009660721, "orthogonal_weight": -0.8639071583747864}, {"head": 1, "memory_to_active": 0.009577648714184761, "contribution_to_active": 0.44667109847068787, "mix_mean": 0.4297338128089905, "mix_low": 0.004520416259765625, "mix_high": 0.008852005004882812, "read_cosine": 0.5599026083946228, "beta_mean": 2.112548827426508e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003488744143396616, "mix_std": 0.29537153244018555, "beta_std": 4.7878187615424395e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009551824769005179, "evidence_weights": [-0.20822188258171082, -0.2389947474002838, 0.5998048782348633], "agreement_weight": -0.02625933475792408, "orthogonal_weight": -0.36910292506217957}, {"head": 2, "memory_to_active": 1.663028359413147, "contribution_to_active": 0.06270547211170197, "mix_mean": 0.02095245197415352, "mix_low": 0.5351676940917969, "mix_high": 0.0, "read_cosine": 0.18854185938835144, "beta_mean": 0.10340280830860138, "nonfinite": 0.0, "weighted_memory_to_active": 0.07428009063005447, "mix_std": 0.04467328265309334, "beta_std": 0.09575007110834122, "beta_low": 0.0010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.016537362709641457, "evidence_weights": [0.08539921045303345, 0.037112366408109665, 0.14936190843582153], "agreement_weight": 0.06629016250371933, "orthogonal_weight": -0.2377723902463913}, {"head": 3, "memory_to_active": 0.16334238648414612, "contribution_to_active": 0.6411898136138916, "mix_mean": 0.6234604716300964, "mix_low": 0.010023117065429688, "mix_high": 0.030439376831054688, "read_cosine": 0.5390164852142334, "beta_mean": 0.23983684182167053, "nonfinite": 0.0, "weighted_memory_to_active": 0.11634458601474762, "mix_std": 0.3434005081653595, "beta_std": 0.27538251876831055, "beta_low": 0.17840576171875, "beta_high": 0.0, "orthogonal_to_active": 0.045446641743183136, "evidence_weights": [-0.1747792810201645, -0.3342457711696625, 0.16037993133068085], "agreement_weight": -0.01507365982979536, "orthogonal_weight": 0.4757356345653534}, {"head": 4, "memory_to_active": 0.7072502970695496, "contribution_to_active": 1.058292031288147, "mix_mean": 0.7089393138885498, "mix_low": 0.0014743804931640625, "mix_high": 0.1051492691040039, "read_cosine": 0.10830873996019363, "beta_mean": 0.12211691588163376, "nonfinite": 0.0, "weighted_memory_to_active": 0.6676979064941406, "mix_std": 0.3086547255516052, "beta_std": 0.1551329642534256, "beta_low": 0.24139404296875, "beta_high": 0.0, "orthogonal_to_active": 0.03638753667473793, "evidence_weights": [-0.24851155281066895, -0.5453839302062988, -0.026992132887244225], "agreement_weight": -0.01635974645614624, "orthogonal_weight": 0.0660874992609024}, {"head": 5, "memory_to_active": 0.051266156136989594, "contribution_to_active": 0.5327349305152893, "mix_mean": 0.5235605239868164, "mix_low": 0.0017614364624023438, "mix_high": 0.0005998611450195312, "read_cosine": 0.2680718004703522, "beta_mean": 0.0003696053463499993, "nonfinite": 0.0, "weighted_memory_to_active": 0.030951766297221184, "mix_std": 0.28756529092788696, "beta_std": 0.0006763006094843149, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.03059455193579197, "evidence_weights": [-0.21831205487251282, -0.26066699624061584, 0.28836485743522644], "agreement_weight": 0.07326515018939972, "orthogonal_weight": -1.0598713159561157}]}, {"layer": 7, "memory_to_active": 2.1002564430236816, "contribution_to_active": 0.1468503773212433, "mix_mean": 0.03272490203380585, "mix_low": 0.5246967077255249, "mix_high": 3.814697265625e-06, "read_cosine": 0.22015641629695892, "beta_mean": 0.14628887176513672, "nonfinite": 0.0, "weighted_memory_to_active": 0.15431007742881775, "mix_std": 0.05770696699619293, "beta_std": 0.1141192764043808, "beta_low": 0.0444437675178051, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.028141209855675697, "output_influence": 0.16538546979427338, "residual_influence": 0.036019742488861084, "state_active": 0.017367638647556305, "state_memory": 0.032643191516399384, "state_peak": 20.392080307006836, "growth_active": 1.0414739847183228, "growth_memory": 1.4024405479431152, "heads": [{"head": 0, "memory_to_active": 2.2419254779815674, "contribution_to_active": 0.23135238885879517, "mix_mean": 0.025653215125203133, "mix_low": 0.5443687438964844, "mix_high": 2.002716064453125e-05, "read_cosine": 0.3733057379722595, "beta_mean": 0.20829880237579346, "nonfinite": 0.0, "weighted_memory_to_active": 0.21022368967533112, "mix_std": 0.055471260100603104, "beta_std": 0.14544899761676788, "beta_low": 0.00201416015625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.03481088951230049, "evidence_weights": [0.00015289249131456017, -0.07778962701559067, 0.08240435272455215], "agreement_weight": -0.04474810138344765, "orthogonal_weight": 0.2095382660627365}, {"head": 1, "memory_to_active": 2.365079641342163, "contribution_to_active": 0.26869773864746094, "mix_mean": 0.07903888821601868, "mix_low": 0.29319095611572266, "mix_high": 0.0, "read_cosine": 0.08116543292999268, "beta_mean": 0.19875186681747437, "nonfinite": 0.0, "weighted_memory_to_active": 0.33202749490737915, "mix_std": 0.11281972378492355, "beta_std": 0.11773060262203217, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06958892196416855, "evidence_weights": [-0.11882630735635757, -0.09617535769939423, -0.09275194257497787], "agreement_weight": 0.013071810826659203, "orthogonal_weight": -0.21641045808792114}, {"head": 2, "memory_to_active": 1.364451289176941, "contribution_to_active": 0.23702803254127502, "mix_mean": 0.054175399243831635, "mix_low": 0.2950105667114258, "mix_high": 2.86102294921875e-06, "read_cosine": 0.15907713770866394, "beta_mean": 0.1374698430299759, "nonfinite": 0.0, "weighted_memory_to_active": 0.193965882062912, "mix_std": 0.10135110467672348, "beta_std": 0.08551150560379028, "beta_low": 0.00762939453125, "beta_high": 0.0, "orthogonal_to_active": 0.013876538723707199, "evidence_weights": [-0.031949613243341446, -0.10800211131572723, 0.2138044536113739], "agreement_weight": 0.02504946105182171, "orthogonal_weight": -0.07453400641679764}, {"head": 3, "memory_to_active": 3.042914390563965, "contribution_to_active": 0.03975071385502815, "mix_mean": 0.012209547683596611, "mix_low": 0.6459884643554688, "mix_high": 0.0, "read_cosine": 0.12054659426212311, "beta_mean": 0.12343035638332367, "nonfinite": 0.0, "weighted_memory_to_active": 0.0718793049454689, "mix_std": 0.025752997025847435, "beta_std": 0.13170002400875092, "beta_low": 0.05877685546875, "beta_high": 0.0, "orthogonal_to_active": 0.03850235044956207, "evidence_weights": [0.12159678339958191, 0.08337713032960892, 0.26836347579956055], "agreement_weight": 0.07248087972402573, "orthogonal_weight": -0.5712257623672485}, {"head": 4, "memory_to_active": 1.8339838981628418, "contribution_to_active": 0.04482521116733551, "mix_mean": 0.010762812569737434, "mix_low": 0.7384023666381836, "mix_high": 0.0, "read_cosine": 0.3317195177078247, "beta_mean": 0.16023772954940796, "nonfinite": 0.0, "weighted_memory_to_active": 0.04398006945848465, "mix_std": 0.024495970457792282, "beta_std": 0.14001403748989105, "beta_low": 0.02264404296875, "beta_high": 0.0, "orthogonal_to_active": 0.0030378454830497503, "evidence_weights": [0.08837424963712692, 0.045128244906663895, 0.0467284694314003], "agreement_weight": 0.043913815170526505, "orthogonal_weight": -0.07767406851053238}, {"head": 5, "memory_to_active": 1.753183126449585, "contribution_to_active": 0.059448208659887314, "mix_mean": 0.0145095344632864, "mix_low": 0.6312189102172852, "mix_high": 0.0, "read_cosine": 0.2551240921020508, "beta_mean": 0.04954458028078079, "nonfinite": 0.0, "weighted_memory_to_active": 0.0737839862704277, "mix_std": 0.02635073848068714, "beta_std": 0.0643104836344719, "beta_low": 0.17559814453125, "beta_high": 0.0, "orthogonal_to_active": 0.009030702523887157, "evidence_weights": [0.06995909661054611, 0.07577443867921829, -0.03946257755160332], "agreement_weight": 0.00780152902007103, "orthogonal_weight": -0.15427792072296143}]}, {"layer": 8, "memory_to_active": 0.7256187796592712, "contribution_to_active": 0.43686771392822266, "mix_mean": 0.3200222849845886, "mix_low": 0.08625397086143494, "mix_high": 0.0008605322218500078, "read_cosine": 0.5287610292434692, "beta_mean": 0.04701242223381996, "nonfinite": 0.0, "weighted_memory_to_active": 0.15376022458076477, "mix_std": 0.22069695591926575, "beta_std": 0.049563851207494736, "beta_low": 0.4266764521598816, "beta_high": 0.0, "orthogonal_to_active": 0.04619734361767769, "output_influence": 0.31553688645362854, "residual_influence": 0.082377590239048, "state_active": 0.05468779429793358, "state_memory": 0.016091249883174896, "state_peak": 28.828954696655273, "growth_active": 1.033806562423706, "growth_memory": 1.375853180885315, "heads": [{"head": 0, "memory_to_active": 0.06280195713043213, "contribution_to_active": 0.504764974117279, "mix_mean": 0.526545524597168, "mix_low": 0.00015163421630859375, "mix_high": 0.00038623809814453125, "read_cosine": 0.7292729616165161, "beta_mean": 0.004099166952073574, "nonfinite": 0.0, "weighted_memory_to_active": 0.03466757759451866, "mix_std": 0.24214327335357666, "beta_std": 0.00885315053164959, "beta_low": 0.9013671875, "beta_high": 0.0, "orthogonal_to_active": 0.017314132302999496, "evidence_weights": [-0.22149881720542908, -0.3338208496570587, 0.6838427186012268], "agreement_weight": 0.06539592891931534, "orthogonal_weight": -0.7610201239585876}, {"head": 1, "memory_to_active": 0.5847187638282776, "contribution_to_active": 0.3293585181236267, "mix_mean": 0.32046014070510864, "mix_low": 0.005764961242675781, "mix_high": 0.00027179718017578125, "read_cosine": 0.5106744766235352, "beta_mean": 0.03703612461686134, "nonfinite": 0.0, "weighted_memory_to_active": 0.21373890340328217, "mix_std": 0.22400687634944916, "beta_std": 0.033214930444955826, "beta_low": 0.117919921875, "beta_high": 0.0, "orthogonal_to_active": 0.03624068200588226, "evidence_weights": [-0.16402269899845123, -0.1863035410642624, 0.36845073103904724], "agreement_weight": 0.0594756044447422, "orthogonal_weight": -0.20381195843219757}, {"head": 2, "memory_to_active": 0.1423359364271164, "contribution_to_active": 0.4738565981388092, "mix_mean": 0.42198309302330017, "mix_low": 0.0019931793212890625, "mix_high": 0.0007381439208984375, "read_cosine": 0.3811768889427185, "beta_mean": 0.002746940590441227, "nonfinite": 0.0, "weighted_memory_to_active": 0.07467245310544968, "mix_std": 0.2598896026611328, "beta_std": 0.004822352901101112, "beta_low": 0.94927978515625, "beta_high": 0.0, "orthogonal_to_active": 0.052653323858976364, "evidence_weights": [-0.18303373456001282, -0.27699464559555054, 0.4390268921852112], "agreement_weight": 0.05431654676795006, "orthogonal_weight": -0.8032413721084595}, {"head": 3, "memory_to_active": 0.29759517312049866, "contribution_to_active": 0.32835036516189575, "mix_mean": 0.2953271269798279, "mix_low": 0.010796546936035156, "mix_high": 8.58306884765625e-06, "read_cosine": 0.7098186612129211, "beta_mean": 0.017728200182318687, "nonfinite": 0.0, "weighted_memory_to_active": 0.1345290243625641, "mix_std": 0.21844518184661865, "beta_std": 0.02115967497229576, "beta_low": 0.3526611328125, "beta_high": 0.0, "orthogonal_to_active": 0.006315438076853752, "evidence_weights": [-0.09092938154935837, -0.19803227484226227, 0.6838260293006897], "agreement_weight": 0.07842699438333511, "orthogonal_weight": -0.07950413227081299}, {"head": 4, "memory_to_active": 3.0389227867126465, "contribution_to_active": 0.31179001927375793, "mix_mean": 0.02825503796339035, "mix_low": 0.3990306854248047, "mix_high": 0.0, "read_cosine": 0.3161231279373169, "beta_mean": 0.13283120095729828, "nonfinite": 0.0, "weighted_memory_to_active": 0.3080942928791046, "mix_std": 0.04546903073787689, "beta_std": 0.10443145781755447, "beta_low": 0.0030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.015590721741318703, "evidence_weights": [0.025102626532316208, -0.0051886956207454205, -0.10863513499498367], "agreement_weight": 0.014748194254934788, "orthogonal_weight": 0.05528120696544647}, {"head": 5, "memory_to_active": 0.22733819484710693, "contribution_to_active": 0.6730857491493225, "mix_mean": 0.32756277918815613, "mix_low": 0.09978675842285156, "mix_high": 0.0037584304809570312, "read_cosine": 0.5255002379417419, "beta_mean": 0.08763287961483002, "nonfinite": 0.0, "weighted_memory_to_active": 0.15685908496379852, "mix_std": 0.33422768115997314, "beta_std": 0.12490154057741165, "beta_low": 0.23577880859375, "beta_high": 0.0, "orthogonal_to_active": 0.14906975626945496, "evidence_weights": [-0.07123185694217682, -0.33062613010406494, -0.435627818107605], "agreement_weight": 0.019206861034035683, "orthogonal_weight": 1.1546986103057861}]}, {"layer": 9, "memory_to_active": 1.2202739715576172, "contribution_to_active": 0.5825364589691162, "mix_mean": 0.22555434703826904, "mix_low": 0.1841297298669815, "mix_high": 0.0010774930706247687, "read_cosine": 0.3563951849937439, "beta_mean": 0.10548123717308044, "nonfinite": 0.0, "weighted_memory_to_active": 0.37346282601356506, "mix_std": 0.20914235711097717, "beta_std": 0.05673566460609436, "beta_low": 0.4954630732536316, "beta_high": 0.0, "orthogonal_to_active": 0.024969972670078278, "output_influence": 0.37038537859916687, "residual_influence": 0.09024176001548767, "state_active": 0.040703777223825455, "state_memory": 0.026905320584774017, "state_peak": 29.753507614135742, "growth_active": 1.2150193452835083, "growth_memory": 1.4158045053482056, "heads": [{"head": 0, "memory_to_active": 0.009922552853822708, "contribution_to_active": 0.44127821922302246, "mix_mean": 0.47180062532424927, "mix_low": 0.004096031188964844, "mix_high": 0.0037870407104492188, "read_cosine": 0.21433648467063904, "beta_mean": 2.0501855033216998e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005176885053515434, "mix_std": 0.2953709363937378, "beta_std": 8.528637408744544e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035216722171753645, "evidence_weights": [-0.25104400515556335, -0.2428310364484787, 0.9514790773391724], "agreement_weight": 0.03414234519004822, "orthogonal_weight": -0.7080647349357605}, {"head": 1, "memory_to_active": 0.032563939690589905, "contribution_to_active": 0.4376922845840454, "mix_mean": 0.2999321222305298, "mix_low": 0.02753925323486328, "mix_high": 0.0009860992431640625, "read_cosine": 0.41575515270233154, "beta_mean": 8.96361016202718e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013788691721856594, "mix_std": 0.2664662003517151, "beta_std": 0.00017707815277390182, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00608176039531827, "evidence_weights": [-0.18201927840709686, -0.18788424134254456, 0.736997127532959], "agreement_weight": 0.31270161271095276, "orthogonal_weight": -0.5074484348297119}, {"head": 2, "memory_to_active": 0.2964458465576172, "contribution_to_active": 0.49908024072647095, "mix_mean": 0.36517077684402466, "mix_low": 0.013324737548828125, "mix_high": 0.0012416839599609375, "read_cosine": 0.3799339532852173, "beta_mean": 0.002141623990610242, "nonfinite": 0.0, "weighted_memory_to_active": 0.15363235771656036, "mix_std": 0.2836436331272125, "beta_std": 0.006213054992258549, "beta_low": 0.97174072265625, "beta_high": 0.0, "orthogonal_to_active": 0.03657260537147522, "evidence_weights": [-0.14238835871219635, -0.1544383317232132, 0.35540926456451416], "agreement_weight": 0.15803883969783783, "orthogonal_weight": -0.293461412191391}, {"head": 3, "memory_to_active": 1.8926855325698853, "contribution_to_active": 0.184677392244339, "mix_mean": 0.038121990859508514, "mix_low": 0.42376136779785156, "mix_high": 5.435943603515625e-05, "read_cosine": 0.4683755934238434, "beta_mean": 0.1277032196521759, "nonfinite": 0.0, "weighted_memory_to_active": 0.15681374073028564, "mix_std": 0.09380576759576797, "beta_std": 0.08136985450983047, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.024616628885269165, "evidence_weights": [0.04370556026697159, 0.03298824280500412, -0.15378837287425995], "agreement_weight": 0.059665970504283905, "orthogonal_weight": 0.15895694494247437}, {"head": 4, "memory_to_active": 3.167771577835083, "contribution_to_active": 1.3709375858306885, "mix_mean": 0.12158143520355225, "mix_low": 0.20975112915039062, "mix_high": 0.000392913818359375, "read_cosine": 0.22563567757606506, "beta_mean": 0.28474801778793335, "nonfinite": 0.0, "weighted_memory_to_active": 1.2422699928283691, "mix_std": 0.19570383429527283, "beta_std": 0.134956955909729, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 5.315886301104911e-05, "evidence_weights": [-0.04383106529712677, -0.12219980359077454, 0.06689821928739548], "agreement_weight": -0.04565979167819023, "orthogonal_weight": 4.2795625631697476e-05}, {"head": 5, "memory_to_active": 1.922254204750061, "contribution_to_active": 0.5615527629852295, "mix_mean": 0.056719087064266205, "mix_low": 0.42630577087402344, "mix_high": 2.86102294921875e-06, "read_cosine": 0.4343342185020447, "beta_mean": 0.21818439662456512, "nonfinite": 0.0, "weighted_memory_to_active": 0.6690952777862549, "mix_std": 0.11986373364925385, "beta_std": 0.11761178076267242, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.07897400110960007, "evidence_weights": [0.04394412785768509, -0.04049757868051529, 0.07254917919635773], "agreement_weight": -0.0004913273733109236, "orthogonal_weight": 0.16757236421108246}]}, {"layer": 10, "memory_to_active": 0.18287672102451324, "contribution_to_active": 0.4282941520214081, "mix_mean": 0.3932417631149292, "mix_low": 0.014903545379638672, "mix_high": 0.008462905883789062, "read_cosine": 0.38543087244033813, "beta_mean": 0.04260905459523201, "nonfinite": 0.0, "weighted_memory_to_active": 0.07983844727277756, "mix_std": 0.2647607624530792, "beta_std": 0.048431478440761566, "beta_low": 0.6530863642692566, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.036536287516355515, "output_influence": 0.383525550365448, "residual_influence": 0.1172516867518425, "state_active": 0.059068597853183746, "state_memory": 0.014499080367386341, "state_peak": 102.85000610351562, "growth_active": 1.0724241733551025, "growth_memory": 1.0598877668380737, "heads": [{"head": 0, "memory_to_active": 0.35211825370788574, "contribution_to_active": 0.7598358392715454, "mix_mean": 0.6274425983428955, "mix_low": 0.01381683349609375, "mix_high": 0.04859352111816406, "read_cosine": 0.23072370886802673, "beta_mean": 0.18490588665008545, "nonfinite": 0.0, "weighted_memory_to_active": 0.2491605430841446, "mix_std": 0.35150614380836487, "beta_std": 0.20794610679149628, "beta_low": 0.135986328125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.11979410797357559, "evidence_weights": [-0.16787178814411163, -0.3433499336242676, -0.2675008177757263], "agreement_weight": 0.016689712181687355, "orthogonal_weight": 0.5302460789680481}, {"head": 1, "memory_to_active": 0.0024075359106063843, "contribution_to_active": 0.48755550384521484, "mix_mean": 0.46108901500701904, "mix_low": 0.0025615692138671875, "mix_high": 0.0013170242309570312, "read_cosine": 0.2288258671760559, "beta_mean": 1.4329585610539652e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0011661386815831065, "mix_std": 0.2651461660861969, "beta_std": 3.205869143130258e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000833515077829361, "evidence_weights": [-0.25353527069091797, -0.18151727318763733, 0.2510552406311035], "agreement_weight": 0.11101051419973373, "orthogonal_weight": -0.7757309079170227}, {"head": 2, "memory_to_active": 0.26513412594795227, "contribution_to_active": 0.30785655975341797, "mix_mean": 0.25813770294189453, "mix_low": 0.031229019165039062, "mix_high": 7.152557373046875e-05, "read_cosine": 0.0678347647190094, "beta_mean": 0.0030578896403312683, "nonfinite": 0.0, "weighted_memory_to_active": 0.07394509762525558, "mix_std": 0.25076964497566223, "beta_std": 0.01005056407302618, "beta_low": 0.9393310546875, "beta_high": 0.0, "orthogonal_to_active": 0.019723188132047653, "evidence_weights": [-0.09323050826787949, -0.16134127974510193, 0.3723210394382477], "agreement_weight": 0.02492266334593296, "orthogonal_weight": -0.266279011964798}, {"head": 3, "memory_to_active": 0.2983163893222809, "contribution_to_active": 0.16024059057235718, "mix_mean": 0.2913168966770172, "mix_low": 0.03339958190917969, "mix_high": 0.0007562637329101562, "read_cosine": 0.870460569858551, "beta_mean": 0.061505988240242004, "nonfinite": 0.0, "weighted_memory_to_active": 0.06583686918020248, "mix_std": 0.25123798847198486, "beta_std": 0.06145544350147247, "beta_low": 0.00762939453125, "beta_high": 0.0, "orthogonal_to_active": 0.057997651398181915, "evidence_weights": [-0.12559834122657776, -0.14524845778942108, 0.23051053285598755], "agreement_weight": 0.0030608312226831913, "orthogonal_weight": 1.209608554840088}, {"head": 4, "memory_to_active": 0.02747315540909767, "contribution_to_active": 0.40606242418289185, "mix_mean": 0.3799158036708832, "mix_low": 0.0029449462890625, "mix_high": 3.0517578125e-05, "read_cosine": 0.07523001730442047, "beta_mean": 7.57611051085405e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00741850771009922, "mix_std": 0.24890072643756866, "beta_std": 0.001829545246437192, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0019014443969354033, "evidence_weights": [-0.17550523579120636, -0.18205714225769043, 0.2821212708950043], "agreement_weight": 0.026315905153751373, "orthogonal_weight": -0.26274538040161133}, {"head": 5, "memory_to_active": 0.15181082487106323, "contribution_to_active": 0.4482137858867645, "mix_mean": 0.34154847264289856, "mix_low": 0.005469322204589844, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8395100831985474, "beta_mean": 0.006094461772590876, "nonfinite": 0.0, "weighted_memory_to_active": 0.0815034955739975, "mix_std": 0.22100380063056946, "beta_std": 0.009275143034756184, "beta_low": 0.835693359375, "beta_high": 0.0, "orthogonal_to_active": 0.01896781288087368, "evidence_weights": [-0.177202969789505, -0.1666383147239685, 0.2658904492855072], "agreement_weight": 0.09249583631753922, "orthogonal_weight": -0.5520488023757935}]}, {"layer": 11, "memory_to_active": 0.2615942358970642, "contribution_to_active": 0.45062363147735596, "mix_mean": 0.3762750029563904, "mix_low": 0.053419433534145355, "mix_high": 0.016517480835318565, "read_cosine": 0.5192199945449829, "beta_mean": 0.03882959485054016, "nonfinite": 0.0, "weighted_memory_to_active": 0.09084073454141617, "mix_std": 0.302212655544281, "beta_std": 0.05581095069646835, "beta_low": 0.488494873046875, "beta_high": 0.0, "orthogonal_to_active": 0.042133770883083344, "output_influence": 0.4554765522480011, "residual_influence": 0.11978469043970108, "state_active": 0.11701321601867676, "state_memory": 0.01191781647503376, "state_peak": 26.757171630859375, "growth_active": 1.1565515995025635, "growth_memory": 1.432801365852356, "heads": [{"head": 0, "memory_to_active": 0.02387763373553753, "contribution_to_active": 0.5272702574729919, "mix_mean": 0.5191734433174133, "mix_low": 0.0026721954345703125, "mix_high": 0.006188392639160156, "read_cosine": 0.43272221088409424, "beta_mean": 0.00014924989955034107, "nonfinite": 0.0, "weighted_memory_to_active": 0.014545930549502373, "mix_std": 0.27941155433654785, "beta_std": 0.004400098696351051, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.007899098098278046, "evidence_weights": [-0.1596992313861847, -0.31606435775756836, 0.5198233723640442], "agreement_weight": 0.13678251206874847, "orthogonal_weight": -0.6619293093681335}, {"head": 1, "memory_to_active": 0.5286012291908264, "contribution_to_active": 0.2325354516506195, "mix_mean": 0.24873942136764526, "mix_low": 0.030500411987304688, "mix_high": 0.00016880035400390625, "read_cosine": 0.6060135364532471, "beta_mean": 0.039591919630765915, "nonfinite": 0.0, "weighted_memory_to_active": 0.18613755702972412, "mix_std": 0.23790320754051208, "beta_std": 0.06732214242219925, "beta_low": 0.3199462890625, "beta_high": 0.0, "orthogonal_to_active": 0.02453383430838585, "evidence_weights": [-0.15445813536643982, -0.11076084524393082, 0.14556632936000824], "agreement_weight": 0.08950107544660568, "orthogonal_weight": -0.1823192685842514}, {"head": 2, "memory_to_active": 0.056617602705955505, "contribution_to_active": 0.8301777839660645, "mix_mean": 0.46158427000045776, "mix_low": 0.06511592864990234, "mix_high": 0.05751228332519531, "read_cosine": 0.3327847719192505, "beta_mean": 0.07416289299726486, "nonfinite": 0.0, "weighted_memory_to_active": 0.04562649875879288, "mix_std": 0.40989840030670166, "beta_std": 0.10117034614086151, "beta_low": 0.42840576171875, "beta_high": 0.0, "orthogonal_to_active": 0.10070588439702988, "evidence_weights": [-0.12467126548290253, -0.42563939094543457, -0.7436548471450806], "agreement_weight": -0.011270889081060886, "orthogonal_weight": 2.398383617401123}, {"head": 3, "memory_to_active": 0.027566242963075638, "contribution_to_active": 0.47342637181282043, "mix_mean": 0.4219989776611328, "mix_low": 0.1443958282470703, "mix_high": 0.03472423553466797, "read_cosine": 0.5726652145385742, "beta_mean": 0.06790521740913391, "nonfinite": 0.0, "weighted_memory_to_active": 0.01496544573456049, "mix_std": 0.4180857539176941, "beta_std": 0.09923112392425537, "beta_low": 0.4498291015625, "beta_high": 0.0, "orthogonal_to_active": 0.03159265220165253, "evidence_weights": [-0.1306118220090866, -0.3653180003166199, -0.25499841570854187], "agreement_weight": 0.020417368039488792, "orthogonal_weight": 2.609370470046997}, {"head": 4, "memory_to_active": 0.7461569309234619, "contribution_to_active": 0.26693961024284363, "mix_mean": 0.18927770853042603, "mix_low": 0.07676982879638672, "mix_high": 3.814697265625e-06, "read_cosine": 0.25415584444999695, "beta_mean": 0.03685706481337547, "nonfinite": 0.0, "weighted_memory_to_active": 0.19670888781547546, "mix_std": 0.2091827541589737, "beta_std": 0.04166898503899574, "beta_low": 0.16009521484375, "beta_high": 0.0, "orthogonal_to_active": 0.06727875024080276, "evidence_weights": [-0.0907883495092392, -0.07104241102933884, 1.0232958793640137], "agreement_weight": -0.0900811180472374, "orthogonal_weight": -0.3763827383518219}, {"head": 5, "memory_to_active": 0.18674562871456146, "contribution_to_active": 0.37339216470718384, "mix_mean": 0.41687604784965515, "mix_low": 0.0010623931884765625, "mix_high": 0.000507354736328125, "read_cosine": 0.9169781804084778, "beta_mean": 0.014311224222183228, "nonfinite": 0.0, "weighted_memory_to_active": 0.0870601087808609, "mix_std": 0.2587941288948059, "beta_std": 0.02107299119234085, "beta_low": 0.573486328125, "beta_high": 0.0, "orthogonal_to_active": 0.020792419090867043, "evidence_weights": [-0.1907576471567154, -0.2342451512813568, 0.3862520754337311], "agreement_weight": 0.04121233522891998, "orthogonal_weight": -0.5719768404960632}]}, {"layer": 12, "memory_to_active": 0.4791065454483032, "contribution_to_active": 0.5149927735328674, "mix_mean": 0.39672476053237915, "mix_low": 0.037317436188459396, "mix_high": 0.0021707217674702406, "read_cosine": 0.31043851375579834, "beta_mean": 0.01889028213918209, "nonfinite": 0.0, "weighted_memory_to_active": 0.22350820899009705, "mix_std": 0.24170497059822083, "beta_std": 0.0210718996822834, "beta_low": 0.7744649648666382, "beta_high": 0.0, "orthogonal_to_active": 0.07001777738332748, "output_influence": 0.36008796095848083, "residual_influence": 0.09701777994632721, "state_active": 0.04640632122755051, "state_memory": 0.013213081285357475, "state_peak": 29.966506958007812, "growth_active": 1.068442463874817, "growth_memory": 1.2263126373291016, "heads": [{"head": 0, "memory_to_active": 0.3073875308036804, "contribution_to_active": 0.3349696695804596, "mix_mean": 0.3134732246398926, "mix_low": 0.016257286071777344, "mix_high": 9.34600830078125e-05, "read_cosine": 0.34345725178718567, "beta_mean": 0.003855537623167038, "nonfinite": 0.0, "weighted_memory_to_active": 0.12484433501958847, "mix_std": 0.24530497193336487, "beta_std": 0.014292190782725811, "beta_low": 0.91900634765625, "beta_high": 0.0, "orthogonal_to_active": 0.04750465974211693, "evidence_weights": [-0.14063329994678497, -0.19971942901611328, 0.24943609535694122], "agreement_weight": 0.09354904294013977, "orthogonal_weight": -0.49508610367774963}, {"head": 1, "memory_to_active": 0.13968777656555176, "contribution_to_active": 0.4353291392326355, "mix_mean": 0.4244719445705414, "mix_low": 0.0010175704956054688, "mix_high": 0.000194549560546875, "read_cosine": 0.5933310985565186, "beta_mean": 0.0012901915470138192, "nonfinite": 0.0, "weighted_memory_to_active": 0.06442557275295258, "mix_std": 0.2575676739215851, "beta_std": 0.005351824685931206, "beta_low": 0.98944091796875, "beta_high": 0.0, "orthogonal_to_active": 0.044305071234703064, "evidence_weights": [-0.18362422287464142, -0.23294034600257874, 0.6693841218948364], "agreement_weight": 0.08960013091564178, "orthogonal_weight": -0.8799682855606079}, {"head": 2, "memory_to_active": 0.6223317384719849, "contribution_to_active": 0.41460251808166504, "mix_mean": 0.3137114346027374, "mix_low": 0.021829605102539062, "mix_high": 0.0007658004760742188, "read_cosine": 0.43289557099342346, "beta_mean": 0.09666646271944046, "nonfinite": 0.0, "weighted_memory_to_active": 0.26219576597213745, "mix_std": 0.24552664160728455, "beta_std": 0.07497882843017578, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.15460577607154846, "evidence_weights": [-0.2263842523097992, -0.05304000526666641, -0.022057484835386276], "agreement_weight": 0.039212923496961594, "orthogonal_weight": 0.7344381213188171}, {"head": 3, "memory_to_active": 0.03270920738577843, "contribution_to_active": 0.6867405772209167, "mix_mean": 0.6555455327033997, "mix_low": 0.00048065185546875, "mix_high": 0.009829521179199219, "read_cosine": 0.09247094392776489, "beta_mean": 0.000277855375315994, "nonfinite": 0.0, "weighted_memory_to_active": 0.0218760184943676, "mix_std": 0.24699966609477997, "beta_std": 0.0009774034842848778, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.018343139439821243, "evidence_weights": [-0.3090631663799286, -0.309569776058197, 0.6543833017349243], "agreement_weight": 0.18680450320243835, "orthogonal_weight": -0.8706842660903931}, {"head": 4, "memory_to_active": 1.7177677154541016, "contribution_to_active": 0.7934443950653076, "mix_mean": 0.1306002140045166, "mix_low": 0.1791849136352539, "mix_high": 0.0010766983032226562, "read_cosine": 0.008539395406842232, "beta_mean": 0.010604934766888618, "nonfinite": 0.0, "weighted_memory_to_active": 0.8424228429794312, "mix_std": 0.18191766738891602, "beta_std": 0.02838781476020813, "beta_low": 0.7481689453125, "beta_high": 0.0, "orthogonal_to_active": 0.13648644089698792, "evidence_weights": [0.023184677585959435, -0.12166627496480942, -9.871648217085749e-05], "agreement_weight": -0.0035676283296197653, "orthogonal_weight": -0.17690041661262512}, {"head": 5, "memory_to_active": 0.05475540831685066, "contribution_to_active": 0.42487025260925293, "mix_mean": 0.5425457954406738, "mix_low": 0.00513458251953125, "mix_high": 0.001064300537109375, "read_cosine": 0.391936719417572, "beta_mean": 0.0006467087659984827, "nonfinite": 0.0, "weighted_memory_to_active": 0.025284672155976295, "mix_std": 0.2729131877422333, "beta_std": 0.002443337347358465, "beta_low": 0.9903564453125, "beta_high": 0.0, "orthogonal_to_active": 0.018861550837755203, "evidence_weights": [-0.26363733410835266, -0.28026083111763, 0.7056828141212463], "agreement_weight": 0.07261999696493149, "orthogonal_weight": -0.8241118788719177}]}, {"layer": 13, "memory_to_active": 0.2160361111164093, "contribution_to_active": 0.4477815628051758, "mix_mean": 0.4479585886001587, "mix_low": 0.004890601150691509, "mix_high": 0.001115957973524928, "read_cosine": 0.39151859283447266, "beta_mean": 0.017726171761751175, "nonfinite": 0.0, "weighted_memory_to_active": 0.09619691967964172, "mix_std": 0.25545310974121094, "beta_std": 0.017507847398519516, "beta_low": 0.7710368037223816, "beta_high": 0.0, "orthogonal_to_active": 0.07885514199733734, "output_influence": 0.3951898217201233, "residual_influence": 0.12852713465690613, "state_active": 0.08285346627235413, "state_memory": 0.03558673337101936, "state_peak": 91.79261016845703, "growth_active": 0.9608997702598572, "growth_memory": 1.024938941001892, "heads": [{"head": 0, "memory_to_active": 0.13370801508426666, "contribution_to_active": 0.4631589353084564, "mix_mean": 0.4420040249824524, "mix_low": 0.00133514404296875, "mix_high": 0.0013341903686523438, "read_cosine": 0.3157726526260376, "beta_mean": 0.0009069289080798626, "nonfinite": 0.0, "weighted_memory_to_active": 0.0655129998922348, "mix_std": 0.2618177831172943, "beta_std": 0.0019391776295378804, "beta_low": 0.99072265625, "beta_high": 0.0, "orthogonal_to_active": 0.06172327324748039, "evidence_weights": [-0.20181027054786682, -0.22593523561954498, 0.7048728466033936], "agreement_weight": 0.04183272272348404, "orthogonal_weight": -1.0203382968902588}, {"head": 1, "memory_to_active": 0.08178651332855225, "contribution_to_active": 0.5007774829864502, "mix_mean": 0.4315970540046692, "mix_low": 0.0028362274169921875, "mix_high": 4.38690185546875e-05, "read_cosine": 0.24271129071712494, "beta_mean": 0.0004415297880768776, "nonfinite": 0.0, "weighted_memory_to_active": 0.04056887701153755, "mix_std": 0.2589148283004761, "beta_std": 0.0011751605197787285, "beta_low": 0.99871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.037220828235149384, "evidence_weights": [-0.18900664150714874, -0.2348109483718872, 0.7209779024124146], "agreement_weight": 0.10238620638847351, "orthogonal_weight": -0.9720580577850342}, {"head": 2, "memory_to_active": 0.219206303358078, "contribution_to_active": 0.32831087708473206, "mix_mean": 0.3286070227622986, "mix_low": 0.0021686553955078125, "mix_high": 4.1961669921875e-05, "read_cosine": 0.68574458360672, "beta_mean": 0.013953826390206814, "nonfinite": 0.0, "weighted_memory_to_active": 0.08337744325399399, "mix_std": 0.21564838290214539, "beta_std": 0.030008815228939056, "beta_low": 0.688232421875, "beta_high": 0.0, "orthogonal_to_active": 0.035155512392520905, "evidence_weights": [-0.21587897837162018, -0.1537521630525589, 0.4880419373512268], "agreement_weight": 0.04584881663322449, "orthogonal_weight": -0.6799716353416443}, {"head": 3, "memory_to_active": 0.18647485971450806, "contribution_to_active": 0.5203033089637756, "mix_mean": 0.558971107006073, "mix_low": 0.007457733154296875, "mix_high": 0.0026760101318359375, "read_cosine": -0.03693556785583496, "beta_mean": 0.00115145870950073, "nonfinite": 0.0, "weighted_memory_to_active": 0.13109487295150757, "mix_std": 0.2665809690952301, "beta_std": 0.005668829660862684, "beta_low": 0.98822021484375, "beta_high": 0.0, "orthogonal_to_active": 0.134272962808609, "evidence_weights": [-0.3524917662143707, -0.23277415335178375, 1.5341012477874756], "agreement_weight": 1.3838229179382324, "orthogonal_weight": -1.0403910875320435}, {"head": 4, "memory_to_active": 0.47521087527275085, "contribution_to_active": 0.30250200629234314, "mix_mean": 0.43826451897621155, "mix_low": 0.009438514709472656, "mix_high": 0.0009908676147460938, "read_cosine": 0.8660030364990234, "beta_mean": 0.08782400190830231, "nonfinite": 0.0, "weighted_memory_to_active": 0.15201549232006073, "mix_std": 0.24697481095790863, "beta_std": 0.05942937731742859, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.13813185691833496, "evidence_weights": [-0.2378523200750351, -0.2444521188735962, 0.47364935278892517], "agreement_weight": -0.00256986147724092, "orthogonal_weight": 1.321765661239624}, {"head": 5, "memory_to_active": 0.1998300701379776, "contribution_to_active": 0.5716366767883301, "mix_mean": 0.48830780386924744, "mix_low": 0.006107330322265625, "mix_high": 0.0016088485717773438, "read_cosine": 0.2758154273033142, "beta_mean": 0.0020792887080460787, "nonfinite": 0.0, "weighted_memory_to_active": 0.1046118438243866, "mix_std": 0.28278177976608276, "beta_std": 0.0068257153034210205, "beta_low": 0.9603271484375, "beta_high": 0.0, "orthogonal_to_active": 0.06662638485431671, "evidence_weights": [-0.21027567982673645, -0.24912184476852417, 0.41359245777130127], "agreement_weight": 0.06652645766735077, "orthogonal_weight": -0.7123682498931885}]}, {"layer": 14, "memory_to_active": 0.28064557909965515, "contribution_to_active": 0.43217137455940247, "mix_mean": 0.3980172872543335, "mix_low": 0.022666772827506065, "mix_high": 0.0036031408235430717, "read_cosine": 0.6168500185012817, "beta_mean": 0.06945444643497467, "nonfinite": 0.0, "weighted_memory_to_active": 0.1333625614643097, "mix_std": 0.25115931034088135, "beta_std": 0.06538070738315582, "beta_low": 0.4012959897518158, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04247787967324257, "output_influence": 0.2650712728500366, "residual_influence": 0.10027720779180527, "state_active": 0.07667078077793121, "state_memory": 0.016943400725722313, "state_peak": 29.53316879272461, "growth_active": 1.1617234945297241, "growth_memory": 1.3667138814926147, "heads": [{"head": 0, "memory_to_active": 0.30554434657096863, "contribution_to_active": 0.26410478353500366, "mix_mean": 0.2552281618118286, "mix_low": 0.014948844909667969, "mix_high": 4.38690185546875e-05, "read_cosine": 0.7273635864257812, "beta_mean": 0.062914177775383, "nonfinite": 0.0, "weighted_memory_to_active": 0.1008550226688385, "mix_std": 0.2234896868467331, "beta_std": 0.07548071444034576, "beta_low": 0.102783203125, "beta_high": 0.0, "orthogonal_to_active": 0.021505091339349747, "evidence_weights": [-0.06293075531721115, -0.11379290372133255, 0.7258459329605103], "agreement_weight": 0.044011954218149185, "orthogonal_weight": 0.32546019554138184}, {"head": 1, "memory_to_active": 0.47381144762039185, "contribution_to_active": 0.43476590514183044, "mix_mean": 0.2794911861419678, "mix_low": 0.09386062622070312, "mix_high": 0.0005235671997070312, "read_cosine": 0.6197577118873596, "beta_mean": 0.287038117647171, "nonfinite": 0.0, "weighted_memory_to_active": 0.23804166913032532, "mix_std": 0.27813395857810974, "beta_std": 0.2530447244644165, "beta_low": 0.0767822265625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.02337954379618168, "evidence_weights": [0.0014899305533617735, -0.26837122440338135, -0.04232065752148628], "agreement_weight": 0.05709696188569069, "orthogonal_weight": 0.1536024510860443}, {"head": 2, "memory_to_active": 0.26649996638298035, "contribution_to_active": 0.48956024646759033, "mix_mean": 0.4835115075111389, "mix_low": 0.004787445068359375, "mix_high": 0.0013151168823242188, "read_cosine": 0.40104126930236816, "beta_mean": 0.009387264959514141, "nonfinite": 0.0, "weighted_memory_to_active": 0.1534547209739685, "mix_std": 0.27115339040756226, "beta_std": 0.011144949123263359, "beta_low": 0.698486328125, "beta_high": 0.0, "orthogonal_to_active": 0.10969125479459763, "evidence_weights": [-0.19140642881393433, -0.23720912635326385, 0.6721351146697998], "agreement_weight": 0.08334391564130783, "orthogonal_weight": -0.8136007785797119}, {"head": 3, "memory_to_active": 0.07040775567293167, "contribution_to_active": 0.6557927131652832, "mix_mean": 0.6572363972663879, "mix_low": 0.0021734237670898438, "mix_high": 0.013004302978515625, "read_cosine": 0.4783627390861511, "beta_mean": 0.011620664037764072, "nonfinite": 0.0, "weighted_memory_to_active": 0.04809516668319702, "mix_std": 0.2424558699131012, "beta_std": 0.010472348891198635, "beta_low": 0.5634765625, "beta_high": 0.0, "orthogonal_to_active": 0.02293708547949791, "evidence_weights": [-0.38102272152900696, -0.32740452885627747, 0.42762911319732666], "agreement_weight": 0.07475671172142029, "orthogonal_weight": -0.6023232340812683}, {"head": 4, "memory_to_active": 0.2134505808353424, "contribution_to_active": 0.4787331223487854, "mix_mean": 0.45182347297668457, "mix_low": 0.003986358642578125, "mix_high": 0.0062236785888671875, "read_cosine": 0.7800267338752747, "beta_mean": 0.04178766906261444, "nonfinite": 0.0, "weighted_memory_to_active": 0.12059218436479568, "mix_std": 0.262231707572937, "beta_std": 0.029825519770383835, "beta_low": 0.0567626953125, "beta_high": 0.0, "orthogonal_to_active": 0.029320277273654938, "evidence_weights": [-0.2164444774389267, -0.26579713821411133, 0.3088570833206177], "agreement_weight": 0.10708819329738617, "orthogonal_weight": -0.4169938266277313}, {"head": 5, "memory_to_active": 0.35415932536125183, "contribution_to_active": 0.2700713276863098, "mix_mean": 0.26081275939941406, "mix_low": 0.016243934631347656, "mix_high": 0.0005083084106445312, "read_cosine": 0.6945478320121765, "beta_mean": 0.003978770226240158, "nonfinite": 0.0, "weighted_memory_to_active": 0.13913653790950775, "mix_std": 0.22949102520942688, "beta_std": 0.012316006235778332, "beta_low": 0.90948486328125, "beta_high": 0.0, "orthogonal_to_active": 0.04803401231765747, "evidence_weights": [-0.18493066728115082, -0.13654664158821106, 0.1375875174999237], "agreement_weight": 0.09478422999382019, "orthogonal_weight": -0.5557712912559509}]}, {"layer": 15, "memory_to_active": 0.07775256037712097, "contribution_to_active": 0.46300509572029114, "mix_mean": 0.3889213800430298, "mix_low": 0.006324768532067537, "mix_high": 0.0021452903747558594, "read_cosine": 0.6730901598930359, "beta_mean": 0.04342591017484665, "nonfinite": 0.0, "weighted_memory_to_active": 0.03507358208298683, "mix_std": 0.2723003029823303, "beta_std": 0.047137610614299774, "beta_low": 0.7071635127067566, "beta_high": 0.0, "orthogonal_to_active": 0.007592420559376478, "output_influence": 0.2952514588832855, "residual_influence": 0.15392687916755676, "state_active": 0.10870922356843948, "state_memory": 0.008431063033640385, "state_peak": 28.767728805541992, "growth_active": 1.2875274419784546, "growth_memory": 1.6899876594543457, "heads": [{"head": 0, "memory_to_active": 0.10320602357387543, "contribution_to_active": 0.4784761667251587, "mix_mean": 0.34656208753585815, "mix_low": 0.004955291748046875, "mix_high": 0.0020151138305664062, "read_cosine": 0.745182991027832, "beta_mean": 0.037882693111896515, "nonfinite": 0.0, "weighted_memory_to_active": 0.048798903822898865, "mix_std": 0.258137047290802, "beta_std": 0.029996130615472794, "beta_low": 0.126220703125, "beta_high": 0.0, "orthogonal_to_active": 0.013364359736442566, "evidence_weights": [-0.16447801887989044, -0.19893711805343628, 0.3464609682559967], "agreement_weight": 0.08414972573518753, "orthogonal_weight": -0.5171559453010559}, {"head": 1, "memory_to_active": 0.1859065294265747, "contribution_to_active": 0.40158313512802124, "mix_mean": 0.37614330649375916, "mix_low": 0.0061855316162109375, "mix_high": 0.004734992980957031, "read_cosine": 0.8451638221740723, "beta_mean": 0.0017376483883708715, "nonfinite": 0.0, "weighted_memory_to_active": 0.07233073562383652, "mix_std": 0.2713354229927063, "beta_std": 0.0037337718531489372, "beta_low": 0.9781494140625, "beta_high": 0.0, "orthogonal_to_active": 0.005531918257474899, "evidence_weights": [-0.19163364171981812, -0.19562631845474243, 0.2827649414539337], "agreement_weight": 0.09570861607789993, "orthogonal_weight": -0.11648466438055038}, {"head": 2, "memory_to_active": 0.009502429515123367, "contribution_to_active": 0.43900009989738464, "mix_mean": 0.35809260606765747, "mix_low": 0.008522987365722656, "mix_high": 0.00015926361083984375, "read_cosine": 0.7937624454498291, "beta_mean": 0.00017146870959550142, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038961037062108517, "mix_std": 0.26904788613319397, "beta_std": 0.0005229118396528065, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.0007835190044716001, "evidence_weights": [-0.21012067794799805, -0.27264320850372314, 0.2885117530822754], "agreement_weight": 0.041995447129011154, "orthogonal_weight": -0.32148170471191406}, {"head": 3, "memory_to_active": 0.06388905644416809, "contribution_to_active": 0.444504976272583, "mix_mean": 0.32012003660202026, "mix_low": 0.0031566619873046875, "mix_high": 1.9073486328125e-05, "read_cosine": 0.6943334937095642, "beta_mean": 0.0013262005522847176, "nonfinite": 0.0, "weighted_memory_to_active": 0.02884560450911522, "mix_std": 0.24406728148460388, "beta_std": 0.002852687146514654, "beta_low": 0.977783203125, "beta_high": 0.0, "orthogonal_to_active": 0.015446385368704796, "evidence_weights": [-0.17973467707633972, -0.22274230420589447, 0.4395928680896759], "agreement_weight": 0.01502224151045084, "orthogonal_weight": -0.7772126197814941}, {"head": 4, "memory_to_active": 0.09995003044605255, "contribution_to_active": 0.5530945062637329, "mix_mean": 0.41739681363105774, "mix_low": 0.010054588317871094, "mix_high": 0.000926971435546875, "read_cosine": 0.42882558703422546, "beta_mean": 0.21933555603027344, "nonfinite": 0.0, "weighted_memory_to_active": 0.054851822555065155, "mix_std": 0.3095903694629669, "beta_std": 0.2455340474843979, "beta_low": 0.161376953125, "beta_high": 0.0, "orthogonal_to_active": 0.009135554544627666, "evidence_weights": [-0.26036524772644043, -0.25064441561698914, 0.2363671362400055], "agreement_weight": 0.007963123731315136, "orthogonal_weight": -0.21340471506118774}, {"head": 5, "memory_to_active": 0.0040612551383674145, "contribution_to_active": 0.46137160062789917, "mix_mean": 0.5152132511138916, "mix_low": 0.00507354736328125, "mix_high": 0.005016326904296875, "read_cosine": 0.5312724113464355, "beta_mean": 0.00010188606393057853, "nonfinite": 0.0, "weighted_memory_to_active": 0.0017183253075927496, "mix_std": 0.28162360191345215, "beta_std": 0.0001861242053564638, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001292785513214767, "evidence_weights": [-0.3616291284561157, -0.29864072799682617, 0.3891849219799042], "agreement_weight": 0.12894940376281738, "orthogonal_weight": 0.8446988463401794}]}]}, {"step": 16000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.778339964999759, "memory_to_active": 0.9837832450866699, "contribution_to_active": 0.4455021023750305, "mix_mean": 0.30441510677337646, "mix_low": 0.17273473739624023, "mix_high": 0.005153040401637554, "read_cosine": 0.3973504304885864, "beta_mean": 0.09501641988754272, "nonfinite": 0.0, "weighted_memory_to_active": 0.2030547857284546, "mix_std": 0.2093692421913147, "beta_std": 0.07588212937116623, "beta_low": 0.45133593678474426, "beta_high": 1.4623006791225635e-05, "orthogonal_to_active": 0.04086025804281235, "samples": 4, "scored_tokens": 899, "normal_loss": 3.171722888946533, "normal_token_accuracy": 0.4304783046245575, "state_active": 0.06008689105510712, "state_memory": 0.023529119789600372, "state_peak": 104.47718048095703, "growth_active": 1.142276644706726, "growth_memory": 1.38402259349823, "output_influence": 0.3076280653476715, "residual_influence": 0.09728918969631195, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.255784273147583, "no_memory_loss_delta": 0.143346905708313, "scrambled_loss_delta": 0.14909695088863373, "memory_only_loss_delta": 2.293271541595459, "no_evidence_loss_delta": 0.07653464376926422, "no_agreement_loss_delta": 0.004233572632074356, "no_orthogonal_loss_delta": 0.002266431227326393}, {"index": 94, "tokens": 256, "normal_loss": 3.235344409942627, "no_memory_loss_delta": 0.3025071620941162, "scrambled_loss_delta": 0.4216829538345337, "memory_only_loss_delta": 2.37626051902771, "no_evidence_loss_delta": 0.13431765139102936, "no_agreement_loss_delta": 0.028211161494255066, "no_orthogonal_loss_delta": 0.02507925033569336}, {"index": 188, "tokens": 256, "normal_loss": 3.5825204849243164, "no_memory_loss_delta": 0.18389618396759033, "scrambled_loss_delta": 0.11733649671077728, "memory_only_loss_delta": 2.1740880012512207, "no_evidence_loss_delta": 0.10199062526226044, "no_agreement_loss_delta": -0.002681662794202566, "no_orthogonal_loss_delta": 0.002851699246093631}, {"index": 283, "tokens": 131, "normal_loss": 2.080343008041382, "no_memory_loss_delta": 0.11073523759841919, "scrambled_loss_delta": 0.29768267273902893, "memory_only_loss_delta": 2.397491693496704, "no_evidence_loss_delta": 0.0012979853199794888, "no_agreement_loss_delta": -0.0008462835685350001, "no_orthogonal_loss_delta": -0.018067875877022743}], "agreement_weight_abs_mean": 0.08651388436555862, "agreement_weight_abs_max": 1.5211271047592163, "agreement_weight_mean": 0.07086168974637985, "orthogonal_weight_abs_mean": 0.48104986548423767, "orthogonal_weight_abs_max": 2.788325071334839, "orthogonal_weight_mean": -0.16705022752285004, "no_memory_loss": 3.367187261581421, "no_memory_loss_delta": 0.19546426832675934, "no_memory_token_accuracy_gain": 0.004449388012290001, "no_memory_token_win_fraction": 0.5895439386367798, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 17.75472693486181, "scrambled_loss": 3.4110491275787354, "scrambled_loss_delta": 0.23932616412639618, "scrambled_token_accuracy_gain": 0.026696329936385155, "scrambled_token_win_fraction": 0.5995550751686096, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 21.28419022352924, "memory_only_loss": 5.469874382019043, "memory_only_loss_delta": 2.2981514930725098, "memory_only_token_accuracy_gain": 0.22358176112174988, "memory_only_token_win_fraction": 0.8865405917167664, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 89.95556557133148, "no_evidence_loss": 3.260997772216797, "no_evidence_loss_delta": 0.08927455544471741, "no_evidence_token_accuracy_gain": -0.004449388012290001, "no_evidence_token_win_fraction": 0.5339266061782837, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 8.54055677811576, "no_agreement_loss": 3.1800754070281982, "no_agreement_loss_delta": 0.008352039381861687, "no_agreement_token_accuracy_gain": -0.004449388012290001, "no_agreement_token_win_fraction": 0.44160178303718567, "no_agreement_probe_win_fraction": 0.5, "no_agreement_ppl_reduction_pct": 0.8317258000124659, "no_orthogonal_loss": 3.17768931388855, "no_orthogonal_loss_delta": 0.005966227035969496, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5494994521141052, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.5948464446217729, "short_no_memory_loss_delta": 0.16566941142082214, "long_context_memory_gain": 0.029794856905937195, "normal_context_loss_gain": 0.14267702400684357, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.003302812576294, "contribution_to_active": 0.7926848530769348, "mix_mean": 0.09456969797611237, "mix_low": 0.46533286571502686, "mix_high": 0.003131548874080181, "read_cosine": 0.47495585680007935, "beta_mean": 0.27445539832115173, "nonfinite": 0.0, "weighted_memory_to_active": 0.839084804058075, "mix_std": 0.08279835432767868, "beta_std": 0.15947291254997253, "beta_low": 0.00323486328125, "beta_high": 0.0, "orthogonal_to_active": 0.03784472867846489, "output_influence": 0.12674164772033691, "residual_influence": 0.18515212833881378, "state_active": 0.08491470664739609, "state_memory": 0.051016878336668015, "state_peak": 51.55500793457031, "growth_active": 1.2942750453948975, "growth_memory": 1.283237099647522, "heads": [{"head": 0, "memory_to_active": 2.407487154006958, "contribution_to_active": 0.35871970653533936, "mix_mean": 0.02082796022295952, "mix_low": 0.5254974365234375, "mix_high": 0.0, "read_cosine": 0.3342715799808502, "beta_mean": 0.21512869000434875, "nonfinite": 0.0, "weighted_memory_to_active": 0.44589343667030334, "mix_std": 0.04969773441553116, "beta_std": 0.09833056479692459, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0337873250246048, "evidence_weights": [-0.04312673583626747, -0.0755947157740593, -0.06667342782020569], "agreement_weight": -0.0818139985203743, "orthogonal_weight": -0.09064080566167831}, {"head": 1, "memory_to_active": 3.8772668838500977, "contribution_to_active": 1.3487374782562256, "mix_mean": 0.01860050857067108, "mix_low": 0.5278339385986328, "mix_high": 0.0, "read_cosine": 0.4685550928115845, "beta_mean": 0.38144683837890625, "nonfinite": 0.0, "weighted_memory_to_active": 1.6779156923294067, "mix_std": 0.04531228169798851, "beta_std": 0.1571807563304901, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1164628341794014, "evidence_weights": [0.01095200888812542, -0.0401376374065876, 0.17544612288475037], "agreement_weight": 0.01247665099799633, "orthogonal_weight": -0.16757813096046448}, {"head": 2, "memory_to_active": 2.617666721343994, "contribution_to_active": 0.4943564534187317, "mix_mean": 0.015099611133337021, "mix_low": 0.6274185180664062, "mix_high": 0.0, "read_cosine": 0.331825852394104, "beta_mean": 0.3407246470451355, "nonfinite": 0.0, "weighted_memory_to_active": 0.5666022300720215, "mix_std": 0.03140885755419731, "beta_std": 0.2522563636302948, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.058031730353832245, "evidence_weights": [0.04326840490102768, 0.04904969036579132, 0.002131973160430789], "agreement_weight": -0.005049595609307289, "orthogonal_weight": -0.16758401691913605}, {"head": 3, "memory_to_active": 5.391897201538086, "contribution_to_active": 2.1305556297302246, "mix_mean": 0.04835505783557892, "mix_low": 0.27035045623779297, "mix_high": 0.0, "read_cosine": 0.42605963349342346, "beta_mean": 0.3536938428878784, "nonfinite": 0.0, "weighted_memory_to_active": 2.217988967895508, "mix_std": 0.08453462272882462, "beta_std": 0.2087394744157791, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005764790694229305, "evidence_weights": [-0.09624991565942764, -0.12402883917093277, -0.03752925992012024], "agreement_weight": -0.01767471432685852, "orthogonal_weight": 0.0003416429099161178}, {"head": 4, "memory_to_active": 3.5844409465789795, "contribution_to_active": 0.07459472119808197, "mix_mean": 0.006859129294753075, "mix_low": 0.8144369125366211, "mix_high": 0.0, "read_cosine": 0.33609694242477417, "beta_mean": 0.2944783568382263, "nonfinite": 0.0, "weighted_memory_to_active": 0.06851484626531601, "mix_std": 0.012214860878884792, "beta_std": 0.18705256283283234, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.009423986077308655, "evidence_weights": [0.07266999036073685, 0.025660738348960876, -0.2915012240409851], "agreement_weight": -0.03995194658637047, "orthogonal_weight": 0.15502238273620605}, {"head": 5, "memory_to_active": 0.1410578340291977, "contribution_to_active": 0.34914493560791016, "mix_mean": 0.45767587423324585, "mix_low": 0.026459693908691406, "mix_high": 0.018789291381835938, "read_cosine": 0.9529260396957397, "beta_mean": 0.061259958893060684, "nonfinite": 0.0, "weighted_memory_to_active": 0.05759317800402641, "mix_std": 0.27362173795700073, "beta_std": 0.05327768623828888, "beta_low": 0.0194091796875, "beta_high": 0.0, "orthogonal_to_active": 0.008785994723439217, "evidence_weights": [-0.2896239459514618, -0.27509835362434387, 0.28534433245658875], "agreement_weight": 0.1695067286491394, "orthogonal_weight": -0.35238024592399597}]}, {"layer": 1, "memory_to_active": 2.344672203063965, "contribution_to_active": 0.3963303864002228, "mix_mean": 0.11169502139091492, "mix_low": 0.3994058072566986, "mix_high": 0.0004774729604832828, "read_cosine": 0.3289135694503784, "beta_mean": 0.2160032093524933, "nonfinite": 0.0, "weighted_memory_to_active": 0.3271214962005615, "mix_std": 0.11134768277406693, "beta_std": 0.13649654388427734, "beta_low": 0.1694030910730362, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03483777493238449, "output_influence": 0.13752812147140503, "residual_influence": 0.03173099830746651, "state_active": 0.04464441537857056, "state_memory": 0.04531564936041832, "state_peak": 24.419681549072266, "growth_active": 1.1583503484725952, "growth_memory": 1.5329574346542358, "heads": [{"head": 0, "memory_to_active": 0.046283334493637085, "contribution_to_active": 0.3167349100112915, "mix_mean": 0.2582702040672302, "mix_low": 0.03632545471191406, "mix_high": 3.24249267578125e-05, "read_cosine": -0.052793391048908234, "beta_mean": 0.0002328681293874979, "nonfinite": 0.0, "weighted_memory_to_active": 0.018598152324557304, "mix_std": 0.2522767186164856, "beta_std": 0.000619767582975328, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.010278764180839062, "evidence_weights": [-0.13201797008514404, -0.16400104761123657, 0.03101065568625927], "agreement_weight": 0.12868496775627136, "orthogonal_weight": -0.6216769218444824}, {"head": 1, "memory_to_active": 3.409188747406006, "contribution_to_active": 0.15546035766601562, "mix_mean": 0.0215715654194355, "mix_low": 0.46929359436035156, "mix_high": 0.0, "read_cosine": 0.4420987665653229, "beta_mean": 0.35588735342025757, "nonfinite": 0.0, "weighted_memory_to_active": 0.16520532965660095, "mix_std": 0.04646334797143936, "beta_std": 0.2227167934179306, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00553237646818161, "evidence_weights": [-0.010418194346129894, 0.005294837523251772, 0.0012697812635451555], "agreement_weight": -0.011181498877704144, "orthogonal_weight": 0.0561990886926651}, {"head": 2, "memory_to_active": 0.5966190695762634, "contribution_to_active": 0.3916759788990021, "mix_mean": 0.3363720774650574, "mix_low": 0.006493568420410156, "mix_high": 0.0028257369995117188, "read_cosine": 0.8008536696434021, "beta_mean": 0.07891137897968292, "nonfinite": 0.0, "weighted_memory_to_active": 0.42908743023872375, "mix_std": 0.24958303570747375, "beta_std": 0.03865678980946541, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.004173655528575182, "evidence_weights": [-0.2175835818052292, -0.3737410306930542, 0.2069186568260193], "agreement_weight": 0.00999278575181961, "orthogonal_weight": -0.020257513970136642}, {"head": 3, "memory_to_active": 2.9509944915771484, "contribution_to_active": 0.06028776615858078, "mix_mean": 0.01016453467309475, "mix_low": 0.7654886245727539, "mix_high": 0.0, "read_cosine": 0.19677335023880005, "beta_mean": 0.3989408612251282, "nonfinite": 0.0, "weighted_memory_to_active": 0.07557807117700577, "mix_std": 0.027321696281433105, "beta_std": 0.2473216950893402, "beta_low": 0.000732421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.016379818320274353, "evidence_weights": [0.037738095968961716, 0.015345516614615917, 0.02301209233701229], "agreement_weight": -0.017213590443134308, "orthogonal_weight": -0.2298446148633957}, {"head": 4, "memory_to_active": 3.1942319869995117, "contribution_to_active": 1.4000194072723389, "mix_mean": 0.03624545410275459, "mix_low": 0.29343700408935547, "mix_high": 1.9073486328125e-06, "read_cosine": 0.40124163031578064, "beta_mean": 0.188925102353096, "nonfinite": 0.0, "weighted_memory_to_active": 1.2237106561660767, "mix_std": 0.06669710576534271, "beta_std": 0.10228417813777924, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17135578393936157, "evidence_weights": [-0.02360294759273529, -0.01187354139983654, 0.09937124699354172], "agreement_weight": -0.0369737446308136, "orthogonal_weight": 0.160659059882164}, {"head": 5, "memory_to_active": 3.870715379714966, "contribution_to_active": 0.053803637623786926, "mix_mean": 0.007546243257820606, "mix_low": 0.8253965377807617, "mix_high": 4.76837158203125e-06, "read_cosine": 0.1853073239326477, "beta_mean": 0.27312159538269043, "nonfinite": 0.0, "weighted_memory_to_active": 0.05054916441440582, "mix_std": 0.025744151324033737, "beta_std": 0.207380011677742, "beta_low": 0.0155029296875, "beta_high": 0.0, "orthogonal_to_active": 0.001306264428421855, "evidence_weights": [0.1306767463684082, 0.06340567022562027, 0.31835994124412537], "agreement_weight": -0.13862550258636475, "orthogonal_weight": 0.025873709470033646}]}, {"layer": 2, "memory_to_active": 1.2027053833007812, "contribution_to_active": 0.29817867279052734, "mix_mean": 0.2622547745704651, "mix_low": 0.2716129720211029, "mix_high": 0.002536773681640625, "read_cosine": 0.3453405201435089, "beta_mean": 0.1176706999540329, "nonfinite": 0.0, "weighted_memory_to_active": 0.07525994628667831, "mix_std": 0.19246554374694824, "beta_std": 0.07620828598737717, "beta_low": 0.54766845703125, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.016379741951823235, "output_influence": 0.25489354133605957, "residual_influence": 0.05995919555425644, "state_active": 0.04101363569498062, "state_memory": 0.0244818776845932, "state_peak": 47.92687225341797, "growth_active": 1.0897325277328491, "growth_memory": 1.554211139678955, "heads": [{"head": 0, "memory_to_active": 0.017049897462129593, "contribution_to_active": 0.17602990567684174, "mix_mean": 0.31978386640548706, "mix_low": 0.016771316528320312, "mix_high": 0.0012292861938476562, "read_cosine": 0.24708989262580872, "beta_mean": 0.00011419433576520532, "nonfinite": 0.0, "weighted_memory_to_active": 0.006874836049973965, "mix_std": 0.279355525970459, "beta_std": 0.00019460011390037835, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00021629960974678397, "evidence_weights": [-0.18500709533691406, -0.12358458340167999, 0.19770082831382751], "agreement_weight": 0.07581906020641327, "orthogonal_weight": -0.03724348917603493}, {"head": 1, "memory_to_active": 3.3364734649658203, "contribution_to_active": 0.056113678961992264, "mix_mean": 0.004919928498566151, "mix_low": 0.9017658233642578, "mix_high": 9.5367431640625e-07, "read_cosine": 0.17741860449314117, "beta_mean": 0.37147587537765503, "nonfinite": 0.0, "weighted_memory_to_active": 0.0507270023226738, "mix_std": 0.012396168895065784, "beta_std": 0.16351063549518585, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012862707488238811, "evidence_weights": [0.0953485444188118, 0.10176915675401688, 0.23216742277145386], "agreement_weight": 0.03383325785398483, "orthogonal_weight": 0.30287960171699524}, {"head": 2, "memory_to_active": 0.05152719467878342, "contribution_to_active": 0.43125391006469727, "mix_mean": 0.39019906520843506, "mix_low": 0.004046440124511719, "mix_high": 0.0018167495727539062, "read_cosine": 0.2886906862258911, "beta_mean": 8.436707139480859e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.02064855955541134, "mix_std": 0.27833977341651917, "beta_std": 0.0002521820133551955, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003818379482254386, "evidence_weights": [-0.18832768499851227, -0.22764328122138977, 0.4837613105773926], "agreement_weight": 0.027803640812635422, "orthogonal_weight": -0.2279493510723114}, {"head": 3, "memory_to_active": 0.2795286774635315, "contribution_to_active": 0.42309486865997314, "mix_mean": 0.3214070796966553, "mix_low": 0.0051250457763671875, "mix_high": 0.0044651031494140625, "read_cosine": 0.6592815518379211, "beta_mean": 0.0385456457734108, "nonfinite": 0.0, "weighted_memory_to_active": 0.12362286448478699, "mix_std": 0.25547268986701965, "beta_std": 0.053948260843753815, "beta_low": 0.24188232421875, "beta_high": 0.0, "orthogonal_to_active": 0.06739862263202667, "evidence_weights": [-0.1280970275402069, -0.1948414444923401, 0.661474883556366], "agreement_weight": 0.07032249122858047, "orthogonal_weight": -0.7821795344352722}, {"head": 4, "memory_to_active": 3.496797561645508, "contribution_to_active": 0.19275471568107605, "mix_mean": 0.014308903366327286, "mix_low": 0.6995649337768555, "mix_high": 0.0, "read_cosine": 0.19219860434532166, "beta_mean": 0.29572272300720215, "nonfinite": 0.0, "weighted_memory_to_active": 0.22835448384284973, "mix_std": 0.0455327183008194, "beta_std": 0.23922249674797058, "beta_low": 0.04412841796875, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.012368819676339626, "evidence_weights": [0.04200621321797371, 0.03565188869833946, 0.15108218789100647], "agreement_weight": 0.02944157086312771, "orthogonal_weight": -0.06275925785303116}, {"head": 5, "memory_to_active": 0.0348554365336895, "contribution_to_active": 0.5098248720169067, "mix_mean": 0.5229097604751587, "mix_low": 0.0024042129516601562, "mix_high": 0.007708549499511719, "read_cosine": 0.5073637962341309, "beta_mean": 8.140920544974506e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.021331900730729103, "mix_std": 0.2836962044239044, "beta_std": 0.0001215323936776258, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016136214835569263, "evidence_weights": [-0.24264851212501526, -0.25275951623916626, 0.6666716933250427], "agreement_weight": 0.027770865708589554, "orthogonal_weight": 0.13511647284030914}]}, {"layer": 3, "memory_to_active": 1.3307253122329712, "contribution_to_active": 0.32567012310028076, "mix_mean": 0.19582656025886536, "mix_low": 0.26138418912887573, "mix_high": 0.002728621242567897, "read_cosine": 0.2685061991214752, "beta_mean": 0.09836384654045105, "nonfinite": 0.0, "weighted_memory_to_active": 0.14259091019630432, "mix_std": 0.20026236772537231, "beta_std": 0.07529847323894501, "beta_low": 0.394744873046875, "beta_high": 0.0, "orthogonal_to_active": 0.03270381689071655, "output_influence": 0.23972126841545105, "residual_influence": 0.04265092313289642, "state_active": 0.031108442693948746, "state_memory": 0.021188488230109215, "state_peak": 26.511621475219727, "growth_active": 1.0729249715805054, "growth_memory": 1.8052690029144287, "heads": [{"head": 0, "memory_to_active": 0.7480148077011108, "contribution_to_active": 0.340725302696228, "mix_mean": 0.2206983119249344, "mix_low": 0.1295757293701172, "mix_high": 0.001430511474609375, "read_cosine": 0.23349133133888245, "beta_mean": 0.01847969926893711, "nonfinite": 0.0, "weighted_memory_to_active": 0.21358740329742432, "mix_std": 0.27165693044662476, "beta_std": 0.01582258939743042, "beta_low": 0.32989501953125, "beta_high": 0.0, "orthogonal_to_active": 0.083292655646801, "evidence_weights": [-0.06806045770645142, -0.13870738446712494, 0.019817998632788658], "agreement_weight": 0.08893471211194992, "orthogonal_weight": -0.4024171531200409}, {"head": 1, "memory_to_active": 0.03222567215561867, "contribution_to_active": 0.28203660249710083, "mix_mean": 0.26060324907302856, "mix_low": 0.0637350082397461, "mix_high": 0.002269744873046875, "read_cosine": 0.20894187688827515, "beta_mean": 0.00013981980737298727, "nonfinite": 0.0, "weighted_memory_to_active": 0.00922270119190216, "mix_std": 0.27675098180770874, "beta_std": 0.00036469317274168134, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0004646379384212196, "evidence_weights": [-0.1457696408033371, -0.11623890697956085, 0.3574088215827942], "agreement_weight": 0.10493776947259903, "orthogonal_weight": -0.055130258202552795}, {"head": 2, "memory_to_active": 3.2146544456481934, "contribution_to_active": 0.2591221034526825, "mix_mean": 0.03219097852706909, "mix_low": 0.47006893157958984, "mix_high": 3.814697265625e-06, "read_cosine": 0.17280124127864838, "beta_mean": 0.21651563048362732, "nonfinite": 0.0, "weighted_memory_to_active": 0.3289932310581207, "mix_std": 0.07874815165996552, "beta_std": 0.14257685840129852, "beta_low": 0.00390625, "beta_high": 0.0, "orthogonal_to_active": 0.02286611869931221, "evidence_weights": [-0.03710830584168434, 0.020546309649944305, 0.057149164378643036], "agreement_weight": -0.02520090527832508, "orthogonal_weight": -0.08772232383489609}, {"head": 3, "memory_to_active": 0.034022435545921326, "contribution_to_active": 0.4525284171104431, "mix_mean": 0.44678887724876404, "mix_low": 0.030704498291015625, "mix_high": 0.008190155029296875, "read_cosine": 0.28355181217193604, "beta_mean": 0.00012713560136035085, "nonfinite": 0.0, "weighted_memory_to_active": 0.019205879420042038, "mix_std": 0.31694793701171875, "beta_std": 0.00031502838828600943, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.005316015332937241, "evidence_weights": [-0.1831008493900299, -0.20517104864120483, 1.1501529216766357], "agreement_weight": 0.6433047652244568, "orthogonal_weight": -0.3622206747531891}, {"head": 4, "memory_to_active": 3.5284852981567383, "contribution_to_active": 0.04452243819832802, "mix_mean": 0.00773556437343359, "mix_low": 0.8054561614990234, "mix_high": 0.0, "read_cosine": 0.2022787481546402, "beta_mean": 0.2537279725074768, "nonfinite": 0.0, "weighted_memory_to_active": 0.06054455786943436, "mix_std": 0.019145073369145393, "beta_std": 0.1846127212047577, "beta_low": 0.0013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.021678553894162178, "evidence_weights": [0.14865945279598236, 0.1154697835445404, -0.06956388056278229], "agreement_weight": -0.05914486572146416, "orthogonal_weight": -0.3851591944694519}, {"head": 5, "memory_to_active": 0.4269491136074066, "contribution_to_active": 0.5750858783721924, "mix_mean": 0.2069423794746399, "mix_low": 0.06876468658447266, "mix_high": 0.004477500915527344, "read_cosine": 0.5099721550941467, "beta_mean": 0.10119280219078064, "nonfinite": 0.0, "weighted_memory_to_active": 0.22399167716503143, "mix_std": 0.2383250892162323, "beta_std": 0.10809893161058426, "beta_low": 0.03369140625, "beta_high": 0.0, "orthogonal_to_active": 0.06260489672422409, "evidence_weights": [-0.09178242832422256, -0.1660064458847046, 0.21438980102539062], "agreement_weight": 0.16264471411705017, "orthogonal_weight": -0.4612688720226288}]}, {"layer": 4, "memory_to_active": 1.2871718406677246, "contribution_to_active": 0.38424405455589294, "mix_mean": 0.28144705295562744, "mix_low": 0.2711409032344818, "mix_high": 0.0022110939025878906, "read_cosine": 0.40790116786956787, "beta_mean": 0.1377500295639038, "nonfinite": 0.0, "weighted_memory_to_active": 0.20114260911941528, "mix_std": 0.16135495901107788, "beta_std": 0.11091845482587814, "beta_low": 0.2861735224723816, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.048079948872327805, "output_influence": 0.26011279225349426, "residual_influence": 0.06114441901445389, "state_active": 0.04419553652405739, "state_memory": 0.030875859782099724, "state_peak": 46.30631637573242, "growth_active": 1.1332114934921265, "growth_memory": 1.6261266469955444, "heads": [{"head": 0, "memory_to_active": 0.37763267755508423, "contribution_to_active": 0.5725609064102173, "mix_mean": 0.5286791920661926, "mix_low": 0.010608673095703125, "mix_high": 0.0030994415283203125, "read_cosine": 0.5844389796257019, "beta_mean": 0.19630131125450134, "nonfinite": 0.0, "weighted_memory_to_active": 0.2658415734767914, "mix_std": 0.3179059624671936, "beta_std": 0.21486513316631317, "beta_low": 0.15960693359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.028174620121717453, "evidence_weights": [-0.22688263654708862, -0.2837406098842621, 0.44036048650741577], "agreement_weight": 0.05170050263404846, "orthogonal_weight": 0.14532068371772766}, {"head": 1, "memory_to_active": 0.31114423274993896, "contribution_to_active": 0.40553590655326843, "mix_mean": 0.4513130486011505, "mix_low": 0.00180816650390625, "mix_high": 0.001918792724609375, "read_cosine": 0.6074737310409546, "beta_mean": 0.015633605420589447, "nonfinite": 0.0, "weighted_memory_to_active": 0.15303277969360352, "mix_std": 0.2686002850532532, "beta_std": 0.024861421436071396, "beta_low": 0.5194091796875, "beta_high": 0.0, "orthogonal_to_active": 0.07589463144540787, "evidence_weights": [-0.2262154519557953, -0.20751598477363586, 0.6562179923057556], "agreement_weight": 0.04340362548828125, "orthogonal_weight": -0.6464848518371582}, {"head": 2, "memory_to_active": 2.5569961071014404, "contribution_to_active": 0.12800128757953644, "mix_mean": 0.014829403720796108, "mix_low": 0.6140661239624023, "mix_high": 0.0, "read_cosine": 0.31427156925201416, "beta_mean": 0.1254827082157135, "nonfinite": 0.0, "weighted_memory_to_active": 0.17647574841976166, "mix_std": 0.025036858394742012, "beta_std": 0.09688013792037964, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.004677152261137962, "evidence_weights": [0.04330473393201828, 0.021870989352464676, -0.04047393426299095], "agreement_weight": 0.06523498147726059, "orthogonal_weight": -0.043543290346860886}, {"head": 3, "memory_to_active": 1.6856420040130615, "contribution_to_active": 0.1263870894908905, "mix_mean": 0.03385446220636368, "mix_low": 0.3824644088745117, "mix_high": 0.0, "read_cosine": 0.2508702278137207, "beta_mean": 0.19638720154762268, "nonfinite": 0.0, "weighted_memory_to_active": 0.2318369746208191, "mix_std": 0.06138006970286369, "beta_std": 0.15790098905563354, "beta_low": 0.0369873046875, "beta_high": 0.0, "orthogonal_to_active": 0.020541230216622353, "evidence_weights": [-0.016999462619423866, 0.04692849516868591, -0.1825520247220993], "agreement_weight": 0.03601042553782463, "orthogonal_weight": -0.16927412152290344}, {"head": 4, "memory_to_active": 0.02692086063325405, "contribution_to_active": 0.6098872423171997, "mix_mean": 0.6441518068313599, "mix_low": 0.0011348724365234375, "mix_high": 0.008220672607421875, "read_cosine": 0.31167513132095337, "beta_mean": 5.425757262855768e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.02001068741083145, "mix_std": 0.25461000204086304, "beta_std": 0.0001251662033610046, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019073312869295478, "evidence_weights": [-0.2822391390800476, -0.3235939145088196, 0.748684287071228], "agreement_weight": 0.04180171713232994, "orthogonal_weight": -0.11431651562452316}, {"head": 5, "memory_to_active": 2.764694929122925, "contribution_to_active": 0.4630919098854065, "mix_mean": 0.015854282304644585, "mix_low": 0.6167631149291992, "mix_high": 2.765655517578125e-05, "read_cosine": 0.3786774277687073, "beta_mean": 0.29264113306999207, "nonfinite": 0.0, "weighted_memory_to_active": 0.35965782403945923, "mix_std": 0.04059653356671333, "beta_std": 0.1708778589963913, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1572847068309784, "evidence_weights": [0.03574606031179428, -0.030517475679516792, -0.12185525894165039], "agreement_weight": -0.04387751221656799, "orthogonal_weight": 0.6493573188781738}]}, {"layer": 5, "memory_to_active": 0.6842645406723022, "contribution_to_active": 0.43292197585105896, "mix_mean": 0.38392412662506104, "mix_low": 0.10391156375408173, "mix_high": 0.008236567489802837, "read_cosine": 0.22361576557159424, "beta_mean": 0.060697175562381744, "nonfinite": 0.0, "weighted_memory_to_active": 0.16547651588916779, "mix_std": 0.24662527441978455, "beta_std": 0.0737881287932396, "beta_low": 0.578277587890625, "beta_high": 0.0, "orthogonal_to_active": 0.05067111924290657, "output_influence": 0.4078604280948639, "residual_influence": 0.09176594018936157, "state_active": 0.05806853249669075, "state_memory": 0.01729777082800865, "state_peak": 66.00870513916016, "growth_active": 1.1158502101898193, "growth_memory": 1.3174622058868408, "heads": [{"head": 0, "memory_to_active": 1.3531440496444702, "contribution_to_active": 0.26621729135513306, "mix_mean": 0.1209796741604805, "mix_low": 0.15105628967285156, "mix_high": 0.0, "read_cosine": -0.27298811078071594, "beta_mean": 0.0735047310590744, "nonfinite": 0.0, "weighted_memory_to_active": 0.18578986823558807, "mix_std": 0.16530297696590424, "beta_std": 0.0889335423707962, "beta_low": 0.07073974609375, "beta_high": 0.0, "orthogonal_to_active": 0.04071890190243721, "evidence_weights": [-0.03949886932969093, -0.07586146891117096, -0.3613806366920471], "agreement_weight": -0.014612695202231407, "orthogonal_weight": -0.21295230090618134}, {"head": 1, "memory_to_active": 0.029365602880716324, "contribution_to_active": 0.45826655626296997, "mix_mean": 0.5126700401306152, "mix_low": 0.009809494018554688, "mix_high": 0.008267402648925781, "read_cosine": 0.22713185846805573, "beta_mean": 7.640750845894217e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013572719879448414, "mix_std": 0.3150336444377899, "beta_std": 0.00015786674339324236, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011947587132453918, "evidence_weights": [-0.22108742594718933, -0.23179826140403748, 0.6683517694473267], "agreement_weight": 0.26379287242889404, "orthogonal_weight": -0.9162400960922241}, {"head": 2, "memory_to_active": 0.07367300242185593, "contribution_to_active": 0.4783991575241089, "mix_mean": 0.5586906671524048, "mix_low": 0.00296783447265625, "mix_high": 0.010287284851074219, "read_cosine": 0.45639678835868835, "beta_mean": 0.00213621580041945, "nonfinite": 0.0, "weighted_memory_to_active": 0.04877277836203575, "mix_std": 0.2809649705886841, "beta_std": 0.002390495967119932, "beta_low": 0.9840087890625, "beta_high": 0.0, "orthogonal_to_active": 0.006283791735768318, "evidence_weights": [-0.26956260204315186, -0.3073023557662964, 0.6987795233726501], "agreement_weight": 0.03026263415813446, "orthogonal_weight": -0.17856758832931519}, {"head": 3, "memory_to_active": 2.046938180923462, "contribution_to_active": 0.19658976793289185, "mix_mean": 0.06826506555080414, "mix_low": 0.26889514923095703, "mix_high": 0.0, "read_cosine": 0.1328052133321762, "beta_mean": 0.09266841411590576, "nonfinite": 0.0, "weighted_memory_to_active": 0.33230680227279663, "mix_std": 0.11696690320968628, "beta_std": 0.12620553374290466, "beta_low": 0.183349609375, "beta_high": 0.0, "orthogonal_to_active": 0.12482593953609467, "evidence_weights": [-0.012092539109289646, -0.06101461872458458, 0.02428312785923481], "agreement_weight": 0.05621566250920296, "orthogonal_weight": -0.4512191712856293}, {"head": 4, "memory_to_active": 0.5011136531829834, "contribution_to_active": 0.5899415016174316, "mix_mean": 0.32352977991104126, "mix_low": 0.1871013641357422, "mix_high": 0.012659072875976562, "read_cosine": 0.5723367929458618, "beta_mean": 0.195431187748909, "nonfinite": 0.0, "weighted_memory_to_active": 0.34652507305145264, "mix_std": 0.3390964865684509, "beta_std": 0.22337606549263, "beta_low": 0.23675537109375, "beta_high": 0.0, "orthogonal_to_active": 0.0841413140296936, "evidence_weights": [-0.13396400213241577, -0.2764100730419159, 0.012162106111645699], "agreement_weight": 0.046177856624126434, "orthogonal_weight": 0.38241147994995117}, {"head": 5, "memory_to_active": 0.10135230422019958, "contribution_to_active": 0.608117401599884, "mix_mean": 0.7194092273712158, "mix_low": 0.00363922119140625, "mix_high": 0.018205642700195312, "read_cosine": 0.22601203620433807, "beta_mean": 0.0003660752554424107, "nonfinite": 0.0, "weighted_memory_to_active": 0.06589178740978241, "mix_std": 0.26238659024238586, "beta_std": 0.0016652251360937953, "beta_low": 0.99481201171875, "beta_high": 0.0, "orthogonal_to_active": 0.03610915690660477, "evidence_weights": [-0.3190010190010071, -0.30555540323257446, 1.5206156969070435], "agreement_weight": 0.30783504247665405, "orthogonal_weight": -0.589026153087616}]}, {"layer": 6, "memory_to_active": 0.4562537670135498, "contribution_to_active": 0.5499317646026611, "mix_mean": 0.49757421016693115, "mix_low": 0.08471298217773438, "mix_high": 0.02465025708079338, "read_cosine": 0.30779892206192017, "beta_mean": 0.08333754539489746, "nonfinite": 0.0, "weighted_memory_to_active": 0.16772355139255524, "mix_std": 0.2580360770225525, "beta_std": 0.09540286660194397, "beta_low": 0.507537841796875, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03603413701057434, "output_influence": 0.4292459785938263, "residual_influence": 0.13219870626926422, "state_active": 0.05332517996430397, "state_memory": 0.01182597130537033, "state_peak": 62.38289260864258, "growth_active": 1.1842631101608276, "growth_memory": 1.2791671752929688, "heads": [{"head": 0, "memory_to_active": 0.15767452120780945, "contribution_to_active": 0.5600842237472534, "mix_mean": 0.6387509107589722, "mix_low": 0.0004482269287109375, "mix_high": 0.004952430725097656, "read_cosine": 0.140025794506073, "beta_mean": 0.013850485906004906, "nonfinite": 0.0, "weighted_memory_to_active": 0.11577367782592773, "mix_std": 0.2570328712463379, "beta_std": 0.031220775097608566, "beta_low": 0.63690185546875, "beta_high": 0.0, "orthogonal_to_active": 0.08606909960508347, "evidence_weights": [-0.2925455570220947, -0.289207398891449, 0.40578481554985046], "agreement_weight": 0.09399939328432083, "orthogonal_weight": -0.8760758638381958}, {"head": 1, "memory_to_active": 0.01014731079339981, "contribution_to_active": 0.4749935567378998, "mix_mean": 0.45203453302383423, "mix_low": 0.0029249191284179688, "mix_high": 0.009683609008789062, "read_cosine": 0.5788475275039673, "beta_mean": 2.2659416572423652e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0036553777754306793, "mix_std": 0.2942344546318054, "beta_std": 5.867784420843236e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010414939606562257, "evidence_weights": [-0.21346965432167053, -0.24990402162075043, 0.6444419026374817], "agreement_weight": -0.026995357125997543, "orthogonal_weight": -0.3666137456893921}, {"head": 2, "memory_to_active": 1.659014105796814, "contribution_to_active": 0.07740983366966248, "mix_mean": 0.026434235274791718, "mix_low": 0.48870849609375, "mix_high": 0.0, "read_cosine": 0.18757842481136322, "beta_mean": 0.1087072566151619, "nonfinite": 0.0, "weighted_memory_to_active": 0.09726361930370331, "mix_std": 0.05543455481529236, "beta_std": 0.09483791142702103, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.02089623361825943, "evidence_weights": [0.07644051313400269, 0.024784527719020844, 0.1627836376428604], "agreement_weight": 0.06666121631860733, "orthogonal_weight": -0.23299868404865265}, {"head": 3, "memory_to_active": 0.16278311610221863, "contribution_to_active": 0.6285690069198608, "mix_mean": 0.62043297290802, "mix_low": 0.01259613037109375, "mix_high": 0.03170490264892578, "read_cosine": 0.47497618198394775, "beta_mean": 0.24116654694080353, "nonfinite": 0.0, "weighted_memory_to_active": 0.1159878745675087, "mix_std": 0.34484657645225525, "beta_std": 0.2795480489730835, "beta_low": 0.20196533203125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.047683194279670715, "evidence_weights": [-0.17475473880767822, -0.3354140520095825, 0.15751416981220245], "agreement_weight": -0.014668405055999756, "orthogonal_weight": 0.483858197927475}, {"head": 4, "memory_to_active": 0.7031515836715698, "contribution_to_active": 1.0053391456604004, "mix_mean": 0.7037148475646973, "mix_low": 0.0020904541015625, "mix_high": 0.10074424743652344, "read_cosine": 0.1990719735622406, "beta_mean": 0.1359110325574875, "nonfinite": 0.0, "weighted_memory_to_active": 0.6455426216125488, "mix_std": 0.3105921447277069, "beta_std": 0.16602322459220886, "beta_low": 0.2066650390625, "beta_high": 0.0, "orthogonal_to_active": 0.03247246891260147, "evidence_weights": [-0.24479492008686066, -0.5466269254684448, -0.03501248359680176], "agreement_weight": -0.016829609870910645, "orthogonal_weight": 0.06079251319169998}, {"head": 5, "memory_to_active": 0.04475201666355133, "contribution_to_active": 0.553194522857666, "mix_mean": 0.5440776348114014, "mix_low": 0.0015096664428710938, "mix_high": 0.00081634521484375, "read_cosine": 0.2662935256958008, "beta_mean": 0.0003672788152471185, "nonfinite": 0.0, "weighted_memory_to_active": 0.02811809629201889, "mix_std": 0.28607574105262756, "beta_std": 0.0007285698084160686, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.028042327612638474, "evidence_weights": [-0.22242318093776703, -0.2657533288002014, 0.2994205951690674], "agreement_weight": 0.07563267648220062, "orthogonal_weight": -1.065375566482544}]}, {"layer": 7, "memory_to_active": 2.083895206451416, "contribution_to_active": 0.15623995661735535, "mix_mean": 0.03681837022304535, "mix_low": 0.5040326118469238, "mix_high": 9.218852028425317e-06, "read_cosine": 0.23200494050979614, "beta_mean": 0.13813698291778564, "nonfinite": 0.0, "weighted_memory_to_active": 0.16691449284553528, "mix_std": 0.06536580622196198, "beta_std": 0.11220304667949677, "beta_low": 0.0368143729865551, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.032498784363269806, "output_influence": 0.1772935390472412, "residual_influence": 0.038770437240600586, "state_active": 0.016280192881822586, "state_memory": 0.03197339549660683, "state_peak": 18.412294387817383, "growth_active": 1.015763759613037, "growth_memory": 1.4066591262817383, "heads": [{"head": 0, "memory_to_active": 2.1066465377807617, "contribution_to_active": 0.21865133941173553, "mix_mean": 0.028852060437202454, "mix_low": 0.529911994934082, "mix_high": 2.288818359375e-05, "read_cosine": 0.381987988948822, "beta_mean": 0.15645408630371094, "nonfinite": 0.0, "weighted_memory_to_active": 0.1984812468290329, "mix_std": 0.06297831237316132, "beta_std": 0.12043291330337524, "beta_low": 0.0089111328125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.03136766701936722, "evidence_weights": [-0.000551700359210372, -0.07937964051961899, 0.08025124669075012], "agreement_weight": -0.04246640205383301, "orthogonal_weight": 0.19847793877124786}, {"head": 1, "memory_to_active": 2.1774966716766357, "contribution_to_active": 0.2725287675857544, "mix_mean": 0.08713700622320175, "mix_low": 0.29480552673339844, "mix_high": 0.0, "read_cosine": 0.11086296290159225, "beta_mean": 0.17042377591133118, "nonfinite": 0.0, "weighted_memory_to_active": 0.32930421829223633, "mix_std": 0.123744435608387, "beta_std": 0.10447751730680466, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06689535826444626, "evidence_weights": [-0.12388458102941513, -0.09860531240701675, -0.10124938935041428], "agreement_weight": 0.014635373838245869, "orthogonal_weight": -0.21397541463375092}, {"head": 2, "memory_to_active": 1.295151710510254, "contribution_to_active": 0.2803340256214142, "mix_mean": 0.06132851541042328, "mix_low": 0.28778553009033203, "mix_high": 3.24249267578125e-05, "read_cosine": 0.1516314297914505, "beta_mean": 0.12431000173091888, "nonfinite": 0.0, "weighted_memory_to_active": 0.22738279402256012, "mix_std": 0.11587465554475784, "beta_std": 0.08159638941287994, "beta_low": 0.0145263671875, "beta_high": 0.0, "orthogonal_to_active": 0.01827027089893818, "evidence_weights": [-0.03356405347585678, -0.11223678290843964, 0.21322070062160492], "agreement_weight": 0.027959972620010376, "orthogonal_weight": -0.08383462578058243}, {"head": 3, "memory_to_active": 3.343660354614258, "contribution_to_active": 0.04793958365917206, "mix_mean": 0.015661846846342087, "mix_low": 0.5777797698974609, "mix_high": 0.0, "read_cosine": 0.1357956826686859, "beta_mean": 0.17958271503448486, "nonfinite": 0.0, "weighted_memory_to_active": 0.10455024242401123, "mix_std": 0.031446631997823715, "beta_std": 0.16441260278224945, "beta_low": 0.0264892578125, "beta_high": 0.0, "orthogonal_to_active": 0.06133522465825081, "evidence_weights": [0.1188943162560463, 0.07856844365596771, 0.3040279150009155], "agreement_weight": 0.07798492163419724, "orthogonal_weight": -0.6328413486480713}, {"head": 4, "memory_to_active": 1.8443958759307861, "contribution_to_active": 0.047618966549634933, "mix_mean": 0.011238026432693005, "mix_low": 0.7296457290649414, "mix_high": 0.0, "read_cosine": 0.3640778064727783, "beta_mean": 0.14504684507846832, "nonfinite": 0.0, "weighted_memory_to_active": 0.05366218835115433, "mix_std": 0.025923505425453186, "beta_std": 0.13684570789337158, "beta_low": 0.02325439453125, "beta_high": 0.0, "orthogonal_to_active": 0.003268270520493388, "evidence_weights": [0.09174978733062744, 0.04743523523211479, 0.04651331901550293], "agreement_weight": 0.04969223588705063, "orthogonal_weight": -0.07296071946620941}, {"head": 5, "memory_to_active": 1.7360199689865112, "contribution_to_active": 0.0703669935464859, "mix_mean": 0.01669275388121605, "mix_low": 0.6042671203613281, "mix_high": 0.0, "read_cosine": 0.24767373502254486, "beta_mean": 0.053004372864961624, "nonfinite": 0.0, "weighted_memory_to_active": 0.08810625225305557, "mix_std": 0.0322272889316082, "beta_std": 0.06545314937829971, "beta_low": 0.147705078125, "beta_high": 0.0, "orthogonal_to_active": 0.01385590061545372, "evidence_weights": [0.0689576268196106, 0.06970077008008957, -0.03236672654747963], "agreement_weight": 0.015981502830982208, "orthogonal_weight": -0.19704295694828033}]}, {"layer": 8, "memory_to_active": 0.7048299312591553, "contribution_to_active": 0.443337082862854, "mix_mean": 0.3334364593029022, "mix_low": 0.09024302661418915, "mix_high": 0.0006726583815179765, "read_cosine": 0.5331072211265564, "beta_mean": 0.05115991085767746, "nonfinite": 0.0, "weighted_memory_to_active": 0.15135496854782104, "mix_std": 0.22226165235042572, "beta_std": 0.05030502378940582, "beta_low": 0.4251708984375, "beta_high": 0.0, "orthogonal_to_active": 0.0488477386534214, "output_influence": 0.31927070021629333, "residual_influence": 0.08597408980131149, "state_active": 0.053357355296611786, "state_memory": 0.015951473265886307, "state_peak": 29.15033531188965, "growth_active": 1.0324735641479492, "growth_memory": 1.3653727769851685, "heads": [{"head": 0, "memory_to_active": 0.06213803589344025, "contribution_to_active": 0.5003109574317932, "mix_mean": 0.5279700756072998, "mix_low": 0.00016880035400390625, "mix_high": 0.000316619873046875, "read_cosine": 0.7298622131347656, "beta_mean": 0.004427354317158461, "nonfinite": 0.0, "weighted_memory_to_active": 0.034427762031555176, "mix_std": 0.24308542907238007, "beta_std": 0.009192043915390968, "beta_low": 0.89190673828125, "beta_high": 0.0, "orthogonal_to_active": 0.017534010112285614, "evidence_weights": [-0.21400851011276245, -0.339069128036499, 0.6943208575248718], "agreement_weight": 0.06803763657808304, "orthogonal_weight": -0.7754846811294556}, {"head": 1, "memory_to_active": 0.5496051907539368, "contribution_to_active": 0.3386344015598297, "mix_mean": 0.3382771611213684, "mix_low": 0.005892753601074219, "mix_high": 0.00010395050048828125, "read_cosine": 0.5084502696990967, "beta_mean": 0.031609710305929184, "nonfinite": 0.0, "weighted_memory_to_active": 0.21531856060028076, "mix_std": 0.23303666710853577, "beta_std": 0.029648562893271446, "beta_low": 0.13653564453125, "beta_high": 0.0, "orthogonal_to_active": 0.04745052009820938, "evidence_weights": [-0.16928167641162872, -0.19395405054092407, 0.39320674538612366], "agreement_weight": 0.060534048825502396, "orthogonal_weight": -0.2629896104335785}, {"head": 2, "memory_to_active": 0.12565484642982483, "contribution_to_active": 0.5119510889053345, "mix_mean": 0.46484076976776123, "mix_low": 0.0012359619140625, "mix_high": 0.0008916854858398438, "read_cosine": 0.3868749737739563, "beta_mean": 0.0023494060151278973, "nonfinite": 0.0, "weighted_memory_to_active": 0.07009685039520264, "mix_std": 0.2598690688610077, "beta_std": 0.0045598093420267105, "beta_low": 0.9647216796875, "beta_high": 0.0, "orthogonal_to_active": 0.049703434109687805, "evidence_weights": [-0.1951085478067398, -0.2916904091835022, 0.47926637530326843], "agreement_weight": 0.05539318546652794, "orthogonal_weight": -0.8132711052894592}, {"head": 3, "memory_to_active": 0.29128438234329224, "contribution_to_active": 0.3533758521080017, "mix_mean": 0.32574838399887085, "mix_low": 0.0074481964111328125, "mix_high": 2.002716064453125e-05, "read_cosine": 0.7214411497116089, "beta_mean": 0.019620293751358986, "nonfinite": 0.0, "weighted_memory_to_active": 0.14306846261024475, "mix_std": 0.22619260847568512, "beta_std": 0.021205557510256767, "beta_low": 0.28887939453125, "beta_high": 0.0, "orthogonal_to_active": 0.014015834778547287, "evidence_weights": [-0.10150358080863953, -0.21059177815914154, 0.7068503499031067], "agreement_weight": 0.07835567742586136, "orthogonal_weight": -0.16528867185115814}, {"head": 4, "memory_to_active": 2.972683906555176, "contribution_to_active": 0.2925664186477661, "mix_mean": 0.025835581123828888, "mix_low": 0.4271697998046875, "mix_high": 0.0, "read_cosine": 0.3350842595100403, "beta_mean": 0.1659635454416275, "nonfinite": 0.0, "weighted_memory_to_active": 0.2907721996307373, "mix_std": 0.0429227352142334, "beta_std": 0.1169586032629013, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.012256646528840065, "evidence_weights": [0.02974841557443142, -0.001806657761335373, -0.11841925233602524], "agreement_weight": 0.023232130333781242, "orthogonal_weight": 0.04674457386136055}, {"head": 5, "memory_to_active": 0.22761301696300507, "contribution_to_active": 0.6631836891174316, "mix_mean": 0.3179466426372528, "mix_low": 0.09954261779785156, "mix_high": 0.0027036666870117188, "read_cosine": 0.5169302225112915, "beta_mean": 0.08298914134502411, "nonfinite": 0.0, "weighted_memory_to_active": 0.15444597601890564, "mix_std": 0.3284634053707123, "beta_std": 0.12026555836200714, "beta_low": 0.268310546875, "beta_high": 0.0, "orthogonal_to_active": 0.15212595462799072, "evidence_weights": [-0.07003866881132126, -0.3264818787574768, -0.4415399432182312], "agreement_weight": 0.033209990710020065, "orthogonal_weight": 1.187217354774475}]}, {"layer": 9, "memory_to_active": 1.1829861402511597, "contribution_to_active": 0.5835201144218445, "mix_mean": 0.23270010948181152, "mix_low": 0.17893871665000916, "mix_high": 0.0011944770812988281, "read_cosine": 0.362562894821167, "beta_mean": 0.09956437349319458, "nonfinite": 0.0, "weighted_memory_to_active": 0.3678779602050781, "mix_std": 0.20966312289237976, "beta_std": 0.055606067180633545, "beta_low": 0.4981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.027663055807352066, "output_influence": 0.38126039505004883, "residual_influence": 0.09524643421173096, "state_active": 0.043325550854206085, "state_memory": 0.026147330179810524, "state_peak": 30.575319290161133, "growth_active": 1.3055009841918945, "growth_memory": 1.4487727880477905, "heads": [{"head": 0, "memory_to_active": 0.007345537655055523, "contribution_to_active": 0.45152294635772705, "mix_mean": 0.4909680485725403, "mix_low": 0.0028972625732421875, "mix_high": 0.004604339599609375, "read_cosine": 0.23392944037914276, "beta_mean": 1.6381116438424215e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038986308500170708, "mix_std": 0.2962070405483246, "beta_std": 6.272656173678115e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028245265129953623, "evidence_weights": [-0.25751230120658875, -0.25095006823539734, 0.9348037838935852], "agreement_weight": 0.037269227206707, "orthogonal_weight": -0.7566708922386169}, {"head": 1, "memory_to_active": 0.0316869355738163, "contribution_to_active": 0.46268969774246216, "mix_mean": 0.31066253781318665, "mix_low": 0.02271747589111328, "mix_high": 0.0005712509155273438, "read_cosine": 0.4152284264564514, "beta_mean": 9.764688729774207e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013913780450820923, "mix_std": 0.2702029347419739, "beta_std": 0.00020408954878803343, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006336208898574114, "evidence_weights": [-0.18506579101085663, -0.19428884983062744, 0.7981181144714355], "agreement_weight": 0.30061784386634827, "orthogonal_weight": -0.5300930738449097}, {"head": 2, "memory_to_active": 0.24572429060935974, "contribution_to_active": 0.5111818313598633, "mix_mean": 0.37815719842910767, "mix_low": 0.011241912841796875, "mix_high": 0.00173187255859375, "read_cosine": 0.4170576333999634, "beta_mean": 0.001324888551607728, "nonfinite": 0.0, "weighted_memory_to_active": 0.1301446259021759, "mix_std": 0.2863163948059082, "beta_std": 0.005377218592911959, "beta_low": 0.9884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.032746296375989914, "evidence_weights": [-0.1490880846977234, -0.16405433416366577, 0.3832423686981201], "agreement_weight": 0.16533806920051575, "orthogonal_weight": -0.31597283482551575}, {"head": 3, "memory_to_active": 2.120246410369873, "contribution_to_active": 0.24392244219779968, "mix_mean": 0.037659190595149994, "mix_low": 0.4230165481567383, "mix_high": 2.193450927734375e-05, "read_cosine": 0.45959901809692383, "beta_mean": 0.14663048088550568, "nonfinite": 0.0, "weighted_memory_to_active": 0.2241121232509613, "mix_std": 0.09370819479227066, "beta_std": 0.09162771701812744, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.03385818749666214, "evidence_weights": [0.04427460581064224, 0.03315972909331322, -0.1641269028186798], "agreement_weight": 0.05364774167537689, "orthogonal_weight": 0.1701439768075943}, {"head": 4, "memory_to_active": 2.858461618423462, "contribution_to_active": 1.2123196125030518, "mix_mean": 0.12140554189682007, "mix_low": 0.2060871124267578, "mix_high": 0.000232696533203125, "read_cosine": 0.23948892951011658, "beta_mean": 0.2684484124183655, "nonfinite": 0.0, "weighted_memory_to_active": 1.1114320755004883, "mix_std": 0.1946311742067337, "beta_std": 0.12858235836029053, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014505370054394007, "evidence_weights": [-0.046406906098127365, -0.12794999778270721, 0.0663214847445488], "agreement_weight": -0.048289548605680466, "orthogonal_weight": -0.0013111865846440196}, {"head": 5, "memory_to_active": 1.834451675415039, "contribution_to_active": 0.619484007358551, "mix_mean": 0.05734807252883911, "mix_low": 0.4076719284057617, "mix_high": 4.76837158203125e-06, "read_cosine": 0.41007381677627563, "beta_mean": 0.18086838722229004, "nonfinite": 0.0, "weighted_memory_to_active": 0.7237664461135864, "mix_std": 0.11691299080848694, "beta_std": 0.1077822744846344, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.088762566447258, "evidence_weights": [0.048030175268650055, -0.040243808180093765, 0.07100506126880646], "agreement_weight": 0.012665363028645515, "orthogonal_weight": 0.166819229722023}]}, {"layer": 10, "memory_to_active": 0.17552761733531952, "contribution_to_active": 0.42946499586105347, "mix_mean": 0.4080486595630646, "mix_low": 0.013003190979361534, "mix_high": 0.007797400467097759, "read_cosine": 0.39573413133621216, "beta_mean": 0.04514271765947342, "nonfinite": 0.0, "weighted_memory_to_active": 0.08051149547100067, "mix_std": 0.2689422369003296, "beta_std": 0.051984548568725586, "beta_low": 0.63555908203125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.036980319768190384, "output_influence": 0.3885882496833801, "residual_influence": 0.12200141698122025, "state_active": 0.06166422739624977, "state_memory": 0.014004084281623363, "state_peak": 104.47718048095703, "growth_active": 1.12522554397583, "growth_memory": 1.0721259117126465, "heads": [{"head": 0, "memory_to_active": 0.3341723382472992, "contribution_to_active": 0.7584578990936279, "mix_mean": 0.6234843134880066, "mix_low": 0.019620895385742188, "mix_high": 0.04449939727783203, "read_cosine": 0.2452981024980545, "beta_mean": 0.19507595896720886, "nonfinite": 0.0, "weighted_memory_to_active": 0.2443106323480606, "mix_std": 0.35518762469291687, "beta_std": 0.21984222531318665, "beta_low": 0.13983154296875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12145932763814926, "evidence_weights": [-0.16644659638404846, -0.343560129404068, -0.29095810651779175], "agreement_weight": 0.022447051480412483, "orthogonal_weight": 0.5542271137237549}, {"head": 1, "memory_to_active": 0.00318640423938632, "contribution_to_active": 0.4951338768005371, "mix_mean": 0.4754776954650879, "mix_low": 0.002178192138671875, "mix_high": 0.0018100738525390625, "read_cosine": 0.2502818703651428, "beta_mean": 1.7035546989063732e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0015826919116079807, "mix_std": 0.26726117730140686, "beta_std": 3.605955862440169e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011964376317337155, "evidence_weights": [-0.2576580345630646, -0.18379095196723938, 0.2826927900314331], "agreement_weight": 0.13201098144054413, "orthogonal_weight": -0.8302451968193054}, {"head": 2, "memory_to_active": 0.23718911409378052, "contribution_to_active": 0.33900487422943115, "mix_mean": 0.3012223243713379, "mix_low": 0.015280723571777344, "mix_high": 5.91278076171875e-05, "read_cosine": 0.06806454062461853, "beta_mean": 0.003643201431259513, "nonfinite": 0.0, "weighted_memory_to_active": 0.0877019464969635, "mix_std": 0.2598136365413666, "beta_std": 0.014565842226147652, "beta_low": 0.934326171875, "beta_high": 0.0, "orthogonal_to_active": 0.025363612920045853, "evidence_weights": [-0.1122843399643898, -0.1756018102169037, 0.40283071994781494], "agreement_weight": 0.009519385173916817, "orthogonal_weight": -0.2919310927391052}, {"head": 3, "memory_to_active": 0.28257089853286743, "contribution_to_active": 0.1468503773212433, "mix_mean": 0.2888064384460449, "mix_low": 0.03383159637451172, "mix_high": 0.00037288665771484375, "read_cosine": 0.8809800148010254, "beta_mean": 0.06321059167385101, "nonfinite": 0.0, "weighted_memory_to_active": 0.06030045449733734, "mix_std": 0.2556118369102478, "beta_std": 0.06231246143579483, "beta_low": 0.0069580078125, "beta_high": 0.0, "orthogonal_to_active": 0.05151554197072983, "evidence_weights": [-0.12567879259586334, -0.1457112431526184, 0.22576552629470825], "agreement_weight": 0.00523273553699255, "orthogonal_weight": 1.2558064460754395}, {"head": 4, "memory_to_active": 0.037482328712940216, "contribution_to_active": 0.41863614320755005, "mix_mean": 0.40736880898475647, "mix_low": 0.0020456314086914062, "mix_high": 3.4332275390625e-05, "read_cosine": 0.09611508250236511, "beta_mean": 7.003280916251242e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011221863329410553, "mix_std": 0.25200557708740234, "beta_std": 0.0028425180353224277, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0027451985515654087, "evidence_weights": [-0.1803690493106842, -0.19383695721626282, 0.34571850299835205], "agreement_weight": 0.020969413220882416, "orthogonal_weight": -0.2516539394855499}, {"head": 5, "memory_to_active": 0.15856461226940155, "contribution_to_active": 0.41870686411857605, "mix_mean": 0.351932168006897, "mix_low": 0.005062103271484375, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8336650133132935, "beta_mean": 0.008839471265673637, "nonfinite": 0.0, "weighted_memory_to_active": 0.0779513567686081, "mix_std": 0.2237735241651535, "beta_std": 0.012308184057474136, "beta_low": 0.7322998046875, "beta_high": 0.0, "orthogonal_to_active": 0.019601788371801376, "evidence_weights": [-0.17493346333503723, -0.1686297208070755, 0.27706435322761536], "agreement_weight": 0.09206334501504898, "orthogonal_weight": -0.553684413433075}]}, {"layer": 11, "memory_to_active": 0.2441299855709076, "contribution_to_active": 0.46433767676353455, "mix_mean": 0.38359567523002625, "mix_low": 0.04932181537151337, "mix_high": 0.020287197083234787, "read_cosine": 0.5145886540412903, "beta_mean": 0.04182814434170723, "nonfinite": 0.0, "weighted_memory_to_active": 0.0863608568906784, "mix_std": 0.30241498351097107, "beta_std": 0.05946282669901848, "beta_low": 0.48504638671875, "beta_high": 0.0, "orthogonal_to_active": 0.04387028142809868, "output_influence": 0.46781036257743835, "residual_influence": 0.12468808144330978, "state_active": 0.11009073257446289, "state_memory": 0.011624064296483994, "state_peak": 34.551292419433594, "growth_active": 1.1024898290634155, "growth_memory": 1.3707600831985474, "heads": [{"head": 0, "memory_to_active": 0.015495453029870987, "contribution_to_active": 0.5252877473831177, "mix_mean": 0.5246433615684509, "mix_low": 0.002777099609375, "mix_high": 0.0061702728271484375, "read_cosine": 0.4246746301651001, "beta_mean": 0.00012844064622186124, "nonfinite": 0.0, "weighted_memory_to_active": 0.00921366736292839, "mix_std": 0.27692708373069763, "beta_std": 0.003940937574952841, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005142509005963802, "evidence_weights": [-0.15978530049324036, -0.3250304162502289, 0.5474180579185486], "agreement_weight": 0.13989058136940002, "orthogonal_weight": -0.6766262650489807}, {"head": 1, "memory_to_active": 0.48370593786239624, "contribution_to_active": 0.24339301884174347, "mix_mean": 0.2491263598203659, "mix_low": 0.0290374755859375, "mix_high": 7.915496826171875e-05, "read_cosine": 0.6182180643081665, "beta_mean": 0.04316011071205139, "nonfinite": 0.0, "weighted_memory_to_active": 0.16990189254283905, "mix_std": 0.23441952466964722, "beta_std": 0.07488378882408142, "beta_low": 0.3087158203125, "beta_high": 0.0, "orthogonal_to_active": 0.023712459951639175, "evidence_weights": [-0.15631720423698425, -0.11526871472597122, 0.15518984198570251], "agreement_weight": 0.09018170088529587, "orthogonal_weight": -0.19622138142585754}, {"head": 2, "memory_to_active": 0.05087537318468094, "contribution_to_active": 0.8356406092643738, "mix_mean": 0.46895283460617065, "mix_low": 0.05867290496826172, "mix_high": 0.0629129409790039, "read_cosine": 0.331828236579895, "beta_mean": 0.07546383142471313, "nonfinite": 0.0, "weighted_memory_to_active": 0.04091500863432884, "mix_std": 0.4067123532295227, "beta_std": 0.10467223823070526, "beta_low": 0.4490966796875, "beta_high": 0.0, "orthogonal_to_active": 0.09638876467943192, "evidence_weights": [-0.11413867026567459, -0.4274830222129822, -0.782478392124176], "agreement_weight": -0.006552004721015692, "orthogonal_weight": 2.5681192874908447}, {"head": 3, "memory_to_active": 0.030024386942386627, "contribution_to_active": 0.5213194489479065, "mix_mean": 0.4324328899383545, "mix_low": 0.14355754852294922, "mix_high": 0.05167961120605469, "read_cosine": 0.539493203163147, "beta_mean": 0.07005167007446289, "nonfinite": 0.0, "weighted_memory_to_active": 0.016923528164625168, "mix_std": 0.4191989600658417, "beta_std": 0.10103978216648102, "beta_low": 0.4576416015625, "beta_high": 0.0, "orthogonal_to_active": 0.04006579890847206, "evidence_weights": [-0.11532362550497055, -0.3657781481742859, -0.2780733108520508], "agreement_weight": 0.01969936117529869, "orthogonal_weight": 2.788325071334839}, {"head": 4, "memory_to_active": 0.7191404104232788, "contribution_to_active": 0.2804972231388092, "mix_mean": 0.20453497767448425, "mix_low": 0.061301231384277344, "mix_high": 9.5367431640625e-06, "read_cosine": 0.26466071605682373, "beta_mean": 0.04805373400449753, "nonfinite": 0.0, "weighted_memory_to_active": 0.20309893786907196, "mix_std": 0.21594379842281342, "beta_std": 0.0511198565363884, "beta_low": 0.1004638671875, "beta_high": 0.0, "orthogonal_to_active": 0.0785481259226799, "evidence_weights": [-0.10177253931760788, -0.07950736582279205, 1.0943570137023926], "agreement_weight": -0.09890814125537872, "orthogonal_weight": -0.4214760959148407}, {"head": 5, "memory_to_active": 0.16553835570812225, "contribution_to_active": 0.37988781929016113, "mix_mean": 0.42188358306884766, "mix_low": 0.0005846023559570312, "mix_high": 0.0008716583251953125, "read_cosine": 0.9086570739746094, "beta_mean": 0.01411108672618866, "nonfinite": 0.0, "weighted_memory_to_active": 0.07811205089092255, "mix_std": 0.2612881660461426, "beta_std": 0.02112032286822796, "beta_low": 0.594970703125, "beta_high": 0.0, "orthogonal_to_active": 0.01936402916908264, "evidence_weights": [-0.1927681863307953, -0.24005158245563507, 0.4069913625717163], "agreement_weight": 0.0416695661842823, "orthogonal_weight": -0.5780000686645508}]}, {"layer": 12, "memory_to_active": 0.4520040452480316, "contribution_to_active": 0.512046754360199, "mix_mean": 0.4100707173347473, "mix_low": 0.03449312970042229, "mix_high": 0.0023390455171465874, "read_cosine": 0.2884407639503479, "beta_mean": 0.01907358504831791, "nonfinite": 0.0, "weighted_memory_to_active": 0.2051638960838318, "mix_std": 0.24774347245693207, "beta_std": 0.022062579169869423, "beta_low": 0.7713623642921448, "beta_high": 0.0, "orthogonal_to_active": 0.0724504142999649, "output_influence": 0.3702794015407562, "residual_influence": 0.09847744554281235, "state_active": 0.045097772032022476, "state_memory": 0.012820689007639885, "state_peak": 24.36672592163086, "growth_active": 1.1088670492172241, "growth_memory": 1.1886720657348633, "heads": [{"head": 0, "memory_to_active": 0.2790069580078125, "contribution_to_active": 0.35938185453414917, "mix_mean": 0.34124332666397095, "mix_low": 0.012701034545898438, "mix_high": 0.00020599365234375, "read_cosine": 0.3134434223175049, "beta_mean": 0.0047732628881931305, "nonfinite": 0.0, "weighted_memory_to_active": 0.09787975996732712, "mix_std": 0.2555609345436096, "beta_std": 0.01702997088432312, "beta_low": 0.891845703125, "beta_high": 0.0, "orthogonal_to_active": 0.04531589522957802, "evidence_weights": [-0.14735518395900726, -0.209662526845932, 0.2556285262107849], "agreement_weight": 0.10213181376457214, "orthogonal_weight": -0.5161272883415222}, {"head": 1, "memory_to_active": 0.13992102444171906, "contribution_to_active": 0.44162455201148987, "mix_mean": 0.4362148642539978, "mix_low": 0.0007305145263671875, "mix_high": 0.000152587890625, "read_cosine": 0.5780718922615051, "beta_mean": 0.0012704828986898065, "nonfinite": 0.0, "weighted_memory_to_active": 0.06567288935184479, "mix_std": 0.26009657979011536, "beta_std": 0.005423559807240963, "beta_low": 0.9866943359375, "beta_high": 0.0, "orthogonal_to_active": 0.04591982811689377, "evidence_weights": [-0.18535971641540527, -0.23623229563236237, 0.6982149481773376], "agreement_weight": 0.08826857805252075, "orthogonal_weight": -0.8798341751098633}, {"head": 2, "memory_to_active": 0.6629706025123596, "contribution_to_active": 0.43925225734710693, "mix_mean": 0.32311713695526123, "mix_low": 0.02531147003173828, "mix_high": 0.0004444122314453125, "read_cosine": 0.4204428493976593, "beta_mean": 0.097781240940094, "nonfinite": 0.0, "weighted_memory_to_active": 0.27214476466178894, "mix_std": 0.2537164092063904, "beta_std": 0.07662380486726761, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.17278563976287842, "evidence_weights": [-0.22591444849967957, -0.04673127830028534, -0.0362214595079422], "agreement_weight": 0.04201282560825348, "orthogonal_weight": 0.7804436087608337}, {"head": 3, "memory_to_active": 0.034931913018226624, "contribution_to_active": 0.6872963905334473, "mix_mean": 0.6589155197143555, "mix_low": 0.0006961822509765625, "mix_high": 0.01058197021484375, "read_cosine": 0.08304058015346527, "beta_mean": 0.00027872080681845546, "nonfinite": 0.0, "weighted_memory_to_active": 0.023818302899599075, "mix_std": 0.24809443950653076, "beta_std": 0.001328765763901174, "beta_low": 0.99847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.020466722548007965, "evidence_weights": [-0.3066847324371338, -0.3108541965484619, 0.7752745747566223], "agreement_weight": 0.19603148102760315, "orthogonal_weight": -0.8864034414291382}, {"head": 4, "memory_to_active": 1.514380931854248, "contribution_to_active": 0.7127420902252197, "mix_mean": 0.1467789113521576, "mix_low": 0.16156673431396484, "mix_high": 0.0010242462158203125, "read_cosine": 0.008801061660051346, "beta_mean": 0.009324034675955772, "nonfinite": 0.0, "weighted_memory_to_active": 0.7328181266784668, "mix_std": 0.19498717784881592, "beta_std": 0.026605017483234406, "beta_low": 0.77093505859375, "beta_high": 0.0, "orthogonal_to_active": 0.12011787295341492, "evidence_weights": [0.015307833440601826, -0.1332620084285736, -0.002090111142024398], "agreement_weight": -0.004478818736970425, "orthogonal_weight": -0.17893284559249878}, {"head": 5, "memory_to_active": 0.08081300556659698, "contribution_to_active": 0.4319832921028137, "mix_mean": 0.5541545748710632, "mix_low": 0.0059528350830078125, "mix_high": 0.00162506103515625, "read_cosine": 0.32684481143951416, "beta_mean": 0.001013750908896327, "nonfinite": 0.0, "weighted_memory_to_active": 0.03864947333931923, "mix_std": 0.27400532364845276, "beta_std": 0.005364354234188795, "beta_low": 0.97930908203125, "beta_high": 0.0, "orthogonal_to_active": 0.030096519738435745, "evidence_weights": [-0.2652772068977356, -0.2849622070789337, 0.7497245073318481], "agreement_weight": 0.08238308876752853, "orthogonal_weight": -0.8308795094490051}]}, {"layer": 13, "memory_to_active": 0.2202121466398239, "contribution_to_active": 0.46268707513809204, "mix_mean": 0.4579358696937561, "mix_low": 0.004616101738065481, "mix_high": 0.0010641416301950812, "read_cosine": 0.3957483768463135, "beta_mean": 0.0180168729275465, "nonfinite": 0.0, "weighted_memory_to_active": 0.09907311201095581, "mix_std": 0.25663071870803833, "beta_std": 0.01742633804678917, "beta_low": 0.7721659541130066, "beta_high": 0.0, "orthogonal_to_active": 0.08271049708127975, "output_influence": 0.40626952052116394, "residual_influence": 0.13160689175128937, "state_active": 0.07943359017372131, "state_memory": 0.03605115786194801, "state_peak": 90.43522644042969, "growth_active": 0.9769299626350403, "growth_memory": 1.0035983324050903, "heads": [{"head": 0, "memory_to_active": 0.1482943892478943, "contribution_to_active": 0.4721054136753082, "mix_mean": 0.457594633102417, "mix_low": 0.0012903213500976562, "mix_high": 0.0010213851928710938, "read_cosine": 0.33054259419441223, "beta_mean": 0.001171811600215733, "nonfinite": 0.0, "weighted_memory_to_active": 0.07289562374353409, "mix_std": 0.26418012380599976, "beta_std": 0.0026886300183832645, "beta_low": 0.98223876953125, "beta_high": 0.0, "orthogonal_to_active": 0.06872536987066269, "evidence_weights": [-0.2023298442363739, -0.23014414310455322, 0.7288828492164612], "agreement_weight": 0.036394745111465454, "orthogonal_weight": -1.0271810293197632}, {"head": 1, "memory_to_active": 0.06113399937748909, "contribution_to_active": 0.5044494867324829, "mix_mean": 0.4396924674510956, "mix_low": 0.0027856826782226562, "mix_high": 9.34600830078125e-05, "read_cosine": 0.22756290435791016, "beta_mean": 0.0002830504672601819, "nonfinite": 0.0, "weighted_memory_to_active": 0.030399788171052933, "mix_std": 0.2586124539375305, "beta_std": 0.0007379218004643917, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.028518149629235268, "evidence_weights": [-0.18853625655174255, -0.23954635858535767, 0.7242169380187988], "agreement_weight": 0.10449585318565369, "orthogonal_weight": -0.9873772859573364}, {"head": 2, "memory_to_active": 0.2094707041978836, "contribution_to_active": 0.327134370803833, "mix_mean": 0.33347803354263306, "mix_low": 0.002567291259765625, "mix_high": 2.86102294921875e-05, "read_cosine": 0.7093935012817383, "beta_mean": 0.012821022421121597, "nonfinite": 0.0, "weighted_memory_to_active": 0.0784761905670166, "mix_std": 0.21582725644111633, "beta_std": 0.027582507580518723, "beta_low": 0.70159912109375, "beta_high": 0.0, "orthogonal_to_active": 0.034525807946920395, "evidence_weights": [-0.21581976115703583, -0.15287627279758453, 0.4847879707813263], "agreement_weight": 0.04176846146583557, "orthogonal_weight": -0.7062610983848572}, {"head": 3, "memory_to_active": 0.16671425104141235, "contribution_to_active": 0.5426890850067139, "mix_mean": 0.5659308433532715, "mix_low": 0.0076961517333984375, "mix_high": 0.0026798248291015625, "read_cosine": -0.04007489234209061, "beta_mean": 0.0009296347852796316, "nonfinite": 0.0, "weighted_memory_to_active": 0.11868424713611603, "mix_std": 0.2664305865764618, "beta_std": 0.004275176674127579, "beta_low": 0.99169921875, "beta_high": 0.0, "orthogonal_to_active": 0.12067168205976486, "evidence_weights": [-0.35265052318573, -0.23182907700538635, 1.633773684501648], "agreement_weight": 1.5211271047592163, "orthogonal_weight": -1.0353808403015137}, {"head": 4, "memory_to_active": 0.5231135487556458, "contribution_to_active": 0.345409631729126, "mix_mean": 0.45013877749443054, "mix_low": 0.007576942443847656, "mix_high": 0.0007963180541992188, "read_cosine": 0.862617015838623, "beta_mean": 0.09079257398843765, "nonfinite": 0.0, "weighted_memory_to_active": 0.1797381043434143, "mix_std": 0.2532253563404083, "beta_std": 0.06245715916156769, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.16974593698978424, "evidence_weights": [-0.23294013738632202, -0.2447514533996582, 0.4779170751571655], "agreement_weight": -0.0007962617091834545, "orthogonal_weight": 1.4099829196929932}, {"head": 5, "memory_to_active": 0.2125459611415863, "contribution_to_active": 0.5843343138694763, "mix_mean": 0.5007804036140442, "mix_low": 0.005780220031738281, "mix_high": 0.0017652511596679688, "read_cosine": 0.2844489812850952, "beta_mean": 0.002103141974657774, "nonfinite": 0.0, "weighted_memory_to_active": 0.11424471437931061, "mix_std": 0.2815086543560028, "beta_std": 0.006816626060754061, "beta_low": 0.95770263671875, "beta_high": 0.0, "orthogonal_to_active": 0.07407601177692413, "evidence_weights": [-0.21239785850048065, -0.2546664774417877, 0.44461339712142944], "agreement_weight": 0.06516577303409576, "orthogonal_weight": -0.7236334085464478}]}, {"layer": 14, "memory_to_active": 0.28784307837486267, "contribution_to_active": 0.4264511168003082, "mix_mean": 0.3956782817840576, "mix_low": 0.024120330810546875, "mix_high": 0.0029231710359454155, "read_cosine": 0.620049774646759, "beta_mean": 0.07152865827083588, "nonfinite": 0.0, "weighted_memory_to_active": 0.13700464367866516, "mix_std": 0.2502880096435547, "beta_std": 0.06765212118625641, "beta_low": 0.4009602963924408, "beta_high": 0.0, "orthogonal_to_active": 0.04534425958991051, "output_influence": 0.2661091387271881, "residual_influence": 0.1031111553311348, "state_active": 0.08086897432804108, "state_memory": 0.017015360295772552, "state_peak": 34.68592834472656, "growth_active": 1.2113813161849976, "growth_memory": 1.3123606443405151, "heads": [{"head": 0, "memory_to_active": 0.3185425102710724, "contribution_to_active": 0.2520681917667389, "mix_mean": 0.2508675158023834, "mix_low": 0.015293121337890625, "mix_high": 3.4332275390625e-05, "read_cosine": 0.7108494639396667, "beta_mean": 0.0668826475739479, "nonfinite": 0.0, "weighted_memory_to_active": 0.10463809967041016, "mix_std": 0.22373376786708832, "beta_std": 0.07880404591560364, "beta_low": 0.078125, "beta_high": 0.0, "orthogonal_to_active": 0.02638668194413185, "evidence_weights": [-0.06595677882432938, -0.10666771978139877, 0.7331073880195618], "agreement_weight": 0.050072699785232544, "orthogonal_weight": 0.37028875946998596}, {"head": 1, "memory_to_active": 0.47892701625823975, "contribution_to_active": 0.4382890462875366, "mix_mean": 0.28058305382728577, "mix_low": 0.10191822052001953, "mix_high": 0.00028228759765625, "read_cosine": 0.648080587387085, "beta_mean": 0.29017484188079834, "nonfinite": 0.0, "weighted_memory_to_active": 0.24241358041763306, "mix_std": 0.2774258852005005, "beta_std": 0.25711387395858765, "beta_low": 0.08245849609375, "beta_high": 0.0, "orthogonal_to_active": 0.023653248324990273, "evidence_weights": [0.0021574327256530523, -0.2740093171596527, -0.08186858892440796], "agreement_weight": 0.05802483856678009, "orthogonal_weight": 0.1659632921218872}, {"head": 2, "memory_to_active": 0.2890881896018982, "contribution_to_active": 0.4839196801185608, "mix_mean": 0.484505295753479, "mix_low": 0.005562782287597656, "mix_high": 0.0010623931884765625, "read_cosine": 0.3763054311275482, "beta_mean": 0.011569496244192123, "nonfinite": 0.0, "weighted_memory_to_active": 0.1638425588607788, "mix_std": 0.2703663110733032, "beta_std": 0.01496044546365738, "beta_low": 0.63177490234375, "beta_high": 0.0, "orthogonal_to_active": 0.11738617718219757, "evidence_weights": [-0.18932299315929413, -0.2439892590045929, 0.702254056930542], "agreement_weight": 0.08189938217401505, "orthogonal_weight": -0.8096657991409302}, {"head": 3, "memory_to_active": 0.0723680704832077, "contribution_to_active": 0.6380674242973328, "mix_mean": 0.6492602825164795, "mix_low": 0.0026750564575195312, "mix_high": 0.0109100341796875, "read_cosine": 0.5133481025695801, "beta_mean": 0.010083850473165512, "nonfinite": 0.0, "weighted_memory_to_active": 0.04950272664427757, "mix_std": 0.24345165491104126, "beta_std": 0.00873037800192833, "beta_low": 0.63153076171875, "beta_high": 0.0, "orthogonal_to_active": 0.023638829588890076, "evidence_weights": [-0.3784918785095215, -0.32526496052742004, 0.4403471052646637], "agreement_weight": 0.07742606103420258, "orthogonal_weight": -0.6266670823097229}, {"head": 4, "memory_to_active": 0.21208971738815308, "contribution_to_active": 0.4785059988498688, "mix_mean": 0.4436488449573517, "mix_low": 0.004067420959472656, "mix_high": 0.004927635192871094, "read_cosine": 0.7764971256256104, "beta_mean": 0.047238364815711975, "nonfinite": 0.0, "weighted_memory_to_active": 0.12268801033496857, "mix_std": 0.25872811675071716, "beta_std": 0.03468558192253113, "beta_low": 0.0450439453125, "beta_high": 0.0, "orthogonal_to_active": 0.031831420958042145, "evidence_weights": [-0.2153906673192978, -0.26523590087890625, 0.32325461506843567], "agreement_weight": 0.11107945442199707, "orthogonal_weight": -0.4465509057044983}, {"head": 5, "memory_to_active": 0.3560429513454437, "contribution_to_active": 0.2678562104701996, "mix_mean": 0.26520463824272156, "mix_low": 0.01520538330078125, "mix_high": 0.0003223419189453125, "read_cosine": 0.6952179074287415, "beta_mean": 0.0032227132469415665, "nonfinite": 0.0, "weighted_memory_to_active": 0.13894276320934296, "mix_std": 0.22802232205867767, "beta_std": 0.011618386954069138, "beta_low": 0.93682861328125, "beta_high": 0.0, "orthogonal_to_active": 0.04916919022798538, "evidence_weights": [-0.18427857756614685, -0.13738451898097992, 0.14803236722946167], "agreement_weight": 0.0931035652756691, "orthogonal_weight": -0.564872682094574}]}, {"layer": 15, "memory_to_active": 0.08000801503658295, "contribution_to_active": 0.46998679637908936, "mix_mean": 0.38506561517715454, "mix_low": 0.007485389709472656, "mix_high": 0.002189000602811575, "read_cosine": 0.6583383083343506, "beta_mean": 0.04753350466489792, "nonfinite": 0.0, "weighted_memory_to_active": 0.03621513769030571, "mix_std": 0.2737078070640564, "beta_std": 0.049825768917798996, "beta_low": 0.7090861797332764, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.006847543641924858, "output_influence": 0.28906387090682983, "residual_influence": 0.1521487832069397, "state_active": 0.11400144547224045, "state_memory": 0.008875860832631588, "state_peak": 29.56561279296875, "growth_active": 1.3491864204406738, "growth_memory": 1.577609658241272, "heads": [{"head": 0, "memory_to_active": 0.11404989659786224, "contribution_to_active": 0.4738155007362366, "mix_mean": 0.3530401289463043, "mix_low": 0.004818916320800781, "mix_high": 0.0024614334106445312, "read_cosine": 0.693890392780304, "beta_mean": 0.03983495384454727, "nonfinite": 0.0, "weighted_memory_to_active": 0.052205782383680344, "mix_std": 0.2641682028770447, "beta_std": 0.031883031129837036, "beta_low": 0.15106201171875, "beta_high": 0.0, "orthogonal_to_active": 0.014418724924325943, "evidence_weights": [-0.16247090697288513, -0.19566436111927032, 0.3463380038738251], "agreement_weight": 0.08244667947292328, "orthogonal_weight": -0.5106076598167419}, {"head": 1, "memory_to_active": 0.19772446155548096, "contribution_to_active": 0.3998185396194458, "mix_mean": 0.3817305266857147, "mix_low": 0.005767822265625, "mix_high": 0.00457000732421875, "read_cosine": 0.8519051671028137, "beta_mean": 0.002278723753988743, "nonfinite": 0.0, "weighted_memory_to_active": 0.07553710788488388, "mix_std": 0.2722586989402771, "beta_std": 0.005351203493773937, "beta_low": 0.9609375, "beta_high": 0.0, "orthogonal_to_active": 0.004982668906450272, "evidence_weights": [-0.19048060476779938, -0.19480577111244202, 0.2854000926017761], "agreement_weight": 0.0948345959186554, "orthogonal_weight": -0.10197953879833221}, {"head": 2, "memory_to_active": 0.00645443145185709, "contribution_to_active": 0.4373377561569214, "mix_mean": 0.3489694893360138, "mix_low": 0.012530326843261719, "mix_high": 0.000141143798828125, "read_cosine": 0.7426113486289978, "beta_mean": 0.00011458709195721895, "nonfinite": 0.0, "weighted_memory_to_active": 0.002673287643119693, "mix_std": 0.2685113549232483, "beta_std": 0.0004320566076785326, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.0005121073336340487, "evidence_weights": [-0.21150149405002594, -0.2738681435585022, 0.2775147259235382], "agreement_weight": 0.050681229680776596, "orthogonal_weight": -0.284515917301178}, {"head": 3, "memory_to_active": 0.04267222434282303, "contribution_to_active": 0.4678162932395935, "mix_mean": 0.31085404753685, "mix_low": 0.0039157867431640625, "mix_high": 5.054473876953125e-05, "read_cosine": 0.674078106880188, "beta_mean": 0.0005965759046375751, "nonfinite": 0.0, "weighted_memory_to_active": 0.02044498361647129, "mix_std": 0.24624621868133545, "beta_std": 0.0012844761367887259, "beta_low": 0.99688720703125, "beta_high": 0.0, "orthogonal_to_active": 0.010985327884554863, "evidence_weights": [-0.18202027678489685, -0.22817440330982208, 0.4423341453075409], "agreement_weight": 0.007108013611286879, "orthogonal_weight": -0.8233774900436401}, {"head": 4, "memory_to_active": 0.11300401389598846, "contribution_to_active": 0.5821954011917114, "mix_mean": 0.41046643257141113, "mix_low": 0.011782646179199219, "mix_high": 0.0009403228759765625, "read_cosine": 0.433438777923584, "beta_mean": 0.2422008514404297, "nonfinite": 0.0, "weighted_memory_to_active": 0.0638844445347786, "mix_std": 0.3087717890739441, "beta_std": 0.25969135761260986, "beta_low": 0.14599609375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.008348105475306511, "evidence_weights": [-0.25619298219680786, -0.2542727291584015, 0.22198013961315155], "agreement_weight": 0.010956586338579655, "orthogonal_weight": -0.17028075456619263}, {"head": 5, "memory_to_active": 0.006143032107502222, "contribution_to_active": 0.4589369595050812, "mix_mean": 0.5053331255912781, "mix_low": 0.006096839904785156, "mix_high": 0.004970550537109375, "read_cosine": 0.5541055202484131, "beta_mean": 0.00017533291247673333, "nonfinite": 0.0, "weighted_memory_to_active": 0.0025452033150941133, "mix_std": 0.2822904884815216, "beta_std": 0.00031247769948095083, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018383250571787357, "evidence_weights": [-0.35801371932029724, -0.296637624502182, 0.3827577829360962], "agreement_weight": 0.14417360723018646, "orthogonal_weight": 0.8515912294387817}]}]}, {"step": 17000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7763928879976447, "memory_to_active": 0.9716812968254089, "contribution_to_active": 0.45242083072662354, "mix_mean": 0.3098180294036865, "mix_low": 0.17253652215003967, "mix_high": 0.005352377891540527, "read_cosine": 0.40165412425994873, "beta_mean": 0.09382399916648865, "nonfinite": 0.0, "weighted_memory_to_active": 0.20425471663475037, "mix_std": 0.20925916731357574, "beta_std": 0.07490257918834686, "beta_low": 0.45860546827316284, "beta_high": 1.1444091796875e-05, "orthogonal_to_active": 0.04120985418558121, "samples": 4, "scored_tokens": 899, "normal_loss": 3.187927484512329, "normal_token_accuracy": 0.42046719789505005, "state_active": 0.0594978965818882, "state_memory": 0.022733410820364952, "state_peak": 106.39469146728516, "growth_active": 1.1297634840011597, "growth_memory": 1.4044039249420166, "output_influence": 0.31571438908576965, "residual_influence": 0.09916277974843979, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.294847249984741, "no_memory_loss_delta": 0.22219786047935486, "scrambled_loss_delta": 0.1816326081752777, "memory_only_loss_delta": 2.305877208709717, "no_evidence_loss_delta": 0.11943747103214264, "no_agreement_loss_delta": -0.0066162776201963425, "no_orthogonal_loss_delta": -0.008703031577169895}, {"index": 94, "tokens": 256, "normal_loss": 3.238424777984619, "no_memory_loss_delta": 0.3326241970062256, "scrambled_loss_delta": 0.47914063930511475, "memory_only_loss_delta": 2.391831874847412, "no_evidence_loss_delta": 0.15748752653598785, "no_agreement_loss_delta": 0.039445262402296066, "no_orthogonal_loss_delta": 0.027275821194052696}, {"index": 188, "tokens": 256, "normal_loss": 3.59049391746521, "no_memory_loss_delta": 0.2837826907634735, "scrambled_loss_delta": 0.15666872262954712, "memory_only_loss_delta": 2.166726589202881, "no_evidence_loss_delta": 0.16651034355163574, "no_agreement_loss_delta": 0.0022161074448376894, "no_orthogonal_loss_delta": 0.0013190205208957195}, {"index": 283, "tokens": 131, "normal_loss": 2.093608856201172, "no_memory_loss_delta": 0.11359159648418427, "scrambled_loss_delta": 0.31137701869010925, "memory_only_loss_delta": 2.3029351234436035, "no_evidence_loss_delta": 0.03148287162184715, "no_agreement_loss_delta": -4.750080552184954e-05, "no_orthogonal_loss_delta": -0.01255219429731369}], "agreement_weight_abs_mean": 0.09135081619024277, "agreement_weight_abs_max": 1.664100170135498, "agreement_weight_mean": 0.0755663514137268, "orthogonal_weight_abs_mean": 0.4987926781177521, "orthogonal_weight_abs_max": 2.961101531982422, "orthogonal_weight_mean": -0.16858547925949097, "no_memory_loss": 3.443281650543213, "no_memory_loss_delta": 0.2553540766239166, "no_memory_token_accuracy_gain": 0.025583982467651367, "no_memory_token_win_fraction": 0.5928809642791748, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 22.53578332863274, "scrambled_loss": 3.466076135635376, "scrambled_loss_delta": 0.2781485319137573, "scrambled_token_accuracy_gain": 0.030033370479941368, "scrambled_token_win_fraction": 0.6384872198104858, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 24.28156528786211, "memory_only_loss": 5.478227615356445, "memory_only_loss_delta": 2.2903006076812744, "memory_only_token_accuracy_gain": 0.21579532325267792, "memory_only_token_win_fraction": 0.8809788823127747, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 89.8763975047393, "no_evidence_loss": 3.3187882900238037, "no_evidence_loss_delta": 0.13086062669754028, "no_evidence_token_accuracy_gain": 0.01001112349331379, "no_evidence_token_win_fraction": 0.5528364777565002, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 12.265995635042595, "no_agreement_loss": 3.1979000568389893, "no_agreement_loss_delta": 0.009972547180950642, "no_agreement_token_accuracy_gain": -0.0033370412420481443, "no_agreement_token_win_fraction": 0.46718576550483704, "no_agreement_probe_win_fraction": 0.5, "no_agreement_ppl_reduction_pct": 0.9922986218813639, "no_orthogonal_loss": 3.191763162612915, "no_orthogonal_loss_delta": 0.003835334675386548, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5472747683525085, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 0.3827989173170866, "short_no_memory_loss_delta": 0.19780398905277252, "long_context_memory_gain": 0.057550087571144104, "normal_context_loss_gain": 0.1305040717124939, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.0283703804016113, "contribution_to_active": 0.8329009413719177, "mix_mean": 0.09398271143436432, "mix_low": 0.4845067858695984, "mix_high": 0.0034639041405171156, "read_cosine": 0.4793497323989868, "beta_mean": 0.2612777352333069, "nonfinite": 0.0, "weighted_memory_to_active": 0.8866159915924072, "mix_std": 0.08394457399845123, "beta_std": 0.15339554846286774, "beta_low": 0.001129150390625, "beta_high": 0.0, "orthogonal_to_active": 0.04222952947020531, "output_influence": 0.12412942200899124, "residual_influence": 0.17723461985588074, "state_active": 0.08374587446451187, "state_memory": 0.050607673823833466, "state_peak": 51.256656646728516, "growth_active": 1.2917509078979492, "growth_memory": 1.2879695892333984, "heads": [{"head": 0, "memory_to_active": 2.2389397621154785, "contribution_to_active": 0.3098176419734955, "mix_mean": 0.020201731473207474, "mix_low": 0.533879280090332, "mix_high": 0.0, "read_cosine": 0.334826797246933, "beta_mean": 0.20257149636745453, "nonfinite": 0.0, "weighted_memory_to_active": 0.386146605014801, "mix_std": 0.04957716912031174, "beta_std": 0.09691283106803894, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.029836121946573257, "evidence_weights": [-0.040836531668901443, -0.07424390316009521, -0.0677337720990181], "agreement_weight": -0.0845680832862854, "orthogonal_weight": -0.09089861810207367}, {"head": 1, "memory_to_active": 4.149675369262695, "contribution_to_active": 1.47332763671875, "mix_mean": 0.016895681619644165, "mix_low": 0.5683021545410156, "mix_high": 0.0, "read_cosine": 0.4771993160247803, "beta_mean": 0.3616392910480499, "nonfinite": 0.0, "weighted_memory_to_active": 1.8204195499420166, "mix_std": 0.04511093348264694, "beta_std": 0.16328106820583344, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1259235143661499, "evidence_weights": [0.019125575199723244, -0.0344025082886219, 0.19198447465896606], "agreement_weight": 0.015285483561456203, "orthogonal_weight": -0.17318721115589142}, {"head": 2, "memory_to_active": 2.7081263065338135, "contribution_to_active": 0.4734271764755249, "mix_mean": 0.01473526656627655, "mix_low": 0.6329355239868164, "mix_high": 0.0, "read_cosine": 0.3316385746002197, "beta_mean": 0.3564070463180542, "nonfinite": 0.0, "weighted_memory_to_active": 0.5499420762062073, "mix_std": 0.029675278812646866, "beta_std": 0.25141602754592896, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.057835690677165985, "evidence_weights": [0.04671850427985191, 0.051746245473623276, 0.032854918390512466], "agreement_weight": -0.0015739444643259048, "orthogonal_weight": -0.1723909080028534}, {"head": 3, "memory_to_active": 5.551097869873047, "contribution_to_active": 2.340054512023926, "mix_mean": 0.04822132736444473, "mix_low": 0.30950164794921875, "mix_high": 0.0, "read_cosine": 0.4422771632671356, "beta_mean": 0.3182773292064667, "nonfinite": 0.0, "weighted_memory_to_active": 2.446990489959717, "mix_std": 0.0896427184343338, "beta_std": 0.2020643651485443, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02312658727169037, "evidence_weights": [-0.09451963752508163, -0.12207848578691483, -0.049673765897750854], "agreement_weight": -0.0133329713717103, "orthogonal_weight": -0.012267142534255981}, {"head": 4, "memory_to_active": 3.382953643798828, "contribution_to_active": 0.06614046543836594, "mix_mean": 0.006125187501311302, "mix_low": 0.8354835510253906, "mix_high": 0.0, "read_cosine": 0.3365861177444458, "beta_mean": 0.27439266443252563, "nonfinite": 0.0, "weighted_memory_to_active": 0.061579667031764984, "mix_std": 0.011114733293652534, "beta_std": 0.16393020749092102, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00779770826920867, "evidence_weights": [0.08502799272537231, 0.03596101328730583, -0.3086613714694977], "agreement_weight": -0.041086457669734955, "orthogonal_weight": 0.14215625822544098}, {"head": 5, "memory_to_active": 0.1394299864768982, "contribution_to_active": 0.3346382677555084, "mix_mean": 0.45771706104278564, "mix_low": 0.026938438415527344, "mix_high": 0.020783424377441406, "read_cosine": 0.9535704851150513, "beta_mean": 0.05437863618135452, "nonfinite": 0.0, "weighted_memory_to_active": 0.05461724102497101, "mix_std": 0.27854666113853455, "beta_std": 0.042768776416778564, "beta_low": 0.00677490234375, "beta_high": 0.0, "orthogonal_to_active": 0.008857561275362968, "evidence_weights": [-0.284644216299057, -0.26815158128738403, 0.2683473229408264], "agreement_weight": 0.17031843960285187, "orthogonal_weight": -0.3726831078529358}]}, {"layer": 1, "memory_to_active": 2.3165037631988525, "contribution_to_active": 0.3983767032623291, "mix_mean": 0.11837338656187057, "mix_low": 0.3988614082336426, "mix_high": 0.0003431638178881258, "read_cosine": 0.3280773460865021, "beta_mean": 0.2156294286251068, "nonfinite": 0.0, "weighted_memory_to_active": 0.33546775579452515, "mix_std": 0.11279888451099396, "beta_std": 0.13871246576309204, "beta_low": 0.1695048213005066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.035273801535367966, "output_influence": 0.140511617064476, "residual_influence": 0.03302904590964317, "state_active": 0.03924661874771118, "state_memory": 0.043999165296554565, "state_peak": 27.024993896484375, "growth_active": 1.0352380275726318, "growth_memory": 1.522934913635254, "heads": [{"head": 0, "memory_to_active": 0.06798622012138367, "contribution_to_active": 0.3426790237426758, "mix_mean": 0.26672932505607605, "mix_low": 0.028344154357910156, "mix_high": 3.4332275390625e-05, "read_cosine": -0.07700534909963608, "beta_mean": 0.00035395496524870396, "nonfinite": 0.0, "weighted_memory_to_active": 0.030049417167901993, "mix_std": 0.25593480467796326, "beta_std": 0.0006903539760969579, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.01839083805680275, "evidence_weights": [-0.13723377883434296, -0.17055320739746094, 0.014426000416278839], "agreement_weight": 0.1495804488658905, "orthogonal_weight": -0.6911202073097229}, {"head": 1, "memory_to_active": 3.5180554389953613, "contribution_to_active": 0.19507955014705658, "mix_mean": 0.022607915103435516, "mix_low": 0.47202587127685547, "mix_high": 9.5367431640625e-07, "read_cosine": 0.46238866448402405, "beta_mean": 0.3639950752258301, "nonfinite": 0.0, "weighted_memory_to_active": 0.2101619392633438, "mix_std": 0.049205634742975235, "beta_std": 0.22199127078056335, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006793681997805834, "evidence_weights": [-0.009424981661140919, 0.004332949873059988, -0.007568857632577419], "agreement_weight": -0.010554585605859756, "orthogonal_weight": 0.05766676738858223}, {"head": 2, "memory_to_active": 0.618721604347229, "contribution_to_active": 0.42426079511642456, "mix_mean": 0.36681491136550903, "mix_low": 0.00511932373046875, "mix_high": 0.0020132064819335938, "read_cosine": 0.8000421524047852, "beta_mean": 0.08317559957504272, "nonfinite": 0.0, "weighted_memory_to_active": 0.48287898302078247, "mix_std": 0.25291410088539124, "beta_std": 0.04166898503899574, "beta_low": 0.00128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.009922998026013374, "evidence_weights": [-0.23057293891906738, -0.38335949182510376, 0.21924279630184174], "agreement_weight": 0.014041421934962273, "orthogonal_weight": -0.043308671563863754}, {"head": 3, "memory_to_active": 2.8187079429626465, "contribution_to_active": 0.06144339218735695, "mix_mean": 0.010738499462604523, "mix_low": 0.765162467956543, "mix_high": 0.0, "read_cosine": 0.17975392937660217, "beta_mean": 0.34511443972587585, "nonfinite": 0.0, "weighted_memory_to_active": 0.08001518249511719, "mix_std": 0.030469000339508057, "beta_std": 0.24120676517486572, "beta_low": 0.00115966796875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.018156105652451515, "evidence_weights": [0.041099388152360916, 0.016278179362416267, 0.02272503636777401], "agreement_weight": -0.025994960218667984, "orthogonal_weight": -0.24127961695194244}, {"head": 4, "memory_to_active": 3.1565134525299072, "contribution_to_active": 1.3135164976119995, "mix_mean": 0.03592962026596069, "mix_low": 0.2953338623046875, "mix_high": 2.86102294921875e-06, "read_cosine": 0.3883938193321228, "beta_mean": 0.1918906569480896, "nonfinite": 0.0, "weighted_memory_to_active": 1.159355640411377, "mix_std": 0.06604712456464767, "beta_std": 0.09812302142381668, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.15699350833892822, "evidence_weights": [-0.02152658812701702, -0.007758143357932568, 0.07466431707143784], "agreement_weight": -0.02646048739552498, "orthogonal_weight": 0.15425661206245422}, {"head": 5, "memory_to_active": 3.7190377712249756, "contribution_to_active": 0.05328091233968735, "mix_mean": 0.007420085370540619, "mix_low": 0.8271827697753906, "mix_high": 7.62939453125e-06, "read_cosine": 0.21489068865776062, "beta_mean": 0.30924680829048157, "nonfinite": 0.0, "weighted_memory_to_active": 0.05034523829817772, "mix_std": 0.022222649306058884, "beta_std": 0.22859430313110352, "beta_low": 0.015380859375, "beta_high": 0.0, "orthogonal_to_active": 0.0013856629375368357, "evidence_weights": [0.13712434470653534, 0.06864883750677109, 0.3212853968143463], "agreement_weight": -0.15340964496135712, "orthogonal_weight": 0.027698982506990433}]}, {"layer": 2, "memory_to_active": 1.16802978515625, "contribution_to_active": 0.2964860498905182, "mix_mean": 0.26687413454055786, "mix_low": 0.27464932203292847, "mix_high": 0.002404371974989772, "read_cosine": 0.36024415493011475, "beta_mean": 0.114137202501297, "nonfinite": 0.0, "weighted_memory_to_active": 0.06809515506029129, "mix_std": 0.19225934147834778, "beta_std": 0.07555553317070007, "beta_low": 0.5424296259880066, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.015077861025929451, "output_influence": 0.25871068239212036, "residual_influence": 0.06176430732011795, "state_active": 0.03829149901866913, "state_memory": 0.023582879453897476, "state_peak": 41.64177322387695, "growth_active": 1.0400148630142212, "growth_memory": 1.551147222518921, "heads": [{"head": 0, "memory_to_active": 0.01161007210612297, "contribution_to_active": 0.18518885970115662, "mix_mean": 0.32411515712738037, "mix_low": 0.017395973205566406, "mix_high": 0.00063323974609375, "read_cosine": 0.23418018221855164, "beta_mean": 8.000600792001933e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00433559762313962, "mix_std": 0.2787502408027649, "beta_std": 0.0001273571833735332, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00021987457876093686, "evidence_weights": [-0.1957246959209442, -0.13071848452091217, 0.20813769102096558], "agreement_weight": 0.13170990347862244, "orthogonal_weight": -0.05828522890806198}, {"head": 1, "memory_to_active": 3.0666427612304688, "contribution_to_active": 0.0373736135661602, "mix_mean": 0.004334889352321625, "mix_low": 0.9175624847412109, "mix_high": 9.5367431640625e-07, "read_cosine": 0.1964818835258484, "beta_mean": 0.3337629437446594, "nonfinite": 0.0, "weighted_memory_to_active": 0.02838590368628502, "mix_std": 0.010963449254631996, "beta_std": 0.151351198554039, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008705012500286102, "evidence_weights": [0.10276814550161362, 0.11061368137598038, 0.23980708420276642], "agreement_weight": 0.03354519605636597, "orthogonal_weight": 0.31926265358924866}, {"head": 2, "memory_to_active": 0.029371850192546844, "contribution_to_active": 0.4295133352279663, "mix_mean": 0.3998444676399231, "mix_low": 0.0037374496459960938, "mix_high": 0.0019559860229492188, "read_cosine": 0.3032154440879822, "beta_mean": 4.936256300425157e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012396170757710934, "mix_std": 0.2791329026222229, "beta_std": 0.00010345331247663125, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002297662664204836, "evidence_weights": [-0.1949649304151535, -0.23412294685840607, 0.48728102445602417], "agreement_weight": 0.022425150498747826, "orthogonal_weight": -0.24422909319400787}, {"head": 3, "memory_to_active": 0.25754350423812866, "contribution_to_active": 0.44047558307647705, "mix_mean": 0.3413020968437195, "mix_low": 0.004481315612792969, "mix_high": 0.00406646728515625, "read_cosine": 0.6738137602806091, "beta_mean": 0.04781470075249672, "nonfinite": 0.0, "weighted_memory_to_active": 0.12181849777698517, "mix_std": 0.25673341751098633, "beta_std": 0.06474217772483826, "beta_low": 0.22613525390625, "beta_high": 0.0, "orthogonal_to_active": 0.06359078735113144, "evidence_weights": [-0.13498272001743317, -0.2028186023235321, 0.6980893611907959], "agreement_weight": 0.06509839743375778, "orthogonal_weight": -0.8012216687202454}, {"head": 4, "memory_to_active": 3.613198757171631, "contribution_to_active": 0.1836349070072174, "mix_mean": 0.013677847571671009, "mix_low": 0.7018899917602539, "mix_high": 0.0, "read_cosine": 0.19449838995933533, "beta_mean": 0.3030395209789276, "nonfinite": 0.0, "weighted_memory_to_active": 0.2231806516647339, "mix_std": 0.040870022028684616, "beta_std": 0.236899733543396, "beta_low": 0.0284423828125, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.014407211914658546, "evidence_weights": [0.03985128551721573, 0.0343037024140358, 0.14008043706417084], "agreement_weight": 0.03116529807448387, "orthogonal_weight": -0.07679634541273117}, {"head": 5, "memory_to_active": 0.029811182990670204, "contribution_to_active": 0.5027300119400024, "mix_mean": 0.5179702043533325, "mix_low": 0.0028285980224609375, "mix_high": 0.007769584655761719, "read_cosine": 0.5592750906944275, "beta_mean": 7.667452882742509e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.018454095348715782, "mix_std": 0.28710606694221497, "beta_std": 0.00010921764624072239, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012466153129935265, "evidence_weights": [-0.24097104370594025, -0.25661832094192505, 0.7056687474250793], "agreement_weight": 0.03025968372821808, "orthogonal_weight": 0.1337176263332367}]}, {"layer": 3, "memory_to_active": 1.3058016300201416, "contribution_to_active": 0.3328097462654114, "mix_mean": 0.20639392733573914, "mix_low": 0.24488529562950134, "mix_high": 0.0029195151291787624, "read_cosine": 0.2879524230957031, "beta_mean": 0.1124781146645546, "nonfinite": 0.0, "weighted_memory_to_active": 0.15986603498458862, "mix_std": 0.2051810473203659, "beta_std": 0.08209525048732758, "beta_low": 0.4106852412223816, "beta_high": 0.0, "orthogonal_to_active": 0.036529697477817535, "output_influence": 0.247798353433609, "residual_influence": 0.04469525068998337, "state_active": 0.02655874751508236, "state_memory": 0.0204615481197834, "state_peak": 22.869407653808594, "growth_active": 0.9962069988250732, "growth_memory": 1.7195407152175903, "heads": [{"head": 0, "memory_to_active": 0.6748125553131104, "contribution_to_active": 0.3317623734474182, "mix_mean": 0.23441411554813385, "mix_low": 0.10849666595458984, "mix_high": 0.0012731552124023438, "read_cosine": 0.277275949716568, "beta_mean": 0.01618942804634571, "nonfinite": 0.0, "weighted_memory_to_active": 0.19541966915130615, "mix_std": 0.27146822214126587, "beta_std": 0.01520693302154541, "beta_low": 0.4178466796875, "beta_high": 0.0, "orthogonal_to_active": 0.08211207389831543, "evidence_weights": [-0.07709866017103195, -0.14599426090717316, 0.07260210067033768], "agreement_weight": 0.08842038363218307, "orthogonal_weight": -0.44305935502052307}, {"head": 1, "memory_to_active": 0.02104254812002182, "contribution_to_active": 0.26566699147224426, "mix_mean": 0.2726432681083679, "mix_low": 0.056929588317871094, "mix_high": 0.0018129348754882812, "read_cosine": 0.20385421812534332, "beta_mean": 8.896542567526922e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005763005930930376, "mix_std": 0.27741357684135437, "beta_std": 0.00024189072428271174, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00045234645949676633, "evidence_weights": [-0.15664006769657135, -0.12526053190231323, 0.34780144691467285], "agreement_weight": 0.12029227614402771, "orthogonal_weight": -0.08434373885393143}, {"head": 2, "memory_to_active": 3.1529462337493896, "contribution_to_active": 0.30802562832832336, "mix_mean": 0.03989469259977341, "mix_low": 0.4282398223876953, "mix_high": 2.765655517578125e-05, "read_cosine": 0.19941295683383942, "beta_mean": 0.23174726963043213, "nonfinite": 0.0, "weighted_memory_to_active": 0.41201847791671753, "mix_std": 0.09750649333000183, "beta_std": 0.148481547832489, "beta_low": 0.00469970703125, "beta_high": 0.0, "orthogonal_to_active": 0.023278648033738136, "evidence_weights": [-0.048037439584732056, 0.009550394490361214, 0.0628441646695137], "agreement_weight": -0.014598666690289974, "orthogonal_weight": -0.07593235373497009}, {"head": 3, "memory_to_active": 0.035307396203279495, "contribution_to_active": 0.4586053788661957, "mix_mean": 0.4586745500564575, "mix_low": 0.027136802673339844, "mix_high": 0.009949684143066406, "read_cosine": 0.2805618643760681, "beta_mean": 0.00012526166392490268, "nonfinite": 0.0, "weighted_memory_to_active": 0.020152384415268898, "mix_std": 0.31871870160102844, "beta_std": 0.00029730709502473474, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006009866017848253, "evidence_weights": [-0.18328648805618286, -0.20563285052776337, 1.2512750625610352], "agreement_weight": 0.7083695530891418, "orthogonal_weight": -0.38271790742874146}, {"head": 4, "memory_to_active": 3.5150187015533447, "contribution_to_active": 0.05513186752796173, "mix_mean": 0.009173755533993244, "mix_low": 0.7847518920898438, "mix_high": 0.0, "read_cosine": 0.21919435262680054, "beta_mean": 0.32879751920700073, "nonfinite": 0.0, "weighted_memory_to_active": 0.08915982395410538, "mix_std": 0.022660139948129654, "beta_std": 0.2211696207523346, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.034775510430336, "evidence_weights": [0.1469474881887436, 0.11305295675992966, -0.050360389053821564], "agreement_weight": -0.0469205342233181, "orthogonal_weight": -0.40959450602531433}, {"head": 5, "memory_to_active": 0.4356817603111267, "contribution_to_active": 0.5776660442352295, "mix_mean": 0.22356320917606354, "mix_low": 0.06375694274902344, "mix_high": 0.0044536590576171875, "read_cosine": 0.5474151372909546, "beta_mean": 0.09792015701532364, "nonfinite": 0.0, "weighted_memory_to_active": 0.23668277263641357, "mix_std": 0.24331916868686676, "beta_std": 0.10717422515153885, "beta_low": 0.04095458984375, "beta_high": 0.0, "orthogonal_to_active": 0.0725497454404831, "evidence_weights": [-0.10091133415699005, -0.17247721552848816, 0.25697827339172363], "agreement_weight": 0.16583549976348877, "orthogonal_weight": -0.4729044437408447}]}, {"layer": 4, "memory_to_active": 1.2929039001464844, "contribution_to_active": 0.3731834888458252, "mix_mean": 0.2861749529838562, "mix_low": 0.27492237091064453, "mix_high": 0.0026488304138183594, "read_cosine": 0.40715569257736206, "beta_mean": 0.13329045474529266, "nonfinite": 0.0, "weighted_memory_to_active": 0.1637670397758484, "mix_std": 0.15738451480865479, "beta_std": 0.1094101294875145, "beta_low": 0.316162109375, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04090716689825058, "output_influence": 0.26286613941192627, "residual_influence": 0.06295057386159897, "state_active": 0.042889393866062164, "state_memory": 0.029874123632907867, "state_peak": 53.26459884643555, "growth_active": 1.1120926141738892, "growth_memory": 1.6249701976776123, "heads": [{"head": 0, "memory_to_active": 0.3494144380092621, "contribution_to_active": 0.5920630693435669, "mix_mean": 0.5359780788421631, "mix_low": 0.01067352294921875, "mix_high": 0.006073951721191406, "read_cosine": 0.5744373798370361, "beta_mean": 0.19844190776348114, "nonfinite": 0.0, "weighted_memory_to_active": 0.24330057203769684, "mix_std": 0.31723397970199585, "beta_std": 0.21702472865581512, "beta_low": 0.17706298828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.03232894837856293, "evidence_weights": [-0.23332370817661285, -0.29435840249061584, 0.423033744096756], "agreement_weight": 0.039594363421201706, "orthogonal_weight": 0.17600181698799133}, {"head": 1, "memory_to_active": 0.2416834831237793, "contribution_to_active": 0.4210694432258606, "mix_mean": 0.46399086713790894, "mix_low": 0.0013265609741210938, "mix_high": 0.0017347335815429688, "read_cosine": 0.6805030703544617, "beta_mean": 0.01034548133611679, "nonfinite": 0.0, "weighted_memory_to_active": 0.1197037547826767, "mix_std": 0.26847484707832336, "beta_std": 0.01367670577019453, "beta_low": 0.66632080078125, "beta_high": 0.0, "orthogonal_to_active": 0.055720359086990356, "evidence_weights": [-0.22760175168514252, -0.21248874068260193, 0.6527838706970215], "agreement_weight": 0.03911983221769333, "orthogonal_weight": -0.6450589895248413}, {"head": 2, "memory_to_active": 2.7199037075042725, "contribution_to_active": 0.11587311327457428, "mix_mean": 0.01424635760486126, "mix_low": 0.6308183670043945, "mix_high": 0.0, "read_cosine": 0.29481497406959534, "beta_mean": 0.16508269309997559, "nonfinite": 0.0, "weighted_memory_to_active": 0.15767420828342438, "mix_std": 0.023903824388980865, "beta_std": 0.10660267621278763, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.005664187483489513, "evidence_weights": [0.04819625988602638, 0.02578037977218628, -0.041285187005996704], "agreement_weight": 0.06365227699279785, "orthogonal_weight": -0.057068344205617905}, {"head": 3, "memory_to_active": 1.7665843963623047, "contribution_to_active": 0.1129070296883583, "mix_mean": 0.030126657336950302, "mix_low": 0.3812828063964844, "mix_high": 0.0, "read_cosine": 0.2222384363412857, "beta_mean": 0.17148280143737793, "nonfinite": 0.0, "weighted_memory_to_active": 0.15829157829284668, "mix_std": 0.05019459128379822, "beta_std": 0.15072491765022278, "beta_low": 0.05267333984375, "beta_high": 0.0, "orthogonal_to_active": 0.02266337163746357, "evidence_weights": [-0.018050679937005043, 0.04755407199263573, -0.19690677523612976], "agreement_weight": 0.030468981713056564, "orthogonal_weight": -0.18017922341823578}, {"head": 4, "memory_to_active": 0.02657444030046463, "contribution_to_active": 0.6169266700744629, "mix_mean": 0.6583161354064941, "mix_low": 0.0006122589111328125, "mix_high": 0.008066177368164062, "read_cosine": 0.3234509825706482, "beta_mean": 5.2833573136013e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.019555697217583656, "mix_std": 0.24890092015266418, "beta_std": 0.00014427056885324419, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017777351895347238, "evidence_weights": [-0.2850530743598938, -0.3272048234939575, 0.7753568887710571], "agreement_weight": 0.08001765608787537, "orthogonal_weight": -0.11384402960538864}, {"head": 5, "memory_to_active": 2.6532630920410156, "contribution_to_active": 0.38026145100593567, "mix_mean": 0.014391853474080563, "mix_low": 0.6248207092285156, "mix_high": 1.811981201171875e-05, "read_cosine": 0.34748923778533936, "beta_mean": 0.25433701276779175, "nonfinite": 0.0, "weighted_memory_to_active": 0.28407639265060425, "mix_std": 0.03559894487261772, "beta_std": 0.1682874858379364, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12728838622570038, "evidence_weights": [0.03710709512233734, -0.032427724450826645, -0.13670609891414642], "agreement_weight": -0.0435761883854866, "orthogonal_weight": 0.6713557839393616}]}, {"layer": 5, "memory_to_active": 0.6744163036346436, "contribution_to_active": 0.44266048073768616, "mix_mean": 0.39280933141708374, "mix_low": 0.09920597821474075, "mix_high": 0.00979471206665039, "read_cosine": 0.24805700778961182, "beta_mean": 0.05916760489344597, "nonfinite": 0.0, "weighted_memory_to_active": 0.1668984591960907, "mix_std": 0.2482418566942215, "beta_std": 0.07397840917110443, "beta_low": 0.5975748896598816, "beta_high": 0.0, "orthogonal_to_active": 0.05111569166183472, "output_influence": 0.4229317903518677, "residual_influence": 0.0964944139122963, "state_active": 0.05633269622921944, "state_memory": 0.016620643436908722, "state_peak": 65.00455474853516, "growth_active": 1.11197829246521, "growth_memory": 1.2889398336410522, "heads": [{"head": 0, "memory_to_active": 1.2196640968322754, "contribution_to_active": 0.24761955440044403, "mix_mean": 0.11972438544034958, "mix_low": 0.15497875213623047, "mix_high": 0.0, "read_cosine": -0.14059269428253174, "beta_mean": 0.05972664803266525, "nonfinite": 0.0, "weighted_memory_to_active": 0.17332559823989868, "mix_std": 0.1646919697523117, "beta_std": 0.088216133415699, "beta_low": 0.179443359375, "beta_high": 0.0, "orthogonal_to_active": 0.03505302220582962, "evidence_weights": [-0.04231167212128639, -0.07812902331352234, -0.366004079580307], "agreement_weight": -0.019308805465698242, "orthogonal_weight": -0.19852663576602936}, {"head": 1, "memory_to_active": 0.030929189175367355, "contribution_to_active": 0.4803081750869751, "mix_mean": 0.5286152362823486, "mix_low": 0.009591102600097656, "mix_high": 0.01303863525390625, "read_cosine": 0.19258353114128113, "beta_mean": 9.34005729504861e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014242276549339294, "mix_std": 0.31666821241378784, "beta_std": 0.00020630474318750203, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01241404376924038, "evidence_weights": [-0.22159411013126373, -0.2367362082004547, 0.6905936002731323], "agreement_weight": 0.27455517649650574, "orthogonal_weight": -0.9034488201141357}, {"head": 2, "memory_to_active": 0.07966557145118713, "contribution_to_active": 0.49751967191696167, "mix_mean": 0.5763808488845825, "mix_low": 0.00194549560546875, "mix_high": 0.0116729736328125, "read_cosine": 0.4720176160335541, "beta_mean": 0.0021307363640516996, "nonfinite": 0.0, "weighted_memory_to_active": 0.05365757644176483, "mix_std": 0.27904045581817627, "beta_std": 0.0027573781553655863, "beta_low": 0.9783935546875, "beta_high": 0.0, "orthogonal_to_active": 0.0062910206615924835, "evidence_weights": [-0.2645641565322876, -0.3078646659851074, 0.72816002368927], "agreement_weight": 0.028872713446617126, "orthogonal_weight": -0.16742371022701263}, {"head": 3, "memory_to_active": 2.1338255405426025, "contribution_to_active": 0.21224631369113922, "mix_mean": 0.07827163487672806, "mix_low": 0.23793888092041016, "mix_high": 0.0, "read_cosine": 0.14844545722007751, "beta_mean": 0.07502609491348267, "nonfinite": 0.0, "weighted_memory_to_active": 0.36492621898651123, "mix_std": 0.12558653950691223, "beta_std": 0.11201070249080658, "beta_low": 0.22161865234375, "beta_high": 0.0, "orthogonal_to_active": 0.133770152926445, "evidence_weights": [-0.024873895570635796, -0.07209967821836472, -0.0019257229287177324], "agreement_weight": 0.06132232025265694, "orthogonal_weight": -0.45046284794807434}, {"head": 4, "memory_to_active": 0.5034024119377136, "contribution_to_active": 0.604318380355835, "mix_mean": 0.3266274333000183, "mix_low": 0.18747997283935547, "mix_high": 0.010706901550292969, "read_cosine": 0.5660662651062012, "beta_mean": 0.21772921085357666, "nonfinite": 0.0, "weighted_memory_to_active": 0.34314200282096863, "mix_std": 0.34137317538261414, "beta_std": 0.23941123485565186, "beta_low": 0.20941162109375, "beta_high": 0.0, "orthogonal_to_active": 0.08970580995082855, "evidence_weights": [-0.13068297505378723, -0.279448926448822, 0.004678803030401468], "agreement_weight": 0.04153819754719734, "orthogonal_weight": 0.39194852113723755}, {"head": 5, "memory_to_active": 0.07901059836149216, "contribution_to_active": 0.6139507293701172, "mix_mean": 0.7272363901138306, "mix_low": 0.0033016204833984375, "mix_high": 0.023349761962890625, "read_cosine": 0.24982184171676636, "beta_mean": 0.00029952675686217844, "nonfinite": 0.0, "weighted_memory_to_active": 0.05209711194038391, "mix_std": 0.2620907127857208, "beta_std": 0.0012686687987297773, "beta_low": 0.99664306640625, "beta_high": 0.0, "orthogonal_to_active": 0.02946009859442711, "evidence_weights": [-0.320874959230423, -0.3084743618965149, 1.6312321424484253], "agreement_weight": 0.315601646900177, "orthogonal_weight": -0.6115430593490601}]}, {"layer": 6, "memory_to_active": 0.4402313232421875, "contribution_to_active": 0.5515722632408142, "mix_mean": 0.5049033164978027, "mix_low": 0.0855778157711029, "mix_high": 0.024253051728010178, "read_cosine": 0.3015173077583313, "beta_mean": 0.08301298320293427, "nonfinite": 0.0, "weighted_memory_to_active": 0.16594745218753815, "mix_std": 0.2509409785270691, "beta_std": 0.09542134404182434, "beta_low": 0.5098063349723816, "beta_high": 0.0, "orthogonal_to_active": 0.037799008190631866, "output_influence": 0.4396163523197174, "residual_influence": 0.13188865780830383, "state_active": 0.05214226245880127, "state_memory": 0.011000905185937881, "state_peak": 60.60014343261719, "growth_active": 1.1834967136383057, "growth_memory": 1.31048583984375, "heads": [{"head": 0, "memory_to_active": 0.16011160612106323, "contribution_to_active": 0.5492478013038635, "mix_mean": 0.6494746804237366, "mix_low": 0.0002079010009765625, "mix_high": 0.0036382675170898438, "read_cosine": 0.15016593039035797, "beta_mean": 0.0121806301176548, "nonfinite": 0.0, "weighted_memory_to_active": 0.11585212498903275, "mix_std": 0.24676848948001862, "beta_std": 0.022109871730208397, "beta_low": 0.64453125, "beta_high": 0.0, "orthogonal_to_active": 0.085902638733387, "evidence_weights": [-0.2958377003669739, -0.29371926188468933, 0.43878549337387085], "agreement_weight": 0.09316322207450867, "orthogonal_weight": -0.8842183351516724}, {"head": 1, "memory_to_active": 0.01202117558568716, "contribution_to_active": 0.46013402938842773, "mix_mean": 0.4552949070930481, "mix_low": 0.0026884078979492188, "mix_high": 0.008802413940429688, "read_cosine": 0.5753017663955688, "beta_mean": 2.82408764178399e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004386513493955135, "mix_std": 0.29237794876098633, "beta_std": 6.40377911622636e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012235647300258279, "evidence_weights": [-0.21150152385234833, -0.2531736195087433, 0.692290723323822], "agreement_weight": -0.02608194574713707, "orthogonal_weight": -0.3705897033214569}, {"head": 2, "memory_to_active": 1.563520073890686, "contribution_to_active": 0.06487420946359634, "mix_mean": 0.022561751306056976, "mix_low": 0.499420166015625, "mix_high": 0.0, "read_cosine": 0.20193783938884735, "beta_mean": 0.10793288052082062, "nonfinite": 0.0, "weighted_memory_to_active": 0.0772884264588356, "mix_std": 0.043458543717861176, "beta_std": 0.10390904545783997, "beta_low": 0.001953125, "beta_high": 0.0, "orthogonal_to_active": 0.014613260515034199, "evidence_weights": [0.0853196382522583, 0.0297319944947958, 0.1797657161951065], "agreement_weight": 0.07250135391950607, "orthogonal_weight": -0.21166324615478516}, {"head": 3, "memory_to_active": 0.1496078372001648, "contribution_to_active": 0.6484242081642151, "mix_mean": 0.6299810409545898, "mix_low": 0.008021354675292969, "mix_high": 0.02870655059814453, "read_cosine": 0.4865957200527191, "beta_mean": 0.23668469488620758, "nonfinite": 0.0, "weighted_memory_to_active": 0.10869763791561127, "mix_std": 0.3387536406517029, "beta_std": 0.2733549475669861, "beta_low": 0.20379638671875, "beta_high": 0.0, "orthogonal_to_active": 0.04685123264789581, "evidence_weights": [-0.1787596493959427, -0.3389548659324646, 0.15162311494350433], "agreement_weight": -0.01406589150428772, "orthogonal_weight": 0.507806658744812}, {"head": 4, "memory_to_active": 0.6972076296806335, "contribution_to_active": 1.0424261093139648, "mix_mean": 0.7200543880462646, "mix_low": 0.0015277862548828125, "mix_high": 0.10334491729736328, "read_cosine": 0.13496103882789612, "beta_mean": 0.14073452353477478, "nonfinite": 0.0, "weighted_memory_to_active": 0.6532022356987, "mix_std": 0.30092719197273254, "beta_std": 0.17217719554901123, "beta_low": 0.2098388671875, "beta_high": 0.0, "orthogonal_to_active": 0.041642092168331146, "evidence_weights": [-0.24693159759044647, -0.5561345815658569, -0.024656392633914948], "agreement_weight": -0.015149353072047234, "orthogonal_weight": 0.07413654774427414}, {"head": 5, "memory_to_active": 0.05891932174563408, "contribution_to_active": 0.5443270206451416, "mix_mean": 0.5520529747009277, "mix_low": 0.0016012191772460938, "mix_high": 0.001026153564453125, "read_cosine": 0.2601414620876312, "beta_mean": 0.0005169397336430848, "nonfinite": 0.0, "weighted_memory_to_active": 0.03625774383544922, "mix_std": 0.2833600640296936, "beta_std": 0.000912955729290843, "beta_low": 0.99871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.03656123951077461, "evidence_weights": [-0.22574575245380402, -0.2705428898334503, 0.3096306324005127], "agreement_weight": 0.08337577432394028, "orthogonal_weight": -1.0753840208053589}]}, {"layer": 7, "memory_to_active": 2.118773937225342, "contribution_to_active": 0.18865829706192017, "mix_mean": 0.03668421506881714, "mix_low": 0.5114997625350952, "mix_high": 8.265178621513769e-06, "read_cosine": 0.21818485856056213, "beta_mean": 0.1456417739391327, "nonfinite": 0.0, "weighted_memory_to_active": 0.20558592677116394, "mix_std": 0.06490151584148407, "beta_std": 0.11700709909200668, "beta_low": 0.0398356169462204, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.04185386002063751, "output_influence": 0.21491968631744385, "residual_influence": 0.04695571959018707, "state_active": 0.015927083790302277, "state_memory": 0.031658586114645004, "state_peak": 19.931310653686523, "growth_active": 1.0509119033813477, "growth_memory": 1.3650341033935547, "heads": [{"head": 0, "memory_to_active": 2.252824306488037, "contribution_to_active": 0.4486653804779053, "mix_mean": 0.02712203934788704, "mix_low": 0.5490617752075195, "mix_high": 4.1961669921875e-05, "read_cosine": 0.36432501673698425, "beta_mean": 0.18715284764766693, "nonfinite": 0.0, "weighted_memory_to_active": 0.419700562953949, "mix_std": 0.06202954053878784, "beta_std": 0.13818469643592834, "beta_low": 0.00152587890625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.05167153477668762, "evidence_weights": [0.0008558612898923457, -0.07943609356880188, 0.10021716356277466], "agreement_weight": -0.03689521923661232, "orthogonal_weight": 0.1836671531200409}, {"head": 1, "memory_to_active": 2.0461225509643555, "contribution_to_active": 0.28892529010772705, "mix_mean": 0.09488382935523987, "mix_low": 0.28301429748535156, "mix_high": 2.86102294921875e-06, "read_cosine": 0.10232406109571457, "beta_mean": 0.1908877044916153, "nonfinite": 0.0, "weighted_memory_to_active": 0.35144132375717163, "mix_std": 0.13396325707435608, "beta_std": 0.11287194490432739, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0656118094921112, "evidence_weights": [-0.12856809794902802, -0.10026679933071136, -0.1338770091533661], "agreement_weight": 0.008308662101626396, "orthogonal_weight": -0.1969825178384781}, {"head": 2, "memory_to_active": 1.3013675212860107, "contribution_to_active": 0.21487048268318176, "mix_mean": 0.053507253527641296, "mix_low": 0.3046531677246094, "mix_high": 4.76837158203125e-06, "read_cosine": 0.1543245166540146, "beta_mean": 0.13762038946151733, "nonfinite": 0.0, "weighted_memory_to_active": 0.18074941635131836, "mix_std": 0.10097325593233109, "beta_std": 0.08158302307128906, "beta_low": 0.00640869140625, "beta_high": 0.0, "orthogonal_to_active": 0.0175656545907259, "evidence_weights": [-0.027369868010282516, -0.10741404443979263, 0.21734920144081116], "agreement_weight": 0.02457553520798683, "orthogonal_weight": -0.10027486830949783}, {"head": 3, "memory_to_active": 3.4407379627227783, "contribution_to_active": 0.06629101186990738, "mix_mean": 0.017118219286203384, "mix_low": 0.5654745101928711, "mix_high": 0.0, "read_cosine": 0.13280007243156433, "beta_mean": 0.16160957515239716, "nonfinite": 0.0, "weighted_memory_to_active": 0.15182673931121826, "mix_std": 0.03160838037729263, "beta_std": 0.16656753420829773, "beta_low": 0.049560546875, "beta_high": 0.0, "orthogonal_to_active": 0.09830717742443085, "evidence_weights": [0.10792374610900879, 0.06788255274295807, 0.33529868721961975], "agreement_weight": 0.07840712368488312, "orthogonal_weight": -0.6842405200004578}, {"head": 4, "memory_to_active": 1.903978705406189, "contribution_to_active": 0.05361895635724068, "mix_mean": 0.01049063727259636, "mix_low": 0.7584266662597656, "mix_high": 0.0, "read_cosine": 0.3131066858768463, "beta_mean": 0.148659348487854, "nonfinite": 0.0, "weighted_memory_to_active": 0.05496165156364441, "mix_std": 0.02786533161997795, "beta_std": 0.14695027470588684, "beta_low": 0.0333251953125, "beta_high": 0.0, "orthogonal_to_active": 0.003463811706751585, "evidence_weights": [0.09714218229055405, 0.046602990478277206, 0.05033637955784798], "agreement_weight": 0.042432621121406555, "orthogonal_weight": -0.06869769841432571}, {"head": 5, "memory_to_active": 1.7676122188568115, "contribution_to_active": 0.05957874655723572, "mix_mean": 0.016983311623334885, "mix_low": 0.608367919921875, "mix_high": 0.0, "read_cosine": 0.24222877621650696, "beta_mean": 0.04792077839374542, "nonfinite": 0.0, "weighted_memory_to_active": 0.07483578473329544, "mix_std": 0.03296932578086853, "beta_std": 0.055885083973407745, "beta_low": 0.148193359375, "beta_high": 0.0, "orthogonal_to_active": 0.01450316421687603, "evidence_weights": [0.07194258272647858, 0.06759890913963318, -0.02288111299276352], "agreement_weight": 0.01690342091023922, "orthogonal_weight": -0.24256692826747894}]}, {"layer": 8, "memory_to_active": 0.6460275650024414, "contribution_to_active": 0.4581581950187683, "mix_mean": 0.34493598341941833, "mix_low": 0.08674319833517075, "mix_high": 0.0007936160545796156, "read_cosine": 0.523490309715271, "beta_mean": 0.04552968591451645, "nonfinite": 0.0, "weighted_memory_to_active": 0.1444067806005478, "mix_std": 0.22473697364330292, "beta_std": 0.04556693136692047, "beta_low": 0.4311930537223816, "beta_high": 0.0, "orthogonal_to_active": 0.05367299169301987, "output_influence": 0.34095531702041626, "residual_influence": 0.0935918539762497, "state_active": 0.05351390317082405, "state_memory": 0.0147800138220191, "state_peak": 30.957244873046875, "growth_active": 1.0819278955459595, "growth_memory": 1.414547324180603, "heads": [{"head": 0, "memory_to_active": 0.0523919016122818, "contribution_to_active": 0.5014718770980835, "mix_mean": 0.5289954543113708, "mix_low": 0.0001811981201171875, "mix_high": 0.00030612945556640625, "read_cosine": 0.6984388828277588, "beta_mean": 0.003313462482765317, "nonfinite": 0.0, "weighted_memory_to_active": 0.03001510724425316, "mix_std": 0.24204671382904053, "beta_std": 0.007411427795886993, "beta_low": 0.9197998046875, "beta_high": 0.0, "orthogonal_to_active": 0.016096869483590126, "evidence_weights": [-0.21004930138587952, -0.3459286689758301, 0.7082764506340027], "agreement_weight": 0.06618992984294891, "orthogonal_weight": -0.78570955991745}, {"head": 1, "memory_to_active": 0.5136805772781372, "contribution_to_active": 0.37061628699302673, "mix_mean": 0.3569927215576172, "mix_low": 0.0043010711669921875, "mix_high": 0.00038051605224609375, "read_cosine": 0.462485134601593, "beta_mean": 0.03432236611843109, "nonfinite": 0.0, "weighted_memory_to_active": 0.2155260443687439, "mix_std": 0.24089325964450836, "beta_std": 0.03088689222931862, "beta_low": 0.12078857421875, "beta_high": 0.0, "orthogonal_to_active": 0.059911273419857025, "evidence_weights": [-0.17312677204608917, -0.19946889579296112, 0.39736631512641907], "agreement_weight": 0.06288603693246841, "orthogonal_weight": -0.32450467348098755}, {"head": 2, "memory_to_active": 0.09028968214988708, "contribution_to_active": 0.5299009680747986, "mix_mean": 0.47627565264701843, "mix_low": 0.0009946823120117188, "mix_high": 0.001064300537109375, "read_cosine": 0.41765087842941284, "beta_mean": 0.0013385855127125978, "nonfinite": 0.0, "weighted_memory_to_active": 0.05459727719426155, "mix_std": 0.2578091621398926, "beta_std": 0.00251195952296257, "beta_low": 0.98834228515625, "beta_high": 0.0, "orthogonal_to_active": 0.03913316875696182, "evidence_weights": [-0.19786573946475983, -0.29728108644485474, 0.5050076842308044], "agreement_weight": 0.07310232520103455, "orthogonal_weight": -0.8435969948768616}, {"head": 3, "memory_to_active": 0.25049030780792236, "contribution_to_active": 0.40238863229751587, "mix_mean": 0.36194270849227905, "mix_low": 0.0043182373046875, "mix_high": 2.47955322265625e-05, "read_cosine": 0.725043535232544, "beta_mean": 0.018426377326250076, "nonfinite": 0.0, "weighted_memory_to_active": 0.1303328275680542, "mix_std": 0.23556861281394958, "beta_std": 0.01846114546060562, "beta_low": 0.31756591796875, "beta_high": 0.0, "orthogonal_to_active": 0.019359655678272247, "evidence_weights": [-0.11376020312309265, -0.22576981782913208, 0.7223238348960876], "agreement_weight": 0.08000917732715607, "orthogonal_weight": -0.24904906749725342}, {"head": 4, "memory_to_active": 2.7110438346862793, "contribution_to_active": 0.2681492567062378, "mix_mean": 0.027254335582256317, "mix_low": 0.4159564971923828, "mix_high": 0.0, "read_cosine": 0.3251754641532898, "beta_mean": 0.13018110394477844, "nonfinite": 0.0, "weighted_memory_to_active": 0.26134487986564636, "mix_std": 0.04412109777331352, "beta_std": 0.09379179775714874, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.010404093191027641, "evidence_weights": [0.030965644866228104, -0.0034580868668854237, -0.13748589158058167], "agreement_weight": 0.007421595044434071, "orthogonal_weight": 0.04224296659231186}, {"head": 5, "memory_to_active": 0.2582687437534332, "contribution_to_active": 0.6764219403266907, "mix_mean": 0.31815487146377563, "mix_low": 0.09470748901367188, "mix_high": 0.0029859542846679688, "read_cosine": 0.512147843837738, "beta_mean": 0.08559620380401611, "nonfinite": 0.0, "weighted_memory_to_active": 0.17462453246116638, "mix_std": 0.32798293232917786, "beta_std": 0.12033838033676147, "beta_low": 0.2396240234375, "beta_high": 0.0, "orthogonal_to_active": 0.17713285982608795, "evidence_weights": [-0.06949952989816666, -0.32362261414527893, -0.4489659368991852], "agreement_weight": 0.018545983359217644, "orthogonal_weight": 1.216500163078308}]}, {"layer": 9, "memory_to_active": 1.1998536586761475, "contribution_to_active": 0.5879796147346497, "mix_mean": 0.24383395910263062, "mix_low": 0.1773131787776947, "mix_high": 0.0014168422203511, "read_cosine": 0.37320077419281006, "beta_mean": 0.08917029201984406, "nonfinite": 0.0, "weighted_memory_to_active": 0.36822426319122314, "mix_std": 0.2111087292432785, "beta_std": 0.050343044102191925, "beta_low": 0.4990132749080658, "beta_high": 0.0, "orthogonal_to_active": 0.028986763209104538, "output_influence": 0.3930150270462036, "residual_influence": 0.09808701276779175, "state_active": 0.04310539364814758, "state_memory": 0.024415025487542152, "state_peak": 34.15913009643555, "growth_active": 1.2300461530685425, "growth_memory": 1.428863525390625, "heads": [{"head": 0, "memory_to_active": 0.007708301767706871, "contribution_to_active": 0.4602569341659546, "mix_mean": 0.5021029114723206, "mix_low": 0.0031595230102539062, "mix_high": 0.005871772766113281, "read_cosine": 0.23725274205207825, "beta_mean": 2.4388311430811882e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004240445792675018, "mix_std": 0.29607275128364563, "beta_std": 9.151679114438593e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0031354029197245836, "evidence_weights": [-0.26043930649757385, -0.2548525631427765, 0.9301871657371521], "agreement_weight": 0.06216242164373398, "orthogonal_weight": -0.768514096736908}, {"head": 1, "memory_to_active": 0.041410405188798904, "contribution_to_active": 0.4884122610092163, "mix_mean": 0.34071534872055054, "mix_low": 0.01568317413330078, "mix_high": 0.0010423660278320312, "read_cosine": 0.40313130617141724, "beta_mean": 0.00012359094398561865, "nonfinite": 0.0, "weighted_memory_to_active": 0.020369892939925194, "mix_std": 0.2778680920600891, "beta_std": 0.0002753654553089291, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009578571654856205, "evidence_weights": [-0.19255925714969635, -0.20518863201141357, 0.8469867706298828], "agreement_weight": 0.2991640269756317, "orthogonal_weight": -0.5507389307022095}, {"head": 2, "memory_to_active": 0.25026047229766846, "contribution_to_active": 0.5274523496627808, "mix_mean": 0.4060637950897217, "mix_low": 0.008595466613769531, "mix_high": 0.0013952255249023438, "read_cosine": 0.43754151463508606, "beta_mean": 0.0012822799617424607, "nonfinite": 0.0, "weighted_memory_to_active": 0.14200907945632935, "mix_std": 0.2889217734336853, "beta_std": 0.003887255908921361, "beta_low": 0.987060546875, "beta_high": 0.0, "orthogonal_to_active": 0.038005441427230835, "evidence_weights": [-0.15236876904964447, -0.17288260161876678, 0.415569007396698], "agreement_weight": 0.18032588064670563, "orthogonal_weight": -0.34199443459510803}, {"head": 3, "memory_to_active": 1.8530755043029785, "contribution_to_active": 0.18955643475055695, "mix_mean": 0.0375218465924263, "mix_low": 0.4221792221069336, "mix_high": 8.296966552734375e-05, "read_cosine": 0.4450390338897705, "beta_mean": 0.09793345630168915, "nonfinite": 0.0, "weighted_memory_to_active": 0.16053399443626404, "mix_std": 0.09195329248905182, "beta_std": 0.06561256945133209, "beta_low": 0.00616455078125, "beta_high": 0.0, "orthogonal_to_active": 0.027973828837275505, "evidence_weights": [0.04695067182183266, 0.03488098829984665, -0.1648416966199875], "agreement_weight": 0.047902170568704605, "orthogonal_weight": 0.18254047632217407}, {"head": 4, "memory_to_active": 3.1417317390441895, "contribution_to_active": 1.2306114435195923, "mix_mean": 0.11533379554748535, "mix_low": 0.209625244140625, "mix_high": 0.0001068115234375, "read_cosine": 0.2873295545578003, "beta_mean": 0.2651894688606262, "nonfinite": 0.0, "weighted_memory_to_active": 1.1391029357910156, "mix_std": 0.18635061383247375, "beta_std": 0.12374161183834076, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019158325158059597, "evidence_weights": [-0.04433996230363846, -0.1273583322763443, 0.05502575263381004], "agreement_weight": -0.045776404440402985, "orthogonal_weight": 0.0016607792349532247}, {"head": 5, "memory_to_active": 1.9049354791641235, "contribution_to_active": 0.6315882205963135, "mix_mean": 0.0612659715116024, "mix_low": 0.4046363830566406, "mix_high": 1.9073486328125e-06, "read_cosine": 0.428910493850708, "beta_mean": 0.17046859860420227, "nonfinite": 0.0, "weighted_memory_to_active": 0.7430893778800964, "mix_std": 0.1254858672618866, "beta_std": 0.10844992846250534, "beta_low": 0.0008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.09331150352954865, "evidence_weights": [0.05036204308271408, -0.04285035654902458, 0.08337943255901337], "agreement_weight": 0.016756433993577957, "orthogonal_weight": 0.17844684422016144}]}, {"layer": 10, "memory_to_active": 0.17899729311466217, "contribution_to_active": 0.4410315454006195, "mix_mean": 0.4190128445625305, "mix_low": 0.00964323803782463, "mix_high": 0.006493250839412212, "read_cosine": 0.3909434676170349, "beta_mean": 0.04738643392920494, "nonfinite": 0.0, "weighted_memory_to_active": 0.08184072375297546, "mix_std": 0.26656627655029297, "beta_std": 0.05163612961769104, "beta_low": 0.63916015625, "beta_high": 0.0, "orthogonal_to_active": 0.04017406329512596, "output_influence": 0.38007986545562744, "residual_influence": 0.11882282793521881, "state_active": 0.05930141359567642, "state_memory": 0.01436670683324337, "state_peak": 106.39469146728516, "growth_active": 1.0863890647888184, "growth_memory": 1.144114375114441, "heads": [{"head": 0, "memory_to_active": 0.3651142716407776, "contribution_to_active": 0.7434203624725342, "mix_mean": 0.6271005868911743, "mix_low": 0.012195587158203125, "mix_high": 0.036408424377441406, "read_cosine": 0.2156720906496048, "beta_mean": 0.21026375889778137, "nonfinite": 0.0, "weighted_memory_to_active": 0.2612649202346802, "mix_std": 0.34751683473587036, "beta_std": 0.22785930335521698, "beta_low": 0.1121826171875, "beta_high": 0.0, "orthogonal_to_active": 0.13979074358940125, "evidence_weights": [-0.1702016144990921, -0.34515514969825745, -0.32889899611473083], "agreement_weight": 0.018870048224925995, "orthogonal_weight": 0.5923280119895935}, {"head": 1, "memory_to_active": 0.0059698778204619884, "contribution_to_active": 0.5078687071800232, "mix_mean": 0.4910729229450226, "mix_low": 0.0016155242919921875, "mix_high": 0.0018815994262695312, "read_cosine": 0.24957317113876343, "beta_mean": 3.239406942157075e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0030807654839009047, "mix_std": 0.2650965452194214, "beta_std": 6.85286067891866e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025615603663027287, "evidence_weights": [-0.2615968883037567, -0.18684640526771545, 0.323972225189209], "agreement_weight": 0.16664384305477142, "orthogonal_weight": -0.9220690727233887}, {"head": 2, "memory_to_active": 0.2591305673122406, "contribution_to_active": 0.374592125415802, "mix_mean": 0.33287501335144043, "mix_low": 0.0062255859375, "mix_high": 9.250640869140625e-05, "read_cosine": 0.0682608112692833, "beta_mean": 0.0035832575522363186, "nonfinite": 0.0, "weighted_memory_to_active": 0.08755431324243546, "mix_std": 0.26419544219970703, "beta_std": 0.013064062222838402, "beta_low": 0.9290771484375, "beta_high": 0.0, "orthogonal_to_active": 0.027310747653245926, "evidence_weights": [-0.12727099657058716, -0.18758465349674225, 0.4823622405529022], "agreement_weight": 0.02951911836862564, "orthogonal_weight": -0.31019067764282227}, {"head": 3, "memory_to_active": 0.2862779200077057, "contribution_to_active": 0.14429953694343567, "mix_mean": 0.29147249460220337, "mix_low": 0.03220176696777344, "mix_high": 0.0004949569702148438, "read_cosine": 0.8781689405441284, "beta_mean": 0.0629970133304596, "nonfinite": 0.0, "weighted_memory_to_active": 0.05807178467512131, "mix_std": 0.24701669812202454, "beta_std": 0.058365169912576675, "beta_low": 0.0047607421875, "beta_high": 0.0, "orthogonal_to_active": 0.05160578712821007, "evidence_weights": [-0.12735654413700104, -0.1464703530073166, 0.21912603080272675], "agreement_weight": 0.0020175743848085403, "orthogonal_weight": 1.2932841777801514}, {"head": 4, "memory_to_active": 0.006971114315092564, "contribution_to_active": 0.43825089931488037, "mix_mean": 0.42199814319610596, "mix_low": 0.0014820098876953125, "mix_high": 7.43865966796875e-05, "read_cosine": 0.0895872637629509, "beta_mean": 3.795917291427031e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0025038227904587984, "mix_std": 0.2524564862251282, "beta_std": 0.00044425312080420554, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0006204204401001334, "evidence_weights": [-0.18082566559314728, -0.20042411983013153, 0.37735915184020996], "agreement_weight": 0.02704969421029091, "orthogonal_weight": -0.25913137197494507}, {"head": 5, "memory_to_active": 0.1505199670791626, "contribution_to_active": 0.4377574324607849, "mix_mean": 0.34955769777297974, "mix_low": 0.004138946533203125, "mix_high": 7.62939453125e-06, "read_cosine": 0.8443982601165771, "beta_mean": 0.007404227741062641, "nonfinite": 0.0, "weighted_memory_to_active": 0.07856873422861099, "mix_std": 0.22311560809612274, "beta_std": 0.010015458799898624, "beta_low": 0.78900146484375, "beta_high": 0.0, "orthogonal_to_active": 0.019155116751790047, "evidence_weights": [-0.17075835168361664, -0.1679704636335373, 0.2814916968345642], "agreement_weight": 0.09184250980615616, "orthogonal_weight": -0.565514862537384}]}, {"layer": 11, "memory_to_active": 0.22538872063159943, "contribution_to_active": 0.4663320481777191, "mix_mean": 0.3939821124076843, "mix_low": 0.04560788720846176, "mix_high": 0.021701335906982422, "read_cosine": 0.5033982396125793, "beta_mean": 0.04066363722085953, "nonfinite": 0.0, "weighted_memory_to_active": 0.08213615417480469, "mix_std": 0.3033979833126068, "beta_std": 0.056377656757831573, "beta_low": 0.4959615468978882, "beta_high": 0.0, "orthogonal_to_active": 0.04576751962304115, "output_influence": 0.47028419375419617, "residual_influence": 0.1232009083032608, "state_active": 0.11500176787376404, "state_memory": 0.010830244049429893, "state_peak": 30.40044403076172, "growth_active": 1.1806464195251465, "growth_memory": 1.4624885320663452, "heads": [{"head": 0, "memory_to_active": 0.020133651793003082, "contribution_to_active": 0.546699047088623, "mix_mean": 0.5464649200439453, "mix_low": 0.0018739700317382812, "mix_high": 0.0071659088134765625, "read_cosine": 0.4025239050388336, "beta_mean": 0.0001397300948156044, "nonfinite": 0.0, "weighted_memory_to_active": 0.012815490365028381, "mix_std": 0.2772247791290283, "beta_std": 0.0035951663739979267, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.007331796456128359, "evidence_weights": [-0.1631045937538147, -0.33334803581237793, 0.5663250088691711], "agreement_weight": 0.1459885835647583, "orthogonal_weight": -0.6843690872192383}, {"head": 1, "memory_to_active": 0.464608758687973, "contribution_to_active": 0.23309817910194397, "mix_mean": 0.25338053703308105, "mix_low": 0.029807090759277344, "mix_high": 8.296966552734375e-05, "read_cosine": 0.624843955039978, "beta_mean": 0.04101340472698212, "nonfinite": 0.0, "weighted_memory_to_active": 0.16749367117881775, "mix_std": 0.23914073407649994, "beta_std": 0.07124806940555573, "beta_low": 0.32391357421875, "beta_high": 0.0, "orthogonal_to_active": 0.024236731231212616, "evidence_weights": [-0.15748153626918793, -0.1180618479847908, 0.16362307965755463], "agreement_weight": 0.09149238467216492, "orthogonal_weight": -0.20339027047157288}, {"head": 2, "memory_to_active": 0.049831584095954895, "contribution_to_active": 0.8624142408370972, "mix_mean": 0.4835657775402069, "mix_low": 0.050830841064453125, "mix_high": 0.07085227966308594, "read_cosine": 0.3137964606285095, "beta_mean": 0.0814988985657692, "nonfinite": 0.0, "weighted_memory_to_active": 0.040980204939842224, "mix_std": 0.40526506304740906, "beta_std": 0.10840603709220886, "beta_low": 0.4075927734375, "beta_high": 0.0, "orthogonal_to_active": 0.10411493480205536, "evidence_weights": [-0.11204829812049866, -0.4368229806423187, -0.8066028356552124], "agreement_weight": -0.008946625515818596, "orthogonal_weight": 2.744204521179199}, {"head": 3, "memory_to_active": 0.026865826919674873, "contribution_to_active": 0.4946061670780182, "mix_mean": 0.4337804317474365, "mix_low": 0.13807392120361328, "mix_high": 0.05129241943359375, "read_cosine": 0.5302532911300659, "beta_mean": 0.07244584709405899, "nonfinite": 0.0, "weighted_memory_to_active": 0.014937077648937702, "mix_std": 0.4193935990333557, "beta_std": 0.10365359485149384, "beta_low": 0.428955078125, "beta_high": 0.0, "orthogonal_to_active": 0.0378568135201931, "evidence_weights": [-0.09958940744400024, -0.36580419540405273, -0.3135035037994385], "agreement_weight": 0.020708270370960236, "orthogonal_weight": 2.961101531982422}, {"head": 4, "memory_to_active": 0.6328600645065308, "contribution_to_active": 0.2916399836540222, "mix_mean": 0.21412625908851624, "mix_low": 0.052544593811035156, "mix_high": 6.389617919921875e-05, "read_cosine": 0.23651528358459473, "beta_mean": 0.03849644213914871, "nonfinite": 0.0, "weighted_memory_to_active": 0.18444882333278656, "mix_std": 0.2173345983028412, "beta_std": 0.03616851940751076, "beta_low": 0.1270751953125, "beta_high": 0.0, "orthogonal_to_active": 0.0818738266825676, "evidence_weights": [-0.11330367624759674, -0.08898069709539413, 1.1502565145492554], "agreement_weight": -0.11196428537368774, "orthogonal_weight": -0.4780004918575287}, {"head": 5, "memory_to_active": 0.15803241729736328, "contribution_to_active": 0.369534432888031, "mix_mean": 0.43257468938827515, "mix_low": 0.0005168914794921875, "mix_high": 0.0007505416870117188, "read_cosine": 0.9124564528465271, "beta_mean": 0.010387481190264225, "nonfinite": 0.0, "weighted_memory_to_active": 0.07214158773422241, "mix_std": 0.26202911138534546, "beta_std": 0.015194541774690151, "beta_low": 0.68878173828125, "beta_high": 0.0, "orthogonal_to_active": 0.01919100433588028, "evidence_weights": [-0.1906440556049347, -0.2408793568611145, 0.43317726254463196], "agreement_weight": 0.04103400185704231, "orthogonal_weight": -0.5943576097488403}]}, {"layer": 12, "memory_to_active": 0.4046003222465515, "contribution_to_active": 0.5007542967796326, "mix_mean": 0.40688952803611755, "mix_low": 0.0313924178481102, "mix_high": 0.0021996498107910156, "read_cosine": 0.28928664326667786, "beta_mean": 0.018076542764902115, "nonfinite": 0.0, "weighted_memory_to_active": 0.1865125298500061, "mix_std": 0.2464355081319809, "beta_std": 0.018013663589954376, "beta_low": 0.7906189560890198, "beta_high": 0.0, "orthogonal_to_active": 0.06717219948768616, "output_influence": 0.3712632358074188, "residual_influence": 0.10167264938354492, "state_active": 0.04613562673330307, "state_memory": 0.012121303007006645, "state_peak": 24.786373138427734, "growth_active": 1.1102442741394043, "growth_memory": 1.2539336681365967, "heads": [{"head": 0, "memory_to_active": 0.24192535877227783, "contribution_to_active": 0.3647295832633972, "mix_mean": 0.332852840423584, "mix_low": 0.013442039489746094, "mix_high": 4.863739013671875e-05, "read_cosine": 0.2947588562965393, "beta_mean": 0.0035574110224843025, "nonfinite": 0.0, "weighted_memory_to_active": 0.09385453164577484, "mix_std": 0.24856649339199066, "beta_std": 0.013094818219542503, "beta_low": 0.92437744140625, "beta_high": 0.0, "orthogonal_to_active": 0.04188307374715805, "evidence_weights": [-0.14751273393630981, -0.21181398630142212, 0.24558821320533752], "agreement_weight": 0.10484393686056137, "orthogonal_weight": -0.5187711119651794}, {"head": 1, "memory_to_active": 0.1450205147266388, "contribution_to_active": 0.4408910870552063, "mix_mean": 0.43968015909194946, "mix_low": 0.0006113052368164062, "mix_high": 0.00014019012451171875, "read_cosine": 0.576457679271698, "beta_mean": 0.0015343017876148224, "nonfinite": 0.0, "weighted_memory_to_active": 0.06669392436742783, "mix_std": 0.26036009192466736, "beta_std": 0.004499527625739574, "beta_low": 0.9783935546875, "beta_high": 0.0, "orthogonal_to_active": 0.047653406858444214, "evidence_weights": [-0.18752321600914001, -0.23984850943088531, 0.7241688966751099], "agreement_weight": 0.09520483016967773, "orthogonal_weight": -0.8878840208053589}, {"head": 2, "memory_to_active": 0.6633808612823486, "contribution_to_active": 0.42519131302833557, "mix_mean": 0.2968887686729431, "mix_low": 0.029273033142089844, "mix_high": 0.0001697540283203125, "read_cosine": 0.43543460965156555, "beta_mean": 0.09670942276716232, "nonfinite": 0.0, "weighted_memory_to_active": 0.2593528926372528, "mix_std": 0.2402869462966919, "beta_std": 0.06837494671344757, "beta_low": 0.0013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.17363210022449493, "evidence_weights": [-0.22975273430347443, -0.04474814236164093, -0.04094070941209793], "agreement_weight": 0.032409895211458206, "orthogonal_weight": 0.8219321966171265}, {"head": 3, "memory_to_active": 0.02545444294810295, "contribution_to_active": 0.6829378008842468, "mix_mean": 0.6551705002784729, "mix_low": 0.000629425048828125, "mix_high": 0.010279655456542969, "read_cosine": 0.047344233840703964, "beta_mean": 0.00022245061700232327, "nonfinite": 0.0, "weighted_memory_to_active": 0.017578039318323135, "mix_std": 0.2524084448814392, "beta_std": 0.0007480279891751707, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.015489339828491211, "evidence_weights": [-0.3061293661594391, -0.31409403681755066, 0.8560610413551331], "agreement_weight": 0.20617598295211792, "orthogonal_weight": -0.9036742448806763}, {"head": 4, "memory_to_active": 1.3043527603149414, "contribution_to_active": 0.6589309573173523, "mix_mean": 0.15822452306747437, "mix_low": 0.13781356811523438, "mix_high": 0.0011377334594726562, "read_cosine": 0.0005293805152177811, "beta_mean": 0.00591433048248291, "nonfinite": 0.0, "weighted_memory_to_active": 0.6574114561080933, "mix_std": 0.2021225243806839, "beta_std": 0.01893114298582077, "beta_low": 0.84783935546875, "beta_high": 0.0, "orthogonal_to_active": 0.10609271377325058, "evidence_weights": [0.009767971001565456, -0.14315728843212128, -0.018925275653600693], "agreement_weight": -0.003443973371759057, "orthogonal_weight": -0.1759137213230133}, {"head": 5, "memory_to_active": 0.0474679172039032, "contribution_to_active": 0.43184512853622437, "mix_mean": 0.5585202574729919, "mix_low": 0.006585121154785156, "mix_high": 0.0014219284057617188, "read_cosine": 0.38119491934776306, "beta_mean": 0.0005213387776166201, "nonfinite": 0.0, "weighted_memory_to_active": 0.024184228852391243, "mix_std": 0.2748684287071228, "beta_std": 0.0024335207417607307, "beta_low": 0.99267578125, "beta_high": 0.0, "orthogonal_to_active": 0.018282556906342506, "evidence_weights": [-0.26553019881248474, -0.286872923374176, 0.7894610166549683], "agreement_weight": 0.0943262055516243, "orthogonal_weight": -0.8298505544662476}]}, {"layer": 13, "memory_to_active": 0.20009250938892365, "contribution_to_active": 0.46739906072616577, "mix_mean": 0.46049821376800537, "mix_low": 0.005679925438016653, "mix_high": 0.0015988349914550781, "read_cosine": 0.4113072454929352, "beta_mean": 0.018307678401470184, "nonfinite": 0.0, "weighted_memory_to_active": 0.08964227885007858, "mix_std": 0.2568613886833191, "beta_std": 0.018461093306541443, "beta_low": 0.7574666738510132, "beta_high": 0.0, "orthogonal_to_active": 0.07563832402229309, "output_influence": 0.40467554330825806, "residual_influence": 0.13227035105228424, "state_active": 0.08565422147512436, "state_memory": 0.03469719737768173, "state_peak": 92.5177230834961, "growth_active": 1.0112577676773071, "growth_memory": 1.0893075466156006, "heads": [{"head": 0, "memory_to_active": 0.10108141601085663, "contribution_to_active": 0.49096471071243286, "mix_mean": 0.4740442931652069, "mix_low": 0.0010776519775390625, "mix_high": 0.0018215179443359375, "read_cosine": 0.35661426186561584, "beta_mean": 0.0005459614330902696, "nonfinite": 0.0, "weighted_memory_to_active": 0.05186540260910988, "mix_std": 0.26666444540023804, "beta_std": 0.001211046939715743, "beta_low": 0.99798583984375, "beta_high": 0.0, "orthogonal_to_active": 0.04941834881901741, "evidence_weights": [-0.20417501032352448, -0.234816774725914, 0.7458466291427612], "agreement_weight": 0.04490124061703682, "orthogonal_weight": -1.0498358011245728}, {"head": 1, "memory_to_active": 0.06395576894283295, "contribution_to_active": 0.5218273401260376, "mix_mean": 0.4554334282875061, "mix_low": 0.0029230117797851562, "mix_high": 8.869171142578125e-05, "read_cosine": 0.24735210835933685, "beta_mean": 0.0003190395073033869, "nonfinite": 0.0, "weighted_memory_to_active": 0.03334987908601761, "mix_std": 0.2645873725414276, "beta_std": 0.0009346621227450669, "beta_low": 0.99835205078125, "beta_high": 0.0, "orthogonal_to_active": 0.031256359070539474, "evidence_weights": [-0.18876054883003235, -0.2438027262687683, 0.7335991263389587], "agreement_weight": 0.1264069676399231, "orthogonal_weight": -0.9908010959625244}, {"head": 2, "memory_to_active": 0.17986899614334106, "contribution_to_active": 0.34677889943122864, "mix_mean": 0.3377310633659363, "mix_low": 0.0025415420532226562, "mix_high": 4.100799560546875e-05, "read_cosine": 0.698502779006958, "beta_mean": 0.020613618195056915, "nonfinite": 0.0, "weighted_memory_to_active": 0.07066170871257782, "mix_std": 0.21441102027893066, "beta_std": 0.04425825923681259, "beta_low": 0.61737060546875, "beta_high": 0.0, "orthogonal_to_active": 0.030321713536977768, "evidence_weights": [-0.21953675150871277, -0.15420596301555634, 0.48683902621269226], "agreement_weight": 0.04256568104028702, "orthogonal_weight": -0.7119709253311157}, {"head": 3, "memory_to_active": 0.16557714343070984, "contribution_to_active": 0.5729649066925049, "mix_mean": 0.5767979025840759, "mix_low": 0.00891876220703125, "mix_high": 0.005381584167480469, "read_cosine": 0.017483126372098923, "beta_mean": 0.0010127455461770296, "nonfinite": 0.0, "weighted_memory_to_active": 0.11962559819221497, "mix_std": 0.2733283042907715, "beta_std": 0.003955229185521603, "beta_low": 0.98883056640625, "beta_high": 0.0, "orthogonal_to_active": 0.12040852010250092, "evidence_weights": [-0.35278424620628357, -0.2320737987756729, 1.7238320112228394], "agreement_weight": 1.664100170135498, "orthogonal_weight": -1.046340823173523}, {"head": 4, "memory_to_active": 0.4570268988609314, "contribution_to_active": 0.2892551124095917, "mix_mean": 0.4069975018501282, "mix_low": 0.01232147216796875, "mix_high": 0.0001697540283203125, "read_cosine": 0.867911696434021, "beta_mean": 0.08479616045951843, "nonfinite": 0.0, "weighted_memory_to_active": 0.13466458022594452, "mix_std": 0.23734967410564423, "beta_std": 0.05287721008062363, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.138694629073143, "evidence_weights": [-0.22484347224235535, -0.24240742623806, 0.48556479811668396], "agreement_weight": -0.013945543207228184, "orthogonal_weight": 1.4886951446533203}, {"head": 5, "memory_to_active": 0.23304475843906403, "contribution_to_active": 0.582603394985199, "mix_mean": 0.5119850635528564, "mix_low": 0.006297111511230469, "mix_high": 0.0020904541015625, "read_cosine": 0.27997952699661255, "beta_mean": 0.0025585440453141928, "nonfinite": 0.0, "weighted_memory_to_active": 0.12768647074699402, "mix_std": 0.28482744097709656, "beta_std": 0.007530140224844217, "beta_low": 0.9422607421875, "beta_high": 0.0, "orthogonal_to_active": 0.08373036980628967, "evidence_weights": [-0.2140054851770401, -0.2585374414920807, 0.4601759612560272], "agreement_weight": 0.07363572716712952, "orthogonal_weight": -0.7344276309013367}]}, {"layer": 14, "memory_to_active": 0.2720497250556946, "contribution_to_active": 0.4293396472930908, "mix_mean": 0.3979197144508362, "mix_low": 0.02290996164083481, "mix_high": 0.0034546852111816406, "read_cosine": 0.6275792717933655, "beta_mean": 0.07203938066959381, "nonfinite": 0.0, "weighted_memory_to_active": 0.12954497337341309, "mix_std": 0.2515505850315094, "beta_std": 0.06522126495838165, "beta_low": 0.4266154170036316, "beta_high": 0.0, "orthogonal_to_active": 0.04089481756091118, "output_influence": 0.2779645621776581, "residual_influence": 0.1077699363231659, "state_active": 0.07671596109867096, "state_memory": 0.016624726355075836, "state_peak": 29.217357635498047, "growth_active": 1.1683554649353027, "growth_memory": 1.3799668550491333, "heads": [{"head": 0, "memory_to_active": 0.3172137141227722, "contribution_to_active": 0.24099832773208618, "mix_mean": 0.24139311909675598, "mix_low": 0.014545440673828125, "mix_high": 9.5367431640625e-06, "read_cosine": 0.7368988394737244, "beta_mean": 0.06955373287200928, "nonfinite": 0.0, "weighted_memory_to_active": 0.09920746088027954, "mix_std": 0.21481706202030182, "beta_std": 0.07441176474094391, "beta_low": 0.06109619140625, "beta_high": 0.0, "orthogonal_to_active": 0.02762937732040882, "evidence_weights": [-0.06981141865253448, -0.09885277599096298, 0.7404080033302307], "agreement_weight": 0.044097401201725006, "orthogonal_weight": 0.42770662903785706}, {"head": 1, "memory_to_active": 0.49541205167770386, "contribution_to_active": 0.429756760597229, "mix_mean": 0.28493010997772217, "mix_low": 0.09727954864501953, "mix_high": 0.00043201446533203125, "read_cosine": 0.666243314743042, "beta_mean": 0.298048734664917, "nonfinite": 0.0, "weighted_memory_to_active": 0.258174866437912, "mix_std": 0.2834489345550537, "beta_std": 0.2575298547744751, "beta_low": 0.0748291015625, "beta_high": 0.0, "orthogonal_to_active": 0.026415258646011353, "evidence_weights": [0.0021713420283049345, -0.2770565152168274, -0.09459789842367172], "agreement_weight": 0.05817669257521629, "orthogonal_weight": 0.17466798424720764}, {"head": 2, "memory_to_active": 0.2295115888118744, "contribution_to_active": 0.4954688251018524, "mix_mean": 0.4886598587036133, "mix_low": 0.00536346435546875, "mix_high": 0.0018663406372070312, "read_cosine": 0.3989100456237793, "beta_mean": 0.007604035548865795, "nonfinite": 0.0, "weighted_memory_to_active": 0.13107195496559143, "mix_std": 0.27250978350639343, "beta_std": 0.009256716817617416, "beta_low": 0.76885986328125, "beta_high": 0.0, "orthogonal_to_active": 0.09379176050424576, "evidence_weights": [-0.18716008961200714, -0.24889163672924042, 0.7190281748771667], "agreement_weight": 0.07742054015398026, "orthogonal_weight": -0.8127369284629822}, {"head": 3, "memory_to_active": 0.06756524741649628, "contribution_to_active": 0.6522843837738037, "mix_mean": 0.6544163227081299, "mix_low": 0.002696990966796875, "mix_high": 0.012427330017089844, "read_cosine": 0.4729292690753937, "beta_mean": 0.009193884208798409, "nonfinite": 0.0, "weighted_memory_to_active": 0.04594672471284866, "mix_std": 0.24567930400371552, "beta_std": 0.00811040960252285, "beta_low": 0.6798095703125, "beta_high": 0.0, "orthogonal_to_active": 0.02256077527999878, "evidence_weights": [-0.3784354031085968, -0.324624627828598, 0.45204728841781616], "agreement_weight": 0.07220973074436188, "orthogonal_weight": -0.6214019656181335}, {"head": 4, "memory_to_active": 0.18941240012645721, "contribution_to_active": 0.47751128673553467, "mix_mean": 0.44443556666374207, "mix_low": 0.004366874694824219, "mix_high": 0.005415916442871094, "read_cosine": 0.7847937941551208, "beta_mean": 0.044926710426807404, "nonfinite": 0.0, "weighted_memory_to_active": 0.10552874952554703, "mix_std": 0.2606387436389923, "beta_std": 0.03169037401676178, "beta_low": 0.02783203125, "beta_high": 0.0, "orthogonal_to_active": 0.02686528116464615, "evidence_weights": [-0.21804513037204742, -0.2660672068595886, 0.340516597032547], "agreement_weight": 0.10970159620046616, "orthogonal_weight": -0.4544961750507355}, {"head": 5, "memory_to_active": 0.33318328857421875, "contribution_to_active": 0.2800183892250061, "mix_mean": 0.2736831605434418, "mix_low": 0.013207435607910156, "mix_high": 0.0005769729614257812, "read_cosine": 0.7057000994682312, "beta_mean": 0.002909185830503702, "nonfinite": 0.0, "weighted_memory_to_active": 0.1373400092124939, "mix_std": 0.23220960795879364, "beta_std": 0.010328449308872223, "beta_low": 0.947265625, "beta_high": 0.0, "orthogonal_to_active": 0.048106446862220764, "evidence_weights": [-0.18598419427871704, -0.141458660364151, 0.1584799587726593], "agreement_weight": 0.09359724819660187, "orthogonal_weight": -0.5750887989997864}]}, {"layer": 15, "memory_to_active": 0.0748603567481041, "contribution_to_active": 0.47109076380729675, "mix_mean": 0.3838198781013489, "mix_low": 0.007185935974121094, "mix_high": 0.0021440188866108656, "read_cosine": 0.6767219305038452, "beta_mean": 0.045375023037195206, "nonfinite": 0.0, "weighted_memory_to_active": 0.0335237979888916, "mix_std": 0.2718365490436554, "beta_std": 0.047245658934116364, "beta_low": 0.7105306386947632, "beta_high": 0.0, "orthogonal_to_active": 0.006264353170990944, "output_influence": 0.3017084002494812, "residual_influence": 0.15617631375789642, "state_active": 0.11740390956401825, "state_memory": 0.00809382926672697, "state_peak": 33.52898025512695, "growth_active": 1.3856593370437622, "growth_memory": 1.6262201070785522, "heads": [{"head": 0, "memory_to_active": 0.0988544151186943, "contribution_to_active": 0.4852237105369568, "mix_mean": 0.33810901641845703, "mix_low": 0.00742340087890625, "mix_high": 0.0023012161254882812, "read_cosine": 0.6558889150619507, "beta_mean": 0.03840111196041107, "nonfinite": 0.0, "weighted_memory_to_active": 0.04588725045323372, "mix_std": 0.25896012783050537, "beta_std": 0.030903639271855354, "beta_low": 0.14532470703125, "beta_high": 0.0, "orthogonal_to_active": 0.014193357899785042, "evidence_weights": [-0.16267409920692444, -0.19280123710632324, 0.34299764037132263], "agreement_weight": 0.08197122812271118, "orthogonal_weight": -0.5210889577865601}, {"head": 1, "memory_to_active": 0.1869233399629593, "contribution_to_active": 0.41031336784362793, "mix_mean": 0.3887479603290558, "mix_low": 0.0056667327880859375, "mix_high": 0.00482177734375, "read_cosine": 0.8459396362304688, "beta_mean": 0.0018169415416195989, "nonfinite": 0.0, "weighted_memory_to_active": 0.07307958602905273, "mix_std": 0.27347469329833984, "beta_std": 0.00401658471673727, "beta_low": 0.9722900390625, "beta_high": 0.0, "orthogonal_to_active": 0.004278651904314756, "evidence_weights": [-0.19183743000030518, -0.19676882028579712, 0.2885418236255646], "agreement_weight": 0.09322239458560944, "orthogonal_weight": -0.08872067183256149}, {"head": 2, "memory_to_active": 0.006867671385407448, "contribution_to_active": 0.42835795879364014, "mix_mean": 0.34616297483444214, "mix_low": 0.011006355285644531, "mix_high": 0.00017070770263671875, "read_cosine": 0.7571430802345276, "beta_mean": 0.0001185911605716683, "nonfinite": 0.0, "weighted_memory_to_active": 0.002656984608620405, "mix_std": 0.26695752143859863, "beta_std": 0.0002925105800386518, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.00038682855665683746, "evidence_weights": [-0.20587602257728577, -0.2716900110244751, 0.2932021915912628], "agreement_weight": 0.047917526215314865, "orthogonal_weight": -0.24297048151493073}, {"head": 3, "memory_to_active": 0.048224225640296936, "contribution_to_active": 0.46924397349357605, "mix_mean": 0.30680325627326965, "mix_low": 0.0029449462890625, "mix_high": 2.288818359375e-05, "read_cosine": 0.6885401010513306, "beta_mean": 0.0007309404900297523, "nonfinite": 0.0, "weighted_memory_to_active": 0.02213948406279087, "mix_std": 0.2424393743276596, "beta_std": 0.0015630465932190418, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.011994019150733948, "evidence_weights": [-0.1816118359565735, -0.22713543474674225, 0.43170642852783203], "agreement_weight": 0.006426352541893721, "orthogonal_weight": -0.8549047708511353}, {"head": 4, "memory_to_active": 0.10296355187892914, "contribution_to_active": 0.5759264230728149, "mix_mean": 0.40929415822029114, "mix_low": 0.010369300842285156, "mix_high": 0.0007505416870117188, "read_cosine": 0.5697157382965088, "beta_mean": 0.23105326294898987, "nonfinite": 0.0, "weighted_memory_to_active": 0.05517595261335373, "mix_std": 0.30797356367111206, "beta_std": 0.24642658233642578, "beta_low": 0.15032958984375, "beta_high": 0.0, "orthogonal_to_active": 0.004913931712508202, "evidence_weights": [-0.24905633926391602, -0.2512491047382355, 0.18474583327770233], "agreement_weight": 0.009174498729407787, "orthogonal_weight": -0.12374871224164963}, {"head": 5, "memory_to_active": 0.005328932777047157, "contribution_to_active": 0.4574790596961975, "mix_mean": 0.5138019323348999, "mix_low": 0.0057048797607421875, "mix_high": 0.0047969818115234375, "read_cosine": 0.5431039929389954, "beta_mean": 0.00012929549848195165, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022035236470401287, "mix_std": 0.28121399879455566, "beta_std": 0.0002715765731409192, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0018193279393017292, "evidence_weights": [-0.3575696647167206, -0.29707226157188416, 0.3821885287761688], "agreement_weight": 0.1431480497121811, "orthogonal_weight": 0.8849583864212036}]}]}, {"step": 18000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9041793409996899, "memory_to_active": 0.9559821486473083, "contribution_to_active": 0.46123337745666504, "mix_mean": 0.31672215461730957, "mix_low": 0.1703178584575653, "mix_high": 0.005940149538218975, "read_cosine": 0.39806339144706726, "beta_mean": 0.09053549915552139, "nonfinite": 0.0, "weighted_memory_to_active": 0.20509032905101776, "mix_std": 0.21160228550434113, "beta_std": 0.07287804037332535, "beta_low": 0.46631181240081787, "beta_high": 1.3987223610456567e-05, "orthogonal_to_active": 0.04232507199048996, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1840171813964844, "normal_token_accuracy": 0.439377099275589, "state_active": 0.05999453738331795, "state_memory": 0.02239951677620411, "state_peak": 108.54845428466797, "growth_active": 1.139553427696228, "growth_memory": 1.3898552656173706, "output_influence": 0.31935346126556396, "residual_influence": 0.10059776157140732, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.248753547668457, "no_memory_loss_delta": 0.16624322533607483, "scrambled_loss_delta": 0.14520087838172913, "memory_only_loss_delta": 2.3812851905822754, "no_evidence_loss_delta": 0.08414968848228455, "no_agreement_loss_delta": -0.008449153043329716, "no_orthogonal_loss_delta": -0.003747769631445408}, {"index": 94, "tokens": 256, "normal_loss": 3.286602258682251, "no_memory_loss_delta": 0.2948053181171417, "scrambled_loss_delta": 0.35469356179237366, "memory_only_loss_delta": 2.378228187561035, "no_evidence_loss_delta": 0.16053572297096252, "no_agreement_loss_delta": 0.02105267532169819, "no_orthogonal_loss_delta": 0.01796768605709076}, {"index": 188, "tokens": 256, "normal_loss": 3.5775604248046875, "no_memory_loss_delta": 0.21568520367145538, "scrambled_loss_delta": 0.11105486750602722, "memory_only_loss_delta": 2.1239707469940186, "no_evidence_loss_delta": 0.1313427835702896, "no_agreement_loss_delta": -0.008534939959645271, "no_orthogonal_loss_delta": 0.012184534221887589}, {"index": 283, "tokens": 131, "normal_loss": 2.087977886199951, "no_memory_loss_delta": 0.12597982585430145, "scrambled_loss_delta": 0.31622806191444397, "memory_only_loss_delta": 2.653822422027588, "no_evidence_loss_delta": 0.055728036910295486, "no_agreement_loss_delta": -0.009485912509262562, "no_orthogonal_loss_delta": 0.010839174501597881}], "agreement_weight_abs_mean": 0.09483402967453003, "agreement_weight_abs_max": 1.7908620834350586, "agreement_weight_mean": 0.0795939639210701, "orthogonal_weight_abs_mean": 0.5160690546035767, "orthogonal_weight_abs_max": 3.1447558403015137, "orthogonal_weight_mean": -0.17111222445964813, "no_memory_loss": 3.3950819969177246, "no_memory_loss_delta": 0.21106471121311188, "no_memory_token_accuracy_gain": 0.025583982467651367, "no_memory_token_win_fraction": 0.5984427332878113, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 19.027833288549363, "scrambled_loss": 3.404071807861328, "scrambled_loss_delta": 0.22005437314510345, "scrambled_token_accuracy_gain": 0.03337041288614273, "scrambled_token_win_fraction": 0.6229143738746643, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.7524836322294, "memory_only_loss": 5.530872344970703, "memory_only_loss_delta": 2.3468551635742188, "memory_only_token_accuracy_gain": 0.23692992329597473, "memory_only_token_win_fraction": 0.8865405917167664, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 90.43304452741378, "no_evidence_loss": 3.2992162704467773, "no_evidence_loss_delta": 0.11519866436719894, "no_evidence_token_accuracy_gain": 0.007786429487168789, "no_evidence_token_win_fraction": 0.5394883155822754, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 10.881092119524485, "no_agreement_loss": 3.183793544769287, "no_agreement_loss_delta": -0.0002236907312180847, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.4371523857116699, "no_agreement_probe_win_fraction": 0.25, "no_agreement_ppl_reduction_pct": -0.022371575185529454, "no_orthogonal_loss": 3.193115472793579, "no_orthogonal_loss_delta": 0.009098409675061703, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5394883155822754, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.9057144390074311, "short_no_memory_loss_delta": 0.18125870823860168, "long_context_memory_gain": 0.029806002974510193, "normal_context_loss_gain": 0.14565525949001312, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 2.9696884155273438, "contribution_to_active": 0.8190103769302368, "mix_mean": 0.09446742385625839, "mix_low": 0.48842352628707886, "mix_high": 0.003498554229736328, "read_cosine": 0.4668083190917969, "beta_mean": 0.27672040462493896, "nonfinite": 0.0, "weighted_memory_to_active": 0.8811370134353638, "mix_std": 0.08593188971281052, "beta_std": 0.16162976622581482, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.05314135178923607, "output_influence": 0.125863716006279, "residual_influence": 0.17695015668869019, "state_active": 0.08032021671533585, "state_memory": 0.0516359843313694, "state_peak": 51.18924331665039, "growth_active": 1.2591254711151123, "growth_memory": 1.287630558013916, "heads": [{"head": 0, "memory_to_active": 2.306016683578491, "contribution_to_active": 0.3519330322742462, "mix_mean": 0.02021126076579094, "mix_low": 0.5508708953857422, "mix_high": 0.0, "read_cosine": 0.28691500425338745, "beta_mean": 0.21712318062782288, "nonfinite": 0.0, "weighted_memory_to_active": 0.4400021731853485, "mix_std": 0.05430161952972412, "beta_std": 0.10747673362493515, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.036511145532131195, "evidence_weights": [-0.039626482874155045, -0.07521986216306686, -0.06473502516746521], "agreement_weight": -0.09214247763156891, "orthogonal_weight": -0.09725809097290039}, {"head": 1, "memory_to_active": 4.056994438171387, "contribution_to_active": 1.4897211790084839, "mix_mean": 0.01729646697640419, "mix_low": 0.5823736190795898, "mix_high": 0.0, "read_cosine": 0.49225640296936035, "beta_mean": 0.4206646978855133, "nonfinite": 0.0, "weighted_memory_to_active": 1.8728946447372437, "mix_std": 0.04750892147421837, "beta_std": 0.17396341264247894, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.15893816947937012, "evidence_weights": [0.01980682834982872, -0.04019026830792427, 0.1901702880859375], "agreement_weight": 0.012936028651893139, "orthogonal_weight": -0.20272250473499298}, {"head": 2, "memory_to_active": 2.436190605163574, "contribution_to_active": 0.45273396372795105, "mix_mean": 0.014216096140444279, "mix_low": 0.6194801330566406, "mix_high": 0.0, "read_cosine": 0.3304581046104431, "beta_mean": 0.33252444863319397, "nonfinite": 0.0, "weighted_memory_to_active": 0.5315389633178711, "mix_std": 0.03065057285130024, "beta_std": 0.2134808599948883, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.059303104877471924, "evidence_weights": [0.048089399933815, 0.05189108848571777, 0.005180639214813709], "agreement_weight": -0.0021953443065285683, "orthogonal_weight": -0.1938340812921524}, {"head": 3, "memory_to_active": 5.272896766662598, "contribution_to_active": 2.200148820877075, "mix_mean": 0.04785614833235741, "mix_low": 0.3291025161743164, "mix_high": 0.0, "read_cosine": 0.4266340136528015, "beta_mean": 0.3335929811000824, "nonfinite": 0.0, "weighted_memory_to_active": 2.3295552730560303, "mix_std": 0.08991389721632004, "beta_std": 0.2254667580127716, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04817769676446915, "evidence_weights": [-0.09091132879257202, -0.11792926490306854, -0.06020747125148773], "agreement_weight": -0.014506918378174305, "orthogonal_weight": -0.02696189098060131}, {"head": 4, "memory_to_active": 3.614928722381592, "contribution_to_active": 0.06596560031175613, "mix_mean": 0.006551257334649563, "mix_low": 0.8209114074707031, "mix_high": 0.0, "read_cosine": 0.31037652492523193, "beta_mean": 0.30305296182632446, "nonfinite": 0.0, "weighted_memory_to_active": 0.061415039002895355, "mix_std": 0.012393541634082794, "beta_std": 0.21004459261894226, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007692024111747742, "evidence_weights": [0.08667624741792679, 0.03457694128155708, -0.31217822432518005], "agreement_weight": -0.04059821739792824, "orthogonal_weight": 0.13811683654785156}, {"head": 5, "memory_to_active": 0.13110379874706268, "contribution_to_active": 0.35355907678604126, "mix_mean": 0.46067333221435547, "mix_low": 0.027802467346191406, "mix_high": 0.02099132537841797, "read_cosine": 0.9542097449302673, "beta_mean": 0.05336397886276245, "nonfinite": 0.0, "weighted_memory_to_active": 0.05141593888401985, "mix_std": 0.2808227837085724, "beta_std": 0.03934623673558235, "beta_low": 0.0040283203125, "beta_high": 0.0, "orthogonal_to_active": 0.008225963450968266, "evidence_weights": [-0.28486865758895874, -0.2655021548271179, 0.2572886347770691], "agreement_weight": 0.17739824950695038, "orthogonal_weight": -0.38681066036224365}]}, {"layer": 1, "memory_to_active": 2.2040822505950928, "contribution_to_active": 0.4226831793785095, "mix_mean": 0.12960432469844818, "mix_low": 0.39272356033325195, "mix_high": 0.0006537438021041453, "read_cosine": 0.3529009521007538, "beta_mean": 0.21257802844047546, "nonfinite": 0.0, "weighted_memory_to_active": 0.3451269865036011, "mix_std": 0.11766175180673599, "beta_std": 0.1383836716413498, "beta_low": 0.1704305112361908, "beta_high": 4.069010537932627e-05, "orthogonal_to_active": 0.04009893536567688, "output_influence": 0.14692315459251404, "residual_influence": 0.03478617966175079, "state_active": 0.04268213361501694, "state_memory": 0.04237283393740654, "state_peak": 24.466665267944336, "growth_active": 1.1545579433441162, "growth_memory": 1.5089489221572876, "heads": [{"head": 0, "memory_to_active": 0.06204410642385483, "contribution_to_active": 0.3501919209957123, "mix_mean": 0.3041576147079468, "mix_low": 0.023360252380371094, "mix_high": 7.534027099609375e-05, "read_cosine": -0.03309369832277298, "beta_mean": 0.0003475213597994298, "nonfinite": 0.0, "weighted_memory_to_active": 0.0269579216837883, "mix_std": 0.270811527967453, "beta_std": 0.0007887347019277513, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.018203532323241234, "evidence_weights": [-0.147890105843544, -0.1817660927772522, -0.02787499502301216], "agreement_weight": 0.1458297073841095, "orthogonal_weight": -0.7513658404350281}, {"head": 1, "memory_to_active": 3.2867724895477295, "contribution_to_active": 0.18878750503063202, "mix_mean": 0.021143997088074684, "mix_low": 0.48775196075439453, "mix_high": 0.0, "read_cosine": 0.48176127672195435, "beta_mean": 0.3697526454925537, "nonfinite": 0.0, "weighted_memory_to_active": 0.21065807342529297, "mix_std": 0.04404096305370331, "beta_std": 0.23457714915275574, "beta_low": 0.00030517578125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.008012419566512108, "evidence_weights": [-0.004294734913855791, 0.0033966400660574436, -0.02221810631453991], "agreement_weight": -0.007041829638183117, "orthogonal_weight": 0.0737561360001564}, {"head": 2, "memory_to_active": 0.5507144927978516, "contribution_to_active": 0.432046115398407, "mix_mean": 0.3956058919429779, "mix_low": 0.004082679748535156, "mix_high": 0.0038433074951171875, "read_cosine": 0.8076150417327881, "beta_mean": 0.06761373579502106, "nonfinite": 0.0, "weighted_memory_to_active": 0.4253385365009308, "mix_std": 0.26549142599105835, "beta_std": 0.03442048281431198, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.011561676859855652, "evidence_weights": [-0.2383226603269577, -0.38978514075279236, 0.2260414958000183], "agreement_weight": 0.01559456903487444, "orthogonal_weight": -0.05731315165758133}, {"head": 3, "memory_to_active": 2.9136786460876465, "contribution_to_active": 0.07056619971990585, "mix_mean": 0.011523444205522537, "mix_low": 0.7513008117675781, "mix_high": 0.0, "read_cosine": 0.2124217450618744, "beta_mean": 0.3672722280025482, "nonfinite": 0.0, "weighted_memory_to_active": 0.09348994493484497, "mix_std": 0.032606810331344604, "beta_std": 0.2374640703201294, "beta_low": 0.00152587890625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.021653249859809875, "evidence_weights": [0.046618107706308365, 0.018331149592995644, 0.05980777367949486], "agreement_weight": -0.023076379671692848, "orthogonal_weight": -0.24674668908119202}, {"head": 4, "memory_to_active": 3.174830913543701, "contribution_to_active": 1.447049617767334, "mix_mean": 0.03694994002580643, "mix_low": 0.2947578430175781, "mix_high": 3.814697265625e-06, "read_cosine": 0.3858802914619446, "beta_mean": 0.20291346311569214, "nonfinite": 0.0, "weighted_memory_to_active": 1.268818736076355, "mix_std": 0.07193470001220703, "beta_std": 0.10600261390209198, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17995858192443848, "evidence_weights": [-0.02262970618903637, -0.009512281976640224, 0.07318252325057983], "agreement_weight": -0.031603604555130005, "orthogonal_weight": 0.16764794290065765}, {"head": 5, "memory_to_active": 3.2364513874053955, "contribution_to_active": 0.04745769128203392, "mix_mean": 0.008245021104812622, "mix_low": 0.7950878143310547, "mix_high": 0.0, "read_cosine": 0.26282089948654175, "beta_mean": 0.26756855845451355, "nonfinite": 0.0, "weighted_memory_to_active": 0.04549865052103996, "mix_std": 0.021085072308778763, "beta_std": 0.21704897284507751, "beta_low": 0.020263671875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0012041499139741063, "evidence_weights": [0.13316866755485535, 0.0618891678750515, 0.3404904305934906], "agreement_weight": -0.14294491708278656, "orthogonal_weight": 0.0276575218886137}]}, {"layer": 2, "memory_to_active": 1.2148346900939941, "contribution_to_active": 0.31142833828926086, "mix_mean": 0.278911828994751, "mix_low": 0.26906079053878784, "mix_high": 0.003223737236112356, "read_cosine": 0.3486723005771637, "beta_mean": 0.11718486994504929, "nonfinite": 0.0, "weighted_memory_to_active": 0.08028232306241989, "mix_std": 0.19750532507896423, "beta_std": 0.07920823246240616, "beta_low": 0.53875732421875, "beta_high": 0.00010172526526730508, "orthogonal_to_active": 0.018017349764704704, "output_influence": 0.2662739157676697, "residual_influence": 0.06433223932981491, "state_active": 0.040503375232219696, "state_memory": 0.023797381669282913, "state_peak": 44.65210723876953, "growth_active": 1.0540047883987427, "growth_memory": 1.543188452720642, "heads": [{"head": 0, "memory_to_active": 0.012434989213943481, "contribution_to_active": 0.19639191031455994, "mix_mean": 0.3638501465320587, "mix_low": 0.012820243835449219, "mix_high": 0.0018482208251953125, "read_cosine": 0.2533200979232788, "beta_mean": 8.093791984720156e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005327248014509678, "mix_std": 0.2932139039039612, "beta_std": 9.994082211051136e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001613055355846882, "evidence_weights": [-0.20946387946605682, -0.13947363197803497, 0.2064216136932373], "agreement_weight": 0.11149071156978607, "orthogonal_weight": -0.034851860255002975}, {"head": 1, "memory_to_active": 3.190453052520752, "contribution_to_active": 0.04996836930513382, "mix_mean": 0.004906671587377787, "mix_low": 0.9017963409423828, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19646260142326355, "beta_mean": 0.3528793454170227, "nonfinite": 0.0, "weighted_memory_to_active": 0.03647741675376892, "mix_std": 0.012568582780659199, "beta_std": 0.17011858522891998, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012586753815412521, "evidence_weights": [0.09707576036453247, 0.10599291324615479, 0.2674121558666229], "agreement_weight": 0.02559708245098591, "orthogonal_weight": 0.35222217440605164}, {"head": 2, "memory_to_active": 0.03145124763250351, "contribution_to_active": 0.43307557702064514, "mix_mean": 0.39699411392211914, "mix_low": 0.0038547515869140625, "mix_high": 0.0023050308227539062, "read_cosine": 0.2777722477912903, "beta_mean": 5.6235268857562914e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013705112040042877, "mix_std": 0.2808927893638611, "beta_std": 0.00011607155465753749, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029027515556663275, "evidence_weights": [-0.1917881965637207, -0.2319844514131546, 0.5004392266273499], "agreement_weight": 0.03919915109872818, "orthogonal_weight": -0.275462806224823}, {"head": 3, "memory_to_active": 0.28197115659713745, "contribution_to_active": 0.4396740794181824, "mix_mean": 0.35951095819473267, "mix_low": 0.0033369064331054688, "mix_high": 0.005889892578125, "read_cosine": 0.6641912460327148, "beta_mean": 0.05202273279428482, "nonfinite": 0.0, "weighted_memory_to_active": 0.13221658766269684, "mix_std": 0.2652374804019928, "beta_std": 0.06944329291582108, "beta_low": 0.1986083984375, "beta_high": 0.0, "orthogonal_to_active": 0.07114515453577042, "evidence_weights": [-0.13892017304897308, -0.20826925337314606, 0.7223249673843384], "agreement_weight": 0.06825438141822815, "orthogonal_weight": -0.8114152550697327}, {"head": 4, "memory_to_active": 3.7485156059265137, "contribution_to_active": 0.24018844962120056, "mix_mean": 0.014764570631086826, "mix_low": 0.6905050277709961, "mix_high": 0.0, "read_cosine": 0.17476339638233185, "beta_mean": 0.29800426959991455, "nonfinite": 0.0, "weighted_memory_to_active": 0.27926844358444214, "mix_std": 0.04521919786930084, "beta_std": 0.2353820502758026, "beta_low": 0.033935546875, "beta_high": 0.0006103515625, "orthogonal_to_active": 0.020383041352033615, "evidence_weights": [0.043182410299777985, 0.03944264352321625, 0.15321680903434753], "agreement_weight": 0.03534189984202385, "orthogonal_weight": -0.08179783821105957}, {"head": 5, "memory_to_active": 0.024181708693504333, "contribution_to_active": 0.5092716813087463, "mix_mean": 0.5334445238113403, "mix_low": 0.0020513534545898438, "mix_high": 0.009298324584960938, "read_cosine": 0.5255240201950073, "beta_mean": 6.566150113940239e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014699119143188, "mix_std": 0.2878999710083008, "beta_std": 8.943353168433532e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009250869043171406, "evidence_weights": [-0.24114196002483368, -0.2592487335205078, 0.6915934085845947], "agreement_weight": 0.02937028370797634, "orthogonal_weight": 0.11890356242656708}]}, {"layer": 3, "memory_to_active": 1.2975194454193115, "contribution_to_active": 0.3424561619758606, "mix_mean": 0.21961776912212372, "mix_low": 0.23623323440551758, "mix_high": 0.002862453693524003, "read_cosine": 0.2567278742790222, "beta_mean": 0.1072218045592308, "nonfinite": 0.0, "weighted_memory_to_active": 0.16401860117912292, "mix_std": 0.2104807198047638, "beta_std": 0.07852163165807724, "beta_low": 0.4055684804916382, "beta_high": 0.0, "orthogonal_to_active": 0.04290241748094559, "output_influence": 0.24908031523227692, "residual_influence": 0.04687092825770378, "state_active": 0.029422074556350708, "state_memory": 0.020111946389079094, "state_peak": 22.20427894592285, "growth_active": 1.0846405029296875, "growth_memory": 1.7019249200820923, "heads": [{"head": 0, "memory_to_active": 0.6814484596252441, "contribution_to_active": 0.39970892667770386, "mix_mean": 0.2691311240196228, "mix_low": 0.10539627075195312, "mix_high": 0.0027685165405273438, "read_cosine": 0.16450266540050507, "beta_mean": 0.016185205429792404, "nonfinite": 0.0, "weighted_memory_to_active": 0.23906049132347107, "mix_std": 0.29356154799461365, "beta_std": 0.014816923066973686, "beta_low": 0.3990478515625, "beta_high": 0.0, "orthogonal_to_active": 0.11431141942739487, "evidence_weights": [-0.08971945196390152, -0.15773142874240875, 0.11635686457157135], "agreement_weight": 0.09097401052713394, "orthogonal_weight": -0.4955483675003052}, {"head": 1, "memory_to_active": 0.02576429583132267, "contribution_to_active": 0.29155924916267395, "mix_mean": 0.28877413272857666, "mix_low": 0.04937267303466797, "mix_high": 0.0016574859619140625, "read_cosine": 0.20579761266708374, "beta_mean": 0.00011681381874950603, "nonfinite": 0.0, "weighted_memory_to_active": 0.007454561069607735, "mix_std": 0.2815777361392975, "beta_std": 0.0002507409662939608, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007711778162047267, "evidence_weights": [-0.1615106165409088, -0.12994204461574554, 0.35424235463142395], "agreement_weight": 0.13492725789546967, "orthogonal_weight": -0.11267929524183273}, {"head": 2, "memory_to_active": 3.492358684539795, "contribution_to_active": 0.31365159153938293, "mix_mean": 0.04103265702724457, "mix_low": 0.40035438537597656, "mix_high": 0.0, "read_cosine": 0.17467713356018066, "beta_mean": 0.2591637670993805, "nonfinite": 0.0, "weighted_memory_to_active": 0.40414392948150635, "mix_std": 0.09385078400373459, "beta_std": 0.1616775393486023, "beta_low": 0.00445556640625, "beta_high": 0.0, "orthogonal_to_active": 0.022111324593424797, "evidence_weights": [-0.04931808263063431, 0.010823986493051052, 0.05110416188836098], "agreement_weight": -0.008975616656243801, "orthogonal_weight": -0.07100483030080795}, {"head": 3, "memory_to_active": 0.037029899656772614, "contribution_to_active": 0.46905839443206787, "mix_mean": 0.46142345666885376, "mix_low": 0.02444934844970703, "mix_high": 0.008597373962402344, "read_cosine": 0.28977400064468384, "beta_mean": 0.00014364541857503355, "nonfinite": 0.0, "weighted_memory_to_active": 0.021155720576643944, "mix_std": 0.3179602026939392, "beta_std": 0.0002918170939665288, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006238730624318123, "evidence_weights": [-0.1834149956703186, -0.2047261893749237, 1.3305319547653198], "agreement_weight": 0.7587639093399048, "orthogonal_weight": -0.38694050908088684}, {"head": 4, "memory_to_active": 3.0897021293640137, "contribution_to_active": 0.04350799322128296, "mix_mean": 0.0092630535364151, "mix_low": 0.784912109375, "mix_high": 0.0, "read_cosine": 0.2043064534664154, "beta_mean": 0.2625386416912079, "nonfinite": 0.0, "weighted_memory_to_active": 0.06580054014921188, "mix_std": 0.024215739220380783, "beta_std": 0.19097426533699036, "beta_low": 0.00164794921875, "beta_high": 0.0, "orthogonal_to_active": 0.028525175526738167, "evidence_weights": [0.15485705435276031, 0.11947476118803024, -0.05334563925862312], "agreement_weight": -0.04716106131672859, "orthogonal_weight": -0.45395737886428833}, {"head": 5, "memory_to_active": 0.4588128626346588, "contribution_to_active": 0.5372506380081177, "mix_mean": 0.24808217585086823, "mix_low": 0.05291461944580078, "mix_high": 0.004151344299316406, "read_cosine": 0.5013092756271362, "beta_mean": 0.10518273711204529, "nonfinite": 0.0, "weighted_memory_to_active": 0.24649640917778015, "mix_std": 0.2517182230949402, "beta_std": 0.10311847180128098, "beta_low": 0.0283203125, "beta_high": 0.0, "orthogonal_to_active": 0.08545665442943573, "evidence_weights": [-0.11203408241271973, -0.1806788146495819, 0.27312546968460083], "agreement_weight": 0.17101049423217773, "orthogonal_weight": -0.5139684081077576}]}, {"layer": 4, "memory_to_active": 1.265932321548462, "contribution_to_active": 0.37441521883010864, "mix_mean": 0.29340070486068726, "mix_low": 0.2764662206172943, "mix_high": 0.002643903251737356, "read_cosine": 0.4136437773704529, "beta_mean": 0.12296439707279205, "nonfinite": 0.0, "weighted_memory_to_active": 0.15512841939926147, "mix_std": 0.15963563323020935, "beta_std": 0.10337644815444946, "beta_low": 0.3448486626148224, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.034436315298080444, "output_influence": 0.27732253074645996, "residual_influence": 0.0658613070845604, "state_active": 0.0446959063410759, "state_memory": 0.027845585718750954, "state_peak": 52.426273345947266, "growth_active": 1.1691851615905762, "growth_memory": 1.6130162477493286, "heads": [{"head": 0, "memory_to_active": 0.32798531651496887, "contribution_to_active": 0.612591028213501, "mix_mean": 0.5457708835601807, "mix_low": 0.01366424560546875, "mix_high": 0.0037078857421875, "read_cosine": 0.5701634883880615, "beta_mean": 0.19701026380062103, "nonfinite": 0.0, "weighted_memory_to_active": 0.24191449582576752, "mix_std": 0.3250151574611664, "beta_std": 0.22030401229858398, "beta_low": 0.19842529296875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.03427448868751526, "evidence_weights": [-0.23261943459510803, -0.2988510727882385, 0.4274440407752991], "agreement_weight": 0.04913495108485222, "orthogonal_weight": 0.19871823489665985}, {"head": 1, "memory_to_active": 0.20541812479496002, "contribution_to_active": 0.45304715633392334, "mix_mean": 0.4870949983596802, "mix_low": 0.0009012222290039062, "mix_high": 0.002460479736328125, "read_cosine": 0.7324246168136597, "beta_mean": 0.006888113915920258, "nonfinite": 0.0, "weighted_memory_to_active": 0.10866500437259674, "mix_std": 0.2693231701850891, "beta_std": 0.010962649248540401, "beta_low": 0.81787109375, "beta_high": 0.0, "orthogonal_to_active": 0.04667474702000618, "evidence_weights": [-0.2313646823167801, -0.22188112139701843, 0.6577243208885193], "agreement_weight": 0.049678198993206024, "orthogonal_weight": -0.6558703780174255}, {"head": 2, "memory_to_active": 2.6195578575134277, "contribution_to_active": 0.13133636116981506, "mix_mean": 0.015489117242395878, "mix_low": 0.6244068145751953, "mix_high": 0.0, "read_cosine": 0.3201998174190521, "beta_mean": 0.12194061279296875, "nonfinite": 0.0, "weighted_memory_to_active": 0.16923201084136963, "mix_std": 0.028255600482225418, "beta_std": 0.08852874487638474, "beta_low": 0.00152587890625, "beta_high": 0.0, "orthogonal_to_active": 0.007081092335283756, "evidence_weights": [0.048027753829956055, 0.02514319121837616, -0.03859449550509453], "agreement_weight": 0.0677562728524208, "orthogonal_weight": -0.06099117174744606}, {"head": 3, "memory_to_active": 1.6362087726593018, "contribution_to_active": 0.1105763390660286, "mix_mean": 0.03198995441198349, "mix_low": 0.3742694854736328, "mix_high": 0.0, "read_cosine": 0.19315588474273682, "beta_mean": 0.1553598940372467, "nonfinite": 0.0, "weighted_memory_to_active": 0.17459692060947418, "mix_std": 0.05529201030731201, "beta_std": 0.1409694403409958, "beta_low": 0.05126953125, "beta_high": 0.0, "orthogonal_to_active": 0.019205480813980103, "evidence_weights": [-0.018398262560367584, 0.04966476932168007, -0.22179442644119263], "agreement_weight": 0.04551507160067558, "orthogonal_weight": -0.1694619357585907}, {"head": 4, "memory_to_active": 0.03423968702554703, "contribution_to_active": 0.6313668489456177, "mix_mean": 0.6662111878395081, "mix_low": 0.0004062652587890625, "mix_high": 0.009688377380371094, "read_cosine": 0.3073650002479553, "beta_mean": 6.651321018580347e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0256439708173275, "mix_std": 0.2472350150346756, "beta_std": 0.00016728133778087795, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020473587792366743, "evidence_weights": [-0.2882027328014374, -0.3285849392414093, 0.7974423170089722], "agreement_weight": 0.0745384469628334, "orthogonal_weight": -0.097713403403759}, {"head": 5, "memory_to_active": 2.7721831798553467, "contribution_to_active": 0.30757343769073486, "mix_mean": 0.0138480793684721, "mix_low": 0.6451492309570312, "mix_high": 6.67572021484375e-06, "read_cosine": 0.35855376720428467, "beta_mean": 0.25652095675468445, "nonfinite": 0.0, "weighted_memory_to_active": 0.2107180953025818, "mix_std": 0.03269275650382042, "beta_std": 0.15932656824588776, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.09733471274375916, "evidence_weights": [0.03684946522116661, -0.0345340333878994, -0.13599136471748352], "agreement_weight": -0.047077029943466187, "orthogonal_weight": 0.6886631846427917}]}, {"layer": 5, "memory_to_active": 0.6475077867507935, "contribution_to_active": 0.4474474787712097, "mix_mean": 0.4002746641635895, "mix_low": 0.09725523740053177, "mix_high": 0.010420323349535465, "read_cosine": 0.2238166481256485, "beta_mean": 0.05169152840971947, "nonfinite": 0.0, "weighted_memory_to_active": 0.1597796082496643, "mix_std": 0.24795275926589966, "beta_std": 0.06683022528886795, "beta_low": 0.6084798574447632, "beta_high": 0.0, "orthogonal_to_active": 0.04970872402191162, "output_influence": 0.4328197240829468, "residual_influence": 0.09854739904403687, "state_active": 0.05750669538974762, "state_memory": 0.01618003472685814, "state_peak": 66.24176788330078, "growth_active": 1.1416152715682983, "growth_memory": 1.2892160415649414, "heads": [{"head": 0, "memory_to_active": 1.3113511800765991, "contribution_to_active": 0.2644544243812561, "mix_mean": 0.13258853554725647, "mix_low": 0.14528369903564453, "mix_high": 0.0, "read_cosine": -0.18337149918079376, "beta_mean": 0.05260097235441208, "nonfinite": 0.0, "weighted_memory_to_active": 0.19192785024642944, "mix_std": 0.17122463881969452, "beta_std": 0.07816403359174728, "beta_low": 0.19134521484375, "beta_high": 0.0, "orthogonal_to_active": 0.0432397723197937, "evidence_weights": [-0.04970075190067291, -0.08444274216890335, -0.35850924253463745], "agreement_weight": -0.021451985463500023, "orthogonal_weight": -0.21740815043449402}, {"head": 1, "memory_to_active": 0.0406796932220459, "contribution_to_active": 0.4728322923183441, "mix_mean": 0.5380051136016846, "mix_low": 0.009036064147949219, "mix_high": 0.011119842529296875, "read_cosine": 0.19736269116401672, "beta_mean": 0.00011604341852944344, "nonfinite": 0.0, "weighted_memory_to_active": 0.018720729276537895, "mix_std": 0.31579136848449707, "beta_std": 0.0002778050838969648, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.01638329029083252, "evidence_weights": [-0.21993590891361237, -0.23664915561676025, 0.736235499382019], "agreement_weight": 0.28751567006111145, "orthogonal_weight": -0.9169849157333374}, {"head": 2, "memory_to_active": 0.06104365736246109, "contribution_to_active": 0.48962441086769104, "mix_mean": 0.5766326189041138, "mix_low": 0.0020799636840820312, "mix_high": 0.011674880981445312, "read_cosine": 0.3677336573600769, "beta_mean": 0.0014455904019996524, "nonfinite": 0.0, "weighted_memory_to_active": 0.0406591072678566, "mix_std": 0.280468225479126, "beta_std": 0.0017370892455801368, "beta_low": 0.99365234375, "beta_high": 0.0, "orthogonal_to_active": 0.005468529649078846, "evidence_weights": [-0.26344195008277893, -0.312322199344635, 0.7507427930831909], "agreement_weight": 0.019061066210269928, "orthogonal_weight": -0.17830458283424377}, {"head": 3, "memory_to_active": 1.95688796043396, "contribution_to_active": 0.20830626785755157, "mix_mean": 0.07744704186916351, "mix_low": 0.23705673217773438, "mix_high": 0.0, "read_cosine": 0.14566762745380402, "beta_mean": 0.07176525890827179, "nonfinite": 0.0, "weighted_memory_to_active": 0.3460986912250519, "mix_std": 0.12393023073673248, "beta_std": 0.10744168609380722, "beta_low": 0.22357177734375, "beta_high": 0.0, "orthogonal_to_active": 0.1259455680847168, "evidence_weights": [-0.025977525860071182, -0.07556787878274918, -0.01146644540131092], "agreement_weight": 0.05171192064881325, "orthogonal_weight": -0.45869624614715576}, {"head": 4, "memory_to_active": 0.44477689266204834, "contribution_to_active": 0.6217821836471558, "mix_mean": 0.33304810523986816, "mix_low": 0.18707656860351562, "mix_high": 0.01096343994140625, "read_cosine": 0.5856074690818787, "beta_mean": 0.18394137918949127, "nonfinite": 0.0, "weighted_memory_to_active": 0.31353631615638733, "mix_std": 0.3428347110748291, "beta_std": 0.21206071972846985, "beta_low": 0.2454833984375, "beta_high": 0.0, "orthogonal_to_active": 0.0797891765832901, "evidence_weights": [-0.12651099264621735, -0.280843049287796, -0.010977165773510933], "agreement_weight": 0.04928044229745865, "orthogonal_weight": 0.39790067076683044}, {"head": 5, "memory_to_active": 0.07030714303255081, "contribution_to_active": 0.6276851892471313, "mix_mean": 0.7439265251159668, "mix_low": 0.00299835205078125, "mix_high": 0.028763771057128906, "read_cosine": 0.22989995777606964, "beta_mean": 0.00027991452952846885, "nonfinite": 0.0, "weighted_memory_to_active": 0.04773489013314247, "mix_std": 0.25346729159355164, "beta_std": 0.0013000309700146317, "beta_low": 0.9969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.02742597460746765, "evidence_weights": [-0.32339733839035034, -0.3120138645172119, 1.6637884378433228], "agreement_weight": 0.31524690985679626, "orthogonal_weight": -0.6143634915351868}]}, {"layer": 6, "memory_to_active": 0.39362338185310364, "contribution_to_active": 0.5556145906448364, "mix_mean": 0.509679913520813, "mix_low": 0.08404874801635742, "mix_high": 0.025522232055664062, "read_cosine": 0.30406561493873596, "beta_mean": 0.07545360922813416, "nonfinite": 0.0, "weighted_memory_to_active": 0.15824830532073975, "mix_std": 0.2507745325565338, "beta_std": 0.08971402794122696, "beta_low": 0.5096842646598816, "beta_high": 0.0, "orthogonal_to_active": 0.035036370158195496, "output_influence": 0.43160271644592285, "residual_influence": 0.13156014680862427, "state_active": 0.05211278051137924, "state_memory": 0.010345468297600746, "state_peak": 63.70640563964844, "growth_active": 1.1771901845932007, "growth_memory": 1.257115125656128, "heads": [{"head": 0, "memory_to_active": 0.14524197578430176, "contribution_to_active": 0.5704478025436401, "mix_mean": 0.6757240295410156, "mix_low": 0.00024127960205078125, "mix_high": 0.004893302917480469, "read_cosine": 0.1088305115699768, "beta_mean": 0.013932068832218647, "nonfinite": 0.0, "weighted_memory_to_active": 0.10612566769123077, "mix_std": 0.24052855372428894, "beta_std": 0.03307662904262543, "beta_low": 0.62286376953125, "beta_high": 0.0, "orthogonal_to_active": 0.08349914103746414, "evidence_weights": [-0.30750003457069397, -0.30823880434036255, 0.4456581473350525], "agreement_weight": 0.09629955142736435, "orthogonal_weight": -0.8985474705696106}, {"head": 1, "memory_to_active": 0.007763821631669998, "contribution_to_active": 0.45739561319351196, "mix_mean": 0.46217554807662964, "mix_low": 0.0024061203002929688, "mix_high": 0.009715080261230469, "read_cosine": 0.6020698547363281, "beta_mean": 1.8222188373329118e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0028853663243353367, "mix_std": 0.294128954410553, "beta_std": 4.7274064854718745e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007383580668829381, "evidence_weights": [-0.21209457516670227, -0.2575237452983856, 0.7355618476867676], "agreement_weight": -0.02423926442861557, "orthogonal_weight": -0.3508279025554657}, {"head": 2, "memory_to_active": 1.3178430795669556, "contribution_to_active": 0.05274217575788498, "mix_mean": 0.022933345288038254, "mix_low": 0.4897632598876953, "mix_high": 0.0, "read_cosine": 0.20276345312595367, "beta_mean": 0.0755196362733841, "nonfinite": 0.0, "weighted_memory_to_active": 0.058976855129003525, "mix_std": 0.04217329993844032, "beta_std": 0.07738062739372253, "beta_low": 0.00994873046875, "beta_high": 0.0, "orthogonal_to_active": 0.010248408652842045, "evidence_weights": [0.08575981855392456, 0.02521783858537674, 0.20205576717853546], "agreement_weight": 0.08715615421533585, "orthogonal_weight": -0.1935131549835205}, {"head": 3, "memory_to_active": 0.1465892195701599, "contribution_to_active": 0.6394509077072144, "mix_mean": 0.6135483980178833, "mix_low": 0.009905815124511719, "mix_high": 0.024866104125976562, "read_cosine": 0.5050851702690125, "beta_mean": 0.23040203750133514, "nonfinite": 0.0, "weighted_memory_to_active": 0.1032409742474556, "mix_std": 0.34346887469291687, "beta_std": 0.26652881503105164, "beta_low": 0.212158203125, "beta_high": 0.0, "orthogonal_to_active": 0.04478011652827263, "evidence_weights": [-0.1733619123697281, -0.33349528908729553, 0.15002501010894775], "agreement_weight": -0.010452952235937119, "orthogonal_weight": 0.5148612260818481}, {"head": 4, "memory_to_active": 0.6965194940567017, "contribution_to_active": 1.0568327903747559, "mix_mean": 0.7230775952339172, "mix_low": 0.001560211181640625, "mix_high": 0.1123647689819336, "read_cosine": 0.12180939316749573, "beta_mean": 0.13247999548912048, "nonfinite": 0.0, "weighted_memory_to_active": 0.6478885412216187, "mix_std": 0.30298131704330444, "beta_std": 0.1604899913072586, "beta_low": 0.2142333984375, "beta_high": 0.0, "orthogonal_to_active": 0.04068419709801674, "evidence_weights": [-0.24677379429340363, -0.5564538240432739, -0.036508649587631226], "agreement_weight": -0.019726945087313652, "orthogonal_weight": 0.07392458617687225}, {"head": 5, "memory_to_active": 0.04778248816728592, "contribution_to_active": 0.556818425655365, "mix_mean": 0.5606206059455872, "mix_low": 0.000415802001953125, "mix_high": 0.0012941360473632812, "read_cosine": 0.28383517265319824, "beta_mean": 0.0003696702769957483, "nonfinite": 0.0, "weighted_memory_to_active": 0.030372420325875282, "mix_std": 0.28136613965034485, "beta_std": 0.0007608255255036056, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.03026800975203514, "evidence_weights": [-0.22811411321163177, -0.2747616469860077, 0.3392769992351532], "agreement_weight": 0.08209529519081116, "orthogonal_weight": -1.071642518043518}]}, {"layer": 7, "memory_to_active": 2.0874757766723633, "contribution_to_active": 0.17776606976985931, "mix_mean": 0.03970973938703537, "mix_low": 0.48650187253952026, "mix_high": 4.927317604597192e-06, "read_cosine": 0.22982355952262878, "beta_mean": 0.13475266098976135, "nonfinite": 0.0, "weighted_memory_to_active": 0.19596779346466064, "mix_std": 0.06821957230567932, "beta_std": 0.11069430410861969, "beta_low": 0.0456644743680954, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.040405310690402985, "output_influence": 0.19487415254116058, "residual_influence": 0.042692698538303375, "state_active": 0.015653889626264572, "state_memory": 0.03143927454948425, "state_peak": 21.898048400878906, "growth_active": 1.0627634525299072, "growth_memory": 1.3926924467086792, "heads": [{"head": 0, "memory_to_active": 2.2598459720611572, "contribution_to_active": 0.36736994981765747, "mix_mean": 0.027997393161058426, "mix_low": 0.5276222229003906, "mix_high": 1.239776611328125e-05, "read_cosine": 0.3689807653427124, "beta_mean": 0.16379877924919128, "nonfinite": 0.0, "weighted_memory_to_active": 0.3423721194267273, "mix_std": 0.06166272982954979, "beta_std": 0.11914660036563873, "beta_low": 0.003662109375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.03885510191321373, "evidence_weights": [0.0014015758642926812, -0.0817716047167778, 0.08812077343463898], "agreement_weight": -0.03812359273433685, "orthogonal_weight": 0.15804660320281982}, {"head": 1, "memory_to_active": 1.8092190027236938, "contribution_to_active": 0.28243961930274963, "mix_mean": 0.10241768509149551, "mix_low": 0.2704582214355469, "mix_high": 5.7220458984375e-06, "read_cosine": 0.14723360538482666, "beta_mean": 0.17553091049194336, "nonfinite": 0.0, "weighted_memory_to_active": 0.355762779712677, "mix_std": 0.14317142963409424, "beta_std": 0.10160152614116669, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06115572154521942, "evidence_weights": [-0.13650454580783844, -0.1054174080491066, -0.173089399933815], "agreement_weight": 0.009650597348809242, "orthogonal_weight": -0.18596304953098297}, {"head": 2, "memory_to_active": 1.2349517345428467, "contribution_to_active": 0.2195827066898346, "mix_mean": 0.057553041726350784, "mix_low": 0.2933464050292969, "mix_high": 1.1444091796875e-05, "read_cosine": 0.1755611002445221, "beta_mean": 0.1117362231016159, "nonfinite": 0.0, "weighted_memory_to_active": 0.17737160623073578, "mix_std": 0.1064080148935318, "beta_std": 0.0713101401925087, "beta_low": 0.02093505859375, "beta_high": 0.0, "orthogonal_to_active": 0.019333023577928543, "evidence_weights": [-0.030129218474030495, -0.11066873371601105, 0.22255070507526398], "agreement_weight": 0.026794834062457085, "orthogonal_weight": -0.11407677829265594}, {"head": 3, "memory_to_active": 3.4245822429656982, "contribution_to_active": 0.06323298811912537, "mix_mean": 0.018296653404831886, "mix_low": 0.5279426574707031, "mix_high": 0.0, "read_cosine": 0.12200918793678284, "beta_mean": 0.1671832650899887, "nonfinite": 0.0, "weighted_memory_to_active": 0.14742015302181244, "mix_std": 0.030172109603881836, "beta_std": 0.17125269770622253, "beta_low": 0.0389404296875, "beta_high": 0.0, "orthogonal_to_active": 0.09960577636957169, "evidence_weights": [0.10335183888673782, 0.06288635730743408, 0.3731667399406433], "agreement_weight": 0.08154121041297913, "orthogonal_weight": -0.7446392178535461}, {"head": 4, "memory_to_active": 1.8784351348876953, "contribution_to_active": 0.05434310436248779, "mix_mean": 0.010688289999961853, "mix_low": 0.7602863311767578, "mix_high": 0.0, "read_cosine": 0.33863282203674316, "beta_mean": 0.1360861212015152, "nonfinite": 0.0, "weighted_memory_to_active": 0.05152240768074989, "mix_std": 0.028010614216327667, "beta_std": 0.13259069621562958, "beta_low": 0.0433349609375, "beta_high": 0.0, "orthogonal_to_active": 0.0028663701377809048, "evidence_weights": [0.10132195055484772, 0.049166806042194366, 0.04486706852912903], "agreement_weight": 0.04801815003156662, "orthogonal_weight": -0.06260167807340622}, {"head": 5, "memory_to_active": 1.9178204536437988, "contribution_to_active": 0.07962794601917267, "mix_mean": 0.02130538783967495, "mix_low": 0.5393552780151367, "mix_high": 0.0, "read_cosine": 0.22652381658554077, "beta_mean": 0.054180655628442764, "nonfinite": 0.0, "weighted_memory_to_active": 0.10135766118764877, "mix_std": 0.0398925319314003, "beta_std": 0.06826414167881012, "beta_low": 0.1671142578125, "beta_high": 0.0, "orthogonal_to_active": 0.020615870133042336, "evidence_weights": [0.06832949817180634, 0.06035386398434639, 0.03014938160777092], "agreement_weight": 0.01908392459154129, "orthogonal_weight": -0.26023584604263306}]}, {"layer": 8, "memory_to_active": 0.6137710213661194, "contribution_to_active": 0.47357743978500366, "mix_mean": 0.3628588318824768, "mix_low": 0.08410152047872543, "mix_high": 0.0009509722585789859, "read_cosine": 0.5307572484016418, "beta_mean": 0.03872133046388626, "nonfinite": 0.0, "weighted_memory_to_active": 0.14257289469242096, "mix_std": 0.2280711680650711, "beta_std": 0.040760770440101624, "beta_low": 0.46881103515625, "beta_high": 0.0, "orthogonal_to_active": 0.049919381737709045, "output_influence": 0.3483859598636627, "residual_influence": 0.09703344106674194, "state_active": 0.06083551421761513, "state_memory": 0.013903690502047539, "state_peak": 34.90372848510742, "growth_active": 1.1550264358520508, "growth_memory": 1.4034974575042725, "heads": [{"head": 0, "memory_to_active": 0.04866943508386612, "contribution_to_active": 0.5099401473999023, "mix_mean": 0.5376801490783691, "mix_low": 0.00016880035400390625, "mix_high": 0.0004749298095703125, "read_cosine": 0.7151975631713867, "beta_mean": 0.003264164552092552, "nonfinite": 0.0, "weighted_memory_to_active": 0.027814878150820732, "mix_std": 0.24236372113227844, "beta_std": 0.007269232999533415, "beta_low": 0.92376708984375, "beta_high": 0.0, "orthogonal_to_active": 0.014743429608643055, "evidence_weights": [-0.20427143573760986, -0.35173529386520386, 0.7196851372718811], "agreement_weight": 0.0676649808883667, "orthogonal_weight": -0.7852610349655151}, {"head": 1, "memory_to_active": 0.49284297227859497, "contribution_to_active": 0.362947940826416, "mix_mean": 0.3685033917427063, "mix_low": 0.005034446716308594, "mix_high": 0.000202178955078125, "read_cosine": 0.4945911765098572, "beta_mean": 0.0237730760127306, "nonfinite": 0.0, "weighted_memory_to_active": 0.20458155870437622, "mix_std": 0.24552173912525177, "beta_std": 0.025717411190271378, "beta_low": 0.27398681640625, "beta_high": 0.0, "orthogonal_to_active": 0.06466163694858551, "evidence_weights": [-0.17210589349269867, -0.2007371038198471, 0.42305025458335876], "agreement_weight": 0.056811437010765076, "orthogonal_weight": -0.3712727129459381}, {"head": 2, "memory_to_active": 0.10886917263269424, "contribution_to_active": 0.5559848546981812, "mix_mean": 0.5118155479431152, "mix_low": 0.0007266998291015625, "mix_high": 0.0017423629760742188, "read_cosine": 0.36610177159309387, "beta_mean": 0.0014884925913065672, "nonfinite": 0.0, "weighted_memory_to_active": 0.0624205581843853, "mix_std": 0.25754207372665405, "beta_std": 0.0037030749954283237, "beta_low": 0.9776611328125, "beta_high": 0.0, "orthogonal_to_active": 0.045790478587150574, "evidence_weights": [-0.2018812894821167, -0.3040939271450043, 0.5203133821487427], "agreement_weight": 0.07307230681180954, "orthogonal_weight": -0.8458834290504456}, {"head": 3, "memory_to_active": 0.22071030735969543, "contribution_to_active": 0.45078498125076294, "mix_mean": 0.3984560966491699, "mix_low": 0.0028276443481445312, "mix_high": 5.245208740234375e-05, "read_cosine": 0.7282453179359436, "beta_mean": 0.016519492492079735, "nonfinite": 0.0, "weighted_memory_to_active": 0.12778735160827637, "mix_std": 0.24225057661533356, "beta_std": 0.017516329884529114, "beta_low": 0.37017822265625, "beta_high": 0.0, "orthogonal_to_active": 0.025289904326200485, "evidence_weights": [-0.12658053636550903, -0.239202618598938, 0.7283012270927429], "agreement_weight": 0.08577979356050491, "orthogonal_weight": -0.33155199885368347}, {"head": 4, "memory_to_active": 2.6144495010375977, "contribution_to_active": 0.2975451648235321, "mix_mean": 0.02872798964381218, "mix_low": 0.4066944122314453, "mix_high": 0.0, "read_cosine": 0.3347315192222595, "beta_mean": 0.11453178524971008, "nonfinite": 0.0, "weighted_memory_to_active": 0.29652783274650574, "mix_std": 0.04992746189236641, "beta_std": 0.0871371403336525, "beta_low": 0.00244140625, "beta_high": 0.0, "orthogonal_to_active": 0.010182691738009453, "evidence_weights": [0.03504406660795212, -0.003568005980923772, -0.15355990827083588], "agreement_weight": 0.01666485331952572, "orthogonal_weight": 0.03662969917058945}, {"head": 5, "memory_to_active": 0.197084441781044, "contribution_to_active": 0.6642614006996155, "mix_mean": 0.3319697380065918, "mix_low": 0.0891571044921875, "mix_high": 0.0032339096069335938, "read_cosine": 0.5456761121749878, "beta_mean": 0.07275094091892242, "nonfinite": 0.0, "weighted_memory_to_active": 0.13630512356758118, "mix_std": 0.3308213949203491, "beta_std": 0.10322140902280807, "beta_low": 0.26483154296875, "beta_high": 0.0, "orthogonal_to_active": 0.1388481706380844, "evidence_weights": [-0.07210618257522583, -0.3224088251590729, -0.43902695178985596], "agreement_weight": 0.030819538980722427, "orthogonal_weight": 1.247130274772644}]}, {"layer": 9, "memory_to_active": 1.2375035285949707, "contribution_to_active": 0.6247949600219727, "mix_mean": 0.24949662387371063, "mix_low": 0.17945513129234314, "mix_high": 0.0013446807861328125, "read_cosine": 0.3633941411972046, "beta_mean": 0.0851907879114151, "nonfinite": 0.0, "weighted_memory_to_active": 0.38867253065109253, "mix_std": 0.21051347255706787, "beta_std": 0.04942445084452629, "beta_low": 0.4977518916130066, "beta_high": 0.0, "orthogonal_to_active": 0.02973860688507557, "output_influence": 0.4062407314777374, "residual_influence": 0.10283096134662628, "state_active": 0.046429216861724854, "state_memory": 0.024507440626621246, "state_peak": 34.56564712524414, "growth_active": 1.2892670631408691, "growth_memory": 1.4113459587097168, "heads": [{"head": 0, "memory_to_active": 0.0066735208965837955, "contribution_to_active": 0.4725914001464844, "mix_mean": 0.5213475823402405, "mix_low": 0.0026292800903320312, "mix_high": 0.005730628967285156, "read_cosine": 0.22196276485919952, "beta_mean": 1.8903801901615225e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003675096668303013, "mix_std": 0.29801687598228455, "beta_std": 6.982782360864803e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028923763893544674, "evidence_weights": [-0.26549604535102844, -0.2633461356163025, 0.9133269786834717], "agreement_weight": 0.05716132000088692, "orthogonal_weight": -0.8177974820137024}, {"head": 1, "memory_to_active": 0.03246646746993065, "contribution_to_active": 0.504181981086731, "mix_mean": 0.3595913350582123, "mix_low": 0.014746665954589844, "mix_high": 0.0006256103515625, "read_cosine": 0.4029461443424225, "beta_mean": 9.53906710492447e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01609860546886921, "mix_std": 0.2826656103134155, "beta_std": 0.00022468986571766436, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008048349060118198, "evidence_weights": [-0.19817395508289337, -0.21441778540611267, 0.8883110880851746], "agreement_weight": 0.289812296628952, "orthogonal_weight": -0.5802522301673889}, {"head": 2, "memory_to_active": 0.26175206899642944, "contribution_to_active": 0.5237019658088684, "mix_mean": 0.4071287512779236, "mix_low": 0.008322715759277344, "mix_high": 0.0014734268188476562, "read_cosine": 0.42101114988327026, "beta_mean": 0.0013600109377875924, "nonfinite": 0.0, "weighted_memory_to_active": 0.14763182401657104, "mix_std": 0.2870405912399292, "beta_std": 0.005821858532726765, "beta_low": 0.98516845703125, "beta_high": 0.0, "orthogonal_to_active": 0.04310081526637077, "evidence_weights": [-0.15426993370056152, -0.17854934930801392, 0.44746774435043335], "agreement_weight": 0.1787138730287552, "orthogonal_weight": -0.3647323250770569}, {"head": 3, "memory_to_active": 1.9709885120391846, "contribution_to_active": 0.17810004949569702, "mix_mean": 0.035019658505916595, "mix_low": 0.43503761291503906, "mix_high": 2.47955322265625e-05, "read_cosine": 0.4790990352630615, "beta_mean": 0.10630086064338684, "nonfinite": 0.0, "weighted_memory_to_active": 0.1524682343006134, "mix_std": 0.08625713735818863, "beta_std": 0.07181821763515472, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.027276992797851562, "evidence_weights": [0.04976532608270645, 0.036970123648643494, -0.1604105830192566], "agreement_weight": 0.0603548102080822, "orthogonal_weight": 0.18706084787845612}, {"head": 4, "memory_to_active": 3.3213441371917725, "contribution_to_active": 1.439490556716919, "mix_mean": 0.1154232993721962, "mix_low": 0.20976734161376953, "mix_high": 0.0002117156982421875, "read_cosine": 0.2336064577102661, "beta_mean": 0.231114462018013, "nonfinite": 0.0, "weighted_memory_to_active": 1.292330265045166, "mix_std": 0.18755432963371277, "beta_std": 0.1122409850358963, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004056815814692527, "evidence_weights": [-0.04621744528412819, -0.13164149224758148, 0.04254043102264404], "agreement_weight": -0.044706862419843674, "orthogonal_weight": -0.0003122851194348186}, {"head": 5, "memory_to_active": 1.8317956924438477, "contribution_to_active": 0.6307040452957153, "mix_mean": 0.058469127863645554, "mix_low": 0.40622711181640625, "mix_high": 1.9073486328125e-06, "read_cosine": 0.4217391908168793, "beta_mean": 0.17225511372089386, "nonfinite": 0.0, "weighted_memory_to_active": 0.719831109046936, "mix_std": 0.12154630571603775, "beta_std": 0.10637109726667404, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.09670741856098175, "evidence_weights": [0.057005174458026886, -0.04073012247681618, 0.09081678837537766], "agreement_weight": 0.011712427251040936, "orthogonal_weight": 0.18573160469532013}]}, {"layer": 10, "memory_to_active": 0.17097361385822296, "contribution_to_active": 0.4441855549812317, "mix_mean": 0.4272463321685791, "mix_low": 0.01026519201695919, "mix_high": 0.00936492346227169, "read_cosine": 0.39316561818122864, "beta_mean": 0.04492984339594841, "nonfinite": 0.0, "weighted_memory_to_active": 0.07413674890995026, "mix_std": 0.2693607807159424, "beta_std": 0.05026281997561455, "beta_low": 0.6542562246322632, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03808530420064926, "output_influence": 0.3849419951438904, "residual_influence": 0.12210877239704132, "state_active": 0.05949337035417557, "state_memory": 0.014407671056687832, "state_peak": 108.54845428466797, "growth_active": 1.0887930393218994, "growth_memory": 1.0913877487182617, "heads": [{"head": 0, "memory_to_active": 0.3168734908103943, "contribution_to_active": 0.7384687662124634, "mix_mean": 0.63347327709198, "mix_low": 0.01636791229248047, "mix_high": 0.052771568298339844, "read_cosine": 0.2381516695022583, "beta_mean": 0.19559550285339355, "nonfinite": 0.0, "weighted_memory_to_active": 0.22881272435188293, "mix_std": 0.356266587972641, "beta_std": 0.21721811592578888, "beta_low": 0.1368408203125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12488022446632385, "evidence_weights": [-0.171694815158844, -0.347616583108902, -0.34060758352279663], "agreement_weight": 0.022174928337335587, "orthogonal_weight": 0.6163197755813599}, {"head": 1, "memory_to_active": 0.015808092430233955, "contribution_to_active": 0.5134953856468201, "mix_mean": 0.49934548139572144, "mix_low": 0.0018482208251953125, "mix_high": 0.0027894973754882812, "read_cosine": 0.2522163391113281, "beta_mean": 9.652346489019692e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008508719503879547, "mix_std": 0.26796016097068787, "beta_std": 0.00021231398568488657, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007921546697616577, "evidence_weights": [-0.26270416378974915, -0.1876066029071808, 0.3965991139411926], "agreement_weight": 0.21914483606815338, "orthogonal_weight": -1.0465052127838135}, {"head": 2, "memory_to_active": 0.19615299999713898, "contribution_to_active": 0.3871133029460907, "mix_mean": 0.3623403310775757, "mix_low": 0.004558563232421875, "mix_high": 0.0001049041748046875, "read_cosine": 0.05744132399559021, "beta_mean": 0.0020896096248179674, "nonfinite": 0.0, "weighted_memory_to_active": 0.06687839329242706, "mix_std": 0.2702587842941284, "beta_std": 0.008908923715353012, "beta_low": 0.95477294921875, "beta_high": 0.0, "orthogonal_to_active": 0.02211751788854599, "evidence_weights": [-0.14063760638237, -0.19803601503372192, 0.4933158755302429], "agreement_weight": 0.0019415973220020533, "orthogonal_weight": -0.3270319998264313}, {"head": 3, "memory_to_active": 0.32592836022377014, "contribution_to_active": 0.12984170019626617, "mix_mean": 0.2744067311286926, "mix_low": 0.03400421142578125, "mix_high": 0.0004711151123046875, "read_cosine": 0.8894044160842896, "beta_mean": 0.06521661579608917, "nonfinite": 0.0, "weighted_memory_to_active": 0.055776577442884445, "mix_std": 0.24359868466854095, "beta_std": 0.06436973810195923, "beta_low": 0.00640869140625, "beta_high": 0.0, "orthogonal_to_active": 0.053130216896533966, "evidence_weights": [-0.1299736052751541, -0.14812259376049042, 0.20956166088581085], "agreement_weight": 0.0031351225916296244, "orthogonal_weight": 1.335499882698059}, {"head": 4, "memory_to_active": 0.027651416137814522, "contribution_to_active": 0.45389431715011597, "mix_mean": 0.4403570294380188, "mix_low": 0.0012464523315429688, "mix_high": 4.57763671875e-05, "read_cosine": 0.06986947357654572, "beta_mean": 6.0036920331185684e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009349338710308075, "mix_std": 0.25426965951919556, "beta_std": 0.0023720432072877884, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0025279128458350897, "evidence_weights": [-0.18348367512226105, -0.2085035890340805, 0.42523130774497986], "agreement_weight": 0.03210386633872986, "orthogonal_weight": -0.27581095695495605}, {"head": 5, "memory_to_active": 0.14342722296714783, "contribution_to_active": 0.4422997534275055, "mix_mean": 0.35355493426322937, "mix_low": 0.0035657882690429688, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8519102931022644, "beta_mean": 0.006520754657685757, "nonfinite": 0.0, "weighted_memory_to_active": 0.07549475133419037, "mix_std": 0.2238108217716217, "beta_std": 0.008495795540511608, "beta_low": 0.82757568359375, "beta_high": 0.0, "orthogonal_to_active": 0.017934411764144897, "evidence_weights": [-0.16824671626091003, -0.16819022595882416, 0.2857912480831146], "agreement_weight": 0.0928652286529541, "orthogonal_weight": -0.563988983631134}]}, {"layer": 11, "memory_to_active": 0.22216689586639404, "contribution_to_active": 0.47520190477371216, "mix_mean": 0.3948848843574524, "mix_low": 0.055047035217285156, "mix_high": 0.024416448548436165, "read_cosine": 0.4951317310333252, "beta_mean": 0.038926441222429276, "nonfinite": 0.0, "weighted_memory_to_active": 0.08142753690481186, "mix_std": 0.3070598244667053, "beta_std": 0.053640395402908325, "beta_low": 0.503814697265625, "beta_high": 0.0, "orthogonal_to_active": 0.04761429876089096, "output_influence": 0.460878849029541, "residual_influence": 0.11838482320308685, "state_active": 0.12113329768180847, "state_memory": 0.010564461350440979, "state_peak": 36.391998291015625, "growth_active": 1.1298941373825073, "growth_memory": 1.4677385091781616, "heads": [{"head": 0, "memory_to_active": 0.01570737734436989, "contribution_to_active": 0.5393857359886169, "mix_mean": 0.5386552810668945, "mix_low": 0.002349853515625, "mix_high": 0.0076389312744140625, "read_cosine": 0.4227476418018341, "beta_mean": 0.00011747390090022236, "nonfinite": 0.0, "weighted_memory_to_active": 0.010139494203031063, "mix_std": 0.2773854732513428, "beta_std": 0.003552426351234317, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.0056838421151041985, "evidence_weights": [-0.15711431205272675, -0.3329995274543762, 0.5797935128211975], "agreement_weight": 0.1519192010164261, "orthogonal_weight": -0.6887195110321045}, {"head": 1, "memory_to_active": 0.45400553941726685, "contribution_to_active": 0.23675711452960968, "mix_mean": 0.25492316484451294, "mix_low": 0.033156394958496094, "mix_high": 0.000194549560546875, "read_cosine": 0.5967334508895874, "beta_mean": 0.034203335642814636, "nonfinite": 0.0, "weighted_memory_to_active": 0.1600342094898224, "mix_std": 0.24058404564857483, "beta_std": 0.06052779033780098, "beta_low": 0.39599609375, "beta_high": 0.0, "orthogonal_to_active": 0.02429421804845333, "evidence_weights": [-0.16081172227859497, -0.12196460366249084, 0.17094890773296356], "agreement_weight": 0.08921916037797928, "orthogonal_weight": -0.20818760991096497}, {"head": 2, "memory_to_active": 0.046764541417360306, "contribution_to_active": 0.875953197479248, "mix_mean": 0.47966229915618896, "mix_low": 0.06628608703613281, "mix_high": 0.07421207427978516, "read_cosine": 0.3189179599285126, "beta_mean": 0.07375285774469376, "nonfinite": 0.0, "weighted_memory_to_active": 0.03844226896762848, "mix_std": 0.4095921516418457, "beta_std": 0.10009337216615677, "beta_low": 0.44073486328125, "beta_high": 0.0, "orthogonal_to_active": 0.10287223756313324, "evidence_weights": [-0.10629542917013168, -0.4389042258262634, -0.8256774544715881], "agreement_weight": 0.004081566818058491, "orthogonal_weight": 2.9131782054901123}, {"head": 3, "memory_to_active": 0.023668009787797928, "contribution_to_active": 0.5506722927093506, "mix_mean": 0.4279025197029114, "mix_low": 0.18010234832763672, "mix_high": 0.06357288360595703, "read_cosine": 0.4953003525733948, "beta_mean": 0.06709257513284683, "nonfinite": 0.0, "weighted_memory_to_active": 0.013321725651621819, "mix_std": 0.4269033968448639, "beta_std": 0.0962282121181488, "beta_low": 0.45587158203125, "beta_high": 0.0, "orthogonal_to_active": 0.03674167022109032, "evidence_weights": [-0.08355863392353058, -0.36516669392585754, -0.3420017957687378], "agreement_weight": 0.022281428799033165, "orthogonal_weight": 3.1447558403015137}, {"head": 4, "memory_to_active": 0.6346738338470459, "contribution_to_active": 0.29254603385925293, "mix_mean": 0.2300187051296234, "mix_low": 0.047863006591796875, "mix_high": 0.00017452239990234375, "read_cosine": 0.24499647319316864, "beta_mean": 0.046191051602363586, "nonfinite": 0.0, "weighted_memory_to_active": 0.19622176885604858, "mix_std": 0.2271420657634735, "beta_std": 0.04340410977602005, "beta_low": 0.0965576171875, "beta_high": 0.0, "orthogonal_to_active": 0.094712033867836, "evidence_weights": [-0.12573911249637604, -0.09978074580430984, 1.2085468769073486], "agreement_weight": -0.11068253964185715, "orthogonal_weight": -0.5158687829971313}, {"head": 5, "memory_to_active": 0.1581820547580719, "contribution_to_active": 0.35589683055877686, "mix_mean": 0.43814703822135925, "mix_low": 0.0005245208740234375, "mix_high": 0.000705718994140625, "read_cosine": 0.8920942544937134, "beta_mean": 0.012201343663036823, "nonfinite": 0.0, "weighted_memory_to_active": 0.07040572166442871, "mix_std": 0.2607515752315521, "beta_std": 0.018036462366580963, "beta_low": 0.63427734375, "beta_high": 0.0, "orthogonal_to_active": 0.021381773054599762, "evidence_weights": [-0.19293327629566193, -0.24491316080093384, 0.4600520431995392], "agreement_weight": 0.041029177606105804, "orthogonal_weight": -0.5968164205551147}]}, {"layer": 12, "memory_to_active": 0.4350318908691406, "contribution_to_active": 0.5304198861122131, "mix_mean": 0.4179525375366211, "mix_low": 0.027811529114842415, "mix_high": 0.0023390452843159437, "read_cosine": 0.29257237911224365, "beta_mean": 0.01723681017756462, "nonfinite": 0.0, "weighted_memory_to_active": 0.20842403173446655, "mix_std": 0.2510601282119751, "beta_std": 0.018870271742343903, "beta_low": 0.7932332754135132, "beta_high": 0.0, "orthogonal_to_active": 0.07645092904567719, "output_influence": 0.3821292519569397, "residual_influence": 0.10276944935321808, "state_active": 0.04273981228470802, "state_memory": 0.011888278648257256, "state_peak": 24.894365310668945, "growth_active": 1.0570576190948486, "growth_memory": 1.209063172340393, "heads": [{"head": 0, "memory_to_active": 0.22890113294124603, "contribution_to_active": 0.3695051968097687, "mix_mean": 0.34656965732574463, "mix_low": 0.010173797607421875, "mix_high": 0.0001659393310546875, "read_cosine": 0.31221240758895874, "beta_mean": 0.0035668066702783108, "nonfinite": 0.0, "weighted_memory_to_active": 0.08422365039587021, "mix_std": 0.2531833052635193, "beta_std": 0.012163170613348484, "beta_low": 0.9188232421875, "beta_high": 0.0, "orthogonal_to_active": 0.03945806249976158, "evidence_weights": [-0.15163551270961761, -0.2165893018245697, 0.24017366766929626], "agreement_weight": 0.10654424130916595, "orthogonal_weight": -0.516588568687439}, {"head": 1, "memory_to_active": 0.14741390943527222, "contribution_to_active": 0.4504489302635193, "mix_mean": 0.4519430696964264, "mix_low": 0.000598907470703125, "mix_high": 0.0002231597900390625, "read_cosine": 0.5869652032852173, "beta_mean": 0.0015294349286705256, "nonfinite": 0.0, "weighted_memory_to_active": 0.06872710585594177, "mix_std": 0.26062023639678955, "beta_std": 0.005770943593233824, "beta_low": 0.9783935546875, "beta_high": 0.0, "orthogonal_to_active": 0.04814118891954422, "evidence_weights": [-0.19061625003814697, -0.24441012740135193, 0.7554584741592407], "agreement_weight": 0.10307646542787552, "orthogonal_weight": -0.8865983486175537}, {"head": 2, "memory_to_active": 0.7075843214988708, "contribution_to_active": 0.5006213188171387, "mix_mean": 0.32840484380722046, "mix_low": 0.022792816162109375, "mix_high": 0.0012865066528320312, "read_cosine": 0.429517924785614, "beta_mean": 0.09188475459814072, "nonfinite": 0.0, "weighted_memory_to_active": 0.29959094524383545, "mix_std": 0.2530776560306549, "beta_std": 0.06882689893245697, "beta_low": 0.001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.21160484850406647, "evidence_weights": [-0.23692195117473602, -0.048145607113838196, -0.044397637248039246], "agreement_weight": 0.02952967770397663, "orthogonal_weight": 0.8540621399879456}, {"head": 3, "memory_to_active": 0.03299044072628021, "contribution_to_active": 0.6898672580718994, "mix_mean": 0.6573658585548401, "mix_low": 0.0003986358642578125, "mix_high": 0.009550094604492188, "read_cosine": 0.08046816289424896, "beta_mean": 0.00029066100250929594, "nonfinite": 0.0, "weighted_memory_to_active": 0.022901076823472977, "mix_std": 0.2506362795829773, "beta_std": 0.0018303561955690384, "beta_low": 0.9986572265625, "beta_high": 0.0, "orthogonal_to_active": 0.019966254010796547, "evidence_weights": [-0.3051452934741974, -0.31747758388519287, 0.8966162800788879], "agreement_weight": 0.21050037443637848, "orthogonal_weight": -0.8941016793251038}, {"head": 4, "memory_to_active": 1.4438843727111816, "contribution_to_active": 0.7472245693206787, "mix_mean": 0.16855774819850922, "mix_low": 0.1256694793701172, "mix_high": 0.0010890960693359375, "read_cosine": -0.03437534719705582, "beta_mean": 0.005620857700705528, "nonfinite": 0.0, "weighted_memory_to_active": 0.7502263784408569, "mix_std": 0.21134847402572632, "beta_std": 0.022236311808228493, "beta_low": 0.86798095703125, "beta_high": 0.0, "orthogonal_to_active": 0.12059718370437622, "evidence_weights": [0.0007773010875098407, -0.15263672173023224, -0.02593991905450821], "agreement_weight": 0.001904225442558527, "orthogonal_weight": -0.17714932560920715}, {"head": 5, "memory_to_active": 0.04941713064908981, "contribution_to_active": 0.4248518943786621, "mix_mean": 0.5548741221427917, "mix_low": 0.007235527038574219, "mix_high": 0.0017194747924804688, "read_cosine": 0.3806459903717041, "beta_mean": 0.0005283483769744635, "nonfinite": 0.0, "weighted_memory_to_active": 0.024874981492757797, "mix_std": 0.27749475836753845, "beta_std": 0.002393947681412101, "beta_low": 0.99383544921875, "beta_high": 0.0, "orthogonal_to_active": 0.018938058987259865, "evidence_weights": [-0.26272067427635193, -0.28650981187820435, 0.8078662753105164], "agreement_weight": 0.1008889377117157, "orthogonal_weight": -0.8397084474563599}]}, {"layer": 13, "memory_to_active": 0.19099122285842896, "contribution_to_active": 0.4726918935775757, "mix_mean": 0.4639662802219391, "mix_low": 0.004978816024959087, "mix_high": 0.00183534633833915, "read_cosine": 0.40164849162101746, "beta_mean": 0.018483871594071388, "nonfinite": 0.0, "weighted_memory_to_active": 0.08417508751153946, "mix_std": 0.25733092427253723, "beta_std": 0.017747236415743828, "beta_low": 0.77593994140625, "beta_high": 0.0, "orthogonal_to_active": 0.07328062504529953, "output_influence": 0.4212397038936615, "residual_influence": 0.13954010605812073, "state_active": 0.08079209178686142, "state_memory": 0.03485282137989998, "state_peak": 93.37261962890625, "growth_active": 0.9453604817390442, "growth_memory": 1.0527716875076294, "heads": [{"head": 0, "memory_to_active": 0.1249493956565857, "contribution_to_active": 0.49326786398887634, "mix_mean": 0.4819942116737366, "mix_low": 0.0009698867797851562, "mix_high": 0.0012302398681640625, "read_cosine": 0.3519061803817749, "beta_mean": 0.0007912508444860578, "nonfinite": 0.0, "weighted_memory_to_active": 0.06576713919639587, "mix_std": 0.26356932520866394, "beta_std": 0.001872644992545247, "beta_low": 0.9927978515625, "beta_high": 0.0, "orthogonal_to_active": 0.06342744827270508, "evidence_weights": [-0.20519113540649414, -0.2386045902967453, 0.7562705874443054], "agreement_weight": 0.04175661876797676, "orthogonal_weight": -1.0635653734207153}, {"head": 1, "memory_to_active": 0.046991199254989624, "contribution_to_active": 0.5238258838653564, "mix_mean": 0.4639253616333008, "mix_low": 0.0021810531616210938, "mix_high": 0.000156402587890625, "read_cosine": 0.24748185276985168, "beta_mean": 0.0001891575666377321, "nonfinite": 0.0, "weighted_memory_to_active": 0.024942733347415924, "mix_std": 0.26226696372032166, "beta_std": 0.0004355418204795569, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.023678485304117203, "evidence_weights": [-0.18828509747982025, -0.24686336517333984, 0.7378734946250916], "agreement_weight": 0.13966374099254608, "orthogonal_weight": -1.0096536874771118}, {"head": 2, "memory_to_active": 0.18592700362205505, "contribution_to_active": 0.34882602095603943, "mix_mean": 0.33817753195762634, "mix_low": 0.0026674270629882812, "mix_high": 3.719329833984375e-05, "read_cosine": 0.6369935274124146, "beta_mean": 0.016772817820310593, "nonfinite": 0.0, "weighted_memory_to_active": 0.06990441679954529, "mix_std": 0.21442276239395142, "beta_std": 0.03811147063970566, "beta_low": 0.69000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.03385215625166893, "evidence_weights": [-0.22470824420452118, -0.15942387282848358, 0.49987471103668213], "agreement_weight": 0.04256729036569595, "orthogonal_weight": -0.724364697933197}, {"head": 3, "memory_to_active": 0.16943266987800598, "contribution_to_active": 0.5871329307556152, "mix_mean": 0.5794881582260132, "mix_low": 0.009380340576171875, "mix_high": 0.006855010986328125, "read_cosine": 0.023224923759698868, "beta_mean": 0.0011313771829009056, "nonfinite": 0.0, "weighted_memory_to_active": 0.1221047192811966, "mix_std": 0.2769719362258911, "beta_std": 0.003976565320044756, "beta_low": 0.98809814453125, "beta_high": 0.0, "orthogonal_to_active": 0.1205843910574913, "evidence_weights": [-0.35081103444099426, -0.23174884915351868, 1.8061466217041016], "agreement_weight": 1.7908620834350586, "orthogonal_weight": -1.0381642580032349}, {"head": 4, "memory_to_active": 0.46616122126579285, "contribution_to_active": 0.2890399098396301, "mix_mean": 0.40199095010757446, "mix_low": 0.009454727172851562, "mix_high": 0.0005731582641601562, "read_cosine": 0.8750221729278564, "beta_mean": 0.09084412455558777, "nonfinite": 0.0, "weighted_memory_to_active": 0.1371641308069229, "mix_std": 0.2446225881576538, "beta_std": 0.05719014257192612, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14055395126342773, "evidence_weights": [-0.219217911362648, -0.24253837764263153, 0.48971283435821533], "agreement_weight": -0.004814967047423124, "orthogonal_weight": 1.5597015619277954}, {"head": 5, "memory_to_active": 0.15248581767082214, "contribution_to_active": 0.5940585732460022, "mix_mean": 0.5182213187217712, "mix_low": 0.005219459533691406, "mix_high": 0.0021600723266601562, "read_cosine": 0.2752622663974762, "beta_mean": 0.0011744998628273606, "nonfinite": 0.0, "weighted_memory_to_active": 0.08516736328601837, "mix_std": 0.2821318805217743, "beta_std": 0.004897044040262699, "beta_low": 0.98480224609375, "beta_high": 0.0, "orthogonal_to_active": 0.057587288320064545, "evidence_weights": [-0.21187013387680054, -0.2606877088546753, 0.4883822500705719], "agreement_weight": 0.07252170145511627, "orthogonal_weight": -0.7506541609764099}]}, {"layer": 14, "memory_to_active": 0.2688676416873932, "contribution_to_active": 0.435748815536499, "mix_mean": 0.4028932750225067, "mix_low": 0.02514839358627796, "mix_high": 0.003475507255643606, "read_cosine": 0.6353567838668823, "beta_mean": 0.06493384391069412, "nonfinite": 0.0, "weighted_memory_to_active": 0.12847402691841125, "mix_std": 0.25031059980392456, "beta_std": 0.06153612583875656, "beta_low": 0.4320068657398224, "beta_high": 0.0, "orthogonal_to_active": 0.04232275113463402, "output_influence": 0.2788756489753723, "residual_influence": 0.10726296156644821, "state_active": 0.07245176285505295, "state_memory": 0.01597856730222702, "state_peak": 23.15028190612793, "growth_active": 1.136389970779419, "growth_memory": 1.3295344114303589, "heads": [{"head": 0, "memory_to_active": 0.3315636217594147, "contribution_to_active": 0.2361237108707428, "mix_mean": 0.25197041034698486, "mix_low": 0.013644218444824219, "mix_high": 8.58306884765625e-06, "read_cosine": 0.7405827045440674, "beta_mean": 0.06197698041796684, "nonfinite": 0.0, "weighted_memory_to_active": 0.10545633733272552, "mix_std": 0.22153200209140778, "beta_std": 0.07017543166875839, "beta_low": 0.08233642578125, "beta_high": 0.0, "orthogonal_to_active": 0.03201630339026451, "evidence_weights": [-0.07385113090276718, -0.09333422034978867, 0.7430594563484192], "agreement_weight": 0.04224636033177376, "orthogonal_weight": 0.47128015756607056}, {"head": 1, "memory_to_active": 0.46902307868003845, "contribution_to_active": 0.4278714656829834, "mix_mean": 0.27034175395965576, "mix_low": 0.11301231384277344, "mix_high": 0.00021839141845703125, "read_cosine": 0.6406183242797852, "beta_mean": 0.2681077718734741, "nonfinite": 0.0, "weighted_memory_to_active": 0.24638670682907104, "mix_std": 0.27433615922927856, "beta_std": 0.2462272047996521, "beta_low": 0.09576416015625, "beta_high": 0.0, "orthogonal_to_active": 0.023953411728143692, "evidence_weights": [0.005037079099565744, -0.27571430802345276, -0.10932096093893051], "agreement_weight": 0.058652862906455994, "orthogonal_weight": 0.1710372120141983}, {"head": 2, "memory_to_active": 0.2440096139907837, "contribution_to_active": 0.5045595765113831, "mix_mean": 0.5007466077804565, "mix_low": 0.0046100616455078125, "mix_high": 0.0013217926025390625, "read_cosine": 0.4231603443622589, "beta_mean": 0.0076395501382648945, "nonfinite": 0.0, "weighted_memory_to_active": 0.14119066298007965, "mix_std": 0.2707650363445282, "beta_std": 0.00949910655617714, "beta_low": 0.77032470703125, "beta_high": 0.0, "orthogonal_to_active": 0.10173658281564713, "evidence_weights": [-0.18637582659721375, -0.25634559988975525, 0.7417994737625122], "agreement_weight": 0.08214009553194046, "orthogonal_weight": -0.8295415043830872}, {"head": 3, "memory_to_active": 0.06379754096269608, "contribution_to_active": 0.6776605844497681, "mix_mean": 0.662774920463562, "mix_low": 0.0026655197143554688, "mix_high": 0.01386260986328125, "read_cosine": 0.5035384893417358, "beta_mean": 0.009226254187524319, "nonfinite": 0.0, "weighted_memory_to_active": 0.044872887432575226, "mix_std": 0.24444670975208282, "beta_std": 0.00805654563009739, "beta_low": 0.666015625, "beta_high": 0.0, "orthogonal_to_active": 0.02148965746164322, "evidence_weights": [-0.38340404629707336, -0.33021602034568787, 0.45844554901123047], "agreement_weight": 0.0808010995388031, "orthogonal_weight": -0.635653018951416}, {"head": 4, "memory_to_active": 0.18143895268440247, "contribution_to_active": 0.486137330532074, "mix_mean": 0.454519659280777, "mix_low": 0.004059791564941406, "mix_high": 0.00499725341796875, "read_cosine": 0.7921665906906128, "beta_mean": 0.03978797793388367, "nonfinite": 0.0, "weighted_memory_to_active": 0.10158760100603104, "mix_std": 0.25912946462631226, "beta_std": 0.026352250948548317, "beta_low": 0.032958984375, "beta_high": 0.0, "orthogonal_to_active": 0.027486082166433334, "evidence_weights": [-0.22140318155288696, -0.27045324444770813, 0.3597105145454407], "agreement_weight": 0.11029448360204697, "orthogonal_weight": -0.4809989035129547}, {"head": 5, "memory_to_active": 0.323373019695282, "contribution_to_active": 0.28214031457901, "mix_mean": 0.2770063579082489, "mix_low": 0.012898445129394531, "mix_high": 0.0004444122314453125, "read_cosine": 0.7120741605758667, "beta_mean": 0.0028645070269703865, "nonfinite": 0.0, "weighted_memory_to_active": 0.13134998083114624, "mix_std": 0.2316541224718094, "beta_std": 0.008906194940209389, "beta_low": 0.94464111328125, "beta_high": 0.0, "orthogonal_to_active": 0.04725445806980133, "evidence_weights": [-0.18503020703792572, -0.1423090547323227, 0.16368302702903748], "agreement_weight": 0.0927065908908844, "orthogonal_weight": -0.5838595628738403}]}, {"layer": 15, "memory_to_active": 0.07574436813592911, "contribution_to_active": 0.47229233384132385, "mix_mean": 0.3825892508029938, "mix_low": 0.007564862724393606, "mix_high": 0.0024855933152139187, "read_cosine": 0.6605287790298462, "beta_mean": 0.04157772660255432, "nonfinite": 0.0, "weighted_memory_to_active": 0.03387345373630524, "mix_std": 0.2737674415111542, "beta_std": 0.04544825106859207, "beta_low": 0.7110698223114014, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.006042451597750187, "output_influence": 0.30220291018486023, "residual_influence": 0.15803253650665283, "state_active": 0.11314046382904053, "state_memory": 0.008560825139284134, "state_peak": 33.51646423339844, "growth_active": 1.3279831409454346, "growth_memory": 1.6786128282546997, "heads": [{"head": 0, "memory_to_active": 0.10298215597867966, "contribution_to_active": 0.4754147529602051, "mix_mean": 0.3465757966041565, "mix_low": 0.0057849884033203125, "mix_high": 0.0027027130126953125, "read_cosine": 0.6697217226028442, "beta_mean": 0.03772864118218422, "nonfinite": 0.0, "weighted_memory_to_active": 0.048370588570833206, "mix_std": 0.2626882493495941, "beta_std": 0.02902003563940525, "beta_low": 0.13531494140625, "beta_high": 0.0, "orthogonal_to_active": 0.014997793361544609, "evidence_weights": [-0.16457746922969818, -0.19466882944107056, 0.345802366733551], "agreement_weight": 0.0822393074631691, "orthogonal_weight": -0.5419498682022095}, {"head": 1, "memory_to_active": 0.18998463451862335, "contribution_to_active": 0.402907133102417, "mix_mean": 0.3855583667755127, "mix_low": 0.005518913269042969, "mix_high": 0.005059242248535156, "read_cosine": 0.8554723858833313, "beta_mean": 0.0018836336676031351, "nonfinite": 0.0, "weighted_memory_to_active": 0.07248605787754059, "mix_std": 0.2750566899776459, "beta_std": 0.0043998719193041325, "beta_low": 0.96844482421875, "beta_high": 0.0, "orthogonal_to_active": 0.0038049903232604265, "evidence_weights": [-0.18787088990211487, -0.19334067404270172, 0.2886217534542084], "agreement_weight": 0.09491803497076035, "orthogonal_weight": -0.07957947254180908}, {"head": 2, "memory_to_active": 0.007831070572137833, "contribution_to_active": 0.4316304624080658, "mix_mean": 0.3394920825958252, "mix_low": 0.012804031372070312, "mix_high": 0.00015544891357421875, "read_cosine": 0.764563798904419, "beta_mean": 0.00013944751117378473, "nonfinite": 0.0, "weighted_memory_to_active": 0.003204419044777751, "mix_std": 0.2681507170200348, "beta_std": 0.0003141058550681919, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.00041186160524375737, "evidence_weights": [-0.20568254590034485, -0.2716522812843323, 0.3027718961238861], "agreement_weight": 0.04211953282356262, "orthogonal_weight": -0.23026536405086517}, {"head": 3, "memory_to_active": 0.041612472385168076, "contribution_to_active": 0.4637947678565979, "mix_mean": 0.31122756004333496, "mix_low": 0.0029392242431640625, "mix_high": 3.62396240234375e-05, "read_cosine": 0.654725968837738, "beta_mean": 0.0006939024897292256, "nonfinite": 0.0, "weighted_memory_to_active": 0.019350284710526466, "mix_std": 0.24678178131580353, "beta_std": 0.001540444791316986, "beta_low": 0.994384765625, "beta_high": 0.0, "orthogonal_to_active": 0.011104646138846874, "evidence_weights": [-0.18529485166072845, -0.23006820678710938, 0.4393502175807953], "agreement_weight": 0.00852230191230774, "orthogonal_weight": -0.8407846093177795}, {"head": 4, "memory_to_active": 0.10549520701169968, "contribution_to_active": 0.5866116285324097, "mix_mean": 0.39879119396209717, "mix_low": 0.01299285888671875, "mix_high": 0.000789642333984375, "read_cosine": 0.49579378962516785, "beta_mean": 0.2088356912136078, "nonfinite": 0.0, "weighted_memory_to_active": 0.0570717453956604, "mix_std": 0.30660107731819153, "beta_std": 0.2370566725730896, "beta_low": 0.16839599609375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.0037604430690407753, "evidence_weights": [-0.24971795082092285, -0.25532761216163635, 0.15783818066120148], "agreement_weight": 0.01133356336504221, "orthogonal_weight": -0.08904688060283661}, {"head": 5, "memory_to_active": 0.006560631562024355, "contribution_to_active": 0.47339513897895813, "mix_mean": 0.5138903856277466, "mix_low": 0.005349159240722656, "mix_high": 0.0061702728271484375, "read_cosine": 0.5228949785232544, "beta_mean": 0.00018504884792491794, "nonfinite": 0.0, "weighted_memory_to_active": 0.002757615176960826, "mix_std": 0.28332608938217163, "beta_std": 0.0003583274665288627, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002174972090870142, "evidence_weights": [-0.35721468925476074, -0.2974756956100464, 0.3652190864086151], "agreement_weight": 0.1415243297815323, "orthogonal_weight": 0.9251235723495483}]}]}, {"step": 19000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8933908389990393, "memory_to_active": 0.9484463334083557, "contribution_to_active": 0.4640418589115143, "mix_mean": 0.3214016556739807, "mix_low": 0.1667865514755249, "mix_high": 0.0061056120321154594, "read_cosine": 0.39423060417175293, "beta_mean": 0.0871034562587738, "nonfinite": 0.0, "weighted_memory_to_active": 0.2039511799812317, "mix_std": 0.21129345893859863, "beta_std": 0.07158082723617554, "beta_low": 0.4723052978515625, "beta_high": 1.52587890625e-05, "orthogonal_to_active": 0.043487146496772766, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1931233406066895, "normal_token_accuracy": 0.42380422353744507, "state_active": 0.059416063129901886, "state_memory": 0.02160402573645115, "state_peak": 111.58720397949219, "growth_active": 1.1379952430725098, "growth_memory": 1.3727951049804688, "output_influence": 0.3237469792366028, "residual_influence": 0.10224208235740662, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.224900722503662, "no_memory_loss_delta": 0.2720737159252167, "scrambled_loss_delta": 0.22441835701465607, "memory_only_loss_delta": 2.1495959758758545, "no_evidence_loss_delta": 0.16353009641170502, "no_agreement_loss_delta": 0.012426519766449928, "no_orthogonal_loss_delta": 0.013035094365477562}, {"index": 94, "tokens": 256, "normal_loss": 3.2793240547180176, "no_memory_loss_delta": 0.299832820892334, "scrambled_loss_delta": 0.3346395194530487, "memory_only_loss_delta": 2.463418960571289, "no_evidence_loss_delta": 0.14851343631744385, "no_agreement_loss_delta": 0.026812074705958366, "no_orthogonal_loss_delta": 0.003580664750188589}, {"index": 188, "tokens": 256, "normal_loss": 3.596040725708008, "no_memory_loss_delta": 0.2177639752626419, "scrambled_loss_delta": 0.09188878536224365, "memory_only_loss_delta": 2.2408885955810547, "no_evidence_loss_delta": 0.13191388547420502, "no_agreement_loss_delta": 0.006950491573661566, "no_orthogonal_loss_delta": 0.014232498593628407}, {"index": 283, "tokens": 131, "normal_loss": 2.175189733505249, "no_memory_loss_delta": 0.03267265483736992, "scrambled_loss_delta": 0.21959927678108215, "memory_only_loss_delta": 2.466860771179199, "no_evidence_loss_delta": -0.030886882916092873, "no_agreement_loss_delta": -0.03281869366765022, "no_orthogonal_loss_delta": -0.017708688974380493}], "agreement_weight_abs_mean": 0.10000551491975784, "agreement_weight_abs_max": 1.9242650270462036, "agreement_weight_mean": 0.08407049626111984, "orthogonal_weight_abs_mean": 0.531144917011261, "orthogonal_weight_abs_max": 3.3267362117767334, "orthogonal_weight_mean": -0.17266792058944702, "no_memory_loss": 3.4227516651153564, "no_memory_loss_delta": 0.22962820529937744, "no_memory_token_accuracy_gain": 0.021134594455361366, "no_memory_token_win_fraction": 0.6051167845726013, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 20.517093919227626, "scrambled_loss": 3.410487174987793, "scrambled_loss_delta": 0.21736356616020203, "scrambled_token_accuracy_gain": 0.0055617354810237885, "scrambled_token_win_fraction": 0.6307007670402527, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.5362622801653, "memory_only_loss": 5.504311561584473, "memory_only_loss_delta": 2.311187982559204, "memory_only_token_accuracy_gain": 0.24694104492664337, "memory_only_token_win_fraction": 0.8654060363769531, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 90.08565990599115, "no_evidence_loss": 3.3150441646575928, "no_evidence_loss_delta": 0.12192094326019287, "no_evidence_token_accuracy_gain": 0.0011123470030725002, "no_evidence_token_win_fraction": 0.5472747683525085, "no_evidence_probe_win_fraction": 0.75, "no_evidence_ppl_reduction_pct": 11.478165178696607, "no_agreement_loss": 3.201493740081787, "no_agreement_loss_delta": 0.008370587602257729, "no_agreement_token_accuracy_gain": 0.0011123470030725002, "no_agreement_token_win_fraction": 0.45939934253692627, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 0.8335651779599758, "no_orthogonal_loss": 3.1993274688720703, "no_orthogonal_loss_delta": 0.006203911267220974, "no_orthogonal_token_accuracy_gain": -0.0033370412420481443, "no_orthogonal_token_win_fraction": 0.5628476142883301, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.6184706744623453, "short_no_memory_loss_delta": 0.18479377031326294, "long_context_memory_gain": 0.0448344349861145, "normal_context_loss_gain": 0.12817251682281494, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.0516035556793213, "contribution_to_active": 0.8438066840171814, "mix_mean": 0.09388743340969086, "mix_low": 0.49612367153167725, "mix_high": 0.0036392214242368937, "read_cosine": 0.47061124444007874, "beta_mean": 0.2663889527320862, "nonfinite": 0.0, "weighted_memory_to_active": 0.9017066955566406, "mix_std": 0.08427239209413528, "beta_std": 0.1563262641429901, "beta_low": 0.0018717448692768812, "beta_high": 0.0, "orthogonal_to_active": 0.052024051547050476, "output_influence": 0.1286415159702301, "residual_influence": 0.17383751273155212, "state_active": 0.07786691188812256, "state_memory": 0.05024281144142151, "state_peak": 51.844757080078125, "growth_active": 1.2428356409072876, "growth_memory": 1.2783044576644897, "heads": [{"head": 0, "memory_to_active": 2.3176777362823486, "contribution_to_active": 0.35607147216796875, "mix_mean": 0.019839303568005562, "mix_low": 0.556117057800293, "mix_high": 0.0, "read_cosine": 0.2725333273410797, "beta_mean": 0.2230556458234787, "nonfinite": 0.0, "weighted_memory_to_active": 0.43583545088768005, "mix_std": 0.05263025313615799, "beta_std": 0.10156641900539398, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03741057962179184, "evidence_weights": [-0.03593974933028221, -0.07358089834451675, -0.06864342838525772], "agreement_weight": -0.09345453977584839, "orthogonal_weight": -0.09777341037988663}, {"head": 1, "memory_to_active": 4.362993240356445, "contribution_to_active": 1.4997639656066895, "mix_mean": 0.016510076820850372, "mix_low": 0.5935697555541992, "mix_high": 0.0, "read_cosine": 0.5148628950119019, "beta_mean": 0.38156643509864807, "nonfinite": 0.0, "weighted_memory_to_active": 1.852311372756958, "mix_std": 0.04428525269031525, "beta_std": 0.171608105301857, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.13623939454555511, "evidence_weights": [0.018577564507722855, -0.045332279056310654, 0.20456139743328094], "agreement_weight": 0.017185332253575325, "orthogonal_weight": -0.1896544247865677}, {"head": 2, "memory_to_active": 2.582035779953003, "contribution_to_active": 0.5106207132339478, "mix_mean": 0.01494666188955307, "mix_low": 0.634648323059082, "mix_high": 0.0, "read_cosine": 0.35588812828063965, "beta_mean": 0.3274646997451782, "nonfinite": 0.0, "weighted_memory_to_active": 0.5937914252281189, "mix_std": 0.0314427874982357, "beta_std": 0.23082099854946136, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06475576013326645, "evidence_weights": [0.042538393288850784, 0.048692602664232254, 0.011015824042260647], "agreement_weight": -0.003497013822197914, "orthogonal_weight": -0.18579256534576416}, {"head": 3, "memory_to_active": 5.677544593811035, "contribution_to_active": 2.2915313243865967, "mix_mean": 0.04391022026538849, "mix_low": 0.3440895080566406, "mix_high": 0.0, "read_cosine": 0.39450323581695557, "beta_mean": 0.34777045249938965, "nonfinite": 0.0, "weighted_memory_to_active": 2.4192559719085693, "mix_std": 0.08523549139499664, "beta_std": 0.24439159035682678, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05695188790559769, "evidence_weights": [-0.08478965610265732, -0.10959524661302567, -0.07394640892744064], "agreement_weight": -0.009064625017344952, "orthogonal_weight": -0.030353493988513947}, {"head": 4, "memory_to_active": 3.233799695968628, "contribution_to_active": 0.060807034373283386, "mix_mean": 0.0063150860369205475, "mix_low": 0.8227062225341797, "mix_high": 0.0, "read_cosine": 0.33062028884887695, "beta_mean": 0.26486021280288696, "nonfinite": 0.0, "weighted_memory_to_active": 0.05714473873376846, "mix_std": 0.011188376694917679, "beta_std": 0.1439269781112671, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006669710390269756, "evidence_weights": [0.0872846245765686, 0.03705470636487007, -0.33791589736938477], "agreement_weight": -0.04146326333284378, "orthogonal_weight": 0.13114650547504425}, {"head": 5, "memory_to_active": 0.1355685591697693, "contribution_to_active": 0.34404540061950684, "mix_mean": 0.4618031978607178, "mix_low": 0.025610923767089844, "mix_high": 0.0218353271484375, "read_cosine": 0.9552595615386963, "beta_mean": 0.05361633002758026, "nonfinite": 0.0, "weighted_memory_to_active": 0.05190115049481392, "mix_std": 0.28085219860076904, "beta_std": 0.045643534511327744, "beta_low": 0.01123046875, "beta_high": 0.0, "orthogonal_to_active": 0.010116982273757458, "evidence_weights": [-0.2793404459953308, -0.26026755571365356, 0.2451174259185791], "agreement_weight": 0.1827959418296814, "orthogonal_weight": -0.44511863589286804}]}, {"layer": 1, "memory_to_active": 2.3769092559814453, "contribution_to_active": 0.4206109941005707, "mix_mean": 0.13736256957054138, "mix_low": 0.39366215467453003, "mix_high": 0.0007473628502339125, "read_cosine": 0.33109188079833984, "beta_mean": 0.22012175619602203, "nonfinite": 0.0, "weighted_memory_to_active": 0.34429532289505005, "mix_std": 0.11746597290039062, "beta_std": 0.1411600410938263, "beta_low": 0.1703593134880066, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.0415726974606514, "output_influence": 0.15622079372406006, "residual_influence": 0.03702523931860924, "state_active": 0.03923911601305008, "state_memory": 0.04193073883652687, "state_peak": 32.0347785949707, "growth_active": 1.0677051544189453, "growth_memory": 1.4911903142929077, "heads": [{"head": 0, "memory_to_active": 0.06914370507001877, "contribution_to_active": 0.3715537190437317, "mix_mean": 0.3259059190750122, "mix_low": 0.0222930908203125, "mix_high": 0.0001392364501953125, "read_cosine": -0.0810537040233612, "beta_mean": 0.00037757051177322865, "nonfinite": 0.0, "weighted_memory_to_active": 0.03199588507413864, "mix_std": 0.27630043029785156, "beta_std": 0.0007509029237553477, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.02247271127998829, "evidence_weights": [-0.15386436879634857, -0.18976151943206787, -0.047064270824193954], "agreement_weight": 0.13001783192157745, "orthogonal_weight": -0.7851284146308899}, {"head": 1, "memory_to_active": 3.5766091346740723, "contribution_to_active": 0.19523867964744568, "mix_mean": 0.020397959277033806, "mix_low": 0.5044193267822266, "mix_high": 0.0, "read_cosine": 0.4869210422039032, "beta_mean": 0.39905789494514465, "nonfinite": 0.0, "weighted_memory_to_active": 0.211788147687912, "mix_std": 0.04297737032175064, "beta_std": 0.24214568734169006, "beta_low": 0.0, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.008362717926502228, "evidence_weights": [0.0005390027654357255, 0.0038106366991996765, -0.016992997378110886], "agreement_weight": -0.008802661672234535, "orthogonal_weight": 0.0765129029750824}, {"head": 2, "memory_to_active": 0.5680661797523499, "contribution_to_active": 0.443708598613739, "mix_mean": 0.4199365973472595, "mix_low": 0.0035848617553710938, "mix_high": 0.004336357116699219, "read_cosine": 0.7818188071250916, "beta_mean": 0.059229526668787, "nonfinite": 0.0, "weighted_memory_to_active": 0.45381295680999756, "mix_std": 0.263935923576355, "beta_std": 0.03267611563205719, "beta_low": 0.00579833984375, "beta_high": 0.0, "orthogonal_to_active": 0.018194513395428658, "evidence_weights": [-0.24254029989242554, -0.39126238226890564, 0.2234048694372177], "agreement_weight": 0.016264937818050385, "orthogonal_weight": -0.0797000303864479}, {"head": 3, "memory_to_active": 3.0702481269836426, "contribution_to_active": 0.07024327665567398, "mix_mean": 0.011595344170928001, "mix_low": 0.7479381561279297, "mix_high": 0.0, "read_cosine": 0.2107737958431244, "beta_mean": 0.39336419105529785, "nonfinite": 0.0, "weighted_memory_to_active": 0.09560713917016983, "mix_std": 0.03324448689818382, "beta_std": 0.24485011398792267, "beta_low": 0.00341796875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.02336648851633072, "evidence_weights": [0.0442548505961895, 0.015015403740108013, 0.05893203243613243], "agreement_weight": -0.018227674067020416, "orthogonal_weight": -0.26222795248031616}, {"head": 4, "memory_to_active": 3.123753309249878, "contribution_to_active": 1.3942173719406128, "mix_mean": 0.03856215626001358, "mix_low": 0.27610015869140625, "mix_high": 8.58306884765625e-06, "read_cosine": 0.38235554099082947, "beta_mean": 0.17424297332763672, "nonfinite": 0.0, "weighted_memory_to_active": 1.2270106077194214, "mix_std": 0.07191067188978195, "beta_std": 0.0978768914937973, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.17563383281230927, "evidence_weights": [-0.026047848165035248, -0.010044111870229244, 0.07997077703475952], "agreement_weight": -0.027103472501039505, "orthogonal_weight": 0.1691163331270218}, {"head": 5, "memory_to_active": 3.853635311126709, "contribution_to_active": 0.04870428144931793, "mix_mean": 0.0077774859964847565, "mix_low": 0.8076372146606445, "mix_high": 0.0, "read_cosine": 0.20573563873767853, "beta_mean": 0.2944582998752594, "nonfinite": 0.0, "weighted_memory_to_active": 0.04555704444646835, "mix_std": 0.016427000984549522, "beta_std": 0.2286604344844818, "beta_low": 0.0133056640625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0014059219975024462, "evidence_weights": [0.13830246031284332, 0.06602302938699722, 0.34628087282180786], "agreement_weight": -0.13467253744602203, "orthogonal_weight": 0.030867815017700195}]}, {"layer": 2, "memory_to_active": 1.1513776779174805, "contribution_to_active": 0.30187734961509705, "mix_mean": 0.2789945900440216, "mix_low": 0.27252501249313354, "mix_high": 0.002379894256591797, "read_cosine": 0.3352086544036865, "beta_mean": 0.1049424260854721, "nonfinite": 0.0, "weighted_memory_to_active": 0.07517440617084503, "mix_std": 0.19311630725860596, "beta_std": 0.07623881101608276, "beta_low": 0.5371907949447632, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.018532251939177513, "output_influence": 0.25000709295272827, "residual_influence": 0.05820874124765396, "state_active": 0.03489541634917259, "state_memory": 0.02333822473883629, "state_peak": 48.278282165527344, "growth_active": 0.9626195430755615, "growth_memory": 1.5688575506210327, "heads": [{"head": 0, "memory_to_active": 0.01531665213406086, "contribution_to_active": 0.20093205571174622, "mix_mean": 0.3681686520576477, "mix_low": 0.010164260864257812, "mix_high": 0.0023097991943359375, "read_cosine": 0.23109745979309082, "beta_mean": 0.00011211472883587703, "nonfinite": 0.0, "weighted_memory_to_active": 0.006918803788721561, "mix_std": 0.29050153493881226, "beta_std": 0.00016727471665944904, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00019678298849612474, "evidence_weights": [-0.22120381891727448, -0.14505517482757568, 0.2582648694515228], "agreement_weight": 0.16007883846759796, "orthogonal_weight": -0.03194349259138107}, {"head": 1, "memory_to_active": 3.014403820037842, "contribution_to_active": 0.05195870250463486, "mix_mean": 0.005454306956380606, "mix_low": 0.8884983062744141, "mix_high": 1.9073486328125e-06, "read_cosine": 0.19212910532951355, "beta_mean": 0.2780756950378418, "nonfinite": 0.0, "weighted_memory_to_active": 0.03867817297577858, "mix_std": 0.014321437105536461, "beta_std": 0.1505233496427536, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013157455250620842, "evidence_weights": [0.0944972038269043, 0.10412860661745071, 0.3182486891746521], "agreement_weight": 0.031695496290922165, "orthogonal_weight": 0.3621588349342346}, {"head": 2, "memory_to_active": 0.032512303441762924, "contribution_to_active": 0.4478323757648468, "mix_mean": 0.40717118978500366, "mix_low": 0.0035142898559570312, "mix_high": 0.0019426345825195312, "read_cosine": 0.223405122756958, "beta_mean": 5.570750727201812e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014938978478312492, "mix_std": 0.27560001611709595, "beta_std": 0.00010226698213955387, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0033054081723093987, "evidence_weights": [-0.19882214069366455, -0.2389378547668457, 0.5210911631584167], "agreement_weight": 0.030276382341980934, "orthogonal_weight": -0.2851135730743408}, {"head": 3, "memory_to_active": 0.3088000416755676, "contribution_to_active": 0.4205586910247803, "mix_mean": 0.3565371036529541, "mix_low": 0.0037889480590820312, "mix_high": 0.0043792724609375, "read_cosine": 0.6632737517356873, "beta_mean": 0.05578434094786644, "nonfinite": 0.0, "weighted_memory_to_active": 0.14521019160747528, "mix_std": 0.25674688816070557, "beta_std": 0.07407085597515106, "beta_low": 0.19476318359375, "beta_high": 0.0, "orthogonal_to_active": 0.07740828394889832, "evidence_weights": [-0.14068610966205597, -0.2126472145318985, 0.7395774126052856], "agreement_weight": 0.06871077418327332, "orthogonal_weight": -0.8267839550971985}, {"head": 4, "memory_to_active": 3.508575916290283, "contribution_to_active": 0.19256357848644257, "mix_mean": 0.012616841122508049, "mix_low": 0.7267341613769531, "mix_high": 0.0, "read_cosine": 0.1926993429660797, "beta_mean": 0.2955530285835266, "nonfinite": 0.0, "weighted_memory_to_active": 0.2285614311695099, "mix_std": 0.03662693873047829, "beta_std": 0.23246735334396362, "beta_low": 0.02838134765625, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.01599770039319992, "evidence_weights": [0.05027775093913078, 0.04473438858985901, 0.15597976744174957], "agreement_weight": 0.024993250146508217, "orthogonal_weight": -0.08007507771253586}, {"head": 5, "memory_to_active": 0.02865629829466343, "contribution_to_active": 0.49741852283477783, "mix_mean": 0.5240192413330078, "mix_low": 0.0024499893188476562, "mix_high": 0.005645751953125, "read_cosine": 0.5086469054222107, "beta_mean": 7.365673081949353e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.016738882288336754, "mix_std": 0.28490111231803894, "beta_std": 0.00010171406756853685, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001127876341342926, "evidence_weights": [-0.24031321704387665, -0.259526789188385, 0.7014225125312805], "agreement_weight": 0.04822469875216484, "orthogonal_weight": 0.1252823770046234}]}, {"layer": 3, "memory_to_active": 1.2269116640090942, "contribution_to_active": 0.3458477854728699, "mix_mean": 0.22543933987617493, "mix_low": 0.2244909703731537, "mix_high": 0.002735455986112356, "read_cosine": 0.2581287622451782, "beta_mean": 0.09024911373853683, "nonfinite": 0.0, "weighted_memory_to_active": 0.16013452410697937, "mix_std": 0.21073018014431, "beta_std": 0.07249931246042252, "beta_low": 0.4249064326286316, "beta_high": 0.0, "orthogonal_to_active": 0.04399929568171501, "output_influence": 0.24841855466365814, "residual_influence": 0.04785507172346115, "state_active": 0.031023845076560974, "state_memory": 0.01880030333995819, "state_peak": 19.554641723632812, "growth_active": 1.1352518796920776, "growth_memory": 1.692955732345581, "heads": [{"head": 0, "memory_to_active": 0.6485908031463623, "contribution_to_active": 0.40170133113861084, "mix_mean": 0.2856408655643463, "mix_low": 0.08408451080322266, "mix_high": 0.0022840499877929688, "read_cosine": 0.19602778553962708, "beta_mean": 0.014176994562149048, "nonfinite": 0.0, "weighted_memory_to_active": 0.22353190183639526, "mix_std": 0.2979682683944702, "beta_std": 0.012897305190563202, "beta_low": 0.4832763671875, "beta_high": 0.0, "orthogonal_to_active": 0.11257006227970123, "evidence_weights": [-0.10097169131040573, -0.16664886474609375, 0.15233711898326874], "agreement_weight": 0.12076855450868607, "orthogonal_weight": -0.5289809107780457}, {"head": 1, "memory_to_active": 0.03779887035489082, "contribution_to_active": 0.27461421489715576, "mix_mean": 0.28459432721138, "mix_low": 0.04349327087402344, "mix_high": 0.001251220703125, "read_cosine": 0.1991880238056183, "beta_mean": 0.00016483390936627984, "nonfinite": 0.0, "weighted_memory_to_active": 0.011447958648204803, "mix_std": 0.27387407422065735, "beta_std": 0.0003747029113583267, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0012913000537082553, "evidence_weights": [-0.16561970114707947, -0.1349811851978302, 0.3577403128147125], "agreement_weight": 0.1652957946062088, "orthogonal_weight": -0.12307923287153244}, {"head": 2, "memory_to_active": 3.1804637908935547, "contribution_to_active": 0.3098812699317932, "mix_mean": 0.04008465260267258, "mix_low": 0.3973217010498047, "mix_high": 0.0, "read_cosine": 0.16242742538452148, "beta_mean": 0.23586684465408325, "nonfinite": 0.0, "weighted_memory_to_active": 0.39141935110092163, "mix_std": 0.0905960202217102, "beta_std": 0.1593906730413437, "beta_low": 0.004150390625, "beta_high": 0.0, "orthogonal_to_active": 0.020132288336753845, "evidence_weights": [-0.049606114625930786, 0.013551855459809303, 0.03888870030641556], "agreement_weight": -0.024307791143655777, "orthogonal_weight": -0.06565974652767181}, {"head": 3, "memory_to_active": 0.030321504920721054, "contribution_to_active": 0.46795183420181274, "mix_mean": 0.4633150100708008, "mix_low": 0.023090362548828125, "mix_high": 0.008029937744140625, "read_cosine": 0.304263174533844, "beta_mean": 0.00012081722525181249, "nonfinite": 0.0, "weighted_memory_to_active": 0.016954973340034485, "mix_std": 0.31703925132751465, "beta_std": 0.00025795979308895767, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0048986682668328285, "evidence_weights": [-0.18652425706386566, -0.20801596343517303, 1.401032567024231], "agreement_weight": 0.8030511736869812, "orthogonal_weight": -0.38499027490615845}, {"head": 4, "memory_to_active": 3.056539297103882, "contribution_to_active": 0.049575112760066986, "mix_mean": 0.010450929403305054, "mix_low": 0.7562551498413086, "mix_high": 0.0, "read_cosine": 0.19297118484973907, "beta_mean": 0.20006480813026428, "nonfinite": 0.0, "weighted_memory_to_active": 0.08815870434045792, "mix_std": 0.026088014245033264, "beta_std": 0.1656484305858612, "beta_low": 0.00555419921875, "beta_high": 0.0, "orthogonal_to_active": 0.03996892645955086, "evidence_weights": [0.1464652717113495, 0.11340969055891037, -0.043834298849105835], "agreement_weight": -0.05288443714380264, "orthogonal_weight": -0.46611538529396057}, {"head": 5, "memory_to_active": 0.4077560007572174, "contribution_to_active": 0.5713628530502319, "mix_mean": 0.26855015754699707, "mix_low": 0.042700767517089844, "mix_high": 0.004847526550292969, "read_cosine": 0.4938947856426239, "beta_mean": 0.09110032021999359, "nonfinite": 0.0, "weighted_memory_to_active": 0.2292942851781845, "mix_std": 0.2588154375553131, "beta_std": 0.09642680734395981, "beta_low": 0.0565185546875, "beta_high": 0.0, "orthogonal_to_active": 0.08513452857732773, "evidence_weights": [-0.12067992985248566, -0.1873059719800949, 0.2914734184741974], "agreement_weight": 0.1600901335477829, "orthogonal_weight": -0.5412507057189941}]}, {"layer": 4, "memory_to_active": 1.2127289772033691, "contribution_to_active": 0.4132031798362732, "mix_mean": 0.3009220361709595, "mix_low": 0.2740628123283386, "mix_high": 0.003795306198298931, "read_cosine": 0.41379839181900024, "beta_mean": 0.12099431455135345, "nonfinite": 0.0, "weighted_memory_to_active": 0.17768588662147522, "mix_std": 0.1601671427488327, "beta_std": 0.10187237709760666, "beta_low": 0.3572896420955658, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.04838881641626358, "output_influence": 0.28872406482696533, "residual_influence": 0.07032737880945206, "state_active": 0.043226830661296844, "state_memory": 0.025361260399222374, "state_peak": 52.04909896850586, "growth_active": 1.1728802919387817, "growth_memory": 1.5632842779159546, "heads": [{"head": 0, "memory_to_active": 0.3159785866737366, "contribution_to_active": 0.6156318783760071, "mix_mean": 0.5714647769927979, "mix_low": 0.009815216064453125, "mix_high": 0.009479522705078125, "read_cosine": 0.5792680978775024, "beta_mean": 0.220433309674263, "nonfinite": 0.0, "weighted_memory_to_active": 0.23290185630321503, "mix_std": 0.3242988586425781, "beta_std": 0.2389247566461563, "beta_low": 0.18829345703125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.03780980780720711, "evidence_weights": [-0.23569215834140778, -0.30625349283218384, 0.42423930764198303], "agreement_weight": 0.04823462665081024, "orthogonal_weight": 0.21962766349315643}, {"head": 1, "memory_to_active": 0.16041512787342072, "contribution_to_active": 0.4654161036014557, "mix_mean": 0.49247968196868896, "mix_low": 0.0007753372192382812, "mix_high": 0.0024890899658203125, "read_cosine": 0.7350553870201111, "beta_mean": 0.004547827411442995, "nonfinite": 0.0, "weighted_memory_to_active": 0.08608333766460419, "mix_std": 0.2687467336654663, "beta_std": 0.007087268400937319, "beta_low": 0.90704345703125, "beta_high": 0.0, "orthogonal_to_active": 0.03831081837415695, "evidence_weights": [-0.229167178273201, -0.22768908739089966, 0.6549580693244934], "agreement_weight": 0.04647361487150192, "orthogonal_weight": -0.6864188313484192}, {"head": 2, "memory_to_active": 2.5132322311401367, "contribution_to_active": 0.1314081847667694, "mix_mean": 0.01551220566034317, "mix_low": 0.6059103012084961, "mix_high": 0.0, "read_cosine": 0.31163668632507324, "beta_mean": 0.12334530055522919, "nonfinite": 0.0, "weighted_memory_to_active": 0.167868509888649, "mix_std": 0.026968006044626236, "beta_std": 0.08653523027896881, "beta_low": 0.00213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.008059075102210045, "evidence_weights": [0.04596179723739624, 0.0238727405667305, -0.040781959891319275], "agreement_weight": 0.07802849262952805, "orthogonal_weight": -0.06930920481681824}, {"head": 3, "memory_to_active": 1.719138741493225, "contribution_to_active": 0.1413167268037796, "mix_mean": 0.03344547748565674, "mix_low": 0.38284873962402344, "mix_high": 0.0, "read_cosine": 0.20787066221237183, "beta_mean": 0.1438295990228653, "nonfinite": 0.0, "weighted_memory_to_active": 0.20931380987167358, "mix_std": 0.06230970472097397, "beta_std": 0.13159790635108948, "beta_low": 0.0462646484375, "beta_high": 0.0, "orthogonal_to_active": 0.02472717873752117, "evidence_weights": [-0.02211102843284607, 0.049381889402866364, -0.2222897857427597], "agreement_weight": 0.05571889504790306, "orthogonal_weight": -0.15909625589847565}, {"head": 4, "memory_to_active": 0.02005251683294773, "contribution_to_active": 0.6271281242370605, "mix_mean": 0.6784766912460327, "mix_low": 0.0002918243408203125, "mix_high": 0.010797500610351562, "read_cosine": 0.29352259635925293, "beta_mean": 3.913840191671625e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014546794816851616, "mix_std": 0.241791769862175, "beta_std": 9.240736835636199e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001569140236824751, "evidence_weights": [-0.2925530970096588, -0.3345470726490021, 0.7792039513587952], "agreement_weight": 0.0783795714378357, "orthogonal_weight": -0.12672442197799683}, {"head": 5, "memory_to_active": 2.5475568771362305, "contribution_to_active": 0.49831801652908325, "mix_mean": 0.014153281226754189, "mix_low": 0.6447353363037109, "mix_high": 5.7220458984375e-06, "read_cosine": 0.35543686151504517, "beta_mean": 0.23377065360546112, "nonfinite": 0.0, "weighted_memory_to_active": 0.35540106892585754, "mix_std": 0.03688773140311241, "beta_std": 0.1469966620206833, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17985688149929047, "evidence_weights": [0.04051202908158302, -0.03286312520503998, -0.1481495350599289], "agreement_weight": -0.040475573390722275, "orthogonal_weight": 0.7003052234649658}]}, {"layer": 5, "memory_to_active": 0.6267495155334473, "contribution_to_active": 0.45091134309768677, "mix_mean": 0.40502771735191345, "mix_low": 0.0874578207731247, "mix_high": 0.010241350159049034, "read_cosine": 0.20883116126060486, "beta_mean": 0.04992677643895149, "nonfinite": 0.0, "weighted_memory_to_active": 0.16630671918392181, "mix_std": 0.24833287298679352, "beta_std": 0.06628505885601044, "beta_low": 0.618621826171875, "beta_high": 0.0, "orthogonal_to_active": 0.05454782024025917, "output_influence": 0.4406495988368988, "residual_influence": 0.10250114649534225, "state_active": 0.054478719830513, "state_memory": 0.014459433034062386, "state_peak": 62.25197982788086, "growth_active": 1.1012316942214966, "growth_memory": 1.2696574926376343, "heads": [{"head": 0, "memory_to_active": 1.2251346111297607, "contribution_to_active": 0.26360949873924255, "mix_mean": 0.1403147280216217, "mix_low": 0.13828468322753906, "mix_high": 0.0, "read_cosine": -0.1528005599975586, "beta_mean": 0.05166503041982651, "nonfinite": 0.0, "weighted_memory_to_active": 0.1913706511259079, "mix_std": 0.18371132016181946, "beta_std": 0.08154600858688354, "beta_low": 0.20330810546875, "beta_high": 0.0, "orthogonal_to_active": 0.042359765619039536, "evidence_weights": [-0.05113548785448074, -0.08755579590797424, -0.35483458638191223], "agreement_weight": -0.021204892545938492, "orthogonal_weight": -0.2179640382528305}, {"head": 1, "memory_to_active": 0.04185561090707779, "contribution_to_active": 0.49290239810943604, "mix_mean": 0.5519524216651917, "mix_low": 0.0069179534912109375, "mix_high": 0.012102127075195312, "read_cosine": 0.17477461695671082, "beta_mean": 0.00011140183778479695, "nonfinite": 0.0, "weighted_memory_to_active": 0.02050570212304592, "mix_std": 0.3110853135585785, "beta_std": 0.00023444164253305644, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.018130572512745857, "evidence_weights": [-0.22675907611846924, -0.24511216580867767, 0.7504964470863342], "agreement_weight": 0.29887521266937256, "orthogonal_weight": -0.9196109175682068}, {"head": 2, "memory_to_active": 0.05735069513320923, "contribution_to_active": 0.5021289587020874, "mix_mean": 0.5716392993927002, "mix_low": 0.0020351409912109375, "mix_high": 0.011639595031738281, "read_cosine": 0.2946403920650482, "beta_mean": 0.0015158667229115963, "nonfinite": 0.0, "weighted_memory_to_active": 0.038059599697589874, "mix_std": 0.27857649326324463, "beta_std": 0.0020964283030480146, "beta_low": 0.9891357421875, "beta_high": 0.0, "orthogonal_to_active": 0.005112173967063427, "evidence_weights": [-0.26058846712112427, -0.31598302721977234, 0.767472505569458], "agreement_weight": 0.02283596806228161, "orthogonal_weight": -0.1842251718044281}, {"head": 3, "memory_to_active": 1.8790932893753052, "contribution_to_active": 0.22376835346221924, "mix_mean": 0.08560682088136673, "mix_low": 0.20069313049316406, "mix_high": 0.0, "read_cosine": 0.16432231664657593, "beta_mean": 0.0618363693356514, "nonfinite": 0.0, "weighted_memory_to_active": 0.37156549096107483, "mix_std": 0.1291269212961197, "beta_std": 0.10067801922559738, "beta_low": 0.26727294921875, "beta_high": 0.0, "orthogonal_to_active": 0.1398925632238388, "evidence_weights": [-0.03240114822983742, -0.08170730620622635, -0.030102526769042015], "agreement_weight": 0.06535124778747559, "orthogonal_weight": -0.4572222828865051}, {"head": 4, "memory_to_active": 0.46585994958877563, "contribution_to_active": 0.5897940397262573, "mix_mean": 0.33363351225852966, "mix_low": 0.17406749725341797, "mix_high": 0.009891510009765625, "read_cosine": 0.5567987561225891, "beta_mean": 0.1841217428445816, "nonfinite": 0.0, "weighted_memory_to_active": 0.3138153851032257, "mix_std": 0.3374454975128174, "beta_std": 0.21173685789108276, "beta_low": 0.25653076171875, "beta_high": 0.0, "orthogonal_to_active": 0.08412769436836243, "evidence_weights": [-0.12706197798252106, -0.28489789366722107, -0.01752527616918087], "agreement_weight": 0.046013619750738144, "orthogonal_weight": 0.4049874246120453}, {"head": 5, "memory_to_active": 0.0912027582526207, "contribution_to_active": 0.6332648992538452, "mix_mean": 0.7470194697380066, "mix_low": 0.0027484893798828125, "mix_high": 0.027814865112304688, "read_cosine": 0.21525149047374725, "beta_mean": 0.0003102283808402717, "nonfinite": 0.0, "weighted_memory_to_active": 0.06252346187829971, "mix_std": 0.2500516176223755, "beta_std": 0.0014185653999447823, "beta_low": 0.99554443359375, "beta_high": 0.0, "orthogonal_to_active": 0.037664152681827545, "evidence_weights": [-0.32581812143325806, -0.31360146403312683, 1.708081603050232], "agreement_weight": 0.3177528977394104, "orthogonal_weight": -0.6447875499725342}]}, {"layer": 6, "memory_to_active": 0.39344480633735657, "contribution_to_active": 0.5581602454185486, "mix_mean": 0.5155483484268188, "mix_low": 0.07909949868917465, "mix_high": 0.023525238037109375, "read_cosine": 0.28922945261001587, "beta_mean": 0.07156006246805191, "nonfinite": 0.0, "weighted_memory_to_active": 0.16440042853355408, "mix_std": 0.25142961740493774, "beta_std": 0.0847114622592926, "beta_low": 0.50457763671875, "beta_high": 0.0, "orthogonal_to_active": 0.03711257874965668, "output_influence": 0.4414205849170685, "residual_influence": 0.13658775389194489, "state_active": 0.04918298125267029, "state_memory": 0.009862251579761505, "state_peak": 68.33963775634766, "growth_active": 1.1666384935379028, "growth_memory": 1.221245527267456, "heads": [{"head": 0, "memory_to_active": 0.17553691565990448, "contribution_to_active": 0.5611392855644226, "mix_mean": 0.671295166015625, "mix_low": 0.000217437744140625, "mix_high": 0.0038118362426757812, "read_cosine": 0.08309771120548248, "beta_mean": 0.013192170299589634, "nonfinite": 0.0, "weighted_memory_to_active": 0.12465732544660568, "mix_std": 0.24611543118953705, "beta_std": 0.023367203772068024, "beta_low": 0.58453369140625, "beta_high": 0.0, "orthogonal_to_active": 0.10070660710334778, "evidence_weights": [-0.30661851167678833, -0.30994221568107605, 0.4683704078197479], "agreement_weight": 0.10194772481918335, "orthogonal_weight": -0.911223828792572}, {"head": 1, "memory_to_active": 0.009604649618268013, "contribution_to_active": 0.45497560501098633, "mix_mean": 0.46663835644721985, "mix_low": 0.00148773193359375, "mix_high": 0.008194923400878906, "read_cosine": 0.592525064945221, "beta_mean": 2.1238472982076928e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003618038957938552, "mix_std": 0.2933181822299957, "beta_std": 5.836103446199559e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009357529925182462, "evidence_weights": [-0.20971918106079102, -0.2587471306324005, 0.7470176815986633], "agreement_weight": -0.025874849408864975, "orthogonal_weight": -0.35890161991119385}, {"head": 2, "memory_to_active": 1.2836045026779175, "contribution_to_active": 0.06190432235598564, "mix_mean": 0.027375593781471252, "mix_low": 0.45940685272216797, "mix_high": 0.0, "read_cosine": 0.1930485963821411, "beta_mean": 0.07008574157953262, "nonfinite": 0.0, "weighted_memory_to_active": 0.06736156344413757, "mix_std": 0.05367130786180496, "beta_std": 0.07392370700836182, "beta_low": 0.01336669921875, "beta_high": 0.0, "orthogonal_to_active": 0.010694753378629684, "evidence_weights": [0.08157749474048615, 0.015572884120047092, 0.22470629215240479], "agreement_weight": 0.10444801300764084, "orthogonal_weight": -0.17640288174152374}, {"head": 3, "memory_to_active": 0.14365489780902863, "contribution_to_active": 0.6261701583862305, "mix_mean": 0.6222171783447266, "mix_low": 0.011224746704101562, "mix_high": 0.018693923950195312, "read_cosine": 0.48363643884658813, "beta_mean": 0.22015193104743958, "nonfinite": 0.0, "weighted_memory_to_active": 0.1006334125995636, "mix_std": 0.340307354927063, "beta_std": 0.25990644097328186, "beta_low": 0.2110595703125, "beta_high": 0.0, "orthogonal_to_active": 0.04504021257162094, "evidence_weights": [-0.17676950991153717, -0.3361390233039856, 0.1527813971042633], "agreement_weight": -0.016602128744125366, "orthogonal_weight": 0.5251408815383911}, {"head": 4, "memory_to_active": 0.7048888206481934, "contribution_to_active": 1.07475745677948, "mix_mean": 0.7333905696868896, "mix_low": 0.0014505386352539062, "mix_high": 0.10935115814208984, "read_cosine": 0.11069047451019287, "beta_mean": 0.12559545040130615, "nonfinite": 0.0, "weighted_memory_to_active": 0.6623203754425049, "mix_std": 0.29725727438926697, "beta_std": 0.15043078362941742, "beta_low": 0.21893310546875, "beta_high": 0.0, "orthogonal_to_active": 0.03744565322995186, "evidence_weights": [-0.25047817826271057, -0.5634258389472961, -0.06092048063874245], "agreement_weight": -0.015221025794744492, "orthogonal_weight": 0.06706954538822174}, {"head": 5, "memory_to_active": 0.04337894916534424, "contribution_to_active": 0.5700145363807678, "mix_mean": 0.5723729133605957, "mix_low": 0.0008096694946289062, "mix_high": 0.0010995864868164062, "read_cosine": 0.2723783552646637, "beta_mean": 0.0003138408064842224, "nonfinite": 0.0, "weighted_memory_to_active": 0.027811843901872635, "mix_std": 0.2779080271720886, "beta_std": 0.0005822587991133332, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.027852464467287064, "evidence_weights": [-0.2301413118839264, -0.2794014513492584, 0.3527075946331024], "agreement_weight": 0.09116654843091965, "orthogonal_weight": -1.0686514377593994}]}, {"layer": 7, "memory_to_active": 2.047811985015869, "contribution_to_active": 0.16730178892612457, "mix_mean": 0.04301917552947998, "mix_low": 0.4563048779964447, "mix_high": 1.3987222700961865e-05, "read_cosine": 0.23303303122520447, "beta_mean": 0.1270814836025238, "nonfinite": 0.0, "weighted_memory_to_active": 0.1829056590795517, "mix_std": 0.0725981593132019, "beta_std": 0.10932334512472153, "beta_low": 0.0568745955824852, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.039174310863018036, "output_influence": 0.1917857527732849, "residual_influence": 0.04248868301510811, "state_active": 0.015393446199595928, "state_memory": 0.029761649668216705, "state_peak": 22.857603073120117, "growth_active": 1.0525835752487183, "growth_memory": 1.3664103746414185, "heads": [{"head": 0, "memory_to_active": 2.164407253265381, "contribution_to_active": 0.3166772723197937, "mix_mean": 0.030752195045351982, "mix_low": 0.49831104278564453, "mix_high": 3.528594970703125e-05, "read_cosine": 0.3675408959388733, "beta_mean": 0.14838501811027527, "nonfinite": 0.0, "weighted_memory_to_active": 0.2895756661891937, "mix_std": 0.06739649921655655, "beta_std": 0.11917182058095932, "beta_low": 0.0059814453125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.030019385740160942, "evidence_weights": [-0.003293414134532213, -0.08734563738107681, 0.0991511270403862], "agreement_weight": -0.04655098542571068, "orthogonal_weight": 0.14417356252670288}, {"head": 1, "memory_to_active": 1.6611852645874023, "contribution_to_active": 0.2726966440677643, "mix_mean": 0.1068350225687027, "mix_low": 0.2738456726074219, "mix_high": 1.33514404296875e-05, "read_cosine": 0.14957915246486664, "beta_mean": 0.13542440533638, "nonfinite": 0.0, "weighted_memory_to_active": 0.328621506690979, "mix_std": 0.14650200307369232, "beta_std": 0.08969206362962723, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.05159064382314682, "evidence_weights": [-0.1407630741596222, -0.10676590353250504, -0.18954384326934814], "agreement_weight": 0.005435851868242025, "orthogonal_weight": -0.17135611176490784}, {"head": 2, "memory_to_active": 1.2372369766235352, "contribution_to_active": 0.2291293442249298, "mix_mean": 0.062409766018390656, "mix_low": 0.2752552032470703, "mix_high": 3.528594970703125e-05, "read_cosine": 0.1713171750307083, "beta_mean": 0.1028607115149498, "nonfinite": 0.0, "weighted_memory_to_active": 0.18013857305049896, "mix_std": 0.11680153012275696, "beta_std": 0.06801054626703262, "beta_low": 0.02166748046875, "beta_high": 0.0, "orthogonal_to_active": 0.021760720759630203, "evidence_weights": [-0.03323657438158989, -0.1154996007680893, 0.21327689290046692], "agreement_weight": 0.024513432756066322, "orthogonal_weight": -0.12505991756916046}, {"head": 3, "memory_to_active": 3.641054153442383, "contribution_to_active": 0.0514083132147789, "mix_mean": 0.02206921949982643, "mix_low": 0.46578407287597656, "mix_high": 0.0, "read_cosine": 0.12693843245506287, "beta_mean": 0.19497886300086975, "nonfinite": 0.0, "weighted_memory_to_active": 0.14952324330806732, "mix_std": 0.03448185324668884, "beta_std": 0.18812400102615356, "beta_low": 0.02203369140625, "beta_high": 0.0, "orthogonal_to_active": 0.10742023587226868, "evidence_weights": [0.0963117703795433, 0.054120756685733795, 0.4132121205329895], "agreement_weight": 0.08031218498945236, "orthogonal_weight": -0.7878648638725281}, {"head": 4, "memory_to_active": 1.9941003322601318, "contribution_to_active": 0.06085614860057831, "mix_mean": 0.01102122850716114, "mix_low": 0.7439565658569336, "mix_high": 0.0, "read_cosine": 0.3625744581222534, "beta_mean": 0.13975197076797485, "nonfinite": 0.0, "weighted_memory_to_active": 0.05890475586056709, "mix_std": 0.026077020913362503, "beta_std": 0.1320050060749054, "beta_low": 0.03167724609375, "beta_high": 0.0, "orthogonal_to_active": 0.0034984678495675325, "evidence_weights": [0.10300110280513763, 0.04942997172474861, 0.026870068162679672], "agreement_weight": 0.047984592616558075, "orthogonal_weight": -0.06695374846458435}, {"head": 5, "memory_to_active": 1.5888874530792236, "contribution_to_active": 0.07304297387599945, "mix_mean": 0.025027595460414886, "mix_low": 0.48067665100097656, "mix_high": 0.0, "read_cosine": 0.2202480137348175, "beta_mean": 0.04108794406056404, "nonfinite": 0.0, "weighted_memory_to_active": 0.0906701534986496, "mix_std": 0.04433005303144455, "beta_std": 0.058936648070812225, "beta_low": 0.2596435546875, "beta_high": 0.0, "orthogonal_to_active": 0.02075641043484211, "evidence_weights": [0.0616280622780323, 0.049317460507154465, 0.03149367868900299], "agreement_weight": 0.020562194287776947, "orthogonal_weight": -0.2848304808139801}]}, {"layer": 8, "memory_to_active": 0.6145553588867188, "contribution_to_active": 0.4677850306034088, "mix_mean": 0.364446222782135, "mix_low": 0.08711720257997513, "mix_high": 0.0006837845430709422, "read_cosine": 0.5260292887687683, "beta_mean": 0.03504781797528267, "nonfinite": 0.0, "weighted_memory_to_active": 0.13393867015838623, "mix_std": 0.2252410650253296, "beta_std": 0.03949303552508354, "beta_low": 0.4932963252067566, "beta_high": 0.0, "orthogonal_to_active": 0.05252251401543617, "output_influence": 0.3577192425727844, "residual_influence": 0.1011563390493393, "state_active": 0.05545523017644882, "state_memory": 0.012832165695726871, "state_peak": 35.711002349853516, "growth_active": 1.0997964143753052, "growth_memory": 1.3277186155319214, "heads": [{"head": 0, "memory_to_active": 0.05110296234488487, "contribution_to_active": 0.5180282592773438, "mix_mean": 0.5511982440948486, "mix_low": 0.00017452239990234375, "mix_high": 0.0004978179931640625, "read_cosine": 0.6925313472747803, "beta_mean": 0.003832722082734108, "nonfinite": 0.0, "weighted_memory_to_active": 0.029957585036754608, "mix_std": 0.24060723185539246, "beta_std": 0.008236343041062355, "beta_low": 0.9012451171875, "beta_high": 0.0, "orthogonal_to_active": 0.01661870628595352, "evidence_weights": [-0.20265772938728333, -0.35927510261535645, 0.726732611656189], "agreement_weight": 0.06860465556383133, "orthogonal_weight": -0.7965478301048279}, {"head": 1, "memory_to_active": 0.4975799024105072, "contribution_to_active": 0.3667979836463928, "mix_mean": 0.36620956659317017, "mix_low": 0.0048809051513671875, "mix_high": 0.00023651123046875, "read_cosine": 0.44304293394088745, "beta_mean": 0.021737154573202133, "nonfinite": 0.0, "weighted_memory_to_active": 0.20760689675807953, "mix_std": 0.24377061426639557, "beta_std": 0.02529274672269821, "beta_low": 0.3592529296875, "beta_high": 0.0, "orthogonal_to_active": 0.07461509108543396, "evidence_weights": [-0.17274421453475952, -0.20230655372142792, 0.45203909277915955], "agreement_weight": 0.0633251741528511, "orthogonal_weight": -0.41261452436447144}, {"head": 2, "memory_to_active": 0.07488149404525757, "contribution_to_active": 0.5688115954399109, "mix_mean": 0.5236166715621948, "mix_low": 0.0007772445678710938, "mix_high": 0.0016689300537109375, "read_cosine": 0.3911972939968109, "beta_mean": 0.0008869227021932602, "nonfinite": 0.0, "weighted_memory_to_active": 0.04690597206354141, "mix_std": 0.25507044792175293, "beta_std": 0.0026001150254160166, "beta_low": 0.99334716796875, "beta_high": 0.0, "orthogonal_to_active": 0.03470277041196823, "evidence_weights": [-0.20372532308101654, -0.3077723979949951, 0.5310369729995728], "agreement_weight": 0.07429651916027069, "orthogonal_weight": -0.8510383367538452}, {"head": 3, "memory_to_active": 0.20987169444561005, "contribution_to_active": 0.4578248858451843, "mix_mean": 0.4145619869232178, "mix_low": 0.0020151138305664062, "mix_high": 0.00013446807861328125, "read_cosine": 0.7209255695343018, "beta_mean": 0.014970174059271812, "nonfinite": 0.0, "weighted_memory_to_active": 0.1257198601961136, "mix_std": 0.24531736969947815, "beta_std": 0.01614289917051792, "beta_low": 0.4219970703125, "beta_high": 0.0, "orthogonal_to_active": 0.029627814888954163, "evidence_weights": [-0.13153667747974396, -0.24586984515190125, 0.7362999320030212], "agreement_weight": 0.09519639611244202, "orthogonal_weight": -0.3911024034023285}, {"head": 4, "memory_to_active": 2.6350629329681396, "contribution_to_active": 0.2452823966741562, "mix_mean": 0.026757851243019104, "mix_low": 0.4120750427246094, "mix_high": 0.0, "read_cosine": 0.36056146025657654, "beta_mean": 0.10151728987693787, "nonfinite": 0.0, "weighted_memory_to_active": 0.24578726291656494, "mix_std": 0.04339729994535446, "beta_std": 0.08731509745121002, "beta_low": 0.006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.005817196797579527, "evidence_weights": [0.03507952019572258, -0.005376901011914015, -0.16114269196987152], "agreement_weight": 0.022926636040210724, "orthogonal_weight": 0.025709640234708786}, {"head": 5, "memory_to_active": 0.2188330441713333, "contribution_to_active": 0.6499649286270142, "mix_mean": 0.30433300137519836, "mix_low": 0.10278034210205078, "mix_high": 0.0015649795532226562, "read_cosine": 0.5479171276092529, "beta_mean": 0.06734264642000198, "nonfinite": 0.0, "weighted_memory_to_active": 0.14765439927577972, "mix_std": 0.32328343391418457, "beta_std": 0.09737101197242737, "beta_low": 0.27783203125, "beta_high": 0.0, "orthogonal_to_active": 0.15375347435474396, "evidence_weights": [-0.07048580795526505, -0.3179176449775696, -0.4332963824272156], "agreement_weight": 0.023901857435703278, "orthogonal_weight": 1.2867131233215332}]}, {"layer": 9, "memory_to_active": 1.1943880319595337, "contribution_to_active": 0.6232657432556152, "mix_mean": 0.255969762802124, "mix_low": 0.17330121994018555, "mix_high": 0.0015678406925871968, "read_cosine": 0.35772544145584106, "beta_mean": 0.0863637775182724, "nonfinite": 0.0, "weighted_memory_to_active": 0.37952589988708496, "mix_std": 0.2081700563430786, "beta_std": 0.05217641964554787, "beta_low": 0.4977315664291382, "beta_high": 0.0, "orthogonal_to_active": 0.03373568132519722, "output_influence": 0.41832083463668823, "residual_influence": 0.10744872689247131, "state_active": 0.043801598250865936, "state_memory": 0.023629682138562202, "state_peak": 34.98179244995117, "growth_active": 1.3057548999786377, "growth_memory": 1.3908195495605469, "heads": [{"head": 0, "memory_to_active": 0.005768196191638708, "contribution_to_active": 0.47790423035621643, "mix_mean": 0.5331954956054688, "mix_low": 0.0019044876098632812, "mix_high": 0.006617546081542969, "read_cosine": 0.2145582139492035, "beta_mean": 1.49114948726492e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031976737082004547, "mix_std": 0.2969423830509186, "beta_std": 5.605718251899816e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026608710177242756, "evidence_weights": [-0.2710822820663452, -0.2711813449859619, 0.9607792496681213], "agreement_weight": 0.06750332564115524, "orthogonal_weight": -0.8628270626068115}, {"head": 1, "memory_to_active": 0.042979512363672256, "contribution_to_active": 0.5123423337936401, "mix_mean": 0.3654668927192688, "mix_low": 0.013485908508300781, "mix_high": 0.0006361007690429688, "read_cosine": 0.42706719040870667, "beta_mean": 0.00013573913020081818, "nonfinite": 0.0, "weighted_memory_to_active": 0.021604113280773163, "mix_std": 0.2816166281700134, "beta_std": 0.00032717784051783383, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.011035786010324955, "evidence_weights": [-0.20052587985992432, -0.21973823010921478, 0.9407040476799011], "agreement_weight": 0.2926926612854004, "orthogonal_weight": -0.5997947454452515}, {"head": 2, "memory_to_active": 0.24610789120197296, "contribution_to_active": 0.5499205589294434, "mix_mean": 0.42855387926101685, "mix_low": 0.008263587951660156, "mix_high": 0.0019216537475585938, "read_cosine": 0.3959052562713623, "beta_mean": 0.0013793518301099539, "nonfinite": 0.0, "weighted_memory_to_active": 0.1438361406326294, "mix_std": 0.28919166326522827, "beta_std": 0.006249206140637398, "beta_low": 0.9833984375, "beta_high": 0.0, "orthogonal_to_active": 0.044084955006837845, "evidence_weights": [-0.1604570895433426, -0.18876180052757263, 0.47451141476631165], "agreement_weight": 0.19067129492759705, "orthogonal_weight": -0.38411980867385864}, {"head": 3, "memory_to_active": 1.8924521207809448, "contribution_to_active": 0.16972482204437256, "mix_mean": 0.03496645390987396, "mix_low": 0.4227638244628906, "mix_high": 0.0, "read_cosine": 0.46588748693466187, "beta_mean": 0.10775540769100189, "nonfinite": 0.0, "weighted_memory_to_active": 0.14531511068344116, "mix_std": 0.08061051368713379, "beta_std": 0.07305659353733063, "beta_low": 0.00299072265625, "beta_high": 0.0, "orthogonal_to_active": 0.0258303452283144, "evidence_weights": [0.05418803170323372, 0.039724867790937424, -0.15834416449069977], "agreement_weight": 0.06963137537240982, "orthogonal_weight": 0.18723322451114655}, {"head": 4, "memory_to_active": 3.069319486618042, "contribution_to_active": 1.3356695175170898, "mix_mean": 0.11423277854919434, "mix_low": 0.1952495574951172, "mix_high": 0.0002307891845703125, "read_cosine": 0.21141907572746277, "beta_mean": 0.20374608039855957, "nonfinite": 0.0, "weighted_memory_to_active": 1.1783220767974854, "mix_std": 0.1819307804107666, "beta_std": 0.11132535338401794, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00846776831895113, "evidence_weights": [-0.04772410914301872, -0.1354462206363678, 0.05014847591519356], "agreement_weight": -0.04853403568267822, "orthogonal_weight": -0.007126887794584036}, {"head": 5, "memory_to_active": 1.9097001552581787, "contribution_to_active": 0.694033145904541, "mix_mean": 0.05940324068069458, "mix_low": 0.39813995361328125, "mix_high": 9.5367431640625e-07, "read_cosine": 0.4315153956413269, "beta_mean": 0.20515112578868866, "nonfinite": 0.0, "weighted_memory_to_active": 0.7848802208900452, "mix_std": 0.11872836202383041, "beta_std": 0.12204413115978241, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.1103343516588211, "evidence_weights": [0.061550553888082504, -0.04028516635298729, 0.08964018523693085], "agreement_weight": 0.021313555538654327, "orthogonal_weight": 0.1953003704547882}]}, {"layer": 10, "memory_to_active": 0.1563478708267212, "contribution_to_active": 0.44906580448150635, "mix_mean": 0.43663448095321655, "mix_low": 0.010473729111254215, "mix_high": 0.011263370513916016, "read_cosine": 0.3976958990097046, "beta_mean": 0.045006848871707916, "nonfinite": 0.0, "weighted_memory_to_active": 0.07115674018859863, "mix_std": 0.2674669027328491, "beta_std": 0.048930466175079346, "beta_low": 0.6483256220817566, "beta_high": 0.0, "orthogonal_to_active": 0.038261596113443375, "output_influence": 0.3862919807434082, "residual_influence": 0.1250893473625183, "state_active": 0.06047246232628822, "state_memory": 0.013816716149449348, "state_peak": 111.58720397949219, "growth_active": 1.0944523811340332, "growth_memory": 1.086457371711731, "heads": [{"head": 0, "memory_to_active": 0.3106633424758911, "contribution_to_active": 0.7421640157699585, "mix_mean": 0.6480472087860107, "mix_low": 0.012248039245605469, "mix_high": 0.0633687973022461, "read_cosine": 0.28156381845474243, "beta_mean": 0.20196551084518433, "nonfinite": 0.0, "weighted_memory_to_active": 0.22914263606071472, "mix_std": 0.3552204668521881, "beta_std": 0.2167908251285553, "beta_low": 0.11151123046875, "beta_high": 0.0, "orthogonal_to_active": 0.1340743750333786, "evidence_weights": [-0.1719411462545395, -0.3513292968273163, -0.3620281517505646], "agreement_weight": 0.020045766606926918, "orthogonal_weight": 0.6619578003883362}, {"head": 1, "memory_to_active": 0.017665019258856773, "contribution_to_active": 0.524907648563385, "mix_mean": 0.5185583233833313, "mix_low": 0.0011930465698242188, "mix_high": 0.003757476806640625, "read_cosine": 0.2707922160625458, "beta_mean": 9.798674727790058e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009668131358921528, "mix_std": 0.26835644245147705, "beta_std": 0.0001733042299747467, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009895673021674156, "evidence_weights": [-0.2656683027744293, -0.19037726521492004, 0.4531239867210388], "agreement_weight": 0.2560374438762665, "orthogonal_weight": -1.1544190645217896}, {"head": 2, "memory_to_active": 0.16696403920650482, "contribution_to_active": 0.42083287239074707, "mix_mean": 0.3990134000778198, "mix_low": 0.0025196075439453125, "mix_high": 0.00026988983154296875, "read_cosine": 0.018880002200603485, "beta_mean": 0.0012924724724143744, "nonfinite": 0.0, "weighted_memory_to_active": 0.06838232278823853, "mix_std": 0.27407586574554443, "beta_std": 0.005324537865817547, "beta_low": 0.97186279296875, "beta_high": 0.0, "orthogonal_to_active": 0.023247260600328445, "evidence_weights": [-0.15370094776153564, -0.20847876369953156, 0.5447372198104858], "agreement_weight": -0.01196741871535778, "orthogonal_weight": -0.3386480212211609}, {"head": 3, "memory_to_active": 0.29530423879623413, "contribution_to_active": 0.11110487580299377, "mix_mean": 0.2525582015514374, "mix_low": 0.04306507110595703, "mix_high": 0.00014781951904296875, "read_cosine": 0.8965098857879639, "beta_mean": 0.059596844017505646, "nonfinite": 0.0, "weighted_memory_to_active": 0.045437153428792953, "mix_std": 0.23294031620025635, "beta_std": 0.06104069948196411, "beta_low": 0.011962890625, "beta_high": 0.0, "orthogonal_to_active": 0.04452138766646385, "evidence_weights": [-0.12844102084636688, -0.14653238654136658, 0.20028483867645264], "agreement_weight": 0.00020937653607688844, "orthogonal_weight": 1.3662124872207642}, {"head": 4, "memory_to_active": 0.01098528504371643, "contribution_to_active": 0.45057213306427, "mix_mean": 0.44356635212898254, "mix_low": 0.0011682510375976562, "mix_high": 2.002716064453125e-05, "read_cosine": 0.06365735828876495, "beta_mean": 2.7659927582135424e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003494878299534321, "mix_std": 0.25125181674957275, "beta_std": 0.0010473636211827397, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.000979541800916195, "evidence_weights": [-0.1814945489168167, -0.21144066751003265, 0.4465894401073456], "agreement_weight": 0.027662217617034912, "orthogonal_weight": -0.28545764088630676}, {"head": 5, "memory_to_active": 0.13650524616241455, "contribution_to_active": 0.4448133111000061, "mix_mean": 0.3580634593963623, "mix_low": 0.0026483535766601562, "mix_high": 1.621246337890625e-05, "read_cosine": 0.8547719717025757, "beta_mean": 0.0070606404915452, "nonfinite": 0.0, "weighted_memory_to_active": 0.07081528753042221, "mix_std": 0.22295641899108887, "beta_std": 0.009206057526171207, "beta_low": 0.794677734375, "beta_high": 0.0, "orthogonal_to_active": 0.016851333901286125, "evidence_weights": [-0.16427575051784515, -0.16692796349525452, 0.2897646427154541], "agreement_weight": 0.09347137063741684, "orthogonal_weight": -0.5628223419189453}]}, {"layer": 11, "memory_to_active": 0.2199006825685501, "contribution_to_active": 0.4827866554260254, "mix_mean": 0.4017776846885681, "mix_low": 0.04894733428955078, "mix_high": 0.02729145810008049, "read_cosine": 0.4941783547401428, "beta_mean": 0.036069244146347046, "nonfinite": 0.0, "weighted_memory_to_active": 0.08114854991436005, "mix_std": 0.3071424961090088, "beta_std": 0.05139298737049103, "beta_low": 0.513031005859375, "beta_high": 0.0, "orthogonal_to_active": 0.04978209733963013, "output_influence": 0.4731683135032654, "residual_influence": 0.12237489223480225, "state_active": 0.11909354478120804, "state_memory": 0.010128181427717209, "state_peak": 35.058921813964844, "growth_active": 1.1385366916656494, "growth_memory": 1.4223980903625488, "heads": [{"head": 0, "memory_to_active": 0.017045116052031517, "contribution_to_active": 0.5479146242141724, "mix_mean": 0.552501916885376, "mix_low": 0.002010345458984375, "mix_high": 0.0076160430908203125, "read_cosine": 0.3833877742290497, "beta_mean": 0.00014002659008838236, "nonfinite": 0.0, "weighted_memory_to_active": 0.011392509564757347, "mix_std": 0.27497103810310364, "beta_std": 0.004241839051246643, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.006558192428201437, "evidence_weights": [-0.15890827775001526, -0.3382251262664795, 0.5861101150512695], "agreement_weight": 0.15939761698246002, "orthogonal_weight": -0.697253942489624}, {"head": 1, "memory_to_active": 0.439800500869751, "contribution_to_active": 0.2355361133813858, "mix_mean": 0.25272336602211, "mix_low": 0.028349876403808594, "mix_high": 4.9591064453125e-05, "read_cosine": 0.6018091440200806, "beta_mean": 0.03134186193346977, "nonfinite": 0.0, "weighted_memory_to_active": 0.15901227295398712, "mix_std": 0.236786350607872, "beta_std": 0.058261990547180176, "beta_low": 0.4310302734375, "beta_high": 0.0, "orthogonal_to_active": 0.024893207475543022, "evidence_weights": [-0.1602603793144226, -0.12313556671142578, 0.17715014517307281], "agreement_weight": 0.08439228683710098, "orthogonal_weight": -0.2170848548412323}, {"head": 2, "memory_to_active": 0.04368900880217552, "contribution_to_active": 0.880325436592102, "mix_mean": 0.4804859757423401, "mix_low": 0.06670570373535156, "mix_high": 0.08993911743164062, "read_cosine": 0.31639111042022705, "beta_mean": 0.06962277740240097, "nonfinite": 0.0, "weighted_memory_to_active": 0.03579123318195343, "mix_std": 0.4121231436729431, "beta_std": 0.09459339827299118, "beta_low": 0.4410400390625, "beta_high": 0.0, "orthogonal_to_active": 0.10193034261465073, "evidence_weights": [-0.09496545791625977, -0.44333508610725403, -0.8817583918571472], "agreement_weight": -0.0002661800535861403, "orthogonal_weight": 3.0822811126708984}, {"head": 3, "memory_to_active": 0.023711886256933212, "contribution_to_active": 0.5722575187683105, "mix_mean": 0.43188345432281494, "mix_low": 0.16009140014648438, "mix_high": 0.06549263000488281, "read_cosine": 0.5575183629989624, "beta_mean": 0.05922200158238411, "nonfinite": 0.0, "weighted_memory_to_active": 0.013206513598561287, "mix_std": 0.42462158203125, "beta_std": 0.08772365748882294, "beta_low": 0.4688720703125, "beta_high": 0.0, "orthogonal_to_active": 0.03798533231019974, "evidence_weights": [-0.06819834560155869, -0.3682467043399811, -0.35603514313697815], "agreement_weight": 0.025232721120119095, "orthogonal_weight": 3.3267362117767334}, {"head": 4, "memory_to_active": 0.6303361058235168, "contribution_to_active": 0.3048067092895508, "mix_mean": 0.2544516921043396, "mix_low": 0.035979270935058594, "mix_high": 0.0001125335693359375, "read_cosine": 0.22173050045967102, "beta_mean": 0.04234427213668823, "nonfinite": 0.0, "weighted_memory_to_active": 0.19495265185832977, "mix_std": 0.23579341173171997, "beta_std": 0.04250482842326164, "beta_low": 0.1319580078125, "beta_high": 0.0, "orthogonal_to_active": 0.10391102731227875, "evidence_weights": [-0.13676400482654572, -0.1084783747792244, 1.2777079343795776], "agreement_weight": -0.12055566906929016, "orthogonal_weight": -0.5607137084007263}, {"head": 5, "memory_to_active": 0.16482152044773102, "contribution_to_active": 0.35587942600250244, "mix_mean": 0.4386195242404938, "mix_low": 0.0005474090576171875, "mix_high": 0.0005388259887695312, "read_cosine": 0.8842331171035767, "beta_mean": 0.013744526542723179, "nonfinite": 0.0, "weighted_memory_to_active": 0.07253611087799072, "mix_std": 0.2585594356060028, "beta_std": 0.02103220485150814, "beta_low": 0.60595703125, "beta_high": 0.0, "orthogonal_to_active": 0.023414457216858864, "evidence_weights": [-0.19422566890716553, -0.2469707876443863, 0.4728666841983795], "agreement_weight": 0.041202425956726074, "orthogonal_weight": -0.6153923273086548}]}, {"layer": 12, "memory_to_active": 0.38699281215667725, "contribution_to_active": 0.5221773386001587, "mix_mean": 0.4262675940990448, "mix_low": 0.02485768124461174, "mix_high": 0.0024639766197651625, "read_cosine": 0.2643495798110962, "beta_mean": 0.017331160604953766, "nonfinite": 0.0, "weighted_memory_to_active": 0.18812258541584015, "mix_std": 0.2520807385444641, "beta_std": 0.01889520138502121, "beta_low": 0.8079325556755066, "beta_high": 0.0, "orthogonal_to_active": 0.06862588971853256, "output_influence": 0.39206942915916443, "residual_influence": 0.10701250284910202, "state_active": 0.04394526779651642, "state_memory": 0.01132471114397049, "state_peak": 24.857942581176758, "growth_active": 1.1340633630752563, "growth_memory": 1.1800569295883179, "heads": [{"head": 0, "memory_to_active": 0.23576879501342773, "contribution_to_active": 0.3778117299079895, "mix_mean": 0.3578064441680908, "mix_low": 0.0084381103515625, "mix_high": 0.00019073486328125, "read_cosine": 0.2663515508174896, "beta_mean": 0.0028078933246433735, "nonfinite": 0.0, "weighted_memory_to_active": 0.08620939403772354, "mix_std": 0.25482040643692017, "beta_std": 0.011107989586889744, "beta_low": 0.9432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.04158565029501915, "evidence_weights": [-0.153739333152771, -0.21952714025974274, 0.24913959205150604], "agreement_weight": 0.11115105450153351, "orthogonal_weight": -0.5256891846656799}, {"head": 1, "memory_to_active": 0.1313716173171997, "contribution_to_active": 0.45476365089416504, "mix_mean": 0.4647836983203888, "mix_low": 0.0007228851318359375, "mix_high": 0.000244140625, "read_cosine": 0.5801927447319031, "beta_mean": 0.0015969008672982454, "nonfinite": 0.0, "weighted_memory_to_active": 0.06131301075220108, "mix_std": 0.2625668942928314, "beta_std": 0.006436252500861883, "beta_low": 0.97296142578125, "beta_high": 0.0, "orthogonal_to_active": 0.04363967850804329, "evidence_weights": [-0.19344711303710938, -0.2481411099433899, 0.7878022193908691], "agreement_weight": 0.09423808753490448, "orthogonal_weight": -0.8949147462844849}, {"head": 2, "memory_to_active": 0.6454414129257202, "contribution_to_active": 0.4416348338127136, "mix_mean": 0.3045571744441986, "mix_low": 0.026299476623535156, "mix_high": 0.00028133392333984375, "read_cosine": 0.4404715597629547, "beta_mean": 0.09584657847881317, "nonfinite": 0.0, "weighted_memory_to_active": 0.2645983397960663, "mix_std": 0.24624471366405487, "beta_std": 0.07420827448368073, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.18861566483974457, "evidence_weights": [-0.23814333975315094, -0.046449996531009674, -0.05696571618318558], "agreement_weight": 0.034678272902965546, "orthogonal_weight": 0.889704704284668}, {"head": 3, "memory_to_active": 0.02568337693810463, "contribution_to_active": 0.7070710062980652, "mix_mean": 0.6627936363220215, "mix_low": 0.00048828125, "mix_high": 0.010729789733886719, "read_cosine": -0.024784576147794724, "beta_mean": 0.0002517123648431152, "nonfinite": 0.0, "weighted_memory_to_active": 0.018001165241003036, "mix_std": 0.24718189239501953, "beta_std": 0.0033794103655964136, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.015768203884363174, "evidence_weights": [-0.30450552701950073, -0.31999194622039795, 0.9201849102973938], "agreement_weight": 0.20922575891017914, "orthogonal_weight": -0.8975763916969299}, {"head": 4, "memory_to_active": 1.2353764772415161, "contribution_to_active": 0.6984208822250366, "mix_mean": 0.1932058483362198, "mix_low": 0.10686302185058594, "mix_high": 0.0012645721435546875, "read_cosine": -0.04085848852992058, "beta_mean": 0.0029910379089415073, "nonfinite": 0.0, "weighted_memory_to_active": 0.6723895072937012, "mix_std": 0.22682610154151917, "beta_std": 0.01589372754096985, "beta_low": 0.9383544921875, "beta_high": 0.0, "orthogonal_to_active": 0.10205194354057312, "evidence_weights": [-0.010144947096705437, -0.16404837369918823, -0.011510342359542847], "agreement_weight": -0.0017222893657162786, "orthogonal_weight": -0.1662767231464386}, {"head": 5, "memory_to_active": 0.04831501469016075, "contribution_to_active": 0.4533619284629822, "mix_mean": 0.5744585990905762, "mix_low": 0.0063343048095703125, "mix_high": 0.0020732879638671875, "read_cosine": 0.3647245168685913, "beta_mean": 0.0004928229027427733, "nonfinite": 0.0, "weighted_memory_to_active": 0.026224013417959213, "mix_std": 0.2748444676399231, "beta_std": 0.0023455412592738867, "beta_low": 0.99365234375, "beta_high": 0.0, "orthogonal_to_active": 0.020094161853194237, "evidence_weights": [-0.26333317160606384, -0.29034921526908875, 0.8535911440849304], "agreement_weight": 0.11801367998123169, "orthogonal_weight": -0.8381062746047974}]}, {"layer": 13, "memory_to_active": 0.18248462677001953, "contribution_to_active": 0.4726509153842926, "mix_mean": 0.46530765295028687, "mix_low": 0.006066799163818359, "mix_high": 0.0018992425175383687, "read_cosine": 0.38532108068466187, "beta_mean": 0.018537970259785652, "nonfinite": 0.0, "weighted_memory_to_active": 0.08068126440048218, "mix_std": 0.2570318281650543, "beta_std": 0.020485620945692062, "beta_low": 0.7720134258270264, "beta_high": 0.0, "orthogonal_to_active": 0.07129484415054321, "output_influence": 0.42179688811302185, "residual_influence": 0.13837292790412903, "state_active": 0.09055786579847336, "state_memory": 0.03681815043091774, "state_peak": 94.34730529785156, "growth_active": 1.006062388420105, "growth_memory": 1.0503886938095093, "heads": [{"head": 0, "memory_to_active": 0.08493851125240326, "contribution_to_active": 0.5015137195587158, "mix_mean": 0.4875689744949341, "mix_low": 0.0007181167602539062, "mix_high": 0.001384735107421875, "read_cosine": 0.35222330689430237, "beta_mean": 0.00047240080311894417, "nonfinite": 0.0, "weighted_memory_to_active": 0.045831575989723206, "mix_std": 0.26232242584228516, "beta_std": 0.0010995477205142379, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.04404531046748161, "evidence_weights": [-0.20582734048366547, -0.24221692979335785, 0.7699392437934875], "agreement_weight": 0.04327740892767906, "orthogonal_weight": -1.0620094537734985}, {"head": 1, "memory_to_active": 0.03634434938430786, "contribution_to_active": 0.5324785113334656, "mix_mean": 0.46585699915885925, "mix_low": 0.0021944046020507812, "mix_high": 0.00016689300537109375, "read_cosine": 0.19298188388347626, "beta_mean": 0.00012693478493019938, "nonfinite": 0.0, "weighted_memory_to_active": 0.01887110248208046, "mix_std": 0.26247724890708923, "beta_std": 0.00024255023163277656, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.017984097823500633, "evidence_weights": [-0.1881929188966751, -0.2504459321498871, 0.7275828123092651], "agreement_weight": 0.14684481918811798, "orthogonal_weight": -1.00373113155365}, {"head": 2, "memory_to_active": 0.19034814834594727, "contribution_to_active": 0.33663520216941833, "mix_mean": 0.34166842699050903, "mix_low": 0.0027494430541992188, "mix_high": 3.719329833984375e-05, "read_cosine": 0.619247555732727, "beta_mean": 0.021210096776485443, "nonfinite": 0.0, "weighted_memory_to_active": 0.07326753437519073, "mix_std": 0.21596740186214447, "beta_std": 0.051964059472084045, "beta_low": 0.661865234375, "beta_high": 0.0, "orthogonal_to_active": 0.03822661563754082, "evidence_weights": [-0.2229684293270111, -0.15806980431079865, 0.5059992671012878], "agreement_weight": 0.04119875654578209, "orthogonal_weight": -0.7317273616790771}, {"head": 3, "memory_to_active": 0.18275481462478638, "contribution_to_active": 0.5729876160621643, "mix_mean": 0.5766210556030273, "mix_low": 0.01435089111328125, "mix_high": 0.00724029541015625, "read_cosine": 0.007837429642677307, "beta_mean": 0.0011550933122634888, "nonfinite": 0.0, "weighted_memory_to_active": 0.12831369042396545, "mix_std": 0.2824052572250366, "beta_std": 0.00490217749029398, "beta_low": 0.98748779296875, "beta_high": 0.0, "orthogonal_to_active": 0.12922394275665283, "evidence_weights": [-0.35007303953170776, -0.23213717341423035, 1.867114782333374], "agreement_weight": 1.9242650270462036, "orthogonal_weight": -1.0486568212509155}, {"head": 4, "memory_to_active": 0.45275938510894775, "contribution_to_active": 0.286693811416626, "mix_mean": 0.38627663254737854, "mix_low": 0.010980606079101562, "mix_high": 0.00019741058349609375, "read_cosine": 0.8748137950897217, "beta_mean": 0.08712715655565262, "nonfinite": 0.0, "weighted_memory_to_active": 0.13521841168403625, "mix_std": 0.2371794879436493, "beta_std": 0.059338148683309555, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14127080142498016, "evidence_weights": [-0.20820648968219757, -0.2372741997241974, 0.490178644657135], "agreement_weight": -0.002427715575322509, "orthogonal_weight": 1.6128754615783691}, {"head": 5, "memory_to_active": 0.1477625072002411, "contribution_to_active": 0.6055965423583984, "mix_mean": 0.5338537693023682, "mix_low": 0.0054073333740234375, "mix_high": 0.002368927001953125, "read_cosine": 0.2648225426673889, "beta_mean": 0.0011361311189830303, "nonfinite": 0.0, "weighted_memory_to_active": 0.08258523046970367, "mix_std": 0.28183913230895996, "beta_std": 0.005367240868508816, "beta_low": 0.98468017578125, "beta_high": 0.0, "orthogonal_to_active": 0.05701829493045807, "evidence_weights": [-0.21419814229011536, -0.2666037380695343, 0.50645512342453], "agreement_weight": 0.07990644127130508, "orthogonal_weight": -0.7641038298606873}]}, {"layer": 14, "memory_to_active": 0.2657294273376465, "contribution_to_active": 0.4340665936470032, "mix_mean": 0.4023475646972656, "mix_low": 0.026962440460920334, "mix_high": 0.003261884208768606, "read_cosine": 0.6300644874572754, "beta_mean": 0.06462487578392029, "nonfinite": 0.0, "weighted_memory_to_active": 0.12573184072971344, "mix_std": 0.2505291998386383, "beta_std": 0.06234687566757202, "beta_low": 0.4365743100643158, "beta_high": 0.0, "orthogonal_to_active": 0.041736289858818054, "output_influence": 0.2876979410648346, "residual_influence": 0.10957863926887512, "state_active": 0.07454937696456909, "state_memory": 0.015832427889108658, "state_peak": 27.73839569091797, "growth_active": 1.1734157800674438, "growth_memory": 1.3426430225372314, "heads": [{"head": 0, "memory_to_active": 0.32838761806488037, "contribution_to_active": 0.24684667587280273, "mix_mean": 0.24282732605934143, "mix_low": 0.016097068786621094, "mix_high": 0.00010395050048828125, "read_cosine": 0.745247483253479, "beta_mean": 0.05805259570479393, "nonfinite": 0.0, "weighted_memory_to_active": 0.1052340418100357, "mix_std": 0.22255957126617432, "beta_std": 0.06841835379600525, "beta_low": 0.1002197265625, "beta_high": 0.0, "orthogonal_to_active": 0.033632874488830566, "evidence_weights": [-0.08301451802253723, -0.08928162604570389, 0.7372074723243713], "agreement_weight": 0.04047945886850357, "orthogonal_weight": 0.5128430724143982}, {"head": 1, "memory_to_active": 0.47694575786590576, "contribution_to_active": 0.4107653498649597, "mix_mean": 0.2673841714859009, "mix_low": 0.12255477905273438, "mix_high": 0.00019741058349609375, "read_cosine": 0.6932212710380554, "beta_mean": 0.26342692971229553, "nonfinite": 0.0, "weighted_memory_to_active": 0.2486284375190735, "mix_std": 0.27568912506103516, "beta_std": 0.243535116314888, "beta_low": 0.10186767578125, "beta_high": 0.0, "orthogonal_to_active": 0.021921640262007713, "evidence_weights": [0.007749177515506744, -0.27635446190834045, -0.1369181126356125], "agreement_weight": 0.05837222561240196, "orthogonal_weight": 0.16516880691051483}, {"head": 2, "memory_to_active": 0.2247908115386963, "contribution_to_active": 0.5198670625686646, "mix_mean": 0.5169413685798645, "mix_low": 0.004073143005371094, "mix_high": 0.001560211181640625, "read_cosine": 0.4084310531616211, "beta_mean": 0.0065434398129582405, "nonfinite": 0.0, "weighted_memory_to_active": 0.132651686668396, "mix_std": 0.27060848474502563, "beta_std": 0.007488240487873554, "beta_low": 0.804931640625, "beta_high": 0.0, "orthogonal_to_active": 0.09683921188116074, "evidence_weights": [-0.18420787155628204, -0.26115289330482483, 0.7619166970252991], "agreement_weight": 0.07953117787837982, "orthogonal_weight": -0.8384121656417847}, {"head": 3, "memory_to_active": 0.06445389986038208, "contribution_to_active": 0.6661014556884766, "mix_mean": 0.657969057559967, "mix_low": 0.0030012130737304688, "mix_high": 0.011995315551757812, "read_cosine": 0.44691768288612366, "beta_mean": 0.009877985343337059, "nonfinite": 0.0, "weighted_memory_to_active": 0.04447125643491745, "mix_std": 0.24430783092975616, "beta_std": 0.00872335210442543, "beta_low": 0.642333984375, "beta_high": 0.0, "orthogonal_to_active": 0.02293858677148819, "evidence_weights": [-0.3814419209957123, -0.32840755581855774, 0.45162034034729004], "agreement_weight": 0.07802493125200272, "orthogonal_weight": -0.6521357297897339}, {"head": 4, "memory_to_active": 0.1675349771976471, "contribution_to_active": 0.4875178635120392, "mix_mean": 0.450853168964386, "mix_low": 0.0035686492919921875, "mix_high": 0.0051708221435546875, "read_cosine": 0.7737998962402344, "beta_mean": 0.04667793586850166, "nonfinite": 0.0, "weighted_memory_to_active": 0.09265342354774475, "mix_std": 0.25998392701148987, "beta_std": 0.03372843563556671, "beta_low": 0.03118896484375, "beta_high": 0.0, "orthogonal_to_active": 0.027108337730169296, "evidence_weights": [-0.218867689371109, -0.26869142055511475, 0.3698645830154419], "agreement_weight": 0.11124560236930847, "orthogonal_weight": -0.49979305267333984}, {"head": 5, "memory_to_active": 0.3322635293006897, "contribution_to_active": 0.2733011841773987, "mix_mean": 0.278110146522522, "mix_low": 0.012479782104492188, "mix_high": 0.0005435943603515625, "read_cosine": 0.7127693891525269, "beta_mean": 0.003170380834490061, "nonfinite": 0.0, "weighted_memory_to_active": 0.13075223565101624, "mix_std": 0.23002612590789795, "beta_std": 0.012187761254608631, "beta_low": 0.93890380859375, "beta_high": 0.0, "orthogonal_to_active": 0.047977056354284286, "evidence_weights": [-0.1824226677417755, -0.14201423525810242, 0.16772715747356415], "agreement_weight": 0.0900343656539917, "orthogonal_weight": -0.5969516634941101}]}, {"layer": 15, "memory_to_active": 0.06720556318759918, "contribution_to_active": 0.47115281224250793, "mix_mean": 0.3894745707511902, "mix_low": 0.0071315765380859375, "mix_high": 0.0021804175339639187, "read_cosine": 0.712393045425415, "beta_mean": 0.03940870240330696, "nonfinite": 0.0, "weighted_memory_to_active": 0.030303796753287315, "mix_std": 0.2749205231666565, "beta_std": 0.04315593093633652, "beta_low": 0.7162882685661316, "beta_high": 0.0, "orthogonal_to_active": 0.004483634140342474, "output_influence": 0.2970188856124878, "residual_influence": 0.15600836277008057, "state_active": 0.11747434735298157, "state_memory": 0.007525707595050335, "state_peak": 33.926761627197266, "growth_active": 1.354096531867981, "growth_memory": 1.7123334407806396, "heads": [{"head": 0, "memory_to_active": 0.10099378228187561, "contribution_to_active": 0.47012463212013245, "mix_mean": 0.35543254017829895, "mix_low": 0.005852699279785156, "mix_high": 0.0026483535766601562, "read_cosine": 0.7415308952331543, "beta_mean": 0.03148914873600006, "nonfinite": 0.0, "weighted_memory_to_active": 0.04729185253381729, "mix_std": 0.266427606344223, "beta_std": 0.02327554300427437, "beta_low": 0.1552734375, "beta_high": 0.0, "orthogonal_to_active": 0.0139768086373806, "evidence_weights": [-0.16534192860126495, -0.19356085360050201, 0.342543363571167], "agreement_weight": 0.08134658634662628, "orthogonal_weight": -0.5520349144935608}, {"head": 1, "memory_to_active": 0.17104022204875946, "contribution_to_active": 0.4115143120288849, "mix_mean": 0.3906617760658264, "mix_low": 0.0051288604736328125, "mix_high": 0.0040264129638671875, "read_cosine": 0.8616263270378113, "beta_mean": 0.0014692990807816386, "nonfinite": 0.0, "weighted_memory_to_active": 0.06753569096326828, "mix_std": 0.2758346199989319, "beta_std": 0.003580888267606497, "beta_low": 0.9769287109375, "beta_high": 0.0, "orthogonal_to_active": 0.0030107111670076847, "evidence_weights": [-0.1890045702457428, -0.19475102424621582, 0.29007425904273987], "agreement_weight": 0.09629812091588974, "orthogonal_weight": -0.0705055519938469}, {"head": 2, "memory_to_active": 0.0034586796537041664, "contribution_to_active": 0.4474526643753052, "mix_mean": 0.3465709388256073, "mix_low": 0.0123291015625, "mix_high": 9.72747802734375e-05, "read_cosine": 0.7884476780891418, "beta_mean": 5.848926230100915e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0013843178749084473, "mix_std": 0.2691270411014557, "beta_std": 0.0001100709123420529, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00012506805069278926, "evidence_weights": [-0.2070883959531784, -0.2733355164527893, 0.31179019808769226], "agreement_weight": 0.02723650075495243, "orthogonal_weight": -0.16156524419784546}, {"head": 3, "memory_to_active": 0.022902967408299446, "contribution_to_active": 0.446022629737854, "mix_mean": 0.31401288509368896, "mix_low": 0.003911018371582031, "mix_high": 5.7220458984375e-06, "read_cosine": 0.6480910778045654, "beta_mean": 0.00026546555454842746, "nonfinite": 0.0, "weighted_memory_to_active": 0.009487181901931763, "mix_std": 0.24982580542564392, "beta_std": 0.0008087215246632695, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.0059744929894804955, "evidence_weights": [-0.18680614233016968, -0.2338486760854721, 0.43096402287483215], "agreement_weight": 0.008710145950317383, "orthogonal_weight": -0.8648732304573059}, {"head": 4, "memory_to_active": 0.09846941381692886, "contribution_to_active": 0.5873447060585022, "mix_mean": 0.40480828285217285, "mix_low": 0.009840011596679688, "mix_high": 0.0006456375122070312, "read_cosine": 0.6791470646858215, "beta_mean": 0.20298446714878082, "nonfinite": 0.0, "weighted_memory_to_active": 0.05348493903875351, "mix_std": 0.3069210946559906, "beta_std": 0.23084048926830292, "beta_low": 0.1671142578125, "beta_high": 0.0, "orthogonal_to_active": 0.0016751706134527922, "evidence_weights": [-0.24939197301864624, -0.2576035261154175, 0.13816754519939423], "agreement_weight": 0.012106471695005894, "orthogonal_weight": -0.05064438283443451}, {"head": 5, "memory_to_active": 0.006368331611156464, "contribution_to_active": 0.46445780992507935, "mix_mean": 0.5253608226776123, "mix_low": 0.0057277679443359375, "mix_high": 0.0056591033935546875, "read_cosine": 0.555514931678772, "beta_mean": 0.0001853249705163762, "nonfinite": 0.0, "weighted_memory_to_active": 0.002638796577230096, "mix_std": 0.28138676285743713, "beta_std": 0.00031986203975975513, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002139552729204297, "evidence_weights": [-0.3584372401237488, -0.30055224895477295, 0.39482763409614563], "agreement_weight": 0.15826483070850372, "orthogonal_weight": 0.937770426273346}]}]}, {"step": 20000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7857401559958817, "memory_to_active": 0.9259551167488098, "contribution_to_active": 0.4699249863624573, "mix_mean": 0.32492396235466003, "mix_low": 0.16456088423728943, "mix_high": 0.006284535396844149, "read_cosine": 0.39101681113243103, "beta_mean": 0.0874117910861969, "nonfinite": 0.0, "weighted_memory_to_active": 0.20541197061538696, "mix_std": 0.2124517858028412, "beta_std": 0.07133609801530838, "beta_low": 0.47519367933273315, "beta_high": 1.0172525435336865e-05, "orthogonal_to_active": 0.04401528090238571, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1438651084899902, "normal_token_accuracy": 0.44160178303718567, "state_active": 0.05839140713214874, "state_memory": 0.021436817944049835, "state_peak": 112.7640151977539, "growth_active": 1.1313588619232178, "growth_memory": 1.3891931772232056, "output_influence": 0.3269546329975128, "residual_influence": 0.10336124897003174, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2243311405181885, "no_memory_loss_delta": 0.1924554854631424, "scrambled_loss_delta": 0.21485158801078796, "memory_only_loss_delta": 2.2637860774993896, "no_evidence_loss_delta": 0.09797738492488861, "no_agreement_loss_delta": -0.007757091429084539, "no_orthogonal_loss_delta": 0.010083053261041641}, {"index": 94, "tokens": 256, "normal_loss": 3.1999528408050537, "no_memory_loss_delta": 0.33131515979766846, "scrambled_loss_delta": 0.43825533986091614, "memory_only_loss_delta": 2.593337059020996, "no_evidence_loss_delta": 0.17734000086784363, "no_agreement_loss_delta": 0.02406168170273304, "no_orthogonal_loss_delta": 0.02113236114382744}, {"index": 188, "tokens": 256, "normal_loss": 3.55053448677063, "no_memory_loss_delta": 0.20790433883666992, "scrambled_loss_delta": 0.1461765170097351, "memory_only_loss_delta": 2.3250465393066406, "no_evidence_loss_delta": 0.10682496428489685, "no_agreement_loss_delta": -0.0031498803291469812, "no_orthogonal_loss_delta": 0.009498624131083488}, {"index": 283, "tokens": 131, "normal_loss": 2.082301139831543, "no_memory_loss_delta": 0.11082693189382553, "scrambled_loss_delta": 0.19202056527137756, "memory_only_loss_delta": 2.513601303100586, "no_evidence_loss_delta": 0.027801446616649628, "no_agreement_loss_delta": -0.006061994470655918, "no_orthogonal_loss_delta": -0.005428946577012539}], "agreement_weight_abs_mean": 0.10357777029275894, "agreement_weight_abs_max": 2.056584596633911, "agreement_weight_mean": 0.08794490247964859, "orthogonal_weight_abs_mean": 0.5469009280204773, "orthogonal_weight_abs_max": 3.5152180194854736, "orthogonal_weight_mean": -0.1731872707605362, "no_memory_loss": 3.3683671951293945, "no_memory_loss_delta": 0.2245018035173416, "no_memory_token_accuracy_gain": 0.025583982467651367, "no_memory_token_win_fraction": 0.5995550751686096, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 20.108586415687586, "scrambled_loss": 3.3994505405426025, "scrambled_loss_delta": 0.25558537244796753, "scrambled_token_accuracy_gain": 0.02892102301120758, "scrambled_token_win_fraction": 0.6373748779296875, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 22.553698406540978, "memory_only_loss": 5.5553412437438965, "memory_only_loss_delta": 2.4114766120910645, "memory_only_token_accuracy_gain": 0.26696330308914185, "memory_only_token_win_fraction": 0.8776417970657349, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 91.03172299099744, "no_evidence_loss": 3.2567355632781982, "no_evidence_loss_delta": 0.11287034302949905, "no_evidence_token_accuracy_gain": 0.02002224698662758, "no_evidence_token_win_fraction": 0.5639599561691284, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 10.673352916752183, "no_agreement_loss": 3.146728038787842, "no_agreement_loss_delta": 0.002862607827410102, "no_agreement_token_accuracy_gain": 0.0033370412420481443, "no_agreement_token_win_fraction": 0.4449388086795807, "no_agreement_probe_win_fraction": 0.25, "no_agreement_ppl_reduction_pct": 0.28585144724448897, "no_orthogonal_loss": 3.154668092727661, "no_orthogonal_loss_delta": 0.010802672244608402, "no_orthogonal_token_accuracy_gain": 0.007786429487168789, "no_orthogonal_token_win_fraction": 0.5428253412246704, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.0744532922472154, "short_no_memory_loss_delta": 0.19287368655204773, "long_context_memory_gain": 0.031628116965293884, "normal_context_loss_gain": 0.15819820761680603, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.071373462677002, "contribution_to_active": 0.8818444609642029, "mix_mean": 0.09575606137514114, "mix_low": 0.501003623008728, "mix_high": 0.0033882460556924343, "read_cosine": 0.4735061526298523, "beta_mean": 0.26365500688552856, "nonfinite": 0.0, "weighted_memory_to_active": 0.9493892192840576, "mix_std": 0.08597321808338165, "beta_std": 0.15611319243907928, "beta_low": 0.0031433107797056437, "beta_high": 0.0, "orthogonal_to_active": 0.06027417629957199, "output_influence": 0.1315019428730011, "residual_influence": 0.17492692172527313, "state_active": 0.07595108449459076, "state_memory": 0.05011121556162834, "state_peak": 54.718414306640625, "growth_active": 1.2226417064666748, "growth_memory": 1.2766594886779785, "heads": [{"head": 0, "memory_to_active": 2.3226571083068848, "contribution_to_active": 0.34210705757141113, "mix_mean": 0.020919930189847946, "mix_low": 0.5459299087524414, "mix_high": 0.0, "read_cosine": 0.29624247550964355, "beta_mean": 0.2071285843849182, "nonfinite": 0.0, "weighted_memory_to_active": 0.42674458026885986, "mix_std": 0.055031102150678635, "beta_std": 0.1000497043132782, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03640969470143318, "evidence_weights": [-0.035889383405447006, -0.07560110092163086, -0.07120846211910248], "agreement_weight": -0.09582392871379852, "orthogonal_weight": -0.09801311045885086}, {"head": 1, "memory_to_active": 4.559730529785156, "contribution_to_active": 1.689454197883606, "mix_mean": 0.016243360936641693, "mix_low": 0.6058254241943359, "mix_high": 0.0, "read_cosine": 0.506759762763977, "beta_mean": 0.4089188277721405, "nonfinite": 0.0, "weighted_memory_to_active": 2.089012622833252, "mix_std": 0.04809991270303726, "beta_std": 0.2039768397808075, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17972616851329803, "evidence_weights": [0.016137970611453056, -0.05167688429355621, 0.20487375557422638], "agreement_weight": 0.02055133692920208, "orthogonal_weight": -0.19877244532108307}, {"head": 2, "memory_to_active": 2.622286319732666, "contribution_to_active": 0.5098452568054199, "mix_mean": 0.014130759052932262, "mix_low": 0.6600446701049805, "mix_high": 0.0, "read_cosine": 0.3428668975830078, "beta_mean": 0.33264508843421936, "nonfinite": 0.0, "weighted_memory_to_active": 0.6021404266357422, "mix_std": 0.031668975949287415, "beta_std": 0.20495228469371796, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06638233363628387, "evidence_weights": [0.04426541179418564, 0.05076214298605919, -0.010692283511161804], "agreement_weight": 0.0020570813212543726, "orthogonal_weight": -0.19342350959777832}, {"head": 3, "memory_to_active": 5.6458234786987305, "contribution_to_active": 2.3420095443725586, "mix_mean": 0.04763898253440857, "mix_low": 0.34073448181152344, "mix_high": 0.0, "read_cosine": 0.4137725830078125, "beta_mean": 0.3396297097206116, "nonfinite": 0.0, "weighted_memory_to_active": 2.4648170471191406, "mix_std": 0.08851427584886551, "beta_std": 0.23508919775485992, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05949873477220535, "evidence_weights": [-0.08725213259458542, -0.11411429941654205, -0.06853760033845901], "agreement_weight": -0.019659608602523804, "orthogonal_weight": -0.030868228524923325}, {"head": 4, "memory_to_active": 3.142746925354004, "contribution_to_active": 0.06391075998544693, "mix_mean": 0.006360071711242199, "mix_low": 0.8282241821289062, "mix_high": 0.0, "read_cosine": 0.335155725479126, "beta_mean": 0.2405945062637329, "nonfinite": 0.0, "weighted_memory_to_active": 0.061501964926719666, "mix_std": 0.01141962967813015, "beta_std": 0.14495010673999786, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007389990612864494, "evidence_weights": [0.08722227811813354, 0.03468118980526924, -0.3281955420970917], "agreement_weight": -0.034059084951877594, "orthogonal_weight": 0.1367342472076416}, {"head": 5, "memory_to_active": 0.13499516248703003, "contribution_to_active": 0.3437398076057434, "mix_mean": 0.46924322843551636, "mix_low": 0.025262832641601562, "mix_high": 0.02032947540283203, "read_cosine": 0.9462393522262573, "beta_mean": 0.05301319807767868, "nonfinite": 0.0, "weighted_memory_to_active": 0.05211831256747246, "mix_std": 0.2811053693294525, "beta_std": 0.047661032527685165, "beta_low": 0.01885986328125, "beta_high": 0.0, "orthogonal_to_active": 0.01223813183605671, "evidence_weights": [-0.27783769369125366, -0.25718745589256287, 0.2362900674343109], "agreement_weight": 0.1910986453294754, "orthogonal_weight": -0.4724012613296509}]}, {"layer": 1, "memory_to_active": 2.2582759857177734, "contribution_to_active": 0.4305683374404907, "mix_mean": 0.13587909936904907, "mix_low": 0.3809129595756531, "mix_high": 0.0005747477989643812, "read_cosine": 0.3355984091758728, "beta_mean": 0.20929645001888275, "nonfinite": 0.0, "weighted_memory_to_active": 0.3447777032852173, "mix_std": 0.11852771788835526, "beta_std": 0.14076033234596252, "beta_low": 0.1733093410730362, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.040699783712625504, "output_influence": 0.14860734343528748, "residual_influence": 0.03531990200281143, "state_active": 0.04000477492809296, "state_memory": 0.04070138931274414, "state_peak": 28.691524505615234, "growth_active": 1.127677321434021, "growth_memory": 1.5119056701660156, "heads": [{"head": 0, "memory_to_active": 0.04725131019949913, "contribution_to_active": 0.33928635716438293, "mix_mean": 0.31426751613616943, "mix_low": 0.0234832763671875, "mix_high": 0.000274658203125, "read_cosine": -0.04729297384619713, "beta_mean": 0.00025227805599570274, "nonfinite": 0.0, "weighted_memory_to_active": 0.02046506479382515, "mix_std": 0.26752549409866333, "beta_std": 0.0006767333834432065, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.01551910676062107, "evidence_weights": [-0.15250729024410248, -0.19121703505516052, -0.0591706819832325], "agreement_weight": 0.11855999380350113, "orthogonal_weight": -0.8324975371360779}, {"head": 1, "memory_to_active": 3.3081188201904297, "contribution_to_active": 0.17514854669570923, "mix_mean": 0.02181917056441307, "mix_low": 0.4783802032470703, "mix_high": 0.0, "read_cosine": 0.4541867673397064, "beta_mean": 0.3837687075138092, "nonfinite": 0.0, "weighted_memory_to_active": 0.1892615109682083, "mix_std": 0.0492972657084465, "beta_std": 0.23718291521072388, "beta_low": 0.0, "beta_high": 0.000244140625, "orthogonal_to_active": 0.00791575200855732, "evidence_weights": [0.0010694382945075631, -0.0020489038433879614, -0.021092846989631653], "agreement_weight": -0.008698660880327225, "orthogonal_weight": 0.07659541070461273}, {"head": 2, "memory_to_active": 0.49651774764060974, "contribution_to_active": 0.4408556818962097, "mix_mean": 0.4174213111400604, "mix_low": 0.00524139404296875, "mix_high": 0.0031728744506835938, "read_cosine": 0.8027436137199402, "beta_mean": 0.059210069477558136, "nonfinite": 0.0, "weighted_memory_to_active": 0.3818854093551636, "mix_std": 0.2598646879196167, "beta_std": 0.03585787117481232, "beta_low": 0.00677490234375, "beta_high": 0.0, "orthogonal_to_active": 0.020243532955646515, "evidence_weights": [-0.23944757878780365, -0.384865939617157, 0.22296567261219025], "agreement_weight": 0.01878499984741211, "orthogonal_weight": -0.1105383113026619}, {"head": 3, "memory_to_active": 2.837583303451538, "contribution_to_active": 0.06547857075929642, "mix_mean": 0.013800602406263351, "mix_low": 0.7172727584838867, "mix_high": 0.0, "read_cosine": 0.2163134217262268, "beta_mean": 0.353412389755249, "nonfinite": 0.0, "weighted_memory_to_active": 0.08778033405542374, "mix_std": 0.03956296294927597, "beta_std": 0.2352897822856903, "beta_low": 0.009765625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.023022914305329323, "evidence_weights": [0.04389806091785431, 0.014648453332483768, 0.08616481721401215], "agreement_weight": -0.01624627783894539, "orthogonal_weight": -0.2764277160167694}, {"head": 4, "memory_to_active": 3.2048139572143555, "contribution_to_active": 1.511815071105957, "mix_mean": 0.03938644006848335, "mix_low": 0.2644376754760742, "mix_high": 0.0, "read_cosine": 0.3749563694000244, "beta_mean": 0.17316417396068573, "nonfinite": 0.0, "weighted_memory_to_active": 1.3418288230895996, "mix_std": 0.07116319239139557, "beta_std": 0.10361704975366592, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.1753641813993454, "evidence_weights": [-0.026146862655878067, -0.005465567111968994, 0.06819210946559906], "agreement_weight": -0.025874560698866844, "orthogonal_weight": 0.15229645371437073}, {"head": 5, "memory_to_active": 3.6553702354431152, "contribution_to_active": 0.05082599073648453, "mix_mean": 0.00857958197593689, "mix_low": 0.7966623306274414, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21268321573734283, "beta_mean": 0.2859710454940796, "nonfinite": 0.0, "weighted_memory_to_active": 0.0474449023604393, "mix_std": 0.023752715438604355, "beta_std": 0.23193767666816711, "beta_low": 0.02337646484375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.002133200876414776, "evidence_weights": [0.13860692083835602, 0.06547137349843979, 0.3303748071193695], "agreement_weight": -0.13716131448745728, "orthogonal_weight": 0.04514465853571892}]}, {"layer": 2, "memory_to_active": 1.141968846321106, "contribution_to_active": 0.2925175726413727, "mix_mean": 0.2882106900215149, "mix_low": 0.26658758521080017, "mix_high": 0.002140045166015625, "read_cosine": 0.34868595004081726, "beta_mean": 0.10511825233697891, "nonfinite": 0.0, "weighted_memory_to_active": 0.05892857909202576, "mix_std": 0.1943180114030838, "beta_std": 0.07496491819620132, "beta_low": 0.54217529296875, "beta_high": 7.120767986634746e-05, "orthogonal_to_active": 0.015589414164423943, "output_influence": 0.2744773328304291, "residual_influence": 0.06643494963645935, "state_active": 0.04011921584606171, "state_memory": 0.021890690550208092, "state_peak": 41.444278717041016, "growth_active": 1.0469415187835693, "growth_memory": 1.5916229486465454, "heads": [{"head": 0, "memory_to_active": 0.010949506424367428, "contribution_to_active": 0.22055786848068237, "mix_mean": 0.3909805417060852, "mix_low": 0.007912635803222656, "mix_high": 0.0012941360473632812, "read_cosine": 0.23794275522232056, "beta_mean": 7.646398444194347e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00494797807186842, "mix_std": 0.29111260175704956, "beta_std": 9.381279232911766e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0002626061614137143, "evidence_weights": [-0.23190145194530487, -0.1514948606491089, 0.2674473226070404], "agreement_weight": 0.16936583817005157, "orthogonal_weight": -0.058617353439331055}, {"head": 1, "memory_to_active": 3.1215834617614746, "contribution_to_active": 0.05278410017490387, "mix_mean": 0.005774192977696657, "mix_low": 0.8845415115356445, "mix_high": 0.0, "read_cosine": 0.2047371119260788, "beta_mean": 0.31286001205444336, "nonfinite": 0.0, "weighted_memory_to_active": 0.04194668307900429, "mix_std": 0.01642354018986225, "beta_std": 0.1575288474559784, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013427220284938812, "evidence_weights": [0.09474201500415802, 0.10271898657083511, 0.3338708281517029], "agreement_weight": 0.025518840178847313, "orthogonal_weight": 0.3592908978462219}, {"head": 2, "memory_to_active": 0.02924272231757641, "contribution_to_active": 0.4104159474372864, "mix_mean": 0.4182513356208801, "mix_low": 0.0027561187744140625, "mix_high": 0.0019702911376953125, "read_cosine": 0.31567585468292236, "beta_mean": 5.224768028710969e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011615787632763386, "mix_std": 0.27718567848205566, "beta_std": 0.00010341421148041263, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026124315336346626, "evidence_weights": [-0.20417872071266174, -0.24472780525684357, 0.5416952967643738], "agreement_weight": 0.028777113184332848, "orthogonal_weight": -0.29315829277038574}, {"head": 3, "memory_to_active": 0.22987546026706696, "contribution_to_active": 0.4362810552120209, "mix_mean": 0.36778387427330017, "mix_low": 0.002788543701171875, "mix_high": 0.004504203796386719, "read_cosine": 0.617537260055542, "beta_mean": 0.04786432534456253, "nonfinite": 0.0, "weighted_memory_to_active": 0.10382598638534546, "mix_std": 0.26104509830474854, "beta_std": 0.06414991617202759, "beta_low": 0.226806640625, "beta_high": 0.0, "orthogonal_to_active": 0.06441839784383774, "evidence_weights": [-0.1430908441543579, -0.2177964299917221, 0.7715431451797485], "agreement_weight": 0.06123614311218262, "orthogonal_weight": -0.8261370062828064}, {"head": 4, "memory_to_active": 3.4426450729370117, "contribution_to_active": 0.147551029920578, "mix_mean": 0.01354922540485859, "mix_low": 0.6998138427734375, "mix_high": 0.0, "read_cosine": 0.20369160175323486, "beta_mean": 0.2698107361793518, "nonfinite": 0.0, "weighted_memory_to_active": 0.18133938312530518, "mix_std": 0.03822606801986694, "beta_std": 0.22784703969955444, "beta_low": 0.0262451171875, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.01222930196672678, "evidence_weights": [0.0457615964114666, 0.04307900741696358, 0.1426164209842682], "agreement_weight": 0.030348412692546844, "orthogonal_weight": -0.08161936700344086}, {"head": 5, "memory_to_active": 0.017516352236270905, "contribution_to_active": 0.487515389919281, "mix_mean": 0.5329248309135437, "mix_low": 0.001712799072265625, "mix_high": 0.0050716400146484375, "read_cosine": 0.5125311017036438, "beta_mean": 4.570027158479206e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009895626455545425, "mix_std": 0.28191494941711426, "beta_std": 6.646742986049503e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005865253624506295, "evidence_weights": [-0.23978914320468903, -0.26187822222709656, 0.7164101004600525], "agreement_weight": 0.04836297780275345, "orthogonal_weight": 0.10083664953708649}]}, {"layer": 3, "memory_to_active": 1.1703801155090332, "contribution_to_active": 0.35554179549217224, "mix_mean": 0.2385302186012268, "mix_low": 0.22100624442100525, "mix_high": 0.003329118248075247, "read_cosine": 0.2542736828327179, "beta_mean": 0.09400662779808044, "nonfinite": 0.0, "weighted_memory_to_active": 0.15558302402496338, "mix_std": 0.215742290019989, "beta_std": 0.07158683985471725, "beta_low": 0.4256693720817566, "beta_high": 0.0, "orthogonal_to_active": 0.0443902425467968, "output_influence": 0.264690101146698, "residual_influence": 0.048476122319698334, "state_active": 0.02733747847378254, "state_memory": 0.017797721549868584, "state_peak": 21.19858169555664, "growth_active": 1.0404342412948608, "growth_memory": 1.7612414360046387, "heads": [{"head": 0, "memory_to_active": 0.5857734680175781, "contribution_to_active": 0.4021153748035431, "mix_mean": 0.306885302066803, "mix_low": 0.07483482360839844, "mix_high": 0.003826141357421875, "read_cosine": 0.2541477084159851, "beta_mean": 0.014618460088968277, "nonfinite": 0.0, "weighted_memory_to_active": 0.2338184118270874, "mix_std": 0.3058416247367859, "beta_std": 0.01608360931277275, "beta_low": 0.4945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.12064504623413086, "evidence_weights": [-0.10852377116680145, -0.17227311432361603, 0.1748034656047821], "agreement_weight": 0.12967929244041443, "orthogonal_weight": -0.5592558979988098}, {"head": 1, "memory_to_active": 0.02372048795223236, "contribution_to_active": 0.28256750106811523, "mix_mean": 0.3124319911003113, "mix_low": 0.039463043212890625, "mix_high": 0.0017404556274414062, "read_cosine": 0.20210275053977966, "beta_mean": 0.00010560775990597904, "nonfinite": 0.0, "weighted_memory_to_active": 0.007470578886568546, "mix_std": 0.2834950387477875, "beta_std": 0.00019191682804375887, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00108176632784307, "evidence_weights": [-0.17186769843101501, -0.14074622094631195, 0.36147671937942505], "agreement_weight": 0.1851043403148651, "orthogonal_weight": -0.15879106521606445}, {"head": 2, "memory_to_active": 2.8141634464263916, "contribution_to_active": 0.3005003333091736, "mix_mean": 0.04528427496552467, "mix_low": 0.38417625427246094, "mix_high": 9.5367431640625e-07, "read_cosine": 0.1740262359380722, "beta_mean": 0.23343336582183838, "nonfinite": 0.0, "weighted_memory_to_active": 0.38768577575683594, "mix_std": 0.09886208176612854, "beta_std": 0.1550893932580948, "beta_low": 0.0042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.01873544044792652, "evidence_weights": [-0.052371539175510406, 0.01254623755812645, 0.028138447552919388], "agreement_weight": -0.010970097966492176, "orthogonal_weight": -0.06299666315317154}, {"head": 3, "memory_to_active": 0.024243604391813278, "contribution_to_active": 0.4761565029621124, "mix_mean": 0.4751482605934143, "mix_low": 0.024518966674804688, "mix_high": 0.009973526000976562, "read_cosine": 0.2755011320114136, "beta_mean": 0.0001008036924758926, "nonfinite": 0.0, "weighted_memory_to_active": 0.014190598390996456, "mix_std": 0.3179376721382141, "beta_std": 0.0002039204991888255, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00445577735081315, "evidence_weights": [-0.18487155437469482, -0.20616811513900757, 1.448899507522583], "agreement_weight": 0.8463566899299622, "orthogonal_weight": -0.40438857674598694}, {"head": 4, "memory_to_active": 3.1962904930114746, "contribution_to_active": 0.040606480091810226, "mix_mean": 0.010092196054756641, "mix_low": 0.7617654800415039, "mix_high": 0.0, "read_cosine": 0.2068888396024704, "beta_mean": 0.23175299167633057, "nonfinite": 0.0, "weighted_memory_to_active": 0.0717364251613617, "mix_std": 0.02427438087761402, "beta_std": 0.1776241809129715, "beta_low": 0.003173828125, "beta_high": 0.0, "orthogonal_to_active": 0.03441351652145386, "evidence_weights": [0.1474582701921463, 0.11442361772060394, -0.041787777096033096], "agreement_weight": -0.040867842733860016, "orthogonal_weight": -0.5014422535896301}, {"head": 5, "memory_to_active": 0.37808918952941895, "contribution_to_active": 0.6313045024871826, "mix_mean": 0.2813393175601959, "mix_low": 0.041278839111328125, "mix_high": 0.004433631896972656, "read_cosine": 0.4129753112792969, "beta_mean": 0.08402849733829498, "nonfinite": 0.0, "weighted_memory_to_active": 0.21859636902809143, "mix_std": 0.2640429735183716, "beta_std": 0.08032802492380142, "beta_low": 0.05206298828125, "beta_high": 0.0, "orthogonal_to_active": 0.08700989186763763, "evidence_weights": [-0.12319937348365784, -0.1869809627532959, 0.3079376816749573], "agreement_weight": 0.1702837198972702, "orthogonal_weight": -0.5554628372192383}]}, {"layer": 4, "memory_to_active": 1.2564324140548706, "contribution_to_active": 0.40175795555114746, "mix_mean": 0.2991517186164856, "mix_low": 0.2754976153373718, "mix_high": 0.003477891441434622, "read_cosine": 0.40225809812545776, "beta_mean": 0.13333669304847717, "nonfinite": 0.0, "weighted_memory_to_active": 0.16694995760917664, "mix_std": 0.1582065224647522, "beta_std": 0.1055026724934578, "beta_low": 0.348785400390625, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.046734195202589035, "output_influence": 0.2845117449760437, "residual_influence": 0.06931572407484055, "state_active": 0.03894045203924179, "state_memory": 0.02644307166337967, "state_peak": 48.45431137084961, "growth_active": 1.1137363910675049, "growth_memory": 1.5979526042938232, "heads": [{"head": 0, "memory_to_active": 0.32122862339019775, "contribution_to_active": 0.6293076276779175, "mix_mean": 0.552608847618103, "mix_low": 0.013332366943359375, "mix_high": 0.005188941955566406, "read_cosine": 0.5574561357498169, "beta_mean": 0.22944842278957367, "nonfinite": 0.0, "weighted_memory_to_active": 0.2371591031551361, "mix_std": 0.32530677318573, "beta_std": 0.24002167582511902, "beta_low": 0.17669677734375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.04526130110025406, "evidence_weights": [-0.24160286784172058, -0.3167820870876312, 0.42138534784317017], "agreement_weight": 0.03907013684511185, "orthogonal_weight": 0.2504490911960602}, {"head": 1, "memory_to_active": 0.16818049550056458, "contribution_to_active": 0.4717893600463867, "mix_mean": 0.5033957958221436, "mix_low": 0.00029277801513671875, "mix_high": 0.0033702850341796875, "read_cosine": 0.744683027267456, "beta_mean": 0.005455205217003822, "nonfinite": 0.0, "weighted_memory_to_active": 0.08997824788093567, "mix_std": 0.26773473620414734, "beta_std": 0.00841966737061739, "beta_low": 0.8665771484375, "beta_high": 0.0, "orthogonal_to_active": 0.039555471390485764, "evidence_weights": [-0.23082324862480164, -0.23743033409118652, 0.6665874123573303], "agreement_weight": 0.053089503198862076, "orthogonal_weight": -0.7018334269523621}, {"head": 2, "memory_to_active": 2.4914438724517822, "contribution_to_active": 0.12865349650382996, "mix_mean": 0.015164075419306755, "mix_low": 0.615325927734375, "mix_high": 0.0, "read_cosine": 0.29682496190071106, "beta_mean": 0.1423778235912323, "nonfinite": 0.0, "weighted_memory_to_active": 0.15408799052238464, "mix_std": 0.027435552328824997, "beta_std": 0.09440292418003082, "beta_low": 0.0013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.008898583240807056, "evidence_weights": [0.049421314150094986, 0.028691163286566734, -0.0558178573846817], "agreement_weight": 0.067258819937706, "orthogonal_weight": -0.07995229214429855}, {"head": 3, "memory_to_active": 1.6126781702041626, "contribution_to_active": 0.1100459024310112, "mix_mean": 0.0299631766974926, "mix_low": 0.3953208923339844, "mix_high": 0.0, "read_cosine": 0.21649953722953796, "beta_mean": 0.13481593132019043, "nonfinite": 0.0, "weighted_memory_to_active": 0.17166733741760254, "mix_std": 0.050607092678546906, "beta_std": 0.12178026139736176, "beta_low": 0.048095703125, "beta_high": 0.0, "orthogonal_to_active": 0.017041049897670746, "evidence_weights": [-0.01912929117679596, 0.05321769416332245, -0.2389156073331833], "agreement_weight": 0.03710624948143959, "orthogonal_weight": -0.15495167672634125}, {"head": 4, "memory_to_active": 0.037866126745939255, "contribution_to_active": 0.6238768696784973, "mix_mean": 0.6793283820152283, "mix_low": 0.0004930496215820312, "mix_high": 0.012301445007324219, "read_cosine": 0.24572275578975677, "beta_mean": 6.071654934203252e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.028264809399843216, "mix_std": 0.24340036511421204, "beta_std": 0.0002005203568842262, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003396999789401889, "evidence_weights": [-0.2919526994228363, -0.33338356018066406, 0.7726272344589233], "agreement_weight": 0.09650083631277084, "orthogonal_weight": -0.14226149022579193}, {"head": 5, "memory_to_active": 2.9071969985961914, "contribution_to_active": 0.44687432050704956, "mix_mean": 0.014449995011091232, "mix_low": 0.6282205581665039, "mix_high": 6.67572021484375e-06, "read_cosine": 0.3523620367050171, "beta_mean": 0.28786200284957886, "nonfinite": 0.0, "weighted_memory_to_active": 0.3205421566963196, "mix_std": 0.034754592925310135, "beta_std": 0.16819097101688385, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1662517488002777, "evidence_weights": [0.04139329493045807, -0.0318438895046711, -0.17500872910022736], "agreement_weight": -0.046337470412254333, "orthogonal_weight": 0.7152034044265747}]}, {"layer": 5, "memory_to_active": 0.609402060508728, "contribution_to_active": 0.4581141173839569, "mix_mean": 0.4069560170173645, "mix_low": 0.09084733575582504, "mix_high": 0.011302472092211246, "read_cosine": 0.18782420456409454, "beta_mean": 0.055357348173856735, "nonfinite": 0.0, "weighted_memory_to_active": 0.16600468754768372, "mix_std": 0.2536589205265045, "beta_std": 0.071175716817379, "beta_low": 0.6144816279411316, "beta_high": 0.0, "orthogonal_to_active": 0.05391595512628555, "output_influence": 0.4336903989315033, "residual_influence": 0.10292556136846542, "state_active": 0.054182279855012894, "state_memory": 0.014221366494894028, "state_peak": 67.8643569946289, "growth_active": 1.1154086589813232, "growth_memory": 1.263022780418396, "heads": [{"head": 0, "memory_to_active": 1.1726415157318115, "contribution_to_active": 0.30711740255355835, "mix_mean": 0.14798559248447418, "mix_low": 0.14116668701171875, "mix_high": 2.86102294921875e-06, "read_cosine": -0.21972505748271942, "beta_mean": 0.049149300903081894, "nonfinite": 0.0, "weighted_memory_to_active": 0.18704432249069214, "mix_std": 0.19588254392147064, "beta_std": 0.08101045340299606, "beta_low": 0.22705078125, "beta_high": 0.0, "orthogonal_to_active": 0.04283132776618004, "evidence_weights": [-0.052033986896276474, -0.08924352377653122, -0.3615890443325043], "agreement_weight": -0.015349216759204865, "orthogonal_weight": -0.21727354824543}, {"head": 1, "memory_to_active": 0.028276484459638596, "contribution_to_active": 0.5037840604782104, "mix_mean": 0.5552347898483276, "mix_low": 0.0075206756591796875, "mix_high": 0.014589309692382812, "read_cosine": 0.15513202548027039, "beta_mean": 7.088725396897644e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014160960912704468, "mix_std": 0.3150571286678314, "beta_std": 0.00011746353993657976, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012580826878547668, "evidence_weights": [-0.22599080204963684, -0.24594834446907043, 0.7703348398208618], "agreement_weight": 0.30616495013237, "orthogonal_weight": -0.9244974851608276}, {"head": 2, "memory_to_active": 0.049735184758901596, "contribution_to_active": 0.5093169212341309, "mix_mean": 0.5833666324615479, "mix_low": 0.0022401809692382812, "mix_high": 0.014288902282714844, "read_cosine": 0.24853189289569855, "beta_mean": 0.0009340584510937333, "nonfinite": 0.0, "weighted_memory_to_active": 0.03354061394929886, "mix_std": 0.28269898891448975, "beta_std": 0.0012893332168459892, "beta_low": 0.99749755859375, "beta_high": 0.0, "orthogonal_to_active": 0.005014071241021156, "evidence_weights": [-0.2609994113445282, -0.3218546509742737, 0.7680574655532837], "agreement_weight": 0.029430503025650978, "orthogonal_weight": -0.19285935163497925}, {"head": 3, "memory_to_active": 1.8474085330963135, "contribution_to_active": 0.2180103361606598, "mix_mean": 0.08589909970760345, "mix_low": 0.2040414810180664, "mix_high": 9.5367431640625e-07, "read_cosine": 0.17302298545837402, "beta_mean": 0.06771564483642578, "nonfinite": 0.0, "weighted_memory_to_active": 0.3842279016971588, "mix_std": 0.12991416454315186, "beta_std": 0.10656020790338516, "beta_low": 0.2447509765625, "beta_high": 0.0, "orthogonal_to_active": 0.1345238834619522, "evidence_weights": [-0.03372294083237648, -0.08112408965826035, -0.05439252406358719], "agreement_weight": 0.047845788300037384, "orthogonal_weight": -0.4621654748916626}, {"head": 4, "memory_to_active": 0.44167453050613403, "contribution_to_active": 0.6061007380485535, "mix_mean": 0.3310595750808716, "mix_low": 0.18676471710205078, "mix_high": 0.009696006774902344, "read_cosine": 0.5609972476959229, "beta_mean": 0.21382424235343933, "nonfinite": 0.0, "weighted_memory_to_active": 0.2993986904621124, "mix_std": 0.34009820222854614, "beta_std": 0.23579511046409607, "beta_low": 0.22509765625, "beta_high": 0.0, "orthogonal_to_active": 0.08096186816692352, "evidence_weights": [-0.1268952488899231, -0.2887250781059265, -0.028603089973330498], "agreement_weight": 0.043518684804439545, "orthogonal_weight": 0.40333500504493713}, {"head": 5, "memory_to_active": 0.11667590588331223, "contribution_to_active": 0.6043552160263062, "mix_mean": 0.7381902933120728, "mix_low": 0.0033502578735351562, "mix_high": 0.029236793518066406, "read_cosine": 0.20898616313934326, "beta_mean": 0.00044994510244578123, "nonfinite": 0.0, "weighted_memory_to_active": 0.07765553891658783, "mix_std": 0.2583024501800537, "beta_std": 0.0022817235440015793, "beta_low": 0.99249267578125, "beta_high": 0.0, "orthogonal_to_active": 0.047583743929862976, "evidence_weights": [-0.322623074054718, -0.30995187163352966, 1.761615514755249], "agreement_weight": 0.33200597763061523, "orthogonal_weight": -0.6545518636703491}]}, {"layer": 6, "memory_to_active": 0.4149811863899231, "contribution_to_active": 0.5680189728736877, "mix_mean": 0.5120030045509338, "mix_low": 0.08083248138427734, "mix_high": 0.02284097857773304, "read_cosine": 0.2619304656982422, "beta_mean": 0.0714520812034607, "nonfinite": 0.0, "weighted_memory_to_active": 0.1790560781955719, "mix_std": 0.25080156326293945, "beta_std": 0.08161747455596924, "beta_low": 0.5218505859375, "beta_high": 0.0, "orthogonal_to_active": 0.03878437727689743, "output_influence": 0.4588526487350464, "residual_influence": 0.13814766705036163, "state_active": 0.04925759136676788, "state_memory": 0.010141528211534023, "state_peak": 72.82765197753906, "growth_active": 1.1680667400360107, "growth_memory": 1.2282500267028809, "heads": [{"head": 0, "memory_to_active": 0.15423601865768433, "contribution_to_active": 0.5609017014503479, "mix_mean": 0.6757762432098389, "mix_low": 0.0001010894775390625, "mix_high": 0.004230499267578125, "read_cosine": -0.04779829829931259, "beta_mean": 0.009928352199494839, "nonfinite": 0.0, "weighted_memory_to_active": 0.1112232580780983, "mix_std": 0.24020400643348694, "beta_std": 0.014750314876437187, "beta_low": 0.6783447265625, "beta_high": 0.0, "orthogonal_to_active": 0.08697792887687683, "evidence_weights": [-0.3068617880344391, -0.3129733204841614, 0.48833778500556946], "agreement_weight": 0.10326942801475525, "orthogonal_weight": -0.9084189534187317}, {"head": 1, "memory_to_active": 0.005470034666359425, "contribution_to_active": 0.4533511996269226, "mix_mean": 0.4716666340827942, "mix_low": 0.0014753341674804688, "mix_high": 0.009696006774902344, "read_cosine": 0.5683422088623047, "beta_mean": 1.3188657248974778e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0020492931362241507, "mix_std": 0.2937450706958771, "beta_std": 3.529198147589341e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005702116759493947, "evidence_weights": [-0.21190708875656128, -0.26355934143066406, 0.7665730714797974], "agreement_weight": -0.03138783574104309, "orthogonal_weight": -0.386309415102005}, {"head": 2, "memory_to_active": 1.3166695833206177, "contribution_to_active": 0.06105031445622444, "mix_mean": 0.025091882795095444, "mix_low": 0.46759700775146484, "mix_high": 0.0, "read_cosine": 0.20482710003852844, "beta_mean": 0.07487355917692184, "nonfinite": 0.0, "weighted_memory_to_active": 0.06854508817195892, "mix_std": 0.047195471823215485, "beta_std": 0.06621750444173813, "beta_low": 0.00982666015625, "beta_high": 0.0, "orthogonal_to_active": 0.009909271262586117, "evidence_weights": [0.08647292107343674, 0.017844760790467262, 0.21504084765911102], "agreement_weight": 0.0951431393623352, "orthogonal_weight": -0.16083209216594696}, {"head": 3, "memory_to_active": 0.14717638492584229, "contribution_to_active": 0.6494587063789368, "mix_mean": 0.6080858707427979, "mix_low": 0.012813568115234375, "mix_high": 0.016048431396484375, "read_cosine": 0.46821606159210205, "beta_mean": 0.2156074196100235, "nonfinite": 0.0, "weighted_memory_to_active": 0.10519517958164215, "mix_std": 0.34122300148010254, "beta_std": 0.2561147212982178, "beta_low": 0.22406005859375, "beta_high": 0.0, "orthogonal_to_active": 0.04825771972537041, "evidence_weights": [-0.1748685985803604, -0.3342309892177582, 0.15725310146808624], "agreement_weight": -0.012351668439805508, "orthogonal_weight": 0.5345379710197449}, {"head": 4, "memory_to_active": 0.8103587627410889, "contribution_to_active": 1.1201454401016235, "mix_mean": 0.7217060327529907, "mix_low": 0.0021314620971679688, "mix_high": 0.1055145263671875, "read_cosine": 0.11703253537416458, "beta_mean": 0.12786777317523956, "nonfinite": 0.0, "weighted_memory_to_active": 0.7505103945732117, "mix_std": 0.30267810821533203, "beta_std": 0.15143586695194244, "beta_low": 0.2215576171875, "beta_high": 0.0, "orthogonal_to_active": 0.04995378106832504, "evidence_weights": [-0.2521388530731201, -0.5710305571556091, -0.05474621057510376], "agreement_weight": -0.013932581059634686, "orthogonal_weight": 0.07972614467144012}, {"head": 5, "memory_to_active": 0.05597642436623573, "contribution_to_active": 0.5632064938545227, "mix_mean": 0.5696910619735718, "mix_low": 0.0008764266967773438, "mix_high": 0.001556396484375, "read_cosine": 0.2609630227088928, "beta_mean": 0.0004221749259158969, "nonfinite": 0.0, "weighted_memory_to_active": 0.03681328892707825, "mix_std": 0.27976366877555847, "beta_std": 0.0011511038755998015, "beta_low": 0.997314453125, "beta_high": 0.0, "orthogonal_to_active": 0.037037353962659836, "evidence_weights": [-0.23173440992832184, -0.282297819852829, 0.36200353503227234], "agreement_weight": 0.08614134043455124, "orthogonal_weight": -1.072432518005371}]}, {"layer": 7, "memory_to_active": 1.9129903316497803, "contribution_to_active": 0.19374628365039825, "mix_mean": 0.0448262020945549, "mix_low": 0.4392945170402527, "mix_high": 1.5735626220703125e-05, "read_cosine": 0.22467535734176636, "beta_mean": 0.11864404380321503, "nonfinite": 0.0, "weighted_memory_to_active": 0.2096433788537979, "mix_std": 0.07345615327358246, "beta_std": 0.10312885046005249, "beta_low": 0.084808349609375, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.04239562153816223, "output_influence": 0.20419393479824066, "residual_influence": 0.044602248817682266, "state_active": 0.014853492379188538, "state_memory": 0.02860300987958908, "state_peak": 25.662622451782227, "growth_active": 1.0711053609848022, "growth_memory": 1.3578717708587646, "heads": [{"head": 0, "memory_to_active": 2.2222180366516113, "contribution_to_active": 0.4439848065376282, "mix_mean": 0.02875647321343422, "mix_low": 0.5094318389892578, "mix_high": 2.288818359375e-05, "read_cosine": 0.3505237400531769, "beta_mean": 0.15584629774093628, "nonfinite": 0.0, "weighted_memory_to_active": 0.4182804226875305, "mix_std": 0.06309964507818222, "beta_std": 0.12813666462898254, "beta_low": 0.00457763671875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.041852664202451706, "evidence_weights": [0.002325694542378187, -0.08588524907827377, 0.10690661519765854], "agreement_weight": -0.03873828053474426, "orthogonal_weight": 0.14691902697086334}, {"head": 1, "memory_to_active": 1.669418454170227, "contribution_to_active": 0.28452062606811523, "mix_mean": 0.11258973926305771, "mix_low": 0.2501106262207031, "mix_high": 5.054473876953125e-05, "read_cosine": 0.17454415559768677, "beta_mean": 0.15954822301864624, "nonfinite": 0.0, "weighted_memory_to_active": 0.3426291346549988, "mix_std": 0.15051871538162231, "beta_std": 0.09539766609668732, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0494440533220768, "evidence_weights": [-0.14657588303089142, -0.10789662599563599, -0.21060208976268768], "agreement_weight": 0.011234312318265438, "orthogonal_weight": -0.15826046466827393}, {"head": 2, "memory_to_active": 1.1577916145324707, "contribution_to_active": 0.23454596102237701, "mix_mean": 0.06390643864870071, "mix_low": 0.24478626251220703, "mix_high": 2.09808349609375e-05, "read_cosine": 0.12427148222923279, "beta_mean": 0.09096191078424454, "nonfinite": 0.0, "weighted_memory_to_active": 0.17501530051231384, "mix_std": 0.1147691011428833, "beta_std": 0.06389300525188446, "beta_low": 0.03778076171875, "beta_high": 0.0, "orthogonal_to_active": 0.02323489636182785, "evidence_weights": [-0.03207209333777428, -0.11576439440250397, 0.22230301797389984], "agreement_weight": 0.024866055697202682, "orthogonal_weight": -0.1357640027999878}, {"head": 3, "memory_to_active": 3.3152549266815186, "contribution_to_active": 0.05687517300248146, "mix_mean": 0.022518455982208252, "mix_low": 0.4548044204711914, "mix_high": 0.0, "read_cosine": 0.12559688091278076, "beta_mean": 0.1677320897579193, "nonfinite": 0.0, "weighted_memory_to_active": 0.1546381115913391, "mix_std": 0.03430899977684021, "beta_std": 0.17732471227645874, "beta_low": 0.0457763671875, "beta_high": 0.0, "orthogonal_to_active": 0.11456110328435898, "evidence_weights": [0.0924898311495781, 0.047826070338487625, 0.4453659653663635], "agreement_weight": 0.07204556465148926, "orthogonal_weight": -0.8266053795814514}, {"head": 4, "memory_to_active": 1.7099008560180664, "contribution_to_active": 0.06345836818218231, "mix_mean": 0.010996829718351364, "mix_low": 0.7396640777587891, "mix_high": 0.0, "read_cosine": 0.3618555963039398, "beta_mean": 0.10758401453495026, "nonfinite": 0.0, "weighted_memory_to_active": 0.0666244849562645, "mix_std": 0.025915343314409256, "beta_std": 0.10972028970718384, "beta_low": 0.05963134765625, "beta_high": 0.0, "orthogonal_to_active": 0.0026897257193922997, "evidence_weights": [0.10709129273891449, 0.052325762808322906, 0.0449993871152401], "agreement_weight": 0.04387996718287468, "orthogonal_weight": -0.05947992205619812}, {"head": 5, "memory_to_active": 1.4033575057983398, "contribution_to_active": 0.07909277826547623, "mix_mean": 0.030189285054802895, "mix_low": 0.4369697570800781, "mix_high": 0.0, "read_cosine": 0.21126024425029755, "beta_mean": 0.030191659927368164, "nonfinite": 0.0, "weighted_memory_to_active": 0.10067278146743774, "mix_std": 0.05212508141994476, "beta_std": 0.04430074244737625, "beta_low": 0.361083984375, "beta_high": 0.0, "orthogonal_to_active": 0.02259128727018833, "evidence_weights": [0.05311325564980507, 0.03756674379110336, 0.04266608506441116], "agreement_weight": 0.03556077554821968, "orthogonal_weight": -0.3098004460334778}]}, {"layer": 8, "memory_to_active": 0.572452962398529, "contribution_to_active": 0.4912436008453369, "mix_mean": 0.3734557032585144, "mix_low": 0.08444007486104965, "mix_high": 0.0007332166424021125, "read_cosine": 0.532145082950592, "beta_mean": 0.03382563218474388, "nonfinite": 0.0, "weighted_memory_to_active": 0.1370822638273239, "mix_std": 0.22601886093616486, "beta_std": 0.03791186213493347, "beta_low": 0.5107218623161316, "beta_high": 0.0, "orthogonal_to_active": 0.053399838507175446, "output_influence": 0.36759239435195923, "residual_influence": 0.10444149374961853, "state_active": 0.05503671243786812, "state_memory": 0.012544050812721252, "state_peak": 34.39226531982422, "growth_active": 1.0748844146728516, "growth_memory": 1.3690862655639648, "heads": [{"head": 0, "memory_to_active": 0.04134466126561165, "contribution_to_active": 0.5161693692207336, "mix_mean": 0.5496274828910828, "mix_low": 0.00018596649169921875, "mix_high": 0.00047779083251953125, "read_cosine": 0.6795294284820557, "beta_mean": 0.003108678152784705, "nonfinite": 0.0, "weighted_memory_to_active": 0.024045666679739952, "mix_std": 0.24079322814941406, "beta_std": 0.006924206856638193, "beta_low": 0.9283447265625, "beta_high": 0.0, "orthogonal_to_active": 0.013737259432673454, "evidence_weights": [-0.19843539595603943, -0.3628000319004059, 0.7308641672134399], "agreement_weight": 0.06624801456928253, "orthogonal_weight": -0.8059382438659668}, {"head": 1, "memory_to_active": 0.4447817802429199, "contribution_to_active": 0.38641712069511414, "mix_mean": 0.3850401043891907, "mix_low": 0.0030775070190429688, "mix_high": 0.00034236907958984375, "read_cosine": 0.4635901153087616, "beta_mean": 0.019075734540820122, "nonfinite": 0.0, "weighted_memory_to_active": 0.19662734866142273, "mix_std": 0.24747909605503082, "beta_std": 0.022385280579328537, "beta_low": 0.3907470703125, "beta_high": 0.0, "orthogonal_to_active": 0.07627098262310028, "evidence_weights": [-0.1767576038837433, -0.20832712948322296, 0.492035835981369], "agreement_weight": 0.07025043666362762, "orthogonal_weight": -0.4544019401073456}, {"head": 2, "memory_to_active": 0.07162053883075714, "contribution_to_active": 0.5776925683021545, "mix_mean": 0.5341652631759644, "mix_low": 0.0007982254028320312, "mix_high": 0.0018930435180664062, "read_cosine": 0.38402435183525085, "beta_mean": 0.0008330655400641263, "nonfinite": 0.0, "weighted_memory_to_active": 0.04638645052909851, "mix_std": 0.2530008852481842, "beta_std": 0.0027377584483474493, "beta_low": 0.99395751953125, "beta_high": 0.0, "orthogonal_to_active": 0.034016549587249756, "evidence_weights": [-0.2063557356595993, -0.312593549489975, 0.5493091940879822], "agreement_weight": 0.089146688580513, "orthogonal_weight": -0.8445866703987122}, {"head": 3, "memory_to_active": 0.17462435364723206, "contribution_to_active": 0.5091097950935364, "mix_mean": 0.4300365447998047, "mix_low": 0.0015687942504882812, "mix_high": 0.00015735626220703125, "read_cosine": 0.7474733591079712, "beta_mean": 0.012979588471353054, "nonfinite": 0.0, "weighted_memory_to_active": 0.10696347057819366, "mix_std": 0.24642424285411835, "beta_std": 0.013700658455491066, "beta_low": 0.486572265625, "beta_high": 0.0, "orthogonal_to_active": 0.02735363505780697, "evidence_weights": [-0.13770774006843567, -0.2530210316181183, 0.7378722429275513], "agreement_weight": 0.09856384247541428, "orthogonal_weight": -0.436174213886261}, {"head": 4, "memory_to_active": 2.477206230163574, "contribution_to_active": 0.2994920611381531, "mix_mean": 0.027588944882154465, "mix_low": 0.4093809127807617, "mix_high": 0.0, "read_cosine": 0.35263556241989136, "beta_mean": 0.09168727695941925, "nonfinite": 0.0, "weighted_memory_to_active": 0.2966417670249939, "mix_std": 0.046734344214200974, "beta_std": 0.07486789673566818, "beta_low": 0.00958251953125, "beta_high": 0.0, "orthogonal_to_active": 0.008493423461914062, "evidence_weights": [0.03185107558965683, -0.010171215049922466, -0.17158280313014984], "agreement_weight": 0.019397376105189323, "orthogonal_weight": 0.030381472781300545}, {"head": 5, "memory_to_active": 0.22514019906520844, "contribution_to_active": 0.6585807800292969, "mix_mean": 0.31427592039108276, "mix_low": 0.0916290283203125, "mix_high": 0.0015287399291992188, "read_cosine": 0.565617561340332, "beta_mean": 0.07526946067810059, "nonfinite": 0.0, "weighted_memory_to_active": 0.1518288552761078, "mix_std": 0.32168135046958923, "beta_std": 0.10685538500547409, "beta_low": 0.255126953125, "beta_high": 0.0, "orthogonal_to_active": 0.16052716970443726, "evidence_weights": [-0.06925085932016373, -0.3161165714263916, -0.4201856553554535], "agreement_weight": 0.03332358971238136, "orthogonal_weight": 1.317206859588623}]}, {"layer": 9, "memory_to_active": 1.147193193435669, "contribution_to_active": 0.5804600715637207, "mix_mean": 0.2637072801589966, "mix_low": 0.16666826605796814, "mix_high": 0.0014977455139160156, "read_cosine": 0.366848349571228, "beta_mean": 0.0799785628914833, "nonfinite": 0.0, "weighted_memory_to_active": 0.35757899284362793, "mix_std": 0.21019643545150757, "beta_std": 0.047656551003456116, "beta_low": 0.4980469048023224, "beta_high": 0.0, "orthogonal_to_active": 0.03403802961111069, "output_influence": 0.4130062758922577, "residual_influence": 0.10776408016681671, "state_active": 0.04378759115934372, "state_memory": 0.02324634976685047, "state_peak": 33.94084930419922, "growth_active": 1.3052456378936768, "growth_memory": 1.442679524421692, "heads": [{"head": 0, "memory_to_active": 0.006304726004600525, "contribution_to_active": 0.47350382804870605, "mix_mean": 0.5345158576965332, "mix_low": 0.0020599365234375, "mix_high": 0.005429267883300781, "read_cosine": 0.19258064031600952, "beta_mean": 1.1688737686199602e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034941243939101696, "mix_std": 0.29641643166542053, "beta_std": 5.6386328651569784e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029327396769076586, "evidence_weights": [-0.2725587487220764, -0.27401450276374817, 0.9928678274154663], "agreement_weight": 0.06535153090953827, "orthogonal_weight": -0.8710848689079285}, {"head": 1, "memory_to_active": 0.032759882509708405, "contribution_to_active": 0.5174217224121094, "mix_mean": 0.3848949074745178, "mix_low": 0.011919975280761719, "mix_high": 0.0011539459228515625, "read_cosine": 0.40535420179367065, "beta_mean": 9.572102862875909e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01640569604933262, "mix_std": 0.2835698127746582, "beta_std": 0.00024099121219478548, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008955705910921097, "evidence_weights": [-0.20704405009746552, -0.23037810623645782, 0.995790421962738], "agreement_weight": 0.28035104274749756, "orthogonal_weight": -0.6231494545936584}, {"head": 2, "memory_to_active": 0.253256618976593, "contribution_to_active": 0.5488230586051941, "mix_mean": 0.44608545303344727, "mix_low": 0.005942344665527344, "mix_high": 0.0021791458129882812, "read_cosine": 0.38749462366104126, "beta_mean": 0.0012468879576772451, "nonfinite": 0.0, "weighted_memory_to_active": 0.14561864733695984, "mix_std": 0.2864925265312195, "beta_std": 0.005993746221065521, "beta_low": 0.98577880859375, "beta_high": 0.0, "orthogonal_to_active": 0.04890507832169533, "evidence_weights": [-0.16521304845809937, -0.1974191516637802, 0.4901469349861145], "agreement_weight": 0.1984507441520691, "orthogonal_weight": -0.4122929275035858}, {"head": 3, "memory_to_active": 1.6632306575775146, "contribution_to_active": 0.1751186102628708, "mix_mean": 0.03715498000383377, "mix_low": 0.4235687255859375, "mix_high": 2.002716064453125e-05, "read_cosine": 0.43517881631851196, "beta_mean": 0.09610281884670258, "nonfinite": 0.0, "weighted_memory_to_active": 0.1393308788537979, "mix_std": 0.09008882939815521, "beta_std": 0.06533896178007126, "beta_low": 0.00244140625, "beta_high": 0.0, "orthogonal_to_active": 0.02733236365020275, "evidence_weights": [0.05568699911236763, 0.039799779653549194, -0.14890870451927185], "agreement_weight": 0.06617342680692673, "orthogonal_weight": 0.19824738800525665}, {"head": 4, "memory_to_active": 3.233029365539551, "contribution_to_active": 1.202816367149353, "mix_mean": 0.11798091232776642, "mix_low": 0.1804189682006836, "mix_high": 0.0002040863037109375, "read_cosine": 0.32075172662734985, "beta_mean": 0.21512167155742645, "nonfinite": 0.0, "weighted_memory_to_active": 1.172795295715332, "mix_std": 0.18442071974277496, "beta_std": 0.10825447738170624, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016885682940483093, "evidence_weights": [-0.0493588000535965, -0.13803808391094208, 0.03543294593691826], "agreement_weight": -0.0476900115609169, "orthogonal_weight": -0.014514158479869366}, {"head": 5, "memory_to_active": 1.6945781707763672, "contribution_to_active": 0.565076470375061, "mix_mean": 0.06161164492368698, "mix_low": 0.3760995864868164, "mix_high": 0.0, "read_cosine": 0.45972999930381775, "beta_mean": 0.16729256510734558, "nonfinite": 0.0, "weighted_memory_to_active": 0.6678292751312256, "mix_std": 0.12019023299217224, "beta_std": 0.10605470836162567, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.09921661019325256, "evidence_weights": [0.061642345041036606, -0.04424749314785004, 0.09915850311517715], "agreement_weight": 0.01788109540939331, "orthogonal_weight": 0.2132803052663803}]}, {"layer": 10, "memory_to_active": 0.16911597549915314, "contribution_to_active": 0.4559210538864136, "mix_mean": 0.43929988145828247, "mix_low": 0.012405237182974815, "mix_high": 0.011605899780988693, "read_cosine": 0.3834116756916046, "beta_mean": 0.04846814647316933, "nonfinite": 0.0, "weighted_memory_to_active": 0.07320349663496017, "mix_std": 0.26912131905555725, "beta_std": 0.05147522687911987, "beta_low": 0.64862060546875, "beta_high": 0.0, "orthogonal_to_active": 0.0428437814116478, "output_influence": 0.3820064663887024, "residual_influence": 0.12464688718318939, "state_active": 0.05749015510082245, "state_memory": 0.014473140239715576, "state_peak": 112.7640151977539, "growth_active": 1.089315414428711, "growth_memory": 1.0917127132415771, "heads": [{"head": 0, "memory_to_active": 0.36343660950660706, "contribution_to_active": 0.7591022849082947, "mix_mean": 0.6312808990478516, "mix_low": 0.02084064483642578, "mix_high": 0.06475448608398438, "read_cosine": 0.2077166736125946, "beta_mean": 0.2128622978925705, "nonfinite": 0.0, "weighted_memory_to_active": 0.2516387104988098, "mix_std": 0.3657836616039276, "beta_std": 0.22955501079559326, "beta_low": 0.12603759765625, "beta_high": 0.0, "orthogonal_to_active": 0.1625431627035141, "evidence_weights": [-0.16501860320568085, -0.3511243164539337, -0.3775424063205719], "agreement_weight": 0.022655848413705826, "orthogonal_weight": 0.6979280710220337}, {"head": 1, "memory_to_active": 0.016421139240264893, "contribution_to_active": 0.5388654470443726, "mix_mean": 0.5326642394065857, "mix_low": 0.0012645721435546875, "mix_high": 0.004191398620605469, "read_cosine": 0.2635146975517273, "beta_mean": 0.00010704957821872085, "nonfinite": 0.0, "weighted_memory_to_active": 0.008990738540887833, "mix_std": 0.2686649560928345, "beta_std": 0.0001672386861173436, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009842860512435436, "evidence_weights": [-0.2696436643600464, -0.19395481050014496, 0.49786069989204407], "agreement_weight": 0.30221354961395264, "orthogonal_weight": -1.2198965549468994}, {"head": 2, "memory_to_active": 0.145188108086586, "contribution_to_active": 0.4328337609767914, "mix_mean": 0.4155445396900177, "mix_low": 0.0020389556884765625, "mix_high": 0.00046443939208984375, "read_cosine": 0.007335943635553122, "beta_mean": 0.000976938521489501, "nonfinite": 0.0, "weighted_memory_to_active": 0.05025230348110199, "mix_std": 0.2726122736930847, "beta_std": 0.003474922850728035, "beta_low": 0.97991943359375, "beta_high": 0.0, "orthogonal_to_active": 0.017910556867718697, "evidence_weights": [-0.16253988444805145, -0.2157483845949173, 0.5401942729949951], "agreement_weight": -0.02354189194738865, "orthogonal_weight": -0.34925609827041626}, {"head": 3, "memory_to_active": 0.32468265295028687, "contribution_to_active": 0.11470775306224823, "mix_mean": 0.24987547099590302, "mix_low": 0.04550743103027344, "mix_high": 0.00012969970703125, "read_cosine": 0.9005370736122131, "beta_mean": 0.06949411332607269, "nonfinite": 0.0, "weighted_memory_to_active": 0.0501740425825119, "mix_std": 0.2318572998046875, "beta_std": 0.0644172951579094, "beta_low": 0.00262451171875, "beta_high": 0.0, "orthogonal_to_active": 0.047289278358221054, "evidence_weights": [-0.12723597884178162, -0.14469817280769348, 0.18710272014141083], "agreement_weight": -0.0006457218551076949, "orthogonal_weight": 1.3902227878570557}, {"head": 4, "memory_to_active": 0.021366601809859276, "contribution_to_active": 0.4621036648750305, "mix_mean": 0.44958609342575073, "mix_low": 0.0013866424560546875, "mix_high": 8.392333984375e-05, "read_cosine": 0.06804481148719788, "beta_mean": 4.604768400895409e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005890546832233667, "mix_std": 0.25296759605407715, "beta_std": 0.001929478021338582, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0018026754260063171, "evidence_weights": [-0.18092802166938782, -0.21708181500434875, 0.49650073051452637], "agreement_weight": 0.05509001016616821, "orthogonal_weight": -0.31041282415390015}, {"head": 5, "memory_to_active": 0.14360079169273376, "contribution_to_active": 0.42791324853897095, "mix_mean": 0.3568481206893921, "mix_low": 0.0033931732177734375, "mix_high": 1.1444091796875e-05, "read_cosine": 0.8533207774162292, "beta_mean": 0.007322404533624649, "nonfinite": 0.0, "weighted_memory_to_active": 0.07227467000484467, "mix_std": 0.22284214198589325, "beta_std": 0.009307395666837692, "beta_low": 0.783203125, "beta_high": 0.0, "orthogonal_to_active": 0.017674146220088005, "evidence_weights": [-0.16019806265830994, -0.1647149920463562, 0.2899707853794098], "agreement_weight": 0.09309262037277222, "orthogonal_weight": -0.5718417763710022}]}, {"layer": 11, "memory_to_active": 0.21276940405368805, "contribution_to_active": 0.49718332290649414, "mix_mean": 0.4100993275642395, "mix_low": 0.05153592675924301, "mix_high": 0.02854156494140625, "read_cosine": 0.5013861060142517, "beta_mean": 0.03754016011953354, "nonfinite": 0.0, "weighted_memory_to_active": 0.08260087668895721, "mix_std": 0.30988484621047974, "beta_std": 0.05393543094396591, "beta_low": 0.50177001953125, "beta_high": 0.0, "orthogonal_to_active": 0.05216927081346512, "output_influence": 0.4722481966018677, "residual_influence": 0.12278838455677032, "state_active": 0.11681655794382095, "state_memory": 0.010255591943860054, "state_peak": 27.57061004638672, "growth_active": 1.0882575511932373, "growth_memory": 1.463608980178833, "heads": [{"head": 0, "memory_to_active": 0.010869591496884823, "contribution_to_active": 0.5509416460990906, "mix_mean": 0.5571455359458923, "mix_low": 0.0017709732055664062, "mix_high": 0.007313728332519531, "read_cosine": 0.43548259139060974, "beta_mean": 0.0001221718266606331, "nonfinite": 0.0, "weighted_memory_to_active": 0.006890936754643917, "mix_std": 0.2742530107498169, "beta_std": 0.003795053344219923, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.0038959397934377193, "evidence_weights": [-0.15959390997886658, -0.3422360420227051, 0.5977786779403687], "agreement_weight": 0.1711602658033371, "orthogonal_weight": -0.7083463072776794}, {"head": 1, "memory_to_active": 0.4393676817417145, "contribution_to_active": 0.25061726570129395, "mix_mean": 0.270546555519104, "mix_low": 0.023131370544433594, "mix_high": 0.000110626220703125, "read_cosine": 0.6216974258422852, "beta_mean": 0.03806089609861374, "nonfinite": 0.0, "weighted_memory_to_active": 0.16603869199752808, "mix_std": 0.240614652633667, "beta_std": 0.06765073537826538, "beta_low": 0.360595703125, "beta_high": 0.0, "orthogonal_to_active": 0.02650454454123974, "evidence_weights": [-0.16049930453300476, -0.1256209909915924, 0.1939309984445572], "agreement_weight": 0.08957480639219284, "orthogonal_weight": -0.22329996526241302}, {"head": 2, "memory_to_active": 0.04376915097236633, "contribution_to_active": 0.8778387308120728, "mix_mean": 0.47801321744918823, "mix_low": 0.07235336303710938, "mix_high": 0.09090995788574219, "read_cosine": 0.3079540729522705, "beta_mean": 0.06912782043218613, "nonfinite": 0.0, "weighted_memory_to_active": 0.034613706171512604, "mix_std": 0.413396418094635, "beta_std": 0.09450540691614151, "beta_low": 0.454833984375, "beta_high": 0.0, "orthogonal_to_active": 0.10536760091781616, "evidence_weights": [-0.08299262821674347, -0.44625556468963623, -0.9386018514633179], "agreement_weight": -0.004527543671429157, "orthogonal_weight": 3.2725696563720703}, {"head": 3, "memory_to_active": 0.023396460339426994, "contribution_to_active": 0.6285356283187866, "mix_mean": 0.42775195837020874, "mix_low": 0.17928504943847656, "mix_high": 0.07183074951171875, "read_cosine": 0.5555225014686584, "beta_mean": 0.06405580788850784, "nonfinite": 0.0, "weighted_memory_to_active": 0.013245762325823307, "mix_std": 0.42603975534439087, "beta_std": 0.09387262165546417, "beta_low": 0.4638671875, "beta_high": 0.0, "orthogonal_to_active": 0.04034274071455002, "evidence_weights": [-0.05046951770782471, -0.3670993447303772, -0.3796013593673706], "agreement_weight": 0.02499711886048317, "orthogonal_weight": 3.5152180194854736}, {"head": 4, "memory_to_active": 0.6029956340789795, "contribution_to_active": 0.32380008697509766, "mix_mean": 0.2741357088088989, "mix_low": 0.03225135803222656, "mix_high": 0.0004825592041015625, "read_cosine": 0.22235095500946045, "beta_mean": 0.039566680788993835, "nonfinite": 0.0, "weighted_memory_to_active": 0.20696312189102173, "mix_std": 0.24491214752197266, "beta_std": 0.04065629094839096, "beta_low": 0.1302490234375, "beta_high": 0.0, "orthogonal_to_active": 0.11368602514266968, "evidence_weights": [-0.14135204255580902, -0.11242995411157608, 1.3380171060562134], "agreement_weight": -0.1215641051530838, "orthogonal_weight": -0.5843292474746704}, {"head": 5, "memory_to_active": 0.15621793270111084, "contribution_to_active": 0.3513665199279785, "mix_mean": 0.4530026614665985, "mix_low": 0.000423431396484375, "mix_high": 0.0006017684936523438, "read_cosine": 0.8653090596199036, "beta_mean": 0.014307575300335884, "nonfinite": 0.0, "weighted_memory_to_active": 0.06785301119089127, "mix_std": 0.26009315252304077, "beta_std": 0.02313246950507164, "beta_low": 0.601318359375, "beta_high": 0.0, "orthogonal_to_active": 0.023218773305416107, "evidence_weights": [-0.19780930876731873, -0.2521011233329773, 0.4917242228984833], "agreement_weight": 0.04172626882791519, "orthogonal_weight": -0.6139224171638489}]}, {"layer": 12, "memory_to_active": 0.3702775239944458, "contribution_to_active": 0.5278822779655457, "mix_mean": 0.43113353848457336, "mix_low": 0.022345544770359993, "mix_high": 0.0027074813842773438, "read_cosine": 0.277077853679657, "beta_mean": 0.018111877143383026, "nonfinite": 0.0, "weighted_memory_to_active": 0.18182098865509033, "mix_std": 0.2523563802242279, "beta_std": 0.01861380971968174, "beta_low": 0.80548095703125, "beta_high": 0.0, "orthogonal_to_active": 0.0657525509595871, "output_influence": 0.39389768242836, "residual_influence": 0.10805059969425201, "state_active": 0.04336877912282944, "state_memory": 0.011229405179619789, "state_peak": 25.57724952697754, "growth_active": 1.1338123083114624, "growth_memory": 1.2028170824050903, "heads": [{"head": 0, "memory_to_active": 0.14096097648143768, "contribution_to_active": 0.3940313458442688, "mix_mean": 0.36875253915786743, "mix_low": 0.006439208984375, "mix_high": 0.0005636215209960938, "read_cosine": 0.2766866087913513, "beta_mean": 0.00207614921964705, "nonfinite": 0.0, "weighted_memory_to_active": 0.052331969141960144, "mix_std": 0.25772929191589355, "beta_std": 0.0074180178344249725, "beta_low": 0.96240234375, "beta_high": 0.0, "orthogonal_to_active": 0.027200009673833847, "evidence_weights": [-0.15676279366016388, -0.22183384001255035, 0.2589068114757538], "agreement_weight": 0.11203967779874802, "orthogonal_weight": -0.5472869277000427}, {"head": 1, "memory_to_active": 0.14168037474155426, "contribution_to_active": 0.46899113059043884, "mix_mean": 0.47674232721328735, "mix_low": 0.000659942626953125, "mix_high": 0.000270843505859375, "read_cosine": 0.5682748556137085, "beta_mean": 0.0016401861794292927, "nonfinite": 0.0, "weighted_memory_to_active": 0.06817318499088287, "mix_std": 0.2603713572025299, "beta_std": 0.006137563847005367, "beta_low": 0.97314453125, "beta_high": 0.0, "orthogonal_to_active": 0.04938610643148422, "evidence_weights": [-0.19619224965572357, -0.2523563802242279, 0.8037657141685486], "agreement_weight": 0.10543018579483032, "orthogonal_weight": -0.9056594967842102}, {"head": 2, "memory_to_active": 0.6424271464347839, "contribution_to_active": 0.4162968397140503, "mix_mean": 0.2936688959598541, "mix_low": 0.029388427734375, "mix_high": 0.00025653839111328125, "read_cosine": 0.43697986006736755, "beta_mean": 0.10006168484687805, "nonfinite": 0.0, "weighted_memory_to_active": 0.24515417218208313, "mix_std": 0.24091777205467224, "beta_std": 0.07386031746864319, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.18005862832069397, "evidence_weights": [-0.23751504719257355, -0.04356527328491211, -0.06082788482308388], "agreement_weight": 0.02789614163339138, "orthogonal_weight": 0.9165757298469543}, {"head": 3, "memory_to_active": 0.022932035848498344, "contribution_to_active": 0.7229116559028625, "mix_mean": 0.6652607917785645, "mix_low": 0.00040912628173828125, "mix_high": 0.011949539184570312, "read_cosine": 0.0707520842552185, "beta_mean": 0.00022022119082976133, "nonfinite": 0.0, "weighted_memory_to_active": 0.016521187499165535, "mix_std": 0.2497580647468567, "beta_std": 0.0029399802442640066, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.014692004770040512, "evidence_weights": [-0.3026852607727051, -0.3220205008983612, 0.9803009629249573], "agreement_weight": 0.20869027078151703, "orthogonal_weight": -0.9131944179534912}, {"head": 4, "memory_to_active": 1.2224204540252686, "contribution_to_active": 0.7073041200637817, "mix_mean": 0.2043401151895523, "mix_low": 0.0909271240234375, "mix_high": 0.0013189315795898438, "read_cosine": -0.04284105449914932, "beta_mean": 0.004062959924340248, "nonfinite": 0.0, "weighted_memory_to_active": 0.6801928281784058, "mix_std": 0.2301570326089859, "beta_std": 0.018719958141446114, "beta_low": 0.90948486328125, "beta_high": 0.0, "orthogonal_to_active": 0.10128369927406311, "evidence_weights": [-0.019671302288770676, -0.17228811979293823, -0.0038695165421813726], "agreement_weight": 0.00026242496096529067, "orthogonal_weight": -0.1625644564628601}, {"head": 5, "memory_to_active": 0.05124413222074509, "contribution_to_active": 0.45775848627090454, "mix_mean": 0.5780365467071533, "mix_low": 0.006249427795410156, "mix_high": 0.0018854141235351562, "read_cosine": 0.3526146113872528, "beta_mean": 0.0006100604077801108, "nonfinite": 0.0, "weighted_memory_to_active": 0.028552567586302757, "mix_std": 0.27520471811294556, "beta_std": 0.0026070205494761467, "beta_low": 0.98809814453125, "beta_high": 0.0, "orthogonal_to_active": 0.02189485915005207, "evidence_weights": [-0.2643605172634125, -0.2936495244503021, 0.8841521739959717], "agreement_weight": 0.13356523215770721, "orthogonal_weight": -0.843187689781189}]}, {"layer": 13, "memory_to_active": 0.17135095596313477, "contribution_to_active": 0.47907233238220215, "mix_mean": 0.47084861993789673, "mix_low": 0.005772273056209087, "mix_high": 0.0019653639756143093, "read_cosine": 0.3973143696784973, "beta_mean": 0.01805928349494934, "nonfinite": 0.0, "weighted_memory_to_active": 0.07176059484481812, "mix_std": 0.25411975383758545, "beta_std": 0.017646025866270065, "beta_low": 0.7805074453353882, "beta_high": 0.0, "orthogonal_to_active": 0.06555769592523575, "output_influence": 0.42038074135780334, "residual_influence": 0.13886959850788116, "state_active": 0.0852275937795639, "state_memory": 0.03669457882642746, "state_peak": 94.59679412841797, "growth_active": 0.9890717267990112, "growth_memory": 1.046209454536438, "heads": [{"head": 0, "memory_to_active": 0.06495189666748047, "contribution_to_active": 0.5091502070426941, "mix_mean": 0.49564671516418457, "mix_low": 0.000701904296875, "mix_high": 0.00139617919921875, "read_cosine": 0.3526274859905243, "beta_mean": 0.000330612383550033, "nonfinite": 0.0, "weighted_memory_to_active": 0.035135574638843536, "mix_std": 0.26389119029045105, "beta_std": 0.0007055600290186703, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.03396408259868622, "evidence_weights": [-0.20386773347854614, -0.24337470531463623, 0.769770622253418], "agreement_weight": 0.05192941799759865, "orthogonal_weight": -1.0636268854141235}, {"head": 1, "memory_to_active": 0.03935939073562622, "contribution_to_active": 0.5385205149650574, "mix_mean": 0.4733928442001343, "mix_low": 0.0023956298828125, "mix_high": 0.000240325927734375, "read_cosine": 0.23448413610458374, "beta_mean": 0.0001532342575956136, "nonfinite": 0.0, "weighted_memory_to_active": 0.021000375971198082, "mix_std": 0.2641551196575165, "beta_std": 0.00028326374012976885, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.02002251334488392, "evidence_weights": [-0.18781635165214539, -0.2541075348854065, 0.7259677648544312], "agreement_weight": 0.16476432979106903, "orthogonal_weight": -1.011648416519165}, {"head": 2, "memory_to_active": 0.16709980368614197, "contribution_to_active": 0.35083726048469543, "mix_mean": 0.34448525309562683, "mix_low": 0.0024509429931640625, "mix_high": 4.1961669921875e-05, "read_cosine": 0.6418726444244385, "beta_mean": 0.015059834346175194, "nonfinite": 0.0, "weighted_memory_to_active": 0.06401040405035019, "mix_std": 0.2141212821006775, "beta_std": 0.03596169501543045, "beta_low": 0.702880859375, "beta_high": 0.0, "orthogonal_to_active": 0.03271342068910599, "evidence_weights": [-0.224039226770401, -0.1619216948747635, 0.49908867478370667], "agreement_weight": 0.03805117309093475, "orthogonal_weight": -0.7373248934745789}, {"head": 3, "memory_to_active": 0.1522267907857895, "contribution_to_active": 0.5810704231262207, "mix_mean": 0.6016879677772522, "mix_low": 0.012178421020507812, "mix_high": 0.007187843322753906, "read_cosine": 0.00848845299333334, "beta_mean": 0.0010594105115160346, "nonfinite": 0.0, "weighted_memory_to_active": 0.10896232724189758, "mix_std": 0.27491021156311035, "beta_std": 0.003842259058728814, "beta_low": 0.98968505859375, "beta_high": 0.0, "orthogonal_to_active": 0.11024698615074158, "evidence_weights": [-0.3508407473564148, -0.2350061535835266, 1.9450782537460327], "agreement_weight": 2.056584596633911, "orthogonal_weight": -1.052087426185608}, {"head": 4, "memory_to_active": 0.47658559679985046, "contribution_to_active": 0.27488571405410767, "mix_mean": 0.35943129658699036, "mix_low": 0.0130767822265625, "mix_high": 7.915496826171875e-05, "read_cosine": 0.8758856654167175, "beta_mean": 0.09080345928668976, "nonfinite": 0.0, "weighted_memory_to_active": 0.1272873431444168, "mix_std": 0.22668038308620453, "beta_std": 0.06054146960377693, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.1438462883234024, "evidence_weights": [-0.19749930500984192, -0.23409338295459747, 0.48985493183135986], "agreement_weight": -0.004949596710503101, "orthogonal_weight": 1.6757892370224}, {"head": 5, "memory_to_active": 0.127882182598114, "contribution_to_active": 0.6199698448181152, "mix_mean": 0.5504477620124817, "mix_low": 0.0038299560546875, "mix_high": 0.0028467178344726562, "read_cosine": 0.2705278992652893, "beta_mean": 0.0009491483797319233, "nonfinite": 0.0, "weighted_memory_to_active": 0.07416753470897675, "mix_std": 0.2809603810310364, "beta_std": 0.00454191118478775, "beta_low": 0.99090576171875, "beta_high": 0.0, "orthogonal_to_active": 0.05255287140607834, "evidence_weights": [-0.21754315495491028, -0.2720582187175751, 0.5181702375411987], "agreement_weight": 0.07802291959524155, "orthogonal_weight": -0.7873942852020264}]}, {"layer": 14, "memory_to_active": 0.26828619837760925, "contribution_to_active": 0.4280056357383728, "mix_mean": 0.4031953513622284, "mix_low": 0.025876998901367188, "mix_high": 0.0037461917381733656, "read_cosine": 0.6264595985412598, "beta_mean": 0.0723324865102768, "nonfinite": 0.0, "weighted_memory_to_active": 0.12168556451797485, "mix_std": 0.25002801418304443, "beta_std": 0.0657903254032135, "beta_low": 0.4272054135799408, "beta_high": 0.0, "orthogonal_to_active": 0.042650006711483, "output_influence": 0.27891314029693604, "residual_influence": 0.10811427235603333, "state_active": 0.07200714200735092, "state_memory": 0.0166628398001194, "state_peak": 24.49822425842285, "growth_active": 1.1669172048568726, "growth_memory": 1.3178904056549072, "heads": [{"head": 0, "memory_to_active": 0.3541417121887207, "contribution_to_active": 0.21135245263576508, "mix_mean": 0.22334063053131104, "mix_low": 0.019582748413085938, "mix_high": 6.67572021484375e-06, "read_cosine": 0.7457931041717529, "beta_mean": 0.0728902816772461, "nonfinite": 0.0, "weighted_memory_to_active": 0.10232441872358322, "mix_std": 0.2061324268579483, "beta_std": 0.07664690911769867, "beta_low": 0.04071044921875, "beta_high": 0.0, "orthogonal_to_active": 0.03762699291110039, "evidence_weights": [-0.0886140987277031, -0.08386164903640747, 0.7340570688247681], "agreement_weight": 0.038946907967329025, "orthogonal_weight": 0.5615194439888}, {"head": 1, "memory_to_active": 0.4649049639701843, "contribution_to_active": 0.41275858879089355, "mix_mean": 0.2697276175022125, "mix_low": 0.1124715805053711, "mix_high": 0.0002689361572265625, "read_cosine": 0.6622335910797119, "beta_mean": 0.2943524420261383, "nonfinite": 0.0, "weighted_memory_to_active": 0.21975044906139374, "mix_std": 0.27736935019493103, "beta_std": 0.25889891386032104, "beta_low": 0.08856201171875, "beta_high": 0.0, "orthogonal_to_active": 0.022994227707386017, "evidence_weights": [0.010768383741378784, -0.275515615940094, -0.1356031894683838], "agreement_weight": 0.05681661516427994, "orthogonal_weight": 0.17754365503787994}, {"head": 2, "memory_to_active": 0.23154789209365845, "contribution_to_active": 0.5093174576759338, "mix_mean": 0.5078076124191284, "mix_low": 0.004967689514160156, "mix_high": 0.0015478134155273438, "read_cosine": 0.39102351665496826, "beta_mean": 0.006557823158800602, "nonfinite": 0.0, "weighted_memory_to_active": 0.13083815574645996, "mix_std": 0.2734171152114868, "beta_std": 0.008284052833914757, "beta_low": 0.8140869140625, "beta_high": 0.0, "orthogonal_to_active": 0.0958123505115509, "evidence_weights": [-0.1797044426202774, -0.26372820138931274, 0.7775377035140991], "agreement_weight": 0.0761471688747406, "orthogonal_weight": -0.8300487399101257}, {"head": 3, "memory_to_active": 0.07117602974176407, "contribution_to_active": 0.6637270450592041, "mix_mean": 0.6622053980827332, "mix_low": 0.003143310546875, "mix_high": 0.014293670654296875, "read_cosine": 0.4370446801185608, "beta_mean": 0.009832827374339104, "nonfinite": 0.0, "weighted_memory_to_active": 0.049232855439186096, "mix_std": 0.24431845545768738, "beta_std": 0.008973071351647377, "beta_low": 0.655029296875, "beta_high": 0.0, "orthogonal_to_active": 0.025167355313897133, "evidence_weights": [-0.38052788376808167, -0.3275681138038635, 0.4453071653842926], "agreement_weight": 0.07645168155431747, "orthogonal_weight": -0.6465081572532654}, {"head": 4, "memory_to_active": 0.1950923651456833, "contribution_to_active": 0.4786386489868164, "mix_mean": 0.4663945138454437, "mix_low": 0.0036325454711914062, "mix_high": 0.005682945251464844, "read_cosine": 0.8135645389556885, "beta_mean": 0.0476018525660038, "nonfinite": 0.0, "weighted_memory_to_active": 0.11172416061162949, "mix_std": 0.26301059126853943, "beta_std": 0.03238477557897568, "beta_low": 0.0211181640625, "beta_high": 0.0, "orthogonal_to_active": 0.03066597506403923, "evidence_weights": [-0.22122198343276978, -0.27035006880760193, 0.38107115030288696], "agreement_weight": 0.11234637349843979, "orthogonal_weight": -0.508678674697876}, {"head": 5, "memory_to_active": 0.2928542196750641, "contribution_to_active": 0.2922394573688507, "mix_mean": 0.28969627618789673, "mix_low": 0.011464118957519531, "mix_high": 0.0006771087646484375, "read_cosine": 0.7090981006622314, "beta_mean": 0.002759648486971855, "nonfinite": 0.0, "weighted_memory_to_active": 0.11624333262443542, "mix_std": 0.23592011630535126, "beta_std": 0.00955421943217516, "beta_low": 0.9437255859375, "beta_high": 0.0, "orthogonal_to_active": 0.04363314062356949, "evidence_weights": [-0.1835889369249344, -0.1453993171453476, 0.17616702616214752], "agreement_weight": 0.088846355676651, "orthogonal_weight": -0.6023221015930176}]}, {"layer": 15, "memory_to_active": 0.06803138554096222, "contribution_to_active": 0.47692233324050903, "mix_mean": 0.3857305943965912, "mix_low": 0.007947444915771484, "mix_high": 0.002685864921659231, "read_cosine": 0.6828734874725342, "beta_mean": 0.0394059456884861, "nonfinite": 0.0, "weighted_memory_to_active": 0.030526328831911087, "mix_std": 0.2768186926841736, "beta_std": 0.04349828511476517, "beta_low": 0.7165222764015198, "beta_high": 0.0, "orthogonal_to_active": 0.005049573257565498, "output_influence": 0.3027042746543884, "residual_influence": 0.1589556783437729, "state_active": 0.11988157778978348, "state_memory": 0.007973149418830872, "state_peak": 33.16609191894531, "growth_active": 1.3482264280319214, "growth_memory": 1.7045602798461914, "heads": [{"head": 0, "memory_to_active": 0.10182122141122818, "contribution_to_active": 0.47114789485931396, "mix_mean": 0.3526425361633301, "mix_low": 0.0056056976318359375, "mix_high": 0.0029630661010742188, "read_cosine": 0.7069525718688965, "beta_mean": 0.03575475886464119, "nonfinite": 0.0, "weighted_memory_to_active": 0.04753943160176277, "mix_std": 0.26412665843963623, "beta_std": 0.026494791731238365, "beta_low": 0.14373779296875, "beta_high": 0.0, "orthogonal_to_active": 0.01537233404815197, "evidence_weights": [-0.16464653611183167, -0.1919131875038147, 0.34498077630996704], "agreement_weight": 0.08334368467330933, "orthogonal_weight": -0.5549806952476501}, {"head": 1, "memory_to_active": 0.17436237633228302, "contribution_to_active": 0.4103391170501709, "mix_mean": 0.3949849307537079, "mix_low": 0.005274772644042969, "mix_high": 0.00501251220703125, "read_cosine": 0.8571022748947144, "beta_mean": 0.0015526083298027515, "nonfinite": 0.0, "weighted_memory_to_active": 0.0678565576672554, "mix_std": 0.27875447273254395, "beta_std": 0.0036714905872941017, "beta_low": 0.97308349609375, "beta_high": 0.0, "orthogonal_to_active": 0.0027344226837158203, "evidence_weights": [-0.18671415746212006, -0.19269432127475739, 0.2883749306201935], "agreement_weight": 0.09569613635540009, "orthogonal_weight": -0.06279078871011734}, {"head": 2, "memory_to_active": 0.0043046441860497, "contribution_to_active": 0.442724347114563, "mix_mean": 0.3386724591255188, "mix_low": 0.013525009155273438, "mix_high": 0.00013065338134765625, "read_cosine": 0.7810708284378052, "beta_mean": 7.045883103273809e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.001776653341948986, "mix_std": 0.2707483470439911, "beta_std": 0.00016449298709630966, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 9.956457506632432e-05, "evidence_weights": [-0.20639975368976593, -0.2730819284915924, 0.29179197549819946], "agreement_weight": 0.02473357319831848, "orthogonal_weight": -0.10390928387641907}, {"head": 3, "memory_to_active": 0.031688567250967026, "contribution_to_active": 0.4485902786254883, "mix_mean": 0.31967025995254517, "mix_low": 0.0037870407104492188, "mix_high": 2.384185791015625e-05, "read_cosine": 0.6038857102394104, "beta_mean": 0.00040249215089716017, "nonfinite": 0.0, "weighted_memory_to_active": 0.014431595802307129, "mix_std": 0.2546979486942291, "beta_std": 0.001585976337082684, "beta_low": 0.9959716796875, "beta_high": 0.0, "orthogonal_to_active": 0.009653468616306782, "evidence_weights": [-0.18862497806549072, -0.23730356991291046, 0.4193386733531952], "agreement_weight": 0.016967812553048134, "orthogonal_weight": -0.9109170436859131}, {"head": 4, "memory_to_active": 0.09037777781486511, "contribution_to_active": 0.6254531741142273, "mix_mean": 0.3924093246459961, "mix_low": 0.013369560241699219, "mix_high": 0.00092315673828125, "read_cosine": 0.6271880865097046, "beta_mean": 0.19848479330539703, "nonfinite": 0.0, "weighted_memory_to_active": 0.0491333082318306, "mix_std": 0.30864235758781433, "beta_std": 0.22877296805381775, "beta_low": 0.1864013671875, "beta_high": 0.0, "orthogonal_to_active": 0.00043539199396036565, "evidence_weights": [-0.24296541512012482, -0.25576144456863403, 0.10588783770799637], "agreement_weight": 0.010815284214913845, "orthogonal_weight": -0.013596213422715664}, {"head": 5, "memory_to_active": 0.005633704364299774, "contribution_to_active": 0.46327918767929077, "mix_mean": 0.5160039663314819, "mix_low": 0.006122589111328125, "mix_high": 0.007061958312988281, "read_cosine": 0.5210411548614502, "beta_mean": 0.00017055065836757421, "nonfinite": 0.0, "weighted_memory_to_active": 0.002420430537313223, "mix_std": 0.28394225239753723, "beta_std": 0.000299993873341009, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.002002258086577058, "evidence_weights": [-0.3571021854877472, -0.3013570308685303, 0.3950287103652954], "agreement_weight": 0.1588745266199112, "orthogonal_weight": 0.9707054495811462}]}]}, {"step": 21000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 36.03086973499967, "memory_to_active": 0.9256507754325867, "contribution_to_active": 0.47306543588638306, "mix_mean": 0.3306787610054016, "mix_low": 0.16436541080474854, "mix_high": 0.00643274188041687, "read_cosine": 0.3916764557361603, "beta_mean": 0.08679606765508652, "nonfinite": 0.0, "weighted_memory_to_active": 0.20525340735912323, "mix_std": 0.21322110295295715, "beta_std": 0.07116006314754486, "beta_low": 0.47484272718429565, "beta_high": 1.9073488147114404e-05, "orthogonal_to_active": 0.04428798705339432, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1433098316192627, "normal_token_accuracy": 0.4360400438308716, "state_active": 0.05962255969643593, "state_memory": 0.02098235674202442, "state_peak": 112.58544921875, "growth_active": 1.153724193572998, "growth_memory": 1.3893312215805054, "output_influence": 0.33193209767341614, "residual_influence": 0.10441508889198303, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2351019382476807, "no_memory_loss_delta": 0.19673599302768707, "scrambled_loss_delta": 0.17043036222457886, "memory_only_loss_delta": 2.2608461380004883, "no_evidence_loss_delta": 0.1149030327796936, "no_agreement_loss_delta": 0.013662964105606079, "no_orthogonal_loss_delta": 0.006128034088760614}, {"index": 94, "tokens": 256, "normal_loss": 3.1769051551818848, "no_memory_loss_delta": 0.4058891236782074, "scrambled_loss_delta": 0.44794946908950806, "memory_only_loss_delta": 2.439424991607666, "no_evidence_loss_delta": 0.18896399438381195, "no_agreement_loss_delta": 0.030886009335517883, "no_orthogonal_loss_delta": 0.03735295683145523}, {"index": 188, "tokens": 256, "normal_loss": 3.56783127784729, "no_memory_loss_delta": 0.1876906454563141, "scrambled_loss_delta": 0.11024121195077896, "memory_only_loss_delta": 2.1824164390563965, "no_evidence_loss_delta": 0.08798035979270935, "no_agreement_loss_delta": -0.0029384479857981205, "no_orthogonal_loss_delta": 0.00269015459343791}, {"index": 283, "tokens": 131, "normal_loss": 2.0686798095703125, "no_memory_loss_delta": 0.16660013794898987, "scrambled_loss_delta": 0.34729325771331787, "memory_only_loss_delta": 2.757917881011963, "no_evidence_loss_delta": 0.014779197983443737, "no_agreement_loss_delta": -0.015716444700956345, "no_orthogonal_loss_delta": 0.015145551413297653}], "agreement_weight_abs_mean": 0.10859165340662003, "agreement_weight_abs_max": 2.1872799396514893, "agreement_weight_mean": 0.09161144495010376, "orthogonal_weight_abs_mean": 0.5613965392112732, "orthogonal_weight_abs_max": 3.72248911857605, "orthogonal_weight_mean": -0.1722951978445053, "no_memory_loss": 3.3926374912261963, "no_memory_loss_delta": 0.24932754039764404, "no_memory_token_accuracy_gain": 0.04004449397325516, "no_memory_token_win_fraction": 0.621802031993866, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 22.067532873661573, "scrambled_loss": 3.401399612426758, "scrambled_loss_delta": 0.25808945298194885, "scrambled_token_accuracy_gain": 0.023359287530183792, "scrambled_token_win_fraction": 0.6206896305084229, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 22.74738757494254, "memory_only_loss": 5.505106449127197, "memory_only_loss_delta": 2.3617968559265137, "memory_only_token_accuracy_gain": 0.24026696383953094, "memory_only_token_win_fraction": 0.8654060363769531, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 90.57492840057132, "no_evidence_loss": 3.2570464611053467, "no_evidence_loss_delta": 0.11373638361692429, "no_evidence_token_accuracy_gain": 0.01223581749945879, "no_evidence_token_win_fraction": 0.5550611615180969, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 10.750679929665935, "no_agreement_loss": 3.1528685092926025, "no_agreement_loss_delta": 0.009558888152241707, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.45272526144981384, "no_agreement_probe_win_fraction": 0.5, "no_agreement_ppl_reduction_pct": 0.9513347203348482, "no_orthogonal_loss": 3.1586644649505615, "no_orthogonal_loss_delta": 0.015354707837104797, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5684093236923218, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 1.5237425357432282, "short_no_memory_loss_delta": 0.22284671664237976, "long_context_memory_gain": 0.026480823755264282, "normal_context_loss_gain": 0.14252063632011414, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.072143077850342, "contribution_to_active": 0.8916654586791992, "mix_mean": 0.09581349045038223, "mix_low": 0.5123888850212097, "mix_high": 0.0036741893272846937, "read_cosine": 0.4743540287017822, "beta_mean": 0.2636803090572357, "nonfinite": 0.0, "weighted_memory_to_active": 0.9561732411384583, "mix_std": 0.08579529821872711, "beta_std": 0.15021830797195435, "beta_low": 0.004069010727107525, "beta_high": 0.0, "orthogonal_to_active": 0.06062454357743263, "output_influence": 0.135550856590271, "residual_influence": 0.1748102903366089, "state_active": 0.07992792874574661, "state_memory": 0.05041465908288956, "state_peak": 51.8039436340332, "growth_active": 1.2970075607299805, "growth_memory": 1.2897167205810547, "heads": [{"head": 0, "memory_to_active": 2.2735707759857178, "contribution_to_active": 0.3291597366333008, "mix_mean": 0.01964500918984413, "mix_low": 0.563847541809082, "mix_high": 0.0, "read_cosine": 0.2934305667877197, "beta_mean": 0.2121923714876175, "nonfinite": 0.0, "weighted_memory_to_active": 0.400227427482605, "mix_std": 0.052703045308589935, "beta_std": 0.10631775110960007, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03337275609374046, "evidence_weights": [-0.03459487110376358, -0.07541536539793015, -0.09180507063865662], "agreement_weight": -0.09548462182283401, "orthogonal_weight": -0.09350036084651947}, {"head": 1, "memory_to_active": 4.497983932495117, "contribution_to_active": 1.5999373197555542, "mix_mean": 0.014850219711661339, "mix_low": 0.6546497344970703, "mix_high": 0.0, "read_cosine": 0.5166997909545898, "beta_mean": 0.408137708902359, "nonfinite": 0.0, "weighted_memory_to_active": 1.9923444986343384, "mix_std": 0.04672315716743469, "beta_std": 0.1750253289937973, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17992132902145386, "evidence_weights": [0.018188372254371643, -0.05597054213285446, 0.20014968514442444], "agreement_weight": 0.018721813336014748, "orthogonal_weight": -0.20673608779907227}, {"head": 2, "memory_to_active": 2.585798740386963, "contribution_to_active": 0.4963951110839844, "mix_mean": 0.01447266899049282, "mix_low": 0.6521644592285156, "mix_high": 0.0, "read_cosine": 0.3563864827156067, "beta_mean": 0.33880719542503357, "nonfinite": 0.0, "weighted_memory_to_active": 0.5780728459358215, "mix_std": 0.03199559822678566, "beta_std": 0.23314988613128662, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05988144874572754, "evidence_weights": [0.042823296040296555, 0.04988812282681465, -0.008368444629013538], "agreement_weight": 0.0031650513410568237, "orthogonal_weight": -0.18153968453407288}, {"head": 3, "memory_to_active": 5.824214935302734, "contribution_to_active": 2.5093953609466553, "mix_mean": 0.04652160033583641, "mix_low": 0.3472757339477539, "mix_high": 0.0, "read_cosine": 0.4081053137779236, "beta_mean": 0.35020992159843445, "nonfinite": 0.0, "weighted_memory_to_active": 2.656527519226074, "mix_std": 0.09021348506212234, "beta_std": 0.21335721015930176, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07106905430555344, "evidence_weights": [-0.08269913494586945, -0.10904602706432343, -0.0725182443857193], "agreement_weight": -0.01628200337290764, "orthogonal_weight": -0.034362174570560455}, {"head": 4, "memory_to_active": 3.1236658096313477, "contribution_to_active": 0.061574287712574005, "mix_mean": 0.00596359558403492, "mix_low": 0.8315467834472656, "mix_high": 0.0, "read_cosine": 0.329681932926178, "beta_mean": 0.22050143778324127, "nonfinite": 0.0, "weighted_memory_to_active": 0.05890754610300064, "mix_std": 0.010460630990564823, "beta_std": 0.12478560954332352, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0070902579464018345, "evidence_weights": [0.08876392245292664, 0.03656302019953728, -0.3357248902320862], "agreement_weight": -0.03565642982721329, "orthogonal_weight": 0.13547347486019135}, {"head": 5, "memory_to_active": 0.1276252418756485, "contribution_to_active": 0.353531152009964, "mix_mean": 0.47342780232429504, "mix_low": 0.02484893798828125, "mix_high": 0.022045135498046875, "read_cosine": 0.9418200850486755, "beta_mean": 0.05223306268453598, "nonfinite": 0.0, "weighted_memory_to_active": 0.05095910653471947, "mix_std": 0.2826758921146393, "beta_std": 0.04867406189441681, "beta_low": 0.0244140625, "beta_high": 0.0, "orthogonal_to_active": 0.012412410229444504, "evidence_weights": [-0.27993255853652954, -0.25696149468421936, 0.23203374445438385], "agreement_weight": 0.20163361728191376, "orthogonal_weight": -0.49052971601486206}]}, {"layer": 1, "memory_to_active": 2.211284637451172, "contribution_to_active": 0.4202934503555298, "mix_mean": 0.14232495427131653, "mix_low": 0.3802744746208191, "mix_high": 0.0007174810161814094, "read_cosine": 0.3360283076763153, "beta_mean": 0.21005651354789734, "nonfinite": 0.0, "weighted_memory_to_active": 0.343149870634079, "mix_std": 0.11988049745559692, "beta_std": 0.13756850361824036, "beta_low": 0.1740824431180954, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.040342770516872406, "output_influence": 0.15114295482635498, "residual_influence": 0.03665970265865326, "state_active": 0.038515642285346985, "state_memory": 0.039497364312410355, "state_peak": 26.0122127532959, "growth_active": 1.1070727109909058, "growth_memory": 1.489725112915039, "heads": [{"head": 0, "memory_to_active": 0.06742388755083084, "contribution_to_active": 0.3652758002281189, "mix_mean": 0.33587050437927246, "mix_low": 0.019860267639160156, "mix_high": 0.00025844573974609375, "read_cosine": 0.00806386023759842, "beta_mean": 0.0004050054121762514, "nonfinite": 0.0, "weighted_memory_to_active": 0.030368920415639877, "mix_std": 0.27303212881088257, "beta_std": 0.0006812378414906561, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.023322954773902893, "evidence_weights": [-0.15626837313175201, -0.19543825089931488, -0.029286356642842293], "agreement_weight": 0.14912086725234985, "orthogonal_weight": -0.8503104448318481}, {"head": 1, "memory_to_active": 3.112046718597412, "contribution_to_active": 0.2132422775030136, "mix_mean": 0.023297879844903946, "mix_low": 0.46422576904296875, "mix_high": 1.9073486328125e-06, "read_cosine": 0.4470931887626648, "beta_mean": 0.37061071395874023, "nonfinite": 0.0, "weighted_memory_to_active": 0.23755866289138794, "mix_std": 0.04939838498830795, "beta_std": 0.22064316272735596, "beta_low": 0.0, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.008224662393331528, "evidence_weights": [-0.003984076902270317, -0.010987384244799614, -0.015264582820236683], "agreement_weight": -0.006849150639027357, "orthogonal_weight": 0.07366466522216797}, {"head": 2, "memory_to_active": 0.5475568771362305, "contribution_to_active": 0.43351972103118896, "mix_mean": 0.4358787536621094, "mix_low": 0.0038747787475585938, "mix_high": 0.0040416717529296875, "read_cosine": 0.7536723613739014, "beta_mean": 0.05814651399850845, "nonfinite": 0.0, "weighted_memory_to_active": 0.4259325861930847, "mix_std": 0.2653215527534485, "beta_std": 0.03686951845884323, "beta_low": 0.0079345703125, "beta_high": 0.0, "orthogonal_to_active": 0.02834966965019703, "evidence_weights": [-0.2491661012172699, -0.38578853011131287, 0.2323242723941803], "agreement_weight": 0.021783219650387764, "orthogonal_weight": -0.12446201592683792}, {"head": 3, "memory_to_active": 2.8207578659057617, "contribution_to_active": 0.07275628298521042, "mix_mean": 0.014811660163104534, "mix_low": 0.700439453125, "mix_high": 0.0, "read_cosine": 0.22681894898414612, "beta_mean": 0.3600398302078247, "nonfinite": 0.0, "weighted_memory_to_active": 0.09692984819412231, "mix_std": 0.041857097297906876, "beta_std": 0.23910409212112427, "beta_low": 0.0091552734375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.027101170271635056, "evidence_weights": [0.03911834955215454, 0.008561725728213787, 0.12068862468004227], "agreement_weight": -0.029803385958075523, "orthogonal_weight": -0.29537904262542725}, {"head": 4, "memory_to_active": 3.069040060043335, "contribution_to_active": 1.3801602125167847, "mix_mean": 0.03540857136249542, "mix_low": 0.31023311614990234, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3610163927078247, "beta_mean": 0.17341437935829163, "nonfinite": 0.0, "weighted_memory_to_active": 1.2160766124725342, "mix_std": 0.06783334910869598, "beta_std": 0.10057026892900467, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1526602804660797, "evidence_weights": [-0.021972768008708954, 0.0026689860969781876, 0.040562719106674194], "agreement_weight": -0.038460202515125275, "orthogonal_weight": 0.14715257287025452}, {"head": 5, "memory_to_active": 3.650881767272949, "contribution_to_active": 0.05680616945028305, "mix_mean": 0.008682392537593842, "mix_low": 0.7830133438110352, "mix_high": 1.9073486328125e-06, "read_cosine": 0.21950502693653107, "beta_mean": 0.29772254824638367, "nonfinite": 0.0, "weighted_memory_to_active": 0.052032582461833954, "mix_std": 0.021840447559952736, "beta_std": 0.22754274308681488, "beta_low": 0.02813720703125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0023978673852980137, "evidence_weights": [0.13654787838459015, 0.06193361431360245, 0.29128775000572205], "agreement_weight": -0.1394585520029068, "orthogonal_weight": 0.04639347642660141}]}, {"layer": 2, "memory_to_active": 1.0777006149291992, "contribution_to_active": 0.30595332384109497, "mix_mean": 0.293776273727417, "mix_low": 0.26576298475265503, "mix_high": 0.0024857521057128906, "read_cosine": 0.35553309321403503, "beta_mean": 0.10139889270067215, "nonfinite": 0.0, "weighted_memory_to_active": 0.06539727002382278, "mix_std": 0.19618529081344604, "beta_std": 0.07533230632543564, "beta_low": 0.5381266474723816, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.016698788851499557, "output_influence": 0.27445846796035767, "residual_influence": 0.06532348692417145, "state_active": 0.03739558905363083, "state_memory": 0.0209870133548975, "state_peak": 45.87807083129883, "growth_active": 1.019014835357666, "growth_memory": 1.5668904781341553, "heads": [{"head": 0, "memory_to_active": 0.007580106612294912, "contribution_to_active": 0.24373790621757507, "mix_mean": 0.4122729003429413, "mix_low": 0.0077533721923828125, "mix_high": 0.0021390914916992188, "read_cosine": 0.2383066713809967, "beta_mean": 5.416847125161439e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035996083170175552, "mix_std": 0.29597237706184387, "beta_std": 7.723586895735934e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00024108600337058306, "evidence_weights": [-0.24692818522453308, -0.1630561649799347, 0.2899515926837921], "agreement_weight": 0.19711534678936005, "orthogonal_weight": -0.07393089681863785}, {"head": 1, "memory_to_active": 3.092332363128662, "contribution_to_active": 0.05475116893649101, "mix_mean": 0.005621854215860367, "mix_low": 0.8843069076538086, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19513928890228271, "beta_mean": 0.29740145802497864, "nonfinite": 0.0, "weighted_memory_to_active": 0.043312881141901016, "mix_std": 0.014883454889059067, "beta_std": 0.1568942815065384, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.014286932535469532, "evidence_weights": [0.09728574752807617, 0.10489929467439651, 0.3405584990978241], "agreement_weight": 0.04173211380839348, "orthogonal_weight": 0.37293148040771484}, {"head": 2, "memory_to_active": 0.028353536501526833, "contribution_to_active": 0.42331692576408386, "mix_mean": 0.415469229221344, "mix_low": 0.00327301025390625, "mix_high": 0.002025604248046875, "read_cosine": 0.2982870042324066, "beta_mean": 5.471904296427965e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01158863678574562, "mix_std": 0.27679020166397095, "beta_std": 0.000125983715406619, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026312798727303743, "evidence_weights": [-0.2064741849899292, -0.2479163557291031, 0.5535271763801575], "agreement_weight": 0.039585694670677185, "orthogonal_weight": -0.2874200642108917}, {"head": 3, "memory_to_active": 0.2563723623752594, "contribution_to_active": 0.44655975699424744, "mix_mean": 0.39190614223480225, "mix_low": 0.0031557083129882812, "mix_high": 0.005770683288574219, "read_cosine": 0.6727102994918823, "beta_mean": 0.0513630211353302, "nonfinite": 0.0, "weighted_memory_to_active": 0.123783178627491, "mix_std": 0.26408305764198303, "beta_std": 0.06492073833942413, "beta_low": 0.1904296875, "beta_high": 0.0, "orthogonal_to_active": 0.0691400095820427, "evidence_weights": [-0.14737042784690857, -0.2254629284143448, 0.797690212726593], "agreement_weight": 0.0626484751701355, "orthogonal_weight": -0.8380811810493469}, {"head": 4, "memory_to_active": 3.0608413219451904, "contribution_to_active": 0.169855535030365, "mix_mean": 0.014552583917975426, "mix_low": 0.6936511993408203, "mix_high": 0.0, "read_cosine": 0.21415480971336365, "beta_mean": 0.2594667375087738, "nonfinite": 0.0, "weighted_memory_to_active": 0.19817529618740082, "mix_std": 0.041868776082992554, "beta_std": 0.22989553213119507, "beta_low": 0.038330078125, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.013142365962266922, "evidence_weights": [0.04484233632683754, 0.042527008801698685, 0.13554193079471588], "agreement_weight": 0.026276130229234695, "orthogonal_weight": -0.0755363404750824}, {"head": 5, "memory_to_active": 0.020724207162857056, "contribution_to_active": 0.4974987506866455, "mix_mean": 0.5228348970413208, "mix_low": 0.002437591552734375, "mix_high": 0.004978179931640625, "read_cosine": 0.5146005749702454, "beta_mean": 5.3253985242918134e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011924009770154953, "mix_std": 0.2835139036178589, "beta_std": 8.00608831923455e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007510541472584009, "evidence_weights": [-0.24048607051372528, -0.26172617077827454, 0.7167806029319763], "agreement_weight": 0.04771290346980095, "orthogonal_weight": 0.11619473248720169}]}, {"layer": 3, "memory_to_active": 1.1943244934082031, "contribution_to_active": 0.34996891021728516, "mix_mean": 0.2510676980018616, "mix_low": 0.215768501162529, "mix_high": 0.0036622683983296156, "read_cosine": 0.2440822869539261, "beta_mean": 0.0920901671051979, "nonfinite": 0.0, "weighted_memory_to_active": 0.15752440690994263, "mix_std": 0.21881301701068878, "beta_std": 0.07372686266899109, "beta_low": 0.4198201596736908, "beta_high": 0.0, "orthogonal_to_active": 0.04875527322292328, "output_influence": 0.2603132128715515, "residual_influence": 0.05101705715060234, "state_active": 0.027931872755289078, "state_memory": 0.017683468759059906, "state_peak": 16.912464141845703, "growth_active": 1.101035714149475, "growth_memory": 1.721506953239441, "heads": [{"head": 0, "memory_to_active": 0.6182928085327148, "contribution_to_active": 0.4262905716896057, "mix_mean": 0.3202022314071655, "mix_low": 0.063995361328125, "mix_high": 0.005267143249511719, "read_cosine": 0.20492395758628845, "beta_mean": 0.016637641936540604, "nonfinite": 0.0, "weighted_memory_to_active": 0.25420960783958435, "mix_std": 0.31158459186553955, "beta_std": 0.01898912526667118, "beta_low": 0.4315185546875, "beta_high": 0.0, "orthogonal_to_active": 0.13650178909301758, "evidence_weights": [-0.11418809741735458, -0.17450669407844543, 0.2550867199897766], "agreement_weight": 0.12991176545619965, "orthogonal_weight": -0.5720200538635254}, {"head": 1, "memory_to_active": 0.027317844331264496, "contribution_to_active": 0.29646816849708557, "mix_mean": 0.3330625593662262, "mix_low": 0.035552978515625, "mix_high": 0.0018072128295898438, "read_cosine": 0.20341159403324127, "beta_mean": 0.00012518255971372128, "nonfinite": 0.0, "weighted_memory_to_active": 0.009432835504412651, "mix_std": 0.2913757562637329, "beta_std": 0.00027396087534725666, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0016993528697639704, "evidence_weights": [-0.17503264546394348, -0.14531897008419037, 0.4111047387123108], "agreement_weight": 0.2106429934501648, "orthogonal_weight": -0.19631440937519073}, {"head": 2, "memory_to_active": 2.8095078468322754, "contribution_to_active": 0.3024460971355438, "mix_mean": 0.041298724710941315, "mix_low": 0.4069652557373047, "mix_high": 1.239776611328125e-05, "read_cosine": 0.16615131497383118, "beta_mean": 0.21719305217266083, "nonfinite": 0.0, "weighted_memory_to_active": 0.37450313568115234, "mix_std": 0.09216275811195374, "beta_std": 0.14687542617321014, "beta_low": 0.00445556640625, "beta_high": 0.0, "orthogonal_to_active": 0.017326613888144493, "evidence_weights": [-0.053130730986595154, 0.013570656068623066, 0.03908932954072952], "agreement_weight": -0.008688811212778091, "orthogonal_weight": -0.05891350656747818}, {"head": 3, "memory_to_active": 0.021123714745044708, "contribution_to_active": 0.46517908573150635, "mix_mean": 0.4859740734100342, "mix_low": 0.021303176879882812, "mix_high": 0.010733604431152344, "read_cosine": 0.26110249757766724, "beta_mean": 9.442590817343444e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012016926892101765, "mix_std": 0.3171582818031311, "beta_std": 0.00017661077436059713, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004174480214715004, "evidence_weights": [-0.1891385018825531, -0.2099333107471466, 1.5336668491363525], "agreement_weight": 0.8961325883865356, "orthogonal_weight": -0.42752185463905334}, {"head": 4, "memory_to_active": 3.3051249980926514, "contribution_to_active": 0.04220804572105408, "mix_mean": 0.011622155085206032, "mix_low": 0.7303466796875, "mix_high": 0.0, "read_cosine": 0.20522181689739227, "beta_mean": 0.23646482825279236, "nonfinite": 0.0, "weighted_memory_to_active": 0.07531672716140747, "mix_std": 0.027457470074295998, "beta_std": 0.181339830160141, "beta_low": 0.00152587890625, "beta_high": 0.0, "orthogonal_to_active": 0.0379905104637146, "evidence_weights": [0.13975730538368225, 0.10554630309343338, -0.03580820560455322], "agreement_weight": -0.05086907744407654, "orthogonal_weight": -0.5183573961257935}, {"head": 5, "memory_to_active": 0.3845791220664978, "contribution_to_active": 0.5672215819358826, "mix_mean": 0.3142462968826294, "mix_low": 0.03644752502441406, "mix_high": 0.004153251647949219, "read_cosine": 0.42368245124816895, "beta_mean": 0.08202584832906723, "nonfinite": 0.0, "weighted_memory_to_active": 0.21966710686683655, "mix_std": 0.2731391191482544, "beta_std": 0.09470620006322861, "beta_low": 0.08148193359375, "beta_high": 0.0, "orthogonal_to_active": 0.09483889490365982, "evidence_weights": [-0.1349615603685379, -0.19609905779361725, 0.3360670506954193], "agreement_weight": 0.1679440140724182, "orthogonal_weight": -0.5879093408584595}]}, {"layer": 4, "memory_to_active": 1.206634759902954, "contribution_to_active": 0.3707943856716156, "mix_mean": 0.3063983619213104, "mix_low": 0.2789386212825775, "mix_high": 0.0030372939072549343, "read_cosine": 0.39354366064071655, "beta_mean": 0.11877846717834473, "nonfinite": 0.0, "weighted_memory_to_active": 0.14200247824192047, "mix_std": 0.15646658837795258, "beta_std": 0.10301205515861511, "beta_low": 0.3561198115348816, "beta_high": 0.0, "orthogonal_to_active": 0.03402462601661682, "output_influence": 0.2913036346435547, "residual_influence": 0.07074762880802155, "state_active": 0.037920426577329636, "state_memory": 0.024528268724679947, "state_peak": 53.790897369384766, "growth_active": 1.13905668258667, "growth_memory": 1.632899522781372, "heads": [{"head": 0, "memory_to_active": 0.3104891777038574, "contribution_to_active": 0.6278361082077026, "mix_mean": 0.5761326551437378, "mix_low": 0.010903358459472656, "mix_high": 0.0035123825073242188, "read_cosine": 0.5332832932472229, "beta_mean": 0.2187892347574234, "nonfinite": 0.0, "weighted_memory_to_active": 0.2271735668182373, "mix_std": 0.3263450860977173, "beta_std": 0.23622561991214752, "beta_low": 0.19293212890625, "beta_high": 0.0, "orthogonal_to_active": 0.047313444316387177, "evidence_weights": [-0.24199897050857544, -0.3209148347377777, 0.4153761565685272], "agreement_weight": 0.03953762352466583, "orthogonal_weight": 0.2687913179397583}, {"head": 1, "memory_to_active": 0.1516856849193573, "contribution_to_active": 0.47572481632232666, "mix_mean": 0.5192968845367432, "mix_low": 0.0002498626708984375, "mix_high": 0.0034809112548828125, "read_cosine": 0.7463372945785522, "beta_mean": 0.004908377304673195, "nonfinite": 0.0, "weighted_memory_to_active": 0.08178868144750595, "mix_std": 0.26659440994262695, "beta_std": 0.008318226784467697, "beta_low": 0.89117431640625, "beta_high": 0.0, "orthogonal_to_active": 0.03681030496954918, "evidence_weights": [-0.23092982172966003, -0.24636630713939667, 0.6756839752197266], "agreement_weight": 0.0511968657374382, "orthogonal_weight": -0.7055345773696899}, {"head": 2, "memory_to_active": 2.247387409210205, "contribution_to_active": 0.12085269391536713, "mix_mean": 0.01605048030614853, "mix_low": 0.6049604415893555, "mix_high": 0.0, "read_cosine": 0.29134485125541687, "beta_mean": 0.11009643226861954, "nonfinite": 0.0, "weighted_memory_to_active": 0.1560792326927185, "mix_std": 0.02957257442176342, "beta_std": 0.0875067338347435, "beta_low": 0.0030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.008294474333524704, "evidence_weights": [0.04887497425079346, 0.026933953166007996, -0.043705087155103683], "agreement_weight": 0.08460775762796402, "orthogonal_weight": -0.0788525938987732}, {"head": 3, "memory_to_active": 1.7815295457839966, "contribution_to_active": 0.11802661418914795, "mix_mean": 0.02881655842065811, "mix_low": 0.39783763885498047, "mix_high": 0.0, "read_cosine": 0.1949487030506134, "beta_mean": 0.12461832165718079, "nonfinite": 0.0, "weighted_memory_to_active": 0.18433746695518494, "mix_std": 0.0486711785197258, "beta_std": 0.12462718039751053, "beta_low": 0.049560546875, "beta_high": 0.0, "orthogonal_to_active": 0.020870227366685867, "evidence_weights": [-0.01807725429534912, 0.05503784865140915, -0.25076231360435486], "agreement_weight": 0.050929997116327286, "orthogonal_weight": -0.15820427238941193}, {"head": 4, "memory_to_active": 0.04266484081745148, "contribution_to_active": 0.6346993446350098, "mix_mean": 0.6854352355003357, "mix_low": 0.00019741058349609375, "mix_high": 0.011225700378417969, "read_cosine": 0.2462255209684372, "beta_mean": 6.941387982806191e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.031756822019815445, "mix_std": 0.2379543036222458, "beta_std": 0.0002770995197352022, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003840434132143855, "evidence_weights": [-0.2936975359916687, -0.3342575430870056, 0.7787506580352783], "agreement_weight": 0.08142136037349701, "orthogonal_weight": -0.14067864418029785}, {"head": 5, "memory_to_active": 2.706051826477051, "contribution_to_active": 0.24762675166130066, "mix_mean": 0.012658352963626385, "mix_low": 0.6594829559326172, "mix_high": 4.76837158203125e-06, "read_cosine": 0.34912216663360596, "beta_mean": 0.25418904423713684, "nonfinite": 0.0, "weighted_memory_to_active": 0.1708790510892868, "mix_std": 0.029661929234862328, "beta_std": 0.16111746430397034, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08701886236667633, "evidence_weights": [0.04632341489195824, -0.027317974716424942, -0.18561504781246185], "agreement_weight": -0.047474730759859085, "orthogonal_weight": 0.7243868708610535}]}, {"layer": 5, "memory_to_active": 0.6495166420936584, "contribution_to_active": 0.4717966914176941, "mix_mean": 0.4070093631744385, "mix_low": 0.09053055942058563, "mix_high": 0.01108566951006651, "read_cosine": 0.20321501791477203, "beta_mean": 0.05593358352780342, "nonfinite": 0.0, "weighted_memory_to_active": 0.1815098524093628, "mix_std": 0.24887117743492126, "beta_std": 0.07167539745569229, "beta_low": 0.611175537109375, "beta_high": 0.0, "orthogonal_to_active": 0.059135328978300095, "output_influence": 0.45247194170951843, "residual_influence": 0.10768408328294754, "state_active": 0.05431598424911499, "state_memory": 0.014310558326542377, "state_peak": 62.13990783691406, "growth_active": 1.1611381769180298, "growth_memory": 1.3091063499450684, "heads": [{"head": 0, "memory_to_active": 1.0852806568145752, "contribution_to_active": 0.2939215898513794, "mix_mean": 0.14940989017486572, "mix_low": 0.1279315948486328, "mix_high": 8.58306884765625e-06, "read_cosine": -0.1556568145751953, "beta_mean": 0.04889484494924545, "nonfinite": 0.0, "weighted_memory_to_active": 0.1661180853843689, "mix_std": 0.19883069396018982, "beta_std": 0.07524295151233673, "beta_low": 0.23516845703125, "beta_high": 0.0, "orthogonal_to_active": 0.03898841142654419, "evidence_weights": [-0.055498164147138596, -0.09327515959739685, -0.35912057757377625], "agreement_weight": -0.011285080574452877, "orthogonal_weight": -0.22291681170463562}, {"head": 1, "memory_to_active": 0.028115157037973404, "contribution_to_active": 0.4987950325012207, "mix_mean": 0.5565250515937805, "mix_low": 0.006756782531738281, "mix_high": 0.015949249267578125, "read_cosine": 0.17910435795783997, "beta_mean": 7.642529089935124e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013376432470977306, "mix_std": 0.30837470293045044, "beta_std": 0.0001702400913927704, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011839983984827995, "evidence_weights": [-0.2273804247379303, -0.24757814407348633, 0.7483178377151489], "agreement_weight": 0.31048256158828735, "orthogonal_weight": -0.9201042056083679}, {"head": 2, "memory_to_active": 0.053234778344631195, "contribution_to_active": 0.5296447277069092, "mix_mean": 0.5884974598884583, "mix_low": 0.0020475387573242188, "mix_high": 0.014153480529785156, "read_cosine": 0.273642361164093, "beta_mean": 0.0012442826991900802, "nonfinite": 0.0, "weighted_memory_to_active": 0.035187408328056335, "mix_std": 0.27445515990257263, "beta_std": 0.0016110001597553492, "beta_low": 0.99481201171875, "beta_high": 0.0, "orthogonal_to_active": 0.005346429534256458, "evidence_weights": [-0.2599133849143982, -0.3262554407119751, 0.7950571775436401], "agreement_weight": 0.03232858330011368, "orthogonal_weight": -0.2010318636894226}, {"head": 3, "memory_to_active": 2.179295778274536, "contribution_to_active": 0.2910756766796112, "mix_mean": 0.07925941795110703, "mix_low": 0.2039775848388672, "mix_high": 0.0, "read_cosine": 0.1652202159166336, "beta_mean": 0.08816990256309509, "nonfinite": 0.0, "weighted_memory_to_active": 0.5061191916465759, "mix_std": 0.12220057845115662, "beta_std": 0.1291801929473877, "beta_low": 0.1998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.1717979609966278, "evidence_weights": [-0.037591952830553055, -0.0811830461025238, -0.05083450302481651], "agreement_weight": 0.04837874323129654, "orthogonal_weight": -0.4571070969104767}, {"head": 4, "memory_to_active": 0.4518255591392517, "contribution_to_active": 0.5995497703552246, "mix_mean": 0.32430386543273926, "mix_low": 0.19944000244140625, "mix_high": 0.0085601806640625, "read_cosine": 0.5495444536209106, "beta_mean": 0.19686858355998993, "nonfinite": 0.0, "weighted_memory_to_active": 0.30138474702835083, "mix_std": 0.33697834610939026, "beta_std": 0.22219884395599365, "beta_low": 0.24261474609375, "beta_high": 0.0, "orthogonal_to_active": 0.08597283810377121, "evidence_weights": [-0.12417226284742355, -0.28745517134666443, -0.035190898925065994], "agreement_weight": 0.041884373873472214, "orthogonal_weight": 0.4164282977581024}, {"head": 5, "memory_to_active": 0.09934762120246887, "contribution_to_active": 0.6177931427955627, "mix_mean": 0.7440602779388428, "mix_low": 0.0030298233032226562, "mix_high": 0.02784252166748047, "read_cosine": 0.2074354588985443, "beta_mean": 0.0003474782279226929, "nonfinite": 0.0, "weighted_memory_to_active": 0.06687319278717041, "mix_std": 0.2523875832557678, "beta_std": 0.0016491553978994489, "beta_low": 0.99462890625, "beta_high": 0.0, "orthogonal_to_active": 0.04086633771657944, "evidence_weights": [-0.32436874508857727, -0.31334397196769714, 1.7963203191757202], "agreement_weight": 0.32881855964660645, "orthogonal_weight": -0.6528345346450806}]}, {"layer": 6, "memory_to_active": 0.3851443827152252, "contribution_to_active": 0.5717360377311707, "mix_mean": 0.5259370803833008, "mix_low": 0.07859532535076141, "mix_high": 0.027479492127895355, "read_cosine": 0.3006552457809448, "beta_mean": 0.07123418152332306, "nonfinite": 0.0, "weighted_memory_to_active": 0.15658038854599, "mix_std": 0.24798345565795898, "beta_std": 0.08066154271364212, "beta_low": 0.5144755244255066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03230662643909454, "output_influence": 0.4643973112106323, "residual_influence": 0.141298308968544, "state_active": 0.04898996651172638, "state_memory": 0.00949133187532425, "state_peak": 71.5451431274414, "growth_active": 1.223002552986145, "growth_memory": 1.2180246114730835, "heads": [{"head": 0, "memory_to_active": 0.12267202883958817, "contribution_to_active": 0.5817878246307373, "mix_mean": 0.6884560585021973, "mix_low": 2.956390380859375e-05, "mix_high": 0.00373077392578125, "read_cosine": 0.1257409155368805, "beta_mean": 0.010511565022170544, "nonfinite": 0.0, "weighted_memory_to_active": 0.08747375011444092, "mix_std": 0.2369069755077362, "beta_std": 0.013024427928030491, "beta_low": 0.63970947265625, "beta_high": 0.0, "orthogonal_to_active": 0.06628526002168655, "evidence_weights": [-0.3072306215763092, -0.31470730900764465, 0.49497154355049133], "agreement_weight": 0.10647203773260117, "orthogonal_weight": -0.8989086151123047}, {"head": 1, "memory_to_active": 0.006257994100451469, "contribution_to_active": 0.46882641315460205, "mix_mean": 0.4729786813259125, "mix_low": 0.0021495819091796875, "mix_high": 0.009063720703125, "read_cosine": 0.6236728429794312, "beta_mean": 1.4497560187010095e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002437943359836936, "mix_std": 0.29547661542892456, "beta_std": 4.7455971071030945e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006763626006431878, "evidence_weights": [-0.20855334401130676, -0.2629694938659668, 0.8029645681381226], "agreement_weight": -0.01754249818623066, "orthogonal_weight": -0.3855245113372803}, {"head": 2, "memory_to_active": 1.2931745052337646, "contribution_to_active": 0.06016992777585983, "mix_mean": 0.025664664804935455, "mix_low": 0.4597206115722656, "mix_high": 0.0, "read_cosine": 0.18594425916671753, "beta_mean": 0.06816129386425018, "nonfinite": 0.0, "weighted_memory_to_active": 0.06147776544094086, "mix_std": 0.0480906218290329, "beta_std": 0.063397116959095, "beta_low": 0.00946044921875, "beta_high": 0.0, "orthogonal_to_active": 0.007393918465822935, "evidence_weights": [0.08742765337228775, 0.015149261802434921, 0.20974093675613403], "agreement_weight": 0.10007153451442719, "orthogonal_weight": -0.1343011111021042}, {"head": 3, "memory_to_active": 0.14782989025115967, "contribution_to_active": 0.6603295803070068, "mix_mean": 0.6446731090545654, "mix_low": 0.00734710693359375, "mix_high": 0.0244293212890625, "read_cosine": 0.5102716088294983, "beta_mean": 0.22469738125801086, "nonfinite": 0.0, "weighted_memory_to_active": 0.10682480037212372, "mix_std": 0.33494096994400024, "beta_std": 0.2602859139442444, "beta_low": 0.19921875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.049630749970674515, "evidence_weights": [-0.17595571279525757, -0.33666345477104187, 0.15583772957324982], "agreement_weight": -0.019683659076690674, "orthogonal_weight": 0.5505351424217224}, {"head": 4, "memory_to_active": 0.6937950849533081, "contribution_to_active": 1.0884523391723633, "mix_mean": 0.745084285736084, "mix_low": 0.0017566680908203125, "mix_high": 0.1258220672607422, "read_cosine": 0.08931228518486023, "beta_mean": 0.12363851070404053, "nonfinite": 0.0, "weighted_memory_to_active": 0.6502439379692078, "mix_std": 0.29314231872558594, "beta_std": 0.146403968334198, "beta_low": 0.239501953125, "beta_high": 0.0, "orthogonal_to_active": 0.03848576545715332, "evidence_weights": [-0.25209707021713257, -0.5753385424613953, -0.07592209428548813], "agreement_weight": -0.014336911961436272, "orthogonal_weight": 0.07109501957893372}, {"head": 5, "memory_to_active": 0.0471365824341774, "contribution_to_active": 0.5708501935005188, "mix_mean": 0.5787656307220459, "mix_low": 0.000568389892578125, "mix_high": 0.0018310546875, "read_cosine": 0.2689894437789917, "beta_mean": 0.0003818366094492376, "nonfinite": 0.0, "weighted_memory_to_active": 0.031023936346173286, "mix_std": 0.27934327721595764, "beta_std": 0.0008103137370198965, "beta_low": 0.99896240234375, "beta_high": 0.0, "orthogonal_to_active": 0.0313677117228508, "evidence_weights": [-0.23255778849124908, -0.2847236692905426, 0.36943772435188293], "agreement_weight": 0.09769083559513092, "orthogonal_weight": -1.078723430633545}]}, {"layer": 7, "memory_to_active": 1.9923150539398193, "contribution_to_active": 0.17329637706279755, "mix_mean": 0.04949942231178284, "mix_low": 0.4284650683403015, "mix_high": 1.3669331565324683e-05, "read_cosine": 0.22850027680397034, "beta_mean": 0.12720784544944763, "nonfinite": 0.0, "weighted_memory_to_active": 0.20030717551708221, "mix_std": 0.08055094629526138, "beta_std": 0.11082800477743149, "beta_low": 0.061859130859375, "beta_high": 7.120768714230508e-05, "orthogonal_to_active": 0.04779955744743347, "output_influence": 0.19497093558311462, "residual_influence": 0.042299121618270874, "state_active": 0.01634318009018898, "state_memory": 0.028671875596046448, "state_peak": 24.900104522705078, "growth_active": 1.0985122919082642, "growth_memory": 1.355668544769287, "heads": [{"head": 0, "memory_to_active": 2.139259099960327, "contribution_to_active": 0.2652180790901184, "mix_mean": 0.028628887608647346, "mix_low": 0.5285224914550781, "mix_high": 8.58306884765625e-06, "read_cosine": 0.3696202039718628, "beta_mean": 0.13717827200889587, "nonfinite": 0.0, "weighted_memory_to_active": 0.26314330101013184, "mix_std": 0.06486043334007263, "beta_std": 0.11423087120056152, "beta_low": 0.0028076171875, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.027200158685445786, "evidence_weights": [0.003444468602538109, -0.0850532278418541, 0.09993026405572891], "agreement_weight": -0.03739060461521149, "orthogonal_weight": 0.13659530878067017}, {"head": 1, "memory_to_active": 1.5335019826889038, "contribution_to_active": 0.307550847530365, "mix_mean": 0.12647053599357605, "mix_low": 0.25171947479248047, "mix_high": 2.574920654296875e-05, "read_cosine": 0.20024985074996948, "beta_mean": 0.14520767331123352, "nonfinite": 0.0, "weighted_memory_to_active": 0.356564998626709, "mix_std": 0.16706761717796326, "beta_std": 0.09501700848340988, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04692982882261276, "evidence_weights": [-0.15205328166484833, -0.1098853200674057, -0.20553064346313477], "agreement_weight": 0.01260911114513874, "orthogonal_weight": -0.14579935371875763}, {"head": 2, "memory_to_active": 1.157694935798645, "contribution_to_active": 0.24927273392677307, "mix_mean": 0.07198521494865417, "mix_low": 0.2218608856201172, "mix_high": 4.76837158203125e-05, "read_cosine": 0.13933387398719788, "beta_mean": 0.09393304586410522, "nonfinite": 0.0, "weighted_memory_to_active": 0.18697533011436462, "mix_std": 0.12547935545444489, "beta_std": 0.062344420701265335, "beta_low": 0.02667236328125, "beta_high": 0.0, "orthogonal_to_active": 0.027331236749887466, "evidence_weights": [-0.03415358439087868, -0.11785934865474701, 0.24433977901935577], "agreement_weight": 0.024539295583963394, "orthogonal_weight": -0.14936259388923645}, {"head": 3, "memory_to_active": 3.623744487762451, "contribution_to_active": 0.05945140868425369, "mix_mean": 0.026316124945878983, "mix_low": 0.4243183135986328, "mix_high": 0.0, "read_cosine": 0.13713599741458893, "beta_mean": 0.19732630252838135, "nonfinite": 0.0, "weighted_memory_to_active": 0.19027987122535706, "mix_std": 0.0425453707575798, "beta_std": 0.19532924890518188, "beta_low": 0.02569580078125, "beta_high": 0.0, "orthogonal_to_active": 0.14826002717018127, "evidence_weights": [0.08608000725507736, 0.041593872010707855, 0.47286245226860046], "agreement_weight": 0.06893011182546616, "orthogonal_weight": -0.853243350982666}, {"head": 4, "memory_to_active": 1.9811235666275024, "contribution_to_active": 0.06771127134561539, "mix_mean": 0.011257750913500786, "mix_low": 0.7377996444702148, "mix_high": 0.0, "read_cosine": 0.2949885129928589, "beta_mean": 0.1491735875606537, "nonfinite": 0.0, "weighted_memory_to_active": 0.0797850713133812, "mix_std": 0.02741391211748123, "beta_std": 0.13994061946868896, "beta_low": 0.03082275390625, "beta_high": 0.0, "orthogonal_to_active": 0.003178162034600973, "evidence_weights": [0.11301932483911514, 0.05638542398810387, 0.0514681376516819], "agreement_weight": 0.044217728078365326, "orthogonal_weight": -0.06054360046982765}, {"head": 5, "memory_to_active": 1.51856529712677, "contribution_to_active": 0.09057383239269257, "mix_mean": 0.03233799338340759, "mix_low": 0.4065694808959961, "mix_high": 0.0, "read_cosine": 0.22967317700386047, "beta_mean": 0.040428128093481064, "nonfinite": 0.0, "weighted_memory_to_active": 0.1250944584608078, "mix_std": 0.0559389591217041, "beta_std": 0.058105841279029846, "beta_low": 0.28515625, "beta_high": 0.0, "orthogonal_to_active": 0.03389790654182434, "evidence_weights": [0.0526605062186718, 0.03175408020615578, 0.06013074889779091], "agreement_weight": 0.029057659208774567, "orthogonal_weight": -0.3450925648212433}]}, {"layer": 8, "memory_to_active": 0.5899957418441772, "contribution_to_active": 0.4793742001056671, "mix_mean": 0.3732682466506958, "mix_low": 0.08987347781658173, "mix_high": 0.0006529490929096937, "read_cosine": 0.5238686800003052, "beta_mean": 0.03536548092961311, "nonfinite": 0.0, "weighted_memory_to_active": 0.1302758753299713, "mix_std": 0.22419634461402893, "beta_std": 0.04024758189916611, "beta_low": 0.498809814453125, "beta_high": 0.0, "orthogonal_to_active": 0.05597706884145737, "output_influence": 0.3659466505050659, "residual_influence": 0.1035449355840683, "state_active": 0.051963791251182556, "state_memory": 0.012347989715635777, "state_peak": 34.95240020751953, "growth_active": 1.0617176294326782, "growth_memory": 1.3581745624542236, "heads": [{"head": 0, "memory_to_active": 0.04617679491639137, "contribution_to_active": 0.5067422986030579, "mix_mean": 0.5478723645210266, "mix_low": 0.00022411346435546875, "mix_high": 0.0003662109375, "read_cosine": 0.6926993131637573, "beta_mean": 0.002982287434861064, "nonfinite": 0.0, "weighted_memory_to_active": 0.02674822323024273, "mix_std": 0.23916327953338623, "beta_std": 0.006643250584602356, "beta_low": 0.93316650390625, "beta_high": 0.0, "orthogonal_to_active": 0.01501900888979435, "evidence_weights": [-0.19153393805027008, -0.3643472194671631, 0.7261483669281006], "agreement_weight": 0.06792456656694412, "orthogonal_weight": -0.8045855164527893}, {"head": 1, "memory_to_active": 0.40009361505508423, "contribution_to_active": 0.38871997594833374, "mix_mean": 0.3803657293319702, "mix_low": 0.004313468933105469, "mix_high": 0.00028514862060546875, "read_cosine": 0.42264971137046814, "beta_mean": 0.01684427633881569, "nonfinite": 0.0, "weighted_memory_to_active": 0.174104705452919, "mix_std": 0.24347826838493347, "beta_std": 0.021728431805968285, "beta_low": 0.4669189453125, "beta_high": 0.0, "orthogonal_to_active": 0.0761849582195282, "evidence_weights": [-0.17790043354034424, -0.21206359565258026, 0.5136439204216003], "agreement_weight": 0.06186181306838989, "orthogonal_weight": -0.49653345346450806}, {"head": 2, "memory_to_active": 0.06990376114845276, "contribution_to_active": 0.5739677548408508, "mix_mean": 0.5373127460479736, "mix_low": 0.0007762908935546875, "mix_high": 0.0015230178833007812, "read_cosine": 0.3632952570915222, "beta_mean": 0.0007894900045357645, "nonfinite": 0.0, "weighted_memory_to_active": 0.04373706132173538, "mix_std": 0.2507299482822418, "beta_std": 0.0035759008023887873, "beta_low": 0.9931640625, "beta_high": 0.0, "orthogonal_to_active": 0.03252800554037094, "evidence_weights": [-0.20578055083751678, -0.31492719054222107, 0.5551990866661072], "agreement_weight": 0.09248318523168564, "orthogonal_weight": -0.8508237600326538}, {"head": 3, "memory_to_active": 0.19811585545539856, "contribution_to_active": 0.5069436430931091, "mix_mean": 0.445205420255661, "mix_low": 0.000988006591796875, "mix_high": 0.00020599365234375, "read_cosine": 0.7631054520606995, "beta_mean": 0.01772778108716011, "nonfinite": 0.0, "weighted_memory_to_active": 0.12320629507303238, "mix_std": 0.2486322820186615, "beta_std": 0.017494186758995056, "beta_low": 0.3302001953125, "beta_high": 0.0, "orthogonal_to_active": 0.03236529231071472, "evidence_weights": [-0.14532414078712463, -0.26112303137779236, 0.7329564094543457], "agreement_weight": 0.102279432117939, "orthogonal_weight": -0.4783952832221985}, {"head": 4, "memory_to_active": 2.583087682723999, "contribution_to_active": 0.2586909234523773, "mix_mean": 0.025359414517879486, "mix_low": 0.4264364242553711, "mix_high": 0.0, "read_cosine": 0.32404857873916626, "beta_mean": 0.09407379478216171, "nonfinite": 0.0, "weighted_memory_to_active": 0.2526891827583313, "mix_std": 0.04185166582465172, "beta_std": 0.07922329753637314, "beta_low": 0.01055908203125, "beta_high": 0.0, "orthogonal_to_active": 0.007452737540006638, "evidence_weights": [0.03553931042551994, -0.008283325470983982, -0.1882808953523636], "agreement_weight": 0.024612855166196823, "orthogonal_weight": 0.031040946021676064}, {"head": 5, "memory_to_active": 0.242596834897995, "contribution_to_active": 0.6411802768707275, "mix_mean": 0.30349382758140564, "mix_low": 0.10650253295898438, "mix_high": 0.001537322998046875, "read_cosine": 0.5774140357971191, "beta_mean": 0.07977522909641266, "nonfinite": 0.0, "weighted_memory_to_active": 0.16116982698440552, "mix_std": 0.32132259011268616, "beta_std": 0.11282041668891907, "beta_low": 0.25885009765625, "beta_high": 0.0, "orthogonal_to_active": 0.1723124235868454, "evidence_weights": [-0.06767647713422775, -0.3130253851413727, -0.42165642976760864], "agreement_weight": 0.020732535049319267, "orthogonal_weight": 1.3494068384170532}]}, {"layer": 9, "memory_to_active": 1.2163543701171875, "contribution_to_active": 0.6256726980209351, "mix_mean": 0.26657557487487793, "mix_low": 0.15949933230876923, "mix_high": 0.0017698606243357062, "read_cosine": 0.36240074038505554, "beta_mean": 0.07958966493606567, "nonfinite": 0.0, "weighted_memory_to_active": 0.38947010040283203, "mix_std": 0.2109488546848297, "beta_std": 0.047621820122003555, "beta_low": 0.499298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.03778769448399544, "output_influence": 0.4311143755912781, "residual_influence": 0.11196541786193848, "state_active": 0.0461735874414444, "state_memory": 0.023104675114154816, "state_peak": 39.195335388183594, "growth_active": 1.3957439661026, "growth_memory": 1.4186432361602783, "heads": [{"head": 0, "memory_to_active": 0.00552297243848443, "contribution_to_active": 0.48911768198013306, "mix_mean": 0.5457457304000854, "mix_low": 0.0021953582763671875, "mix_high": 0.006524085998535156, "read_cosine": 0.225445955991745, "beta_mean": 1.3513337762560695e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031019223388284445, "mix_std": 0.2980945408344269, "beta_std": 5.266349035082385e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002603420987725258, "evidence_weights": [-0.27533188462257385, -0.27871331572532654, 0.9912550449371338], "agreement_weight": 0.060935452580451965, "orthogonal_weight": -0.8784120082855225}, {"head": 1, "memory_to_active": 0.029820000752806664, "contribution_to_active": 0.5325524806976318, "mix_mean": 0.37802809476852417, "mix_low": 0.013454437255859375, "mix_high": 0.0011281967163085938, "read_cosine": 0.4014461636543274, "beta_mean": 9.018225682666525e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015863588079810143, "mix_std": 0.28165748715400696, "beta_std": 0.000233156475587748, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.009298999793827534, "evidence_weights": [-0.2027091532945633, -0.23103860020637512, 1.0293298959732056], "agreement_weight": 0.28872162103652954, "orthogonal_weight": -0.6748361587524414}, {"head": 2, "memory_to_active": 0.24907760322093964, "contribution_to_active": 0.5514782667160034, "mix_mean": 0.4532245397567749, "mix_low": 0.005385398864746094, "mix_high": 0.0024518966674804688, "read_cosine": 0.39625194668769836, "beta_mean": 0.001172567019239068, "nonfinite": 0.0, "weighted_memory_to_active": 0.14267724752426147, "mix_std": 0.2885740399360657, "beta_std": 0.005803204607218504, "beta_low": 0.98828125, "beta_high": 0.0, "orthogonal_to_active": 0.04999975860118866, "evidence_weights": [-0.1674310863018036, -0.2024160772562027, 0.5233797430992126], "agreement_weight": 0.20827533304691315, "orthogonal_weight": -0.4319670796394348}, {"head": 3, "memory_to_active": 1.7565897703170776, "contribution_to_active": 0.2003590166568756, "mix_mean": 0.039884958416223526, "mix_low": 0.38767051696777344, "mix_high": 2.384185791015625e-05, "read_cosine": 0.42913931608200073, "beta_mean": 0.09694921970367432, "nonfinite": 0.0, "weighted_memory_to_active": 0.1706049144268036, "mix_std": 0.09121857583522797, "beta_std": 0.0660354346036911, "beta_low": 0.007568359375, "beta_high": 0.0, "orthogonal_to_active": 0.030987698584794998, "evidence_weights": [0.05293259769678116, 0.03565507382154465, -0.1344115436077118], "agreement_weight": 0.06996200233697891, "orthogonal_weight": 0.20059029757976532}, {"head": 4, "memory_to_active": 3.5132782459259033, "contribution_to_active": 1.3734928369522095, "mix_mean": 0.11996683478355408, "mix_low": 0.1739826202392578, "mix_high": 0.0004911422729492188, "read_cosine": 0.25775226950645447, "beta_mean": 0.21041592955589294, "nonfinite": 0.0, "weighted_memory_to_active": 1.3061801195144653, "mix_std": 0.18455329537391663, "beta_std": 0.11140304058790207, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.026268182322382927, "evidence_weights": [-0.04965740814805031, -0.13832205533981323, 0.03164278715848923], "agreement_weight": -0.04694056883454323, "orthogonal_weight": -0.020057313144207}, {"head": 5, "memory_to_active": 1.7438371181488037, "contribution_to_active": 0.6070358753204346, "mix_mean": 0.06260322779417038, "mix_low": 0.37430763244628906, "mix_high": 0.0, "read_cosine": 0.46436870098114014, "beta_mean": 0.16889655590057373, "nonfinite": 0.0, "weighted_memory_to_active": 0.6983925700187683, "mix_std": 0.12159519642591476, "beta_std": 0.10220342874526978, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10756810754537582, "evidence_weights": [0.06548269093036652, -0.04530169069766998, 0.10131905227899551], "agreement_weight": 0.023168472573161125, "orthogonal_weight": 0.2217683494091034}]}, {"layer": 10, "memory_to_active": 0.16430965065956116, "contribution_to_active": 0.4605886936187744, "mix_mean": 0.45053350925445557, "mix_low": 0.01260995864868164, "mix_high": 0.008392652496695518, "read_cosine": 0.38528919219970703, "beta_mean": 0.04880034551024437, "nonfinite": 0.0, "weighted_memory_to_active": 0.07367692142724991, "mix_std": 0.2703944742679596, "beta_std": 0.05087057128548622, "beta_low": 0.6508585810661316, "beta_high": 0.0, "orthogonal_to_active": 0.04550149291753769, "output_influence": 0.38636544346809387, "residual_influence": 0.12502461671829224, "state_active": 0.05694764107465744, "state_memory": 0.013841532170772552, "state_peak": 112.58544921875, "growth_active": 1.0968232154846191, "growth_memory": 1.080781102180481, "heads": [{"head": 0, "memory_to_active": 0.3911033868789673, "contribution_to_active": 0.7639645338058472, "mix_mean": 0.620217502117157, "mix_low": 0.027376174926757812, "mix_high": 0.04374504089355469, "read_cosine": 0.23264473676681519, "beta_mean": 0.217827707529068, "nonfinite": 0.0, "weighted_memory_to_active": 0.2636222541332245, "mix_std": 0.36800655722618103, "beta_std": 0.23056048154830933, "beta_low": 0.10101318359375, "beta_high": 0.0, "orthogonal_to_active": 0.17933067679405212, "evidence_weights": [-0.1607123166322708, -0.35146576166152954, -0.421750009059906], "agreement_weight": 0.019450189545750618, "orthogonal_weight": 0.7456135749816895}, {"head": 1, "memory_to_active": 0.016835033893585205, "contribution_to_active": 0.5338307619094849, "mix_mean": 0.5381528735160828, "mix_low": 0.0008745193481445312, "mix_high": 0.00511932373046875, "read_cosine": 0.26573020219802856, "beta_mean": 9.986090299207717e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00900762528181076, "mix_std": 0.26719391345977783, "beta_std": 0.00014910948812030256, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010358039289712906, "evidence_weights": [-0.27163228392601013, -0.19591578841209412, 0.5241485834121704], "agreement_weight": 0.32624515891075134, "orthogonal_weight": -1.273585319519043}, {"head": 2, "memory_to_active": 0.09885525703430176, "contribution_to_active": 0.4483530819416046, "mix_mean": 0.44776034355163574, "mix_low": 0.0011949539184570312, "mix_high": 0.0010547637939453125, "read_cosine": -0.006816583219915628, "beta_mean": 0.00048747070832177997, "nonfinite": 0.0, "weighted_memory_to_active": 0.038896411657333374, "mix_std": 0.27144232392311096, "beta_std": 0.0020301283802837133, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.013712981715798378, "evidence_weights": [-0.1733471155166626, -0.22460727393627167, 0.5346940159797668], "agreement_weight": -0.05066726729273796, "orthogonal_weight": -0.3471750319004059}, {"head": 3, "memory_to_active": 0.30388087034225464, "contribution_to_active": 0.11693936586380005, "mix_mean": 0.2640864849090576, "mix_low": 0.04143238067626953, "mix_high": 0.00034999847412109375, "read_cosine": 0.8878332376480103, "beta_mean": 0.06747827678918839, "nonfinite": 0.0, "weighted_memory_to_active": 0.04733283817768097, "mix_std": 0.23518039286136627, "beta_std": 0.060523007065057755, "beta_low": 0.0025634765625, "beta_high": 0.0, "orthogonal_to_active": 0.04813608154654503, "evidence_weights": [-0.12655854225158691, -0.14463412761688232, 0.18165184557437897], "agreement_weight": 0.00456130038946867, "orthogonal_weight": 1.4156650304794312}, {"head": 4, "memory_to_active": 0.027585038915276527, "contribution_to_active": 0.4759800136089325, "mix_mean": 0.4646655321121216, "mix_low": 0.0012140274047851562, "mix_high": 7.82012939453125e-05, "read_cosine": 0.07260818034410477, "beta_mean": 5.470372343552299e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00816095806658268, "mix_std": 0.25540003180503845, "beta_std": 0.0026627678889781237, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0026187191251665354, "evidence_weights": [-0.18100246787071228, -0.2221985161304474, 0.5374237298965454], "agreement_weight": 0.03984251990914345, "orthogonal_weight": -0.32335683703422546}, {"head": 5, "memory_to_active": 0.14759832620620728, "contribution_to_active": 0.4244644045829773, "mix_mean": 0.3683182895183563, "mix_low": 0.0035676956176757812, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8597352504730225, "beta_mean": 0.006854034960269928, "nonfinite": 0.0, "weighted_memory_to_active": 0.07504140585660934, "mix_std": 0.22514361143112183, "beta_std": 0.009297939017415047, "beta_low": 0.80865478515625, "beta_high": 0.0, "orthogonal_to_active": 0.01885242760181427, "evidence_weights": [-0.1583593785762787, -0.1647738367319107, 0.2916775643825531], "agreement_weight": 0.09252149611711502, "orthogonal_weight": -0.5748788714408875}]}, {"layer": 11, "memory_to_active": 0.1876358538866043, "contribution_to_active": 0.5099949240684509, "mix_mean": 0.4102465510368347, "mix_low": 0.0581364631652832, "mix_high": 0.028271835297346115, "read_cosine": 0.4717975854873657, "beta_mean": 0.037018295377492905, "nonfinite": 0.0, "weighted_memory_to_active": 0.06988157331943512, "mix_std": 0.30899009108543396, "beta_std": 0.05335506051778793, "beta_low": 0.5225830674171448, "beta_high": 0.0, "orthogonal_to_active": 0.046418048441410065, "output_influence": 0.46311938762664795, "residual_influence": 0.1192874163389206, "state_active": 0.1352192759513855, "state_memory": 0.009015440940856934, "state_peak": 32.22330093383789, "growth_active": 1.1199932098388672, "growth_memory": 1.4778834581375122, "heads": [{"head": 0, "memory_to_active": 0.012784305959939957, "contribution_to_active": 0.5469385385513306, "mix_mean": 0.5618388056755066, "mix_low": 0.00185394287109375, "mix_high": 0.006855964660644531, "read_cosine": 0.36809825897216797, "beta_mean": 0.00012204295489937067, "nonfinite": 0.0, "weighted_memory_to_active": 0.008668665774166584, "mix_std": 0.2716503143310547, "beta_std": 0.004287517163902521, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005266091786324978, "evidence_weights": [-0.15888074040412903, -0.34396424889564514, 0.606475830078125], "agreement_weight": 0.17112290859222412, "orthogonal_weight": -0.7113713622093201}, {"head": 1, "memory_to_active": 0.4080488383769989, "contribution_to_active": 0.24058562517166138, "mix_mean": 0.2611731290817261, "mix_low": 0.023908615112304688, "mix_high": 7.05718994140625e-05, "read_cosine": 0.6200040578842163, "beta_mean": 0.03196281939744949, "nonfinite": 0.0, "weighted_memory_to_active": 0.13871701061725616, "mix_std": 0.23449590802192688, "beta_std": 0.06016334518790245, "beta_low": 0.4267578125, "beta_high": 0.0, "orthogonal_to_active": 0.023382358253002167, "evidence_weights": [-0.15691766142845154, -0.12610667943954468, 0.21222108602523804], "agreement_weight": 0.07600341737270355, "orthogonal_weight": -0.2310805320739746}, {"head": 2, "memory_to_active": 0.035971011966466904, "contribution_to_active": 0.8946090936660767, "mix_mean": 0.47253698110580444, "mix_low": 0.09108352661132812, "mix_high": 0.0905618667602539, "read_cosine": 0.29299357533454895, "beta_mean": 0.07640765607357025, "nonfinite": 0.0, "weighted_memory_to_active": 0.028615206480026245, "mix_std": 0.4158787429332733, "beta_std": 0.10360220819711685, "beta_low": 0.42315673828125, "beta_high": 0.0, "orthogonal_to_active": 0.09210846573114395, "evidence_weights": [-0.07160323113203049, -0.4500298500061035, -0.9981197118759155], "agreement_weight": 0.003444696543738246, "orthogonal_weight": 3.4604573249816895}, {"head": 3, "memory_to_active": 0.017243588343262672, "contribution_to_active": 0.7092388868331909, "mix_mean": 0.42304176092147827, "mix_low": 0.20476055145263672, "mix_high": 0.07098102569580078, "read_cosine": 0.4560525715351105, "beta_mean": 0.06461787968873978, "nonfinite": 0.0, "weighted_memory_to_active": 0.00996512919664383, "mix_std": 0.4289581775665283, "beta_std": 0.096550852060318, "beta_low": 0.46429443359375, "beta_high": 0.0, "orthogonal_to_active": 0.034087885171175, "evidence_weights": [-0.03391002118587494, -0.3660414516925812, -0.3995041847229004], "agreement_weight": 0.030520081520080566, "orthogonal_weight": 3.72248911857605}, {"head": 4, "memory_to_active": 0.5150662660598755, "contribution_to_active": 0.3125571608543396, "mix_mean": 0.2811696231365204, "mix_low": 0.026801109313964844, "mix_high": 0.0001621246337890625, "read_cosine": 0.2209574282169342, "beta_mean": 0.038097310811281204, "nonfinite": 0.0, "weighted_memory_to_active": 0.17272035777568817, "mix_std": 0.24304413795471191, "beta_std": 0.03869956359267235, "beta_low": 0.13897705078125, "beta_high": 0.0, "orthogonal_to_active": 0.10257818549871445, "evidence_weights": [-0.15447556972503662, -0.12326601147651672, 1.386326551437378], "agreement_weight": -0.14043433964252472, "orthogonal_weight": -0.6244742274284363}, {"head": 5, "memory_to_active": 0.13670113682746887, "contribution_to_active": 0.35604003071784973, "mix_mean": 0.46171873807907104, "mix_low": 0.00041103363037109375, "mix_high": 0.00099945068359375, "read_cosine": 0.8726795315742493, "beta_mean": 0.0109020434319973, "nonfinite": 0.0, "weighted_memory_to_active": 0.06060308963060379, "mix_std": 0.25991323590278625, "beta_std": 0.01682686060667038, "beta_low": 0.68292236328125, "beta_high": 0.0, "orthogonal_to_active": 0.021085280925035477, "evidence_weights": [-0.20003151893615723, -0.2564277648925781, 0.5065074563026428], "agreement_weight": 0.044423747807741165, "orthogonal_weight": -0.6325802803039551}]}, {"layer": 12, "memory_to_active": 0.36309385299682617, "contribution_to_active": 0.5483360290527344, "mix_mean": 0.4473656415939331, "mix_low": 0.019718807190656662, "mix_high": 0.0028465590439736843, "read_cosine": 0.27408695220947266, "beta_mean": 0.015818912535905838, "nonfinite": 0.0, "weighted_memory_to_active": 0.1927575170993805, "mix_std": 0.25588086247444153, "beta_std": 0.016419444233179092, "beta_low": 0.8127645254135132, "beta_high": 0.0, "orthogonal_to_active": 0.07091875374317169, "output_influence": 0.40845054388046265, "residual_influence": 0.11220807582139969, "state_active": 0.04464145749807358, "state_memory": 0.01052025705575943, "state_peak": 25.507768630981445, "growth_active": 1.1422404050827026, "growth_memory": 1.2459748983383179, "heads": [{"head": 0, "memory_to_active": 0.12116621434688568, "contribution_to_active": 0.41649943590164185, "mix_mean": 0.3901424705982208, "mix_low": 0.005412101745605469, "mix_high": 0.0005769729614257812, "read_cosine": 0.30476653575897217, "beta_mean": 0.002019887790083885, "nonfinite": 0.0, "weighted_memory_to_active": 0.04821881651878357, "mix_std": 0.26208388805389404, "beta_std": 0.006524152122437954, "beta_low": 0.9627685546875, "beta_high": 0.0, "orthogonal_to_active": 0.025301532819867134, "evidence_weights": [-0.15895825624465942, -0.22571928799152374, 0.26111942529678345], "agreement_weight": 0.10855921357870102, "orthogonal_weight": -0.5586935877799988}, {"head": 1, "memory_to_active": 0.13965046405792236, "contribution_to_active": 0.4752657413482666, "mix_mean": 0.48264995217323303, "mix_low": 0.00066375732421875, "mix_high": 0.00024509429931640625, "read_cosine": 0.5434206128120422, "beta_mean": 0.0013159092050045729, "nonfinite": 0.0, "weighted_memory_to_active": 0.06732920557260513, "mix_std": 0.25980183482170105, "beta_std": 0.005541760008782148, "beta_low": 0.98583984375, "beta_high": 0.0, "orthogonal_to_active": 0.050337184220552444, "evidence_weights": [-0.1964280605316162, -0.25436678528785706, 0.8256837725639343], "agreement_weight": 0.10394002497196198, "orthogonal_weight": -0.9127932786941528}, {"head": 2, "memory_to_active": 0.616480827331543, "contribution_to_active": 0.45809727907180786, "mix_mean": 0.32882943749427795, "mix_low": 0.02591419219970703, "mix_high": 0.00026035308837890625, "read_cosine": 0.45617014169692993, "beta_mean": 0.08783634006977081, "nonfinite": 0.0, "weighted_memory_to_active": 0.2560199499130249, "mix_std": 0.25452691316604614, "beta_std": 0.06467276811599731, "beta_low": 0.0013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.19942082464694977, "evidence_weights": [-0.23911716043949127, -0.04160245880484581, -0.056616101413965225], "agreement_weight": 0.025923656299710274, "orthogonal_weight": 0.9510679841041565}, {"head": 3, "memory_to_active": 0.03478196635842323, "contribution_to_active": 0.715328574180603, "mix_mean": 0.6732184886932373, "mix_low": 0.00029659271240234375, "mix_high": 0.012042045593261719, "read_cosine": 0.05658388510346413, "beta_mean": 0.00029858469497412443, "nonfinite": 0.0, "weighted_memory_to_active": 0.02592712640762329, "mix_std": 0.24517124891281128, "beta_std": 0.004017151892185211, "beta_low": 0.997802734375, "beta_high": 0.0, "orthogonal_to_active": 0.023499784991145134, "evidence_weights": [-0.3018147647380829, -0.323882520198822, 1.0303874015808105], "agreement_weight": 0.23566579818725586, "orthogonal_weight": -0.927705705165863}, {"head": 4, "memory_to_active": 1.218345046043396, "contribution_to_active": 0.7615686655044556, "mix_mean": 0.21990357339382172, "mix_low": 0.07981109619140625, "mix_high": 0.0015954971313476562, "read_cosine": -0.06587140262126923, "beta_mean": 0.0027938401326537132, "nonfinite": 0.0, "weighted_memory_to_active": 0.7321087121963501, "mix_std": 0.23842275142669678, "beta_std": 0.015045183710753918, "beta_low": 0.9383544921875, "beta_high": 0.0, "orthogonal_to_active": 0.10635437071323395, "evidence_weights": [-0.028426647186279297, -0.17751078307628632, 0.003790459595620632], "agreement_weight": -0.003669554367661476, "orthogonal_weight": -0.1594141572713852}, {"head": 5, "memory_to_active": 0.048138633370399475, "contribution_to_active": 0.463256299495697, "mix_mean": 0.5894497036933899, "mix_low": 0.006215095520019531, "mix_high": 0.0023593902587890625, "read_cosine": 0.34945181012153625, "beta_mean": 0.0006489109946414828, "nonfinite": 0.0, "weighted_memory_to_active": 0.02694130316376686, "mix_std": 0.27527838945388794, "beta_std": 0.0027156395372003317, "beta_low": 0.990478515625, "beta_high": 0.0, "orthogonal_to_active": 0.020598847419023514, "evidence_weights": [-0.2657257914543152, -0.2974851429462433, 0.9138033986091614], "agreement_weight": 0.14498458802700043, "orthogonal_weight": -0.8409768342971802}]}, {"layer": 13, "memory_to_active": 0.15888512134552002, "contribution_to_active": 0.48920246958732605, "mix_mean": 0.47911733388900757, "mix_low": 0.006055832374840975, "mix_high": 0.0026253066025674343, "read_cosine": 0.4024346172809601, "beta_mean": 0.017856581136584282, "nonfinite": 0.0, "weighted_memory_to_active": 0.06790143251419067, "mix_std": 0.25808531045913696, "beta_std": 0.01662595197558403, "beta_low": 0.7904358506202698, "beta_high": 0.0, "orthogonal_to_active": 0.06321601569652557, "output_influence": 0.4480137228965759, "residual_influence": 0.1484590768814087, "state_active": 0.08982214331626892, "state_memory": 0.037598878145217896, "state_peak": 93.20072174072266, "growth_active": 0.9925459623336792, "growth_memory": 1.0664384365081787, "heads": [{"head": 0, "memory_to_active": 0.054133109748363495, "contribution_to_active": 0.5126955509185791, "mix_mean": 0.5036607384681702, "mix_low": 0.0006704330444335938, "mix_high": 0.0011816024780273438, "read_cosine": 0.3742994964122772, "beta_mean": 0.00023069878807291389, "nonfinite": 0.0, "weighted_memory_to_active": 0.02940240129828453, "mix_std": 0.2647603154182434, "beta_std": 0.00045684646465815604, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.028211817145347595, "evidence_weights": [-0.20362046360969543, -0.24663734436035156, 0.7679044604301453], "agreement_weight": 0.04997551068663597, "orthogonal_weight": -1.0630643367767334}, {"head": 1, "memory_to_active": 0.037783361971378326, "contribution_to_active": 0.5456700325012207, "mix_mean": 0.4831708073616028, "mix_low": 0.0025739669799804688, "mix_high": 0.00032520294189453125, "read_cosine": 0.23044846951961517, "beta_mean": 0.0001351450919173658, "nonfinite": 0.0, "weighted_memory_to_active": 0.01997462660074234, "mix_std": 0.26537156105041504, "beta_std": 0.00027296796906739473, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.019642267376184464, "evidence_weights": [-0.18649783730506897, -0.25567394495010376, 0.7291950583457947], "agreement_weight": 0.17816366255283356, "orthogonal_weight": -1.0351923704147339}, {"head": 2, "memory_to_active": 0.14192040264606476, "contribution_to_active": 0.35882723331451416, "mix_mean": 0.3490563929080963, "mix_low": 0.0027685165405273438, "mix_high": 5.435943603515625e-05, "read_cosine": 0.6511311531066895, "beta_mean": 0.01163388229906559, "nonfinite": 0.0, "weighted_memory_to_active": 0.054142579436302185, "mix_std": 0.21729716658592224, "beta_std": 0.030069604516029358, "beta_low": 0.76300048828125, "beta_high": 0.0, "orthogonal_to_active": 0.026368826627731323, "evidence_weights": [-0.22262878715991974, -0.16339436173439026, 0.49080339074134827], "agreement_weight": 0.038699712604284286, "orthogonal_weight": -0.75654137134552}, {"head": 3, "memory_to_active": 0.14290286600589752, "contribution_to_active": 0.5945470333099365, "mix_mean": 0.5939790606498718, "mix_low": 0.0143890380859375, "mix_high": 0.010387420654296875, "read_cosine": 0.02102036215364933, "beta_mean": 0.0009491737000644207, "nonfinite": 0.0, "weighted_memory_to_active": 0.10304586589336395, "mix_std": 0.28060007095336914, "beta_std": 0.0032270473893731833, "beta_low": 0.991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.10282793641090393, "evidence_weights": [-0.34936773777008057, -0.23586949706077576, 1.984756588935852], "agreement_weight": 2.1872799396514893, "orthogonal_weight": -1.0500140190124512}, {"head": 4, "memory_to_active": 0.4605562090873718, "contribution_to_active": 0.29752272367477417, "mix_mean": 0.3895895779132843, "mix_low": 0.01213836669921875, "mix_high": 0.00013637542724609375, "read_cosine": 0.8690006136894226, "beta_mean": 0.09323295205831528, "nonfinite": 0.0, "weighted_memory_to_active": 0.13698819279670715, "mix_std": 0.23859800398349762, "beta_std": 0.06034332141280174, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.156955748796463, "evidence_weights": [-0.19080448150634766, -0.2341870814561844, 0.4932419955730438], "agreement_weight": -0.004072151146829128, "orthogonal_weight": 1.7308292388916016}, {"head": 5, "memory_to_active": 0.11601472645998001, "contribution_to_active": 0.6259520649909973, "mix_mean": 0.55524742603302, "mix_low": 0.0037946701049804688, "mix_high": 0.0036668777465820312, "read_cosine": 0.26870763301849365, "beta_mean": 0.0009576326119713485, "nonfinite": 0.0, "weighted_memory_to_active": 0.06385488063097, "mix_std": 0.2818847596645355, "beta_std": 0.0053859250620007515, "beta_low": 0.98822021484375, "beta_high": 0.0, "orthogonal_to_active": 0.04528948292136192, "evidence_weights": [-0.21705347299575806, -0.2755221128463745, 0.5603339672088623], "agreement_weight": 0.0795564204454422, "orthogonal_weight": -0.782297670841217}]}, {"layer": 14, "memory_to_active": 0.2730283737182617, "contribution_to_active": 0.42808297276496887, "mix_mean": 0.4059692621231079, "mix_low": 0.02513011358678341, "mix_high": 0.003635883331298828, "read_cosine": 0.6208246946334839, "beta_mean": 0.07355468720197678, "nonfinite": 0.0, "weighted_memory_to_active": 0.12710869312286377, "mix_std": 0.24990792572498322, "beta_std": 0.06601263582706451, "beta_low": 0.4325154721736908, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.0439101904630661, "output_influence": 0.2921898066997528, "residual_influence": 0.11231425404548645, "state_active": 0.07082171738147736, "state_memory": 0.016176793724298477, "state_peak": 26.75347328186035, "growth_active": 1.1771787405014038, "growth_memory": 1.2853516340255737, "heads": [{"head": 0, "memory_to_active": 0.35765379667282104, "contribution_to_active": 0.23086710274219513, "mix_mean": 0.24672776460647583, "mix_low": 0.01232147216796875, "mix_high": 1.811981201171875e-05, "read_cosine": 0.7581391334533691, "beta_mean": 0.06657709926366806, "nonfinite": 0.0, "weighted_memory_to_active": 0.10955910384654999, "mix_std": 0.21224796772003174, "beta_std": 0.06724117696285248, "beta_low": 0.03533935546875, "beta_high": 0.0, "orthogonal_to_active": 0.04249584302306175, "evidence_weights": [-0.0953877866268158, -0.07904696464538574, 0.7316716313362122], "agreement_weight": 0.03441798686981201, "orthogonal_weight": 0.6181772351264954}, {"head": 1, "memory_to_active": 0.49408161640167236, "contribution_to_active": 0.39303556084632874, "mix_mean": 0.2591739892959595, "mix_low": 0.1171722412109375, "mix_high": 9.1552734375e-05, "read_cosine": 0.6975620985031128, "beta_mean": 0.3067866265773773, "nonfinite": 0.0, "weighted_memory_to_active": 0.24298202991485596, "mix_std": 0.27126070857048035, "beta_std": 0.26753801107406616, "beta_low": 0.07916259765625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.022597603499889374, "evidence_weights": [0.010232094675302505, -0.2786829173564911, -0.1465286910533905], "agreement_weight": 0.05452645570039749, "orthogonal_weight": 0.1682080626487732}, {"head": 2, "memory_to_active": 0.22448405623435974, "contribution_to_active": 0.5148376226425171, "mix_mean": 0.5180216431617737, "mix_low": 0.004344940185546875, "mix_high": 0.00177764892578125, "read_cosine": 0.3968863785266876, "beta_mean": 0.007000365760177374, "nonfinite": 0.0, "weighted_memory_to_active": 0.13299459218978882, "mix_std": 0.2725929021835327, "beta_std": 0.008748319931328297, "beta_low": 0.7916259765625, "beta_high": 0.0, "orthogonal_to_active": 0.09761110693216324, "evidence_weights": [-0.17653384804725647, -0.26777687668800354, 0.8000131249427795], "agreement_weight": 0.07937096059322357, "orthogonal_weight": -0.8342716097831726}, {"head": 3, "memory_to_active": 0.060445867478847504, "contribution_to_active": 0.6552395224571228, "mix_mean": 0.6548373699188232, "mix_low": 0.0032052993774414062, "mix_high": 0.012897491455078125, "read_cosine": 0.37189748883247375, "beta_mean": 0.00830870121717453, "nonfinite": 0.0, "weighted_memory_to_active": 0.04106738418340683, "mix_std": 0.2447369396686554, "beta_std": 0.007124456111341715, "beta_low": 0.71728515625, "beta_high": 0.0, "orthogonal_to_active": 0.02251284010708332, "evidence_weights": [-0.38029471039772034, -0.3280000388622284, 0.4374854862689972], "agreement_weight": 0.07766954600811005, "orthogonal_weight": -0.6651723980903625}, {"head": 4, "memory_to_active": 0.18828774988651276, "contribution_to_active": 0.4841819703578949, "mix_mean": 0.46471086144447327, "mix_low": 0.0031795501708984375, "mix_high": 0.006458282470703125, "read_cosine": 0.7899518609046936, "beta_mean": 0.04998230189085007, "nonfinite": 0.0, "weighted_memory_to_active": 0.10816557705402374, "mix_std": 0.2619548738002777, "beta_std": 0.03463535010814667, "beta_low": 0.02557373046875, "beta_high": 0.0, "orthogonal_to_active": 0.030753064900636673, "evidence_weights": [-0.22025437653064728, -0.26862701773643494, 0.38912081718444824], "agreement_weight": 0.1148683950304985, "orthogonal_weight": -0.5029938220977783}, {"head": 5, "memory_to_active": 0.3132171630859375, "contribution_to_active": 0.2903360426425934, "mix_mean": 0.29234379529953003, "mix_low": 0.010557174682617188, "mix_high": 0.00057220458984375, "read_cosine": 0.7105109691619873, "beta_mean": 0.002673005685210228, "nonfinite": 0.0, "weighted_memory_to_active": 0.1278834491968155, "mix_std": 0.2366541028022766, "beta_std": 0.010788444429636002, "beta_low": 0.94610595703125, "beta_high": 0.0, "orthogonal_to_active": 0.04749065637588501, "evidence_weights": [-0.18204617500305176, -0.14547082781791687, 0.1801503449678421], "agreement_weight": 0.08841531723737717, "orthogonal_weight": -0.6106694936752319}]}, {"layer": 15, "memory_to_active": 0.06804502755403519, "contribution_to_active": 0.4722900688648224, "mix_mean": 0.38595786690711975, "mix_low": 0.008098125457763672, "mix_high": 0.0025730133056640625, "read_cosine": 0.6902085542678833, "beta_mean": 0.04035303741693497, "nonfinite": 0.0, "weighted_memory_to_active": 0.030337488278746605, "mix_std": 0.2785876989364624, "beta_std": 0.04438495635986328, "beta_low": 0.7104899883270264, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.0051910290494561195, "output_influence": 0.2911038398742676, "residual_influence": 0.14799799025058746, "state_active": 0.11703072488307953, "state_memory": 0.007527603767812252, "state_peak": 37.2088737487793, "growth_active": 1.3275049924850464, "growth_memory": 1.712514042854309, "heads": [{"head": 0, "memory_to_active": 0.10265973210334778, "contribution_to_active": 0.46491917967796326, "mix_mean": 0.36070919036865234, "mix_low": 0.004210472106933594, "mix_high": 0.003589630126953125, "read_cosine": 0.6863438487052917, "beta_mean": 0.037190645933151245, "nonfinite": 0.0, "weighted_memory_to_active": 0.04546061158180237, "mix_std": 0.26861274242401123, "beta_std": 0.026312947273254395, "beta_low": 0.11419677734375, "beta_high": 0.0, "orthogonal_to_active": 0.017132524400949478, "evidence_weights": [-0.16432048380374908, -0.19031785428524017, 0.34308212995529175], "agreement_weight": 0.08471265435218811, "orthogonal_weight": -0.5729610323905945}, {"head": 1, "memory_to_active": 0.17181023955345154, "contribution_to_active": 0.4207308888435364, "mix_mean": 0.3921505808830261, "mix_low": 0.0056056976318359375, "mix_high": 0.004252433776855469, "read_cosine": 0.8661624193191528, "beta_mean": 0.00147807109169662, "nonfinite": 0.0, "weighted_memory_to_active": 0.06966757029294968, "mix_std": 0.27977481484413147, "beta_std": 0.0035725294146686792, "beta_low": 0.97674560546875, "beta_high": 0.0, "orthogonal_to_active": 0.002782012801617384, "evidence_weights": [-0.18565520644187927, -0.1921740174293518, 0.2864556312561035], "agreement_weight": 0.0972672551870346, "orthogonal_weight": -0.06406097114086151}, {"head": 2, "memory_to_active": 0.003268797881901264, "contribution_to_active": 0.442776620388031, "mix_mean": 0.3376529812812805, "mix_low": 0.014983177185058594, "mix_high": 9.72747802734375e-05, "read_cosine": 0.771043062210083, "beta_mean": 5.313927977113053e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014121655840426683, "mix_std": 0.2718406915664673, "beta_std": 0.00013132930325809866, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 7.511593139497563e-05, "evidence_weights": [-0.20401033759117126, -0.27324843406677246, 0.28111323714256287], "agreement_weight": 0.03412627801299095, "orthogonal_weight": -0.09668208658695221}, {"head": 3, "memory_to_active": 0.030096780508756638, "contribution_to_active": 0.4513823390007019, "mix_mean": 0.3191033601760864, "mix_low": 0.00450897216796875, "mix_high": 5.435943603515625e-05, "read_cosine": 0.6438373327255249, "beta_mean": 0.00044128228910267353, "nonfinite": 0.0, "weighted_memory_to_active": 0.01379963755607605, "mix_std": 0.26015040278434753, "beta_std": 0.001450487063266337, "beta_low": 0.99530029296875, "beta_high": 0.0, "orthogonal_to_active": 0.008718958124518394, "evidence_weights": [-0.1888263076543808, -0.23738263547420502, 0.3910553753376007], "agreement_weight": 0.013596193864941597, "orthogonal_weight": -0.9279932975769043}, {"head": 4, "memory_to_active": 0.09535762667655945, "contribution_to_active": 0.5852720141410828, "mix_mean": 0.3856419622898102, "mix_low": 0.013491630554199219, "mix_high": 0.000446319580078125, "read_cosine": 0.6450917720794678, "beta_mean": 0.20280897617340088, "nonfinite": 0.0, "weighted_memory_to_active": 0.04952932521700859, "mix_std": 0.3086579442024231, "beta_std": 0.2345794141292572, "beta_low": 0.17669677734375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0006389894988387823, "evidence_weights": [-0.2350236177444458, -0.25224316120147705, 0.10908962041139603], "agreement_weight": 0.011225110851228237, "orthogonal_weight": 0.019244153052568436}, {"head": 5, "memory_to_active": 0.005076998844742775, "contribution_to_active": 0.46865931153297424, "mix_mean": 0.5204890966415405, "mix_low": 0.0057888031005859375, "mix_high": 0.0069980621337890625, "read_cosine": 0.5287726521492004, "beta_mean": 0.00014610204380005598, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021556096617132425, "mix_std": 0.282489538192749, "beta_std": 0.0002630297967698425, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017985706217586994, "evidence_weights": [-0.3571011424064636, -0.30140191316604614, 0.3525125980377197], "agreement_weight": 0.17041006684303284, "orthogonal_weight": 0.9826611280441284}]}]}, {"step": 22000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.807179634000022, "memory_to_active": 0.9282733201980591, "contribution_to_active": 0.4819059371948242, "mix_mean": 0.33645379543304443, "mix_low": 0.16198158264160156, "mix_high": 0.006629398092627525, "read_cosine": 0.3907492756843567, "beta_mean": 0.08395792543888092, "nonfinite": 0.0, "weighted_memory_to_active": 0.209171861410141, "mix_std": 0.21341267228126526, "beta_std": 0.07026444375514984, "beta_low": 0.47779592871665955, "beta_high": 1.0808309525600635e-05, "orthogonal_to_active": 0.047248631715774536, "samples": 4, "scored_tokens": 899, "normal_loss": 3.14237380027771, "normal_token_accuracy": 0.43159064650535583, "state_active": 0.058420054614543915, "state_memory": 0.02089233696460724, "state_peak": 112.31742095947266, "growth_active": 1.1383860111236572, "growth_memory": 1.385887861251831, "output_influence": 0.3398914933204651, "residual_influence": 0.10743500292301178, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.213186502456665, "no_memory_loss_delta": 0.2406230866909027, "scrambled_loss_delta": 0.2106676995754242, "memory_only_loss_delta": 2.3932602405548096, "no_evidence_loss_delta": 0.15040871500968933, "no_agreement_loss_delta": 0.012567844241857529, "no_orthogonal_loss_delta": -0.00770237110555172}, {"index": 94, "tokens": 256, "normal_loss": 3.246471405029297, "no_memory_loss_delta": 0.33926132321357727, "scrambled_loss_delta": 0.37848183512687683, "memory_only_loss_delta": 2.46600079536438, "no_evidence_loss_delta": 0.1415204107761383, "no_agreement_loss_delta": 0.0241102185100317, "no_orthogonal_loss_delta": 0.017843952402472496}, {"index": 188, "tokens": 256, "normal_loss": 3.573662757873535, "no_memory_loss_delta": 0.2663642168045044, "scrambled_loss_delta": 0.17455627024173737, "memory_only_loss_delta": 2.2075042724609375, "no_evidence_loss_delta": 0.1532868891954422, "no_agreement_loss_delta": 0.007732358295470476, "no_orthogonal_loss_delta": 0.011470954865217209}, {"index": 283, "tokens": 131, "normal_loss": 1.9577394723892212, "no_memory_loss_delta": 0.17199383676052094, "scrambled_loss_delta": 0.31500303745269775, "memory_only_loss_delta": 2.630640983581543, "no_evidence_loss_delta": 0.02194596268236637, "no_agreement_loss_delta": -0.011824680492281914, "no_orthogonal_loss_delta": -0.008023063652217388}], "agreement_weight_abs_mean": 0.11318445205688477, "agreement_weight_abs_max": 2.312091827392578, "agreement_weight_mean": 0.09638303518295288, "orthogonal_weight_abs_mean": 0.5753890872001648, "orthogonal_weight_abs_max": 3.9174885749816895, "orthogonal_weight_mean": -0.1691107302904129, "no_memory_loss": 3.408414602279663, "no_memory_loss_delta": 0.26604098081588745, "no_memory_token_accuracy_gain": 0.023359287530183792, "no_memory_token_win_fraction": 0.6162402629852295, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 23.359228115617334, "scrambled_loss": 3.4057486057281494, "scrambled_loss_delta": 0.2633749544620514, "scrambled_token_accuracy_gain": 0.021134594455361366, "scrambled_token_win_fraction": 0.6206896305084229, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 23.154629186111794, "memory_only_loss": 5.5380425453186035, "memory_only_loss_delta": 2.3956682682037354, "memory_only_token_accuracy_gain": 0.23804226517677307, "memory_only_token_win_fraction": 0.8731924295425415, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 90.88882285262531, "no_evidence_loss": 3.2723517417907715, "no_evidence_loss_delta": 0.1299780011177063, "no_evidence_token_accuracy_gain": 0.01001112349331379, "no_evidence_token_win_fraction": 0.5695216655731201, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 12.188525174893215, "no_agreement_loss": 3.153297185897827, "no_agreement_loss_delta": 0.010923285968601704, "no_agreement_token_accuracy_gain": -0.007786429487168789, "no_agreement_token_win_fraction": 0.44716352224349976, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.0863843512945153, "no_orthogonal_loss": 3.1473591327667236, "no_orthogonal_loss_delta": 0.0049853031523525715, "no_orthogonal_token_accuracy_gain": 0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5494994521141052, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 0.4972897153043156, "short_no_memory_loss_delta": 0.2188660353422165, "long_context_memory_gain": 0.04717494547367096, "normal_context_loss_gain": 0.14985263347625732, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.134989023208618, "contribution_to_active": 0.9093451499938965, "mix_mean": 0.09681106358766556, "mix_low": 0.5074557065963745, "mix_high": 0.0036288900300860405, "read_cosine": 0.4676762819290161, "beta_mean": 0.250497430562973, "nonfinite": 0.0, "weighted_memory_to_active": 0.975094199180603, "mix_std": 0.08545582741498947, "beta_std": 0.1582866907119751, "beta_low": 0.0019124349346384406, "beta_high": 0.0, "orthogonal_to_active": 0.062455855309963226, "output_influence": 0.14028355479240417, "residual_influence": 0.17693057656288147, "state_active": 0.07854858040809631, "state_memory": 0.04983590915799141, "state_peak": 46.85553741455078, "growth_active": 1.2789775133132935, "growth_memory": 1.284971833229065, "heads": [{"head": 0, "memory_to_active": 2.4768762588500977, "contribution_to_active": 0.38897016644477844, "mix_mean": 0.018933705985546112, "mix_low": 0.5755443572998047, "mix_high": 0.0, "read_cosine": 0.2731110453605652, "beta_mean": 0.2049776017665863, "nonfinite": 0.0, "weighted_memory_to_active": 0.46959829330444336, "mix_std": 0.05147888883948326, "beta_std": 0.10887403786182404, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.037334997206926346, "evidence_weights": [-0.033594001084566116, -0.0758410394191742, -0.07917497307062149], "agreement_weight": -0.10268751531839371, "orthogonal_weight": -0.09109935164451599}, {"head": 1, "memory_to_active": 4.414063453674316, "contribution_to_active": 1.5986311435699463, "mix_mean": 0.016685519367456436, "mix_low": 0.6169347763061523, "mix_high": 0.0, "read_cosine": 0.5067434310913086, "beta_mean": 0.3494662046432495, "nonfinite": 0.0, "weighted_memory_to_active": 1.987135887145996, "mix_std": 0.04861455038189888, "beta_std": 0.1914692223072052, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16542190313339233, "evidence_weights": [0.010851594619452953, -0.06915881484746933, 0.21977746486663818], "agreement_weight": 0.020713061094284058, "orthogonal_weight": -0.19307340681552887}, {"head": 2, "memory_to_active": 2.509439468383789, "contribution_to_active": 0.47173866629600525, "mix_mean": 0.014184538275003433, "mix_low": 0.6477956771850586, "mix_high": 0.0, "read_cosine": 0.3811397850513458, "beta_mean": 0.3324210047721863, "nonfinite": 0.0, "weighted_memory_to_active": 0.5530850291252136, "mix_std": 0.031063757836818695, "beta_std": 0.24064801633358002, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.058444175869226456, "evidence_weights": [0.043500933796167374, 0.0547708123922348, -0.042090535163879395], "agreement_weight": -0.00031224690610542893, "orthogonal_weight": -0.18816451728343964}, {"head": 3, "memory_to_active": 5.951487064361572, "contribution_to_active": 2.574230909347534, "mix_mean": 0.04481447488069534, "mix_low": 0.3706169128417969, "mix_high": 0.0, "read_cosine": 0.38139793276786804, "beta_mean": 0.33020105957984924, "nonfinite": 0.0, "weighted_memory_to_active": 2.720460891723633, "mix_std": 0.08794036507606506, "beta_std": 0.22547827661037445, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.09192212671041489, "evidence_weights": [-0.07948949933052063, -0.10435877740383148, -0.08470918238162994], "agreement_weight": -0.014425774104893208, "orthogonal_weight": -0.041491296142339706}, {"head": 4, "memory_to_active": 3.3292319774627686, "contribution_to_active": 0.071562759578228, "mix_mean": 0.006939461454749107, "mix_low": 0.810338020324707, "mix_high": 0.0, "read_cosine": 0.3239608108997345, "beta_mean": 0.2349238395690918, "nonfinite": 0.0, "weighted_memory_to_active": 0.06885450333356857, "mix_std": 0.01289116870611906, "beta_std": 0.13738703727722168, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008212917484343052, "evidence_weights": [0.08851629495620728, 0.03677649796009064, -0.32364049553871155], "agreement_weight": -0.04224252328276634, "orthogonal_weight": 0.13483880460262299}, {"head": 5, "memory_to_active": 0.1288354992866516, "contribution_to_active": 0.3509371280670166, "mix_mean": 0.47930872440338135, "mix_low": 0.023504257202148438, "mix_high": 0.021773338317871094, "read_cosine": 0.9397045373916626, "beta_mean": 0.050994835793972015, "nonfinite": 0.0, "weighted_memory_to_active": 0.05143073946237564, "mix_std": 0.280746191740036, "beta_std": 0.045863568782806396, "beta_low": 0.011474609375, "beta_high": 0.0, "orthogonal_to_active": 0.013398977927863598, "evidence_weights": [-0.2795848250389099, -0.2562420964241028, 0.22971510887145996], "agreement_weight": 0.21321865916252136, "orthogonal_weight": -0.5069288611412048}]}, {"layer": 1, "memory_to_active": 2.317389965057373, "contribution_to_active": 0.4456981122493744, "mix_mean": 0.14796043932437897, "mix_low": 0.383999228477478, "mix_high": 0.00047842663479968905, "read_cosine": 0.3383432626724243, "beta_mean": 0.2049732506275177, "nonfinite": 0.0, "weighted_memory_to_active": 0.34979191422462463, "mix_std": 0.11849445104598999, "beta_std": 0.13671229779720306, "beta_low": 0.17431640625, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.045189112424850464, "output_influence": 0.16188204288482666, "residual_influence": 0.03909726440906525, "state_active": 0.039433903992176056, "state_memory": 0.03911741077899933, "state_peak": 27.20248794555664, "growth_active": 1.122544288635254, "growth_memory": 1.4966354370117188, "heads": [{"head": 0, "memory_to_active": 0.07598118484020233, "contribution_to_active": 0.3998591899871826, "mix_mean": 0.3567153215408325, "mix_low": 0.01636219024658203, "mix_high": 0.00016689300537109375, "read_cosine": 0.040132708847522736, "beta_mean": 0.0005263875937089324, "nonfinite": 0.0, "weighted_memory_to_active": 0.03666709363460541, "mix_std": 0.27503466606140137, "beta_std": 0.0007767383358441293, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.028455978259444237, "evidence_weights": [-0.1641087681055069, -0.20534923672676086, -0.020285475999116898], "agreement_weight": 0.15319865942001343, "orthogonal_weight": -0.8721089363098145}, {"head": 1, "memory_to_active": 3.375321865081787, "contribution_to_active": 0.20386500656604767, "mix_mean": 0.020693719387054443, "mix_low": 0.49127674102783203, "mix_high": 9.5367431640625e-07, "read_cosine": 0.4390530288219452, "beta_mean": 0.33958250284194946, "nonfinite": 0.0, "weighted_memory_to_active": 0.23090963065624237, "mix_std": 0.04322339966893196, "beta_std": 0.21285982429981232, "beta_low": 0.00018310546875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.007254777476191521, "evidence_weights": [0.004230077378451824, -0.006466996856033802, 0.0004882967914454639], "agreement_weight": -0.003509967355057597, "orthogonal_weight": 0.07083742320537567}, {"head": 2, "memory_to_active": 0.47454530000686646, "contribution_to_active": 0.4630908668041229, "mix_mean": 0.450581431388855, "mix_low": 0.0036497116088867188, "mix_high": 0.0027027130126953125, "read_cosine": 0.7729653716087341, "beta_mean": 0.05188639461994171, "nonfinite": 0.0, "weighted_memory_to_active": 0.37217170000076294, "mix_std": 0.26650920510292053, "beta_std": 0.03250784054398537, "beta_low": 0.01702880859375, "beta_high": 0.0, "orthogonal_to_active": 0.030364269390702248, "evidence_weights": [-0.2549055516719818, -0.3877655565738678, 0.2395390272140503], "agreement_weight": 0.020276017487049103, "orthogonal_weight": -0.1590457558631897}, {"head": 3, "memory_to_active": 2.811617374420166, "contribution_to_active": 0.08066277205944061, "mix_mean": 0.015348418615758419, "mix_low": 0.6950654983520508, "mix_high": 0.0, "read_cosine": 0.20973530411720276, "beta_mean": 0.33571773767471313, "nonfinite": 0.0, "weighted_memory_to_active": 0.10733290016651154, "mix_std": 0.042048320174217224, "beta_std": 0.24186986684799194, "beta_low": 0.01177978515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.030548065900802612, "evidence_weights": [0.036943841725587845, 0.005849510431289673, 0.1024211123585701], "agreement_weight": -0.027554567903280258, "orthogonal_weight": -0.30068376660346985}, {"head": 4, "memory_to_active": 3.274829864501953, "contribution_to_active": 1.4651626348495483, "mix_mean": 0.03604934737086296, "mix_low": 0.3050241470336914, "mix_high": 0.0, "read_cosine": 0.3697481155395508, "beta_mean": 0.19722938537597656, "nonfinite": 0.0, "weighted_memory_to_active": 1.2971868515014648, "mix_std": 0.065151646733284, "beta_std": 0.10711896419525146, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.17151306569576263, "evidence_weights": [-0.03031395561993122, -0.001302648219279945, 0.04986129701137543], "agreement_weight": -0.026690658181905746, "orthogonal_weight": 0.15988709032535553}, {"head": 5, "memory_to_active": 3.8920445442199707, "contribution_to_active": 0.06154811382293701, "mix_mean": 0.008374402299523354, "mix_low": 0.7926168441772461, "mix_high": 0.0, "read_cosine": 0.19842487573623657, "beta_mean": 0.3048969507217407, "nonfinite": 0.0, "weighted_memory_to_active": 0.054483190178871155, "mix_std": 0.018999477848410606, "beta_std": 0.22514048218727112, "beta_low": 0.01708984375, "beta_high": 0.0, "orthogonal_to_active": 0.002998480573296547, "evidence_weights": [0.1359359472990036, 0.056669123470783234, 0.3105257749557495], "agreement_weight": -0.15602214634418488, "orthogonal_weight": 0.053940270096063614}]}, {"layer": 2, "memory_to_active": 1.114418625831604, "contribution_to_active": 0.3231689929962158, "mix_mean": 0.30760133266448975, "mix_low": 0.26097142696380615, "mix_high": 0.0029433569870889187, "read_cosine": 0.3373319208621979, "beta_mean": 0.09416230767965317, "nonfinite": 0.0, "weighted_memory_to_active": 0.07473454624414444, "mix_std": 0.195939302444458, "beta_std": 0.07104668021202087, "beta_low": 0.5460307002067566, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.016583014279603958, "output_influence": 0.2994738221168518, "residual_influence": 0.07193972170352936, "state_active": 0.03614961728453636, "state_memory": 0.02136286348104477, "state_peak": 45.72138595581055, "growth_active": 1.0454280376434326, "growth_memory": 1.5675166845321655, "heads": [{"head": 0, "memory_to_active": 0.013839691877365112, "contribution_to_active": 0.25181129574775696, "mix_mean": 0.43053892254829407, "mix_low": 0.00537872314453125, "mix_high": 0.0021753311157226562, "read_cosine": 0.2301572859287262, "beta_mean": 0.00011105705925729126, "nonfinite": 0.0, "weighted_memory_to_active": 0.00669090123847127, "mix_std": 0.29718929529190063, "beta_std": 0.00017357239266857505, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0004853932186961174, "evidence_weights": [-0.2501068115234375, -0.16420388221740723, 0.29040998220443726], "agreement_weight": 0.20515809953212738, "orthogonal_weight": -0.07874447852373123}, {"head": 1, "memory_to_active": 3.101311445236206, "contribution_to_active": 0.048711150884628296, "mix_mean": 0.0059066955000162125, "mix_low": 0.8679780960083008, "mix_high": 2.86102294921875e-06, "read_cosine": 0.20378711819648743, "beta_mean": 0.2806858420372009, "nonfinite": 0.0, "weighted_memory_to_active": 0.03847425431013107, "mix_std": 0.014853863045573235, "beta_std": 0.14824464917182922, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013172757811844349, "evidence_weights": [0.09276358783245087, 0.10251926630735397, 0.3524012863636017], "agreement_weight": 0.041854605078697205, "orthogonal_weight": 0.394106924533844}, {"head": 2, "memory_to_active": 0.026818880811333656, "contribution_to_active": 0.44420182704925537, "mix_mean": 0.42937687039375305, "mix_low": 0.0029430389404296875, "mix_high": 0.00232696533203125, "read_cosine": 0.24526214599609375, "beta_mean": 4.875303420703858e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011116277426481247, "mix_std": 0.27827632427215576, "beta_std": 9.700877853902057e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025079231709241867, "evidence_weights": [-0.20809133350849152, -0.2481442540884018, 0.5205718874931335], "agreement_weight": 0.0379520058631897, "orthogonal_weight": -0.2736055254936218}, {"head": 3, "memory_to_active": 0.23871153593063354, "contribution_to_active": 0.4493699073791504, "mix_mean": 0.4031287729740143, "mix_low": 0.0021419525146484375, "mix_high": 0.006363868713378906, "read_cosine": 0.6388886570930481, "beta_mean": 0.040910568088293076, "nonfinite": 0.0, "weighted_memory_to_active": 0.11601259559392929, "mix_std": 0.26503968238830566, "beta_std": 0.05311359465122223, "beta_low": 0.23779296875, "beta_high": 0.0, "orthogonal_to_active": 0.06748246401548386, "evidence_weights": [-0.15025179088115692, -0.23174646496772766, 0.8153778910636902], "agreement_weight": 0.06339570879936218, "orthogonal_weight": -0.8431777954101562}, {"head": 4, "memory_to_active": 3.279005289077759, "contribution_to_active": 0.23191355168819427, "mix_mean": 0.014592168852686882, "mix_low": 0.6856527328491211, "mix_high": 0.0, "read_cosine": 0.19044314324855804, "beta_mean": 0.24314628541469574, "nonfinite": 0.0, "weighted_memory_to_active": 0.2598941922187805, "mix_std": 0.04066178575158119, "beta_std": 0.2245645374059677, "beta_low": 0.0384521484375, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.014807094819843769, "evidence_weights": [0.038929279893636703, 0.03858911991119385, 0.14142723381519318], "agreement_weight": 0.033057183027267456, "orthogonal_weight": -0.06134498491883278}, {"head": 5, "memory_to_active": 0.02682461217045784, "contribution_to_active": 0.5130061507225037, "mix_mean": 0.5620646476745605, "mix_low": 0.0017337799072265625, "mix_high": 0.006791114807128906, "read_cosine": 0.5154531002044678, "beta_mean": 7.132173050194979e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.016219090670347214, "mix_std": 0.2796148955821991, "beta_std": 8.668615919305012e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010424555512145162, "evidence_weights": [-0.2463349997997284, -0.2692486345767975, 0.7245983481407166], "agreement_weight": 0.04999827593564987, "orthogonal_weight": 0.11642388999462128}]}, {"layer": 3, "memory_to_active": 1.1140732765197754, "contribution_to_active": 0.352184534072876, "mix_mean": 0.2597411870956421, "mix_low": 0.21630734205245972, "mix_high": 0.003658135887235403, "read_cosine": 0.24979254603385925, "beta_mean": 0.07919857650995255, "nonfinite": 0.0, "weighted_memory_to_active": 0.16609486937522888, "mix_std": 0.21966969966888428, "beta_std": 0.06416566669940948, "beta_low": 0.4301351308822632, "beta_high": 0.0, "orthogonal_to_active": 0.055007513612508774, "output_influence": 0.2650889754295349, "residual_influence": 0.05043046548962593, "state_active": 0.024570899084210396, "state_memory": 0.016723880544304848, "state_peak": 20.098234176635742, "growth_active": 1.068130612373352, "growth_memory": 1.8180491924285889, "heads": [{"head": 0, "memory_to_active": 0.6704888939857483, "contribution_to_active": 0.4367654323577881, "mix_mean": 0.33354300260543823, "mix_low": 0.052308082580566406, "mix_high": 0.004452705383300781, "read_cosine": 0.2239779382944107, "beta_mean": 0.014636639505624771, "nonfinite": 0.0, "weighted_memory_to_active": 0.27151843905448914, "mix_std": 0.31323757767677307, "beta_std": 0.018559463322162628, "beta_low": 0.50189208984375, "beta_high": 0.0, "orthogonal_to_active": 0.15289092063903809, "evidence_weights": [-0.12429806590080261, -0.1809026300907135, 0.30938172340393066], "agreement_weight": 0.12555116415023804, "orthogonal_weight": -0.596786618232727}, {"head": 1, "memory_to_active": 0.017606882378458977, "contribution_to_active": 0.31187352538108826, "mix_mean": 0.3477752208709717, "mix_low": 0.029137611389160156, "mix_high": 0.0017194747924804688, "read_cosine": 0.20899322628974915, "beta_mean": 8.214530680561438e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00631538825109601, "mix_std": 0.29419809579849243, "beta_std": 0.0001507334382040426, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016325765755027533, "evidence_weights": [-0.18325316905975342, -0.15389816462993622, 0.4062742590904236], "agreement_weight": 0.23662377893924713, "orthogonal_weight": -0.27909135818481445}, {"head": 2, "memory_to_active": 2.8838629722595215, "contribution_to_active": 0.2958124279975891, "mix_mean": 0.04071146249771118, "mix_low": 0.41869068145751953, "mix_high": 0.0, "read_cosine": 0.15820464491844177, "beta_mean": 0.20764507353305817, "nonfinite": 0.0, "weighted_memory_to_active": 0.3802081048488617, "mix_std": 0.0929388701915741, "beta_std": 0.1479802429676056, "beta_low": 0.005615234375, "beta_high": 0.0, "orthogonal_to_active": 0.017601201310753822, "evidence_weights": [-0.05620429292321205, 0.015401993878185749, -0.0035002294462174177], "agreement_weight": -0.014237778261303902, "orthogonal_weight": -0.06036437675356865}, {"head": 3, "memory_to_active": 0.028172878548502922, "contribution_to_active": 0.4860006272792816, "mix_mean": 0.4934293329715729, "mix_low": 0.018854141235351562, "mix_high": 0.010834693908691406, "read_cosine": 0.2671739459037781, "beta_mean": 0.00011593414092203602, "nonfinite": 0.0, "weighted_memory_to_active": 0.01730121113359928, "mix_std": 0.31383344531059265, "beta_std": 0.0002000364038394764, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0056518795900046825, "evidence_weights": [-0.19422537088394165, -0.21454879641532898, 1.6068488359451294], "agreement_weight": 0.9432774186134338, "orthogonal_weight": -0.4094540476799011}, {"head": 4, "memory_to_active": 2.662978410720825, "contribution_to_active": 0.03207508102059364, "mix_mean": 0.010980384424328804, "mix_low": 0.7471513748168945, "mix_high": 0.0, "read_cosine": 0.211286723613739, "beta_mean": 0.16559773683547974, "nonfinite": 0.0, "weighted_memory_to_active": 0.05962710082530975, "mix_std": 0.02472582831978798, "beta_std": 0.13228575885295868, "beta_low": 0.00738525390625, "beta_high": 0.0, "orthogonal_to_active": 0.030807137489318848, "evidence_weights": [0.1394275426864624, 0.10515262931585312, -0.04418522119522095], "agreement_weight": -0.04351963847875595, "orthogonal_weight": -0.5379347205162048}, {"head": 5, "memory_to_active": 0.42132923007011414, "contribution_to_active": 0.5505801439285278, "mix_mean": 0.3320077061653137, "mix_low": 0.03170204162597656, "mix_high": 0.0049419403076171875, "read_cosine": 0.4291187524795532, "beta_mean": 0.08711392432451248, "nonfinite": 0.0, "weighted_memory_to_active": 0.2615989148616791, "mix_std": 0.27908435463905334, "beta_std": 0.08581776916980743, "beta_low": 0.06591796875, "beta_high": 0.0, "orthogonal_to_active": 0.12146132439374924, "evidence_weights": [-0.14066354930400848, -0.19963644444942474, 0.349994033575058], "agreement_weight": 0.16865818202495575, "orthogonal_weight": -0.6120600700378418}]}, {"layer": 4, "memory_to_active": 1.2374991178512573, "contribution_to_active": 0.4118322730064392, "mix_mean": 0.31534427404403687, "mix_low": 0.27101120352745056, "mix_high": 0.004089514724910259, "read_cosine": 0.40039002895355225, "beta_mean": 0.12623071670532227, "nonfinite": 0.0, "weighted_memory_to_active": 0.163526251912117, "mix_std": 0.15579518675804138, "beta_std": 0.10499699413776398, "beta_low": 0.3557230830192566, "beta_high": 0.0, "orthogonal_to_active": 0.04782186821103096, "output_influence": 0.31178048253059387, "residual_influence": 0.07659270614385605, "state_active": 0.03849662095308304, "state_memory": 0.024507636204361916, "state_peak": 47.496803283691406, "growth_active": 1.1595616340637207, "growth_memory": 1.574053168296814, "heads": [{"head": 0, "memory_to_active": 0.29480046033859253, "contribution_to_active": 0.6554322242736816, "mix_mean": 0.6004346609115601, "mix_low": 0.005969047546386719, "mix_high": 0.008539199829101562, "read_cosine": 0.5431606769561768, "beta_mean": 0.23277457058429718, "nonfinite": 0.0, "weighted_memory_to_active": 0.22381675243377686, "mix_std": 0.32184359431266785, "beta_std": 0.24367353320121765, "beta_low": 0.18707275390625, "beta_high": 0.0, "orthogonal_to_active": 0.05118414759635925, "evidence_weights": [-0.25024038553237915, -0.33228933811187744, 0.4196883738040924], "agreement_weight": 0.03847246244549751, "orthogonal_weight": 0.2993725538253784}, {"head": 1, "memory_to_active": 0.15489208698272705, "contribution_to_active": 0.4925847351551056, "mix_mean": 0.5355037450790405, "mix_low": 0.00022125244140625, "mix_high": 0.003177642822265625, "read_cosine": 0.7308546304702759, "beta_mean": 0.004347710870206356, "nonfinite": 0.0, "weighted_memory_to_active": 0.08840767294168472, "mix_std": 0.2632475197315216, "beta_std": 0.008142621256411076, "beta_low": 0.90557861328125, "beta_high": 0.0, "orthogonal_to_active": 0.04253440350294113, "evidence_weights": [-0.2322060763835907, -0.25498953461647034, 0.6850127577781677], "agreement_weight": 0.05829453840851784, "orthogonal_weight": -0.7313818335533142}, {"head": 2, "memory_to_active": 2.5305657386779785, "contribution_to_active": 0.12786996364593506, "mix_mean": 0.015375133603811264, "mix_low": 0.6048507690429688, "mix_high": 0.0, "read_cosine": 0.3046755790710449, "beta_mean": 0.14050641655921936, "nonfinite": 0.0, "weighted_memory_to_active": 0.16582560539245605, "mix_std": 0.027764637023210526, "beta_std": 0.08991299569606781, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.008848031982779503, "evidence_weights": [0.051015984266996384, 0.029788725078105927, -0.04604770615696907], "agreement_weight": 0.08482581377029419, "orthogonal_weight": -0.08141425997018814}, {"head": 3, "memory_to_active": 1.6468044519424438, "contribution_to_active": 0.12425382435321808, "mix_mean": 0.029307808727025986, "mix_low": 0.4086036682128906, "mix_high": 0.0, "read_cosine": 0.20058758556842804, "beta_mean": 0.13633468747138977, "nonfinite": 0.0, "weighted_memory_to_active": 0.1680033653974533, "mix_std": 0.05314667895436287, "beta_std": 0.12421239167451859, "beta_low": 0.04095458984375, "beta_high": 0.0, "orthogonal_to_active": 0.019622765481472015, "evidence_weights": [-0.01684674248099327, 0.05682874098420143, -0.26505139470100403], "agreement_weight": 0.044496145099401474, "orthogonal_weight": -0.147843137383461}, {"head": 4, "memory_to_active": 0.013466950505971909, "contribution_to_active": 0.6308588981628418, "mix_mean": 0.6962806582450867, "mix_low": 0.00022983551025390625, "mix_high": 0.012807846069335938, "read_cosine": 0.25778889656066895, "beta_mean": 2.8144902898930013e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009880363941192627, "mix_std": 0.23328429460525513, "beta_std": 7.905576785560697e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012646627146750689, "evidence_weights": [-0.299476683139801, -0.3376297056674957, 0.7778903245925903], "agreement_weight": 0.11615709960460663, "orthogonal_weight": -0.1474282592535019}, {"head": 5, "memory_to_active": 2.7844643592834473, "contribution_to_active": 0.43999385833740234, "mix_mean": 0.01516359206289053, "mix_low": 0.6061925888061523, "mix_high": 1.239776611328125e-05, "read_cosine": 0.36527279019355774, "beta_mean": 0.24339276552200317, "nonfinite": 0.0, "weighted_memory_to_active": 0.32522374391555786, "mix_std": 0.035484425723552704, "beta_std": 0.16396135091781616, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16347716748714447, "evidence_weights": [0.038092803210020065, -0.03791812062263489, -0.1690319925546646], "agreement_weight": -0.0421910360455513, "orthogonal_weight": 0.7518727779388428}]}, {"layer": 5, "memory_to_active": 0.6758881211280823, "contribution_to_active": 0.47024327516555786, "mix_mean": 0.41990429162979126, "mix_low": 0.0888061597943306, "mix_high": 0.012894312851130962, "read_cosine": 0.20176948606967926, "beta_mean": 0.0545988604426384, "nonfinite": 0.0, "weighted_memory_to_active": 0.17001673579216003, "mix_std": 0.2504194974899292, "beta_std": 0.0705728605389595, "beta_low": 0.616851806640625, "beta_high": 0.0, "orthogonal_to_active": 0.05543668940663338, "output_influence": 0.4556381106376648, "residual_influence": 0.10887990146875381, "state_active": 0.05059070140123367, "state_memory": 0.014371372759342194, "state_peak": 63.9531364440918, "growth_active": 1.1293070316314697, "growth_memory": 1.273437261581421, "heads": [{"head": 0, "memory_to_active": 1.166501760482788, "contribution_to_active": 0.3204982280731201, "mix_mean": 0.15895932912826538, "mix_low": 0.12067413330078125, "mix_high": 2.288818359375e-05, "read_cosine": -0.23813524842262268, "beta_mean": 0.052653536200523376, "nonfinite": 0.0, "weighted_memory_to_active": 0.1855987310409546, "mix_std": 0.20376619696617126, "beta_std": 0.07889746874570847, "beta_low": 0.23419189453125, "beta_high": 0.0, "orthogonal_to_active": 0.04091030731797218, "evidence_weights": [-0.055395979434251785, -0.09322236478328705, -0.3555722236633301], "agreement_weight": -0.013965952210128307, "orthogonal_weight": -0.2081208974123001}, {"head": 1, "memory_to_active": 0.04246528446674347, "contribution_to_active": 0.5128740072250366, "mix_mean": 0.5841377377510071, "mix_low": 0.005545616149902344, "mix_high": 0.0177764892578125, "read_cosine": 0.18309707939624786, "beta_mean": 0.00010891709825955331, "nonfinite": 0.0, "weighted_memory_to_active": 0.020255718380212784, "mix_std": 0.3106452524662018, "beta_std": 0.0002404679253231734, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.018433187156915665, "evidence_weights": [-0.22905342280864716, -0.2513108551502228, 0.776679277420044], "agreement_weight": 0.3372569680213928, "orthogonal_weight": -0.9375104904174805}, {"head": 2, "memory_to_active": 0.0506526455283165, "contribution_to_active": 0.5203108191490173, "mix_mean": 0.6053825616836548, "mix_low": 0.001682281494140625, "mix_high": 0.016867637634277344, "read_cosine": 0.35975539684295654, "beta_mean": 0.0010957461781799793, "nonfinite": 0.0, "weighted_memory_to_active": 0.03357505425810814, "mix_std": 0.27453961968421936, "beta_std": 0.0014698090963065624, "beta_low": 0.99639892578125, "beta_high": 0.0, "orthogonal_to_active": 0.005229626782238483, "evidence_weights": [-0.2596944272518158, -0.33200663328170776, 0.8107917904853821], "agreement_weight": 0.026192927733063698, "orthogonal_weight": -0.2124730348587036}, {"head": 3, "memory_to_active": 2.2536234855651855, "contribution_to_active": 0.2435193955898285, "mix_mean": 0.07869094610214233, "mix_low": 0.21086978912353516, "mix_high": 0.0, "read_cosine": 0.12390930950641632, "beta_mean": 0.08010402321815491, "nonfinite": 0.0, "weighted_memory_to_active": 0.4094586968421936, "mix_std": 0.12194863706827164, "beta_std": 0.11849333345890045, "beta_low": 0.2156982421875, "beta_high": 0.0, "orthogonal_to_active": 0.14648792147636414, "evidence_weights": [-0.03965155407786369, -0.08352550864219666, -0.061292581260204315], "agreement_weight": 0.06419778615236282, "orthogonal_weight": -0.45540300011634827}, {"head": 4, "memory_to_active": 0.4503263831138611, "contribution_to_active": 0.5967139005661011, "mix_mean": 0.33739420771598816, "mix_low": 0.19121932983398438, "mix_high": 0.00994110107421875, "read_cosine": 0.5750394463539124, "beta_mean": 0.19331277906894684, "nonfinite": 0.0, "weighted_memory_to_active": 0.3091951608657837, "mix_std": 0.34373870491981506, "beta_std": 0.22282147407531738, "beta_low": 0.2587890625, "beta_high": 0.0, "orthogonal_to_active": 0.08312944322824478, "evidence_weights": [-0.12380247563123703, -0.2920275330543518, -0.052886251360177994], "agreement_weight": 0.04015976935625076, "orthogonal_weight": 0.426037460565567}, {"head": 5, "memory_to_active": 0.09175892919301987, "contribution_to_active": 0.6275432109832764, "mix_mean": 0.7548608779907227, "mix_low": 0.00284576416015625, "mix_high": 0.03275775909423828, "read_cosine": 0.20695079863071442, "beta_mean": 0.00031813594978302717, "nonfinite": 0.0, "weighted_memory_to_active": 0.0620170421898365, "mix_std": 0.24787840247154236, "beta_std": 0.0015145792858675122, "beta_low": 0.9962158203125, "beta_high": 0.0, "orthogonal_to_active": 0.038429658859968185, "evidence_weights": [-0.3229110836982727, -0.3145911693572998, 1.8224866390228271], "agreement_weight": 0.3274194002151489, "orthogonal_weight": -0.6601574420928955}]}, {"layer": 6, "memory_to_active": 0.38096293807029724, "contribution_to_active": 0.5717200636863708, "mix_mean": 0.5304380655288696, "mix_low": 0.07542943954467773, "mix_high": 0.025193532928824425, "read_cosine": 0.2907888889312744, "beta_mean": 0.07373744249343872, "nonfinite": 0.0, "weighted_memory_to_active": 0.16695809364318848, "mix_std": 0.24671965837478638, "beta_std": 0.084633469581604, "beta_low": 0.5015665888786316, "beta_high": 0.0, "orthogonal_to_active": 0.03771018236875534, "output_influence": 0.460480272769928, "residual_influence": 0.14382100105285645, "state_active": 0.047395579516887665, "state_memory": 0.00964218657463789, "state_peak": 69.83965301513672, "growth_active": 1.139108657836914, "growth_memory": 1.167956829071045, "heads": [{"head": 0, "memory_to_active": 0.1724218875169754, "contribution_to_active": 0.574853777885437, "mix_mean": 0.6956660747528076, "mix_low": 0.00014209747314453125, "mix_high": 0.004504203796386719, "read_cosine": 0.08761978894472122, "beta_mean": 0.013815779238939285, "nonfinite": 0.0, "weighted_memory_to_active": 0.12725970149040222, "mix_std": 0.23633737862110138, "beta_std": 0.01969880238175392, "beta_low": 0.5487060546875, "beta_high": 0.0, "orthogonal_to_active": 0.09651502221822739, "evidence_weights": [-0.30955570936203003, -0.3192257285118103, 0.5026297569274902], "agreement_weight": 0.10657963901758194, "orthogonal_weight": -0.8908678889274597}, {"head": 1, "memory_to_active": 0.009681325405836105, "contribution_to_active": 0.4571805000305176, "mix_mean": 0.47836682200431824, "mix_low": 0.001255035400390625, "mix_high": 0.008859634399414062, "read_cosine": 0.6037286520004272, "beta_mean": 2.1995459974277765e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034713027998805046, "mix_std": 0.29185253381729126, "beta_std": 8.06633906904608e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010185092687606812, "evidence_weights": [-0.20793555676937103, -0.2659766376018524, 0.8024863004684448], "agreement_weight": -0.018896052613854408, "orthogonal_weight": -0.39129912853240967}, {"head": 2, "memory_to_active": 1.1936914920806885, "contribution_to_active": 0.06596244126558304, "mix_mean": 0.02791750431060791, "mix_low": 0.44013404846191406, "mix_high": 0.0, "read_cosine": 0.20420342683792114, "beta_mean": 0.06736446917057037, "nonfinite": 0.0, "weighted_memory_to_active": 0.06357842683792114, "mix_std": 0.051258545368909836, "beta_std": 0.06471829861402512, "beta_low": 0.01983642578125, "beta_high": 0.0, "orthogonal_to_active": 0.0063547007739543915, "evidence_weights": [0.08601634949445724, 0.011041373014450073, 0.22504879534244537], "agreement_weight": 0.10611661523580551, "orthogonal_weight": -0.11048918962478638}, {"head": 3, "memory_to_active": 0.15016323328018188, "contribution_to_active": 0.6626306176185608, "mix_mean": 0.641850471496582, "mix_low": 0.008893013000488281, "mix_high": 0.022365570068359375, "read_cosine": 0.4687888026237488, "beta_mean": 0.22290527820587158, "nonfinite": 0.0, "weighted_memory_to_active": 0.11050163209438324, "mix_std": 0.33303651213645935, "beta_std": 0.26047366857528687, "beta_low": 0.22027587890625, "beta_high": 0.0, "orthogonal_to_active": 0.05320843309164047, "evidence_weights": [-0.17591269314289093, -0.3389941453933716, 0.15517336130142212], "agreement_weight": -0.004424927290529013, "orthogonal_weight": 0.5618215203285217}, {"head": 4, "memory_to_active": 0.716910719871521, "contribution_to_active": 1.0826246738433838, "mix_mean": 0.7405227422714233, "mix_low": 0.0017213821411132812, "mix_high": 0.1139516830444336, "read_cosine": 0.10804637521505356, "beta_mean": 0.13799063861370087, "nonfinite": 0.0, "weighted_memory_to_active": 0.6688072681427002, "mix_std": 0.29494187235832214, "beta_std": 0.16226713359355927, "beta_low": 0.22076416015625, "beta_high": 0.0, "orthogonal_to_active": 0.04099470376968384, "evidence_weights": [-0.2516869306564331, -0.5770899057388306, -0.09367372840642929], "agreement_weight": -0.014348197728395462, "orthogonal_weight": 0.07414426654577255}, {"head": 5, "memory_to_active": 0.042908959090709686, "contribution_to_active": 0.5870682001113892, "mix_mean": 0.5983045697212219, "mix_low": 0.000431060791015625, "mix_high": 0.0014801025390625, "read_cosine": 0.2723463177680969, "beta_mean": 0.00032648828346282244, "nonfinite": 0.0, "weighted_memory_to_active": 0.028130294755101204, "mix_std": 0.2728910744190216, "beta_std": 0.0005622001481242478, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.028169743716716766, "evidence_weights": [-0.2362203299999237, -0.2906920611858368, 0.3793628513813019], "agreement_weight": 0.09079598635435104, "orthogonal_weight": -1.0710399150848389}]}, {"layer": 7, "memory_to_active": 1.882827877998352, "contribution_to_active": 0.2093363106250763, "mix_mean": 0.05247950926423073, "mix_low": 0.40901440382003784, "mix_high": 1.9232433260185644e-05, "read_cosine": 0.23618026077747345, "beta_mean": 0.1177835613489151, "nonfinite": 0.0, "weighted_memory_to_active": 0.22652772068977356, "mix_std": 0.08330509066581726, "beta_std": 0.10300756990909576, "beta_low": 0.0869344100356102, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.04945344477891922, "output_influence": 0.20932826399803162, "residual_influence": 0.04624050483107567, "state_active": 0.01489383727312088, "state_memory": 0.0281890407204628, "state_peak": 27.455547332763672, "growth_active": 1.06624436378479, "growth_memory": 1.3271675109863281, "heads": [{"head": 0, "memory_to_active": 2.3443620204925537, "contribution_to_active": 0.5082823038101196, "mix_mean": 0.02925071306526661, "mix_low": 0.5109682083129883, "mix_high": 4.76837158203125e-06, "read_cosine": 0.3684253692626953, "beta_mean": 0.14323070645332336, "nonfinite": 0.0, "weighted_memory_to_active": 0.4807884693145752, "mix_std": 0.0647478699684143, "beta_std": 0.11654511094093323, "beta_low": 0.00811767578125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.047640763223171234, "evidence_weights": [0.0036320944782346487, -0.0868413969874382, 0.11652316153049469], "agreement_weight": -0.0352531373500824, "orthogonal_weight": 0.13086296617984772}, {"head": 1, "memory_to_active": 1.4371055364608765, "contribution_to_active": 0.3065912127494812, "mix_mean": 0.13677406311035156, "mix_low": 0.22827529907226562, "mix_high": 6.866455078125e-05, "read_cosine": 0.21171563863754272, "beta_mean": 0.14137867093086243, "nonfinite": 0.0, "weighted_memory_to_active": 0.3315541446208954, "mix_std": 0.1744903028011322, "beta_std": 0.08693139255046844, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.036599479615688324, "evidence_weights": [-0.1566179096698761, -0.10966809093952179, -0.21993370354175568], "agreement_weight": 0.020913802087306976, "orthogonal_weight": -0.12200804054737091}, {"head": 2, "memory_to_active": 1.0739126205444336, "contribution_to_active": 0.22182972729206085, "mix_mean": 0.06746423244476318, "mix_low": 0.22289276123046875, "mix_high": 4.1961669921875e-05, "read_cosine": 0.13983123004436493, "beta_mean": 0.08282957971096039, "nonfinite": 0.0, "weighted_memory_to_active": 0.15991559624671936, "mix_std": 0.11795499920845032, "beta_std": 0.05736056715250015, "beta_low": 0.026611328125, "beta_high": 0.0, "orthogonal_to_active": 0.0257077869027853, "evidence_weights": [-0.03010125830769539, -0.11604657024145126, 0.2563278079032898], "agreement_weight": 0.021945877000689507, "orthogonal_weight": -0.16626273095607758}, {"head": 3, "memory_to_active": 3.5043039321899414, "contribution_to_active": 0.060008496046066284, "mix_mean": 0.028829794377088547, "mix_low": 0.4038858413696289, "mix_high": 0.0, "read_cosine": 0.1450149416923523, "beta_mean": 0.1875636875629425, "nonfinite": 0.0, "weighted_memory_to_active": 0.1873403787612915, "mix_std": 0.04666610807180405, "beta_std": 0.19224263727664948, "beta_low": 0.0360107421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.147892564535141, "evidence_weights": [0.08143473416566849, 0.037671495229005814, 0.5057607889175415], "agreement_weight": 0.06568509340286255, "orthogonal_weight": -0.8709564805030823}, {"head": 4, "memory_to_active": 1.7121658325195312, "contribution_to_active": 0.06871601194143295, "mix_mean": 0.011314701288938522, "mix_low": 0.7353649139404297, "mix_high": 0.0, "read_cosine": 0.29993218183517456, "beta_mean": 0.12677505612373352, "nonfinite": 0.0, "weighted_memory_to_active": 0.07179282605648041, "mix_std": 0.026653822511434555, "beta_std": 0.1270677149295807, "beta_low": 0.04364013671875, "beta_high": 0.0, "orthogonal_to_active": 0.0035009949933737516, "evidence_weights": [0.12044499069452286, 0.06376629322767258, 0.07367006689310074], "agreement_weight": 0.05820027366280556, "orthogonal_weight": -0.07440491765737534}, {"head": 5, "memory_to_active": 1.2251168489456177, "contribution_to_active": 0.09059008955955505, "mix_mean": 0.0412435457110405, "mix_low": 0.35269927978515625, "mix_high": 0.0, "read_cosine": 0.2521621584892273, "beta_mean": 0.024923652410507202, "nonfinite": 0.0, "weighted_memory_to_active": 0.12777477502822876, "mix_std": 0.06931746006011963, "beta_std": 0.03789796680212021, "beta_low": 0.4072265625, "beta_high": 0.0, "orthogonal_to_active": 0.0353790745139122, "evidence_weights": [0.04170205816626549, 0.015697231516242027, 0.06754955649375916], "agreement_weight": 0.032468609511852264, "orthogonal_weight": -0.37414389848709106}]}, {"layer": 8, "memory_to_active": 0.602256715297699, "contribution_to_active": 0.49271202087402344, "mix_mean": 0.38229066133499146, "mix_low": 0.09014178067445755, "mix_high": 0.0006772677297703922, "read_cosine": 0.5262486934661865, "beta_mean": 0.036184363067150116, "nonfinite": 0.0, "weighted_memory_to_active": 0.1310614049434662, "mix_std": 0.22460511326789856, "beta_std": 0.041135188192129135, "beta_low": 0.5117899775505066, "beta_high": 0.0, "orthogonal_to_active": 0.058738067746162415, "output_influence": 0.3853159546852112, "residual_influence": 0.11114247888326645, "state_active": 0.05436728894710541, "state_memory": 0.012304307892918587, "state_peak": 31.639610290527344, "growth_active": 1.1054173707962036, "growth_memory": 1.3548269271850586, "heads": [{"head": 0, "memory_to_active": 0.04552565515041351, "contribution_to_active": 0.5073363780975342, "mix_mean": 0.5460952520370483, "mix_low": 0.0002574920654296875, "mix_high": 0.00040912628173828125, "read_cosine": 0.6842119693756104, "beta_mean": 0.0030970757361501455, "nonfinite": 0.0, "weighted_memory_to_active": 0.02702072635293007, "mix_std": 0.24220524728298187, "beta_std": 0.007349510211497545, "beta_low": 0.9276123046875, "beta_high": 0.0, "orthogonal_to_active": 0.015455465763807297, "evidence_weights": [-0.18607008457183838, -0.3662852346897125, 0.7369982600212097], "agreement_weight": 0.06891663372516632, "orthogonal_weight": -0.8099432587623596}, {"head": 1, "memory_to_active": 0.4030457139015198, "contribution_to_active": 0.3908120393753052, "mix_mean": 0.40584802627563477, "mix_low": 0.0037069320678710938, "mix_high": 0.000400543212890625, "read_cosine": 0.45632869005203247, "beta_mean": 0.015305573120713234, "nonfinite": 0.0, "weighted_memory_to_active": 0.17645595967769623, "mix_std": 0.2467593252658844, "beta_std": 0.020075611770153046, "beta_low": 0.51422119140625, "beta_high": 0.0, "orthogonal_to_active": 0.08216246217489243, "evidence_weights": [-0.18458090722560883, -0.22099007666110992, 0.5546402931213379], "agreement_weight": 0.06274501979351044, "orthogonal_weight": -0.5338780283927917}, {"head": 2, "memory_to_active": 0.06648658215999603, "contribution_to_active": 0.5947858095169067, "mix_mean": 0.5520738363265991, "mix_low": 0.000759124755859375, "mix_high": 0.0021381378173828125, "read_cosine": 0.3786580562591553, "beta_mean": 0.0007562799146398902, "nonfinite": 0.0, "weighted_memory_to_active": 0.04414171725511551, "mix_std": 0.25019511580467224, "beta_std": 0.004650596994906664, "beta_low": 0.9952392578125, "beta_high": 0.0, "orthogonal_to_active": 0.03229495510458946, "evidence_weights": [-0.20843660831451416, -0.3190891742706299, 0.5749284029006958], "agreement_weight": 0.11205840110778809, "orthogonal_weight": -0.8460577726364136}, {"head": 3, "memory_to_active": 0.18358547985553741, "contribution_to_active": 0.5424110889434814, "mix_mean": 0.46181780099868774, "mix_low": 0.0008068084716796875, "mix_high": 0.0001888275146484375, "read_cosine": 0.7731134295463562, "beta_mean": 0.015655964612960815, "nonfinite": 0.0, "weighted_memory_to_active": 0.11971262842416763, "mix_std": 0.25043007731437683, "beta_std": 0.014123202301561832, "beta_low": 0.367919921875, "beta_high": 0.0, "orthogonal_to_active": 0.033253997564315796, "evidence_weights": [-0.15062201023101807, -0.2677987217903137, 0.7331084609031677], "agreement_weight": 0.1048428937792778, "orthogonal_weight": -0.5155760645866394}, {"head": 4, "memory_to_active": 2.6700117588043213, "contribution_to_active": 0.2669917345046997, "mix_mean": 0.025465596467256546, "mix_low": 0.42624950408935547, "mix_high": 0.0, "read_cosine": 0.3148086965084076, "beta_mean": 0.1073635071516037, "nonfinite": 0.0, "weighted_memory_to_active": 0.258548378944397, "mix_std": 0.04233306646347046, "beta_std": 0.09548301249742508, "beta_low": 0.0081787109375, "beta_high": 0.0, "orthogonal_to_active": 0.00802809651941061, "evidence_weights": [0.03480598330497742, -0.010416798293590546, -0.1820119321346283], "agreement_weight": 0.01714172773063183, "orthogonal_weight": 0.03267714008688927}, {"head": 5, "memory_to_active": 0.2448849380016327, "contribution_to_active": 0.653934895992279, "mix_mean": 0.30244332551956177, "mix_low": 0.1090707778930664, "mix_high": 0.000926971435546875, "read_cosine": 0.5503711700439453, "beta_mean": 0.07492778450250626, "nonfinite": 0.0, "weighted_memory_to_active": 0.16048896312713623, "mix_std": 0.3157077133655548, "beta_std": 0.1051291972398758, "beta_low": 0.257568359375, "beta_high": 0.0, "orthogonal_to_active": 0.18123342096805573, "evidence_weights": [-0.06456608325242996, -0.3082287907600403, -0.4182542860507965], "agreement_weight": 0.02380378171801567, "orthogonal_weight": 1.383259654045105}]}, {"layer": 9, "memory_to_active": 1.1729168891906738, "contribution_to_active": 0.6174570322036743, "mix_mean": 0.2707364559173584, "mix_low": 0.1642923355102539, "mix_high": 0.0016934077721089125, "read_cosine": 0.3578365445137024, "beta_mean": 0.08555548638105392, "nonfinite": 0.0, "weighted_memory_to_active": 0.36687248945236206, "mix_std": 0.20924726128578186, "beta_std": 0.05016329884529114, "beta_low": 0.4988200068473816, "beta_high": 0.0, "orthogonal_to_active": 0.035405900329351425, "output_influence": 0.42864376306533813, "residual_influence": 0.1128493994474411, "state_active": 0.04333307966589928, "state_memory": 0.023087691515684128, "state_peak": 32.07533645629883, "growth_active": 1.2697850465774536, "growth_memory": 1.4215912818908691, "heads": [{"head": 0, "memory_to_active": 0.004049145150929689, "contribution_to_active": 0.4887504577636719, "mix_mean": 0.5492688417434692, "mix_low": 0.0020532608032226562, "mix_high": 0.0068607330322265625, "read_cosine": 0.25790438055992126, "beta_mean": 1.1348774933139794e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022850690875202417, "mix_std": 0.2968050241470337, "beta_std": 4.186773730907589e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018884441815316677, "evidence_weights": [-0.279350221157074, -0.28297391533851624, 0.9812644124031067], "agreement_weight": 0.07053163647651672, "orthogonal_weight": -0.8674588799476624}, {"head": 1, "memory_to_active": 0.02889322303235531, "contribution_to_active": 0.5430557727813721, "mix_mean": 0.40051016211509705, "mix_low": 0.011713027954101562, "mix_high": 0.0008220672607421875, "read_cosine": 0.4169326424598694, "beta_mean": 8.255573629867285e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015412515960633755, "mix_std": 0.28376075625419617, "beta_std": 0.00017414169269613922, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009216823615133762, "evidence_weights": [-0.20850622653961182, -0.2387004941701889, 1.0416216850280762], "agreement_weight": 0.29110923409461975, "orthogonal_weight": -0.6998827457427979}, {"head": 2, "memory_to_active": 0.22670087218284607, "contribution_to_active": 0.5606627464294434, "mix_mean": 0.4571166932582855, "mix_low": 0.0051250457763671875, "mix_high": 0.0020151138305664062, "read_cosine": 0.35425451397895813, "beta_mean": 0.0009419689304195344, "nonfinite": 0.0, "weighted_memory_to_active": 0.1285770684480667, "mix_std": 0.2846170961856842, "beta_std": 0.005331812892109156, "beta_low": 0.99029541015625, "beta_high": 0.0, "orthogonal_to_active": 0.048820704221725464, "evidence_weights": [-0.1706118881702423, -0.20917299389839172, 0.5314511656761169], "agreement_weight": 0.21592772006988525, "orthogonal_weight": -0.4531501829624176}, {"head": 3, "memory_to_active": 1.6771646738052368, "contribution_to_active": 0.1902817189693451, "mix_mean": 0.03856886923313141, "mix_low": 0.4143838882446289, "mix_high": 0.00013065338134765625, "read_cosine": 0.4397469758987427, "beta_mean": 0.10755237936973572, "nonfinite": 0.0, "weighted_memory_to_active": 0.16707120835781097, "mix_std": 0.09267808496952057, "beta_std": 0.07100242376327515, "beta_low": 0.00262451171875, "beta_high": 0.0, "orthogonal_to_active": 0.029891401529312134, "evidence_weights": [0.05764565244317055, 0.03896523267030716, -0.12983612716197968], "agreement_weight": 0.06697829812765121, "orthogonal_weight": 0.19955500960350037}, {"head": 4, "memory_to_active": 3.3347244262695312, "contribution_to_active": 1.3457980155944824, "mix_mean": 0.11973389983177185, "mix_low": 0.1722869873046875, "mix_high": 0.000331878662109375, "read_cosine": 0.23888780176639557, "beta_mean": 0.2385796159505844, "nonfinite": 0.0, "weighted_memory_to_active": 1.2446802854537964, "mix_std": 0.18241091072559357, "beta_std": 0.12012883275747299, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01752028800547123, "evidence_weights": [-0.0522068552672863, -0.14190581440925598, 0.04320760816335678], "agreement_weight": -0.04639158397912979, "orthogonal_weight": -0.014017567969858646}, {"head": 5, "memory_to_active": 1.7659687995910645, "contribution_to_active": 0.5761934518814087, "mix_mean": 0.05922023206949234, "mix_low": 0.3801918029785156, "mix_high": 0.0, "read_cosine": 0.43929263949394226, "beta_mean": 0.16616497933864594, "nonfinite": 0.0, "weighted_memory_to_active": 0.6432085037231445, "mix_std": 0.1152116060256958, "beta_std": 0.10430069267749786, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1050977110862732, "evidence_weights": [0.0734369307756424, -0.043130774050951004, 0.12249451875686646], "agreement_weight": 0.02362988330423832, "orthogonal_weight": 0.22743116319179535}]}, {"layer": 10, "memory_to_active": 0.17615658044815063, "contribution_to_active": 0.47349464893341064, "mix_mean": 0.4526972472667694, "mix_low": 0.011339187622070312, "mix_high": 0.007646243087947369, "read_cosine": 0.3713952600955963, "beta_mean": 0.04282905161380768, "nonfinite": 0.0, "weighted_memory_to_active": 0.07699916511774063, "mix_std": 0.2686609923839569, "beta_std": 0.04726174846291542, "beta_low": 0.66259765625, "beta_high": 0.0, "orthogonal_to_active": 0.052339695394039154, "output_influence": 0.389765202999115, "residual_influence": 0.12707877159118652, "state_active": 0.05615481361746788, "state_memory": 0.013640192337334156, "state_peak": 112.31742095947266, "growth_active": 1.1099793910980225, "growth_memory": 1.0832940340042114, "heads": [{"head": 0, "memory_to_active": 0.43816590309143066, "contribution_to_active": 0.821295440196991, "mix_mean": 0.6181697845458984, "mix_low": 0.023148536682128906, "mix_high": 0.03935527801513672, "read_cosine": 0.1658729612827301, "beta_mean": 0.18918785452842712, "nonfinite": 0.0, "weighted_memory_to_active": 0.288830041885376, "mix_std": 0.3662979006767273, "beta_std": 0.21145059168338776, "beta_low": 0.134521484375, "beta_high": 0.0, "orthogonal_to_active": 0.21564102172851562, "evidence_weights": [-0.15799975395202637, -0.35529109835624695, -0.4526744484901428], "agreement_weight": 0.02078230120241642, "orthogonal_weight": 0.7949219346046448}, {"head": 1, "memory_to_active": 0.01929139904677868, "contribution_to_active": 0.5459374189376831, "mix_mean": 0.5428159832954407, "mix_low": 0.0008039474487304688, "mix_high": 0.0048351287841796875, "read_cosine": 0.2517413794994354, "beta_mean": 0.00011287708184681833, "nonfinite": 0.0, "weighted_memory_to_active": 0.010491601191461086, "mix_std": 0.26704299449920654, "beta_std": 0.00015171588165685534, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012445011176168919, "evidence_weights": [-0.27133142948150635, -0.19654212892055511, 0.5616478323936462], "agreement_weight": 0.35984018445014954, "orthogonal_weight": -1.3068108558654785}, {"head": 2, "memory_to_active": 0.10077987611293793, "contribution_to_active": 0.4492782652378082, "mix_mean": 0.4497544467449188, "mix_low": 0.0012264251708984375, "mix_high": 0.00127410888671875, "read_cosine": 0.005063818302005529, "beta_mean": 0.0004063546657562256, "nonfinite": 0.0, "weighted_memory_to_active": 0.03548654913902283, "mix_std": 0.2693653404712677, "beta_std": 0.0017343445215374231, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.013171536847949028, "evidence_weights": [-0.17634497582912445, -0.22648409008979797, 0.515703558921814], "agreement_weight": -0.058313265442848206, "orthogonal_weight": -0.35954511165618896}, {"head": 3, "memory_to_active": 0.32420945167541504, "contribution_to_active": 0.11714054644107819, "mix_mean": 0.27472439408302307, "mix_low": 0.038611412048339844, "mix_high": 0.00034427642822265625, "read_cosine": 0.8905848264694214, "beta_mean": 0.06110530346632004, "nonfinite": 0.0, "weighted_memory_to_active": 0.04674398899078369, "mix_std": 0.23369163274765015, "beta_std": 0.0594075471162796, "beta_low": 0.00567626953125, "beta_high": 0.0, "orthogonal_to_active": 0.051263000816106796, "evidence_weights": [-0.12726710736751556, -0.14662279188632965, 0.17711375653743744], "agreement_weight": -0.0008441194077022374, "orthogonal_weight": 1.4346253871917725}, {"head": 4, "memory_to_active": 0.030201589688658714, "contribution_to_active": 0.48361581563949585, "mix_mean": 0.4660143256187439, "mix_low": 0.0013828277587890625, "mix_high": 6.771087646484375e-05, "read_cosine": 0.0606275238096714, "beta_mean": 5.093867002869956e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00863267295062542, "mix_std": 0.25373920798301697, "beta_std": 0.002358329715207219, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.002879084786400199, "evidence_weights": [-0.17992867529392242, -0.2262084037065506, 0.5723161697387695], "agreement_weight": 0.04952525347471237, "orthogonal_weight": -0.3344018757343292}, {"head": 5, "memory_to_active": 0.1442912518978119, "contribution_to_active": 0.4237002730369568, "mix_mean": 0.3647046983242035, "mix_low": 0.0028619766235351562, "mix_high": 9.5367431640625e-07, "read_cosine": 0.8544809222221375, "beta_mean": 0.006110966205596924, "nonfinite": 0.0, "weighted_memory_to_active": 0.07181012630462646, "mix_std": 0.22182874381542206, "beta_std": 0.008467944338917732, "beta_low": 0.84002685546875, "beta_high": 0.0, "orthogonal_to_active": 0.018638528883457184, "evidence_weights": [-0.15618184208869934, -0.1635614037513733, 0.2939194440841675], "agreement_weight": 0.09433503448963165, "orthogonal_weight": -0.5823942422866821}]}, {"layer": 11, "memory_to_active": 0.19493511319160461, "contribution_to_active": 0.5097557902336121, "mix_mean": 0.4228397607803345, "mix_low": 0.05541197955608368, "mix_high": 0.031145572662353516, "read_cosine": 0.4863886535167694, "beta_mean": 0.036827947944402695, "nonfinite": 0.0, "weighted_memory_to_active": 0.07811632752418518, "mix_std": 0.31371501088142395, "beta_std": 0.052254028618335724, "beta_low": 0.50341796875, "beta_high": 0.0, "orthogonal_to_active": 0.05288417637348175, "output_influence": 0.47712242603302, "residual_influence": 0.12206342816352844, "state_active": 0.13357853889465332, "state_memory": 0.009768432006239891, "state_peak": 22.477012634277344, "growth_active": 1.1070369482040405, "growth_memory": 1.449876308441162, "heads": [{"head": 0, "memory_to_active": 0.014785051345825195, "contribution_to_active": 0.5455003976821899, "mix_mean": 0.5584811568260193, "mix_low": 0.0018491744995117188, "mix_high": 0.008067131042480469, "read_cosine": 0.3572923541069031, "beta_mean": 0.00013047910761088133, "nonfinite": 0.0, "weighted_memory_to_active": 0.009581551887094975, "mix_std": 0.27419477701187134, "beta_std": 0.0046132029965519905, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.006018701009452343, "evidence_weights": [-0.15516333281993866, -0.34395596385002136, 0.5983127355575562], "agreement_weight": 0.17754682898521423, "orthogonal_weight": -0.7203514575958252}, {"head": 1, "memory_to_active": 0.43162018060684204, "contribution_to_active": 0.2585447132587433, "mix_mean": 0.29539865255355835, "mix_low": 0.018532752990722656, "mix_high": 8.296966552734375e-05, "read_cosine": 0.6233290433883667, "beta_mean": 0.04029005393385887, "nonfinite": 0.0, "weighted_memory_to_active": 0.1706658899784088, "mix_std": 0.24726223945617676, "beta_std": 0.07051403075456619, "beta_low": 0.35028076171875, "beta_high": 0.0, "orthogonal_to_active": 0.029980799183249474, "evidence_weights": [-0.15787553787231445, -0.13050870597362518, 0.22823552787303925], "agreement_weight": 0.08350928872823715, "orthogonal_weight": -0.247125044465065}, {"head": 2, "memory_to_active": 0.03864426538348198, "contribution_to_active": 0.8921651840209961, "mix_mean": 0.47977733612060547, "mix_low": 0.0811767578125, "mix_high": 0.10010433197021484, "read_cosine": 0.31028175354003906, "beta_mean": 0.07118161022663116, "nonfinite": 0.0, "weighted_memory_to_active": 0.03016636148095131, "mix_std": 0.41756945848464966, "beta_std": 0.09346136450767517, "beta_low": 0.419677734375, "beta_high": 0.0, "orthogonal_to_active": 0.10262829810380936, "evidence_weights": [-0.0634383037686348, -0.456843763589859, -1.0321977138519287], "agreement_weight": 0.006044939160346985, "orthogonal_weight": 3.659799814224243}, {"head": 3, "memory_to_active": 0.01895558461546898, "contribution_to_active": 0.6946763396263123, "mix_mean": 0.4250905513763428, "mix_low": 0.20691680908203125, "mix_high": 0.07730674743652344, "read_cosine": 0.5730639696121216, "beta_mean": 0.060028135776519775, "nonfinite": 0.0, "weighted_memory_to_active": 0.010415949858725071, "mix_std": 0.4314509630203247, "beta_std": 0.08682020008563995, "beta_low": 0.45574951171875, "beta_high": 0.0, "orthogonal_to_active": 0.03725757077336311, "evidence_weights": [-0.01941247284412384, -0.3644371032714844, -0.43476125597953796], "agreement_weight": 0.028970377519726753, "orthogonal_weight": 3.9174885749816895}, {"head": 4, "memory_to_active": 0.5281261205673218, "contribution_to_active": 0.32869821786880493, "mix_mean": 0.3092682957649231, "mix_low": 0.023575782775878906, "mix_high": 0.0001678466796875, "read_cosine": 0.20351073145866394, "beta_mean": 0.0370803102850914, "nonfinite": 0.0, "weighted_memory_to_active": 0.18953394889831543, "mix_std": 0.2527014911174774, "beta_std": 0.03960070386528969, "beta_low": 0.15533447265625, "beta_high": 0.0, "orthogonal_to_active": 0.118683360517025, "evidence_weights": [-0.16129431128501892, -0.1296081244945526, 1.4211397171020508], "agreement_weight": -0.13351142406463623, "orthogonal_weight": -0.654210090637207}, {"head": 5, "memory_to_active": 0.13747942447662354, "contribution_to_active": 0.338949978351593, "mix_mean": 0.4690224826335907, "mix_low": 0.00042057037353515625, "mix_high": 0.0011444091796875, "read_cosine": 0.8508539199829102, "beta_mean": 0.01225708145648241, "nonfinite": 0.0, "weighted_memory_to_active": 0.05833420529961586, "mix_std": 0.25911110639572144, "beta_std": 0.018514644354581833, "beta_low": 0.64019775390625, "beta_high": 0.0, "orthogonal_to_active": 0.0227363184094429, "evidence_weights": [-0.19978870451450348, -0.2570474445819855, 0.5080739259719849], "agreement_weight": 0.04805510863661766, "orthogonal_weight": -0.6470367312431335}]}, {"layer": 12, "memory_to_active": 0.3391450047492981, "contribution_to_active": 0.53463214635849, "mix_mean": 0.44772589206695557, "mix_low": 0.019243717193603516, "mix_high": 0.002862135646864772, "read_cosine": 0.2505694627761841, "beta_mean": 0.01615252159535885, "nonfinite": 0.0, "weighted_memory_to_active": 0.173597052693367, "mix_std": 0.2558818459510803, "beta_std": 0.01745130866765976, "beta_low": 0.8119405508041382, "beta_high": 0.0, "orthogonal_to_active": 0.07044956833124161, "output_influence": 0.4080190658569336, "residual_influence": 0.11177732795476913, "state_active": 0.04041524976491928, "state_memory": 0.010555259883403778, "state_peak": 22.651823043823242, "growth_active": 1.1357730627059937, "growth_memory": 1.2276521921157837, "heads": [{"head": 0, "memory_to_active": 0.1581580936908722, "contribution_to_active": 0.4251099228858948, "mix_mean": 0.40284329652786255, "mix_low": 0.005072593688964844, "mix_high": 0.00043201446533203125, "read_cosine": 0.2531968355178833, "beta_mean": 0.0022693395148962736, "nonfinite": 0.0, "weighted_memory_to_active": 0.06294338405132294, "mix_std": 0.264826238155365, "beta_std": 0.008694348856806755, "beta_low": 0.95233154296875, "beta_high": 0.0, "orthogonal_to_active": 0.03403204679489136, "evidence_weights": [-0.16378431022167206, -0.23253363370895386, 0.2727562487125397], "agreement_weight": 0.10552100837230682, "orthogonal_weight": -0.5744324326515198}, {"head": 1, "memory_to_active": 0.11895597726106644, "contribution_to_active": 0.463601291179657, "mix_mean": 0.47639432549476624, "mix_low": 0.0007476806640625, "mix_high": 0.00015735626220703125, "read_cosine": 0.5500780344009399, "beta_mean": 0.0012543099001049995, "nonfinite": 0.0, "weighted_memory_to_active": 0.056476715952157974, "mix_std": 0.25855979323387146, "beta_std": 0.005823069717735052, "beta_low": 0.98297119140625, "beta_high": 0.0, "orthogonal_to_active": 0.042385946959257126, "evidence_weights": [-0.19576115906238556, -0.2549050748348236, 0.8455385565757751], "agreement_weight": 0.09918227791786194, "orthogonal_weight": -0.9051204323768616}, {"head": 2, "memory_to_active": 0.6568489670753479, "contribution_to_active": 0.47882670164108276, "mix_mean": 0.3256218731403351, "mix_low": 0.02606964111328125, "mix_high": 0.0004425048828125, "read_cosine": 0.4412444829940796, "beta_mean": 0.09011221677064896, "nonfinite": 0.0, "weighted_memory_to_active": 0.26992613077163696, "mix_std": 0.2524110674858093, "beta_std": 0.06859979033470154, "beta_low": 0.001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.2156984806060791, "evidence_weights": [-0.23964045941829681, -0.04018106311559677, -0.05783476307988167], "agreement_weight": 0.023250838741660118, "orthogonal_weight": 0.9698473811149597}, {"head": 3, "memory_to_active": 0.02972729131579399, "contribution_to_active": 0.7200550436973572, "mix_mean": 0.6766114830970764, "mix_low": 0.00030231475830078125, "mix_high": 0.012928962707519531, "read_cosine": 0.035220347344875336, "beta_mean": 0.00031458312878385186, "nonfinite": 0.0, "weighted_memory_to_active": 0.02112760581076145, "mix_std": 0.24534626305103302, "beta_std": 0.004416841547936201, "beta_low": 0.9981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.019284550100564957, "evidence_weights": [-0.30144014954566956, -0.3270508646965027, 1.0714905261993408], "agreement_weight": 0.24741590023040771, "orthogonal_weight": -0.9313665628433228}, {"head": 4, "memory_to_active": 1.0074245929718018, "contribution_to_active": 0.6626585721969604, "mix_mean": 0.2231011986732483, "mix_low": 0.07724285125732422, "mix_high": 0.0015268325805664062, "read_cosine": -0.05391457676887512, "beta_mean": 0.002363273873925209, "nonfinite": 0.0, "weighted_memory_to_active": 0.5956994295120239, "mix_std": 0.23888882994651794, "beta_std": 0.014271371066570282, "beta_low": 0.9468994140625, "beta_high": 0.0, "orthogonal_to_active": 0.08327685296535492, "evidence_weights": [-0.03371562063694, -0.18046188354492188, 0.012348046526312828], "agreement_weight": -0.004414606373757124, "orthogonal_weight": -0.15324421226978302}, {"head": 5, "memory_to_active": 0.063755102455616, "contribution_to_active": 0.45754116773605347, "mix_mean": 0.581782877445221, "mix_low": 0.0060272216796875, "mix_high": 0.0016851425170898438, "read_cosine": 0.27759164571762085, "beta_mean": 0.0006014082464389503, "nonfinite": 0.0, "weighted_memory_to_active": 0.03540896624326706, "mix_std": 0.2752586603164673, "beta_std": 0.002902424428611994, "beta_low": 0.99005126953125, "beta_high": 0.0, "orthogonal_to_active": 0.028019502758979797, "evidence_weights": [-0.26347556710243225, -0.2977481782436371, 0.936382532119751], "agreement_weight": 0.16364911198616028, "orthogonal_weight": -0.8425882458686829}]}, {"layer": 13, "memory_to_active": 0.1726721227169037, "contribution_to_active": 0.493124783039093, "mix_mean": 0.4815441966056824, "mix_low": 0.005742550361901522, "mix_high": 0.0027160646859556437, "read_cosine": 0.4185907244682312, "beta_mean": 0.01772412285208702, "nonfinite": 0.0, "weighted_memory_to_active": 0.07531405985355377, "mix_std": 0.2586875557899475, "beta_std": 0.017309552058577538, "beta_low": 0.7857157588005066, "beta_high": 0.0, "orthogonal_to_active": 0.06917108595371246, "output_influence": 0.4452027976512909, "residual_influence": 0.14750435948371887, "state_active": 0.09042541682720184, "state_memory": 0.03765026852488518, "state_peak": 98.03775024414062, "growth_active": 1.0101406574249268, "growth_memory": 1.0247182846069336, "heads": [{"head": 0, "memory_to_active": 0.05307469144463539, "contribution_to_active": 0.5177557468414307, "mix_mean": 0.5052624344825745, "mix_low": 0.0005903244018554688, "mix_high": 0.0018377304077148438, "read_cosine": 0.40532052516937256, "beta_mean": 0.00021567272779066116, "nonfinite": 0.0, "weighted_memory_to_active": 0.029594970867037773, "mix_std": 0.26458483934402466, "beta_std": 0.0004205462755635381, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.027814820408821106, "evidence_weights": [-0.2026699334383011, -0.24979005753993988, 0.7638753056526184], "agreement_weight": 0.05515214800834656, "orthogonal_weight": -1.0632216930389404}, {"head": 1, "memory_to_active": 0.030290139839053154, "contribution_to_active": 0.549340546131134, "mix_mean": 0.4915843904018402, "mix_low": 0.0023174285888671875, "mix_high": 0.00040149688720703125, "read_cosine": 0.24076204001903534, "beta_mean": 0.00011788117262767628, "nonfinite": 0.0, "weighted_memory_to_active": 0.01606096513569355, "mix_std": 0.2658366560935974, "beta_std": 0.0002171996602555737, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.016029249876737595, "evidence_weights": [-0.1879182755947113, -0.25949421525001526, 0.7163965106010437], "agreement_weight": 0.19408266246318817, "orthogonal_weight": -1.052449107170105}, {"head": 2, "memory_to_active": 0.15947014093399048, "contribution_to_active": 0.3743346035480499, "mix_mean": 0.3542059063911438, "mix_low": 0.0025548934936523438, "mix_high": 5.435943603515625e-05, "read_cosine": 0.6804935932159424, "beta_mean": 0.012527395971119404, "nonfinite": 0.0, "weighted_memory_to_active": 0.06483591347932816, "mix_std": 0.21814243495464325, "beta_std": 0.0323917381465435, "beta_low": 0.7388916015625, "beta_high": 0.0, "orthogonal_to_active": 0.030845951288938522, "evidence_weights": [-0.2222786545753479, -0.165266752243042, 0.49942678213119507], "agreement_weight": 0.0333818718791008, "orthogonal_weight": -0.7556240558624268}, {"head": 3, "memory_to_active": 0.16323700547218323, "contribution_to_active": 0.5859357118606567, "mix_mean": 0.5888660550117493, "mix_low": 0.012868881225585938, "mix_high": 0.009608268737792969, "read_cosine": 0.03814509138464928, "beta_mean": 0.0011781807988882065, "nonfinite": 0.0, "weighted_memory_to_active": 0.11801306158304214, "mix_std": 0.28277379274368286, "beta_std": 0.004479588475078344, "beta_low": 0.9886474609375, "beta_high": 0.0, "orthogonal_to_active": 0.11748289316892624, "evidence_weights": [-0.34538084268569946, -0.2337992787361145, 2.0018861293792725], "agreement_weight": 2.312091827392578, "orthogonal_weight": -1.0529613494873047}, {"head": 4, "memory_to_active": 0.4875228703022003, "contribution_to_active": 0.30814728140830994, "mix_mean": 0.3854735493659973, "mix_low": 0.012314796447753906, "mix_high": 0.00026607513427734375, "read_cosine": 0.8724062442779541, "beta_mean": 0.09119808673858643, "nonfinite": 0.0, "weighted_memory_to_active": 0.1424422264099121, "mix_std": 0.239296093583107, "beta_std": 0.060853373259305954, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16457366943359375, "evidence_weights": [-0.18381266295909882, -0.23267939686775208, 0.493678480386734], "agreement_weight": -0.0027109759394079447, "orthogonal_weight": 1.7700623273849487}, {"head": 5, "memory_to_active": 0.14243771135807037, "contribution_to_active": 0.6232348084449768, "mix_mean": 0.5638726949691772, "mix_low": 0.0038089752197265625, "mix_high": 0.004128456115722656, "read_cosine": 0.27441680431365967, "beta_mean": 0.001107522752135992, "nonfinite": 0.0, "weighted_memory_to_active": 0.08093717694282532, "mix_std": 0.28149136900901794, "beta_std": 0.005494855344295502, "beta_low": 0.98681640625, "beta_high": 0.0, "orthogonal_to_active": 0.05827992781996727, "evidence_weights": [-0.2170732468366623, -0.2781127095222473, 0.571520984172821], "agreement_weight": 0.08124948292970657, "orthogonal_weight": -0.802222728729248}]}, {"layer": 14, "memory_to_active": 0.26937204599380493, "contribution_to_active": 0.4206240773200989, "mix_mean": 0.40549466013908386, "mix_low": 0.024801574647426605, "mix_high": 0.003724734066054225, "read_cosine": 0.6186562776565552, "beta_mean": 0.07142653316259384, "nonfinite": 0.0, "weighted_memory_to_active": 0.12199664115905762, "mix_std": 0.24918784201145172, "beta_std": 0.06544728577136993, "beta_low": 0.4430033564567566, "beta_high": 0.0, "orthogonal_to_active": 0.042143404483795166, "output_influence": 0.29170629382133484, "residual_influence": 0.1111866757273674, "state_active": 0.07070956379175186, "state_memory": 0.0159708671271801, "state_peak": 24.932106018066406, "growth_active": 1.1659886837005615, "growth_memory": 1.2990895509719849, "heads": [{"head": 0, "memory_to_active": 0.3322097063064575, "contribution_to_active": 0.21533985435962677, "mix_mean": 0.23532699048519135, "mix_low": 0.015705108642578125, "mix_high": 6.67572021484375e-05, "read_cosine": 0.750234067440033, "beta_mean": 0.06970466673374176, "nonfinite": 0.0, "weighted_memory_to_active": 0.09743429720401764, "mix_std": 0.21347329020500183, "beta_std": 0.0692237839102745, "beta_low": 0.02471923828125, "beta_high": 0.0, "orthogonal_to_active": 0.04305272549390793, "evidence_weights": [-0.09848572313785553, -0.07113990187644958, 0.7137589454650879], "agreement_weight": 0.030911259353160858, "orthogonal_weight": 0.664836049079895}, {"head": 1, "memory_to_active": 0.5154176354408264, "contribution_to_active": 0.36463719606399536, "mix_mean": 0.2608368396759033, "mix_low": 0.11393260955810547, "mix_high": 9.1552734375e-05, "read_cosine": 0.7185103893280029, "beta_mean": 0.2921067476272583, "nonfinite": 0.0, "weighted_memory_to_active": 0.24253875017166138, "mix_std": 0.26991987228393555, "beta_std": 0.2600035071372986, "beta_low": 0.0765380859375, "beta_high": 0.0, "orthogonal_to_active": 0.023102974519133568, "evidence_weights": [0.007376029156148434, -0.2793082892894745, -0.15610231459140778], "agreement_weight": 0.05275021120905876, "orthogonal_weight": 0.17298372089862823}, {"head": 2, "memory_to_active": 0.2095392942428589, "contribution_to_active": 0.5129886269569397, "mix_mean": 0.5166301131248474, "mix_low": 0.00366973876953125, "mix_high": 0.0014810562133789062, "read_cosine": 0.38579490780830383, "beta_mean": 0.005616477690637112, "nonfinite": 0.0, "weighted_memory_to_active": 0.11781121790409088, "mix_std": 0.2700497806072235, "beta_std": 0.007367653306573629, "beta_low": 0.85675048828125, "beta_high": 0.0, "orthogonal_to_active": 0.08738259971141815, "evidence_weights": [-0.1743345558643341, -0.27157458662986755, 0.8138284683227539], "agreement_weight": 0.07452047616243362, "orthogonal_weight": -0.8373416066169739}, {"head": 3, "memory_to_active": 0.05879965424537659, "contribution_to_active": 0.6585808992385864, "mix_mean": 0.6586320400238037, "mix_low": 0.003780364990234375, "mix_high": 0.0131072998046875, "read_cosine": 0.3404049873352051, "beta_mean": 0.008281717076897621, "nonfinite": 0.0, "weighted_memory_to_active": 0.039879463613033295, "mix_std": 0.24384742975234985, "beta_std": 0.007539731916040182, "beta_low": 0.719970703125, "beta_high": 0.0, "orthogonal_to_active": 0.021852806210517883, "evidence_weights": [-0.3821380138397217, -0.3290766477584839, 0.43647339940071106], "agreement_weight": 0.08047927170991898, "orthogonal_weight": -0.6640121936798096}, {"head": 4, "memory_to_active": 0.18832668662071228, "contribution_to_active": 0.47931554913520813, "mix_mean": 0.46388739347457886, "mix_low": 0.0029172897338867188, "mix_high": 0.007035255432128906, "read_cosine": 0.7986240983009338, "beta_mean": 0.050314679741859436, "nonfinite": 0.0, "weighted_memory_to_active": 0.10365158319473267, "mix_std": 0.26218876242637634, "beta_std": 0.038621388375759125, "beta_low": 0.02630615234375, "beta_high": 0.0, "orthogonal_to_active": 0.029253283515572548, "evidence_weights": [-0.21850238740444183, -0.2672644555568695, 0.39322859048843384], "agreement_weight": 0.11674425005912781, "orthogonal_weight": -0.5140045881271362}, {"head": 5, "memory_to_active": 0.31193920969963074, "contribution_to_active": 0.29288214445114136, "mix_mean": 0.29765447974205017, "mix_low": 0.0088043212890625, "mix_high": 0.0005664825439453125, "read_cosine": 0.7183690667152405, "beta_mean": 0.00253489101305604, "nonfinite": 0.0, "weighted_memory_to_active": 0.13066455721855164, "mix_std": 0.23564781248569489, "beta_std": 0.009927639737725258, "beta_low": 0.9537353515625, "beta_high": 0.0, "orthogonal_to_active": 0.04821602255105972, "evidence_weights": [-0.18459203839302063, -0.15049031376838684, 0.18997052311897278], "agreement_weight": 0.0904589593410492, "orthogonal_weight": -0.6122417449951172}]}, {"layer": 15, "memory_to_active": 0.0668688714504242, "contribution_to_active": 0.4751660227775574, "mix_mean": 0.38965171575546265, "mix_low": 0.0077371601946651936, "mix_high": 0.00269953440874815, "read_cosine": 0.7000306248664856, "beta_mean": 0.03544457256793976, "nonfinite": 0.0, "weighted_memory_to_active": 0.030048463493585587, "mix_std": 0.2788185179233551, "beta_std": 0.03978646546602249, "beta_low": 0.7139791250228882, "beta_high": 0.0, "orthogonal_to_active": 0.005188534036278725, "output_influence": 0.30853331089019775, "residual_influence": 0.16142553091049194, "state_active": 0.11565714329481125, "state_memory": 0.007550057955086231, "state_peak": 37.06895065307617, "growth_active": 1.3007533550262451, "growth_memory": 1.8033692836761475, "heads": [{"head": 0, "memory_to_active": 0.11351535469293594, "contribution_to_active": 0.47491395473480225, "mix_mean": 0.36462849378585815, "mix_low": 0.005578041076660156, "mix_high": 0.0034418106079101562, "read_cosine": 0.710991382598877, "beta_mean": 0.03713173419237137, "nonfinite": 0.0, "weighted_memory_to_active": 0.05493218079209328, "mix_std": 0.2695757746696472, "beta_std": 0.027816180139780045, "beta_low": 0.1173095703125, "beta_high": 0.0, "orthogonal_to_active": 0.017784900963306427, "evidence_weights": [-0.16545002162456512, -0.1896502673625946, 0.3443795144557953], "agreement_weight": 0.0865246057510376, "orthogonal_weight": -0.5865693688392639}, {"head": 1, "memory_to_active": 0.16278761625289917, "contribution_to_active": 0.41022375226020813, "mix_mean": 0.39474087953567505, "mix_low": 0.006022453308105469, "mix_high": 0.004757881164550781, "read_cosine": 0.8644723892211914, "beta_mean": 0.001442412962205708, "nonfinite": 0.0, "weighted_memory_to_active": 0.062212806195020676, "mix_std": 0.2806428372859955, "beta_std": 0.0037442294415086508, "beta_low": 0.97283935546875, "beta_high": 0.0, "orthogonal_to_active": 0.0021702363155782223, "evidence_weights": [-0.1835101991891861, -0.18958182632923126, 0.2831153869628906], "agreement_weight": 0.09444425255060196, "orthogonal_weight": -0.05296267569065094}, {"head": 2, "memory_to_active": 0.004589187912642956, "contribution_to_active": 0.4619692265987396, "mix_mean": 0.34725722670555115, "mix_low": 0.013479232788085938, "mix_high": 0.00013446807861328125, "read_cosine": 0.789438009262085, "beta_mean": 7.362817996181548e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0019898342434316874, "mix_std": 0.2755059003829956, "beta_std": 0.00017577162361703813, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 4.8691872507333755e-05, "evidence_weights": [-0.20872710645198822, -0.2774856388568878, 0.28012752532958984], "agreement_weight": 0.02920849248766899, "orthogonal_weight": -0.045988261699676514}, {"head": 3, "memory_to_active": 0.024397753179073334, "contribution_to_active": 0.4312129616737366, "mix_mean": 0.31835058331489563, "mix_low": 0.00335693359375, "mix_high": 2.765655517578125e-05, "read_cosine": 0.6887575387954712, "beta_mean": 0.0002617543505039066, "nonfinite": 0.0, "weighted_memory_to_active": 0.010395278222858906, "mix_std": 0.2597079277038574, "beta_std": 0.0007177978986874223, "beta_low": 0.9990234375, "beta_high": 0.0, "orthogonal_to_active": 0.007237267214804888, "evidence_weights": [-0.19106292724609375, -0.24080263078212738, 0.4009407162666321], "agreement_weight": 0.01745080016553402, "orthogonal_weight": -0.9394339323043823}, {"head": 4, "memory_to_active": 0.0906594842672348, "contribution_to_active": 0.5904169678688049, "mix_mean": 0.38549643754959106, "mix_low": 0.012415885925292969, "mix_high": 0.0006694793701171875, "read_cosine": 0.6508445739746094, "beta_mean": 0.1735903024673462, "nonfinite": 0.0, "weighted_memory_to_active": 0.048322759568691254, "mix_std": 0.3045252561569214, "beta_std": 0.20593315362930298, "beta_low": 0.19476318359375, "beta_high": 0.0, "orthogonal_to_active": 0.0017182085430249572, "evidence_weights": [-0.2340773046016693, -0.2531904876232147, 0.09467530250549316], "agreement_weight": 0.011567111127078533, "orthogonal_weight": 0.06086974963545799}, {"head": 5, "memory_to_active": 0.005263793282210827, "contribution_to_active": 0.4822591245174408, "mix_mean": 0.5274364948272705, "mix_low": 0.005570411682128906, "mix_high": 0.0071659088134765625, "read_cosine": 0.4956798255443573, "beta_mean": 0.00016761211736593395, "nonfinite": 0.0, "weighted_memory_to_active": 0.0024379121605306864, "mix_std": 0.2829534709453583, "beta_std": 0.00033166579669341445, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.002171899890527129, "evidence_weights": [-0.35399308800697327, -0.3017018139362335, 0.3495562672615051], "agreement_weight": 0.17979693412780762, "orthogonal_weight": 1.038853645324707}]}]}, {"step": 23000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.821188161999089, "memory_to_active": 0.9166960716247559, "contribution_to_active": 0.48610788583755493, "mix_mean": 0.3408423662185669, "mix_low": 0.1614629179239273, "mix_high": 0.006808956619352102, "read_cosine": 0.38858020305633545, "beta_mean": 0.08738484978675842, "nonfinite": 0.0, "weighted_memory_to_active": 0.20868229866027832, "mix_std": 0.2145807296037674, "beta_std": 0.07169494777917862, "beta_low": 0.4797922968864441, "beta_high": 1.8437704056850635e-05, "orthogonal_to_active": 0.04728814959526062, "samples": 4, "scored_tokens": 899, "normal_loss": 3.150678873062134, "normal_token_accuracy": 0.4360400438308716, "state_active": 0.057577744126319885, "state_memory": 0.02061862125992775, "state_peak": 112.55970764160156, "growth_active": 1.1126047372817993, "growth_memory": 1.3816957473754883, "output_influence": 0.3435933589935303, "residual_influence": 0.10910183191299438, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.22590970993042, "no_memory_loss_delta": 0.3035438656806946, "scrambled_loss_delta": 0.19532756507396698, "memory_only_loss_delta": 2.5913939476013184, "no_evidence_loss_delta": 0.1591164916753769, "no_agreement_loss_delta": 0.0011003781110048294, "no_orthogonal_loss_delta": -0.005450287368148565}, {"index": 94, "tokens": 256, "normal_loss": 3.274488687515259, "no_memory_loss_delta": 0.3597625195980072, "scrambled_loss_delta": 0.3870626986026764, "memory_only_loss_delta": 2.367952585220337, "no_evidence_loss_delta": 0.18000759184360504, "no_agreement_loss_delta": 0.026065854355692863, "no_orthogonal_loss_delta": 0.010008015669882298}, {"index": 188, "tokens": 256, "normal_loss": 3.4960217475891113, "no_memory_loss_delta": 0.3387802243232727, "scrambled_loss_delta": 0.2203347533941269, "memory_only_loss_delta": 2.2821216583251953, "no_evidence_loss_delta": 0.1752348393201828, "no_agreement_loss_delta": 0.024952562525868416, "no_orthogonal_loss_delta": 0.010676898062229156}, {"index": 283, "tokens": 131, "normal_loss": 2.0868446826934814, "no_memory_loss_delta": 0.18742208182811737, "scrambled_loss_delta": 0.2823888063430786, "memory_only_loss_delta": 2.5782086849212646, "no_evidence_loss_delta": 0.07803332060575485, "no_agreement_loss_delta": -0.01036289893090725, "no_orthogonal_loss_delta": -0.0013334040995687246}], "agreement_weight_abs_mean": 0.1187264546751976, "agreement_weight_abs_max": 2.443385362625122, "agreement_weight_mean": 0.10184323787689209, "orthogonal_weight_abs_mean": 0.5899294018745422, "orthogonal_weight_abs_max": 4.130016326904297, "orthogonal_weight_mean": -0.16935183107852936, "no_memory_loss": 3.463344097137451, "no_memory_loss_delta": 0.312665730714798, "no_memory_token_accuracy_gain": 0.05339265987277031, "no_memory_token_win_fraction": 0.6451612710952759, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 26.85056121875335, "scrambled_loss": 3.420412540435791, "scrambled_loss_delta": 0.26973363757133484, "scrambled_token_accuracy_gain": 0.030033370479941368, "scrambled_token_win_fraction": 0.6329254508018494, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 23.641714296415323, "memory_only_loss": 5.588454246520996, "memory_only_loss_delta": 2.437776803970337, "memory_only_token_accuracy_gain": 0.2602892220020294, "memory_only_token_win_fraction": 0.8698554039001465, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 91.26451573187539, "no_evidence_loss": 3.308518886566162, "no_evidence_loss_delta": 0.15784010291099548, "no_evidence_token_accuracy_gain": 0.032258063554763794, "no_evidence_token_win_fraction": 0.5917686223983765, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 14.601367902113013, "no_agreement_loss": 3.1640100479125977, "no_agreement_loss_delta": 0.013331337831914425, "no_agreement_token_accuracy_gain": -0.0022246940061450005, "no_agreement_token_win_fraction": 0.44938820600509644, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.3242869119514138, "no_orthogonal_loss": 3.154822587966919, "no_orthogonal_loss_delta": 0.004143923986703157, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5617352724075317, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 0.4135349781406493, "short_no_memory_loss_delta": 0.2647603154182434, "long_context_memory_gain": 0.047905415296554565, "normal_context_loss_gain": 0.13246291875839233, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.13669490814209, "contribution_to_active": 0.9334642887115479, "mix_mean": 0.09756141901016235, "mix_low": 0.5056504011154175, "mix_high": 0.003990014549344778, "read_cosine": 0.4564054012298584, "beta_mean": 0.26695337891578674, "nonfinite": 0.0, "weighted_memory_to_active": 1.0100221633911133, "mix_std": 0.08691945672035217, "beta_std": 0.14879146218299866, "beta_low": 0.003275553695857525, "beta_high": 0.0, "orthogonal_to_active": 0.07235468924045563, "output_influence": 0.1408568024635315, "residual_influence": 0.17392554879188538, "state_active": 0.07542985677719116, "state_memory": 0.05007093399763107, "state_peak": 46.013328552246094, "growth_active": 1.233744740486145, "growth_memory": 1.2932976484298706, "heads": [{"head": 0, "memory_to_active": 2.2956652641296387, "contribution_to_active": 0.35262182354927063, "mix_mean": 0.020690441131591797, "mix_low": 0.541961669921875, "mix_high": 0.0, "read_cosine": 0.2875862419605255, "beta_mean": 0.1994161605834961, "nonfinite": 0.0, "weighted_memory_to_active": 0.4280441105365753, "mix_std": 0.05252460390329361, "beta_std": 0.10749399662017822, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03382445126771927, "evidence_weights": [-0.035404499620199203, -0.07900425046682358, -0.08393724262714386], "agreement_weight": -0.10279293358325958, "orthogonal_weight": -0.08974164724349976}, {"head": 1, "memory_to_active": 4.591551780700684, "contribution_to_active": 1.6784172058105469, "mix_mean": 0.01605989784002304, "mix_low": 0.6465120315551758, "mix_high": 0.0, "read_cosine": 0.5083305835723877, "beta_mean": 0.4211113452911377, "nonfinite": 0.0, "weighted_memory_to_active": 2.079235792160034, "mix_std": 0.04909219965338707, "beta_std": 0.18171706795692444, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.18245024979114532, "evidence_weights": [0.016998326405882835, -0.06671622395515442, 0.23261332511901855], "agreement_weight": 0.02168126590549946, "orthogonal_weight": -0.1945130079984665}, {"head": 2, "memory_to_active": 2.7518718242645264, "contribution_to_active": 0.5636600255966187, "mix_mean": 0.015137976035475731, "mix_low": 0.6372442245483398, "mix_high": 0.0, "read_cosine": 0.30006635189056396, "beta_mean": 0.3520382046699524, "nonfinite": 0.0, "weighted_memory_to_active": 0.6568076610565186, "mix_std": 0.033265121281147, "beta_std": 0.21986766159534454, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07325159013271332, "evidence_weights": [0.04210122302174568, 0.05408856272697449, -0.043588463217020035], "agreement_weight": -0.0009564756182953715, "orthogonal_weight": -0.19099895656108856}, {"head": 3, "memory_to_active": 6.06049919128418, "contribution_to_active": 2.6079905033111572, "mix_mean": 0.045310184359550476, "mix_low": 0.36275482177734375, "mix_high": 0.0, "read_cosine": 0.37389060854911804, "beta_mean": 0.34116846323013306, "nonfinite": 0.0, "weighted_memory_to_active": 2.791191577911377, "mix_std": 0.08893951028585434, "beta_std": 0.20587557554244995, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12311479449272156, "evidence_weights": [-0.07876013964414597, -0.1036972627043724, -0.09080170840024948], "agreement_weight": -0.0134242819622159, "orthogonal_weight": -0.05424618721008301}, {"head": 4, "memory_to_active": 2.9862735271453857, "contribution_to_active": 0.05409088358283043, "mix_mean": 0.0064568715170025826, "mix_low": 0.8236684799194336, "mix_high": 0.0, "read_cosine": 0.333789587020874, "beta_mean": 0.23643025755882263, "nonfinite": 0.0, "weighted_memory_to_active": 0.05065678432583809, "mix_std": 0.012986303307116032, "beta_std": 0.1330856829881668, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006271063350141048, "evidence_weights": [0.09461403638124466, 0.042947009205818176, -0.33125802874565125], "agreement_weight": -0.040033891797065735, "orthogonal_weight": 0.13609056174755096}, {"head": 5, "memory_to_active": 0.1343069076538086, "contribution_to_active": 0.34400513768196106, "mix_mean": 0.4817131459712982, "mix_low": 0.021760940551757812, "mix_high": 0.023940086364746094, "read_cosine": 0.9347690343856812, "beta_mean": 0.05155577138066292, "nonfinite": 0.0, "weighted_memory_to_active": 0.05419673025608063, "mix_std": 0.28470897674560547, "beta_std": 0.04470884054899216, "beta_low": 0.0196533203125, "beta_high": 0.0, "orthogonal_to_active": 0.015215978026390076, "evidence_weights": [-0.273006409406662, -0.25390419363975525, 0.23144939541816711], "agreement_weight": 0.2161862999200821, "orthogonal_weight": -0.5283854007720947}]}, {"layer": 1, "memory_to_active": 2.255462408065796, "contribution_to_active": 0.43614041805267334, "mix_mean": 0.15063466131687164, "mix_low": 0.3869249224662781, "mix_high": 0.00042057037353515625, "read_cosine": 0.3363029956817627, "beta_mean": 0.2065235674381256, "nonfinite": 0.0, "weighted_memory_to_active": 0.34148305654525757, "mix_std": 0.11864693462848663, "beta_std": 0.13365282118320465, "beta_low": 0.1774088591337204, "beta_high": 4.069010537932627e-05, "orthogonal_to_active": 0.04172854870557785, "output_influence": 0.1624716818332672, "residual_influence": 0.03995218500494957, "state_active": 0.04132094234228134, "state_memory": 0.03847504407167435, "state_peak": 23.206132888793945, "growth_active": 1.2053567171096802, "growth_memory": 1.4834614992141724, "heads": [{"head": 0, "memory_to_active": 0.06651508808135986, "contribution_to_active": 0.40272051095962524, "mix_mean": 0.35064584016799927, "mix_low": 0.016126632690429688, "mix_high": 0.00017833709716796875, "read_cosine": 0.021219249814748764, "beta_mean": 0.0004493247251957655, "nonfinite": 0.0, "weighted_memory_to_active": 0.03093857318162918, "mix_std": 0.2719019055366516, "beta_std": 0.0006441050791181624, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.02595050446689129, "evidence_weights": [-0.16491228342056274, -0.20654918253421783, -0.03943248838186264], "agreement_weight": 0.15862610936164856, "orthogonal_weight": -0.9171774387359619}, {"head": 1, "memory_to_active": 3.024913787841797, "contribution_to_active": 0.21421432495117188, "mix_mean": 0.021840138360857964, "mix_low": 0.4944496154785156, "mix_high": 3.814697265625e-06, "read_cosine": 0.445357084274292, "beta_mean": 0.3377688527107239, "nonfinite": 0.0, "weighted_memory_to_active": 0.2338554561138153, "mix_std": 0.05042877420783043, "beta_std": 0.2080940306186676, "beta_low": 0.00018310546875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.00889079924672842, "evidence_weights": [0.007616278249770403, -0.007327946834266186, -0.003284129314124584], "agreement_weight": -0.010161573067307472, "orthogonal_weight": 0.07746711373329163}, {"head": 2, "memory_to_active": 0.48644301295280457, "contribution_to_active": 0.47170114517211914, "mix_mean": 0.47248461842536926, "mix_low": 0.0027980804443359375, "mix_high": 0.002307891845703125, "read_cosine": 0.776473879814148, "beta_mean": 0.04569355025887489, "nonfinite": 0.0, "weighted_memory_to_active": 0.3851560652256012, "mix_std": 0.25851738452911377, "beta_std": 0.0318431481719017, "beta_low": 0.0296630859375, "beta_high": 0.0, "orthogonal_to_active": 0.0353022925555706, "evidence_weights": [-0.25821295380592346, -0.3875925838947296, 0.2544346749782562], "agreement_weight": 0.023632226511836052, "orthogonal_weight": -0.17878694832324982}, {"head": 3, "memory_to_active": 2.844967842102051, "contribution_to_active": 0.07669135928153992, "mix_mean": 0.015120552852749825, "mix_low": 0.7044811248779297, "mix_high": 0.0, "read_cosine": 0.23860594630241394, "beta_mean": 0.37467432022094727, "nonfinite": 0.0, "weighted_memory_to_active": 0.10203036665916443, "mix_std": 0.04315574839711189, "beta_std": 0.24398429691791534, "beta_low": 0.00726318359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.029357973486185074, "evidence_weights": [0.042816367000341415, 0.010660683736205101, 0.1049281656742096], "agreement_weight": -0.0300721637904644, "orthogonal_weight": -0.3088262379169464}, {"head": 4, "memory_to_active": 3.3319501876831055, "contribution_to_active": 1.3786882162094116, "mix_mean": 0.0347573459148407, "mix_low": 0.3102684020996094, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3500649333000183, "beta_mean": 0.20826616883277893, "nonfinite": 0.0, "weighted_memory_to_active": 1.2294926643371582, "mix_std": 0.06178508698940277, "beta_std": 0.11362813413143158, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1475980281829834, "evidence_weights": [-0.02943570166826248, 0.0008803459350019693, 0.02924707904458046], "agreement_weight": -0.030509477481245995, "orthogonal_weight": 0.14497019350528717}, {"head": 5, "memory_to_active": 3.7779836654663086, "contribution_to_active": 0.07282692939043045, "mix_mean": 0.008959491737186909, "mix_low": 0.7934255599975586, "mix_high": 3.24249267578125e-05, "read_cosine": 0.18609672784805298, "beta_mean": 0.2722892761230469, "nonfinite": 0.0, "weighted_memory_to_active": 0.06742499023675919, "mix_std": 0.026092667132616043, "beta_std": 0.2037232220172882, "beta_low": 0.02752685546875, "beta_high": 0.0, "orthogonal_to_active": 0.0032716915011405945, "evidence_weights": [0.13975879549980164, 0.05852873623371124, 0.36250704526901245], "agreement_weight": -0.13499493896961212, "orthogonal_weight": 0.04888010397553444}]}, {"layer": 2, "memory_to_active": 1.1019964218139648, "contribution_to_active": 0.3158692717552185, "mix_mean": 0.30789715051651, "mix_low": 0.2647798955440521, "mix_high": 0.002682844875380397, "read_cosine": 0.33857396245002747, "beta_mean": 0.09978922456502914, "nonfinite": 0.0, "weighted_memory_to_active": 0.07202865183353424, "mix_std": 0.19584627449512482, "beta_std": 0.07708672434091568, "beta_low": 0.543426513671875, "beta_high": 0.00020345053053461015, "orthogonal_to_active": 0.016809090971946716, "output_influence": 0.2899942398071289, "residual_influence": 0.07009348273277283, "state_active": 0.03727537393569946, "state_memory": 0.021120144054293633, "state_peak": 47.30268478393555, "growth_active": 1.0245648622512817, "growth_memory": 1.5801124572753906, "heads": [{"head": 0, "memory_to_active": 0.01029248721897602, "contribution_to_active": 0.2495214343070984, "mix_mean": 0.4309238791465759, "mix_low": 0.005614280700683594, "mix_high": 0.0024366378784179688, "read_cosine": 0.22973644733428955, "beta_mean": 7.919431664049625e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0048992205411195755, "mix_std": 0.29430538415908813, "beta_std": 0.0001025286182994023, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005877462099306285, "evidence_weights": [-0.25936630368232727, -0.16980011761188507, 0.2919257879257202], "agreement_weight": 0.22209163010120392, "orthogonal_weight": -0.13127228617668152}, {"head": 1, "memory_to_active": 2.977240562438965, "contribution_to_active": 0.05211197957396507, "mix_mean": 0.0059490809217095375, "mix_low": 0.8657045364379883, "mix_high": 9.5367431640625e-07, "read_cosine": 0.2163371741771698, "beta_mean": 0.26798856258392334, "nonfinite": 0.0, "weighted_memory_to_active": 0.04175271466374397, "mix_std": 0.014083140529692173, "beta_std": 0.1445450335741043, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.01480862870812416, "evidence_weights": [0.09202730655670166, 0.1017356589436531, 0.3902917206287384], "agreement_weight": 0.05294398218393326, "orthogonal_weight": 0.40620025992393494}, {"head": 2, "memory_to_active": 0.0340576246380806, "contribution_to_active": 0.42681193351745605, "mix_mean": 0.4288331866264343, "mix_low": 0.0026826858520507812, "mix_high": 0.0021505355834960938, "read_cosine": 0.20075389742851257, "beta_mean": 4.970945155946538e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014095159247517586, "mix_std": 0.2776799499988556, "beta_std": 8.466927101835608e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0033410065807402134, "evidence_weights": [-0.21187929809093475, -0.2508460581302643, 0.49240055680274963], "agreement_weight": 0.053300123661756516, "orthogonal_weight": -0.2971833050251007}, {"head": 3, "memory_to_active": 0.23509851098060608, "contribution_to_active": 0.44613155722618103, "mix_mean": 0.41643625497817993, "mix_low": 0.0019674301147460938, "mix_high": 0.005509376525878906, "read_cosine": 0.6774832010269165, "beta_mean": 0.05301636457443237, "nonfinite": 0.0, "weighted_memory_to_active": 0.11658649146556854, "mix_std": 0.2652907073497772, "beta_std": 0.07617475837469101, "beta_low": 0.21527099609375, "beta_high": 0.0, "orthogonal_to_active": 0.06775642931461334, "evidence_weights": [-0.1544647216796875, -0.23793834447860718, 0.8152249455451965], "agreement_weight": 0.06433706730604172, "orthogonal_weight": -0.8536980152130127}, {"head": 4, "memory_to_active": 3.3387064933776855, "contribution_to_active": 0.21477001905441284, "mix_mean": 0.013836669735610485, "mix_low": 0.7106304168701172, "mix_high": 0.0, "read_cosine": 0.19389751553535461, "beta_mean": 0.2775609493255615, "nonfinite": 0.0, "weighted_memory_to_active": 0.24459253251552582, "mix_std": 0.04173582047224045, "beta_std": 0.24154692888259888, "beta_low": 0.04522705078125, "beta_high": 0.001220703125, "orthogonal_to_active": 0.013681460171937943, "evidence_weights": [0.04085194319486618, 0.040438491851091385, 0.1286935657262802], "agreement_weight": 0.03594575077295303, "orthogonal_weight": -0.062310591340065}, {"head": 5, "memory_to_active": 0.01658296212553978, "contribution_to_active": 0.5058687329292297, "mix_mean": 0.5514037013053894, "mix_low": 0.0020799636840820312, "mix_high": 0.005999565124511719, "read_cosine": 0.5132354497909546, "beta_mean": 4.0507216908736154e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01024574227631092, "mix_std": 0.28198257088661194, "beta_std": 6.640986248385161e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006792665226384997, "evidence_weights": [-0.25123950839042664, -0.2741013765335083, 0.7704529166221619], "agreement_weight": 0.04848487675189972, "orthogonal_weight": 0.12055762112140656}]}, {"layer": 3, "memory_to_active": 1.1103694438934326, "contribution_to_active": 0.3715628683567047, "mix_mean": 0.27377596497535706, "mix_low": 0.21133598685264587, "mix_high": 0.004562537185847759, "read_cosine": 0.2608947157859802, "beta_mean": 0.08965481072664261, "nonfinite": 0.0, "weighted_memory_to_active": 0.16416579484939575, "mix_std": 0.22321195900440216, "beta_std": 0.07007023692131042, "beta_low": 0.4318644404411316, "beta_high": 0.0, "orthogonal_to_active": 0.052593328058719635, "output_influence": 0.27530455589294434, "residual_influence": 0.053389765322208405, "state_active": 0.022829994559288025, "state_memory": 0.0158004742115736, "state_peak": 17.34484100341797, "growth_active": 0.9947071075439453, "growth_memory": 1.7327427864074707, "heads": [{"head": 0, "memory_to_active": 0.6100859642028809, "contribution_to_active": 0.44296330213546753, "mix_mean": 0.34848538041114807, "mix_low": 0.046669960021972656, "mix_high": 0.0069789886474609375, "read_cosine": 0.23274561762809753, "beta_mean": 0.01549302227795124, "nonfinite": 0.0, "weighted_memory_to_active": 0.25279349088668823, "mix_std": 0.31630581617355347, "beta_std": 0.020434191450476646, "beta_low": 0.47705078125, "beta_high": 0.0, "orthogonal_to_active": 0.14996211230754852, "evidence_weights": [-0.1351609230041504, -0.18943917751312256, 0.3283618986606598], "agreement_weight": 0.11872430145740509, "orthogonal_weight": -0.6223849058151245}, {"head": 1, "memory_to_active": 0.02270582690834999, "contribution_to_active": 0.3298550248146057, "mix_mean": 0.36106619238853455, "mix_low": 0.02670574188232422, "mix_high": 0.0023622512817382812, "read_cosine": 0.182932049036026, "beta_mean": 0.00010758192365756258, "nonfinite": 0.0, "weighted_memory_to_active": 0.008476058021187782, "mix_std": 0.29898354411125183, "beta_std": 0.00020568506442941725, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002298518782481551, "evidence_weights": [-0.18666493892669678, -0.1574629545211792, 0.43974801898002625], "agreement_weight": 0.24925658106803894, "orthogonal_weight": -0.29545798897743225}, {"head": 2, "memory_to_active": 2.9703636169433594, "contribution_to_active": 0.3255240321159363, "mix_mean": 0.042033493518829346, "mix_low": 0.40244197845458984, "mix_high": 5.7220458984375e-06, "read_cosine": 0.17885759472846985, "beta_mean": 0.24847844243049622, "nonfinite": 0.0, "weighted_memory_to_active": 0.40070775151252747, "mix_std": 0.09252133965492249, "beta_std": 0.16414609551429749, "beta_low": 0.0037841796875, "beta_high": 0.0, "orthogonal_to_active": 0.01586110144853592, "evidence_weights": [-0.05313457176089287, 0.019787417724728584, -0.03428047150373459], "agreement_weight": -0.023372303694486618, "orthogonal_weight": -0.05067835748195648}, {"head": 3, "memory_to_active": 0.03177710250020027, "contribution_to_active": 0.4817621111869812, "mix_mean": 0.4995275139808655, "mix_low": 0.01687908172607422, "mix_high": 0.011674880981445312, "read_cosine": 0.294553279876709, "beta_mean": 0.00012273143511265516, "nonfinite": 0.0, "weighted_memory_to_active": 0.019443664699792862, "mix_std": 0.31585898995399475, "beta_std": 0.00026312959380447865, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005953638814389706, "evidence_weights": [-0.19628140330314636, -0.21526691317558289, 1.6697015762329102], "agreement_weight": 1.010519027709961, "orthogonal_weight": -0.406902551651001}, {"head": 4, "memory_to_active": 2.6560451984405518, "contribution_to_active": 0.029334656894207, "mix_mean": 0.010881882160902023, "mix_low": 0.7496776580810547, "mix_high": 0.0, "read_cosine": 0.21189744770526886, "beta_mean": 0.20634134113788605, "nonfinite": 0.0, "weighted_memory_to_active": 0.059139400720596313, "mix_std": 0.024751191958785057, "beta_std": 0.16258832812309265, "beta_low": 0.00311279296875, "beta_high": 0.0, "orthogonal_to_active": 0.031524598598480225, "evidence_weights": [0.1394999921321869, 0.10368509590625763, -0.03696285933256149], "agreement_weight": -0.03333331644535065, "orthogonal_weight": -0.5497452020645142}, {"head": 5, "memory_to_active": 0.3712391257286072, "contribution_to_active": 0.61993807554245, "mix_mean": 0.3806612491607666, "mix_low": 0.025641441345214844, "mix_high": 0.0063533782958984375, "read_cosine": 0.4643822908401489, "beta_mean": 0.0673857182264328, "nonfinite": 0.0, "weighted_memory_to_active": 0.2444343864917755, "mix_std": 0.29085084795951843, "beta_std": 0.07278401404619217, "beta_low": 0.10723876953125, "beta_high": 0.0, "orthogonal_to_active": 0.10996001958847046, "evidence_weights": [-0.1585264503955841, -0.21379400789737701, 0.3541942238807678], "agreement_weight": 0.1664092093706131, "orthogonal_weight": -0.6356346607208252}]}, {"layer": 4, "memory_to_active": 1.2535223960876465, "contribution_to_active": 0.39588940143585205, "mix_mean": 0.3163619637489319, "mix_low": 0.26437029242515564, "mix_high": 0.003519694320857525, "read_cosine": 0.39330407977104187, "beta_mean": 0.12711256742477417, "nonfinite": 0.0, "weighted_memory_to_active": 0.1538032591342926, "mix_std": 0.15867437422275543, "beta_std": 0.11131742596626282, "beta_low": 0.3575643002986908, "beta_high": 0.0, "orthogonal_to_active": 0.041145212948322296, "output_influence": 0.31558483839035034, "residual_influence": 0.08136912435293198, "state_active": 0.035230472683906555, "state_memory": 0.023190142586827278, "state_peak": 36.49898147583008, "growth_active": 1.0841864347457886, "growth_memory": 1.5579957962036133, "heads": [{"head": 0, "memory_to_active": 0.2892735004425049, "contribution_to_active": 0.657835841178894, "mix_mean": 0.5956305861473083, "mix_low": 0.009734153747558594, "mix_high": 0.005440711975097656, "read_cosine": 0.5124114751815796, "beta_mean": 0.2303752452135086, "nonfinite": 0.0, "weighted_memory_to_active": 0.2218627631664276, "mix_std": 0.32654285430908203, "beta_std": 0.24458840489387512, "beta_low": 0.19207763671875, "beta_high": 0.0, "orthogonal_to_active": 0.05453863367438316, "evidence_weights": [-0.25001952052116394, -0.33244699239730835, 0.41664597392082214], "agreement_weight": 0.019890937954187393, "orthogonal_weight": 0.3144393265247345}, {"head": 1, "memory_to_active": 0.1465350240468979, "contribution_to_active": 0.5053308010101318, "mix_mean": 0.554828941822052, "mix_low": 0.00018310546875, "mix_high": 0.004921913146972656, "read_cosine": 0.7427419424057007, "beta_mean": 0.004275647923350334, "nonfinite": 0.0, "weighted_memory_to_active": 0.0847722515463829, "mix_std": 0.26309484243392944, "beta_std": 0.007966523058712482, "beta_low": 0.9039306640625, "beta_high": 0.0, "orthogonal_to_active": 0.04094785451889038, "evidence_weights": [-0.2338641881942749, -0.26445257663726807, 0.6946023106575012], "agreement_weight": 0.06550972163677216, "orthogonal_weight": -0.76130211353302}, {"head": 2, "memory_to_active": 2.6230247020721436, "contribution_to_active": 0.14887908101081848, "mix_mean": 0.018307942897081375, "mix_low": 0.5567245483398438, "mix_high": 0.0, "read_cosine": 0.28393346071243286, "beta_mean": 0.1504121720790863, "nonfinite": 0.0, "weighted_memory_to_active": 0.1912563443183899, "mix_std": 0.03189214691519737, "beta_std": 0.10371081531047821, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.010802829638123512, "evidence_weights": [0.04392813891172409, 0.020761922001838684, -0.030361324548721313], "agreement_weight": 0.09533809870481491, "orthogonal_weight": -0.08017174899578094}, {"head": 3, "memory_to_active": 1.8232717514038086, "contribution_to_active": 0.12244202941656113, "mix_mean": 0.030403699725866318, "mix_low": 0.3991222381591797, "mix_high": 0.0, "read_cosine": 0.22357586026191711, "beta_mean": 0.14574240148067474, "nonfinite": 0.0, "weighted_memory_to_active": 0.19572021067142487, "mix_std": 0.05482032895088196, "beta_std": 0.13928425312042236, "beta_low": 0.04925537109375, "beta_high": 0.0, "orthogonal_to_active": 0.017771130427718163, "evidence_weights": [-0.017738183960318565, 0.05685427039861679, -0.2849530875682831], "agreement_weight": 0.05199030786752701, "orthogonal_weight": -0.13154461979866028}, {"head": 4, "memory_to_active": 0.018760111182928085, "contribution_to_active": 0.6243759393692017, "mix_mean": 0.6833681464195251, "mix_low": 0.00043582916259765625, "mix_high": 0.010747909545898438, "read_cosine": 0.23974452912807465, "beta_mean": 3.361009294167161e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013388142921030521, "mix_std": 0.2385483980178833, "beta_std": 0.00010357807332184166, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018116862047463655, "evidence_weights": [-0.2963061034679413, -0.33330756425857544, 0.7777646780014038], "agreement_weight": 0.12956903874874115, "orthogonal_weight": -0.1572161763906479}, {"head": 5, "memory_to_active": 2.620269298553467, "contribution_to_active": 0.31647270917892456, "mix_mean": 0.015632392838597298, "mix_low": 0.6200218200683594, "mix_high": 7.62939453125e-06, "read_cosine": 0.3574170768260956, "beta_mean": 0.2318362295627594, "nonfinite": 0.0, "weighted_memory_to_active": 0.2158198058605194, "mix_std": 0.03714759647846222, "beta_std": 0.17225094139575958, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12099914252758026, "evidence_weights": [0.03996984660625458, -0.03718819469213486, -0.1894197165966034], "agreement_weight": -0.03848084807395935, "orthogonal_weight": 0.7547927498817444}]}, {"layer": 5, "memory_to_active": 0.6652082800865173, "contribution_to_active": 0.47822436690330505, "mix_mean": 0.4186999499797821, "mix_low": 0.08816735446453094, "mix_high": 0.0126164760440588, "read_cosine": 0.18699215352535248, "beta_mean": 0.059458523988723755, "nonfinite": 0.0, "weighted_memory_to_active": 0.18533942103385925, "mix_std": 0.2533232271671295, "beta_std": 0.07559029757976532, "beta_low": 0.6154683828353882, "beta_high": 0.0, "orthogonal_to_active": 0.06097205728292465, "output_influence": 0.4631726145744324, "residual_influence": 0.11441420018672943, "state_active": 0.048773132264614105, "state_memory": 0.014157968573272228, "state_peak": 57.4361686706543, "growth_active": 1.1311572790145874, "growth_memory": 1.2823610305786133, "heads": [{"head": 0, "memory_to_active": 1.0966323614120483, "contribution_to_active": 0.3090749979019165, "mix_mean": 0.16139104962348938, "mix_low": 0.1186676025390625, "mix_high": 6.961822509765625e-05, "read_cosine": -0.1486506462097168, "beta_mean": 0.045700397342443466, "nonfinite": 0.0, "weighted_memory_to_active": 0.18175290524959564, "mix_std": 0.20497804880142212, "beta_std": 0.07572940737009048, "beta_low": 0.2744140625, "beta_high": 0.0, "orthogonal_to_active": 0.039752718061208725, "evidence_weights": [-0.060374051332473755, -0.09763219952583313, -0.34639355540275574], "agreement_weight": -0.01722855493426323, "orthogonal_weight": -0.20904701948165894}, {"head": 1, "memory_to_active": 0.03142882511019707, "contribution_to_active": 0.5017881393432617, "mix_mean": 0.5775402784347534, "mix_low": 0.0050868988037109375, "mix_high": 0.015484809875488281, "read_cosine": 0.15725556015968323, "beta_mean": 8.292114216601476e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015309091657400131, "mix_std": 0.3111525774002075, "beta_std": 0.00015589639951940626, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013793379068374634, "evidence_weights": [-0.22972367703914642, -0.2539997398853302, 0.8050006031990051], "agreement_weight": 0.3547529876232147, "orthogonal_weight": -0.9348667860031128}, {"head": 2, "memory_to_active": 0.041972607374191284, "contribution_to_active": 0.5162884593009949, "mix_mean": 0.598823070526123, "mix_low": 0.0016622543334960938, "mix_high": 0.013937950134277344, "read_cosine": 0.2411206066608429, "beta_mean": 0.0008042498957365751, "nonfinite": 0.0, "weighted_memory_to_active": 0.026598472148180008, "mix_std": 0.27487146854400635, "beta_std": 0.0010074109304696321, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.004498491063714027, "evidence_weights": [-0.2564125955104828, -0.33340293169021606, 0.8334290981292725], "agreement_weight": 0.026025863364338875, "orthogonal_weight": -0.21435491740703583}, {"head": 3, "memory_to_active": 2.247480869293213, "contribution_to_active": 0.28414401412010193, "mix_mean": 0.0815698429942131, "mix_low": 0.21326637268066406, "mix_high": 0.0, "read_cosine": 0.14448665082454681, "beta_mean": 0.1016557365655899, "nonfinite": 0.0, "weighted_memory_to_active": 0.5010972023010254, "mix_std": 0.13101345300674438, "beta_std": 0.1410636305809021, "beta_low": 0.190185546875, "beta_high": 0.0, "orthogonal_to_active": 0.16954734921455383, "evidence_weights": [-0.04190152511000633, -0.08361306041479111, -0.07729049772024155], "agreement_weight": 0.04641815274953842, "orthogonal_weight": -0.45267030596733093}, {"head": 4, "memory_to_active": 0.45920705795288086, "contribution_to_active": 0.631384015083313, "mix_mean": 0.3390197157859802, "mix_low": 0.18720149993896484, "mix_high": 0.011373519897460938, "read_cosine": 0.5379748344421387, "beta_mean": 0.20812638103961945, "nonfinite": 0.0, "weighted_memory_to_active": 0.310879111289978, "mix_std": 0.3459499180316925, "beta_std": 0.2337355762720108, "beta_low": 0.2344970703125, "beta_high": 0.0, "orthogonal_to_active": 0.0913047268986702, "evidence_weights": [-0.12368577718734741, -0.29490625858306885, -0.04238871857523918], "agreement_weight": 0.044462770223617554, "orthogonal_weight": 0.42529356479644775}, {"head": 5, "memory_to_active": 0.11452778428792953, "contribution_to_active": 0.6266664266586304, "mix_mean": 0.7538556456565857, "mix_low": 0.0031194686889648438, "mix_high": 0.03483295440673828, "read_cosine": 0.18976590037345886, "beta_mean": 0.00038145040161907673, "nonfinite": 0.0, "weighted_memory_to_active": 0.07639973610639572, "mix_std": 0.2519736588001251, "beta_std": 0.001849845633842051, "beta_low": 0.99462890625, "beta_high": 0.0, "orthogonal_to_active": 0.046935684978961945, "evidence_weights": [-0.3243793249130249, -0.31709784269332886, 1.8763184547424316], "agreement_weight": 0.33234184980392456, "orthogonal_weight": -0.6576101779937744}]}, {"layer": 6, "memory_to_active": 0.38197940587997437, "contribution_to_active": 0.5718425512313843, "mix_mean": 0.5260487794876099, "mix_low": 0.07802629470825195, "mix_high": 0.023454030975699425, "read_cosine": 0.25798189640045166, "beta_mean": 0.074867382645607, "nonfinite": 0.0, "weighted_memory_to_active": 0.16227048635482788, "mix_std": 0.24758857488632202, "beta_std": 0.08503636717796326, "beta_low": 0.5207011103630066, "beta_high": 0.0, "orthogonal_to_active": 0.03596412390470505, "output_influence": 0.4597138464450836, "residual_influence": 0.14430087804794312, "state_active": 0.04803351312875748, "state_memory": 0.00970759242773056, "state_peak": 71.47650146484375, "growth_active": 1.1245249509811401, "growth_memory": 1.1579002141952515, "heads": [{"head": 0, "memory_to_active": 0.1408877819776535, "contribution_to_active": 0.5881905555725098, "mix_mean": 0.6905035972595215, "mix_low": 0.0001773834228515625, "mix_high": 0.0037364959716796875, "read_cosine": -0.10736630111932755, "beta_mean": 0.01006033644080162, "nonfinite": 0.0, "weighted_memory_to_active": 0.09965594112873077, "mix_std": 0.23727072775363922, "beta_std": 0.013135867193341255, "beta_low": 0.6597900390625, "beta_high": 0.0, "orthogonal_to_active": 0.08114384114742279, "evidence_weights": [-0.3075369894504547, -0.31899264454841614, 0.49209854006767273], "agreement_weight": 0.10491227358579636, "orthogonal_weight": -0.9180403351783752}, {"head": 1, "memory_to_active": 0.0035791262052953243, "contribution_to_active": 0.47723352909088135, "mix_mean": 0.4863724112510681, "mix_low": 0.001422882080078125, "mix_high": 0.008719444274902344, "read_cosine": 0.5850655436515808, "beta_mean": 8.507185157213826e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014727925881743431, "mix_std": 0.2917163372039795, "beta_std": 2.2022393750376068e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004468547413125634, "evidence_weights": [-0.2106262743473053, -0.2714094817638397, 0.8292471170425415], "agreement_weight": -0.014709191396832466, "orthogonal_weight": -0.4377986490726471}, {"head": 2, "memory_to_active": 1.217439889907837, "contribution_to_active": 0.06623044610023499, "mix_mean": 0.026869038119912148, "mix_low": 0.4508934020996094, "mix_high": 0.0, "read_cosine": 0.20600955188274384, "beta_mean": 0.07285064458847046, "nonfinite": 0.0, "weighted_memory_to_active": 0.06276785582304001, "mix_std": 0.04897736757993698, "beta_std": 0.07529103010892868, "beta_low": 0.01898193359375, "beta_high": 0.0, "orthogonal_to_active": 0.004804098512977362, "evidence_weights": [0.0904819443821907, 0.011585005559027195, 0.23446953296661377], "agreement_weight": 0.11158504337072372, "orthogonal_weight": -0.08725053071975708}, {"head": 3, "memory_to_active": 0.1537356823682785, "contribution_to_active": 0.6343851089477539, "mix_mean": 0.6193094849586487, "mix_low": 0.013607978820800781, "mix_high": 0.015213966369628906, "read_cosine": 0.46711182594299316, "beta_mean": 0.23393213748931885, "nonfinite": 0.0, "weighted_memory_to_active": 0.10621815174818039, "mix_std": 0.33829328417778015, "beta_std": 0.26627129316329956, "beta_low": 0.21563720703125, "beta_high": 0.0, "orthogonal_to_active": 0.052353549748659134, "evidence_weights": [-0.17492756247520447, -0.3380144536495209, 0.14949965476989746], "agreement_weight": -0.015606583096086979, "orthogonal_weight": 0.5735670328140259}, {"head": 4, "memory_to_active": 0.7287673950195312, "contribution_to_active": 1.079617977142334, "mix_mean": 0.7355417609214783, "mix_low": 0.0015850067138671875, "mix_high": 0.11173343658447266, "read_cosine": 0.1273079514503479, "beta_mean": 0.131972074508667, "nonfinite": 0.0, "weighted_memory_to_active": 0.6717073917388916, "mix_std": 0.29720866680145264, "beta_std": 0.1547541320323944, "beta_low": 0.23065185546875, "beta_high": 0.0, "orthogonal_to_active": 0.04486791044473648, "evidence_weights": [-0.25265246629714966, -0.5804421305656433, -0.10468894988298416], "agreement_weight": -0.016343854367733, "orthogonal_weight": 0.08105985075235367}, {"head": 5, "memory_to_active": 0.047466497868299484, "contribution_to_active": 0.5853976011276245, "mix_mean": 0.5976961851119995, "mix_low": 0.0004711151123046875, "mix_high": 0.0013208389282226562, "read_cosine": 0.2697628140449524, "beta_mean": 0.00038057254278101027, "nonfinite": 0.0, "weighted_memory_to_active": 0.03180071339011192, "mix_std": 0.27206510305404663, "beta_std": 0.0007438177708536386, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.03216848522424698, "evidence_weights": [-0.23757235705852509, -0.29361826181411743, 0.3912895619869232], "agreement_weight": 0.10710402578115463, "orthogonal_weight": -1.0833410024642944}]}, {"layer": 7, "memory_to_active": 1.9059243202209473, "contribution_to_active": 0.19703781604766846, "mix_mean": 0.05482512712478638, "mix_low": 0.4000350832939148, "mix_high": 3.9259593904716894e-05, "read_cosine": 0.22503826022148132, "beta_mean": 0.12750670313835144, "nonfinite": 0.0, "weighted_memory_to_active": 0.2193709909915924, "mix_std": 0.08739932626485825, "beta_std": 0.10926111042499542, "beta_low": 0.0882670134305954, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.052879929542541504, "output_influence": 0.21051649749279022, "residual_influence": 0.04536733403801918, "state_active": 0.01337394118309021, "state_memory": 0.028687190264463425, "state_peak": 26.039936065673828, "growth_active": 0.9719944596290588, "growth_memory": 1.3133597373962402, "heads": [{"head": 0, "memory_to_active": 2.2148115634918213, "contribution_to_active": 0.37469470500946045, "mix_mean": 0.030527591705322266, "mix_low": 0.5039100646972656, "mix_high": 6.67572021484375e-06, "read_cosine": 0.3607728183269501, "beta_mean": 0.16102659702301025, "nonfinite": 0.0, "weighted_memory_to_active": 0.37556084990501404, "mix_std": 0.06652726978063583, "beta_std": 0.12159716337919235, "beta_low": 0.00732421875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.030557168647646904, "evidence_weights": [0.0025527847465127707, -0.08952675014734268, 0.12383930385112762], "agreement_weight": -0.03610753268003464, "orthogonal_weight": 0.12336075305938721}, {"head": 1, "memory_to_active": 1.4242055416107178, "contribution_to_active": 0.32660385966300964, "mix_mean": 0.13946139812469482, "mix_low": 0.2205677032470703, "mix_high": 0.0001583099365234375, "read_cosine": 0.20094168186187744, "beta_mean": 0.13121718168258667, "nonfinite": 0.0, "weighted_memory_to_active": 0.34701478481292725, "mix_std": 0.1798195242881775, "beta_std": 0.0858839750289917, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03158879280090332, "evidence_weights": [-0.16399796307086945, -0.1135006844997406, -0.22893771529197693], "agreement_weight": 0.022985903546214104, "orthogonal_weight": -0.09996888041496277}, {"head": 2, "memory_to_active": 1.1521732807159424, "contribution_to_active": 0.24210140109062195, "mix_mean": 0.07169494777917862, "mix_low": 0.22368717193603516, "mix_high": 7.05718994140625e-05, "read_cosine": 0.1239972785115242, "beta_mean": 0.09949402511119843, "nonfinite": 0.0, "weighted_memory_to_active": 0.17742151021957397, "mix_std": 0.1269342005252838, "beta_std": 0.07080741971731186, "beta_low": 0.024169921875, "beta_high": 0.0, "orthogonal_to_active": 0.030547112226486206, "evidence_weights": [-0.030988400802016258, -0.1169988140463829, 0.2537180781364441], "agreement_weight": 0.021419396623969078, "orthogonal_weight": -0.17702928185462952}, {"head": 3, "memory_to_active": 3.653346300125122, "contribution_to_active": 0.07305809110403061, "mix_mean": 0.028766334056854248, "mix_low": 0.39359378814697266, "mix_high": 0.0, "read_cosine": 0.13127003610134125, "beta_mean": 0.2185182273387909, "nonfinite": 0.0, "weighted_memory_to_active": 0.2303217053413391, "mix_std": 0.046541083604097366, "beta_std": 0.21539904177188873, "beta_low": 0.02630615234375, "beta_high": 0.0, "orthogonal_to_active": 0.18110470473766327, "evidence_weights": [0.07909389585256577, 0.03361906483769417, 0.5346193313598633], "agreement_weight": 0.06772427260875702, "orthogonal_weight": -0.8864678144454956}, {"head": 4, "memory_to_active": 1.8102290630340576, "contribution_to_active": 0.07069840282201767, "mix_mean": 0.011433983221650124, "mix_low": 0.7401561737060547, "mix_high": 0.0, "read_cosine": 0.2991594970226288, "beta_mean": 0.13112911581993103, "nonfinite": 0.0, "weighted_memory_to_active": 0.06215802580118179, "mix_std": 0.028945045545697212, "beta_std": 0.12461221218109131, "beta_low": 0.037353515625, "beta_high": 0.0, "orthogonal_to_active": 0.0033793305046856403, "evidence_weights": [0.12280575931072235, 0.06194926053285599, 0.10650970041751862], "agreement_weight": 0.047686971724033356, "orthogonal_weight": -0.061186809092760086}, {"head": 5, "memory_to_active": 1.1807801723480225, "contribution_to_active": 0.09507051110267639, "mix_mean": 0.04706650972366333, "mix_low": 0.3182954788208008, "mix_high": 0.0, "read_cosine": 0.23408827185630798, "beta_mean": 0.0236549973487854, "nonfinite": 0.0, "weighted_memory_to_active": 0.12374904751777649, "mix_std": 0.07562882453203201, "beta_std": 0.0372668132185936, "beta_low": 0.4344482421875, "beta_high": 0.0, "orthogonal_to_active": 0.040102459490299225, "evidence_weights": [0.033693403005599976, 0.002392925089225173, 0.09196767956018448], "agreement_weight": 0.047467902302742004, "orthogonal_weight": -0.41213780641555786}]}, {"layer": 8, "memory_to_active": 0.5850859880447388, "contribution_to_active": 0.5053244233131409, "mix_mean": 0.39130619168281555, "mix_low": 0.09108369052410126, "mix_high": 0.0007390976534225047, "read_cosine": 0.5254844427108765, "beta_mean": 0.03652700409293175, "nonfinite": 0.0, "weighted_memory_to_active": 0.12992680072784424, "mix_std": 0.22444576025009155, "beta_std": 0.040171340107917786, "beta_low": 0.510894775390625, "beta_high": 0.0, "orthogonal_to_active": 0.06078631430864334, "output_influence": 0.3953015208244324, "residual_influence": 0.11622202396392822, "state_active": 0.053218014538288116, "state_memory": 0.012150093913078308, "state_peak": 31.602035522460938, "growth_active": 1.0703836679458618, "growth_memory": 1.3653004169464111, "heads": [{"head": 0, "memory_to_active": 0.04522530362010002, "contribution_to_active": 0.5259278416633606, "mix_mean": 0.5642161965370178, "mix_low": 0.0002346038818359375, "mix_high": 0.000514984130859375, "read_cosine": 0.6904147863388062, "beta_mean": 0.0036065937019884586, "nonfinite": 0.0, "weighted_memory_to_active": 0.027320239692926407, "mix_std": 0.23959708213806152, "beta_std": 0.008130880072712898, "beta_low": 0.91131591796875, "beta_high": 0.0, "orthogonal_to_active": 0.015539146028459072, "evidence_weights": [-0.18478412926197052, -0.3705563247203827, 0.7285078167915344], "agreement_weight": 0.07349840551614761, "orthogonal_weight": -0.8240675926208496}, {"head": 1, "memory_to_active": 0.38837236166000366, "contribution_to_active": 0.4095945656299591, "mix_mean": 0.4138001501560211, "mix_low": 0.00341033935546875, "mix_high": 0.00032901763916015625, "read_cosine": 0.43068093061447144, "beta_mean": 0.015771059319376945, "nonfinite": 0.0, "weighted_memory_to_active": 0.17919792234897614, "mix_std": 0.25016093254089355, "beta_std": 0.022569797933101654, "beta_low": 0.51416015625, "beta_high": 0.0, "orthogonal_to_active": 0.08802898228168488, "evidence_weights": [-0.18493647873401642, -0.22371193766593933, 0.5733616352081299], "agreement_weight": 0.07574382424354553, "orthogonal_weight": -0.562990128993988}, {"head": 2, "memory_to_active": 0.04612809419631958, "contribution_to_active": 0.6064051985740662, "mix_mean": 0.5618125200271606, "mix_low": 0.0008707046508789062, "mix_high": 0.0024709701538085938, "read_cosine": 0.39077669382095337, "beta_mean": 0.0004167621082160622, "nonfinite": 0.0, "weighted_memory_to_active": 0.03166818618774414, "mix_std": 0.24749965965747833, "beta_std": 0.002877660794183612, "beta_low": 0.998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.022957781329751015, "evidence_weights": [-0.20956163108348846, -0.3224945664405823, 0.5851542949676514], "agreement_weight": 0.12109307199716568, "orthogonal_weight": -0.840917706489563}, {"head": 3, "memory_to_active": 0.1621907353401184, "contribution_to_active": 0.5602395534515381, "mix_mean": 0.4853017330169678, "mix_low": 0.000652313232421875, "mix_high": 0.00021266937255859375, "read_cosine": 0.7768193483352661, "beta_mean": 0.014280409552156925, "nonfinite": 0.0, "weighted_memory_to_active": 0.108095183968544, "mix_std": 0.25211837887763977, "beta_std": 0.012665815651416779, "beta_low": 0.40374755859375, "beta_high": 0.0, "orthogonal_to_active": 0.0311744324862957, "evidence_weights": [-0.15569090843200684, -0.27386540174484253, 0.7282164096832275], "agreement_weight": 0.10714519023895264, "orthogonal_weight": -0.5367034077644348}, {"head": 4, "memory_to_active": 2.6044206619262695, "contribution_to_active": 0.27365225553512573, "mix_mean": 0.025835813954472542, "mix_low": 0.41962242126464844, "mix_high": 0.0, "read_cosine": 0.3217943012714386, "beta_mean": 0.10527951270341873, "nonfinite": 0.0, "weighted_memory_to_active": 0.25888341665267944, "mix_std": 0.0419141910970211, "beta_std": 0.08685655891895294, "beta_low": 0.0078125, "beta_high": 0.0, "orthogonal_to_active": 0.008414817042648792, "evidence_weights": [0.0397496223449707, -0.009811743162572384, -0.1771632432937622], "agreement_weight": 0.02868528664112091, "orthogonal_weight": 0.03360714018344879}, {"head": 5, "memory_to_active": 0.2641788125038147, "contribution_to_active": 0.6561269164085388, "mix_mean": 0.29687076807022095, "mix_low": 0.12171173095703125, "mix_high": 0.0009069442749023438, "read_cosine": 0.5424204468727112, "beta_mean": 0.07980769872665405, "nonfinite": 0.0, "weighted_memory_to_active": 0.1743958443403244, "mix_std": 0.31538426876068115, "beta_std": 0.10792731493711472, "beta_low": 0.22955322265625, "beta_high": 0.0, "orthogonal_to_active": 0.19860270619392395, "evidence_weights": [-0.06218075007200241, -0.3046046495437622, -0.42396649718284607], "agreement_weight": 0.023540496826171875, "orthogonal_weight": 1.408668875694275}]}, {"layer": 9, "memory_to_active": 1.0994824171066284, "contribution_to_active": 0.6243740320205688, "mix_mean": 0.2815443277359009, "mix_low": 0.1649429053068161, "mix_high": 0.0018275579204782844, "read_cosine": 0.3542814254760742, "beta_mean": 0.07996074855327606, "nonfinite": 0.0, "weighted_memory_to_active": 0.36492741107940674, "mix_std": 0.21097303926944733, "beta_std": 0.047729574143886566, "beta_low": 0.4986267387866974, "beta_high": 0.0, "orthogonal_to_active": 0.03787766769528389, "output_influence": 0.4460183084011078, "residual_influence": 0.11801774799823761, "state_active": 0.04205929487943649, "state_memory": 0.022149313241243362, "state_peak": 35.090126037597656, "growth_active": 1.2586393356323242, "growth_memory": 1.4006049633026123, "heads": [{"head": 0, "memory_to_active": 0.005962574388831854, "contribution_to_active": 0.5016058683395386, "mix_mean": 0.5591368675231934, "mix_low": 0.0017538070678710938, "mix_high": 0.007040977478027344, "read_cosine": 0.22778382897377014, "beta_mean": 1.1947860912187025e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034704480785876513, "mix_std": 0.2963096797466278, "beta_std": 5.2627721743192524e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029268073849380016, "evidence_weights": [-0.28096193075180054, -0.2842528820037842, 0.958283007144928], "agreement_weight": 0.07941755652427673, "orthogonal_weight": -0.8881475925445557}, {"head": 1, "memory_to_active": 0.02647188864648342, "contribution_to_active": 0.5601088404655457, "mix_mean": 0.42512276768684387, "mix_low": 0.01202392578125, "mix_high": 0.0009517669677734375, "read_cosine": 0.42342329025268555, "beta_mean": 7.372706022579223e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01514640636742115, "mix_std": 0.2887827754020691, "beta_std": 0.00014111767814029008, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009333400055766106, "evidence_weights": [-0.21104350686073303, -0.2436763495206833, 1.0789995193481445], "agreement_weight": 0.30798056721687317, "orthogonal_weight": -0.7352205514907837}, {"head": 2, "memory_to_active": 0.22335635125637054, "contribution_to_active": 0.5785282850265503, "mix_mean": 0.48793354630470276, "mix_low": 0.004329681396484375, "mix_high": 0.0026340484619140625, "read_cosine": 0.40795183181762695, "beta_mean": 0.001086323638446629, "nonfinite": 0.0, "weighted_memory_to_active": 0.1351093351840973, "mix_std": 0.2849489450454712, "beta_std": 0.005745697300881147, "beta_low": 0.98785400390625, "beta_high": 0.0, "orthogonal_to_active": 0.05197688564658165, "evidence_weights": [-0.17511031031608582, -0.21738016605377197, 0.5444649457931519], "agreement_weight": 0.21927420794963837, "orthogonal_weight": -0.4738435745239258}, {"head": 3, "memory_to_active": 1.6593362092971802, "contribution_to_active": 0.18757197260856628, "mix_mean": 0.03863442316651344, "mix_low": 0.4114818572998047, "mix_high": 8.869171142578125e-05, "read_cosine": 0.4228549599647522, "beta_mean": 0.09980456531047821, "nonfinite": 0.0, "weighted_memory_to_active": 0.1585245579481125, "mix_std": 0.09214644134044647, "beta_std": 0.06691890209913254, "beta_low": 0.00372314453125, "beta_high": 0.0, "orthogonal_to_active": 0.029190147295594215, "evidence_weights": [0.05834515020251274, 0.037999507039785385, -0.12676675617694855], "agreement_weight": 0.07765299826860428, "orthogonal_weight": 0.19912661612033844}, {"head": 4, "memory_to_active": 2.957160234451294, "contribution_to_active": 1.2841386795043945, "mix_mean": 0.11788693815469742, "mix_low": 0.18327045440673828, "mix_high": 0.0002498626708984375, "read_cosine": 0.21871228516101837, "beta_mean": 0.22649693489074707, "nonfinite": 0.0, "weighted_memory_to_active": 1.1760727167129517, "mix_std": 0.18419846892356873, "beta_std": 0.11436456441879272, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01607055962085724, "evidence_weights": [-0.04963860288262367, -0.140425443649292, 0.047849155962467194], "agreement_weight": -0.04786144196987152, "orthogonal_weight": -0.013624140992760658}, {"head": 5, "memory_to_active": 1.7246067523956299, "contribution_to_active": 0.6342905759811401, "mix_mean": 0.06055154651403427, "mix_low": 0.3767976760864258, "mix_high": 0.0, "read_cosine": 0.4249621629714966, "beta_mean": 0.1522909551858902, "nonfinite": 0.0, "weighted_memory_to_active": 0.701241135597229, "mix_std": 0.11945183575153351, "beta_std": 0.0991545245051384, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.11776819825172424, "evidence_weights": [0.07707353681325912, -0.04436474293470383, 0.11618870496749878], "agreement_weight": 0.022067254409193993, "orthogonal_weight": 0.23155806958675385}]}, {"layer": 10, "memory_to_active": 0.1694236397743225, "contribution_to_active": 0.4811021685600281, "mix_mean": 0.46567389369010925, "mix_low": 0.011459192261099815, "mix_high": 0.0116182966157794, "read_cosine": 0.37925946712493896, "beta_mean": 0.04773922264575958, "nonfinite": 0.0, "weighted_memory_to_active": 0.07616078108549118, "mix_std": 0.26740291714668274, "beta_std": 0.05089304596185684, "beta_low": 0.6571044921875, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.05328290909528732, "output_influence": 0.40316328406333923, "residual_influence": 0.13152216374874115, "state_active": 0.05497990548610687, "state_memory": 0.013600107282400131, "state_peak": 112.55970764160156, "growth_active": 1.070286512374878, "growth_memory": 1.074392318725586, "heads": [{"head": 0, "memory_to_active": 0.4061100482940674, "contribution_to_active": 0.8340137004852295, "mix_mean": 0.6454192996025085, "mix_low": 0.019168853759765625, "mix_high": 0.06350231170654297, "read_cosine": 0.18065744638442993, "beta_mean": 0.21716181933879852, "nonfinite": 0.0, "weighted_memory_to_active": 0.28648653626441956, "mix_std": 0.3646886646747589, "beta_std": 0.23352348804473877, "beta_low": 0.10845947265625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.22812169790267944, "evidence_weights": [-0.15820282697677612, -0.3601055443286896, -0.4930037558078766], "agreement_weight": 0.028019890189170837, "orthogonal_weight": 0.846971869468689}, {"head": 1, "memory_to_active": 0.014456890523433685, "contribution_to_active": 0.5523360371589661, "mix_mean": 0.5517721176147461, "mix_low": 0.0010919570922851562, "mix_high": 0.004652976989746094, "read_cosine": 0.262362539768219, "beta_mean": 9.013820090331137e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007751734461635351, "mix_std": 0.2660245895385742, "beta_std": 0.000145936559420079, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009493093937635422, "evidence_weights": [-0.27185407280921936, -0.19663088023662567, 0.5782483816146851], "agreement_weight": 0.368490070104599, "orthogonal_weight": -1.3440515995025635}, {"head": 2, "memory_to_active": 0.10331928730010986, "contribution_to_active": 0.46991539001464844, "mix_mean": 0.4806765019893646, "mix_low": 0.000713348388671875, "mix_high": 0.0012683868408203125, "read_cosine": -0.006537657231092453, "beta_mean": 0.0003694244078360498, "nonfinite": 0.0, "weighted_memory_to_active": 0.03812927007675171, "mix_std": 0.26463645696640015, "beta_std": 0.0016102849040180445, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.014177344739437103, "evidence_weights": [-0.1857316493988037, -0.2338562309741974, 0.5200450420379639], "agreement_weight": -0.05438625440001488, "orthogonal_weight": -0.3598998486995697}, {"head": 3, "memory_to_active": 0.32895246148109436, "contribution_to_active": 0.10790874809026718, "mix_mean": 0.26155027747154236, "mix_low": 0.04359245300292969, "mix_high": 0.000244140625, "read_cosine": 0.8976259827613831, "beta_mean": 0.062380578368902206, "nonfinite": 0.0, "weighted_memory_to_active": 0.0418940894305706, "mix_std": 0.23261334002017975, "beta_std": 0.06041506677865982, "beta_low": 0.00421142578125, "beta_high": 0.0, "orthogonal_to_active": 0.046931445598602295, "evidence_weights": [-0.12466512620449066, -0.1455194056034088, 0.1697753220796585], "agreement_weight": -0.0013487562537193298, "orthogonal_weight": 1.4479844570159912}, {"head": 4, "memory_to_active": 0.018040454015135765, "contribution_to_active": 0.4824831485748291, "mix_mean": 0.48110854625701904, "mix_low": 0.0007333755493164062, "mix_high": 3.910064697265625e-05, "read_cosine": 0.08161546289920807, "beta_mean": 3.3598727895878255e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005868344567716122, "mix_std": 0.2515283226966858, "beta_std": 0.0012896379921585321, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0020147301256656647, "evidence_weights": [-0.1808072179555893, -0.2304231971502304, 0.6125092506408691], "agreement_weight": 0.059719301760196686, "orthogonal_weight": -0.34891659021377563}, {"head": 5, "memory_to_active": 0.14566272497177124, "contribution_to_active": 0.43995606899261475, "mix_mean": 0.37351658940315247, "mix_low": 0.0034551620483398438, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8598331212997437, "beta_mean": 0.00639977864921093, "nonfinite": 0.0, "weighted_memory_to_active": 0.07683469355106354, "mix_std": 0.22492615878582, "beta_std": 0.008373860269784927, "beta_low": 0.8345947265625, "beta_high": 0.0, "orthogonal_to_active": 0.018959123641252518, "evidence_weights": [-0.15500184893608093, -0.16344162821769714, 0.2945728898048401], "agreement_weight": 0.0943634882569313, "orthogonal_weight": -0.5822766423225403}]}, {"layer": 11, "memory_to_active": 0.1866106390953064, "contribution_to_active": 0.5201830267906189, "mix_mean": 0.42327651381492615, "mix_low": 0.06233326718211174, "mix_high": 0.0308914203196764, "read_cosine": 0.46683502197265625, "beta_mean": 0.03697359189391136, "nonfinite": 0.0, "weighted_memory_to_active": 0.0733688622713089, "mix_std": 0.31481313705444336, "beta_std": 0.05471838265657425, "beta_low": 0.5190023183822632, "beta_high": 0.0, "orthogonal_to_active": 0.051893144845962524, "output_influence": 0.4733133614063263, "residual_influence": 0.11887872964143753, "state_active": 0.1333879679441452, "state_memory": 0.009190578013658524, "state_peak": 30.249963760375977, "growth_active": 1.0855973958969116, "growth_memory": 1.4643094539642334, "heads": [{"head": 0, "memory_to_active": 0.013778513297438622, "contribution_to_active": 0.557981014251709, "mix_mean": 0.5791125297546387, "mix_low": 0.0012502670288085938, "mix_high": 0.008420944213867188, "read_cosine": 0.37027156352996826, "beta_mean": 0.00013150542508810759, "nonfinite": 0.0, "weighted_memory_to_active": 0.008990451693534851, "mix_std": 0.27170413732528687, "beta_std": 0.004614327568560839, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.005687263794243336, "evidence_weights": [-0.1611100286245346, -0.35006192326545715, 0.6018850803375244], "agreement_weight": 0.18480032682418823, "orthogonal_weight": -0.7340333461761475}, {"head": 1, "memory_to_active": 0.40247952938079834, "contribution_to_active": 0.2563369870185852, "mix_mean": 0.29054611921310425, "mix_low": 0.020021438598632812, "mix_high": 0.000164031982421875, "read_cosine": 0.612483024597168, "beta_mean": 0.04149189591407776, "nonfinite": 0.0, "weighted_memory_to_active": 0.15439969301223755, "mix_std": 0.24957768619060516, "beta_std": 0.08204378932714462, "beta_low": 0.3792724609375, "beta_high": 0.0, "orthogonal_to_active": 0.029158037155866623, "evidence_weights": [-0.1575089395046234, -0.13173504173755646, 0.2407902032136917], "agreement_weight": 0.08564812690019608, "orthogonal_weight": -0.26054567098617554}, {"head": 2, "memory_to_active": 0.033324360847473145, "contribution_to_active": 0.8970085382461548, "mix_mean": 0.4588889479637146, "mix_low": 0.10770511627197266, "mix_high": 0.09433460235595703, "read_cosine": 0.277378112077713, "beta_mean": 0.070833221077919, "nonfinite": 0.0, "weighted_memory_to_active": 0.02579544112086296, "mix_std": 0.42040887475013733, "beta_std": 0.09674021601676941, "beta_low": 0.44793701171875, "beta_high": 0.0, "orthogonal_to_active": 0.0930679440498352, "evidence_weights": [-0.04303096607327461, -0.4523851275444031, -1.0596259832382202], "agreement_weight": -0.006371167954057455, "orthogonal_weight": 3.8546390533447266}, {"head": 3, "memory_to_active": 0.017659220844507217, "contribution_to_active": 0.7295312881469727, "mix_mean": 0.41368550062179565, "mix_low": 0.22183609008789062, "mix_high": 0.08075618743896484, "read_cosine": 0.5113071203231812, "beta_mean": 0.05806686356663704, "nonfinite": 0.0, "weighted_memory_to_active": 0.009763424284756184, "mix_std": 0.43173035979270935, "beta_std": 0.08648835122585297, "beta_low": 0.485107421875, "beta_high": 0.0, "orthogonal_to_active": 0.03700524941086769, "evidence_weights": [-0.0014066449366509914, -0.3654334843158722, -0.4407711923122406], "agreement_weight": 0.03234858810901642, "orthogonal_weight": 4.130016326904297}, {"head": 4, "memory_to_active": 0.5141918659210205, "contribution_to_active": 0.33044975996017456, "mix_mean": 0.3184284567832947, "mix_low": 0.022966384887695312, "mix_high": 0.0002536773681640625, "read_cosine": 0.19728972017765045, "beta_mean": 0.040233928710222244, "nonfinite": 0.0, "weighted_memory_to_active": 0.1832960546016693, "mix_std": 0.2555753290653229, "beta_std": 0.042615775018930435, "beta_low": 0.14337158203125, "beta_high": 0.0, "orthogonal_to_active": 0.12169728428125381, "evidence_weights": [-0.16739846765995026, -0.13389278948307037, 1.4621880054473877], "agreement_weight": -0.14193002879619598, "orthogonal_weight": -0.6891263723373413}, {"head": 5, "memory_to_active": 0.13823026418685913, "contribution_to_active": 0.34979042410850525, "mix_mean": 0.47899746894836426, "mix_low": 0.00022029876708984375, "mix_high": 0.0014190673828125, "read_cosine": 0.8322806358337402, "beta_mean": 0.011084135621786118, "nonfinite": 0.0, "weighted_memory_to_active": 0.0579681321978569, "mix_std": 0.2598823010921478, "beta_std": 0.01580781862139702, "beta_low": 0.6590576171875, "beta_high": 0.0, "orthogonal_to_active": 0.024743087589740753, "evidence_weights": [-0.20086978375911713, -0.25850680470466614, 0.511861264705658], "agreement_weight": 0.052654363214969635, "orthogonal_weight": -0.6537051796913147}]}, {"layer": 12, "memory_to_active": 0.3225385546684265, "contribution_to_active": 0.5413225293159485, "mix_mean": 0.4592236876487732, "mix_low": 0.015823205932974815, "mix_high": 0.0031860671006143093, "read_cosine": 0.27369117736816406, "beta_mean": 0.016365304589271545, "nonfinite": 0.0, "weighted_memory_to_active": 0.1683068871498108, "mix_std": 0.2561044991016388, "beta_std": 0.01720457896590233, "beta_low": 0.8169556260108948, "beta_high": 0.0, "orthogonal_to_active": 0.06556978821754456, "output_influence": 0.4176904857158661, "residual_influence": 0.11359298229217529, "state_active": 0.04070942848920822, "state_memory": 0.010285061784088612, "state_peak": 27.483539581298828, "growth_active": 1.125151515007019, "growth_memory": 1.2323461771011353, "heads": [{"head": 0, "memory_to_active": 0.10665060579776764, "contribution_to_active": 0.4395716190338135, "mix_mean": 0.4157492518424988, "mix_low": 0.003635406494140625, "mix_high": 0.0009746551513671875, "read_cosine": 0.2899487018585205, "beta_mean": 0.0018833852373063564, "nonfinite": 0.0, "weighted_memory_to_active": 0.04381031543016434, "mix_std": 0.26676106452941895, "beta_std": 0.00762533862143755, "beta_low": 0.9656982421875, "beta_high": 0.0, "orthogonal_to_active": 0.02389506623148918, "evidence_weights": [-0.16614846885204315, -0.23665285110473633, 0.27637964487075806], "agreement_weight": 0.10879500955343246, "orthogonal_weight": -0.5815450549125671}, {"head": 1, "memory_to_active": 0.12260168790817261, "contribution_to_active": 0.4872133731842041, "mix_mean": 0.4981687068939209, "mix_low": 0.0005588531494140625, "mix_high": 0.0002269744873046875, "read_cosine": 0.5641833543777466, "beta_mean": 0.0014119623228907585, "nonfinite": 0.0, "weighted_memory_to_active": 0.061481937766075134, "mix_std": 0.25850558280944824, "beta_std": 0.006151672452688217, "beta_low": 0.97747802734375, "beta_high": 0.0, "orthogonal_to_active": 0.04463513568043709, "evidence_weights": [-0.19839438796043396, -0.25875478982925415, 0.8601701259613037], "agreement_weight": 0.1093677505850792, "orthogonal_weight": -0.9030489921569824}, {"head": 2, "memory_to_active": 0.6419160962104797, "contribution_to_active": 0.4589692950248718, "mix_mean": 0.31345587968826294, "mix_low": 0.026353836059570312, "mix_high": 0.0001983642578125, "read_cosine": 0.4412052631378174, "beta_mean": 0.0925530344247818, "nonfinite": 0.0, "weighted_memory_to_active": 0.258086621761322, "mix_std": 0.2449876070022583, "beta_std": 0.07038113474845886, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.20558789372444153, "evidence_weights": [-0.23772265017032623, -0.03701954334974289, -0.05736667662858963], "agreement_weight": 0.03176183998584747, "orthogonal_weight": 0.9796391725540161}, {"head": 3, "memory_to_active": 0.029678523540496826, "contribution_to_active": 0.7198129296302795, "mix_mean": 0.6836032271385193, "mix_low": 0.0002307891845703125, "mix_high": 0.013686180114746094, "read_cosine": 0.0858994722366333, "beta_mean": 0.0002795198233798146, "nonfinite": 0.0, "weighted_memory_to_active": 0.02134746126830578, "mix_std": 0.2410481572151184, "beta_std": 0.004586249589920044, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.01953960955142975, "evidence_weights": [-0.30162313580513, -0.3293980062007904, 1.113982915878296], "agreement_weight": 0.2639296352863312, "orthogonal_weight": -0.9335581660270691}, {"head": 4, "memory_to_active": 1.0012691020965576, "contribution_to_active": 0.6736168265342712, "mix_mean": 0.2503328323364258, "mix_low": 0.0581207275390625, "mix_high": 0.0016050338745117188, "read_cosine": -0.08056357502937317, "beta_mean": 0.001704291207715869, "nonfinite": 0.0, "weighted_memory_to_active": 0.6058631539344788, "mix_std": 0.25125521421432495, "beta_std": 0.013002181425690651, "beta_low": 0.9627685546875, "beta_high": 0.0, "orthogonal_to_active": 0.08500265330076218, "evidence_weights": [-0.045143261551856995, -0.1894046813249588, 0.014805291779339314], "agreement_weight": -0.00014509909669868648, "orthogonal_weight": -0.15259110927581787}, {"head": 5, "memory_to_active": 0.033115204423666, "contribution_to_active": 0.46875083446502686, "mix_mean": 0.5940321087837219, "mix_low": 0.006039619445800781, "mix_high": 0.0024251937866210938, "read_cosine": 0.34147369861602783, "beta_mean": 0.00035962628317065537, "nonfinite": 0.0, "weighted_memory_to_active": 0.019251780584454536, "mix_std": 0.2740693688392639, "beta_std": 0.0014808825217187405, "beta_low": 0.9971923828125, "beta_high": 0.0, "orthogonal_to_active": 0.014758371748030186, "evidence_weights": [-0.26314082741737366, -0.2998999357223511, 0.9610096216201782], "agreement_weight": 0.17732854187488556, "orthogonal_weight": -0.8407139182090759}]}, {"layer": 13, "memory_to_active": 0.1552509367465973, "contribution_to_active": 0.4958048462867737, "mix_mean": 0.4827277362346649, "mix_low": 0.005886237137019634, "mix_high": 0.0029425621032714844, "read_cosine": 0.410918653011322, "beta_mean": 0.017451558262109756, "nonfinite": 0.0, "weighted_memory_to_active": 0.06527036428451538, "mix_std": 0.2584706246852875, "beta_std": 0.01789267174899578, "beta_low": 0.7851664423942566, "beta_high": 0.0, "orthogonal_to_active": 0.06275708973407745, "output_influence": 0.45073601603507996, "residual_influence": 0.14975062012672424, "state_active": 0.08900263905525208, "state_memory": 0.03707846999168396, "state_peak": 92.42474365234375, "growth_active": 0.9790127277374268, "growth_memory": 1.0374526977539062, "heads": [{"head": 0, "memory_to_active": 0.047857433557510376, "contribution_to_active": 0.5270468592643738, "mix_mean": 0.5183022022247314, "mix_low": 0.0007600784301757812, "mix_high": 0.0014352798461914062, "read_cosine": 0.3825796842575073, "beta_mean": 0.0002015228383243084, "nonfinite": 0.0, "weighted_memory_to_active": 0.02646953985095024, "mix_std": 0.26513224840164185, "beta_std": 0.0004723282181657851, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.025631025433540344, "evidence_weights": [-0.20245951414108276, -0.25201645493507385, 0.7666528820991516], "agreement_weight": 0.06218983232975006, "orthogonal_weight": -1.0692499876022339}, {"head": 1, "memory_to_active": 0.029004812240600586, "contribution_to_active": 0.5564844608306885, "mix_mean": 0.4957846999168396, "mix_low": 0.002685546875, "mix_high": 0.0004520416259765625, "read_cosine": 0.23557712137699127, "beta_mean": 9.996250446420163e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015637746080756187, "mix_std": 0.2665729224681854, "beta_std": 0.00017669743101578206, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015812020748853683, "evidence_weights": [-0.1867963671684265, -0.26181867718696594, 0.6999496817588806], "agreement_weight": 0.1994987428188324, "orthogonal_weight": -1.0648568868637085}, {"head": 2, "memory_to_active": 0.1473385989665985, "contribution_to_active": 0.3744733929634094, "mix_mean": 0.35444146394729614, "mix_low": 0.0026826858520507812, "mix_high": 5.817413330078125e-05, "read_cosine": 0.6814848184585571, "beta_mean": 0.014201054349541664, "nonfinite": 0.0, "weighted_memory_to_active": 0.0587758868932724, "mix_std": 0.21984566748142242, "beta_std": 0.037207357585430145, "beta_low": 0.72601318359375, "beta_high": 0.0, "orthogonal_to_active": 0.02829860709607601, "evidence_weights": [-0.2216023951768875, -0.1665772646665573, 0.4951554834842682], "agreement_weight": 0.032957565039396286, "orthogonal_weight": -0.77080237865448}, {"head": 3, "memory_to_active": 0.14796623587608337, "contribution_to_active": 0.5827423930168152, "mix_mean": 0.5816565155982971, "mix_low": 0.015862464904785156, "mix_high": 0.011295318603515625, "read_cosine": 0.0040773022919893265, "beta_mean": 0.0010232559870928526, "nonfinite": 0.0, "weighted_memory_to_active": 0.10439608991146088, "mix_std": 0.2862575054168701, "beta_std": 0.0036198319867253304, "beta_low": 0.9901123046875, "beta_high": 0.0, "orthogonal_to_active": 0.1037302315235138, "evidence_weights": [-0.3448028266429901, -0.23503991961479187, 2.0326173305511475], "agreement_weight": 2.443385362625122, "orthogonal_weight": -1.0537086725234985}, {"head": 4, "memory_to_active": 0.46139413118362427, "contribution_to_active": 0.2905447483062744, "mix_mean": 0.3732964098453522, "mix_low": 0.010109901428222656, "mix_high": 5.7220458984375e-05, "read_cosine": 0.8615124821662903, "beta_mean": 0.0885467380285263, "nonfinite": 0.0, "weighted_memory_to_active": 0.12684160470962524, "mix_std": 0.2327626645565033, "beta_std": 0.06134392321109772, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1591624617576599, "evidence_weights": [-0.17937441170215607, -0.23422831296920776, 0.5020871162414551], "agreement_weight": -0.00022425893985200673, "orthogonal_weight": 1.8127479553222656}, {"head": 5, "memory_to_active": 0.09794434905052185, "contribution_to_active": 0.643537163734436, "mix_mean": 0.5728850364685059, "mix_low": 0.0032167434692382812, "mix_high": 0.004357337951660156, "read_cosine": 0.30028045177459717, "beta_mean": 0.0006368157919496298, "nonfinite": 0.0, "weighted_memory_to_active": 0.0595012828707695, "mix_std": 0.2802525460720062, "beta_std": 0.004535893443971872, "beta_low": 0.9949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.043908171355724335, "evidence_weights": [-0.2194887101650238, -0.2824104130268097, 0.5977393388748169], "agreement_weight": 0.08366240561008453, "orthogonal_weight": -0.8219260573387146}]}, {"layer": 14, "memory_to_active": 0.2691408395767212, "contribution_to_active": 0.43080660700798035, "mix_mean": 0.4126035273075104, "mix_low": 0.024474939331412315, "mix_high": 0.003935655113309622, "read_cosine": 0.6413842439651489, "beta_mean": 0.07197722792625427, "nonfinite": 0.0, "weighted_memory_to_active": 0.12159096449613571, "mix_std": 0.2488514482975006, "beta_std": 0.0652225911617279, "beta_low": 0.4398905634880066, "beta_high": 0.0, "orthogonal_to_active": 0.04423246532678604, "output_influence": 0.29435139894485474, "residual_influence": 0.11566068977117538, "state_active": 0.06912943720817566, "state_memory": 0.016675125807523727, "state_peak": 28.759838104248047, "growth_active": 1.1461553573608398, "growth_memory": 1.286681056022644, "heads": [{"head": 0, "memory_to_active": 0.3748610019683838, "contribution_to_active": 0.2170511931180954, "mix_mean": 0.2369350790977478, "mix_low": 0.012845039367675781, "mix_high": 2.6702880859375e-05, "read_cosine": 0.7548489570617676, "beta_mean": 0.06941614300012589, "nonfinite": 0.0, "weighted_memory_to_active": 0.10783590376377106, "mix_std": 0.21086756885051727, "beta_std": 0.07418215274810791, "beta_low": 0.03106689453125, "beta_high": 0.0, "orthogonal_to_active": 0.05187302082777023, "evidence_weights": [-0.10460170358419418, -0.0655234158039093, 0.7041979432106018], "agreement_weight": 0.02794419787824154, "orthogonal_weight": 0.7297465801239014}, {"head": 1, "memory_to_active": 0.4785807728767395, "contribution_to_active": 0.38804110884666443, "mix_mean": 0.2540507912635803, "mix_low": 0.11481666564941406, "mix_high": 7.43865966796875e-05, "read_cosine": 0.6946238279342651, "beta_mean": 0.3037782907485962, "nonfinite": 0.0, "weighted_memory_to_active": 0.22348417341709137, "mix_std": 0.267177939414978, "beta_std": 0.2615434229373932, "beta_low": 0.0762939453125, "beta_high": 0.0, "orthogonal_to_active": 0.020680051296949387, "evidence_weights": [0.011257897131145, -0.273029625415802, -0.1636095494031906], "agreement_weight": 0.05255056172609329, "orthogonal_weight": 0.16616834700107574}, {"head": 2, "memory_to_active": 0.22306454181671143, "contribution_to_active": 0.5267937779426575, "mix_mean": 0.53277188539505, "mix_low": 0.0031652450561523438, "mix_high": 0.0019073486328125, "read_cosine": 0.42969244718551636, "beta_mean": 0.006108397617936134, "nonfinite": 0.0, "weighted_memory_to_active": 0.1287560611963272, "mix_std": 0.26899197697639465, "beta_std": 0.007182983681559563, "beta_low": 0.83856201171875, "beta_high": 0.0, "orthogonal_to_active": 0.09345166385173798, "evidence_weights": [-0.17195650935173035, -0.2772063612937927, 0.8303936719894409], "agreement_weight": 0.07699789851903915, "orthogonal_weight": -0.8431169986724854}, {"head": 3, "memory_to_active": 0.06716205179691315, "contribution_to_active": 0.6658945679664612, "mix_mean": 0.672344982624054, "mix_low": 0.0036792755126953125, "mix_high": 0.014278411865234375, "read_cosine": 0.44758689403533936, "beta_mean": 0.009182263165712357, "nonfinite": 0.0, "weighted_memory_to_active": 0.047432441264390945, "mix_std": 0.24341267347335815, "beta_std": 0.009310432709753513, "beta_low": 0.69464111328125, "beta_high": 0.0, "orthogonal_to_active": 0.024882156401872635, "evidence_weights": [-0.3828886151313782, -0.3295692503452301, 0.4373510777950287], "agreement_weight": 0.07939033210277557, "orthogonal_weight": -0.6699568629264832}, {"head": 4, "memory_to_active": 0.17244315147399902, "contribution_to_active": 0.4837971031665802, "mix_mean": 0.4714972972869873, "mix_low": 0.003987312316894531, "mix_high": 0.006683349609375, "read_cosine": 0.7947667241096497, "beta_mean": 0.04090741649270058, "nonfinite": 0.0, "weighted_memory_to_active": 0.09768034517765045, "mix_std": 0.2624693512916565, "beta_std": 0.02947678044438362, "beta_low": 0.03790283203125, "beta_high": 0.0, "orthogonal_to_active": 0.02837172895669937, "evidence_weights": [-0.22030392289161682, -0.26735469698905945, 0.40017813444137573], "agreement_weight": 0.11808955669403076, "orthogonal_weight": -0.5144843459129333}, {"head": 5, "memory_to_active": 0.2987334430217743, "contribution_to_active": 0.303261935710907, "mix_mean": 0.3080211281776428, "mix_low": 0.008356094360351562, "mix_high": 0.0006437301635742188, "read_cosine": 0.7267862558364868, "beta_mean": 0.0024708574637770653, "nonfinite": 0.0, "weighted_memory_to_active": 0.12435683608055115, "mix_std": 0.24018917977809906, "beta_std": 0.009639730677008629, "beta_low": 0.96087646484375, "beta_high": 0.0, "orthogonal_to_active": 0.04613616317510605, "evidence_weights": [-0.18420833349227905, -0.15186192095279694, 0.19238147139549255], "agreement_weight": 0.0892004743218422, "orthogonal_weight": -0.6148709654808044}]}, {"layer": 15, "memory_to_active": 0.06844668090343475, "contribution_to_active": 0.47877711057662964, "mix_mean": 0.3913170099258423, "mix_low": 0.008112907409667969, "mix_high": 0.002517223358154297, "read_cosine": 0.70993572473526, "beta_mean": 0.03929681330919266, "nonfinite": 0.0, "weighted_memory_to_active": 0.03088073804974556, "mix_std": 0.280620276927948, "beta_std": 0.042480625212192535, "beta_low": 0.7110595703125, "beta_high": 0.0, "orthogonal_to_active": 0.005764072760939598, "output_influence": 0.29930418729782104, "residual_influence": 0.1591719090938568, "state_active": 0.11648998409509659, "state_memory": 0.007559716235846281, "state_peak": 37.35710144042969, "growth_active": 1.2962114810943604, "growth_memory": 1.8448113203048706, "heads": [{"head": 0, "memory_to_active": 0.10529164969921112, "contribution_to_active": 0.4589247703552246, "mix_mean": 0.355857253074646, "mix_low": 0.007184028625488281, "mix_high": 0.003749847412109375, "read_cosine": 0.7039375305175781, "beta_mean": 0.03386396914720535, "nonfinite": 0.0, "weighted_memory_to_active": 0.047774311155080795, "mix_std": 0.2707782983779907, "beta_std": 0.023826370015740395, "beta_low": 0.126708984375, "beta_high": 0.0, "orthogonal_to_active": 0.016791630536317825, "evidence_weights": [-0.16359412670135498, -0.18704816699028015, 0.34267568588256836], "agreement_weight": 0.08600307255983353, "orthogonal_weight": -0.5971114039421082}, {"head": 1, "memory_to_active": 0.16213935613632202, "contribution_to_active": 0.41083207726478577, "mix_mean": 0.39833611249923706, "mix_low": 0.004382133483886719, "mix_high": 0.0036382675170898438, "read_cosine": 0.8737088441848755, "beta_mean": 0.0014631279045715928, "nonfinite": 0.0, "weighted_memory_to_active": 0.06382209807634354, "mix_std": 0.27769002318382263, "beta_std": 0.0037321834824979305, "beta_low": 0.97369384765625, "beta_high": 0.0, "orthogonal_to_active": 0.002533107530325651, "evidence_weights": [-0.18307816982269287, -0.1891324669122696, 0.2825319468975067], "agreement_weight": 0.09696105122566223, "orthogonal_weight": -0.06324952840805054}, {"head": 2, "memory_to_active": 0.006104448344558477, "contribution_to_active": 0.4576718509197235, "mix_mean": 0.33485525846481323, "mix_low": 0.015825271606445312, "mix_high": 7.915496826171875e-05, "read_cosine": 0.7965332269668579, "beta_mean": 9.862367005553097e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0027325954288244247, "mix_std": 0.272095650434494, "beta_std": 0.00017992252833209932, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 2.620260784169659e-05, "evidence_weights": [-0.2031542807817459, -0.2736254930496216, 0.32412785291671753], "agreement_weight": 0.019922686740756035, "orthogonal_weight": 0.02066006138920784}, {"head": 3, "memory_to_active": 0.034887343645095825, "contribution_to_active": 0.4661959111690521, "mix_mean": 0.3468635678291321, "mix_low": 0.004260063171386719, "mix_high": 4.9591064453125e-05, "read_cosine": 0.642497718334198, "beta_mean": 0.00046423941967077553, "nonfinite": 0.0, "weighted_memory_to_active": 0.01561930775642395, "mix_std": 0.2719404697418213, "beta_std": 0.001468937611207366, "beta_low": 0.99560546875, "beta_high": 0.0, "orthogonal_to_active": 0.0110549321398139, "evidence_weights": [-0.20141355693340302, -0.251628577709198, 0.4013799726963043], "agreement_weight": 0.029731865972280502, "orthogonal_weight": -0.9726881384849548}, {"head": 4, "memory_to_active": 0.09627142548561096, "contribution_to_active": 0.6061489582061768, "mix_mean": 0.3856210708618164, "mix_low": 0.011368751525878906, "mix_high": 0.0006513595581054688, "read_cosine": 0.7170289754867554, "beta_mean": 0.19970202445983887, "nonfinite": 0.0, "weighted_memory_to_active": 0.052651651203632355, "mix_std": 0.3072446286678314, "beta_std": 0.22531983256340027, "beta_low": 0.17034912109375, "beta_high": 0.0, "orthogonal_to_active": 0.0017744209617376328, "evidence_weights": [-0.23091404139995575, -0.25425392389297485, 0.0840008333325386], "agreement_weight": 0.01062198355793953, "orthogonal_weight": 0.06258703768253326}, {"head": 5, "memory_to_active": 0.0059858267195522785, "contribution_to_active": 0.47288888692855835, "mix_mean": 0.5263687968254089, "mix_low": 0.005657196044921875, "mix_high": 0.00693511962890625, "read_cosine": 0.5259076952934265, "beta_mean": 0.00018886561156250536, "nonfinite": 0.0, "weighted_memory_to_active": 0.0026844607200473547, "mix_std": 0.2839724123477936, "beta_std": 0.00035650806967169046, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024041407741606236, "evidence_weights": [-0.35327816009521484, -0.3034621775150299, 0.3554931581020355], "agreement_weight": 0.20558805763721466, "orthogonal_weight": 1.0569208860397339}]}]}, {"step": 24000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7903488049996668, "memory_to_active": 0.9081566333770752, "contribution_to_active": 0.48289981484413147, "mix_mean": 0.34216398000717163, "mix_low": 0.161629319190979, "mix_high": 0.006752450950443745, "read_cosine": 0.38611268997192383, "beta_mean": 0.08497972041368484, "nonfinite": 0.0, "weighted_memory_to_active": 0.20529131591320038, "mix_std": 0.21460947394371033, "beta_std": 0.07064160704612732, "beta_low": 0.48319754004478455, "beta_high": 1.6530355424038135e-05, "orthogonal_to_active": 0.047691795974969864, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1677732467651367, "normal_token_accuracy": 0.4160178005695343, "state_active": 0.05756908282637596, "state_memory": 0.02043220028281212, "state_peak": 115.53536224365234, "growth_active": 1.1199092864990234, "growth_memory": 1.3903356790542603, "output_influence": 0.3465668857097626, "residual_influence": 0.11003981530666351, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2958648204803467, "no_memory_loss_delta": 0.26052558422088623, "scrambled_loss_delta": 0.15555185079574585, "memory_only_loss_delta": 2.335272789001465, "no_evidence_loss_delta": 0.1573958694934845, "no_agreement_loss_delta": 0.013645877130329609, "no_orthogonal_loss_delta": 0.0184182021766901}, {"index": 94, "tokens": 256, "normal_loss": 3.191760540008545, "no_memory_loss_delta": 0.33329060673713684, "scrambled_loss_delta": 0.39966467022895813, "memory_only_loss_delta": 2.471904993057251, "no_evidence_loss_delta": 0.1481064260005951, "no_agreement_loss_delta": 0.0287691168487072, "no_orthogonal_loss_delta": 0.0233734343200922}, {"index": 188, "tokens": 256, "normal_loss": 3.559014081954956, "no_memory_loss_delta": 0.29595547914505005, "scrambled_loss_delta": 0.13165006041526794, "memory_only_loss_delta": 2.3881006240844727, "no_evidence_loss_delta": 0.18563330173492432, "no_agreement_loss_delta": 0.00866973027586937, "no_orthogonal_loss_delta": 0.027061544358730316}, {"index": 283, "tokens": 131, "normal_loss": 2.1060194969177246, "no_memory_loss_delta": 0.26016706228256226, "scrambled_loss_delta": 0.2334044724702835, "memory_only_loss_delta": 2.6603777408599854, "no_evidence_loss_delta": 0.1169547513127327, "no_agreement_loss_delta": -0.002280224347487092, "no_orthogonal_loss_delta": -0.003217894583940506}], "agreement_weight_abs_mean": 0.12175949662923813, "agreement_weight_abs_max": 2.578113079071045, "agreement_weight_mean": 0.10357073694467545, "orthogonal_weight_abs_mean": 0.6021304130554199, "orthogonal_weight_abs_max": 4.336714267730713, "orthogonal_weight_mean": -0.1653110831975937, "no_memory_loss": 3.4590563774108887, "no_memory_loss_delta": 0.2912830114364624, "no_memory_token_accuracy_gain": 0.036707453429698944, "no_memory_token_win_fraction": 0.6395995616912842, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 25.2695847534236, "scrambled_loss": 3.3973770141601562, "scrambled_loss_delta": 0.2296038120985031, "scrambled_token_accuracy_gain": 0.0055617354810237885, "scrambled_token_win_fraction": 0.5973303914070129, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.515155053086044, "memory_only_loss": 5.604370594024658, "memory_only_loss_delta": 2.4365971088409424, "memory_only_token_accuracy_gain": 0.22246940433979034, "memory_only_token_win_fraction": 0.8809788823127747, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 91.25420444273843, "no_evidence_loss": 3.324671983718872, "no_evidence_loss_delta": 0.15689854323863983, "no_evidence_token_accuracy_gain": 0.023359287530183792, "no_evidence_token_win_fraction": 0.5873192548751831, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 14.520922127749994, "no_agreement_loss": 3.181987762451172, "no_agreement_loss_delta": 0.014214660972356796, "no_agreement_token_accuracy_gain": -0.0033370412420481443, "no_agreement_token_win_fraction": 0.47052279114723206, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.4114109677095195, "no_orthogonal_loss": 3.1869113445281982, "no_orthogonal_loss_delta": 0.019137786701321602, "no_orthogonal_token_accuracy_gain": 0.004449388012290001, "no_orthogonal_token_win_fraction": 0.5439376831054688, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.8955821911419048, "short_no_memory_loss_delta": 0.2670254111289978, "long_context_memory_gain": 0.0242576003074646, "normal_context_loss_gain": 0.12931644916534424, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.0784592628479004, "contribution_to_active": 0.8878878951072693, "mix_mean": 0.09784175455570221, "mix_low": 0.5158777236938477, "mix_high": 0.004370053764432669, "read_cosine": 0.4670398235321045, "beta_mean": 0.25790083408355713, "nonfinite": 0.0, "weighted_memory_to_active": 0.950950026512146, "mix_std": 0.08592154830694199, "beta_std": 0.15138712525367737, "beta_low": 0.0058797202073037624, "beta_high": 0.0, "orthogonal_to_active": 0.06252163648605347, "output_influence": 0.14912843704223633, "residual_influence": 0.18176700174808502, "state_active": 0.07174128293991089, "state_memory": 0.05005428194999695, "state_peak": 46.64951705932617, "growth_active": 1.2007672786712646, "growth_memory": 1.2591859102249146, "heads": [{"head": 0, "memory_to_active": 2.2610487937927246, "contribution_to_active": 0.3463544249534607, "mix_mean": 0.020670650526881218, "mix_low": 0.5681896209716797, "mix_high": 0.0, "read_cosine": 0.2778192162513733, "beta_mean": 0.19577741622924805, "nonfinite": 0.0, "weighted_memory_to_active": 0.41413331031799316, "mix_std": 0.054034117609262466, "beta_std": 0.10250143706798553, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.032426029443740845, "evidence_weights": [-0.03436566889286041, -0.07932784408330917, -0.09570872783660889], "agreement_weight": -0.1007189005613327, "orthogonal_weight": -0.0886855348944664}, {"head": 1, "memory_to_active": 4.576144695281982, "contribution_to_active": 1.6594244241714478, "mix_mean": 0.015881989151239395, "mix_low": 0.6366138458251953, "mix_high": 0.0, "read_cosine": 0.5246943235397339, "beta_mean": 0.4055512845516205, "nonfinite": 0.0, "weighted_memory_to_active": 2.052729845046997, "mix_std": 0.047565534710884094, "beta_std": 0.1798311471939087, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16797217726707458, "evidence_weights": [0.016570597887039185, -0.07065502554178238, 0.25087058544158936], "agreement_weight": 0.024100877344608307, "orthogonal_weight": -0.1841181367635727}, {"head": 2, "memory_to_active": 2.451073169708252, "contribution_to_active": 0.38435691595077515, "mix_mean": 0.014004343189299107, "mix_low": 0.6541299819946289, "mix_high": 0.0, "read_cosine": 0.348065584897995, "beta_mean": 0.3241349458694458, "nonfinite": 0.0, "weighted_memory_to_active": 0.4522905647754669, "mix_std": 0.02917371690273285, "beta_std": 0.21353982388973236, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.048918429762125015, "evidence_weights": [0.04898586496710777, 0.06333683431148529, -0.034745700657367706], "agreement_weight": 0.00032892124727368355, "orthogonal_weight": -0.19140686094760895}, {"head": 3, "memory_to_active": 5.875487327575684, "contribution_to_active": 2.5217318534851074, "mix_mean": 0.04392602667212486, "mix_low": 0.3766670227050781, "mix_high": 0.0, "read_cosine": 0.38399600982666016, "beta_mean": 0.32769936323165894, "nonfinite": 0.0, "weighted_memory_to_active": 2.673804521560669, "mix_std": 0.08767762780189514, "beta_std": 0.22015565633773804, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10465247929096222, "evidence_weights": [-0.07544054090976715, -0.10032239556312561, -0.10462020337581635], "agreement_weight": -0.020034192129969597, "orthogonal_weight": -0.04834577813744545}, {"head": 4, "memory_to_active": 3.1773171424865723, "contribution_to_active": 0.06019708514213562, "mix_mean": 0.005994358099997044, "mix_low": 0.8427162170410156, "mix_high": 0.0, "read_cosine": 0.3257342576980591, "beta_mean": 0.24757744371891022, "nonfinite": 0.0, "weighted_memory_to_active": 0.056754838675260544, "mix_std": 0.011135099455714226, "beta_std": 0.1501057893037796, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006692986935377121, "evidence_weights": [0.09943200647830963, 0.048097845166921616, -0.33965152502059937], "agreement_weight": -0.05015554279088974, "orthogonal_weight": 0.13008204102516174}, {"head": 5, "memory_to_active": 0.1296834796667099, "contribution_to_active": 0.35526230931282043, "mix_mean": 0.48657315969467163, "mix_low": 0.01694965362548828, "mix_high": 0.026220321655273438, "read_cosine": 0.9419295787811279, "beta_mean": 0.04666426032781601, "nonfinite": 0.0, "weighted_memory_to_active": 0.05598725378513336, "mix_std": 0.2859431505203247, "beta_std": 0.04218881204724312, "beta_low": 0.0352783203125, "beta_high": 0.0, "orthogonal_to_active": 0.014467701315879822, "evidence_weights": [-0.2711309492588043, -0.2541453242301941, 0.23299965262413025], "agreement_weight": 0.22395169734954834, "orthogonal_weight": -0.5281552076339722}]}, {"layer": 1, "memory_to_active": 2.1946709156036377, "contribution_to_active": 0.4556733965873718, "mix_mean": 0.1569008231163025, "mix_low": 0.37915948033332825, "mix_high": 0.0005362828960642219, "read_cosine": 0.34462279081344604, "beta_mean": 0.18624690175056458, "nonfinite": 0.0, "weighted_memory_to_active": 0.36736607551574707, "mix_std": 0.1205577552318573, "beta_std": 0.1293678730726242, "beta_low": 0.1754048764705658, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.04440220445394516, "output_influence": 0.16629578173160553, "residual_influence": 0.03999600559473038, "state_active": 0.039859041571617126, "state_memory": 0.037920672446489334, "state_peak": 25.45673370361328, "growth_active": 1.1613242626190186, "growth_memory": 1.5087841749191284, "heads": [{"head": 0, "memory_to_active": 0.11085473001003265, "contribution_to_active": 0.38993287086486816, "mix_mean": 0.3703412115573883, "mix_low": 0.017513275146484375, "mix_high": 0.00020599365234375, "read_cosine": 0.008484050631523132, "beta_mean": 0.0007102073286660016, "nonfinite": 0.0, "weighted_memory_to_active": 0.04692758619785309, "mix_std": 0.2806321382522583, "beta_std": 0.0016885127406567335, "beta_low": 0.9893798828125, "beta_high": 0.0, "orthogonal_to_active": 0.04036646708846092, "evidence_weights": [-0.1664443165063858, -0.21069541573524475, -0.004278894979506731], "agreement_weight": 0.1469704806804657, "orthogonal_weight": -0.9095330238342285}, {"head": 1, "memory_to_active": 2.975064992904663, "contribution_to_active": 0.23706987500190735, "mix_mean": 0.02183285541832447, "mix_low": 0.48166370391845703, "mix_high": 0.0, "read_cosine": 0.4706498086452484, "beta_mean": 0.31816667318344116, "nonfinite": 0.0, "weighted_memory_to_active": 0.269308865070343, "mix_std": 0.045032799243927, "beta_std": 0.20175394415855408, "beta_low": 0.0001220703125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.007401997223496437, "evidence_weights": [0.006709626875817776, -0.008532837964594364, 0.004831693135201931], "agreement_weight": -0.00683084549382329, "orthogonal_weight": 0.06710632145404816}, {"head": 2, "memory_to_active": 0.477459192276001, "contribution_to_active": 0.4801126718521118, "mix_mean": 0.4882410168647766, "mix_low": 0.0027494430541992188, "mix_high": 0.0030107498168945312, "read_cosine": 0.7755025625228882, "beta_mean": 0.04494704678654671, "nonfinite": 0.0, "weighted_memory_to_active": 0.38936251401901245, "mix_std": 0.264816552400589, "beta_std": 0.027644077315926552, "beta_low": 0.021728515625, "beta_high": 0.0, "orthogonal_to_active": 0.038747675716876984, "evidence_weights": [-0.26840198040008545, -0.3911425769329071, 0.2553166449069977], "agreement_weight": 0.024713624268770218, "orthogonal_weight": -0.20248286426067352}, {"head": 3, "memory_to_active": 2.7255539894104004, "contribution_to_active": 0.07830765843391418, "mix_mean": 0.016241371631622314, "mix_low": 0.6903467178344727, "mix_high": 0.0, "read_cosine": 0.22925584018230438, "beta_mean": 0.33098310232162476, "nonfinite": 0.0, "weighted_memory_to_active": 0.10929528623819351, "mix_std": 0.04529048502445221, "beta_std": 0.23545871675014496, "beta_low": 0.01678466796875, "beta_high": 0.0, "orthogonal_to_active": 0.0311402827501297, "evidence_weights": [0.04341213405132294, 0.006886150222271681, 0.10834118723869324], "agreement_weight": -0.01919805072247982, "orthogonal_weight": -0.30941835045814514}, {"head": 4, "memory_to_active": 3.4020516872406006, "contribution_to_active": 1.4880056381225586, "mix_mean": 0.03524436801671982, "mix_low": 0.3128337860107422, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3566817343235016, "beta_mean": 0.20002701878547668, "nonfinite": 0.0, "weighted_memory_to_active": 1.334505319595337, "mix_std": 0.06275119632482529, "beta_std": 0.11291263997554779, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.14535172283649445, "evidence_weights": [-0.02716895379126072, 0.005988762713968754, 0.008613440208137035], "agreement_weight": -0.027477702125906944, "orthogonal_weight": 0.1359555572271347}, {"head": 5, "memory_to_active": 3.477041006088257, "contribution_to_active": 0.06061144173145294, "mix_mean": 0.009504105895757675, "mix_low": 0.7698497772216797, "mix_high": 0.0, "read_cosine": 0.22716277837753296, "beta_mean": 0.2226472944021225, "nonfinite": 0.0, "weighted_memory_to_active": 0.05479691922664642, "mix_std": 0.024823302403092384, "beta_std": 0.19674928486347198, "beta_low": 0.0242919921875, "beta_high": 0.0, "orthogonal_to_active": 0.003405054798349738, "evidence_weights": [0.1386667937040329, 0.057993996888399124, 0.3779803216457367], "agreement_weight": -0.14512157440185547, "orthogonal_weight": 0.0611383318901062}]}, {"layer": 2, "memory_to_active": 1.1228152513504028, "contribution_to_active": 0.32493218779563904, "mix_mean": 0.3175629675388336, "mix_low": 0.2636380195617676, "mix_high": 0.0029754638671875, "read_cosine": 0.3094979226589203, "beta_mean": 0.10059159994125366, "nonfinite": 0.0, "weighted_memory_to_active": 0.0702984482049942, "mix_std": 0.1982433795928955, "beta_std": 0.07483691722154617, "beta_low": 0.5425313711166382, "beta_high": 0.00020345053053461015, "orthogonal_to_active": 0.016251521185040474, "output_influence": 0.30515170097351074, "residual_influence": 0.07296723872423172, "state_active": 0.03719037026166916, "state_memory": 0.020391764119267464, "state_peak": 42.978885650634766, "growth_active": 1.0469040870666504, "growth_memory": 1.6055315732955933, "heads": [{"head": 0, "memory_to_active": 0.012658702209591866, "contribution_to_active": 0.2915974259376526, "mix_mean": 0.4482572078704834, "mix_low": 0.003894805908203125, "mix_high": 0.0032567977905273438, "read_cosine": 0.2119297981262207, "beta_mean": 8.480522956233472e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0063337055034935474, "mix_std": 0.2956317663192749, "beta_std": 0.00010927380935754627, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007792380056343973, "evidence_weights": [-0.2643049955368042, -0.17050394415855408, 0.26804497838020325], "agreement_weight": 0.18820734322071075, "orthogonal_weight": -0.13144339621067047}, {"head": 1, "memory_to_active": 3.115609645843506, "contribution_to_active": 0.04896630346775055, "mix_mean": 0.0055553182028234005, "mix_low": 0.8777484893798828, "mix_high": 0.0, "read_cosine": 0.1826821267604828, "beta_mean": 0.27971044182777405, "nonfinite": 0.0, "weighted_memory_to_active": 0.04054207354784012, "mix_std": 0.013615797273814678, "beta_std": 0.15449006855487823, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.014382750727236271, "evidence_weights": [0.09593120962381363, 0.10291028022766113, 0.41672828793525696], "agreement_weight": 0.04672672599554062, "orthogonal_weight": 0.43291428685188293}, {"head": 2, "memory_to_active": 0.04909614473581314, "contribution_to_active": 0.4226541221141815, "mix_mean": 0.44878673553466797, "mix_low": 0.0027952194213867188, "mix_high": 0.002429962158203125, "read_cosine": 0.14538885653018951, "beta_mean": 6.489681254606694e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.020238785073161125, "mix_std": 0.28167811036109924, "beta_std": 0.0001530129520688206, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0054047927260398865, "evidence_weights": [-0.2156052440404892, -0.25377222895622253, 0.4848574101924896], "agreement_weight": 0.029991820454597473, "orthogonal_weight": -0.30576518177986145}, {"head": 3, "memory_to_active": 0.20164024829864502, "contribution_to_active": 0.4711787700653076, "mix_mean": 0.4226624369621277, "mix_low": 0.00185394287109375, "mix_high": 0.005679130554199219, "read_cosine": 0.6282495260238647, "beta_mean": 0.04348677024245262, "nonfinite": 0.0, "weighted_memory_to_active": 0.1031726747751236, "mix_std": 0.2670852243900299, "beta_std": 0.05358768254518509, "beta_low": 0.21429443359375, "beta_high": 0.0, "orthogonal_to_active": 0.06306109577417374, "evidence_weights": [-0.15860624611377716, -0.2447754591703415, 0.827983558177948], "agreement_weight": 0.06600037217140198, "orthogonal_weight": -0.8601171374320984}, {"head": 4, "memory_to_active": 3.3375720977783203, "contribution_to_active": 0.19310219585895538, "mix_mean": 0.016133511438965797, "mix_low": 0.6934728622436523, "mix_high": 0.0, "read_cosine": 0.2045745551586151, "beta_mean": 0.2801465690135956, "nonfinite": 0.0, "weighted_memory_to_active": 0.23874765634536743, "mix_std": 0.05056533217430115, "beta_std": 0.24060040712356567, "beta_low": 0.04083251953125, "beta_high": 0.001220703125, "orthogonal_to_active": 0.012874014675617218, "evidence_weights": [0.03946234658360481, 0.038339726626873016, 0.12837986648082733], "agreement_weight": 0.03031587414443493, "orthogonal_weight": -0.06513432413339615}, {"head": 5, "memory_to_active": 0.020314274355769157, "contribution_to_active": 0.522094190120697, "mix_mean": 0.5639826059341431, "mix_low": 0.0020627975463867188, "mix_high": 0.0064868927001953125, "read_cosine": 0.4841625690460205, "beta_mean": 5.609816071228124e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012755823321640491, "mix_std": 0.2808839678764343, "beta_std": 8.103880827547982e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001007226062938571, "evidence_weights": [-0.24968935549259186, -0.27257221937179565, 0.7878668308258057], "agreement_weight": 0.040149200707674026, "orthogonal_weight": 0.13622944056987762}]}, {"layer": 3, "memory_to_active": 1.1305996179580688, "contribution_to_active": 0.3753946125507355, "mix_mean": 0.2828131318092346, "mix_low": 0.2082107961177826, "mix_high": 0.004676342010498047, "read_cosine": 0.2400723248720169, "beta_mean": 0.07363171875476837, "nonfinite": 0.0, "weighted_memory_to_active": 0.15926052629947662, "mix_std": 0.22589397430419922, "beta_std": 0.06606646627187729, "beta_low": 0.4445699155330658, "beta_high": 0.0, "orthogonal_to_active": 0.054397933185100555, "output_influence": 0.2861497700214386, "residual_influence": 0.055711716413497925, "state_active": 0.024418629705905914, "state_memory": 0.01548916008323431, "state_peak": 18.5543155670166, "growth_active": 1.0322930812835693, "growth_memory": 1.8600050210952759, "heads": [{"head": 0, "memory_to_active": 0.6053183078765869, "contribution_to_active": 0.43573522567749023, "mix_mean": 0.37005743384361267, "mix_low": 0.03854846954345703, "mix_high": 0.007900238037109375, "read_cosine": 0.24053779244422913, "beta_mean": 0.016460973769426346, "nonfinite": 0.0, "weighted_memory_to_active": 0.2526898980140686, "mix_std": 0.32306864857673645, "beta_std": 0.026414144784212112, "beta_low": 0.51885986328125, "beta_high": 0.0, "orthogonal_to_active": 0.14991608262062073, "evidence_weights": [-0.14071619510650635, -0.1926680952310562, 0.395006000995636], "agreement_weight": 0.12046638876199722, "orthogonal_weight": -0.6298031806945801}, {"head": 1, "memory_to_active": 0.025005318224430084, "contribution_to_active": 0.33563724160194397, "mix_mean": 0.3776710331439972, "mix_low": 0.02933216094970703, "mix_high": 0.0028123855590820312, "read_cosine": 0.19490694999694824, "beta_mean": 0.0001245389721589163, "nonfinite": 0.0, "weighted_memory_to_active": 0.00994260236620903, "mix_std": 0.3057652711868286, "beta_std": 0.00022165453992784023, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003000966040417552, "evidence_weights": [-0.1908900886774063, -0.16154159605503082, 0.4787575900554657], "agreement_weight": 0.26973938941955566, "orthogonal_weight": -0.33055171370506287}, {"head": 2, "memory_to_active": 2.8787713050842285, "contribution_to_active": 0.3236188292503357, "mix_mean": 0.03960568085312843, "mix_low": 0.42447566986083984, "mix_high": 0.0, "read_cosine": 0.1700529009103775, "beta_mean": 0.19266638159751892, "nonfinite": 0.0, "weighted_memory_to_active": 0.3890581727027893, "mix_std": 0.08907150477170944, "beta_std": 0.14405879378318787, "beta_low": 0.00689697265625, "beta_high": 0.0, "orthogonal_to_active": 0.012578546069562435, "evidence_weights": [-0.05605626106262207, 0.016707610338926315, -0.02039758674800396], "agreement_weight": -0.010032509453594685, "orthogonal_weight": -0.040997181087732315}, {"head": 3, "memory_to_active": 0.03585682436823845, "contribution_to_active": 0.4801226854324341, "mix_mean": 0.5039284825325012, "mix_low": 0.018288612365722656, "mix_high": 0.012172698974609375, "read_cosine": 0.2825963497161865, "beta_mean": 0.00013195167412050068, "nonfinite": 0.0, "weighted_memory_to_active": 0.02202034369111061, "mix_std": 0.31534451246261597, "beta_std": 0.00025665067369118333, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006733362562954426, "evidence_weights": [-0.19658784568309784, -0.2151467353105545, 1.7275066375732422], "agreement_weight": 1.0558418035507202, "orthogonal_weight": -0.395588755607605}, {"head": 4, "memory_to_active": 2.907144069671631, "contribution_to_active": 0.03364607319235802, "mix_mean": 0.012255982495844364, "mix_low": 0.7111454010009766, "mix_high": 0.0, "read_cosine": 0.19945356249809265, "beta_mean": 0.16923952102661133, "nonfinite": 0.0, "weighted_memory_to_active": 0.06728990375995636, "mix_std": 0.02617769129574299, "beta_std": 0.15303686261177063, "beta_low": 0.01495361328125, "beta_high": 0.0, "orthogonal_to_active": 0.03721819072961807, "evidence_weights": [0.1280382126569748, 0.09124511480331421, -0.05214029923081398], "agreement_weight": -0.05788725987076759, "orthogonal_weight": -0.5685998797416687}, {"head": 5, "memory_to_active": 0.33150190114974976, "contribution_to_active": 0.6436075568199158, "mix_mean": 0.3933600187301636, "mix_low": 0.027474403381347656, "mix_high": 0.0051727294921875, "read_cosine": 0.3528863489627838, "beta_mean": 0.06316692382097244, "nonfinite": 0.0, "weighted_memory_to_active": 0.21456222236156464, "mix_std": 0.2959362864494324, "beta_std": 0.07241067290306091, "beta_low": 0.126708984375, "beta_high": 0.0, "orthogonal_to_active": 0.116940438747406, "evidence_weights": [-0.16067932546138763, -0.21390187740325928, 0.3696295917034149], "agreement_weight": 0.1583028882741928, "orthogonal_weight": -0.6695119738578796}]}, {"layer": 4, "memory_to_active": 1.2467715740203857, "contribution_to_active": 0.39427450299263, "mix_mean": 0.319286972284317, "mix_low": 0.274655818939209, "mix_high": 0.0037190120201557875, "read_cosine": 0.3830055594444275, "beta_mean": 0.13109302520751953, "nonfinite": 0.0, "weighted_memory_to_active": 0.15189452469348907, "mix_std": 0.15837273001670837, "beta_std": 0.11058890074491501, "beta_low": 0.3662109673023224, "beta_high": 0.0, "orthogonal_to_active": 0.03692244365811348, "output_influence": 0.3256734013557434, "residual_influence": 0.08135418593883514, "state_active": 0.03557479381561279, "state_memory": 0.023339254781603813, "state_peak": 39.96186065673828, "growth_active": 1.131005883216858, "growth_memory": 1.545478105545044, "heads": [{"head": 0, "memory_to_active": 0.29735419154167175, "contribution_to_active": 0.6513778567314148, "mix_mean": 0.5965740084648132, "mix_low": 0.01042938232421875, "mix_high": 0.004329681396484375, "read_cosine": 0.509238600730896, "beta_mean": 0.234156996011734, "nonfinite": 0.0, "weighted_memory_to_active": 0.2217463254928589, "mix_std": 0.3304622769355774, "beta_std": 0.2443649023771286, "beta_low": 0.19622802734375, "beta_high": 0.0, "orthogonal_to_active": 0.05743559077382088, "evidence_weights": [-0.25436919927597046, -0.33938726782798767, 0.38612914085388184], "agreement_weight": 0.01931627094745636, "orthogonal_weight": 0.3218558430671692}, {"head": 1, "memory_to_active": 0.12661179900169373, "contribution_to_active": 0.5224890112876892, "mix_mean": 0.5637644529342651, "mix_low": 0.0001506805419921875, "mix_high": 0.0053424835205078125, "read_cosine": 0.7689225673675537, "beta_mean": 0.0030674815643578768, "nonfinite": 0.0, "weighted_memory_to_active": 0.07534243911504745, "mix_std": 0.2624630331993103, "beta_std": 0.006188757251948118, "beta_low": 0.946533203125, "beta_high": 0.0, "orthogonal_to_active": 0.03543362393975258, "evidence_weights": [-0.2322056144475937, -0.2695261538028717, 0.7027438879013062], "agreement_weight": 0.06624124199151993, "orthogonal_weight": -0.7726878523826599}, {"head": 2, "memory_to_active": 2.6805801391601562, "contribution_to_active": 0.17045044898986816, "mix_mean": 0.017500802874565125, "mix_low": 0.5990943908691406, "mix_high": 0.0, "read_cosine": 0.26530033349990845, "beta_mean": 0.16026683151721954, "nonfinite": 0.0, "weighted_memory_to_active": 0.22001665830612183, "mix_std": 0.033897750079631805, "beta_std": 0.10541031509637833, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012827914208173752, "evidence_weights": [0.05168889835476875, 0.028355877846479416, -0.02575959451496601], "agreement_weight": 0.09088924527168274, "orthogonal_weight": -0.0886005163192749}, {"head": 3, "memory_to_active": 1.7477712631225586, "contribution_to_active": 0.1241258978843689, "mix_mean": 0.028869831934571266, "mix_low": 0.40545654296875, "mix_high": 0.0, "read_cosine": 0.18647730350494385, "beta_mean": 0.14711220562458038, "nonfinite": 0.0, "weighted_memory_to_active": 0.18867096304893494, "mix_std": 0.05009261891245842, "beta_std": 0.1362311691045761, "beta_low": 0.05450439453125, "beta_high": 0.0, "orthogonal_to_active": 0.017390305176377296, "evidence_weights": [-0.020628049969673157, 0.05528518185019493, -0.27839186787605286], "agreement_weight": 0.044471949338912964, "orthogonal_weight": -0.1312776803970337}, {"head": 4, "memory_to_active": 0.028315991163253784, "contribution_to_active": 0.6291794776916504, "mix_mean": 0.6940702199935913, "mix_low": 0.00047588348388671875, "mix_high": 0.012636184692382812, "read_cosine": 0.19928810000419617, "beta_mean": 4.708912456408143e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.021162133663892746, "mix_std": 0.23828430473804474, "beta_std": 0.0001346102508250624, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003169753821566701, "evidence_weights": [-0.29908695816993713, -0.33338645100593567, 0.7736042737960815], "agreement_weight": 0.1568991094827652, "orthogonal_weight": -0.16955915093421936}, {"head": 5, "memory_to_active": 2.5999953746795654, "contribution_to_active": 0.26802411675453186, "mix_mean": 0.014942415058612823, "mix_low": 0.6323280334472656, "mix_high": 5.7220458984375e-06, "read_cosine": 0.3688063323497772, "beta_mean": 0.24190747737884521, "nonfinite": 0.0, "weighted_memory_to_active": 0.1844286024570465, "mix_std": 0.03503639996051788, "beta_std": 0.17120367288589478, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.09527747333049774, "evidence_weights": [0.04324081540107727, -0.034547001123428345, -0.20000439882278442], "agreement_weight": -0.04464219510555267, "orthogonal_weight": 0.7643772959709167}]}, {"layer": 5, "memory_to_active": 0.6265496015548706, "contribution_to_active": 0.46938398480415344, "mix_mean": 0.4213920831680298, "mix_low": 0.08241304010152817, "mix_high": 0.011633237823843956, "read_cosine": 0.1938420981168747, "beta_mean": 0.0552995428442955, "nonfinite": 0.0, "weighted_memory_to_active": 0.17189931869506836, "mix_std": 0.2524440288543701, "beta_std": 0.07088169455528259, "beta_low": 0.6168111562728882, "beta_high": 0.0, "orthogonal_to_active": 0.05935455113649368, "output_influence": 0.4593750834465027, "residual_influence": 0.11205258965492249, "state_active": 0.04940267652273178, "state_memory": 0.013282439671456814, "state_peak": 63.21024703979492, "growth_active": 1.1418768167495728, "growth_memory": 1.2657302618026733, "heads": [{"head": 0, "memory_to_active": 1.024489402770996, "contribution_to_active": 0.32448700070381165, "mix_mean": 0.16975843906402588, "mix_low": 0.10542964935302734, "mix_high": 7.915496826171875e-05, "read_cosine": -0.08718404173851013, "beta_mean": 0.049000032246112823, "nonfinite": 0.0, "weighted_memory_to_active": 0.1691068708896637, "mix_std": 0.20872879028320312, "beta_std": 0.07536271214485168, "beta_low": 0.2574462890625, "beta_high": 0.0, "orthogonal_to_active": 0.03635219857096672, "evidence_weights": [-0.0646020695567131, -0.10121674835681915, -0.3563385009765625], "agreement_weight": -0.01584058813750744, "orthogonal_weight": -0.2078322321176529}, {"head": 1, "memory_to_active": 0.03620095178484917, "contribution_to_active": 0.4960463047027588, "mix_mean": 0.5729773044586182, "mix_low": 0.0044765472412109375, "mix_high": 0.01519012451171875, "read_cosine": 0.13672354817390442, "beta_mean": 8.003362017916515e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01867828704416752, "mix_std": 0.3080155849456787, "beta_std": 0.00016187669825740159, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.016787782311439514, "evidence_weights": [-0.23078808188438416, -0.25658103823661804, 0.8167082667350769], "agreement_weight": 0.36516273021698, "orthogonal_weight": -0.9328959584236145}, {"head": 2, "memory_to_active": 0.04001965746283531, "contribution_to_active": 0.5297594666481018, "mix_mean": 0.6068935394287109, "mix_low": 0.0020856857299804688, "mix_high": 0.015117645263671875, "read_cosine": 0.23757576942443848, "beta_mean": 0.000861077569425106, "nonfinite": 0.0, "weighted_memory_to_active": 0.026870843023061752, "mix_std": 0.2742607891559601, "beta_std": 0.0013126758858561516, "beta_low": 0.99749755859375, "beta_high": 0.0, "orthogonal_to_active": 0.004968482535332441, "evidence_weights": [-0.2579009532928467, -0.338152676820755, 0.8311762809753418], "agreement_weight": 0.021798545494675636, "orthogonal_weight": -0.23744863271713257}, {"head": 3, "memory_to_active": 2.0593695640563965, "contribution_to_active": 0.2476535141468048, "mix_mean": 0.08378337323665619, "mix_low": 0.19716835021972656, "mix_high": 0.0, "read_cosine": 0.13569149374961853, "beta_mean": 0.08601001650094986, "nonfinite": 0.0, "weighted_memory_to_active": 0.41086500883102417, "mix_std": 0.1290563941001892, "beta_std": 0.12441930919885635, "beta_low": 0.20513916015625, "beta_high": 0.0, "orthogonal_to_active": 0.1507730484008789, "evidence_weights": [-0.042586494237184525, -0.08442657440900803, -0.08841390162706375], "agreement_weight": 0.05272112786769867, "orthogonal_weight": -0.45747220516204834}, {"head": 4, "memory_to_active": 0.4758187234401703, "contribution_to_active": 0.6066593527793884, "mix_mean": 0.34364527463912964, "mix_low": 0.1824474334716797, "mix_high": 0.009156227111816406, "read_cosine": 0.55197674036026, "beta_mean": 0.1954168826341629, "nonfinite": 0.0, "weighted_memory_to_active": 0.3226316273212433, "mix_std": 0.3464326560497284, "beta_std": 0.22182880342006683, "beta_low": 0.2476806640625, "beta_high": 0.0, "orthogonal_to_active": 0.09473584592342377, "evidence_weights": [-0.11886423081159592, -0.2935229241847992, -0.043358489871025085], "agreement_weight": 0.041818153113126755, "orthogonal_weight": 0.4271094501018524}, {"head": 5, "memory_to_active": 0.12339944392442703, "contribution_to_active": 0.6116981506347656, "mix_mean": 0.7512944340705872, "mix_low": 0.0028705596923828125, "mix_high": 0.030256271362304688, "read_cosine": 0.18826904892921448, "beta_mean": 0.00042921025305986404, "nonfinite": 0.0, "weighted_memory_to_active": 0.08324319124221802, "mix_std": 0.2481699138879776, "beta_std": 0.0022047844249755144, "beta_low": 0.99310302734375, "beta_high": 0.0, "orthogonal_to_active": 0.05250992253422737, "evidence_weights": [-0.3244682252407074, -0.3165704607963562, 1.88217294216156], "agreement_weight": 0.3345779776573181, "orthogonal_weight": -0.6745996475219727}]}, {"layer": 6, "memory_to_active": 0.38020753860473633, "contribution_to_active": 0.5785871744155884, "mix_mean": 0.5310081839561462, "mix_low": 0.07498423755168915, "mix_high": 0.022052448242902756, "read_cosine": 0.283303439617157, "beta_mean": 0.07470351457595825, "nonfinite": 0.0, "weighted_memory_to_active": 0.17932265996932983, "mix_std": 0.24465787410736084, "beta_std": 0.08422350883483887, "beta_low": 0.5130208134651184, "beta_high": 0.0, "orthogonal_to_active": 0.04183864966034889, "output_influence": 0.4668695330619812, "residual_influence": 0.147459015250206, "state_active": 0.048246659338474274, "state_memory": 0.009644853882491589, "state_peak": 86.6934585571289, "growth_active": 1.2141097784042358, "growth_memory": 1.1725784540176392, "heads": [{"head": 0, "memory_to_active": 0.18895317614078522, "contribution_to_active": 0.597743034362793, "mix_mean": 0.7046581506729126, "mix_low": 0.00019359588623046875, "mix_high": 0.004889488220214844, "read_cosine": -0.019807398319244385, "beta_mean": 0.010923240333795547, "nonfinite": 0.0, "weighted_memory_to_active": 0.14441350102424622, "mix_std": 0.2320166975259781, "beta_std": 0.014739410020411015, "beta_low": 0.6322021484375, "beta_high": 0.0, "orthogonal_to_active": 0.11544418334960938, "evidence_weights": [-0.30897390842437744, -0.3236435651779175, 0.5005881190299988], "agreement_weight": 0.11031395941972733, "orthogonal_weight": -0.9180445671081543}, {"head": 1, "memory_to_active": 0.0028652173932641745, "contribution_to_active": 0.4680337905883789, "mix_mean": 0.4925365447998047, "mix_low": 0.0011377334594726562, "mix_high": 0.008608818054199219, "read_cosine": 0.5847468972206116, "beta_mean": 7.2750822255329695e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.001242176629602909, "mix_std": 0.29091858863830566, "beta_std": 1.7515172658022493e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003825027961283922, "evidence_weights": [-0.2074929177761078, -0.27085769176483154, 0.8296023011207581], "agreement_weight": -0.029728997498750687, "orthogonal_weight": -0.450630784034729}, {"head": 2, "memory_to_active": 1.1029832363128662, "contribution_to_active": 0.06923764944076538, "mix_mean": 0.027702022343873978, "mix_low": 0.4341592788696289, "mix_high": 8.58306884765625e-06, "read_cosine": 0.22153276205062866, "beta_mean": 0.05865424871444702, "nonfinite": 0.0, "weighted_memory_to_active": 0.0618693083524704, "mix_std": 0.04853837564587593, "beta_std": 0.0643153041601181, "beta_low": 0.02984619140625, "beta_high": 0.0, "orthogonal_to_active": 0.004006215836852789, "evidence_weights": [0.08585051447153091, 0.0052120620384812355, 0.22117970883846283], "agreement_weight": 0.12319070845842361, "orthogonal_weight": -0.07564586400985718}, {"head": 3, "memory_to_active": 0.15691041946411133, "contribution_to_active": 0.639343798160553, "mix_mean": 0.6227945685386658, "mix_low": 0.012613296508789062, "mix_high": 0.015979766845703125, "read_cosine": 0.5015488266944885, "beta_mean": 0.24172303080558777, "nonfinite": 0.0, "weighted_memory_to_active": 0.10992111265659332, "mix_std": 0.3356831669807434, "beta_std": 0.26775476336479187, "beta_low": 0.18829345703125, "beta_high": 0.0, "orthogonal_to_active": 0.05386616289615631, "evidence_weights": [-0.17334403097629547, -0.33722788095474243, 0.14505179226398468], "agreement_weight": -0.021447159349918365, "orthogonal_weight": 0.5851119756698608}, {"head": 4, "memory_to_active": 0.7803800702095032, "contribution_to_active": 1.1137127876281738, "mix_mean": 0.7347993850708008, "mix_low": 0.0013666152954101562, "mix_high": 0.10151100158691406, "read_cosine": 0.1378699243068695, "beta_mean": 0.13653156161308289, "nonfinite": 0.0, "weighted_memory_to_active": 0.7262950539588928, "mix_std": 0.29456061124801636, "beta_std": 0.15769065916538239, "beta_low": 0.2286376953125, "beta_high": 0.0, "orthogonal_to_active": 0.04483819752931595, "evidence_weights": [-0.2545093595981598, -0.583030104637146, -0.1033894270658493], "agreement_weight": -0.012829073704779148, "orthogonal_weight": 0.07459073513746262}, {"head": 5, "memory_to_active": 0.04915289953351021, "contribution_to_active": 0.5834519863128662, "mix_mean": 0.6035583019256592, "mix_low": 0.00043487548828125, "mix_high": 0.0013170242309570312, "read_cosine": 0.273929625749588, "beta_mean": 0.00038171507185325027, "nonfinite": 0.0, "weighted_memory_to_active": 0.03219471126794815, "mix_std": 0.2662298083305359, "beta_std": 0.0008233390399254858, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.03249462693929672, "evidence_weights": [-0.2379683554172516, -0.29445528984069824, 0.3707919120788574], "agreement_weight": 0.10299716144800186, "orthogonal_weight": -1.0794334411621094}]}, {"layer": 7, "memory_to_active": 1.8727679252624512, "contribution_to_active": 0.18764954805374146, "mix_mean": 0.05560697615146637, "mix_low": 0.39325129985809326, "mix_high": 3.24249267578125e-05, "read_cosine": 0.23546773195266724, "beta_mean": 0.13487249612808228, "nonfinite": 0.0, "weighted_memory_to_active": 0.20728595554828644, "mix_std": 0.0878615528345108, "beta_std": 0.10945998132228851, "beta_low": 0.1024373471736908, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.053243525326251984, "output_influence": 0.21698340773582458, "residual_influence": 0.047047846019268036, "state_active": 0.014907998964190483, "state_memory": 0.029342399910092354, "state_peak": 27.398052215576172, "growth_active": 1.044825792312622, "growth_memory": 1.3280080556869507, "heads": [{"head": 0, "memory_to_active": 2.233790874481201, "contribution_to_active": 0.32077929377555847, "mix_mean": 0.028561856597661972, "mix_low": 0.5113954544067383, "mix_high": 6.67572021484375e-06, "read_cosine": 0.36719056963920593, "beta_mean": 0.16576282680034637, "nonfinite": 0.0, "weighted_memory_to_active": 0.3040577471256256, "mix_std": 0.06331069767475128, "beta_std": 0.11833295226097107, "beta_low": 0.001708984375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.02637888863682747, "evidence_weights": [0.005106959491968155, -0.08905786275863647, 0.11503110826015472], "agreement_weight": -0.048747655004262924, "orthogonal_weight": 0.11686449497938156}, {"head": 1, "memory_to_active": 1.318086862564087, "contribution_to_active": 0.31098395586013794, "mix_mean": 0.13860763609409332, "mix_low": 0.222320556640625, "mix_high": 0.00014972686767578125, "read_cosine": 0.23840108513832092, "beta_mean": 0.1435527801513672, "nonfinite": 0.0, "weighted_memory_to_active": 0.31755754351615906, "mix_std": 0.17846623063087463, "beta_std": 0.08854179084300995, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02348676696419716, "evidence_weights": [-0.1679743230342865, -0.11451825499534607, -0.24257732927799225], "agreement_weight": 0.014054029248654842, "orthogonal_weight": -0.08248161524534225}, {"head": 2, "memory_to_active": 1.0357589721679688, "contribution_to_active": 0.2451428472995758, "mix_mean": 0.07075203210115433, "mix_low": 0.21972370147705078, "mix_high": 3.814697265625e-05, "read_cosine": 0.14410772919654846, "beta_mean": 0.08847348392009735, "nonfinite": 0.0, "weighted_memory_to_active": 0.17685739696025848, "mix_std": 0.12149795144796371, "beta_std": 0.06350581347942352, "beta_low": 0.03375244140625, "beta_high": 0.0, "orthogonal_to_active": 0.03032778762280941, "evidence_weights": [-0.03085949644446373, -0.1167566105723381, 0.2577180564403534], "agreement_weight": 0.020360060036182404, "orthogonal_weight": -0.17781613767147064}, {"head": 3, "memory_to_active": 3.796692371368408, "contribution_to_active": 0.08316121995449066, "mix_mean": 0.028462836518883705, "mix_low": 0.4014148712158203, "mix_high": 0.0, "read_cosine": 0.10759542882442474, "beta_mean": 0.24782413244247437, "nonfinite": 0.0, "weighted_memory_to_active": 0.24996864795684814, "mix_std": 0.04549828916788101, "beta_std": 0.22993583977222443, "beta_low": 0.0247802734375, "beta_high": 0.0, "orthogonal_to_active": 0.1950693577528, "evidence_weights": [0.08015462011098862, 0.030261922627687454, 0.5588828921318054], "agreement_weight": 0.06924442201852798, "orthogonal_weight": -0.9023138284683228}, {"head": 4, "memory_to_active": 1.8915085792541504, "contribution_to_active": 0.07118286937475204, "mix_mean": 0.012500422075390816, "mix_low": 0.7097663879394531, "mix_high": 0.0, "read_cosine": 0.3148643374443054, "beta_mean": 0.14729028940200806, "nonfinite": 0.0, "weighted_memory_to_active": 0.07893341779708862, "mix_std": 0.030212679877877235, "beta_std": 0.13184434175491333, "beta_low": 0.0269775390625, "beta_high": 0.0, "orthogonal_to_active": 0.002777455374598503, "evidence_weights": [0.12213651835918427, 0.05894213914871216, 0.11968342214822769], "agreement_weight": 0.057002536952495575, "orthogonal_weight": -0.050769563764333725}, {"head": 5, "memory_to_active": 0.9607691764831543, "contribution_to_active": 0.09464707970619202, "mix_mean": 0.05475705489516258, "mix_low": 0.29488658905029297, "mix_high": 0.0, "read_cosine": 0.24064719676971436, "beta_mean": 0.01633143611252308, "nonfinite": 0.0, "weighted_memory_to_active": 0.11634091287851334, "mix_std": 0.0881834402680397, "beta_std": 0.024599095806479454, "beta_low": 0.52740478515625, "beta_high": 0.0, "orthogonal_to_active": 0.04142088443040848, "evidence_weights": [0.027817586436867714, -0.006764601916074753, 0.10947686433792114], "agreement_weight": 0.045382432639598846, "orthogonal_weight": -0.46073973178863525}]}, {"layer": 8, "memory_to_active": 0.5628103017807007, "contribution_to_active": 0.5025169849395752, "mix_mean": 0.3899182081222534, "mix_low": 0.09691159427165985, "mix_high": 0.0006918907165527344, "read_cosine": 0.5259723663330078, "beta_mean": 0.03722022846341133, "nonfinite": 0.0, "weighted_memory_to_active": 0.12274988740682602, "mix_std": 0.22463233768939972, "beta_std": 0.04047100618481636, "beta_low": 0.53424072265625, "beta_high": 0.0, "orthogonal_to_active": 0.057738177478313446, "output_influence": 0.3914230763912201, "residual_influence": 0.11384518444538116, "state_active": 0.05440570041537285, "state_memory": 0.01205849926918745, "state_peak": 31.498022079467773, "growth_active": 1.079787015914917, "growth_memory": 1.3653818368911743, "heads": [{"head": 0, "memory_to_active": 0.04746079444885254, "contribution_to_active": 0.5268614888191223, "mix_mean": 0.5604144334793091, "mix_low": 0.0002613067626953125, "mix_high": 0.0004482269287109375, "read_cosine": 0.6802816390991211, "beta_mean": 0.003630208782851696, "nonfinite": 0.0, "weighted_memory_to_active": 0.029324917122721672, "mix_std": 0.24024254083633423, "beta_std": 0.009128166362643242, "beta_low": 0.9091796875, "beta_high": 0.0, "orthogonal_to_active": 0.016485515981912613, "evidence_weights": [-0.18250688910484314, -0.3753660023212433, 0.7265562415122986], "agreement_weight": 0.07207546383142471, "orthogonal_weight": -0.8187485337257385}, {"head": 1, "memory_to_active": 0.3134194016456604, "contribution_to_active": 0.4205135703086853, "mix_mean": 0.41261130571365356, "mix_low": 0.0033817291259765625, "mix_high": 0.000446319580078125, "read_cosine": 0.422574520111084, "beta_mean": 0.012182318605482578, "nonfinite": 0.0, "weighted_memory_to_active": 0.14384488761425018, "mix_std": 0.25058650970458984, "beta_std": 0.018706481903791428, "beta_low": 0.6431884765625, "beta_high": 0.0, "orthogonal_to_active": 0.07307250052690506, "evidence_weights": [-0.18429282307624817, -0.2250336855649948, 0.5990419387817383], "agreement_weight": 0.07099910825490952, "orthogonal_weight": -0.580192506313324}, {"head": 2, "memory_to_active": 0.05461646243929863, "contribution_to_active": 0.6123422384262085, "mix_mean": 0.5716144442558289, "mix_low": 0.0007200241088867188, "mix_high": 0.002044677734375, "read_cosine": 0.3964793086051941, "beta_mean": 0.0005305688828229904, "nonfinite": 0.0, "weighted_memory_to_active": 0.03807263821363449, "mix_std": 0.24827536940574646, "beta_std": 0.0027511613443493843, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.026942308992147446, "evidence_weights": [-0.2111249566078186, -0.3252525329589844, 0.6320157647132874], "agreement_weight": 0.1328454315662384, "orthogonal_weight": -0.821144163608551}, {"head": 3, "memory_to_active": 0.1757294088602066, "contribution_to_active": 0.5616390705108643, "mix_mean": 0.48966431617736816, "mix_low": 0.0005645751953125, "mix_high": 0.0004558563232421875, "read_cosine": 0.8017818927764893, "beta_mean": 0.014979809522628784, "nonfinite": 0.0, "weighted_memory_to_active": 0.11977343261241913, "mix_std": 0.25371190905570984, "beta_std": 0.01298651471734047, "beta_low": 0.3966064453125, "beta_high": 0.0, "orthogonal_to_active": 0.03297523781657219, "evidence_weights": [-0.15706036984920502, -0.2766006588935852, 0.7226157188415527], "agreement_weight": 0.11010397970676422, "orthogonal_weight": -0.5466933846473694}, {"head": 4, "memory_to_active": 2.532567024230957, "contribution_to_active": 0.2545177936553955, "mix_mean": 0.02609829604625702, "mix_low": 0.43862247467041016, "mix_high": 0.0, "read_cosine": 0.3202701807022095, "beta_mean": 0.11587350070476532, "nonfinite": 0.0, "weighted_memory_to_active": 0.2430657148361206, "mix_std": 0.04537405073642731, "beta_std": 0.09197484701871872, "beta_low": 0.0047607421875, "beta_high": 0.0, "orthogonal_to_active": 0.009082471951842308, "evidence_weights": [0.04019862413406372, -0.01105758547782898, -0.1788368672132492], "agreement_weight": 0.009690782986581326, "orthogonal_weight": 0.0383840836584568}, {"head": 5, "memory_to_active": 0.2530682384967804, "contribution_to_active": 0.6392274498939514, "mix_mean": 0.27910637855529785, "mix_low": 0.13791942596435547, "mix_high": 0.0007562637329101562, "read_cosine": 0.5344465970993042, "beta_mean": 0.07612495869398117, "nonfinite": 0.0, "weighted_memory_to_active": 0.16241773962974548, "mix_std": 0.30960360169410706, "beta_std": 0.10727886110544205, "beta_low": 0.2532958984375, "beta_high": 0.0, "orthogonal_to_active": 0.1878710240125656, "evidence_weights": [-0.062436334788799286, -0.30298230051994324, -0.42222046852111816], "agreement_weight": 0.024456970393657684, "orthogonal_weight": 1.4346611499786377}]}, {"layer": 9, "memory_to_active": 1.1121829748153687, "contribution_to_active": 0.593797504901886, "mix_mean": 0.2806379199028015, "mix_low": 0.15997695922851562, "mix_high": 0.0018588702660053968, "read_cosine": 0.37804684042930603, "beta_mean": 0.07808911055326462, "nonfinite": 0.0, "weighted_memory_to_active": 0.34652793407440186, "mix_std": 0.2076563835144043, "beta_std": 0.04717206582427025, "beta_low": 0.498992919921875, "beta_high": 0.0, "orthogonal_to_active": 0.037543248385190964, "output_influence": 0.44472259283065796, "residual_influence": 0.11848243325948715, "state_active": 0.04082740843296051, "state_memory": 0.021635793149471283, "state_peak": 36.20396041870117, "growth_active": 1.2119927406311035, "growth_memory": 1.392657995223999, "heads": [{"head": 0, "memory_to_active": 0.007445279974490404, "contribution_to_active": 0.5052325129508972, "mix_mean": 0.5670862197875977, "mix_low": 0.0016841888427734375, "mix_high": 0.007660865783691406, "read_cosine": 0.2521815896034241, "beta_mean": 1.682842594163958e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004270106554031372, "mix_std": 0.294810950756073, "beta_std": 7.401386392302811e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003707918105646968, "evidence_weights": [-0.28387072682380676, -0.28882521390914917, 0.9736303687095642], "agreement_weight": 0.09160737693309784, "orthogonal_weight": -0.9113078117370605}, {"head": 1, "memory_to_active": 0.03046504035592079, "contribution_to_active": 0.5489268898963928, "mix_mean": 0.4232467710971832, "mix_low": 0.011355400085449219, "mix_high": 0.0010061264038085938, "read_cosine": 0.4077630639076233, "beta_mean": 8.204054029192775e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.017023609951138496, "mix_std": 0.28832149505615234, "beta_std": 0.00015877975965850055, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010767627507448196, "evidence_weights": [-0.2103821337223053, -0.24529239535331726, 1.0649588108062744], "agreement_weight": 0.2980743646621704, "orthogonal_weight": -0.7510913014411926}, {"head": 2, "memory_to_active": 0.22025370597839355, "contribution_to_active": 0.571662962436676, "mix_mean": 0.47882699966430664, "mix_low": 0.0036725997924804688, "mix_high": 0.0022716522216796875, "read_cosine": 0.37788447737693787, "beta_mean": 0.0008555396925657988, "nonfinite": 0.0, "weighted_memory_to_active": 0.13373884558677673, "mix_std": 0.28442150354385376, "beta_std": 0.005185788497328758, "beta_low": 0.99273681640625, "beta_high": 0.0, "orthogonal_to_active": 0.054486967623233795, "evidence_weights": [-0.1745978146791458, -0.22066859900951385, 0.5402840375900269], "agreement_weight": 0.22189925611019135, "orthogonal_weight": -0.49372363090515137}, {"head": 3, "memory_to_active": 1.6514792442321777, "contribution_to_active": 0.15996740758419037, "mix_mean": 0.03810945898294449, "mix_low": 0.40454864501953125, "mix_high": 0.0, "read_cosine": 0.4419093132019043, "beta_mean": 0.09348197281360626, "nonfinite": 0.0, "weighted_memory_to_active": 0.13534992933273315, "mix_std": 0.08334390819072723, "beta_std": 0.06502240896224976, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.025115225464105606, "evidence_weights": [0.06066380813717842, 0.03800615668296814, -0.12090634554624557], "agreement_weight": 0.07612232118844986, "orthogonal_weight": 0.20475167036056519}, {"head": 4, "memory_to_active": 3.040693521499634, "contribution_to_active": 1.1594094038009644, "mix_mean": 0.11646980792284012, "mix_low": 0.1695079803466797, "mix_high": 0.00021457672119140625, "read_cosine": 0.3052305579185486, "beta_mean": 0.22415903210639954, "nonfinite": 0.0, "weighted_memory_to_active": 1.08560311794281, "mix_std": 0.17822889983654022, "beta_std": 0.11199212074279785, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.018585681915283203, "evidence_weights": [-0.053448356688022614, -0.14486132562160492, 0.035014089196920395], "agreement_weight": -0.051787737756967545, "orthogonal_weight": -0.016925564035773277}, {"head": 5, "memory_to_active": 1.7227606773376465, "contribution_to_active": 0.6175857782363892, "mix_mean": 0.06008805334568024, "mix_low": 0.3690929412841797, "mix_high": 0.0, "read_cosine": 0.48331183195114136, "beta_mean": 0.14993923902511597, "nonfinite": 0.0, "weighted_memory_to_active": 0.7031818628311157, "mix_std": 0.11681149899959564, "beta_std": 0.1005992516875267, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.1125960573554039, "evidence_weights": [0.07984693348407745, -0.04720115661621094, 0.1335557997226715], "agreement_weight": 0.022872507572174072, "orthogonal_weight": 0.23413893580436707}]}, {"layer": 10, "memory_to_active": 0.1815102994441986, "contribution_to_active": 0.4922037124633789, "mix_mean": 0.46386778354644775, "mix_low": 0.013351917266845703, "mix_high": 0.011585871689021587, "read_cosine": 0.37485140562057495, "beta_mean": 0.04742678254842758, "nonfinite": 0.0, "weighted_memory_to_active": 0.0810885950922966, "mix_std": 0.26854121685028076, "beta_std": 0.05005896836519241, "beta_low": 0.661468505859375, "beta_high": 0.0, "orthogonal_to_active": 0.06376197189092636, "output_influence": 0.38943231105804443, "residual_influence": 0.1290663480758667, "state_active": 0.05497509986162186, "state_memory": 0.013842499814927578, "state_peak": 115.53536224365234, "growth_active": 1.0870051383972168, "growth_memory": 1.0491809844970703, "heads": [{"head": 0, "memory_to_active": 0.5020611882209778, "contribution_to_active": 0.9069295525550842, "mix_mean": 0.6261432766914368, "mix_low": 0.027321815490722656, "mix_high": 0.060853004455566406, "read_cosine": 0.1418364942073822, "beta_mean": 0.20977473258972168, "nonfinite": 0.0, "weighted_memory_to_active": 0.3351055383682251, "mix_std": 0.3719205856323242, "beta_std": 0.22737538814544678, "beta_low": 0.1243896484375, "beta_high": 0.0, "orthogonal_to_active": 0.291419118642807, "evidence_weights": [-0.1542356014251709, -0.3617454171180725, -0.5157761573791504], "agreement_weight": 0.016050994396209717, "orthogonal_weight": 0.9070339202880859}, {"head": 1, "memory_to_active": 0.020437099039554596, "contribution_to_active": 0.5585339665412903, "mix_mean": 0.5669978857040405, "mix_low": 0.0005702972412109375, "mix_high": 0.00637054443359375, "read_cosine": 0.2670609652996063, "beta_mean": 0.00011974174412898719, "nonfinite": 0.0, "weighted_memory_to_active": 0.011458126828074455, "mix_std": 0.26733481884002686, "beta_std": 0.00013762639719061553, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014240331947803497, "evidence_weights": [-0.27419787645339966, -0.1994735449552536, 0.5936009883880615], "agreement_weight": 0.39173683524131775, "orthogonal_weight": -1.3852423429489136}, {"head": 2, "memory_to_active": 0.06350960582494736, "contribution_to_active": 0.4831881821155548, "mix_mean": 0.48727577924728394, "mix_low": 0.0007123947143554688, "mix_high": 0.0015163421630859375, "read_cosine": 0.0002935739466920495, "beta_mean": 0.00022987136617302895, "nonfinite": 0.0, "weighted_memory_to_active": 0.025635147467255592, "mix_std": 0.26671886444091797, "beta_std": 0.0009292658651247621, "beta_low": 0.99822998046875, "beta_high": 0.0, "orthogonal_to_active": 0.009312246926128864, "evidence_weights": [-0.19156979024410248, -0.23835493624210358, 0.529671847820282], "agreement_weight": -0.053722333163022995, "orthogonal_weight": -0.35566288232803345}, {"head": 3, "memory_to_active": 0.35088902711868286, "contribution_to_active": 0.10623319447040558, "mix_mean": 0.24940627813339233, "mix_low": 0.04807567596435547, "mix_high": 0.0007047653198242188, "read_cosine": 0.9048398733139038, "beta_mean": 0.06826145201921463, "nonfinite": 0.0, "weighted_memory_to_active": 0.0423717275261879, "mix_std": 0.22834455966949463, "beta_std": 0.06348162889480591, "beta_low": 0.00225830078125, "beta_high": 0.0, "orthogonal_to_active": 0.04780563712120056, "evidence_weights": [-0.12565341591835022, -0.1475408524274826, 0.16129951179027557], "agreement_weight": -0.003617184702306986, "orthogonal_weight": 1.4656920433044434}, {"head": 4, "memory_to_active": 0.008794091641902924, "contribution_to_active": 0.4887276291847229, "mix_mean": 0.4846112132072449, "mix_low": 0.0011272430419921875, "mix_high": 6.4849853515625e-05, "read_cosine": 0.07847634702920914, "beta_mean": 2.7740446967072785e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002934602089226246, "mix_std": 0.2551281750202179, "beta_std": 0.0007577344658784568, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0010280739516019821, "evidence_weights": [-0.17998605966567993, -0.2337283343076706, 0.6765922904014587], "agreement_weight": 0.0680127814412117, "orthogonal_weight": -0.3543926477432251}, {"head": 5, "memory_to_active": 0.14337074756622314, "contribution_to_active": 0.4096095860004425, "mix_mean": 0.3687722086906433, "mix_low": 0.0023040771484375, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8566011786460876, "beta_mean": 0.00614715414121747, "nonfinite": 0.0, "weighted_memory_to_active": 0.06902642548084259, "mix_std": 0.22180020809173584, "beta_std": 0.007672157138586044, "beta_low": 0.843994140625, "beta_high": 0.0, "orthogonal_to_active": 0.018766388297080994, "evidence_weights": [-0.15186183154582977, -0.16153070330619812, 0.2952319383621216], "agreement_weight": 0.09379397332668304, "orthogonal_weight": -0.5940636396408081}]}, {"layer": 11, "memory_to_active": 0.18952593207359314, "contribution_to_active": 0.5234612822532654, "mix_mean": 0.4257045388221741, "mix_low": 0.0681660994887352, "mix_high": 0.03129434585571289, "read_cosine": 0.47585529088974, "beta_mean": 0.038982655853033066, "nonfinite": 0.0, "weighted_memory_to_active": 0.07732029259204865, "mix_std": 0.31646522879600525, "beta_std": 0.05482245236635208, "beta_low": 0.514556884765625, "beta_high": 0.0, "orthogonal_to_active": 0.05648002400994301, "output_influence": 0.4843750298023224, "residual_influence": 0.12106962502002716, "state_active": 0.12966041266918182, "state_memory": 0.008920136839151382, "state_peak": 25.11005973815918, "growth_active": 1.0811724662780762, "growth_memory": 1.4729896783828735, "heads": [{"head": 0, "memory_to_active": 0.008394724689424038, "contribution_to_active": 0.5586515665054321, "mix_mean": 0.5805531144142151, "mix_low": 0.0013332366943359375, "mix_high": 0.007205009460449219, "read_cosine": 0.42107170820236206, "beta_mean": 9.802217391552404e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0052624973468482494, "mix_std": 0.2705402374267578, "beta_std": 0.003361881710588932, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.003093755804002285, "evidence_weights": [-0.16178381443023682, -0.3519662320613861, 0.5985754728317261], "agreement_weight": 0.18930654227733612, "orthogonal_weight": -0.7325953245162964}, {"head": 1, "memory_to_active": 0.43213605880737305, "contribution_to_active": 0.2518690824508667, "mix_mean": 0.28774967789649963, "mix_low": 0.022897720336914062, "mix_high": 9.5367431640625e-05, "read_cosine": 0.6296924352645874, "beta_mean": 0.040896665304899216, "nonfinite": 0.0, "weighted_memory_to_active": 0.16848152875900269, "mix_std": 0.2466021329164505, "beta_std": 0.07703007012605667, "beta_low": 0.35467529296875, "beta_high": 0.0, "orthogonal_to_active": 0.03264252096414566, "evidence_weights": [-0.15691880881786346, -0.13356861472129822, 0.2586001455783844], "agreement_weight": 0.08320584893226624, "orthogonal_weight": -0.27322500944137573}, {"head": 2, "memory_to_active": 0.03330477327108383, "contribution_to_active": 0.9137350916862488, "mix_mean": 0.4567878842353821, "mix_low": 0.12258720397949219, "mix_high": 0.09893417358398438, "read_cosine": 0.26940667629241943, "beta_mean": 0.0794595256447792, "nonfinite": 0.0, "weighted_memory_to_active": 0.026151882484555244, "mix_std": 0.4248059093952179, "beta_std": 0.10428152978420258, "beta_low": 0.4224853515625, "beta_high": 0.0, "orthogonal_to_active": 0.10041286796331406, "evidence_weights": [-0.03096252679824829, -0.4529702663421631, -1.1173685789108276], "agreement_weight": -0.001343368086963892, "orthogonal_weight": 4.046028137207031}, {"head": 3, "memory_to_active": 0.018434077501296997, "contribution_to_active": 0.7349691390991211, "mix_mean": 0.4102199673652649, "mix_low": 0.24290180206298828, "mix_high": 0.07998371124267578, "read_cosine": 0.5278418660163879, "beta_mean": 0.06467115134000778, "nonfinite": 0.0, "weighted_memory_to_active": 0.010415764525532722, "mix_std": 0.43411576747894287, "beta_std": 0.09388698637485504, "beta_low": 0.46893310546875, "beta_high": 0.0, "orthogonal_to_active": 0.04105585813522339, "evidence_weights": [0.017052223905920982, -0.3592832386493683, -0.45773008465766907], "agreement_weight": 0.029686367139220238, "orthogonal_weight": 4.336714267730713}, {"head": 4, "memory_to_active": 0.5259715914726257, "contribution_to_active": 0.33761510252952576, "mix_mean": 0.35197097063064575, "mix_low": 0.018833160400390625, "mix_high": 0.00034427642822265625, "read_cosine": 0.19142568111419678, "beta_mean": 0.039463743567466736, "nonfinite": 0.0, "weighted_memory_to_active": 0.20371264219284058, "mix_std": 0.2632022202014923, "beta_std": 0.036209575831890106, "beta_low": 0.11517333984375, "beta_high": 0.0, "orthogonal_to_active": 0.1406337022781372, "evidence_weights": [-0.17829491198062897, -0.14585642516613007, 1.5283572673797607], "agreement_weight": -0.1497034877538681, "orthogonal_weight": -0.7180569767951965}, {"head": 5, "memory_to_active": 0.11891429871320724, "contribution_to_active": 0.3439274728298187, "mix_mean": 0.4669455289840698, "mix_low": 0.00044345855712890625, "mix_high": 0.0012035369873046875, "read_cosine": 0.8156933784484863, "beta_mean": 0.00930680800229311, "nonfinite": 0.0, "weighted_memory_to_active": 0.0498974435031414, "mix_std": 0.25952503085136414, "beta_std": 0.014164669439196587, "beta_low": 0.72650146484375, "beta_high": 0.0, "orthogonal_to_active": 0.021041441708803177, "evidence_weights": [-0.2009250521659851, -0.2573888599872589, 0.5079566240310669], "agreement_weight": 0.05627826973795891, "orthogonal_weight": -0.6594576835632324}]}, {"layer": 12, "memory_to_active": 0.35172855854034424, "contribution_to_active": 0.5416167974472046, "mix_mean": 0.4502906799316406, "mix_low": 0.016784191131591797, "mix_high": 0.002838134765625, "read_cosine": 0.2712363600730896, "beta_mean": 0.016135962679982185, "nonfinite": 0.0, "weighted_memory_to_active": 0.18292048573493958, "mix_std": 0.25381940603256226, "beta_std": 0.016615865752100945, "beta_low": 0.8151550889015198, "beta_high": 0.0, "orthogonal_to_active": 0.06605630367994308, "output_influence": 0.4070581793785095, "residual_influence": 0.11240552365779877, "state_active": 0.03974956274032593, "state_memory": 0.010377388447523117, "state_peak": 25.603113174438477, "growth_active": 1.078035831451416, "growth_memory": 1.233632206916809, "heads": [{"head": 0, "memory_to_active": 0.08967513591051102, "contribution_to_active": 0.43973270058631897, "mix_mean": 0.41622671484947205, "mix_low": 0.003711700439453125, "mix_high": 0.00051116943359375, "read_cosine": 0.2817363142967224, "beta_mean": 0.0015583874192088842, "nonfinite": 0.0, "weighted_memory_to_active": 0.03961542248725891, "mix_std": 0.2652231752872467, "beta_std": 0.006056491285562515, "beta_low": 0.9766845703125, "beta_high": 0.0, "orthogonal_to_active": 0.022116689011454582, "evidence_weights": [-0.16911470890045166, -0.24090415239334106, 0.28941044211387634], "agreement_weight": 0.10990350693464279, "orthogonal_weight": -0.5864497423171997}, {"head": 1, "memory_to_active": 0.11901765316724777, "contribution_to_active": 0.47715914249420166, "mix_mean": 0.48669296503067017, "mix_low": 0.0008544921875, "mix_high": 0.00017833709716796875, "read_cosine": 0.5618964433670044, "beta_mean": 0.0013554405886679888, "nonfinite": 0.0, "weighted_memory_to_active": 0.057873986661434174, "mix_std": 0.26043128967285156, "beta_std": 0.005861623678356409, "beta_low": 0.98028564453125, "beta_high": 0.0, "orthogonal_to_active": 0.042945969849824905, "evidence_weights": [-0.1984771192073822, -0.2606194317340851, 0.8633853793144226], "agreement_weight": 0.10096898674964905, "orthogonal_weight": -0.9088748097419739}, {"head": 2, "memory_to_active": 0.6505897641181946, "contribution_to_active": 0.41404762864112854, "mix_mean": 0.2921532988548279, "mix_low": 0.02994537353515625, "mix_high": 6.008148193359375e-05, "read_cosine": 0.44462159276008606, "beta_mean": 0.09070147573947906, "nonfinite": 0.0, "weighted_memory_to_active": 0.23281148076057434, "mix_std": 0.23305675387382507, "beta_std": 0.06733822077512741, "beta_low": 0.0010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.18918079137802124, "evidence_weights": [-0.24133263528347015, -0.039804857224226, -0.06175042316317558], "agreement_weight": 0.03533428907394409, "orthogonal_weight": 1.0021181106567383}, {"head": 3, "memory_to_active": 0.0292771365493536, "contribution_to_active": 0.709046483039856, "mix_mean": 0.6775675415992737, "mix_low": 0.00038814544677734375, "mix_high": 0.012621879577636719, "read_cosine": 0.07389139384031296, "beta_mean": 0.00024361004761885852, "nonfinite": 0.0, "weighted_memory_to_active": 0.02179957553744316, "mix_std": 0.24431049823760986, "beta_std": 0.0033958833664655685, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.01990477181971073, "evidence_weights": [-0.29974687099456787, -0.33022230863571167, 1.1877955198287964], "agreement_weight": 0.26871970295906067, "orthogonal_weight": -0.9351863861083984}, {"head": 4, "memory_to_active": 1.1612939834594727, "contribution_to_active": 0.7551993727684021, "mix_mean": 0.24321798980236053, "mix_low": 0.05920886993408203, "mix_high": 0.0013866424560546875, "read_cosine": -0.057335201650857925, "beta_mean": 0.002359705278649926, "nonfinite": 0.0, "weighted_memory_to_active": 0.7107099294662476, "mix_std": 0.24527673423290253, "beta_std": 0.014078138396143913, "beta_low": 0.9456787109375, "beta_high": 0.0, "orthogonal_to_active": 0.09577502310276031, "evidence_weights": [-0.0468154214322567, -0.18753650784492493, 0.017714980989694595], "agreement_weight": -0.002193559892475605, "orthogonal_weight": -0.14628203213214874}, {"head": 5, "memory_to_active": 0.06051767244935036, "contribution_to_active": 0.4545154571533203, "mix_mean": 0.5858854651451111, "mix_low": 0.006596565246582031, "mix_high": 0.0022706985473632812, "read_cosine": 0.3226074278354645, "beta_mean": 0.000597151112742722, "nonfinite": 0.0, "weighted_memory_to_active": 0.03471250832080841, "mix_std": 0.27461808919906616, "beta_std": 0.00296483701094985, "beta_low": 0.98828125, "beta_high": 0.0, "orthogonal_to_active": 0.026414580643177032, "evidence_weights": [-0.2606041133403778, -0.29953086376190186, 0.9973767399787903], "agreement_weight": 0.18773216009140015, "orthogonal_weight": -0.8363237977027893}]}, {"layer": 13, "memory_to_active": 0.15340347588062286, "contribution_to_active": 0.493398517370224, "mix_mean": 0.48527729511260986, "mix_low": 0.006176789756864309, "mix_high": 0.003222624771296978, "read_cosine": 0.40541261434555054, "beta_mean": 0.017366033047437668, "nonfinite": 0.0, "weighted_memory_to_active": 0.06641456484794617, "mix_std": 0.2585922181606293, "beta_std": 0.017525026574730873, "beta_low": 0.7883199453353882, "beta_high": 0.0, "orthogonal_to_active": 0.06348039954900742, "output_influence": 0.4465709626674652, "residual_influence": 0.1479136049747467, "state_active": 0.0944700837135315, "state_memory": 0.0369253009557724, "state_peak": 97.60086822509766, "growth_active": 0.979236900806427, "growth_memory": 1.0369375944137573, "heads": [{"head": 0, "memory_to_active": 0.046919893473386765, "contribution_to_active": 0.5242675542831421, "mix_mean": 0.5157832503318787, "mix_low": 0.0006685256958007812, "mix_high": 0.0011692047119140625, "read_cosine": 0.3741307854652405, "beta_mean": 0.00019531532598193735, "nonfinite": 0.0, "weighted_memory_to_active": 0.026152510195970535, "mix_std": 0.26404452323913574, "beta_std": 0.0004484587989281863, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.02550298161804676, "evidence_weights": [-0.20177756249904633, -0.2530694305896759, 0.7487257719039917], "agreement_weight": 0.06729809194803238, "orthogonal_weight": -1.0700076818466187}, {"head": 1, "memory_to_active": 0.029859405010938644, "contribution_to_active": 0.5576762557029724, "mix_mean": 0.4998292922973633, "mix_low": 0.0024404525756835938, "mix_high": 0.0005950927734375, "read_cosine": 0.2181437611579895, "beta_mean": 0.00010403933993075043, "nonfinite": 0.0, "weighted_memory_to_active": 0.016352100297808647, "mix_std": 0.26682761311531067, "beta_std": 0.0001727466587908566, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.016771210357546806, "evidence_weights": [-0.18760384619235992, -0.26476630568504333, 0.6995285153388977], "agreement_weight": 0.21110783517360687, "orthogonal_weight": -1.0800505876541138}, {"head": 2, "memory_to_active": 0.1328699290752411, "contribution_to_active": 0.38856303691864014, "mix_mean": 0.3494226038455963, "mix_low": 0.0024824142456054688, "mix_high": 5.054473876953125e-05, "read_cosine": 0.6715279817581177, "beta_mean": 0.014158208854496479, "nonfinite": 0.0, "weighted_memory_to_active": 0.05342644825577736, "mix_std": 0.21884848177433014, "beta_std": 0.03689263388514519, "beta_low": 0.7481689453125, "beta_high": 0.0, "orthogonal_to_active": 0.026712164282798767, "evidence_weights": [-0.21994362771511078, -0.16797111928462982, 0.48690372705459595], "agreement_weight": 0.027363844215869904, "orthogonal_weight": -0.7759861946105957}, {"head": 3, "memory_to_active": 0.16721414029598236, "contribution_to_active": 0.570901095867157, "mix_mean": 0.5983439683914185, "mix_low": 0.0172271728515625, "mix_high": 0.013685226440429688, "read_cosine": 0.011825351044535637, "beta_mean": 0.0012968472437933087, "nonfinite": 0.0, "weighted_memory_to_active": 0.11882895976305008, "mix_std": 0.2898330092430115, "beta_std": 0.004848668351769447, "beta_low": 0.98699951171875, "beta_high": 0.0, "orthogonal_to_active": 0.11864235997200012, "evidence_weights": [-0.34522658586502075, -0.2368287891149521, 2.0378499031066895], "agreement_weight": 2.578113079071045, "orthogonal_weight": -1.0534389019012451}, {"head": 4, "memory_to_active": 0.44155678153038025, "contribution_to_active": 0.28195738792419434, "mix_mean": 0.3783800005912781, "mix_low": 0.010979652404785156, "mix_high": 1.52587890625e-05, "read_cosine": 0.8766128420829773, "beta_mean": 0.08774673938751221, "nonfinite": 0.0, "weighted_memory_to_active": 0.12438605725765228, "mix_std": 0.23036357760429382, "beta_std": 0.0587579607963562, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1484558880329132, "evidence_weights": [-0.17570145428180695, -0.23586447536945343, 0.5046322345733643], "agreement_weight": 0.0005483758286572993, "orthogonal_weight": 1.85202956199646}, {"head": 5, "memory_to_active": 0.10200069844722748, "contribution_to_active": 0.6370255947113037, "mix_mean": 0.5699045658111572, "mix_low": 0.0032625198364257812, "mix_high": 0.0038204193115234375, "read_cosine": 0.2802349925041199, "beta_mean": 0.0006950429524295032, "nonfinite": 0.0, "weighted_memory_to_active": 0.05934130400419235, "mix_std": 0.2816360592842102, "beta_std": 0.004029694478958845, "beta_low": 0.99481201171875, "beta_high": 0.0, "orthogonal_to_active": 0.04479779303073883, "evidence_weights": [-0.21849797666072845, -0.28416070342063904, 0.6147670745849609], "agreement_weight": 0.0917028859257698, "orthogonal_weight": -0.8370421528816223}]}, {"layer": 14, "memory_to_active": 0.2621328830718994, "contribution_to_active": 0.4319838285446167, "mix_mean": 0.4073915183544159, "mix_low": 0.02422475814819336, "mix_high": 0.0039722127839922905, "read_cosine": 0.6174954175949097, "beta_mean": 0.06966495513916016, "nonfinite": 0.0, "weighted_memory_to_active": 0.12046986818313599, "mix_std": 0.24961942434310913, "beta_std": 0.06379513442516327, "beta_low": 0.4417623281478882, "beta_high": 0.0, "orthogonal_to_active": 0.042710475623607635, "output_influence": 0.30217671394348145, "residual_influence": 0.11595013737678528, "state_active": 0.06879100203514099, "state_memory": 0.015973353758454323, "state_peak": 33.054603576660156, "growth_active": 1.1491459608078003, "growth_memory": 1.3003110885620117, "heads": [{"head": 0, "memory_to_active": 0.35809963941574097, "contribution_to_active": 0.23166748881340027, "mix_mean": 0.2321198284626007, "mix_low": 0.013075828552246094, "mix_high": 9.5367431640625e-06, "read_cosine": 0.7603988647460938, "beta_mean": 0.0734134167432785, "nonfinite": 0.0, "weighted_memory_to_active": 0.10624685138463974, "mix_std": 0.2049993872642517, "beta_std": 0.0705309584736824, "beta_low": 0.01495361328125, "beta_high": 0.0, "orthogonal_to_active": 0.050404686480760574, "evidence_weights": [-0.11104138195514679, -0.06307704746723175, 0.6947084069252014], "agreement_weight": 0.029135648161172867, "orthogonal_weight": 0.7774612307548523}, {"head": 1, "memory_to_active": 0.4753846824169159, "contribution_to_active": 0.38759833574295044, "mix_mean": 0.2619989514350891, "mix_low": 0.11256027221679688, "mix_high": 0.0001354217529296875, "read_cosine": 0.7156969904899597, "beta_mean": 0.2892381250858307, "nonfinite": 0.0, "weighted_memory_to_active": 0.2323329746723175, "mix_std": 0.2749674618244171, "beta_std": 0.2605457305908203, "beta_low": 0.09136962890625, "beta_high": 0.0, "orthogonal_to_active": 0.019833479076623917, "evidence_weights": [0.00706316577270627, -0.27252253890037537, -0.16812123358249664], "agreement_weight": 0.05319976434111595, "orthogonal_weight": 0.1606307327747345}, {"head": 2, "memory_to_active": 0.2117908000946045, "contribution_to_active": 0.5177974700927734, "mix_mean": 0.5192095637321472, "mix_low": 0.00334930419921875, "mix_high": 0.001117706298828125, "read_cosine": 0.3930884301662445, "beta_mean": 0.006541873794049025, "nonfinite": 0.0, "weighted_memory_to_active": 0.11982855945825577, "mix_std": 0.26826930046081543, "beta_std": 0.008076396770775318, "beta_low": 0.8133544921875, "beta_high": 0.0, "orthogonal_to_active": 0.08836047351360321, "evidence_weights": [-0.16744817793369293, -0.2784864604473114, 0.8377751111984253], "agreement_weight": 0.08214719593524933, "orthogonal_weight": -0.8403430581092834}, {"head": 3, "memory_to_active": 0.0688025951385498, "contribution_to_active": 0.6710745692253113, "mix_mean": 0.6616854667663574, "mix_low": 0.0035505294799804688, "mix_high": 0.015073776245117188, "read_cosine": 0.2726849615573883, "beta_mean": 0.008771183900535107, "nonfinite": 0.0, "weighted_memory_to_active": 0.04704040288925171, "mix_std": 0.24736279249191284, "beta_std": 0.0076191723346710205, "beta_low": 0.699951171875, "beta_high": 0.0, "orthogonal_to_active": 0.026462038978934288, "evidence_weights": [-0.38179656863212585, -0.32812997698783875, 0.44281500577926636], "agreement_weight": 0.07826043665409088, "orthogonal_weight": -0.679336667060852}, {"head": 4, "memory_to_active": 0.1684621125459671, "contribution_to_active": 0.47604992985725403, "mix_mean": 0.461447149515152, "mix_low": 0.004679679870605469, "mix_high": 0.0067844390869140625, "read_cosine": 0.8406696915626526, "beta_mean": 0.037927500903606415, "nonfinite": 0.0, "weighted_memory_to_active": 0.09472718834877014, "mix_std": 0.2612603008747101, "beta_std": 0.027132652699947357, "beta_low": 0.0614013671875, "beta_high": 0.0, "orthogonal_to_active": 0.024914128705859184, "evidence_weights": [-0.21654166281223297, -0.26262152194976807, 0.4071529805660248], "agreement_weight": 0.12094087153673172, "orthogonal_weight": -0.5222810506820679}, {"head": 5, "memory_to_active": 0.2902574837207794, "contribution_to_active": 0.30771511793136597, "mix_mean": 0.30788785219192505, "mix_low": 0.0081329345703125, "mix_high": 0.0007123947143554688, "read_cosine": 0.7224334478378296, "beta_mean": 0.002097622025758028, "nonfinite": 0.0, "weighted_memory_to_active": 0.12264317274093628, "mix_std": 0.2408570796251297, "beta_std": 0.00886586681008339, "beta_low": 0.96954345703125, "beta_high": 0.0, "orthogonal_to_active": 0.04628806188702583, "evidence_weights": [-0.1847476363182068, -0.15278835594654083, 0.19571155309677124], "agreement_weight": 0.08586109429597855, "orthogonal_weight": -0.6204171180725098}]}, {"layer": 15, "memory_to_active": 0.06437064707279205, "contribution_to_active": 0.473635196685791, "mix_mean": 0.38912251591682434, "mix_low": 0.008286476135253906, "mix_high": 0.0025800070725381374, "read_cosine": 0.672080397605896, "beta_mean": 0.04045020416378975, "nonfinite": 0.0, "weighted_memory_to_active": 0.028891779482364655, "mix_std": 0.28047242760658264, "beta_std": 0.042992644011974335, "beta_low": 0.7097982168197632, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.006365678273141384, "output_influence": 0.3036843240261078, "residual_influence": 0.1635485142469406, "state_active": 0.11688459664583206, "state_memory": 0.007717412896454334, "state_peak": 42.200408935546875, "growth_active": 1.279064655303955, "growth_memory": 1.8489785194396973, "heads": [{"head": 0, "memory_to_active": 0.10985581576824188, "contribution_to_active": 0.46345025300979614, "mix_mean": 0.35564500093460083, "mix_low": 0.0063991546630859375, "mix_high": 0.0038785934448242188, "read_cosine": 0.6979789733886719, "beta_mean": 0.036730676889419556, "nonfinite": 0.0, "weighted_memory_to_active": 0.051793087273836136, "mix_std": 0.26851794123649597, "beta_std": 0.026740022003650665, "beta_low": 0.1138916015625, "beta_high": 0.0, "orthogonal_to_active": 0.018623078241944313, "evidence_weights": [-0.16349002718925476, -0.18581627309322357, 0.3410153090953827], "agreement_weight": 0.0871918648481369, "orthogonal_weight": -0.6074789762496948}, {"head": 1, "memory_to_active": 0.14746761322021484, "contribution_to_active": 0.409295916557312, "mix_mean": 0.39122265577316284, "mix_low": 0.005031585693359375, "mix_high": 0.003989219665527344, "read_cosine": 0.8764158487319946, "beta_mean": 0.0013398259179666638, "nonfinite": 0.0, "weighted_memory_to_active": 0.0569833405315876, "mix_std": 0.28013479709625244, "beta_std": 0.0035401438362896442, "beta_low": 0.976318359375, "beta_high": 0.0, "orthogonal_to_active": 0.001871165120974183, "evidence_weights": [-0.1797209084033966, -0.18629996478557587, 0.27496322989463806], "agreement_weight": 0.09712721407413483, "orthogonal_weight": -0.05119999125599861}, {"head": 2, "memory_to_active": 0.005335249938070774, "contribution_to_active": 0.4431799352169037, "mix_mean": 0.3384225070476532, "mix_low": 0.014269828796386719, "mix_high": 4.9591064453125e-05, "read_cosine": 0.7901225686073303, "beta_mean": 8.374424214707687e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00233409833163023, "mix_std": 0.2717041075229645, "beta_std": 0.0001709681237116456, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 7.508356793550774e-05, "evidence_weights": [-0.20499533414840698, -0.27569475769996643, 0.3425973355770111], "agreement_weight": 0.01611645519733429, "orthogonal_weight": 0.06206781789660454}, {"head": 3, "memory_to_active": 0.0330074205994606, "contribution_to_active": 0.45291668176651, "mix_mean": 0.3337913155555725, "mix_low": 0.0052013397216796875, "mix_high": 2.6702880859375e-05, "read_cosine": 0.6338183879852295, "beta_mean": 0.00035002047661691904, "nonfinite": 0.0, "weighted_memory_to_active": 0.015518313273787498, "mix_std": 0.2717527151107788, "beta_std": 0.001115129911340773, "beta_low": 0.99822998046875, "beta_high": 0.0, "orthogonal_to_active": 0.010860440321266651, "evidence_weights": [-0.19837918877601624, -0.2486010491847992, 0.3986404538154602], "agreement_weight": 0.021069740876555443, "orthogonal_weight": -0.9440592527389526}, {"head": 4, "memory_to_active": 0.08216185122728348, "contribution_to_active": 0.5935373902320862, "mix_mean": 0.384138822555542, "mix_low": 0.01250457763671875, "mix_high": 0.0006361007690429688, "read_cosine": 0.5394749641418457, "beta_mean": 0.2039216309785843, "nonfinite": 0.0, "weighted_memory_to_active": 0.04262598976492882, "mix_std": 0.31009039282798767, "beta_std": 0.22572025656700134, "beta_low": 0.17120361328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.003102359129115939, "evidence_weights": [-0.23078162968158722, -0.2551722526550293, 0.07343247532844543], "agreement_weight": 0.011547367088496685, "orthogonal_weight": 0.10232004523277283}, {"head": 5, "memory_to_active": 0.008395910263061523, "contribution_to_active": 0.4794308543205261, "mix_mean": 0.5315148234367371, "mix_low": 0.006312370300292969, "mix_high": 0.006899833679199219, "read_cosine": 0.49467164278030396, "beta_mean": 0.0002753323351498693, "nonfinite": 0.0, "weighted_memory_to_active": 0.004095851443707943, "mix_std": 0.2806345224380493, "beta_std": 0.0006693667965009809, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.0036619440652430058, "evidence_weights": [-0.3502441942691803, -0.3023492693901062, 0.37935757637023926], "agreement_weight": 0.2333931177854538, "orthogonal_weight": 1.0899583101272583}]}]}, {"step": 25000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8111158030005754, "memory_to_active": 0.9092002511024475, "contribution_to_active": 0.4911109209060669, "mix_mean": 0.34680360555648804, "mix_low": 0.1599806696176529, "mix_high": 0.007082839962095022, "read_cosine": 0.3809382915496826, "beta_mean": 0.08372559398412704, "nonfinite": 0.0, "weighted_memory_to_active": 0.2049274444580078, "mix_std": 0.21449290215969086, "beta_std": 0.06987190246582031, "beta_low": 0.48844021558761597, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04788484051823616, "samples": 4, "scored_tokens": 899, "normal_loss": 3.183607816696167, "normal_token_accuracy": 0.42602890729904175, "state_active": 0.058721642941236496, "state_memory": 0.019864723086357117, "state_peak": 109.11646270751953, "growth_active": 1.1307380199432373, "growth_memory": 1.3902983665466309, "output_influence": 0.3494274616241455, "residual_influence": 0.11079331487417221, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2669920921325684, "no_memory_loss_delta": 0.2871580719947815, "scrambled_loss_delta": 0.1970086395740509, "memory_only_loss_delta": 2.3849220275878906, "no_evidence_loss_delta": 0.17312444746494293, "no_agreement_loss_delta": 0.014620322734117508, "no_orthogonal_loss_delta": 0.004435029812157154}, {"index": 94, "tokens": 256, "normal_loss": 3.205732822418213, "no_memory_loss_delta": 0.41941723227500916, "scrambled_loss_delta": 0.39266565442085266, "memory_only_loss_delta": 2.5677988529205322, "no_evidence_loss_delta": 0.20486918091773987, "no_agreement_loss_delta": 0.03618788719177246, "no_orthogonal_loss_delta": 0.03354474902153015}, {"index": 188, "tokens": 256, "normal_loss": 3.5989298820495605, "no_memory_loss_delta": 0.33619368076324463, "scrambled_loss_delta": 0.15192285180091858, "memory_only_loss_delta": 2.3933753967285156, "no_evidence_loss_delta": 0.21168725192546844, "no_agreement_loss_delta": 0.020061597228050232, "no_orthogonal_loss_delta": 0.012692569755017757}, {"index": 283, "tokens": 131, "normal_loss": 2.165802001953125, "no_memory_loss_delta": 0.17171235382556915, "scrambled_loss_delta": 0.11961651593446732, "memory_only_loss_delta": 2.5512239933013916, "no_evidence_loss_delta": 0.07457222789525986, "no_agreement_loss_delta": -0.022747034206986427, "no_orthogonal_loss_delta": -0.016546059399843216}], "agreement_weight_abs_mean": 0.1257483810186386, "agreement_weight_abs_max": 2.6993346214294434, "agreement_weight_mean": 0.10688815265893936, "orthogonal_weight_abs_mean": 0.6132490038871765, "orthogonal_weight_abs_max": 4.534760475158691, "orthogonal_weight_mean": -0.16410312056541443, "no_memory_loss": 3.5055689811706543, "no_memory_loss_delta": 0.32196125388145447, "no_memory_token_accuracy_gain": 0.046718575060367584, "no_memory_token_win_fraction": 0.6273637413978577, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 27.527372988411702, "scrambled_loss": 3.4122159481048584, "scrambled_loss_delta": 0.22860804200172424, "scrambled_token_accuracy_gain": 0.016685206443071365, "scrambled_token_win_fraction": 0.621802031993866, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.43596700133712, "memory_only_loss": 5.647246837615967, "memory_only_loss_delta": 2.4636385440826416, "memory_only_token_accuracy_gain": 0.23136818408966064, "memory_only_token_win_fraction": 0.8676307201385498, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 91.48753430164706, "no_evidence_loss": 3.3623926639556885, "no_evidence_loss_delta": 0.17878451943397522, "no_evidence_token_accuracy_gain": 0.026696329936385155, "no_evidence_token_win_fraction": 0.5839822292327881, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 16.371391661812943, "no_agreement_loss": 3.200474500656128, "no_agreement_loss_delta": 0.016866305842995644, "no_agreement_token_accuracy_gain": -0.0022246940061450005, "no_agreement_token_win_fraction": 0.48275861144065857, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.6724866012149526, "no_orthogonal_loss": 3.1956264972686768, "no_orthogonal_loss_delta": 0.012018452398478985, "no_orthogonal_token_accuracy_gain": 0.004449388012290001, "no_orthogonal_token_win_fraction": 0.5461624264717102, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.1946519262827269, "short_no_memory_loss_delta": 0.27083325386047363, "long_context_memory_gain": 0.051128000020980835, "normal_context_loss_gain": 0.13874025642871857, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.24764084815979, "contribution_to_active": 0.940843403339386, "mix_mean": 0.0969710499048233, "mix_low": 0.5241489410400391, "mix_high": 0.003969510551542044, "read_cosine": 0.451038122177124, "beta_mean": 0.26753440499305725, "nonfinite": 0.0, "weighted_memory_to_active": 1.0164289474487305, "mix_std": 0.08606038987636566, "beta_std": 0.1617458015680313, "beta_low": 0.006805420387536287, "beta_high": 0.0, "orthogonal_to_active": 0.07070083916187286, "output_influence": 0.14343103766441345, "residual_influence": 0.17408093810081482, "state_active": 0.07540322840213776, "state_memory": 0.04903559386730194, "state_peak": 45.985103607177734, "growth_active": 1.2688677310943604, "growth_memory": 1.2646996974945068, "heads": [{"head": 0, "memory_to_active": 2.395113229751587, "contribution_to_active": 0.34912335872650146, "mix_mean": 0.02020854875445366, "mix_low": 0.580780029296875, "mix_high": 0.0, "read_cosine": 0.2710390090942383, "beta_mean": 0.21262700855731964, "nonfinite": 0.0, "weighted_memory_to_active": 0.42280110716819763, "mix_std": 0.05416877940297127, "beta_std": 0.10995952785015106, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03440994396805763, "evidence_weights": [-0.03769025206565857, -0.0828951820731163, -0.08926708996295929], "agreement_weight": -0.10712402313947678, "orthogonal_weight": -0.09462793916463852}, {"head": 1, "memory_to_active": 4.752586841583252, "contribution_to_active": 1.7421327829360962, "mix_mean": 0.014525271020829678, "mix_low": 0.6647758483886719, "mix_high": 0.0, "read_cosine": 0.5022663474082947, "beta_mean": 0.40293094515800476, "nonfinite": 0.0, "weighted_memory_to_active": 2.1420912742614746, "mix_std": 0.04686328023672104, "beta_std": 0.1862795650959015, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16997435688972473, "evidence_weights": [0.024671854451298714, -0.06783155351877213, 0.2629461884498596], "agreement_weight": 0.02711448445916176, "orthogonal_weight": -0.18049860000610352}, {"head": 2, "memory_to_active": 2.884157419204712, "contribution_to_active": 0.5954745411872864, "mix_mean": 0.015035307966172695, "mix_low": 0.6306705474853516, "mix_high": 0.0, "read_cosine": 0.31717097759246826, "beta_mean": 0.33832472562789917, "nonfinite": 0.0, "weighted_memory_to_active": 0.6937870979309082, "mix_std": 0.032930389046669006, "beta_std": 0.23634809255599976, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06605938822031021, "evidence_weights": [0.045981358736753464, 0.06312232464551926, -0.03522671014070511], "agreement_weight": 0.005325881764292717, "orthogonal_weight": -0.18039345741271973}, {"head": 3, "memory_to_active": 6.22447395324707, "contribution_to_active": 2.559986114501953, "mix_mean": 0.040934279561042786, "mix_low": 0.4044532775878906, "mix_high": 0.0, "read_cosine": 0.3713059425354004, "beta_mean": 0.3480897545814514, "nonfinite": 0.0, "weighted_memory_to_active": 2.7383697032928467, "mix_std": 0.08584871888160706, "beta_std": 0.23403213918209076, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1319875568151474, "evidence_weights": [-0.07212349772453308, -0.09569678455591202, -0.11793369054794312], "agreement_weight": -0.021932264789938927, "orthogonal_weight": -0.057613495737314224}, {"head": 4, "memory_to_active": 3.113572120666504, "contribution_to_active": 0.05933674797415733, "mix_mean": 0.005797707010060549, "mix_low": 0.8480195999145508, "mix_high": 0.0, "read_cosine": 0.3220847547054291, "beta_mean": 0.25732147693634033, "nonfinite": 0.0, "weighted_memory_to_active": 0.054135873913764954, "mix_std": 0.010783109813928604, "beta_std": 0.1637086719274521, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0064650485292077065, "evidence_weights": [0.10271421074867249, 0.049770206212997437, -0.3475382328033447], "agreement_weight": -0.07199926674365997, "orthogonal_weight": 0.12875886261463165}, {"head": 5, "memory_to_active": 0.11594069004058838, "contribution_to_active": 0.3390066623687744, "mix_mean": 0.4853251874446869, "mix_low": 0.01619434356689453, "mix_high": 0.023817062377929688, "read_cosine": 0.9223617911338806, "beta_mean": 0.04591234400868416, "nonfinite": 0.0, "weighted_memory_to_active": 0.047388602048158646, "mix_std": 0.28576797246932983, "beta_std": 0.040146827697753906, "beta_low": 0.04083251953125, "beta_high": 0.0, "orthogonal_to_active": 0.015308738686144352, "evidence_weights": [-0.26981720328330994, -0.25346988439559937, 0.23339080810546875], "agreement_weight": 0.2267245352268219, "orthogonal_weight": -0.5516034960746765}]}, {"layer": 1, "memory_to_active": 2.173546552658081, "contribution_to_active": 0.45918071269989014, "mix_mean": 0.15789873898029327, "mix_low": 0.37095263600349426, "mix_high": 0.0004804929194506258, "read_cosine": 0.3399204909801483, "beta_mean": 0.18598613142967224, "nonfinite": 0.0, "weighted_memory_to_active": 0.35102593898773193, "mix_std": 0.12002415210008621, "beta_std": 0.12779201567173004, "beta_low": 0.1787923276424408, "beta_high": 0.0, "orthogonal_to_active": 0.043244000524282455, "output_influence": 0.17557618021965027, "residual_influence": 0.042402058839797974, "state_active": 0.042042702436447144, "state_memory": 0.0375061072409153, "state_peak": 28.762182235717773, "growth_active": 1.1983097791671753, "growth_memory": 1.5051802396774292, "heads": [{"head": 0, "memory_to_active": 0.04620237648487091, "contribution_to_active": 0.3973245322704315, "mix_mean": 0.36888355016708374, "mix_low": 0.013392448425292969, "mix_high": 8.487701416015625e-05, "read_cosine": 0.03904563933610916, "beta_mean": 0.00029687874484807253, "nonfinite": 0.0, "weighted_memory_to_active": 0.02292175404727459, "mix_std": 0.27159640192985535, "beta_std": 0.00041534905903972685, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.01835886389017105, "evidence_weights": [-0.16952374577522278, -0.2156318575143814, 0.04144081845879555], "agreement_weight": 0.16020341217517853, "orthogonal_weight": -0.8992473483085632}, {"head": 1, "memory_to_active": 2.863281726837158, "contribution_to_active": 0.17647260427474976, "mix_mean": 0.022168241441249847, "mix_low": 0.47348785400390625, "mix_high": 9.5367431640625e-07, "read_cosine": 0.4864697754383087, "beta_mean": 0.3172357976436615, "nonfinite": 0.0, "weighted_memory_to_active": 0.2038148045539856, "mix_std": 0.04698805510997772, "beta_std": 0.19156654179096222, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0076735252514481544, "evidence_weights": [0.006219916976988316, -0.011070962063968182, -0.0053177885711193085], "agreement_weight": -0.008222264237701893, "orthogonal_weight": 0.07939932495355606}, {"head": 2, "memory_to_active": 0.41744622588157654, "contribution_to_active": 0.5083003640174866, "mix_mean": 0.49198687076568604, "mix_low": 0.00328826904296875, "mix_high": 0.002796173095703125, "read_cosine": 0.7672623991966248, "beta_mean": 0.042069047689437866, "nonfinite": 0.0, "weighted_memory_to_active": 0.34021681547164917, "mix_std": 0.26122191548347473, "beta_std": 0.029446033760905266, "beta_low": 0.038818359375, "beta_high": 0.0, "orthogonal_to_active": 0.03726430609822273, "evidence_weights": [-0.27160564064979553, -0.38942137360572815, 0.27340027689933777], "agreement_weight": 0.02647506259381771, "orthogonal_weight": -0.2220911830663681}, {"head": 3, "memory_to_active": 2.7972843647003174, "contribution_to_active": 0.08994505554437637, "mix_mean": 0.017328757792711258, "mix_low": 0.6713228225708008, "mix_high": 0.0, "read_cosine": 0.22548022866249084, "beta_mean": 0.3453897535800934, "nonfinite": 0.0, "weighted_memory_to_active": 0.12326956540346146, "mix_std": 0.04748028516769409, "beta_std": 0.23508694767951965, "beta_low": 0.00921630859375, "beta_high": 0.0, "orthogonal_to_active": 0.0346984937787056, "evidence_weights": [0.04219387471675873, 0.0033992857206612825, 0.11499884724617004], "agreement_weight": -0.02998681180179119, "orthogonal_weight": -0.3084140121936798}, {"head": 4, "memory_to_active": 3.329221248626709, "contribution_to_active": 1.5190117359161377, "mix_mean": 0.038238078355789185, "mix_low": 0.27446937561035156, "mix_high": 9.5367431640625e-07, "read_cosine": 0.34336239099502563, "beta_mean": 0.17090719938278198, "nonfinite": 0.0, "weighted_memory_to_active": 1.3582241535186768, "mix_std": 0.06901020556688309, "beta_std": 0.10540871322154999, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.1583693027496338, "evidence_weights": [-0.03639642521739006, -0.0022204818669706583, 0.023290541023015976], "agreement_weight": -0.03091413341462612, "orthogonal_weight": 0.14145486056804657}, {"head": 5, "memory_to_active": 3.587843418121338, "contribution_to_active": 0.0640297681093216, "mix_mean": 0.008786925114691257, "mix_low": 0.7897548675537109, "mix_high": 0.0, "read_cosine": 0.17790231108665466, "beta_mean": 0.2400180995464325, "nonfinite": 0.0, "weighted_memory_to_active": 0.05770851671695709, "mix_std": 0.023848028853535652, "beta_std": 0.20482856035232544, "beta_low": 0.0242919921875, "beta_high": 0.0, "orthogonal_to_active": 0.0030995160341262817, "evidence_weights": [0.1430504024028778, 0.06397608667612076, 0.3546513020992279], "agreement_weight": -0.1512455940246582, "orthogonal_weight": 0.055933624505996704}]}, {"layer": 2, "memory_to_active": 1.1613863706588745, "contribution_to_active": 0.3230655789375305, "mix_mean": 0.3248176574707031, "mix_low": 0.26688575744628906, "mix_high": 0.0032666525803506374, "read_cosine": 0.32113316655158997, "beta_mean": 0.10196642577648163, "nonfinite": 0.0, "weighted_memory_to_active": 0.06146518141031265, "mix_std": 0.1957782357931137, "beta_std": 0.07498027384281158, "beta_low": 0.547943115234375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.016000866889953613, "output_influence": 0.29934218525886536, "residual_influence": 0.07228090614080429, "state_active": 0.037661999464035034, "state_memory": 0.019665740430355072, "state_peak": 42.90117263793945, "growth_active": 1.0363072156906128, "growth_memory": 1.5639954805374146, "heads": [{"head": 0, "memory_to_active": 0.013759409077465534, "contribution_to_active": 0.2985337972640991, "mix_mean": 0.47352200746536255, "mix_low": 0.0029277801513671875, "mix_high": 0.0042057037353515625, "read_cosine": 0.2210490107536316, "beta_mean": 8.084608998615295e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006918936036527157, "mix_std": 0.29632997512817383, "beta_std": 0.00012250459985807538, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009839744307100773, "evidence_weights": [-0.271390825510025, -0.1755012720823288, 0.24582912027835846], "agreement_weight": 0.18160057067871094, "orthogonal_weight": -0.1533913016319275}, {"head": 1, "memory_to_active": 3.406930923461914, "contribution_to_active": 0.04148712754249573, "mix_mean": 0.0050049759447574615, "mix_low": 0.8900184631347656, "mix_high": 0.0, "read_cosine": 0.2136232703924179, "beta_mean": 0.3252458870410919, "nonfinite": 0.0, "weighted_memory_to_active": 0.03415696322917938, "mix_std": 0.011041178368031979, "beta_std": 0.1577426791191101, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012257324531674385, "evidence_weights": [0.09829625487327576, 0.1044299378991127, 0.41565588116645813], "agreement_weight": 0.041083648800849915, "orthogonal_weight": 0.4187597632408142}, {"head": 2, "memory_to_active": 0.02920355089008808, "contribution_to_active": 0.45320308208465576, "mix_mean": 0.4530601501464844, "mix_low": 0.0024633407592773438, "mix_high": 0.0029926300048828125, "read_cosine": 0.18867981433868408, "beta_mean": 4.726862971438095e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01164286583662033, "mix_std": 0.28247883915901184, "beta_std": 8.834235632093623e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030659823678433895, "evidence_weights": [-0.21893298625946045, -0.25708913803100586, 0.48148155212402344], "agreement_weight": 0.04779921844601631, "orthogonal_weight": -0.29647302627563477}, {"head": 3, "memory_to_active": 0.2042175978422165, "contribution_to_active": 0.48237934708595276, "mix_mean": 0.43478521704673767, "mix_low": 0.0011301040649414062, "mix_high": 0.005173683166503906, "read_cosine": 0.6494156122207642, "beta_mean": 0.042335301637649536, "nonfinite": 0.0, "weighted_memory_to_active": 0.11019814014434814, "mix_std": 0.26682114601135254, "beta_std": 0.052945878356695175, "beta_low": 0.239501953125, "beta_high": 0.0, "orthogonal_to_active": 0.06777846813201904, "evidence_weights": [-0.16102521121501923, -0.2491653859615326, 0.8368410468101501], "agreement_weight": 0.06317581981420517, "orthogonal_weight": -0.8632939457893372}, {"head": 4, "memory_to_active": 3.2931246757507324, "contribution_to_active": 0.15951426327228546, "mix_mean": 0.013477875851094723, "mix_low": 0.7028160095214844, "mix_high": 0.0, "read_cosine": 0.20624879002571106, "beta_mean": 0.24403256177902222, "nonfinite": 0.0, "weighted_memory_to_active": 0.19280563294887543, "mix_std": 0.03773472458124161, "beta_std": 0.23889067769050598, "beta_low": 0.04815673828125, "beta_high": 0.000732421875, "orthogonal_to_active": 0.010849370621144772, "evidence_weights": [0.03879864513874054, 0.037235938012599945, 0.12781928479671478], "agreement_weight": 0.0291807372123003, "orthogonal_weight": -0.06654991209506989}, {"head": 5, "memory_to_active": 0.02108180522918701, "contribution_to_active": 0.5032757520675659, "mix_mean": 0.569055438041687, "mix_low": 0.0019588470458984375, "mix_high": 0.007227897644042969, "read_cosine": 0.44778233766555786, "beta_mean": 5.671945109497756e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0130685493350029, "mix_std": 0.2802635431289673, "beta_std": 9.153767314273864e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010700779967010021, "evidence_weights": [-0.2506594955921173, -0.27454274892807007, 0.7920593023300171], "agreement_weight": 0.04727161303162575, "orthogonal_weight": 0.1340814232826233}]}, {"layer": 3, "memory_to_active": 1.0809518098831177, "contribution_to_active": 0.3695097267627716, "mix_mean": 0.2953982651233673, "mix_low": 0.20040561258792877, "mix_high": 0.005003770813345909, "read_cosine": 0.24345287680625916, "beta_mean": 0.07511842250823975, "nonfinite": 0.0, "weighted_memory_to_active": 0.1538422554731369, "mix_std": 0.22897431254386902, "beta_std": 0.06123558431863785, "beta_low": 0.4582112729549408, "beta_high": 0.0, "orthogonal_to_active": 0.054207682609558105, "output_influence": 0.28442877531051636, "residual_influence": 0.05460529774427414, "state_active": 0.025844963267445564, "state_memory": 0.014567196369171143, "state_peak": 22.14252281188965, "growth_active": 1.1126376390457153, "growth_memory": 1.7746728658676147, "heads": [{"head": 0, "memory_to_active": 0.5388375520706177, "contribution_to_active": 0.4642746150493622, "mix_mean": 0.3838805556297302, "mix_low": 0.032360076904296875, "mix_high": 0.007611274719238281, "read_cosine": 0.20886370539665222, "beta_mean": 0.013401789590716362, "nonfinite": 0.0, "weighted_memory_to_active": 0.24470150470733643, "mix_std": 0.3244214653968811, "beta_std": 0.018755994737148285, "beta_low": 0.58990478515625, "beta_high": 0.0, "orthogonal_to_active": 0.14839833974838257, "evidence_weights": [-0.14442870020866394, -0.19266144931316376, 0.46431025862693787], "agreement_weight": 0.11307522654533386, "orthogonal_weight": -0.6440624594688416}, {"head": 1, "memory_to_active": 0.016422700136899948, "contribution_to_active": 0.3275279998779297, "mix_mean": 0.39030927419662476, "mix_low": 0.028102874755859375, "mix_high": 0.0031499862670898438, "read_cosine": 0.21814364194869995, "beta_mean": 7.922571967355907e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006765937432646751, "mix_std": 0.3069510757923126, "beta_std": 0.00013714043598156422, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0021242662332952023, "evidence_weights": [-0.1938052624464035, -0.16602052748203278, 0.5011160969734192], "agreement_weight": 0.2877960205078125, "orthogonal_weight": -0.34136998653411865}, {"head": 2, "memory_to_active": 2.777294635772705, "contribution_to_active": 0.29830095171928406, "mix_mean": 0.04180067032575607, "mix_low": 0.41454219818115234, "mix_high": 0.0, "read_cosine": 0.17228484153747559, "beta_mean": 0.20839940011501312, "nonfinite": 0.0, "weighted_memory_to_active": 0.36547231674194336, "mix_std": 0.09424338489770889, "beta_std": 0.14812898635864258, "beta_low": 0.00445556640625, "beta_high": 0.0, "orthogonal_to_active": 0.012255384586751461, "evidence_weights": [-0.05197715759277344, 0.021593786776065826, -0.02247755415737629], "agreement_weight": -0.020984185859560966, "orthogonal_weight": -0.042961567640304565}, {"head": 3, "memory_to_active": 0.031040318310260773, "contribution_to_active": 0.4819478988647461, "mix_mean": 0.5008848905563354, "mix_low": 0.018587112426757812, "mix_high": 0.011624336242675781, "read_cosine": 0.28264081478118896, "beta_mean": 0.00012008905468974262, "nonfinite": 0.0, "weighted_memory_to_active": 0.018691878765821457, "mix_std": 0.3131314516067505, "beta_std": 0.0002559195854701102, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005356361158192158, "evidence_weights": [-0.19381897151470184, -0.2124788612127304, 1.759549617767334], "agreement_weight": 1.0806481838226318, "orthogonal_weight": -0.3797075152397156}, {"head": 4, "memory_to_active": 2.794321298599243, "contribution_to_active": 0.03592735528945923, "mix_mean": 0.01399158500134945, "mix_low": 0.6883516311645508, "mix_high": 0.0, "read_cosine": 0.19638286530971527, "beta_mean": 0.17681317031383514, "nonfinite": 0.0, "weighted_memory_to_active": 0.07408268749713898, "mix_std": 0.030781211331486702, "beta_std": 0.14867159724235535, "beta_low": 0.009033203125, "beta_high": 0.0, "orthogonal_to_active": 0.042620301246643066, "evidence_weights": [0.12806499004364014, 0.08899994939565659, -0.0372372642159462], "agreement_weight": -0.04366137832403183, "orthogonal_weight": -0.5935527682304382}, {"head": 5, "memory_to_active": 0.3277938961982727, "contribution_to_active": 0.6090793013572693, "mix_mean": 0.44152241945266724, "mix_low": 0.02048969268798828, "mix_high": 0.00763702392578125, "read_cosine": 0.3824012577533722, "beta_mean": 0.051896803081035614, "nonfinite": 0.0, "weighted_memory_to_active": 0.21333914995193481, "mix_std": 0.30431729555130005, "beta_std": 0.05146385729312897, "beta_low": 0.1458740234375, "beta_high": 0.0, "orthogonal_to_active": 0.11449144035577774, "evidence_weights": [-0.17243598401546478, -0.22263523936271667, 0.3845420777797699], "agreement_weight": 0.16087974607944489, "orthogonal_weight": -0.684625506401062}]}, {"layer": 4, "memory_to_active": 1.2333264350891113, "contribution_to_active": 0.41538089513778687, "mix_mean": 0.32800930738449097, "mix_low": 0.2809782028198242, "mix_high": 0.005564212799072266, "read_cosine": 0.38258302211761475, "beta_mean": 0.12727685272693634, "nonfinite": 0.0, "weighted_memory_to_active": 0.14991670846939087, "mix_std": 0.15904325246810913, "beta_std": 0.11201906949281693, "beta_low": 0.3651631772518158, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04363797977566719, "output_influence": 0.33516520261764526, "residual_influence": 0.08499100059270859, "state_active": 0.0364355742931366, "state_memory": 0.022238684818148613, "state_peak": 42.01166915893555, "growth_active": 1.1335861682891846, "growth_memory": 1.5572898387908936, "heads": [{"head": 0, "memory_to_active": 0.2618771195411682, "contribution_to_active": 0.677815854549408, "mix_mean": 0.6171157360076904, "mix_low": 0.010387420654296875, "mix_high": 0.013211250305175781, "read_cosine": 0.5005859136581421, "beta_mean": 0.24326609075069427, "nonfinite": 0.0, "weighted_memory_to_active": 0.2003737986087799, "mix_std": 0.3328996002674103, "beta_std": 0.25555169582366943, "beta_low": 0.2037353515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.053092967718839645, "evidence_weights": [-0.2573026120662689, -0.3442223370075226, 0.3694066107273102], "agreement_weight": 0.02990739978849888, "orthogonal_weight": 0.33373138308525085}, {"head": 1, "memory_to_active": 0.11911197006702423, "contribution_to_active": 0.5418261289596558, "mix_mean": 0.580136775970459, "mix_low": 8.0108642578125e-05, "mix_high": 0.005608558654785156, "read_cosine": 0.7317420244216919, "beta_mean": 0.003204475622624159, "nonfinite": 0.0, "weighted_memory_to_active": 0.07369069010019302, "mix_std": 0.26010841131210327, "beta_std": 0.007614540867507458, "beta_low": 0.9383544921875, "beta_high": 0.0, "orthogonal_to_active": 0.03737518563866615, "evidence_weights": [-0.23386681079864502, -0.27883702516555786, 0.7092570662498474], "agreement_weight": 0.07008028030395508, "orthogonal_weight": -0.8023815751075745}, {"head": 2, "memory_to_active": 2.395071029663086, "contribution_to_active": 0.13204270601272583, "mix_mean": 0.015961376950144768, "mix_low": 0.6206769943237305, "mix_high": 0.0, "read_cosine": 0.3108714520931244, "beta_mean": 0.11194884032011032, "nonfinite": 0.0, "weighted_memory_to_active": 0.16227982938289642, "mix_std": 0.03029409423470497, "beta_std": 0.08937322348356247, "beta_low": 0.00274658203125, "beta_high": 0.0, "orthogonal_to_active": 0.010676075704395771, "evidence_weights": [0.0547855868935585, 0.03200162574648857, -0.03287092223763466], "agreement_weight": 0.09305323660373688, "orthogonal_weight": -0.09323912113904953}, {"head": 3, "memory_to_active": 1.8929109573364258, "contribution_to_active": 0.12648612260818481, "mix_mean": 0.031193846836686134, "mix_low": 0.41875267028808594, "mix_high": 0.0, "read_cosine": 0.1619727611541748, "beta_mean": 0.1574546843767166, "nonfinite": 0.0, "weighted_memory_to_active": 0.19965876638889313, "mix_std": 0.06076239421963692, "beta_std": 0.14137835800647736, "beta_low": 0.046142578125, "beta_high": 0.0, "orthogonal_to_active": 0.018408581614494324, "evidence_weights": [-0.019157245755195618, 0.05903303995728493, -0.2630414664745331], "agreement_weight": 0.04759472236037254, "orthogonal_weight": -0.13541164994239807}, {"head": 4, "memory_to_active": 0.014580819755792618, "contribution_to_active": 0.6474543213844299, "mix_mean": 0.7089077234268188, "mix_low": 0.00048065185546875, "mix_high": 0.014556884765625, "read_cosine": 0.2221744805574417, "beta_mean": 2.6706202334025875e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010528191924095154, "mix_std": 0.2328106313943863, "beta_std": 6.775636575184762e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017386398976668715, "evidence_weights": [-0.3029543459415436, -0.3362954258918762, 0.7461225390434265], "agreement_weight": 0.14353635907173157, "orthogonal_weight": -0.18428274989128113}, {"head": 5, "memory_to_active": 2.7164061069488525, "contribution_to_active": 0.3666602671146393, "mix_mean": 0.014740416780114174, "mix_low": 0.6354913711547852, "mix_high": 8.58306884765625e-06, "read_cosine": 0.3681514859199524, "beta_mean": 0.24776031076908112, "nonfinite": 0.0, "weighted_memory_to_active": 0.2529689371585846, "mix_std": 0.03738433122634888, "beta_std": 0.17812883853912354, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14053639769554138, "evidence_weights": [0.04258362203836441, -0.03650350496172905, -0.19564679265022278], "agreement_weight": -0.050156086683273315, "orthogonal_weight": 0.7800900340080261}]}, {"layer": 5, "memory_to_active": 0.6085449457168579, "contribution_to_active": 0.4743504226207733, "mix_mean": 0.4210496246814728, "mix_low": 0.08569082617759705, "mix_high": 0.010713895782828331, "read_cosine": 0.18147985637187958, "beta_mean": 0.058846473693847656, "nonfinite": 0.0, "weighted_memory_to_active": 0.16036152839660645, "mix_std": 0.25019571185112, "beta_std": 0.07442142069339752, "beta_low": 0.611602783203125, "beta_high": 0.0, "orthogonal_to_active": 0.05363461375236511, "output_influence": 0.45501142740249634, "residual_influence": 0.11180263757705688, "state_active": 0.05099305510520935, "state_memory": 0.01299336738884449, "state_peak": 63.10858154296875, "growth_active": 1.1534479856491089, "growth_memory": 1.278287410736084, "heads": [{"head": 0, "memory_to_active": 1.0212944746017456, "contribution_to_active": 0.33046016097068787, "mix_mean": 0.16478955745697021, "mix_low": 0.10607147216796875, "mix_high": 0.000408172607421875, "read_cosine": -0.16460078954696655, "beta_mean": 0.057350438088178635, "nonfinite": 0.0, "weighted_memory_to_active": 0.16936933994293213, "mix_std": 0.20165479183197021, "beta_std": 0.08321121335029602, "beta_low": 0.22296142578125, "beta_high": 0.0, "orthogonal_to_active": 0.035554490983486176, "evidence_weights": [-0.0637940764427185, -0.09930548816919327, -0.35605189204216003], "agreement_weight": -0.01679414138197899, "orthogonal_weight": -0.20405232906341553}, {"head": 1, "memory_to_active": 0.02684408612549305, "contribution_to_active": 0.529742956161499, "mix_mean": 0.590752899646759, "mix_low": 0.004054069519042969, "mix_high": 0.01569652557373047, "read_cosine": 0.1789339780807495, "beta_mean": 5.485950532602146e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014399691484868526, "mix_std": 0.3038737177848816, "beta_std": 0.00011583997547859326, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013114675879478455, "evidence_weights": [-0.23532572388648987, -0.2622626721858978, 0.8109796643257141], "agreement_weight": 0.3705962002277374, "orthogonal_weight": -0.9351656436920166}, {"head": 2, "memory_to_active": 0.03773342818021774, "contribution_to_active": 0.5320891737937927, "mix_mean": 0.6050083637237549, "mix_low": 0.001972198486328125, "mix_high": 0.013152122497558594, "read_cosine": 0.19173938035964966, "beta_mean": 0.0006919716252014041, "nonfinite": 0.0, "weighted_memory_to_active": 0.02573942020535469, "mix_std": 0.2746126353740692, "beta_std": 0.0010486742248758674, "beta_low": 0.998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.004823347553610802, "evidence_weights": [-0.25341883301734924, -0.3375350832939148, 0.8784304857254028], "agreement_weight": 0.01995963044464588, "orthogonal_weight": -0.236075758934021}, {"head": 3, "memory_to_active": 2.0217247009277344, "contribution_to_active": 0.22880640625953674, "mix_mean": 0.0822717621922493, "mix_low": 0.20585250854492188, "mix_high": 0.0, "read_cosine": 0.15593115985393524, "beta_mean": 0.10545116662979126, "nonfinite": 0.0, "weighted_memory_to_active": 0.38282281160354614, "mix_std": 0.13216567039489746, "beta_std": 0.1426772177219391, "beta_low": 0.18719482421875, "beta_high": 0.0, "orthogonal_to_active": 0.14024662971496582, "evidence_weights": [-0.04379988834261894, -0.08530902862548828, -0.08910585194826126], "agreement_weight": 0.056876737624406815, "orthogonal_weight": -0.46074679493904114}, {"head": 4, "memory_to_active": 0.44286873936653137, "contribution_to_active": 0.6012630462646484, "mix_mean": 0.33271241188049316, "mix_low": 0.19322967529296875, "mix_high": 0.009229660034179688, "read_cosine": 0.5435377359390259, "beta_mean": 0.18916304409503937, "nonfinite": 0.0, "weighted_memory_to_active": 0.3020187020301819, "mix_std": 0.3414214253425598, "beta_std": 0.21749675273895264, "beta_low": 0.26617431640625, "beta_high": 0.0, "orthogonal_to_active": 0.0852588564157486, "evidence_weights": [-0.11706849932670593, -0.29172053933143616, -0.04997768625617027], "agreement_weight": 0.04844246804714203, "orthogonal_weight": 0.4301210343837738}, {"head": 5, "memory_to_active": 0.10080408304929733, "contribution_to_active": 0.6237406730651855, "mix_mean": 0.7507628202438354, "mix_low": 0.0029649734497070312, "mix_high": 0.025796890258789062, "read_cosine": 0.18333770334720612, "beta_mean": 0.00036738079506903887, "nonfinite": 0.0, "weighted_memory_to_active": 0.06781914830207825, "mix_std": 0.2474460005760193, "beta_std": 0.001978820888325572, "beta_low": 0.9949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.042809683829545975, "evidence_weights": [-0.324520081281662, -0.3155260384082794, 1.9270755052566528], "agreement_weight": 0.3460155427455902, "orthogonal_weight": -0.6705501079559326}]}, {"layer": 6, "memory_to_active": 0.3462502360343933, "contribution_to_active": 0.578865647315979, "mix_mean": 0.5369497537612915, "mix_low": 0.07837899774312973, "mix_high": 0.026837985962629318, "read_cosine": 0.2970384359359741, "beta_mean": 0.06861607730388641, "nonfinite": 0.0, "weighted_memory_to_active": 0.1607716679573059, "mix_std": 0.24400606751441956, "beta_std": 0.07781753689050674, "beta_low": 0.5029602646827698, "beta_high": 0.0, "orthogonal_to_active": 0.033299095928668976, "output_influence": 0.4754263758659363, "residual_influence": 0.1441263109445572, "state_active": 0.046982910484075546, "state_memory": 0.008863378316164017, "state_peak": 82.76701354980469, "growth_active": 1.2142609357833862, "growth_memory": 1.190116047859192, "heads": [{"head": 0, "memory_to_active": 0.12987872958183289, "contribution_to_active": 0.60968416929245, "mix_mean": 0.7111232280731201, "mix_low": 4.482269287109375e-05, "mix_high": 0.004944801330566406, "read_cosine": 0.0671239048242569, "beta_mean": 0.013784612528979778, "nonfinite": 0.0, "weighted_memory_to_active": 0.09428410232067108, "mix_std": 0.22924986481666565, "beta_std": 0.01668369770050049, "beta_low": 0.54217529296875, "beta_high": 0.0, "orthogonal_to_active": 0.07904883474111557, "evidence_weights": [-0.30603474378585815, -0.32179880142211914, 0.4903471767902374], "agreement_weight": 0.10188467800617218, "orthogonal_weight": -0.9252004027366638}, {"head": 1, "memory_to_active": 0.005066382233053446, "contribution_to_active": 0.4707605838775635, "mix_mean": 0.5010170340538025, "mix_low": 0.0011453628540039062, "mix_high": 0.010652542114257812, "read_cosine": 0.6116785407066345, "beta_mean": 1.0933099474641494e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.001939298352226615, "mix_std": 0.2917436957359314, "beta_std": 3.352271960466169e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000635860487818718, "evidence_weights": [-0.20733001828193665, -0.27368250489234924, 0.8626757264137268], "agreement_weight": -0.02604520507156849, "orthogonal_weight": -0.4501587748527527}, {"head": 2, "memory_to_active": 1.0228281021118164, "contribution_to_active": 0.06256447732448578, "mix_mean": 0.025646589696407318, "mix_low": 0.4590444564819336, "mix_high": 0.0, "read_cosine": 0.20463091135025024, "beta_mean": 0.05591024458408356, "nonfinite": 0.0, "weighted_memory_to_active": 0.05188499763607979, "mix_std": 0.044649429619312286, "beta_std": 0.05265846103429794, "beta_low": 0.0281982421875, "beta_high": 0.0, "orthogonal_to_active": 0.0025650395546108484, "evidence_weights": [0.09274667501449585, 0.010613197460770607, 0.23594120144844055], "agreement_weight": 0.11968623846769333, "orthogonal_weight": -0.05690180882811546}, {"head": 3, "memory_to_active": 0.15070940554141998, "contribution_to_active": 0.6428674459457397, "mix_mean": 0.6315693259239197, "mix_low": 0.008313179016113281, "mix_high": 0.017330169677734375, "read_cosine": 0.4943605363368988, "beta_mean": 0.21498340368270874, "nonfinite": 0.0, "weighted_memory_to_active": 0.10703476518392563, "mix_std": 0.33462825417518616, "beta_std": 0.2499549686908722, "beta_low": 0.2103271484375, "beta_high": 0.0, "orthogonal_to_active": 0.05322963744401932, "evidence_weights": [-0.17390431463718414, -0.3378009498119354, 0.14894409477710724], "agreement_weight": -0.006145274732261896, "orthogonal_weight": 0.5910972952842712}, {"head": 4, "memory_to_active": 0.7391083836555481, "contribution_to_active": 1.1024326086044312, "mix_mean": 0.7472668886184692, "mix_low": 0.0012807846069335938, "mix_high": 0.1262836456298828, "read_cosine": 0.13347694277763367, "beta_mean": 0.12677550315856934, "nonfinite": 0.0, "weighted_memory_to_active": 0.6894084215164185, "mix_std": 0.29492026567459106, "beta_std": 0.14710162580013275, "beta_low": 0.2373046875, "beta_high": 0.0, "orthogonal_to_active": 0.04391998052597046, "evidence_weights": [-0.25720226764678955, -0.5895270109176636, -0.09856873005628586], "agreement_weight": -0.01621910370886326, "orthogonal_weight": 0.07601156830787659}, {"head": 5, "memory_to_active": 0.029910219833254814, "contribution_to_active": 0.584884524345398, "mix_mean": 0.605075478553772, "mix_low": 0.00044536590576171875, "mix_high": 0.0018167495727539062, "read_cosine": 0.27095961570739746, "beta_mean": 0.00023179070558398962, "nonfinite": 0.0, "weighted_memory_to_active": 0.020078368484973907, "mix_std": 0.2688448429107666, "beta_std": 0.0004729546490125358, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.020395206287503242, "evidence_weights": [-0.23738005757331848, -0.29493290185928345, 0.38354232907295227], "agreement_weight": 0.10641026496887207, "orthogonal_weight": -1.082015037536621}]}, {"layer": 7, "memory_to_active": 1.8670299053192139, "contribution_to_active": 0.1909058392047882, "mix_mean": 0.06297618895769119, "mix_low": 0.36424335837364197, "mix_high": 3.5444896639091894e-05, "read_cosine": 0.23150767385959625, "beta_mean": 0.12607550621032715, "nonfinite": 0.0, "weighted_memory_to_active": 0.21357180178165436, "mix_std": 0.09588775038719177, "beta_std": 0.10715397447347641, "beta_low": 0.11962890625, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.05719809979200363, "output_influence": 0.21531057357788086, "residual_influence": 0.04719869792461395, "state_active": 0.015763215720653534, "state_memory": 0.02858223021030426, "state_peak": 33.07835006713867, "growth_active": 1.0593540668487549, "growth_memory": 1.3052003383636475, "heads": [{"head": 0, "memory_to_active": 2.3157691955566406, "contribution_to_active": 0.31076309084892273, "mix_mean": 0.03226836025714874, "mix_low": 0.47724342346191406, "mix_high": 7.62939453125e-06, "read_cosine": 0.36863940954208374, "beta_mean": 0.15661108493804932, "nonfinite": 0.0, "weighted_memory_to_active": 0.31607526540756226, "mix_std": 0.06938973069190979, "beta_std": 0.11350143700838089, "beta_low": 0.0025634765625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.024721860885620117, "evidence_weights": [0.005710127297788858, -0.08937951922416687, 0.10915157198905945], "agreement_weight": -0.03411327302455902, "orthogonal_weight": 0.1068003848195076}, {"head": 1, "memory_to_active": 1.2915161848068237, "contribution_to_active": 0.3227297067642212, "mix_mean": 0.1503431648015976, "mix_low": 0.21201038360595703, "mix_high": 0.0001201629638671875, "read_cosine": 0.2587941884994507, "beta_mean": 0.12096970528364182, "nonfinite": 0.0, "weighted_memory_to_active": 0.326111376285553, "mix_std": 0.18577814102172852, "beta_std": 0.07865136116743088, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.01859215274453163, "evidence_weights": [-0.1728876531124115, -0.11609863489866257, -0.2484826147556305], "agreement_weight": 0.021741174161434174, "orthogonal_weight": -0.06283649802207947}, {"head": 2, "memory_to_active": 0.9727194309234619, "contribution_to_active": 0.25926709175109863, "mix_mean": 0.08106797188520432, "mix_low": 0.1966266632080078, "mix_high": 8.487701416015625e-05, "read_cosine": 0.11609868705272675, "beta_mean": 0.08092944324016571, "nonfinite": 0.0, "weighted_memory_to_active": 0.17258864641189575, "mix_std": 0.13717123866081238, "beta_std": 0.06076037511229515, "beta_low": 0.0472412109375, "beta_high": 0.0, "orthogonal_to_active": 0.033022426068782806, "evidence_weights": [-0.03216053172945976, -0.11897476762533188, 0.262838751077652], "agreement_weight": 0.018403997644782066, "orthogonal_weight": -0.19563636183738708}, {"head": 3, "memory_to_active": 3.940103769302368, "contribution_to_active": 0.07341795414686203, "mix_mean": 0.03212112560868263, "mix_low": 0.3671121597290039, "mix_high": 0.0, "read_cosine": 0.11434851586818695, "beta_mean": 0.2504124045372009, "nonfinite": 0.0, "weighted_memory_to_active": 0.2670195698738098, "mix_std": 0.050899989902973175, "beta_std": 0.2342994511127472, "beta_low": 0.0223388671875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.21816964447498322, "evidence_weights": [0.07507269084453583, 0.019835859537124634, 0.5720787644386292], "agreement_weight": 0.0635533332824707, "orthogonal_weight": -0.8980880975723267}, {"head": 4, "memory_to_active": 1.8221499919891357, "contribution_to_active": 0.06513551622629166, "mix_mean": 0.012662041001021862, "mix_low": 0.6905832290649414, "mix_high": 0.0, "read_cosine": 0.2919861972332001, "beta_mean": 0.1341170370578766, "nonfinite": 0.0, "weighted_memory_to_active": 0.07722613215446472, "mix_std": 0.026632076129317284, "beta_std": 0.13432469964027405, "beta_low": 0.036865234375, "beta_high": 0.0, "orthogonal_to_active": 0.0035304322373121977, "evidence_weights": [0.122083380818367, 0.05471410974860191, 0.11109793931245804], "agreement_weight": 0.04134894534945488, "orthogonal_weight": -0.0634489580988884}, {"head": 5, "memory_to_active": 0.8599206805229187, "contribution_to_active": 0.11412163078784943, "mix_mean": 0.06939448416233063, "mix_low": 0.2418842315673828, "mix_high": 0.0, "read_cosine": 0.23917900025844574, "beta_mean": 0.013413268141448498, "nonfinite": 0.0, "weighted_memory_to_active": 0.12240982055664062, "mix_std": 0.10545533150434494, "beta_std": 0.02138647437095642, "beta_low": 0.60870361328125, "beta_high": 0.0, "orthogonal_to_active": 0.045152075588703156, "evidence_weights": [0.016938617452979088, -0.023419035598635674, 0.1437438279390335], "agreement_weight": 0.06487495452165604, "orthogonal_weight": -0.48858964443206787}]}, {"layer": 8, "memory_to_active": 0.5629982948303223, "contribution_to_active": 0.5052412748336792, "mix_mean": 0.394196093082428, "mix_low": 0.09775066375732422, "mix_high": 0.0007061958312988281, "read_cosine": 0.5310170650482178, "beta_mean": 0.03738383948802948, "nonfinite": 0.0, "weighted_memory_to_active": 0.11748090386390686, "mix_std": 0.22098594903945923, "beta_std": 0.03807906433939934, "beta_low": 0.5485026240348816, "beta_high": 0.0, "orthogonal_to_active": 0.05240398645401001, "output_influence": 0.40015077590942383, "residual_influence": 0.11832693219184875, "state_active": 0.05382174998521805, "state_memory": 0.011340986005961895, "state_peak": 34.57595443725586, "growth_active": 1.0663559436798096, "growth_memory": 1.351373553276062, "heads": [{"head": 0, "memory_to_active": 0.042338043451309204, "contribution_to_active": 0.5321956872940063, "mix_mean": 0.5676044225692749, "mix_low": 0.00027751922607421875, "mix_high": 0.00048160552978515625, "read_cosine": 0.6855306625366211, "beta_mean": 0.003204901237040758, "nonfinite": 0.0, "weighted_memory_to_active": 0.026323996484279633, "mix_std": 0.2378828227519989, "beta_std": 0.007221434265375137, "beta_low": 0.919921875, "beta_high": 0.0, "orthogonal_to_active": 0.014888131991028786, "evidence_weights": [-0.17997175455093384, -0.3770922124385834, 0.7149716019630432], "agreement_weight": 0.07464096695184708, "orthogonal_weight": -0.811677873134613}, {"head": 1, "memory_to_active": 0.3074471950531006, "contribution_to_active": 0.4243311285972595, "mix_mean": 0.41892921924591064, "mix_low": 0.002918243408203125, "mix_high": 0.000377655029296875, "read_cosine": 0.4435577392578125, "beta_mean": 0.011696094647049904, "nonfinite": 0.0, "weighted_memory_to_active": 0.14192049205303192, "mix_std": 0.2480601966381073, "beta_std": 0.01874512992799282, "beta_low": 0.657958984375, "beta_high": 0.0, "orthogonal_to_active": 0.07139388471841812, "evidence_weights": [-0.1846717894077301, -0.22793519496917725, 0.6281148791313171], "agreement_weight": 0.07101387530565262, "orthogonal_weight": -0.5827870965003967}, {"head": 2, "memory_to_active": 0.03972892463207245, "contribution_to_active": 0.6194154620170593, "mix_mean": 0.5782933831214905, "mix_low": 0.0009250640869140625, "mix_high": 0.0026159286499023438, "read_cosine": 0.39818137884140015, "beta_mean": 0.00036738041671924293, "nonfinite": 0.0, "weighted_memory_to_active": 0.02853032387793064, "mix_std": 0.24561874568462372, "beta_std": 0.002502585295587778, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.020200874656438828, "evidence_weights": [-0.2117338329553604, -0.32697683572769165, 0.6564353108406067], "agreement_weight": 0.13385041058063507, "orthogonal_weight": -0.8171417117118835}, {"head": 3, "memory_to_active": 0.14396342635154724, "contribution_to_active": 0.5699238777160645, "mix_mean": 0.5016888976097107, "mix_low": 0.0005741119384765625, "mix_high": 0.0002841949462890625, "read_cosine": 0.7984830737113953, "beta_mean": 0.013311007060110569, "nonfinite": 0.0, "weighted_memory_to_active": 0.09794744849205017, "mix_std": 0.2514907121658325, "beta_std": 0.012060371227562428, "beta_low": 0.4490966796875, "beta_high": 0.0, "orthogonal_to_active": 0.028810745105147362, "evidence_weights": [-0.16225075721740723, -0.28227534890174866, 0.7173646092414856], "agreement_weight": 0.11850723624229431, "orthogonal_weight": -0.5761962532997131}, {"head": 4, "memory_to_active": 2.6061675548553467, "contribution_to_active": 0.2791575789451599, "mix_mean": 0.023594390600919724, "mix_low": 0.4482421875, "mix_high": 0.0, "read_cosine": 0.30029135942459106, "beta_mean": 0.12651580572128296, "nonfinite": 0.0, "weighted_memory_to_active": 0.2627919614315033, "mix_std": 0.04023313522338867, "beta_std": 0.09208212792873383, "beta_low": 0.00469970703125, "beta_high": 0.0, "orthogonal_to_active": 0.008373817428946495, "evidence_weights": [0.0455879345536232, -0.0082103181630373, -0.1673939973115921], "agreement_weight": 0.03626187518239021, "orthogonal_weight": 0.03267782926559448}, {"head": 5, "memory_to_active": 0.23834455013275146, "contribution_to_active": 0.6064237356185913, "mix_mean": 0.2750660181045532, "mix_low": 0.13356685638427734, "mix_high": 0.00047779083251953125, "read_cosine": 0.5600577592849731, "beta_mean": 0.06920786201953888, "nonfinite": 0.0, "weighted_memory_to_active": 0.14737117290496826, "mix_std": 0.3026300370693207, "beta_std": 0.0958627313375473, "beta_low": 0.25994873046875, "beta_high": 0.0, "orthogonal_to_active": 0.17075645923614502, "evidence_weights": [-0.06259406358003616, -0.2974461317062378, -0.4175311326980591], "agreement_weight": 0.01528361439704895, "orthogonal_weight": 1.4597307443618774}]}, {"layer": 9, "memory_to_active": 1.100621223449707, "contribution_to_active": 0.5957121849060059, "mix_mean": 0.2853248119354248, "mix_low": 0.15794388949871063, "mix_high": 0.0018757185898721218, "read_cosine": 0.3794206380844116, "beta_mean": 0.07650444656610489, "nonfinite": 0.0, "weighted_memory_to_active": 0.3465442657470703, "mix_std": 0.2086181938648224, "beta_std": 0.04757075756788254, "beta_low": 0.499176025390625, "beta_high": 0.0, "orthogonal_to_active": 0.03637273609638214, "output_influence": 0.45002880692481995, "residual_influence": 0.12090742588043213, "state_active": 0.04193957895040512, "state_memory": 0.02139207348227501, "state_peak": 35.161842346191406, "growth_active": 1.224691390991211, "growth_memory": 1.396019697189331, "heads": [{"head": 0, "memory_to_active": 0.007210547104477882, "contribution_to_active": 0.5112389326095581, "mix_mean": 0.5707752704620361, "mix_low": 0.0018491744995117188, "mix_high": 0.0077667236328125, "read_cosine": 0.25071364641189575, "beta_mean": 1.4317870409286115e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004274229519069195, "mix_std": 0.29469919204711914, "beta_std": 6.690953159704804e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003649241989478469, "evidence_weights": [-0.2844947576522827, -0.28944748640060425, 0.9616631269454956], "agreement_weight": 0.08202527463436127, "orthogonal_weight": -0.9059136509895325}, {"head": 1, "memory_to_active": 0.029239404946565628, "contribution_to_active": 0.5619524717330933, "mix_mean": 0.4342954158782959, "mix_low": 0.010967254638671875, "mix_high": 0.00091552734375, "read_cosine": 0.41679853200912476, "beta_mean": 7.939046918181702e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01659283973276615, "mix_std": 0.28824836015701294, "beta_std": 0.00015479084686376154, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010745378211140633, "evidence_weights": [-0.21193401515483856, -0.24953505396842957, 1.0894078016281128], "agreement_weight": 0.3060894012451172, "orthogonal_weight": -0.7605788707733154}, {"head": 2, "memory_to_active": 0.21380361914634705, "contribution_to_active": 0.5801936984062195, "mix_mean": 0.48997989296913147, "mix_low": 0.0033788681030273438, "mix_high": 0.0023660659790039062, "read_cosine": 0.37982290983200073, "beta_mean": 0.0009167434764094651, "nonfinite": 0.0, "weighted_memory_to_active": 0.12835121154785156, "mix_std": 0.28249481320381165, "beta_std": 0.005119456443935633, "beta_low": 0.99188232421875, "beta_high": 0.0, "orthogonal_to_active": 0.05509041249752045, "evidence_weights": [-0.17702022194862366, -0.22473007440567017, 0.5306578278541565], "agreement_weight": 0.23011626303195953, "orthogonal_weight": -0.5158492922782898}, {"head": 3, "memory_to_active": 1.619356393814087, "contribution_to_active": 0.17567062377929688, "mix_mean": 0.03911053389310837, "mix_low": 0.39211273193359375, "mix_high": 1.71661376953125e-05, "read_cosine": 0.41672852635383606, "beta_mean": 0.08376270532608032, "nonfinite": 0.0, "weighted_memory_to_active": 0.14496400952339172, "mix_std": 0.0890347883105278, "beta_std": 0.06119091808795929, "beta_low": 0.00311279296875, "beta_high": 0.0, "orthogonal_to_active": 0.026004279032349586, "evidence_weights": [0.06346283853054047, 0.03884340822696686, -0.09947724640369415], "agreement_weight": 0.07792641222476959, "orthogonal_weight": 0.19398847222328186}, {"head": 4, "memory_to_active": 3.0449581146240234, "contribution_to_active": 1.1706469058990479, "mix_mean": 0.11716103553771973, "mix_low": 0.17374038696289062, "mix_high": 0.0001888275146484375, "read_cosine": 0.3330134153366089, "beta_mean": 0.21255511045455933, "nonfinite": 0.0, "weighted_memory_to_active": 1.128699541091919, "mix_std": 0.18006828427314758, "beta_std": 0.11348628997802734, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016421286389231682, "evidence_weights": [-0.05353347584605217, -0.14481118321418762, 0.03150835633277893], "agreement_weight": -0.05132022127509117, "orthogonal_weight": -0.014766663312911987}, {"head": 5, "memory_to_active": 1.6891595125198364, "contribution_to_active": 0.5745700001716614, "mix_mean": 0.06062661111354828, "mix_low": 0.3656148910522461, "mix_high": 0.0, "read_cosine": 0.4794465899467468, "beta_mean": 0.16169840097427368, "nonfinite": 0.0, "weighted_memory_to_active": 0.6563836336135864, "mix_std": 0.11716373264789581, "beta_std": 0.10540619492530823, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.10632580518722534, "evidence_weights": [0.0850292518734932, -0.0463222973048687, 0.1327875256538391], "agreement_weight": 0.022383583709597588, "orthogonal_weight": 0.23552954196929932}]}, {"layer": 10, "memory_to_active": 0.19110023975372314, "contribution_to_active": 0.5183596611022949, "mix_mean": 0.4685923457145691, "mix_low": 0.011970044113695621, "mix_high": 0.009833971969783306, "read_cosine": 0.3743712306022644, "beta_mean": 0.04465107619762421, "nonfinite": 0.0, "weighted_memory_to_active": 0.09127075225114822, "mix_std": 0.2663494050502777, "beta_std": 0.04844094067811966, "beta_low": 0.6676839590072632, "beta_high": 0.0, "orthogonal_to_active": 0.07562816888093948, "output_influence": 0.39972183108329773, "residual_influence": 0.1338551938533783, "state_active": 0.05319952592253685, "state_memory": 0.013603719882667065, "state_peak": 109.11646270751953, "growth_active": 1.041701078414917, "growth_memory": 0.9886683225631714, "heads": [{"head": 0, "memory_to_active": 0.607075035572052, "contribution_to_active": 1.0251541137695312, "mix_mean": 0.6278682947158813, "mix_low": 0.027029037475585938, "mix_high": 0.05086326599121094, "read_cosine": 0.12409190833568573, "beta_mean": 0.19910046458244324, "nonfinite": 0.0, "weighted_memory_to_active": 0.4085908830165863, "mix_std": 0.37365543842315674, "beta_std": 0.2216988205909729, "beta_low": 0.152099609375, "beta_high": 0.0, "orthogonal_to_active": 0.37194058299064636, "evidence_weights": [-0.14970937371253967, -0.365273118019104, -0.5497121810913086], "agreement_weight": 0.019240370020270348, "orthogonal_weight": 0.9393956065177917}, {"head": 1, "memory_to_active": 0.01514086127281189, "contribution_to_active": 0.5594481229782104, "mix_mean": 0.5676523447036743, "mix_low": 0.00048828125, "mix_high": 0.006199836730957031, "read_cosine": 0.27651911973953247, "beta_mean": 9.424473682884127e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008664562366902828, "mix_std": 0.2666124403476715, "beta_std": 0.00012423074804246426, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010820808820426464, "evidence_weights": [-0.2720504403114319, -0.19828002154827118, 0.5794060826301575], "agreement_weight": 0.4003134071826935, "orthogonal_weight": -1.3944090604782104}, {"head": 2, "memory_to_active": 0.031077250838279724, "contribution_to_active": 0.4917217791080475, "mix_mean": 0.503472089767456, "mix_low": 0.0005044937133789062, "mix_high": 0.001739501953125, "read_cosine": 0.01534334197640419, "beta_mean": 0.0001131677272496745, "nonfinite": 0.0, "weighted_memory_to_active": 0.011340145021677017, "mix_std": 0.2652985453605652, "beta_std": 0.0004491247236728668, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.0043243346735835075, "evidence_weights": [-0.19637760519981384, -0.24233776330947876, 0.5253033638000488], "agreement_weight": -0.05761824548244476, "orthogonal_weight": -0.37011218070983887}, {"head": 3, "memory_to_active": 0.33242520689964294, "contribution_to_active": 0.10361848771572113, "mix_mean": 0.24952802062034607, "mix_low": 0.04026603698730469, "mix_high": 0.00011539459228515625, "read_cosine": 0.8997904658317566, "beta_mean": 0.06255195289850235, "nonfinite": 0.0, "weighted_memory_to_active": 0.04019560664892197, "mix_std": 0.2194186896085739, "beta_std": 0.058885447680950165, "beta_low": 0.00335693359375, "beta_high": 0.0, "orthogonal_to_active": 0.04569970443844795, "evidence_weights": [-0.12320531159639359, -0.14680823683738708, 0.151799738407135], "agreement_weight": 0.003218545578420162, "orthogonal_weight": 1.471989393234253}, {"head": 4, "memory_to_active": 0.0171889066696167, "contribution_to_active": 0.49978891015052795, "mix_mean": 0.49631690979003906, "mix_low": 0.001216888427734375, "mix_high": 7.915496826171875e-05, "read_cosine": 0.07058389484882355, "beta_mean": 3.8595477235503495e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005170756950974464, "mix_std": 0.2509337365627289, "beta_std": 0.0016280735144391656, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0019119760254397988, "evidence_weights": [-0.18112976849079132, -0.23811882734298706, 0.6931743025779724], "agreement_weight": 0.06372233480215073, "orthogonal_weight": -0.3710843324661255}, {"head": 5, "memory_to_active": 0.14369410276412964, "contribution_to_active": 0.4304264485836029, "mix_mean": 0.36671608686447144, "mix_low": 0.002315521240234375, "mix_high": 6.67572021484375e-06, "read_cosine": 0.859898567199707, "beta_mean": 0.006008019670844078, "nonfinite": 0.0, "weighted_memory_to_active": 0.07366253435611725, "mix_std": 0.22217753529548645, "beta_std": 0.007859917357563972, "beta_low": 0.85107421875, "beta_high": 0.0, "orthogonal_to_active": 0.019071614369750023, "evidence_weights": [-0.15109311044216156, -0.1608782261610031, 0.29568344354629517], "agreement_weight": 0.09359578788280487, "orthogonal_weight": -0.597398579120636}]}, {"layer": 11, "memory_to_active": 0.17176419496536255, "contribution_to_active": 0.5362284183502197, "mix_mean": 0.4288647472858429, "mix_low": 0.06659587472677231, "mix_high": 0.032180946320295334, "read_cosine": 0.4400525689125061, "beta_mean": 0.03332392871379852, "nonfinite": 0.0, "weighted_memory_to_active": 0.06844650954008102, "mix_std": 0.3152170181274414, "beta_std": 0.047984812408685684, "beta_low": 0.5488179922103882, "beta_high": 0.0, "orthogonal_to_active": 0.05236881226301193, "output_influence": 0.49033597111701965, "residual_influence": 0.12336060404777527, "state_active": 0.14180153608322144, "state_memory": 0.008089411072432995, "state_peak": 26.16539764404297, "growth_active": 1.044896125793457, "growth_memory": 1.5229880809783936, "heads": [{"head": 0, "memory_to_active": 0.014061196707189083, "contribution_to_active": 0.5675292015075684, "mix_mean": 0.5834043622016907, "mix_low": 0.0010690689086914062, "mix_high": 0.008625030517578125, "read_cosine": 0.32776325941085815, "beta_mean": 0.00014273106353357434, "nonfinite": 0.0, "weighted_memory_to_active": 0.009364485740661621, "mix_std": 0.2704172134399414, "beta_std": 0.004722415469586849, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.0062264250591397285, "evidence_weights": [-0.16154912114143372, -0.35289818048477173, 0.6132216453552246], "agreement_weight": 0.20250171422958374, "orthogonal_weight": -0.7503301501274109}, {"head": 1, "memory_to_active": 0.38393449783325195, "contribution_to_active": 0.2573077082633972, "mix_mean": 0.2882717251777649, "mix_low": 0.020223617553710938, "mix_high": 0.000118255615234375, "read_cosine": 0.6075023412704468, "beta_mean": 0.03191972151398659, "nonfinite": 0.0, "weighted_memory_to_active": 0.14455440640449524, "mix_std": 0.24582047760486603, "beta_std": 0.05916711688041687, "beta_low": 0.4326171875, "beta_high": 0.0, "orthogonal_to_active": 0.030895141884684563, "evidence_weights": [-0.15322332084178925, -0.13347113132476807, 0.2687567174434662], "agreement_weight": 0.07723405957221985, "orthogonal_weight": -0.28613585233688354}, {"head": 2, "memory_to_active": 0.028016190975904465, "contribution_to_active": 0.9222066402435303, "mix_mean": 0.4613705277442932, "mix_low": 0.12153148651123047, "mix_high": 0.10301780700683594, "read_cosine": 0.2606915235519409, "beta_mean": 0.06986518949270248, "nonfinite": 0.0, "weighted_memory_to_active": 0.02172575332224369, "mix_std": 0.42299097776412964, "beta_std": 0.09485912322998047, "beta_low": 0.45379638671875, "beta_high": 0.0, "orthogonal_to_active": 0.08668236434459686, "evidence_weights": [-0.0235617496073246, -0.4562532901763916, -1.14577317237854], "agreement_weight": -0.002383077284321189, "orthogonal_weight": 4.236223220825195}, {"head": 3, "memory_to_active": 0.014469337649643421, "contribution_to_active": 0.7728506326675415, "mix_mean": 0.4136213958263397, "mix_low": 0.23862934112548828, "mix_high": 0.07988548278808594, "read_cosine": 0.51429682970047, "beta_mean": 0.05671442672610283, "nonfinite": 0.0, "weighted_memory_to_active": 0.00829289760440588, "mix_std": 0.4327813684940338, "beta_std": 0.08464129269123077, "beta_low": 0.4906005859375, "beta_high": 0.0, "orthogonal_to_active": 0.032726775854825974, "evidence_weights": [0.036104653030633926, -0.35501739382743835, -0.4644246995449066], "agreement_weight": 0.03887498751282692, "orthogonal_weight": 4.534760475158691}, {"head": 4, "memory_to_active": 0.4923352599143982, "contribution_to_active": 0.33925655484199524, "mix_mean": 0.35363131761550903, "mix_low": 0.017591476440429688, "mix_high": 0.00029754638671875, "read_cosine": 0.16615459322929382, "beta_mean": 0.03321874514222145, "nonfinite": 0.0, "weighted_memory_to_active": 0.1849692314863205, "mix_std": 0.25878599286079407, "beta_std": 0.03157436475157738, "beta_low": 0.16204833984375, "beta_high": 0.0, "orthogonal_to_active": 0.1367940753698349, "evidence_weights": [-0.18696175515651703, -0.15304690599441528, 1.5498390197753906], "agreement_weight": -0.1583370417356491, "orthogonal_weight": -0.7616209387779236}, {"head": 5, "memory_to_active": 0.097768634557724, "contribution_to_active": 0.35821980237960815, "mix_mean": 0.47288912534713745, "mix_low": 0.000530242919921875, "mix_high": 0.0011415481567382812, "read_cosine": 0.7639067769050598, "beta_mean": 0.008082768879830837, "nonfinite": 0.0, "weighted_memory_to_active": 0.041772302240133286, "mix_std": 0.2605059742927551, "beta_std": 0.012944558635354042, "beta_low": 0.754638671875, "beta_high": 0.0, "orthogonal_to_active": 0.020888082683086395, "evidence_weights": [-0.204266756772995, -0.26055780053138733, 0.5172563791275024], "agreement_weight": 0.059817347675561905, "orthogonal_weight": -0.6712160706520081}]}, {"layer": 12, "memory_to_active": 0.3311863839626312, "contribution_to_active": 0.5466862320899963, "mix_mean": 0.46294108033180237, "mix_low": 0.014513015747070312, "mix_high": 0.0029571852646768093, "read_cosine": 0.24365773797035217, "beta_mean": 0.01471199281513691, "nonfinite": 0.0, "weighted_memory_to_active": 0.17915359139442444, "mix_std": 0.2563997507095337, "beta_std": 0.01610920950770378, "beta_low": 0.81500244140625, "beta_high": 0.0, "orthogonal_to_active": 0.06835974752902985, "output_influence": 0.41699254512786865, "residual_influence": 0.11559779196977615, "state_active": 0.041233841329813004, "state_memory": 0.010090887546539307, "state_peak": 24.596223831176758, "growth_active": 1.109013557434082, "growth_memory": 1.269623875617981, "heads": [{"head": 0, "memory_to_active": 0.09748007357120514, "contribution_to_active": 0.44908130168914795, "mix_mean": 0.4288022518157959, "mix_low": 0.0027685165405273438, "mix_high": 0.000690460205078125, "read_cosine": 0.250734806060791, "beta_mean": 0.0016864330973476171, "nonfinite": 0.0, "weighted_memory_to_active": 0.041970327496528625, "mix_std": 0.2640409767627716, "beta_std": 0.005944897420704365, "beta_low": 0.97003173828125, "beta_high": 0.0, "orthogonal_to_active": 0.024824943393468857, "evidence_weights": [-0.1704702526330948, -0.2432618886232376, 0.2911091446876526], "agreement_weight": 0.11036650836467743, "orthogonal_weight": -0.6177757978439331}, {"head": 1, "memory_to_active": 0.10741111636161804, "contribution_to_active": 0.486032634973526, "mix_mean": 0.4929736256599426, "mix_low": 0.0007457733154296875, "mix_high": 0.0002422332763671875, "read_cosine": 0.5620967149734497, "beta_mean": 0.0011783327208831906, "nonfinite": 0.0, "weighted_memory_to_active": 0.05456022545695305, "mix_std": 0.25992095470428467, "beta_std": 0.005535658448934555, "beta_low": 0.9825439453125, "beta_high": 0.0, "orthogonal_to_active": 0.03969583660364151, "evidence_weights": [-0.1992168128490448, -0.26227715611457825, 0.8684377074241638], "agreement_weight": 0.10993853956460953, "orthogonal_weight": -0.9106147289276123}, {"head": 2, "memory_to_active": 0.608502984046936, "contribution_to_active": 0.44111156463623047, "mix_mean": 0.3142716884613037, "mix_low": 0.02538013458251953, "mix_high": 0.00024318695068359375, "read_cosine": 0.43949997425079346, "beta_mean": 0.08224734663963318, "nonfinite": 0.0, "weighted_memory_to_active": 0.24222995340824127, "mix_std": 0.2427445650100708, "beta_std": 0.062207408249378204, "beta_low": 0.00335693359375, "beta_high": 0.0, "orthogonal_to_active": 0.20136983692646027, "evidence_weights": [-0.24131959676742554, -0.040363285690546036, -0.06676296144723892], "agreement_weight": 0.031780797988176346, "orthogonal_weight": 1.022840976715088}, {"head": 3, "memory_to_active": 0.022770872339606285, "contribution_to_active": 0.7179097533226013, "mix_mean": 0.6807363629341125, "mix_low": 0.0002613067626953125, "mix_high": 0.012389183044433594, "read_cosine": 0.03242773935198784, "beta_mean": 0.0002007474540732801, "nonfinite": 0.0, "weighted_memory_to_active": 0.017187412828207016, "mix_std": 0.2412712723016739, "beta_std": 0.0028968777041882277, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.01578175090253353, "evidence_weights": [-0.2992386817932129, -0.33153706789016724, 1.1999454498291016], "agreement_weight": 0.28156912326812744, "orthogonal_weight": -0.9393542408943176}, {"head": 4, "memory_to_active": 1.0654314756393433, "contribution_to_active": 0.7295764088630676, "mix_mean": 0.26617610454559326, "mix_low": 0.05117225646972656, "mix_high": 0.0014009475708007812, "read_cosine": -0.07495652884244919, "beta_mean": 0.002184155862778425, "nonfinite": 0.0, "weighted_memory_to_active": 0.6674108505249023, "mix_std": 0.2542266249656677, "beta_std": 0.01590278558433056, "beta_low": 0.951416015625, "beta_high": 0.0, "orthogonal_to_active": 0.08847475796937943, "evidence_weights": [-0.05648697540163994, -0.19436907768249512, 0.022566823288798332], "agreement_weight": -8.952265488915145e-05, "orthogonal_weight": -0.14238013327121735}, {"head": 5, "memory_to_active": 0.0855216532945633, "contribution_to_active": 0.45640552043914795, "mix_mean": 0.5946863889694214, "mix_low": 0.0067501068115234375, "mix_high": 0.002777099609375, "read_cosine": 0.2521437108516693, "beta_mean": 0.0007749397191219032, "nonfinite": 0.0, "weighted_memory_to_active": 0.051562607288360596, "mix_std": 0.2761939764022827, "beta_std": 0.004167640116065741, "beta_low": 0.98345947265625, "beta_high": 0.0, "orthogonal_to_active": 0.04001133143901825, "evidence_weights": [-0.25823596119880676, -0.2991397976875305, 1.0194873809814453], "agreement_weight": 0.19722123444080353, "orthogonal_weight": -0.8331829309463501}]}, {"layer": 13, "memory_to_active": 0.1459636390209198, "contribution_to_active": 0.5000128149986267, "mix_mean": 0.48480406403541565, "mix_low": 0.006155490875244141, "mix_high": 0.003367900848388672, "read_cosine": 0.40068143606185913, "beta_mean": 0.016030773520469666, "nonfinite": 0.0, "weighted_memory_to_active": 0.06260371953248978, "mix_std": 0.25827863812446594, "beta_std": 0.016941148787736893, "beta_low": 0.7897136211395264, "beta_high": 0.0, "orthogonal_to_active": 0.060412339866161346, "output_influence": 0.4486341178417206, "residual_influence": 0.14975158870220184, "state_active": 0.09180260449647903, "state_memory": 0.035915475338697433, "state_peak": 94.88556671142578, "growth_active": 0.9814356565475464, "growth_memory": 1.083369493484497, "heads": [{"head": 0, "memory_to_active": 0.04388820007443428, "contribution_to_active": 0.5317192673683167, "mix_mean": 0.5212676525115967, "mix_low": 0.00044345855712890625, "mix_high": 0.0011472702026367188, "read_cosine": 0.38163986802101135, "beta_mean": 0.00016853657143656164, "nonfinite": 0.0, "weighted_memory_to_active": 0.02484814077615738, "mix_std": 0.2658027708530426, "beta_std": 0.0003916088317055255, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.024374114349484444, "evidence_weights": [-0.20086343586444855, -0.254006028175354, 0.741318941116333], "agreement_weight": 0.062332503497600555, "orthogonal_weight": -1.084983229637146}, {"head": 1, "memory_to_active": 0.035530947148799896, "contribution_to_active": 0.5537739992141724, "mix_mean": 0.5000687837600708, "mix_low": 0.0022563934326171875, "mix_high": 0.0009222030639648438, "read_cosine": 0.22139544785022736, "beta_mean": 0.0001160221072495915, "nonfinite": 0.0, "weighted_memory_to_active": 0.01899130828678608, "mix_std": 0.26759424805641174, "beta_std": 0.0002267609816044569, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.019650373607873917, "evidence_weights": [-0.18404334783554077, -0.26261693239212036, 0.6852824687957764], "agreement_weight": 0.23149429261684418, "orthogonal_weight": -1.0840972661972046}, {"head": 2, "memory_to_active": 0.12153925746679306, "contribution_to_active": 0.39695030450820923, "mix_mean": 0.35727283358573914, "mix_low": 0.0027675628662109375, "mix_high": 3.910064697265625e-05, "read_cosine": 0.6449706554412842, "beta_mean": 0.014082042500376701, "nonfinite": 0.0, "weighted_memory_to_active": 0.04854495823383331, "mix_std": 0.219273641705513, "beta_std": 0.03948868811130524, "beta_low": 0.75762939453125, "beta_high": 0.0, "orthogonal_to_active": 0.02416682243347168, "evidence_weights": [-0.22024528682231903, -0.1725892871618271, 0.47814038395881653], "agreement_weight": 0.0252374354749918, "orthogonal_weight": -0.7800354361534119}, {"head": 3, "memory_to_active": 0.13746437430381775, "contribution_to_active": 0.5934334993362427, "mix_mean": 0.5962866544723511, "mix_low": 0.01770782470703125, "mix_high": 0.013918876647949219, "read_cosine": 0.025057435035705566, "beta_mean": 0.001221438986249268, "nonfinite": 0.0, "weighted_memory_to_active": 0.09623751789331436, "mix_std": 0.2882310748100281, "beta_std": 0.004113737493753433, "beta_low": 0.9881591796875, "beta_high": 0.0, "orthogonal_to_active": 0.09558374434709549, "evidence_weights": [-0.3427259922027588, -0.23527956008911133, 2.0297608375549316], "agreement_weight": 2.6993346214294434, "orthogonal_weight": -1.0585777759552002}, {"head": 4, "memory_to_active": 0.42560485005378723, "contribution_to_active": 0.27468523383140564, "mix_mean": 0.35163116455078125, "mix_low": 0.011080741882324219, "mix_high": 3.910064697265625e-05, "read_cosine": 0.8661092519760132, "beta_mean": 0.07972486317157745, "nonfinite": 0.0, "weighted_memory_to_active": 0.12146802246570587, "mix_std": 0.2293209433555603, "beta_std": 0.052350424230098724, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.14707204699516296, "evidence_weights": [-0.16924113035202026, -0.23379191756248474, 0.5010985136032104], "agreement_weight": 0.0010839293245226145, "orthogonal_weight": 1.8855655193328857}, {"head": 5, "memory_to_active": 0.1117541491985321, "contribution_to_active": 0.6495145559310913, "mix_mean": 0.582297146320343, "mix_low": 0.0026769638061523438, "mix_high": 0.0041408538818359375, "read_cosine": 0.2649158835411072, "beta_mean": 0.0008717232849448919, "nonfinite": 0.0, "weighted_memory_to_active": 0.0655323714017868, "mix_std": 0.2794489860534668, "beta_std": 0.005075667519122362, "beta_low": 0.9923095703125, "beta_high": 0.0, "orthogonal_to_active": 0.05162695795297623, "evidence_weights": [-0.21938073635101318, -0.2863750457763672, 0.6045725345611572], "agreement_weight": 0.08899666368961334, "orthogonal_weight": -0.8631125688552856}]}, {"layer": 14, "memory_to_active": 0.2582516074180603, "contribution_to_active": 0.4221966862678528, "mix_mean": 0.4024977684020996, "mix_low": 0.02533864974975586, "mix_high": 0.0038940112572163343, "read_cosine": 0.6153545379638672, "beta_mean": 0.06581000238656998, "nonfinite": 0.0, "weighted_memory_to_active": 0.11539734154939651, "mix_std": 0.24518302083015442, "beta_std": 0.06191246211528778, "beta_low": 0.444122314453125, "beta_high": 0.0, "orthogonal_to_active": 0.041289448738098145, "output_influence": 0.29694268107414246, "residual_influence": 0.11474008858203888, "state_active": 0.0665682852268219, "state_memory": 0.015439126640558243, "state_peak": 27.408855438232422, "growth_active": 1.132947564125061, "growth_memory": 1.3046283721923828, "heads": [{"head": 0, "memory_to_active": 0.34057095646858215, "contribution_to_active": 0.20413216948509216, "mix_mean": 0.21795357763767242, "mix_low": 0.01662158966064453, "mix_high": 4.76837158203125e-06, "read_cosine": 0.7461867332458496, "beta_mean": 0.06295627355575562, "nonfinite": 0.0, "weighted_memory_to_active": 0.09480778872966766, "mix_std": 0.19918504357337952, "beta_std": 0.06275396794080734, "beta_low": 0.03466796875, "beta_high": 0.0, "orthogonal_to_active": 0.04931732639670372, "evidence_weights": [-0.11650518327951431, -0.06020652875304222, 0.6797988414764404], "agreement_weight": 0.027780979871749878, "orthogonal_weight": 0.8240150809288025}, {"head": 1, "memory_to_active": 0.4909448027610779, "contribution_to_active": 0.3722311556339264, "mix_mean": 0.24241380393505096, "mix_low": 0.11550521850585938, "mix_high": 5.817413330078125e-05, "read_cosine": 0.7200099229812622, "beta_mean": 0.27351126074790955, "nonfinite": 0.0, "weighted_memory_to_active": 0.22730328142642975, "mix_std": 0.25982466340065, "beta_std": 0.25136497616767883, "beta_low": 0.101318359375, "beta_high": 0.0, "orthogonal_to_active": 0.018312465399503708, "evidence_weights": [0.007542169187217951, -0.2660558521747589, -0.17577621340751648], "agreement_weight": 0.05244402959942818, "orthogonal_weight": 0.15268005430698395}, {"head": 2, "memory_to_active": 0.18060629069805145, "contribution_to_active": 0.5231158137321472, "mix_mean": 0.525337815284729, "mix_low": 0.002712249755859375, "mix_high": 0.0012989044189453125, "read_cosine": 0.4028500020503998, "beta_mean": 0.00498782517388463, "nonfinite": 0.0, "weighted_memory_to_active": 0.10387396812438965, "mix_std": 0.2664957344532013, "beta_std": 0.007031954824924469, "beta_low": 0.8748779296875, "beta_high": 0.0, "orthogonal_to_active": 0.07763518393039703, "evidence_weights": [-0.1708386093378067, -0.28650256991386414, 0.853972315788269], "agreement_weight": 0.07984359562397003, "orthogonal_weight": -0.8506051301956177}, {"head": 3, "memory_to_active": 0.065925233066082, "contribution_to_active": 0.6631127595901489, "mix_mean": 0.6631850004196167, "mix_low": 0.003939628601074219, "mix_high": 0.014904022216796875, "read_cosine": 0.3058510422706604, "beta_mean": 0.009664366021752357, "nonfinite": 0.0, "weighted_memory_to_active": 0.04570874944329262, "mix_std": 0.24426525831222534, "beta_std": 0.008972764015197754, "beta_low": 0.650146484375, "beta_high": 0.0, "orthogonal_to_active": 0.02680302783846855, "evidence_weights": [-0.37942302227020264, -0.3271542489528656, 0.44104430079460144], "agreement_weight": 0.08413340151309967, "orthogonal_weight": -0.6988585591316223}, {"head": 4, "memory_to_active": 0.17387452721595764, "contribution_to_active": 0.469125896692276, "mix_mean": 0.46086758375167847, "mix_low": 0.004534721374511719, "mix_high": 0.006587028503417969, "read_cosine": 0.7992533445358276, "beta_mean": 0.04162364453077316, "nonfinite": 0.0, "weighted_memory_to_active": 0.09522461891174316, "mix_std": 0.2623213231563568, "beta_std": 0.030704081058502197, "beta_low": 0.03765869140625, "beta_high": 0.0, "orthogonal_to_active": 0.02833222597837448, "evidence_weights": [-0.21682338416576385, -0.26230478286743164, 0.4069058895111084], "agreement_weight": 0.12000615149736404, "orthogonal_weight": -0.5357305407524109}, {"head": 5, "memory_to_active": 0.29758766293525696, "contribution_to_active": 0.30146223306655884, "mix_mean": 0.3052288293838501, "mix_low": 0.008718490600585938, "mix_high": 0.00051116943359375, "read_cosine": 0.7179760932922363, "beta_mean": 0.0021166508086025715, "nonfinite": 0.0, "weighted_memory_to_active": 0.12546560168266296, "mix_std": 0.23900601267814636, "beta_std": 0.010647023096680641, "beta_low": 0.966064453125, "beta_high": 0.0, "orthogonal_to_active": 0.047336433082818985, "evidence_weights": [-0.18248210847377777, -0.1521952897310257, 0.2002372294664383], "agreement_weight": 0.08638542890548706, "orthogonal_weight": -0.6203498244285583}]}, {"layer": 15, "memory_to_active": 0.06664074212312698, "contribution_to_active": 0.48123496770858765, "mix_mean": 0.3975665867328644, "mix_low": 0.007738749496638775, "mix_high": 0.0026375455781817436, "read_cosine": 0.6623044013977051, "beta_mean": 0.03977308049798012, "nonfinite": 0.0, "weighted_memory_to_active": 0.03055817261338234, "mix_std": 0.2808844745159149, "beta_std": 0.04374627023935318, "beta_low": 0.7109171748161316, "beta_high": 0.0, "orthogonal_to_active": 0.007399043068289757, "output_influence": 0.3043411374092102, "residual_influence": 0.16466563940048218, "state_active": 0.11805148422718048, "state_memory": 0.008511570282280445, "state_peak": 41.91395950317383, "growth_active": 1.3139967918395996, "growth_memory": 1.8886619806289673, "heads": [{"head": 0, "memory_to_active": 0.10861652344465256, "contribution_to_active": 0.4810173809528351, "mix_mean": 0.36357390880584717, "mix_low": 0.0077304840087890625, "mix_high": 0.004444122314453125, "read_cosine": 0.6924582719802856, "beta_mean": 0.036706991493701935, "nonfinite": 0.0, "weighted_memory_to_active": 0.05085235834121704, "mix_std": 0.2722412943840027, "beta_std": 0.026165705174207687, "beta_low": 0.11212158203125, "beta_high": 0.0, "orthogonal_to_active": 0.018895098939538002, "evidence_weights": [-0.16467656195163727, -0.18543004989624023, 0.346231073141098], "agreement_weight": 0.08754228055477142, "orthogonal_weight": -0.6109395623207092}, {"head": 1, "memory_to_active": 0.13890878856182098, "contribution_to_active": 0.4141123294830322, "mix_mean": 0.4034508168697357, "mix_low": 0.0041294097900390625, "mix_high": 0.003930091857910156, "read_cosine": 0.8761062622070312, "beta_mean": 0.001191003480926156, "nonfinite": 0.0, "weighted_memory_to_active": 0.053217578679323196, "mix_std": 0.28086769580841064, "beta_std": 0.0028642688412219286, "beta_low": 0.981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.001720495754852891, "evidence_weights": [-0.17794956266880035, -0.18467403948307037, 0.27141401171684265], "agreement_weight": 0.09719981998205185, "orthogonal_weight": -0.05003893002867699}, {"head": 2, "memory_to_active": 0.008509445935487747, "contribution_to_active": 0.442279577255249, "mix_mean": 0.34265395998954773, "mix_low": 0.013401031494140625, "mix_high": 6.008148193359375e-05, "read_cosine": 0.7894922494888306, "beta_mean": 0.00014714623102918267, "nonfinite": 0.0, "weighted_memory_to_active": 0.003743608482182026, "mix_std": 0.27259549498558044, "beta_std": 0.0003494624688755721, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00014089148316998035, "evidence_weights": [-0.20579208433628082, -0.2758943438529968, 0.3186262249946594], "agreement_weight": 0.016199784353375435, "orthogonal_weight": 0.07607444375753403}, {"head": 3, "memory_to_active": 0.03868700563907623, "contribution_to_active": 0.4499925374984741, "mix_mean": 0.33749690651893616, "mix_low": 0.0049686431884765625, "mix_high": 4.863739013671875e-05, "read_cosine": 0.49137717485427856, "beta_mean": 0.0005783118540421128, "nonfinite": 0.0, "weighted_memory_to_active": 0.018480218946933746, "mix_std": 0.2692071795463562, "beta_std": 0.0027584664057940245, "beta_low": 0.98992919921875, "beta_high": 0.0, "orthogonal_to_active": 0.013382804580032825, "evidence_weights": [-0.19977326691150665, -0.24951694905757904, 0.38197407126426697], "agreement_weight": 0.020772691816091537, "orthogonal_weight": -0.9397878646850586}, {"head": 4, "memory_to_active": 0.09267427027225494, "contribution_to_active": 0.6192641854286194, "mix_mean": 0.39698493480682373, "mix_low": 0.008378982543945312, "mix_high": 0.000629425048828125, "read_cosine": 0.7051426768302917, "beta_mean": 0.1995343565940857, "nonfinite": 0.0, "weighted_memory_to_active": 0.04980309680104256, "mix_std": 0.30866914987564087, "beta_std": 0.22777962684631348, "beta_low": 0.19036865234375, "beta_high": 0.0, "orthogonal_to_active": 0.0035462016239762306, "evidence_weights": [-0.23157279193401337, -0.2589778006076813, 0.056445080786943436], "agreement_weight": 0.0128939189016819, "orthogonal_weight": 0.1296347677707672}, {"head": 5, "memory_to_active": 0.012448383495211601, "contribution_to_active": 0.48074355721473694, "mix_mean": 0.5412390232086182, "mix_low": 0.007823944091796875, "mix_high": 0.006712913513183594, "read_cosine": 0.41924944519996643, "beta_mean": 0.0004806581127922982, "nonfinite": 0.0, "weighted_memory_to_active": 0.007252159528434277, "mix_std": 0.2817259430885315, "beta_std": 0.0025600732769817114, "beta_low": 0.991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.006708764936774969, "evidence_weights": [-0.3482900559902191, -0.3046712875366211, 0.43450728058815], "agreement_weight": 0.25305816531181335, "orthogonal_weight": 1.0876567363739014}]}]}, {"step": 26000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.824142903999018, "memory_to_active": 0.9128516912460327, "contribution_to_active": 0.49700239300727844, "mix_mean": 0.34953537583351135, "mix_low": 0.15959596633911133, "mix_high": 0.007176349870860577, "read_cosine": 0.38081109523773193, "beta_mean": 0.08556346595287323, "nonfinite": 0.0, "weighted_memory_to_active": 0.21026703715324402, "mix_std": 0.21509656310081482, "beta_std": 0.07099580019712448, "beta_low": 0.4851163327693939, "beta_high": 1.3987224519951269e-05, "orthogonal_to_active": 0.05050566792488098, "samples": 4, "scored_tokens": 899, "normal_loss": 3.119915246963501, "normal_token_accuracy": 0.43492770195007324, "state_active": 0.057234108448028564, "state_memory": 0.020180508494377136, "state_peak": 112.71537017822266, "growth_active": 1.1205132007598877, "growth_memory": 1.381299376487732, "output_influence": 0.35249894857406616, "residual_influence": 0.11252976208925247, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2289323806762695, "no_memory_loss_delta": 0.29916632175445557, "scrambled_loss_delta": 0.24327819049358368, "memory_only_loss_delta": 2.411208391189575, "no_evidence_loss_delta": 0.18161779642105103, "no_agreement_loss_delta": 0.011388782411813736, "no_orthogonal_loss_delta": 0.015157656744122505}, {"index": 94, "tokens": 256, "normal_loss": 3.197967767715454, "no_memory_loss_delta": 0.39901095628738403, "scrambled_loss_delta": 0.4447954297065735, "memory_only_loss_delta": 2.3986334800720215, "no_evidence_loss_delta": 0.19506965577602386, "no_agreement_loss_delta": 0.027748282998800278, "no_orthogonal_loss_delta": 0.030393771827220917}, {"index": 188, "tokens": 256, "normal_loss": 3.511054515838623, "no_memory_loss_delta": 0.35769328474998474, "scrambled_loss_delta": 0.17750751972198486, "memory_only_loss_delta": 2.358711004257202, "no_evidence_loss_delta": 0.22369444370269775, "no_agreement_loss_delta": 0.01795828528702259, "no_orthogonal_loss_delta": 0.02389959618449211}, {"index": 283, "tokens": 131, "normal_loss": 1.9899790287017822, "no_memory_loss_delta": 0.1748283952474594, "scrambled_loss_delta": 0.21987564861774445, "memory_only_loss_delta": 2.7251527309417725, "no_evidence_loss_delta": 0.03773009032011032, "no_agreement_loss_delta": 0.007896944880485535, "no_orthogonal_loss_delta": -0.0006765109137631953}], "agreement_weight_abs_mean": 0.12993746995925903, "agreement_weight_abs_max": 2.8321328163146973, "agreement_weight_mean": 0.11044447869062424, "orthogonal_weight_abs_mean": 0.6228691935539246, "orthogonal_weight_abs_max": 4.726424217224121, "orthogonal_weight_mean": -0.16369347274303436, "no_memory_loss": 3.446061134338379, "no_memory_loss_delta": 0.3261461555957794, "no_memory_token_accuracy_gain": 0.04338153451681137, "no_memory_token_win_fraction": 0.6307007670402527, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 27.830030072645982, "scrambled_loss": 3.3984386920928955, "scrambled_loss_delta": 0.27852335572242737, "scrambled_token_accuracy_gain": 0.02892102301120758, "scrambled_token_win_fraction": 0.6206896305084229, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 24.309941041661176, "memory_only_loss": 5.558340549468994, "memory_only_loss_delta": 2.4384255409240723, "memory_only_token_accuracy_gain": 0.24249166250228882, "memory_only_token_win_fraction": 0.8676307201385498, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 91.27018092551997, "no_evidence_loss": 3.2963783740997314, "no_evidence_loss_delta": 0.17646317183971405, "no_evidence_token_accuracy_gain": 0.017797552049160004, "no_evidence_token_win_fraction": 0.5750834345817566, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 16.177035095738834, "no_agreement_loss": 3.137324094772339, "no_agreement_loss_delta": 0.017409242689609528, "no_agreement_token_accuracy_gain": -0.0033370412420481443, "no_agreement_token_win_fraction": 0.46384871006011963, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 1.7258577413847374, "no_orthogonal_loss": 3.1395931243896484, "no_orthogonal_loss_delta": 0.019678350538015366, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5583982467651367, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.9485995607019049, "short_no_memory_loss_delta": 0.25494253635406494, "long_context_memory_gain": 0.07120361924171448, "normal_context_loss_gain": 0.14769059419631958, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.1845054626464844, "contribution_to_active": 0.9300929307937622, "mix_mean": 0.09881628304719925, "mix_low": 0.5330405235290527, "mix_high": 0.00457000732421875, "read_cosine": 0.46608826518058777, "beta_mean": 0.2665702998638153, "nonfinite": 0.0, "weighted_memory_to_active": 1.0061607360839844, "mix_std": 0.08613263070583344, "beta_std": 0.15761694312095642, "beta_low": 0.00533040426671505, "beta_high": 0.0, "orthogonal_to_active": 0.07208360731601715, "output_influence": 0.15133966505527496, "residual_influence": 0.18090370297431946, "state_active": 0.07627131044864655, "state_memory": 0.04957270249724388, "state_peak": 44.820499420166016, "growth_active": 1.2709944248199463, "growth_memory": 1.261561632156372, "heads": [{"head": 0, "memory_to_active": 2.3723561763763428, "contribution_to_active": 0.37031129002571106, "mix_mean": 0.018810201436281204, "mix_low": 0.62109375, "mix_high": 0.0, "read_cosine": 0.28565311431884766, "beta_mean": 0.2152516096830368, "nonfinite": 0.0, "weighted_memory_to_active": 0.4640446901321411, "mix_std": 0.05450701341032982, "beta_std": 0.10776307433843613, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03588153049349785, "evidence_weights": [-0.03367733210325241, -0.07990778982639313, -0.09909740090370178], "agreement_weight": -0.10611876845359802, "orthogonal_weight": -0.09039337933063507}, {"head": 1, "memory_to_active": 4.756599426269531, "contribution_to_active": 1.721333622932434, "mix_mean": 0.015407603234052658, "mix_low": 0.656773567199707, "mix_high": 0.0, "read_cosine": 0.5162780284881592, "beta_mean": 0.4233260452747345, "nonfinite": 0.0, "weighted_memory_to_active": 2.127824544906616, "mix_std": 0.04854767024517059, "beta_std": 0.18916776776313782, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17887581884860992, "evidence_weights": [0.019642772153019905, -0.07710956782102585, 0.2710467576980591], "agreement_weight": 0.027409318834543228, "orthogonal_weight": -0.18631592392921448}, {"head": 2, "memory_to_active": 2.6968724727630615, "contribution_to_active": 0.49392345547676086, "mix_mean": 0.015455951914191246, "mix_low": 0.624424934387207, "mix_high": 0.0, "read_cosine": 0.33363109827041626, "beta_mean": 0.3347368836402893, "nonfinite": 0.0, "weighted_memory_to_active": 0.5737436413764954, "mix_std": 0.031606730073690414, "beta_std": 0.2355378270149231, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05869264528155327, "evidence_weights": [0.045110467821359634, 0.061205632984638214, -0.04060399904847145], "agreement_weight": -0.0002411417372059077, "orthogonal_weight": -0.18539510667324066}, {"head": 3, "memory_to_active": 6.219286918640137, "contribution_to_active": 2.5859081745147705, "mix_mean": 0.04021108150482178, "mix_low": 0.4198904037475586, "mix_high": 0.0, "read_cosine": 0.4070047438144684, "beta_mean": 0.3624633848667145, "nonfinite": 0.0, "weighted_memory_to_active": 2.7757720947265625, "mix_std": 0.08576042205095291, "beta_std": 0.23244141042232513, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.137691468000412, "evidence_weights": [-0.06783123314380646, -0.09019827097654343, -0.13642831146717072], "agreement_weight": -0.021779412403702736, "orthogonal_weight": -0.060370687395334244}, {"head": 4, "memory_to_active": 2.9472453594207764, "contribution_to_active": 0.05293460562825203, "mix_mean": 0.005320906639099121, "mix_low": 0.8616600036621094, "mix_high": 0.0, "read_cosine": 0.32944056391716003, "beta_mean": 0.21444030106067657, "nonfinite": 0.0, "weighted_memory_to_active": 0.047303199768066406, "mix_std": 0.010342733934521675, "beta_std": 0.1397777944803238, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005527906585484743, "evidence_weights": [0.11201365292072296, 0.058881308883428574, -0.3288053274154663], "agreement_weight": -0.08665613830089569, "orthogonal_weight": 0.12185529619455338}, {"head": 5, "memory_to_active": 0.11467242240905762, "contribution_to_active": 0.35614633560180664, "mix_mean": 0.49769195914268494, "mix_low": 0.014400482177734375, "mix_high": 0.0274200439453125, "read_cosine": 0.9245219230651855, "beta_mean": 0.04920358955860138, "nonfinite": 0.0, "weighted_memory_to_active": 0.04827587679028511, "mix_std": 0.28603118658065796, "beta_std": 0.04101380705833435, "beta_low": 0.031982421875, "beta_high": 0.0, "orthogonal_to_active": 0.01583227328956127, "evidence_weights": [-0.2696804404258728, -0.25384876132011414, 0.2389696091413498], "agreement_weight": 0.23120687901973724, "orthogonal_weight": -0.5732612609863281}]}, {"layer": 1, "memory_to_active": 2.261810302734375, "contribution_to_active": 0.473263144493103, "mix_mean": 0.16297093033790588, "mix_low": 0.38509324193000793, "mix_high": 0.0006376902456395328, "read_cosine": 0.33059316873550415, "beta_mean": 0.19665952026844025, "nonfinite": 0.0, "weighted_memory_to_active": 0.37010258436203003, "mix_std": 0.11805998533964157, "beta_std": 0.1322690099477768, "beta_low": 0.1857706755399704, "beta_high": 4.069010537932627e-05, "orthogonal_to_active": 0.04633302614092827, "output_influence": 0.16524918377399445, "residual_influence": 0.04036802053451538, "state_active": 0.0345681831240654, "state_memory": 0.03787246346473694, "state_peak": 28.78143882751465, "growth_active": 1.0846691131591797, "growth_memory": 1.5037426948547363, "heads": [{"head": 0, "memory_to_active": 0.06083573400974274, "contribution_to_active": 0.4144493043422699, "mix_mean": 0.37543785572052, "mix_low": 0.015181541442871094, "mix_high": 0.0001468658447265625, "read_cosine": -0.029204227030277252, "beta_mean": 0.000382178375730291, "nonfinite": 0.0, "weighted_memory_to_active": 0.03057907335460186, "mix_std": 0.27351993322372437, "beta_std": 0.0005080265109427273, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.02543206885457039, "evidence_weights": [-0.16915296018123627, -0.21752116084098816, 0.06330227851867676], "agreement_weight": 0.15682831406593323, "orthogonal_weight": -0.9201619625091553}, {"head": 1, "memory_to_active": 3.206390857696533, "contribution_to_active": 0.1888032704591751, "mix_mean": 0.019732950255274773, "mix_low": 0.5227899551391602, "mix_high": 9.5367431640625e-07, "read_cosine": 0.4717252850532532, "beta_mean": 0.3464261293411255, "nonfinite": 0.0, "weighted_memory_to_active": 0.20518134534358978, "mix_std": 0.044037848711013794, "beta_std": 0.20658019185066223, "beta_low": 0.0, "beta_high": 0.000244140625, "orthogonal_to_active": 0.007774456404149532, "evidence_weights": [0.010178814642131329, -0.008213039487600327, -0.01486069057136774], "agreement_weight": -0.01165901031345129, "orthogonal_weight": 0.07352785021066666}, {"head": 2, "memory_to_active": 0.474909245967865, "contribution_to_active": 0.4909482002258301, "mix_mean": 0.5236738324165344, "mix_low": 0.0025482177734375, "mix_high": 0.0036773681640625, "read_cosine": 0.7820833921432495, "beta_mean": 0.033269643783569336, "nonfinite": 0.0, "weighted_memory_to_active": 0.38480567932128906, "mix_std": 0.26152244210243225, "beta_std": 0.02332298830151558, "beta_low": 0.08154296875, "beta_high": 0.0, "orthogonal_to_active": 0.046229779720306396, "evidence_weights": [-0.27445271611213684, -0.38684701919555664, 0.2764834463596344], "agreement_weight": 0.02711411565542221, "orthogonal_weight": -0.2325085997581482}, {"head": 3, "memory_to_active": 2.69922137260437, "contribution_to_active": 0.07922127842903137, "mix_mean": 0.016372349113225937, "mix_low": 0.6878261566162109, "mix_high": 0.0, "read_cosine": 0.24567437171936035, "beta_mean": 0.332053542137146, "nonfinite": 0.0, "weighted_memory_to_active": 0.11187051236629486, "mix_std": 0.045679204165935516, "beta_std": 0.2381969839334488, "beta_low": 0.013671875, "beta_high": 0.0, "orthogonal_to_active": 0.03262511268258095, "evidence_weights": [0.04419345036149025, 0.004423557315021753, 0.13225819170475006], "agreement_weight": -0.024888118728995323, "orthogonal_weight": -0.3183474540710449}, {"head": 4, "memory_to_active": 3.4373667240142822, "contribution_to_active": 1.6119472980499268, "mix_mean": 0.03370948135852814, "mix_low": 0.3124704360961914, "mix_high": 9.5367431640625e-07, "read_cosine": 0.34153637290000916, "beta_mean": 0.19975732266902924, "nonfinite": 0.0, "weighted_memory_to_active": 1.4394633769989014, "mix_std": 0.062052808701992035, "beta_std": 0.10944250971078873, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.16322487592697144, "evidence_weights": [-0.03278999775648117, 0.0008655399433337152, -0.010992595925927162], "agreement_weight": -0.026349537074565887, "orthogonal_weight": 0.13924364745616913}, {"head": 5, "memory_to_active": 3.6921372413635254, "contribution_to_active": 0.05420927703380585, "mix_mean": 0.008899062871932983, "mix_low": 0.7697429656982422, "mix_high": 0.0, "read_cosine": 0.17174383997917175, "beta_mean": 0.2680683135986328, "nonfinite": 0.0, "weighted_memory_to_active": 0.048715345561504364, "mix_std": 0.021547667682170868, "beta_std": 0.21556337177753448, "beta_low": 0.0191650390625, "beta_high": 0.0, "orthogonal_to_active": 0.0027118553407490253, "evidence_weights": [0.14313942193984985, 0.06433924287557602, 0.35636621713638306], "agreement_weight": -0.1559208333492279, "orthogonal_weight": 0.055529747158288956}]}, {"layer": 2, "memory_to_active": 1.1717345714569092, "contribution_to_active": 0.3492056429386139, "mix_mean": 0.32691508531570435, "mix_low": 0.26165375113487244, "mix_high": 0.0027190844994038343, "read_cosine": 0.3236008882522583, "beta_mean": 0.09967586398124695, "nonfinite": 0.0, "weighted_memory_to_active": 0.08655718713998795, "mix_std": 0.19512848556041718, "beta_std": 0.07495914399623871, "beta_low": 0.5471903681755066, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.016716832295060158, "output_influence": 0.32496118545532227, "residual_influence": 0.07936494052410126, "state_active": 0.038380272686481476, "state_memory": 0.020017890259623528, "state_peak": 36.23189163208008, "growth_active": 1.0539289712905884, "growth_memory": 1.539523959159851, "heads": [{"head": 0, "memory_to_active": 0.009708081372082233, "contribution_to_active": 0.289105087518692, "mix_mean": 0.47831061482429504, "mix_low": 0.0026111602783203125, "mix_high": 0.0040435791015625, "read_cosine": 0.20846161246299744, "beta_mean": 8.465708378935233e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004925948102027178, "mix_std": 0.2932102680206299, "beta_std": 0.000119530908705201, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007572676986455917, "evidence_weights": [-0.279662549495697, -0.179314985871315, 0.24146433174610138], "agreement_weight": 0.18674884736537933, "orthogonal_weight": -0.16341730952262878}, {"head": 1, "memory_to_active": 3.12701678276062, "contribution_to_active": 0.041693929582834244, "mix_mean": 0.005100348964333534, "mix_low": 0.8907785415649414, "mix_high": 0.0, "read_cosine": 0.18334923684597015, "beta_mean": 0.2843581438064575, "nonfinite": 0.0, "weighted_memory_to_active": 0.034778304398059845, "mix_std": 0.012008599936962128, "beta_std": 0.15581610798835754, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.01201090682297945, "evidence_weights": [0.10110531747341156, 0.1079455018043518, 0.4228273034095764], "agreement_weight": 0.04121776670217514, "orthogonal_weight": 0.41150930523872375}, {"head": 2, "memory_to_active": 0.023597411811351776, "contribution_to_active": 0.4380794167518616, "mix_mean": 0.4472590386867523, "mix_low": 0.0024461746215820312, "mix_high": 0.0019350051879882812, "read_cosine": 0.2395496517419815, "beta_mean": 4.2046474845847115e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009257491677999496, "mix_std": 0.27904337644577026, "beta_std": 8.8381020759698e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023886298295110464, "evidence_weights": [-0.2196839302778244, -0.2566390633583069, 0.4494435489177704], "agreement_weight": 0.03733431175351143, "orthogonal_weight": -0.29789116978645325}, {"head": 3, "memory_to_active": 0.2000133991241455, "contribution_to_active": 0.4926873743534088, "mix_mean": 0.43780866265296936, "mix_low": 0.0010442733764648438, "mix_high": 0.005194664001464844, "read_cosine": 0.6733608245849609, "beta_mean": 0.044388096779584885, "nonfinite": 0.0, "weighted_memory_to_active": 0.10808786749839783, "mix_std": 0.2658424377441406, "beta_std": 0.0588887594640255, "beta_low": 0.2529296875, "beta_high": 0.0, "orthogonal_to_active": 0.06131011247634888, "evidence_weights": [-0.16021834313869476, -0.25101643800735474, 0.8532410264015198], "agreement_weight": 0.0645582526922226, "orthogonal_weight": -0.857661247253418}, {"head": 4, "memory_to_active": 3.6468820571899414, "contribution_to_active": 0.3177717328071594, "mix_mean": 0.015998845919966698, "mix_low": 0.6713962554931641, "mix_high": 0.0, "read_cosine": 0.17749714851379395, "beta_mean": 0.2691183090209961, "nonfinite": 0.0, "weighted_memory_to_active": 0.34739410877227783, "mix_std": 0.04505394399166107, "beta_std": 0.2347535938024521, "beta_low": 0.03009033203125, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.022636957466602325, "evidence_weights": [0.03254628926515579, 0.031854428350925446, 0.11809049546718597], "agreement_weight": 0.028340725228190422, "orthogonal_weight": -0.06731777638196945}, {"head": 5, "memory_to_active": 0.023188937455415726, "contribution_to_active": 0.5158963203430176, "mix_mean": 0.577012836933136, "mix_low": 0.0016460418701171875, "mix_high": 0.005141258239746094, "read_cosine": 0.45938676595687866, "beta_mean": 6.391435454133898e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014899397268891335, "mix_std": 0.2756122648715973, "beta_std": 8.848656580084935e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011971208732575178, "evidence_weights": [-0.25453126430511475, -0.27928924560546875, 0.810863733291626], "agreement_weight": 0.06601807475090027, "orthogonal_weight": 0.13115695118904114}]}, {"layer": 3, "memory_to_active": 0.9778703451156616, "contribution_to_active": 0.3657482862472534, "mix_mean": 0.30090829730033875, "mix_low": 0.1950591504573822, "mix_high": 0.004912853240966797, "read_cosine": 0.24408826231956482, "beta_mean": 0.06836241483688354, "nonfinite": 0.0, "weighted_memory_to_active": 0.16165649890899658, "mix_std": 0.22976061701774597, "beta_std": 0.05601341649889946, "beta_low": 0.4460957944393158, "beta_high": 0.0, "orthogonal_to_active": 0.05992942303419113, "output_influence": 0.2795652151107788, "residual_influence": 0.05576007813215256, "state_active": 0.024676134809851646, "state_memory": 0.014372823759913445, "state_peak": 18.026844024658203, "growth_active": 1.0680346488952637, "growth_memory": 1.856889247894287, "heads": [{"head": 0, "memory_to_active": 0.5808428525924683, "contribution_to_active": 0.4651981592178345, "mix_mean": 0.39580392837524414, "mix_low": 0.024658203125, "mix_high": 0.00739288330078125, "read_cosine": 0.20898471772670746, "beta_mean": 0.01475043036043644, "nonfinite": 0.0, "weighted_memory_to_active": 0.26380982995033264, "mix_std": 0.3241971731185913, "beta_std": 0.01783602684736252, "beta_low": 0.5130615234375, "beta_high": 0.0, "orthogonal_to_active": 0.166563481092453, "evidence_weights": [-0.1560126543045044, -0.2013772577047348, 0.5320430994033813], "agreement_weight": 0.13324607908725739, "orthogonal_weight": -0.6608737707138062}, {"head": 1, "memory_to_active": 0.01851331815123558, "contribution_to_active": 0.3141000270843506, "mix_mean": 0.3890771269798279, "mix_low": 0.0220794677734375, "mix_high": 0.0023069381713867188, "read_cosine": 0.1841837465763092, "beta_mean": 9.540952305542305e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007606157101690769, "mix_std": 0.30103522539138794, "beta_std": 0.00020817702170461416, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002469611121341586, "evidence_weights": [-0.19605523347854614, -0.16916514933109283, 0.5162282586097717], "agreement_weight": 0.2840435206890106, "orthogonal_weight": -0.3542838990688324}, {"head": 2, "memory_to_active": 2.658796548843384, "contribution_to_active": 0.3073277175426483, "mix_mean": 0.04300929605960846, "mix_low": 0.40183162689208984, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19895075261592865, "beta_mean": 0.21787835657596588, "nonfinite": 0.0, "weighted_memory_to_active": 0.3883947730064392, "mix_std": 0.09651049226522446, "beta_std": 0.149357408285141, "beta_low": 0.00555419921875, "beta_high": 0.0, "orthogonal_to_active": 0.014134733006358147, "evidence_weights": [-0.05635065212845802, 0.01927162893116474, -0.038503725081682205], "agreement_weight": -0.01289557944983244, "orthogonal_weight": -0.04782116413116455}, {"head": 3, "memory_to_active": 0.03486711531877518, "contribution_to_active": 0.48180490732192993, "mix_mean": 0.5058509707450867, "mix_low": 0.01751232147216797, "mix_high": 0.01168060302734375, "read_cosine": 0.2891877293586731, "beta_mean": 0.00014879560330882668, "nonfinite": 0.0, "weighted_memory_to_active": 0.0211343914270401, "mix_std": 0.3141477704048157, "beta_std": 0.00030501000583171844, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0064287553541362286, "evidence_weights": [-0.19810372591018677, -0.21625365316867828, 1.8016562461853027], "agreement_weight": 1.1431962251663208, "orthogonal_weight": -0.3909365236759186}, {"head": 4, "memory_to_active": 2.2328953742980957, "contribution_to_active": 0.029119547456502914, "mix_mean": 0.014836259186267853, "mix_low": 0.6833314895629883, "mix_high": 0.0, "read_cosine": 0.18571245670318604, "beta_mean": 0.11240758001804352, "nonfinite": 0.0, "weighted_memory_to_active": 0.061866745352745056, "mix_std": 0.03377636522054672, "beta_std": 0.10073237866163254, "beta_low": 0.03155517578125, "beta_high": 0.0, "orthogonal_to_active": 0.03893756866455078, "evidence_weights": [0.12457596510648727, 0.08156543970108032, -0.0036905088927596807], "agreement_weight": -0.0504390187561512, "orthogonal_weight": -0.6443116068840027}, {"head": 5, "memory_to_active": 0.34130656719207764, "contribution_to_active": 0.5969392657279968, "mix_mean": 0.45687204599380493, "mix_low": 0.020941734313964844, "mix_high": 0.008095741271972656, "read_cosine": 0.3975101113319397, "beta_mean": 0.06489390879869461, "nonfinite": 0.0, "weighted_memory_to_active": 0.2271271049976349, "mix_std": 0.308896541595459, "beta_std": 0.0676414966583252, "beta_low": 0.12640380859375, "beta_high": 0.0, "orthogonal_to_active": 0.13104237616062164, "evidence_weights": [-0.17993007600307465, -0.22758591175079346, 0.40750735998153687], "agreement_weight": 0.14918160438537598, "orthogonal_weight": -0.711342990398407}]}, {"layer": 4, "memory_to_active": 1.2072938680648804, "contribution_to_active": 0.40109267830848694, "mix_mean": 0.3288678824901581, "mix_low": 0.2730283737182617, "mix_high": 0.004262447357177734, "read_cosine": 0.3788111209869385, "beta_mean": 0.12909799814224243, "nonfinite": 0.0, "weighted_memory_to_active": 0.13471262156963348, "mix_std": 0.1552208811044693, "beta_std": 0.1138744130730629, "beta_low": 0.3628641963005066, "beta_high": 0.0, "orthogonal_to_active": 0.0380050390958786, "output_influence": 0.3353908061981201, "residual_influence": 0.08569824695587158, "state_active": 0.0355970598757267, "state_memory": 0.021444860845804214, "state_peak": 35.362430572509766, "growth_active": 1.1458325386047363, "growth_memory": 1.5550200939178467, "heads": [{"head": 0, "memory_to_active": 0.26535236835479736, "contribution_to_active": 0.6701690554618835, "mix_mean": 0.6196094751358032, "mix_low": 0.008286476135253906, "mix_high": 0.006102561950683594, "read_cosine": 0.5154032111167908, "beta_mean": 0.25407785177230835, "nonfinite": 0.0, "weighted_memory_to_active": 0.20021483302116394, "mix_std": 0.3271957039833069, "beta_std": 0.2619803547859192, "beta_low": 0.18670654296875, "beta_high": 0.0, "orthogonal_to_active": 0.056144990026950836, "evidence_weights": [-0.25949880480766296, -0.3464460074901581, 0.35197678208351135], "agreement_weight": 0.03472437709569931, "orthogonal_weight": 0.34756383299827576}, {"head": 1, "memory_to_active": 0.11385582387447357, "contribution_to_active": 0.5456464886665344, "mix_mean": 0.5832692980766296, "mix_low": 3.0517578125e-05, "mix_high": 0.004288673400878906, "read_cosine": 0.7555922269821167, "beta_mean": 0.0027941460721194744, "nonfinite": 0.0, "weighted_memory_to_active": 0.07099658250808716, "mix_std": 0.2567142844200134, "beta_std": 0.005966723430901766, "beta_low": 0.9482421875, "beta_high": 0.0, "orthogonal_to_active": 0.034726642072200775, "evidence_weights": [-0.23265953361988068, -0.28348949551582336, 0.7229526042938232], "agreement_weight": 0.07617134600877762, "orthogonal_weight": -0.7919301986694336}, {"head": 2, "memory_to_active": 2.5267086029052734, "contribution_to_active": 0.1577613353729248, "mix_mean": 0.017274159938097, "mix_low": 0.5903224945068359, "mix_high": 0.0, "read_cosine": 0.26579582691192627, "beta_mean": 0.1408383697271347, "nonfinite": 0.0, "weighted_memory_to_active": 0.18129786849021912, "mix_std": 0.03259062021970749, "beta_std": 0.10479851067066193, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.012289782986044884, "evidence_weights": [0.05035426467657089, 0.02652212418615818, -0.026443686336278915], "agreement_weight": 0.09714161604642868, "orthogonal_weight": -0.09659935534000397}, {"head": 3, "memory_to_active": 1.7294145822525024, "contribution_to_active": 0.11002856492996216, "mix_mean": 0.029502445831894875, "mix_low": 0.4168853759765625, "mix_high": 0.0, "read_cosine": 0.1656467616558075, "beta_mean": 0.14056526124477386, "nonfinite": 0.0, "weighted_memory_to_active": 0.14986112713813782, "mix_std": 0.0520758181810379, "beta_std": 0.13312461972236633, "beta_low": 0.0418701171875, "beta_high": 0.0, "orthogonal_to_active": 0.014285009354352951, "evidence_weights": [-0.020509928464889526, 0.05753893777728081, -0.26744621992111206], "agreement_weight": 0.05222024768590927, "orthogonal_weight": -0.12207432091236115}, {"head": 4, "memory_to_active": 0.020990770310163498, "contribution_to_active": 0.6460301280021667, "mix_mean": 0.7089661955833435, "mix_low": 0.0001926422119140625, "mix_high": 0.015178680419921875, "read_cosine": 0.2236977517604828, "beta_mean": 3.4827262425096706e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01568586565554142, "mix_std": 0.23157118260860443, "beta_std": 9.714582847664133e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024181450717151165, "evidence_weights": [-0.30650758743286133, -0.33777254819869995, 0.7538440823554993], "agreement_weight": 0.12371253967285156, "orthogonal_weight": -0.17463304102420807}, {"head": 5, "memory_to_active": 2.587440013885498, "contribution_to_active": 0.276920348405838, "mix_mean": 0.014585699886083603, "mix_low": 0.6224527359008789, "mix_high": 4.76837158203125e-06, "read_cosine": 0.346731036901474, "beta_mean": 0.23627743124961853, "nonfinite": 0.0, "weighted_memory_to_active": 0.1902194768190384, "mix_std": 0.031177690252661705, "beta_std": 0.17727912962436676, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10816565155982971, "evidence_weights": [0.041169606149196625, -0.03774707019329071, -0.1997724175453186], "agreement_weight": -0.045094333589076996, "orthogonal_weight": 0.7873532176017761}]}, {"layer": 5, "memory_to_active": 0.6216375827789307, "contribution_to_active": 0.4747735261917114, "mix_mean": 0.42139458656311035, "mix_low": 0.08130741119384766, "mix_high": 0.011307716369628906, "read_cosine": 0.16830968856811523, "beta_mean": 0.05630706995725632, "nonfinite": 0.0, "weighted_memory_to_active": 0.16432301700115204, "mix_std": 0.2505751848220825, "beta_std": 0.07159532606601715, "beta_low": 0.6199747920036316, "beta_high": 0.0, "orthogonal_to_active": 0.056000303477048874, "output_influence": 0.4651545286178589, "residual_influence": 0.1130513921380043, "state_active": 0.049725666642189026, "state_memory": 0.012820187956094742, "state_peak": 63.504154205322266, "growth_active": 1.1648377180099487, "growth_memory": 1.23359215259552, "heads": [{"head": 0, "memory_to_active": 1.1235288381576538, "contribution_to_active": 0.3378174602985382, "mix_mean": 0.16688600182533264, "mix_low": 0.1000070571899414, "mix_high": 0.00021457672119140625, "read_cosine": -0.18926982581615448, "beta_mean": 0.03906282037496567, "nonfinite": 0.0, "weighted_memory_to_active": 0.16961996257305145, "mix_std": 0.2027885913848877, "beta_std": 0.06302698701620102, "beta_low": 0.29486083984375, "beta_high": 0.0, "orthogonal_to_active": 0.03661160543560982, "evidence_weights": [-0.06945253908634186, -0.10471698641777039, -0.35632938146591187], "agreement_weight": -0.013964802958071232, "orthogonal_weight": -0.20696225762367249}, {"head": 1, "memory_to_active": 0.03572250157594681, "contribution_to_active": 0.5187141299247742, "mix_mean": 0.5900189876556396, "mix_low": 0.0036382675170898438, "mix_high": 0.016297340393066406, "read_cosine": 0.14465686678886414, "beta_mean": 9.017947741085663e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.017896173521876335, "mix_std": 0.3033713102340698, "beta_std": 0.0001526008127257228, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01593499444425106, "evidence_weights": [-0.23426376283168793, -0.26333194971084595, 0.8106762170791626], "agreement_weight": 0.38681721687316895, "orthogonal_weight": -0.9257726669311523}, {"head": 2, "memory_to_active": 0.03534526750445366, "contribution_to_active": 0.5395283699035645, "mix_mean": 0.6019366979598999, "mix_low": 0.00246429443359375, "mix_high": 0.012824058532714844, "read_cosine": 0.19463790953159332, "beta_mean": 0.000500772031955421, "nonfinite": 0.0, "weighted_memory_to_active": 0.023527704179286957, "mix_std": 0.2768091559410095, "beta_std": 0.0008465479477308691, "beta_low": 0.9986572265625, "beta_high": 0.0, "orthogonal_to_active": 0.0045210812240839005, "evidence_weights": [-0.25258538126945496, -0.3415773808956146, 0.8885945081710815], "agreement_weight": 0.014795968309044838, "orthogonal_weight": -0.24268348515033722}, {"head": 3, "memory_to_active": 1.9626426696777344, "contribution_to_active": 0.23062747716903687, "mix_mean": 0.08342644572257996, "mix_low": 0.19859695434570312, "mix_high": 0.0, "read_cosine": 0.14407390356063843, "beta_mean": 0.09227845817804337, "nonfinite": 0.0, "weighted_memory_to_active": 0.385993629693985, "mix_std": 0.13331589102745056, "beta_std": 0.13386759161949158, "beta_low": 0.19317626953125, "beta_high": 0.0, "orthogonal_to_active": 0.14133059978485107, "evidence_weights": [-0.0488099530339241, -0.08656109869480133, -0.08913902193307877], "agreement_weight": 0.05935010686516762, "orthogonal_weight": -0.45309916138648987}, {"head": 4, "memory_to_active": 0.4581156373023987, "contribution_to_active": 0.5982898473739624, "mix_mean": 0.3306536376476288, "mix_low": 0.18035221099853516, "mix_high": 0.009589195251464844, "read_cosine": 0.551903247833252, "beta_mean": 0.205488383769989, "nonfinite": 0.0, "weighted_memory_to_active": 0.31094658374786377, "mix_std": 0.33942681550979614, "beta_std": 0.22974759340286255, "beta_low": 0.23980712890625, "beta_high": 0.0, "orthogonal_to_active": 0.08890443295240402, "evidence_weights": [-0.11444930732250214, -0.2883300483226776, -0.053227685391902924], "agreement_weight": 0.05257486552000046, "orthogonal_weight": 0.4389054477214813}, {"head": 5, "memory_to_active": 0.11447075754404068, "contribution_to_active": 0.6236639022827148, "mix_mean": 0.755445659160614, "mix_low": 0.0027856826782226562, "mix_high": 0.028921127319335938, "read_cosine": 0.16385598480701447, "beta_mean": 0.0004218068206682801, "nonfinite": 0.0, "weighted_memory_to_active": 0.07795405387878418, "mix_std": 0.2477392554283142, "beta_std": 0.0019306052708998322, "beta_low": 0.99334716796875, "beta_high": 0.0, "orthogonal_to_active": 0.04869909584522247, "evidence_weights": [-0.3255469799041748, -0.31712353229522705, 1.9460103511810303], "agreement_weight": 0.3529207110404968, "orthogonal_weight": -0.6636615991592407}]}, {"layer": 6, "memory_to_active": 0.38216304779052734, "contribution_to_active": 0.583160400390625, "mix_mean": 0.5353661775588989, "mix_low": 0.07717959582805634, "mix_high": 0.025329750031232834, "read_cosine": 0.2967291474342346, "beta_mean": 0.07309625297784805, "nonfinite": 0.0, "weighted_memory_to_active": 0.17208881676197052, "mix_std": 0.2462848424911499, "beta_std": 0.08354860544204712, "beta_low": 0.5029093623161316, "beta_high": 0.0, "orthogonal_to_active": 0.03885466605424881, "output_influence": 0.47715166211128235, "residual_influence": 0.1518050730228424, "state_active": 0.04695432260632515, "state_memory": 0.009553509764373302, "state_peak": 76.46794128417969, "growth_active": 1.1686805486679077, "growth_memory": 1.182559609413147, "heads": [{"head": 0, "memory_to_active": 0.15576840937137604, "contribution_to_active": 0.5957823991775513, "mix_mean": 0.7113428115844727, "mix_low": 0.0002231597900390625, "mix_high": 0.005061149597167969, "read_cosine": 0.05275886505842209, "beta_mean": 0.01413712091743946, "nonfinite": 0.0, "weighted_memory_to_active": 0.11070242524147034, "mix_std": 0.23225009441375732, "beta_std": 0.017400039359927177, "beta_low": 0.52313232421875, "beta_high": 0.0, "orthogonal_to_active": 0.09120549261569977, "evidence_weights": [-0.3077389895915985, -0.32423436641693115, 0.4918789863586426], "agreement_weight": 0.10740038007497787, "orthogonal_weight": -0.922224760055542}, {"head": 1, "memory_to_active": 0.00345388101413846, "contribution_to_active": 0.5014936923980713, "mix_mean": 0.488646924495697, "mix_low": 0.0012969970703125, "mix_high": 0.008549690246582031, "read_cosine": 0.6018418073654175, "beta_mean": 7.619812549819471e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014422689564526081, "mix_std": 0.292732834815979, "beta_std": 1.55570924107451e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00046609615674242377, "evidence_weights": [-0.2030964344739914, -0.27313125133514404, 0.886701226234436], "agreement_weight": -0.02495412528514862, "orthogonal_weight": -0.4601164758205414}, {"head": 2, "memory_to_active": 1.1735761165618896, "contribution_to_active": 0.07655638456344604, "mix_mean": 0.027905933558940887, "mix_low": 0.44943904876708984, "mix_high": 2.86102294921875e-06, "read_cosine": 0.21573933959007263, "beta_mean": 0.07757888734340668, "nonfinite": 0.0, "weighted_memory_to_active": 0.06834712624549866, "mix_std": 0.053426191210746765, "beta_std": 0.078834168612957, "beta_low": 0.01055908203125, "beta_high": 0.0, "orthogonal_to_active": 0.0029075108468532562, "evidence_weights": [0.09084149450063705, 0.0058746058493852615, 0.2519080936908722], "agreement_weight": 0.12557415664196014, "orthogonal_weight": -0.050438862293958664}, {"head": 3, "memory_to_active": 0.15250492095947266, "contribution_to_active": 0.6526450514793396, "mix_mean": 0.6307281255722046, "mix_low": 0.010172843933105469, "mix_high": 0.01561737060546875, "read_cosine": 0.4768783748149872, "beta_mean": 0.21078352630138397, "nonfinite": 0.0, "weighted_memory_to_active": 0.10918687283992767, "mix_std": 0.33486679196357727, "beta_std": 0.24912723898887634, "beta_low": 0.24346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.05491568520665169, "evidence_weights": [-0.1757931411266327, -0.33853650093078613, 0.15017051994800568], "agreement_weight": -0.011434733867645264, "orthogonal_weight": 0.5976735949516296}, {"head": 4, "memory_to_active": 0.7618386745452881, "contribution_to_active": 1.0884430408477783, "mix_mean": 0.7443769574165344, "mix_low": 0.00162506103515625, "mix_high": 0.1211709976196289, "read_cosine": 0.15729786455631256, "beta_mean": 0.1356838196516037, "nonfinite": 0.0, "weighted_memory_to_active": 0.712273359298706, "mix_std": 0.2973094880580902, "beta_std": 0.15512952208518982, "beta_low": 0.24127197265625, "beta_high": 0.0, "orthogonal_to_active": 0.05263730511069298, "evidence_weights": [-0.25613513588905334, -0.5870087146759033, -0.11803293228149414], "agreement_weight": -0.013506158255040646, "orthogonal_weight": 0.08697813004255295}, {"head": 5, "memory_to_active": 0.045836191624403, "contribution_to_active": 0.5840415954589844, "mix_mean": 0.6091961860656738, "mix_low": 0.0003204345703125, "mix_high": 0.0015764236450195312, "read_cosine": 0.2758585512638092, "beta_mean": 0.0003865391481667757, "nonfinite": 0.0, "weighted_memory_to_active": 0.030580934137105942, "mix_std": 0.2671236991882324, "beta_std": 0.0007850778638385236, "beta_low": 0.9990234375, "beta_high": 0.0, "orthogonal_to_active": 0.030995896086096764, "evidence_weights": [-0.23647648096084595, -0.2956811487674713, 0.385951429605484], "agreement_weight": 0.11459042131900787, "orthogonal_weight": -1.0800621509552002}]}, {"layer": 7, "memory_to_active": 1.8666434288024902, "contribution_to_active": 0.20685502886772156, "mix_mean": 0.06459850072860718, "mix_low": 0.35478833317756653, "mix_high": 4.609425741364248e-05, "read_cosine": 0.24164801836013794, "beta_mean": 0.12681424617767334, "nonfinite": 0.0, "weighted_memory_to_active": 0.22452570497989655, "mix_std": 0.09705384820699692, "beta_std": 0.10476206243038177, "beta_low": 0.11383056640625, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.0510420985519886, "output_influence": 0.21334236860275269, "residual_influence": 0.04731311649084091, "state_active": 0.014544779434800148, "state_memory": 0.028421878814697266, "state_peak": 29.336965560913086, "growth_active": 1.117173194885254, "growth_memory": 1.2958871126174927, "heads": [{"head": 0, "memory_to_active": 2.2255337238311768, "contribution_to_active": 0.3916164040565491, "mix_mean": 0.03368823230266571, "mix_low": 0.4559659957885742, "mix_high": 1.9073486328125e-06, "read_cosine": 0.36260032653808594, "beta_mean": 0.14617815613746643, "nonfinite": 0.0, "weighted_memory_to_active": 0.39568188786506653, "mix_std": 0.06985477358102798, "beta_std": 0.10972615331411362, "beta_low": 0.00299072265625, "beta_high": 0.000244140625, "orthogonal_to_active": 0.026526980102062225, "evidence_weights": [0.004280851222574711, -0.09188728779554367, 0.1283639520406723], "agreement_weight": -0.03906645625829697, "orthogonal_weight": 0.09872028976678848}, {"head": 1, "memory_to_active": 1.2213308811187744, "contribution_to_active": 0.32253560423851013, "mix_mean": 0.15249714255332947, "mix_low": 0.20823001861572266, "mix_high": 0.00020599365234375, "read_cosine": 0.2890070676803589, "beta_mean": 0.11371001601219177, "nonfinite": 0.0, "weighted_memory_to_active": 0.3284224569797516, "mix_std": 0.18868441879749298, "beta_std": 0.07427442073822021, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.015448163263499737, "evidence_weights": [-0.17431007325649261, -0.11544498056173325, -0.25873416662216187], "agreement_weight": 0.014367139898240566, "orthogonal_weight": -0.052920863032341}, {"head": 2, "memory_to_active": 1.0364065170288086, "contribution_to_active": 0.26322272419929504, "mix_mean": 0.08132235705852509, "mix_low": 0.1887836456298828, "mix_high": 6.866455078125e-05, "read_cosine": 0.13011416792869568, "beta_mean": 0.08552242070436478, "nonfinite": 0.0, "weighted_memory_to_active": 0.18799453973770142, "mix_std": 0.13539129495620728, "beta_std": 0.05978409945964813, "beta_low": 0.0279541015625, "beta_high": 0.0, "orthogonal_to_active": 0.03837431222200394, "evidence_weights": [-0.030270975083112717, -0.11812656372785568, 0.28530630469322205], "agreement_weight": 0.027671124786138535, "orthogonal_weight": -0.20876146852970123}, {"head": 3, "memory_to_active": 3.902428388595581, "contribution_to_active": 0.06921582669019699, "mix_mean": 0.027205344289541245, "mix_low": 0.40614986419677734, "mix_high": 0.0, "read_cosine": 0.12546555697917938, "beta_mean": 0.24789053201675415, "nonfinite": 0.0, "weighted_memory_to_active": 0.21367433667182922, "mix_std": 0.04257825389504433, "beta_std": 0.22526893019676208, "beta_low": 0.01849365234375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1740293651819229, "evidence_weights": [0.08666105568408966, 0.030152056366205215, 0.5743902325630188], "agreement_weight": 0.05569334700703621, "orthogonal_weight": -0.909384548664093}, {"head": 4, "memory_to_active": 1.9893133640289307, "contribution_to_active": 0.08079689741134644, "mix_mean": 0.013887949287891388, "mix_low": 0.6735010147094727, "mix_high": 0.0, "read_cosine": 0.3085838854312897, "beta_mean": 0.15432661771774292, "nonfinite": 0.0, "weighted_memory_to_active": 0.09442955255508423, "mix_std": 0.030816245824098587, "beta_std": 0.1368386447429657, "beta_low": 0.017822265625, "beta_high": 0.0, "orthogonal_to_active": 0.00395771162584424, "evidence_weights": [0.12143710255622864, 0.0523240827023983, 0.13425719738006592], "agreement_weight": 0.04662763699889183, "orthogonal_weight": -0.06846921890974045}, {"head": 5, "memory_to_active": 0.8248476386070251, "contribution_to_active": 0.1137426570057869, "mix_mean": 0.07898993045091629, "mix_low": 0.19609928131103516, "mix_high": 0.0, "read_cosine": 0.2341170608997345, "beta_mean": 0.013257608748972416, "nonfinite": 0.0, "weighted_memory_to_active": 0.1269514560699463, "mix_std": 0.11499806493520737, "beta_std": 0.022680118680000305, "beta_low": 0.61553955078125, "beta_high": 0.0, "orthogonal_to_active": 0.04791604354977608, "evidence_weights": [0.004498898517340422, -0.03743625804781914, 0.15848027169704437], "agreement_weight": 0.06951417773962021, "orthogonal_weight": -0.5240522623062134}]}, {"layer": 8, "memory_to_active": 0.5877541303634644, "contribution_to_active": 0.5177167654037476, "mix_mean": 0.3980656862258911, "mix_low": 0.09608158469200134, "mix_high": 0.0007036527385935187, "read_cosine": 0.5175055265426636, "beta_mean": 0.04174547269940376, "nonfinite": 0.0, "weighted_memory_to_active": 0.11889342218637466, "mix_std": 0.22469201683998108, "beta_std": 0.043411724269390106, "beta_low": 0.5667623281478882, "beta_high": 0.0, "orthogonal_to_active": 0.056885283440351486, "output_influence": 0.4165874123573303, "residual_influence": 0.12272009998559952, "state_active": 0.055856913328170776, "state_memory": 0.011726902797818184, "state_peak": 34.46521759033203, "growth_active": 1.1220060586929321, "growth_memory": 1.3707051277160645, "heads": [{"head": 0, "memory_to_active": 0.04142265021800995, "contribution_to_active": 0.5226406455039978, "mix_mean": 0.5645453929901123, "mix_low": 0.0002574920654296875, "mix_high": 0.0005445480346679688, "read_cosine": 0.6667964458465576, "beta_mean": 0.0032046008855104446, "nonfinite": 0.0, "weighted_memory_to_active": 0.025567222386598587, "mix_std": 0.2413288652896881, "beta_std": 0.007490777410566807, "beta_low": 0.9222412109375, "beta_high": 0.0, "orthogonal_to_active": 0.014760082587599754, "evidence_weights": [-0.17802263796329498, -0.3790914714336395, 0.7209094166755676], "agreement_weight": 0.0766262412071228, "orthogonal_weight": -0.812643826007843}, {"head": 1, "memory_to_active": 0.2572150230407715, "contribution_to_active": 0.4520795941352844, "mix_mean": 0.43222665786743164, "mix_low": 0.0027065277099609375, "mix_high": 0.00033855438232421875, "read_cosine": 0.39701026678085327, "beta_mean": 0.008709571324288845, "nonfinite": 0.0, "weighted_memory_to_active": 0.12059669196605682, "mix_std": 0.25491636991500854, "beta_std": 0.016818281263113022, "beta_low": 0.76409912109375, "beta_high": 0.0, "orthogonal_to_active": 0.06398005038499832, "evidence_weights": [-0.18724380433559418, -0.23277059197425842, 0.641438364982605], "agreement_weight": 0.07205358147621155, "orthogonal_weight": -0.6060985922813416}, {"head": 2, "memory_to_active": 0.04507526010274887, "contribution_to_active": 0.6184183955192566, "mix_mean": 0.584200382232666, "mix_low": 0.0008668899536132812, "mix_high": 0.002277374267578125, "read_cosine": 0.39505958557128906, "beta_mean": 0.0004223985015414655, "nonfinite": 0.0, "weighted_memory_to_active": 0.031976938247680664, "mix_std": 0.24529090523719788, "beta_std": 0.00331157841719687, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.022373922169208527, "evidence_weights": [-0.21402402222156525, -0.3296840190887451, 0.6911494731903076], "agreement_weight": 0.14564016461372375, "orthogonal_weight": -0.8101690411567688}, {"head": 3, "memory_to_active": 0.13360673189163208, "contribution_to_active": 0.5969981551170349, "mix_mean": 0.505014955997467, "mix_low": 0.0007772445678710938, "mix_high": 0.00042629241943359375, "read_cosine": 0.784516453742981, "beta_mean": 0.012239508330821991, "nonfinite": 0.0, "weighted_memory_to_active": 0.0933673307299614, "mix_std": 0.25491398572921753, "beta_std": 0.010314779356122017, "beta_low": 0.4786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.02776813507080078, "evidence_weights": [-0.16323748230934143, -0.28463679552078247, 0.7129752039909363], "agreement_weight": 0.12316163629293442, "orthogonal_weight": -0.5805424451828003}, {"head": 4, "memory_to_active": 2.770070791244507, "contribution_to_active": 0.27920717000961304, "mix_mean": 0.02517837844789028, "mix_low": 0.4490528106689453, "mix_high": 0.0, "read_cosine": 0.3140512704849243, "beta_mean": 0.1450968086719513, "nonfinite": 0.0, "weighted_memory_to_active": 0.2689170837402344, "mix_std": 0.04516998678445816, "beta_std": 0.11044760048389435, "beta_low": 0.00457763671875, "beta_high": 0.0, "orthogonal_to_active": 0.006581037770956755, "evidence_weights": [0.04867551848292351, -0.006221781484782696, -0.1583942472934723], "agreement_weight": 0.03391130641102791, "orthogonal_weight": 0.025356395170092583}, {"head": 5, "memory_to_active": 0.27913475036621094, "contribution_to_active": 0.6369563341140747, "mix_mean": 0.27722829580307007, "mix_low": 0.12282848358154297, "mix_high": 0.0006351470947265625, "read_cosine": 0.5475990772247314, "beta_mean": 0.08079995214939117, "nonfinite": 0.0, "weighted_memory_to_active": 0.17293526232242584, "mix_std": 0.30653196573257446, "beta_std": 0.11208730190992355, "beta_low": 0.23187255859375, "beta_high": 0.0, "orthogonal_to_active": 0.20584845542907715, "evidence_weights": [-0.06260889768600464, -0.29292890429496765, -0.4084266722202301], "agreement_weight": 0.023365898057818413, "orthogonal_weight": 1.4774715900421143}]}, {"layer": 9, "memory_to_active": 1.149223804473877, "contribution_to_active": 0.6176602840423584, "mix_mean": 0.28925731778144836, "mix_low": 0.15712229907512665, "mix_high": 0.0022409758530557156, "read_cosine": 0.3658066987991333, "beta_mean": 0.07815924286842346, "nonfinite": 0.0, "weighted_memory_to_active": 0.3589693605899811, "mix_std": 0.20864766836166382, "beta_std": 0.04873731732368469, "beta_low": 0.4995524287223816, "beta_high": 0.0, "orthogonal_to_active": 0.039527926594018936, "output_influence": 0.4580807089805603, "residual_influence": 0.12300185859203339, "state_active": 0.04298314079642296, "state_memory": 0.021546468138694763, "state_peak": 37.77910232543945, "growth_active": 1.223513126373291, "growth_memory": 1.37788724899292, "heads": [{"head": 0, "memory_to_active": 0.005255566444247961, "contribution_to_active": 0.5144057869911194, "mix_mean": 0.5761983394622803, "mix_low": 0.0022153854370117188, "mix_high": 0.00960540771484375, "read_cosine": 0.23782223463058472, "beta_mean": 1.0234709407086484e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031360238790512085, "mix_std": 0.2962547540664673, "beta_std": 4.662390347220935e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002731545362621546, "evidence_weights": [-0.28722116351127625, -0.2930489778518677, 0.9473242163658142], "agreement_weight": 0.09002716094255447, "orthogonal_weight": -0.9134711623191833}, {"head": 1, "memory_to_active": 0.03195267170667648, "contribution_to_active": 0.5670634508132935, "mix_mean": 0.441371887922287, "mix_low": 0.010443687438964844, "mix_high": 0.0009632110595703125, "read_cosine": 0.4186152517795563, "beta_mean": 8.837162749841809e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01849670335650444, "mix_std": 0.2908344864845276, "beta_std": 0.0001769252703525126, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012330835685133934, "evidence_weights": [-0.21279191970825195, -0.2528667449951172, 1.1112463474273682], "agreement_weight": 0.3124927580356598, "orthogonal_weight": -0.7817313075065613}, {"head": 2, "memory_to_active": 0.22223103046417236, "contribution_to_active": 0.5920825600624084, "mix_mean": 0.5010279417037964, "mix_low": 0.0027112960815429688, "mix_high": 0.0024957656860351562, "read_cosine": 0.379962682723999, "beta_mean": 0.000842099660076201, "nonfinite": 0.0, "weighted_memory_to_active": 0.14110717177391052, "mix_std": 0.2825659215450287, "beta_std": 0.005103676579892635, "beta_low": 0.9935302734375, "beta_high": 0.0, "orthogonal_to_active": 0.06123434752225876, "evidence_weights": [-0.17994019389152527, -0.23148714005947113, 0.5549874305725098], "agreement_weight": 0.23253923654556274, "orthogonal_weight": -0.5305700898170471}, {"head": 3, "memory_to_active": 1.6440277099609375, "contribution_to_active": 0.17745938897132874, "mix_mean": 0.03861198574304581, "mix_low": 0.39310741424560547, "mix_high": 7.2479248046875e-05, "read_cosine": 0.43828728795051575, "beta_mean": 0.08998386561870575, "nonfinite": 0.0, "weighted_memory_to_active": 0.14521577954292297, "mix_std": 0.08814289420843124, "beta_std": 0.06395450979471207, "beta_low": 0.003173828125, "beta_high": 0.0, "orthogonal_to_active": 0.027721025049686432, "evidence_weights": [0.06548526883125305, 0.03632109612226486, -0.09314827620983124], "agreement_weight": 0.08028854429721832, "orthogonal_weight": 0.20257200300693512}, {"head": 4, "memory_to_active": 3.202575445175171, "contribution_to_active": 1.2216846942901611, "mix_mean": 0.1174706220626831, "mix_low": 0.16656494140625, "mix_high": 0.000308990478515625, "read_cosine": 0.2946700155735016, "beta_mean": 0.2070649415254593, "nonfinite": 0.0, "weighted_memory_to_active": 1.1580190658569336, "mix_std": 0.17731884121894836, "beta_std": 0.10707230865955353, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.015376436524093151, "evidence_weights": [-0.055428117513656616, -0.1472560614347458, 0.02483637072145939], "agreement_weight": -0.052469100803136826, "orthogonal_weight": -0.013308643363416195}, {"head": 5, "memory_to_active": 1.7892999649047852, "contribution_to_active": 0.6332655549049377, "mix_mean": 0.06086311116814613, "mix_low": 0.3676910400390625, "mix_high": 0.0, "read_cosine": 0.42548245191574097, "beta_mean": 0.17096596956253052, "nonfinite": 0.0, "weighted_memory_to_active": 0.6878413558006287, "mix_std": 0.11676903069019318, "beta_std": 0.11606983840465546, "beta_low": 0.0006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.11777336150407791, "evidence_weights": [0.0857188031077385, -0.05082283169031143, 0.13674500584602356], "agreement_weight": 0.018317805603146553, "orthogonal_weight": 0.24015039205551147}]}, {"layer": 10, "memory_to_active": 0.2162080705165863, "contribution_to_active": 0.5406519174575806, "mix_mean": 0.4702022671699524, "mix_low": 0.012677988037467003, "mix_high": 0.007963022217154503, "read_cosine": 0.37141627073287964, "beta_mean": 0.0485127754509449, "nonfinite": 0.0, "weighted_memory_to_active": 0.10599148273468018, "mix_std": 0.26811811327934265, "beta_std": 0.048922501504421234, "beta_low": 0.6559346914291382, "beta_high": 0.0, "orthogonal_to_active": 0.09324595332145691, "output_influence": 0.39108413457870483, "residual_influence": 0.13068948686122894, "state_active": 0.05279482156038284, "state_memory": 0.013431484811007977, "state_peak": 112.71537017822266, "growth_active": 1.0546936988830566, "growth_memory": 0.9948474764823914, "heads": [{"head": 0, "memory_to_active": 0.7410616874694824, "contribution_to_active": 1.1669776439666748, "mix_mean": 0.6144986152648926, "mix_low": 0.03295707702636719, "mix_high": 0.03798198699951172, "read_cosine": 0.09068857133388519, "beta_mean": 0.21962273120880127, "nonfinite": 0.0, "weighted_memory_to_active": 0.49368011951446533, "mix_std": 0.375880628824234, "beta_std": 0.22711819410324097, "beta_low": 0.10888671875, "beta_high": 0.0, "orthogonal_to_active": 0.47498437762260437, "evidence_weights": [-0.14925517141819, -0.36955082416534424, -0.6008044481277466], "agreement_weight": 0.02033095620572567, "orthogonal_weight": 0.9846088290214539}, {"head": 1, "memory_to_active": 0.014978840947151184, "contribution_to_active": 0.5639866590499878, "mix_mean": 0.5747848749160767, "mix_low": 0.0006694793701171875, "mix_high": 0.007279396057128906, "read_cosine": 0.26379525661468506, "beta_mean": 9.632451110519469e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00859140232205391, "mix_std": 0.26499295234680176, "beta_std": 0.00013101589865982533, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01079991739243269, "evidence_weights": [-0.2746182680130005, -0.20059984922409058, 0.5773177146911621], "agreement_weight": 0.41247615218162537, "orthogonal_weight": -1.3820809125900269}, {"head": 2, "memory_to_active": 0.0395403616130352, "contribution_to_active": 0.4959348738193512, "mix_mean": 0.5049632787704468, "mix_low": 0.00054931640625, "mix_high": 0.0021276473999023438, "read_cosine": 0.034200266003608704, "beta_mean": 0.00012879632413387299, "nonfinite": 0.0, "weighted_memory_to_active": 0.014089684002101421, "mix_std": 0.26310473680496216, "beta_std": 0.0005361068760976195, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.005648239981383085, "evidence_weights": [-0.1988801509141922, -0.24347463250160217, 0.54997718334198], "agreement_weight": -0.07728128135204315, "orthogonal_weight": -0.3833956718444824}, {"head": 3, "memory_to_active": 0.3422260582447052, "contribution_to_active": 0.10862575471401215, "mix_mean": 0.2638856768608093, "mix_low": 0.03762054443359375, "mix_high": 0.0002918243408203125, "read_cosine": 0.9092621803283691, "beta_mean": 0.06472804397344589, "nonfinite": 0.0, "weighted_memory_to_active": 0.042855873703956604, "mix_std": 0.22739899158477783, "beta_std": 0.056978847831487656, "beta_low": 0.00213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.04677838087081909, "evidence_weights": [-0.12468817085027695, -0.14904391765594482, 0.1462719440460205], "agreement_weight": 0.0016326340846717358, "orthogonal_weight": 1.4798082113265991}, {"head": 4, "memory_to_active": 0.006779786199331284, "contribution_to_active": 0.49835753440856934, "mix_mean": 0.4969606399536133, "mix_low": 0.0010957717895507812, "mix_high": 8.7738037109375e-05, "read_cosine": 0.07691612839698792, "beta_mean": 2.557361040089745e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0023051220923662186, "mix_std": 0.2538374066352844, "beta_std": 0.0008073191856965423, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0009369560866616666, "evidence_weights": [-0.18081790208816528, -0.24101506173610687, 0.7602634429931641], "agreement_weight": 0.07712200284004211, "orthogonal_weight": -0.40977054834365845}, {"head": 5, "memory_to_active": 0.15266162157058716, "contribution_to_active": 0.4100291132926941, "mix_mean": 0.3661203980445862, "mix_low": 0.0031757354736328125, "mix_high": 9.5367431640625e-06, "read_cosine": 0.8536351919174194, "beta_mean": 0.006475153844803572, "nonfinite": 0.0, "weighted_memory_to_active": 0.07442664355039597, "mix_std": 0.22349373996257782, "beta_std": 0.007963535375893116, "beta_low": 0.8251953125, "beta_high": 0.0, "orthogonal_to_active": 0.02032780647277832, "evidence_weights": [-0.1503152698278427, -0.16174232959747314, 0.29836779832839966], "agreement_weight": 0.0932849571108818, "orthogonal_weight": -0.5931954979896545}]}, {"layer": 11, "memory_to_active": 0.1747610718011856, "contribution_to_active": 0.5333563685417175, "mix_mean": 0.43622010946273804, "mix_low": 0.06958293914794922, "mix_high": 0.037080924957990646, "read_cosine": 0.4437771439552307, "beta_mean": 0.03713574260473251, "nonfinite": 0.0, "weighted_memory_to_active": 0.07183842360973358, "mix_std": 0.3172599673271179, "beta_std": 0.05245742201805115, "beta_low": 0.5153300166130066, "beta_high": 0.0, "orthogonal_to_active": 0.05597427114844322, "output_influence": 0.47966843843460083, "residual_influence": 0.12016460299491882, "state_active": 0.13445574045181274, "state_memory": 0.008624020032584667, "state_peak": 23.622657775878906, "growth_active": 1.0231684446334839, "growth_memory": 1.5064961910247803, "heads": [{"head": 0, "memory_to_active": 0.00946508627384901, "contribution_to_active": 0.5555629730224609, "mix_mean": 0.5881999731063843, "mix_low": 0.0013341903686523438, "mix_high": 0.00701141357421875, "read_cosine": 0.35157984495162964, "beta_mean": 0.00010884855146287009, "nonfinite": 0.0, "weighted_memory_to_active": 0.0059155928902328014, "mix_std": 0.26930686831474304, "beta_std": 0.004051758907735348, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.003921199589967728, "evidence_weights": [-0.16210529208183289, -0.35306504368782043, 0.6301807165145874], "agreement_weight": 0.2061600536108017, "orthogonal_weight": -0.7499460577964783}, {"head": 1, "memory_to_active": 0.4104965627193451, "contribution_to_active": 0.26501214504241943, "mix_mean": 0.30324727296829224, "mix_low": 0.017253875732421875, "mix_high": 7.343292236328125e-05, "read_cosine": 0.6120572090148926, "beta_mean": 0.040826670825481415, "nonfinite": 0.0, "weighted_memory_to_active": 0.15981075167655945, "mix_std": 0.2509034276008606, "beta_std": 0.07601700723171234, "beta_low": 0.35955810546875, "beta_high": 0.0, "orthogonal_to_active": 0.034480419009923935, "evidence_weights": [-0.15986575186252594, -0.14216923713684082, 0.2856377363204956], "agreement_weight": 0.08006460964679718, "orthogonal_weight": -0.29533204436302185}, {"head": 2, "memory_to_active": 0.028141329064965248, "contribution_to_active": 0.9343536496162415, "mix_mean": 0.45926332473754883, "mix_low": 0.13323497772216797, "mix_high": 0.11860466003417969, "read_cosine": 0.26132285594940186, "beta_mean": 0.07479746639728546, "nonfinite": 0.0, "weighted_memory_to_active": 0.02147325500845909, "mix_std": 0.4261752963066101, "beta_std": 0.09792819619178772, "beta_low": 0.4324951171875, "beta_high": 0.0, "orthogonal_to_active": 0.08979053795337677, "evidence_weights": [-0.013315959833562374, -0.4540650248527527, -1.1539198160171509], "agreement_weight": -0.001539792981930077, "orthogonal_weight": 4.408664703369141}, {"head": 3, "memory_to_active": 0.017521239817142487, "contribution_to_active": 0.7553138136863708, "mix_mean": 0.41479408740997314, "mix_low": 0.24939918518066406, "mix_high": 0.09471797943115234, "read_cosine": 0.5290622115135193, "beta_mean": 0.06469959020614624, "nonfinite": 0.0, "weighted_memory_to_active": 0.010069143027067184, "mix_std": 0.4337646961212158, "beta_std": 0.09105704724788666, "beta_low": 0.45263671875, "beta_high": 0.0, "orthogonal_to_active": 0.04083970934152603, "evidence_weights": [0.047024767845869064, -0.35384809970855713, -0.459909588098526], "agreement_weight": 0.03768695518374443, "orthogonal_weight": 4.726424217224121}, {"head": 4, "memory_to_active": 0.4686470031738281, "contribution_to_active": 0.3482511639595032, "mix_mean": 0.3687490224838257, "mix_low": 0.015784263610839844, "mix_high": 0.0004825592041015625, "read_cosine": 0.12840676307678223, "beta_mean": 0.03188817948102951, "nonfinite": 0.0, "weighted_memory_to_active": 0.18509887158870697, "mix_std": 0.2620055377483368, "beta_std": 0.03138254955410957, "beta_low": 0.182861328125, "beta_high": 0.0, "orthogonal_to_active": 0.1431119590997696, "evidence_weights": [-0.1917690634727478, -0.15870295464992523, 1.60638427734375], "agreement_weight": -0.1554107367992401, "orthogonal_weight": -0.792965292930603}, {"head": 5, "memory_to_active": 0.11429518461227417, "contribution_to_active": 0.3416440486907959, "mix_mean": 0.4830668270587921, "mix_low": 0.0004911422729492188, "mix_high": 0.0015954971313476562, "read_cosine": 0.7802338600158691, "beta_mean": 0.010493675246834755, "nonfinite": 0.0, "weighted_memory_to_active": 0.04866289347410202, "mix_std": 0.2614039182662964, "beta_std": 0.014307992532849312, "beta_low": 0.6649169921875, "beta_high": 0.0, "orthogonal_to_active": 0.02370178885757923, "evidence_weights": [-0.2046416997909546, -0.2624199688434601, 0.5231576561927795], "agreement_weight": 0.062046658247709274, "orthogonal_weight": -0.679661214351654}]}, {"layer": 12, "memory_to_active": 0.31488490104675293, "contribution_to_active": 0.5468661785125732, "mix_mean": 0.46447786688804626, "mix_low": 0.014799913391470909, "mix_high": 0.00273227714933455, "read_cosine": 0.26388028264045715, "beta_mean": 0.01581994816660881, "nonfinite": 0.0, "weighted_memory_to_active": 0.16869992017745972, "mix_std": 0.2564237117767334, "beta_std": 0.016577603295445442, "beta_low": 0.8160502314567566, "beta_high": 0.0, "orthogonal_to_active": 0.06820856034755707, "output_influence": 0.42415428161621094, "residual_influence": 0.11826326698064804, "state_active": 0.0386815071105957, "state_memory": 0.009982280433177948, "state_peak": 27.45755386352539, "growth_active": 1.1116145849227905, "growth_memory": 1.2578566074371338, "heads": [{"head": 0, "memory_to_active": 0.0910768136382103, "contribution_to_active": 0.45300373435020447, "mix_mean": 0.43132779002189636, "mix_low": 0.002521514892578125, "mix_high": 0.0006046295166015625, "read_cosine": 0.30451005697250366, "beta_mean": 0.0015416857786476612, "nonfinite": 0.0, "weighted_memory_to_active": 0.03965560346841812, "mix_std": 0.26556605100631714, "beta_std": 0.005654987879097462, "beta_low": 0.97808837890625, "beta_high": 0.0, "orthogonal_to_active": 0.023019572719931602, "evidence_weights": [-0.17219528555870056, -0.2461872398853302, 0.29327672719955444], "agreement_weight": 0.11253221333026886, "orthogonal_weight": -0.6192135810852051}, {"head": 1, "memory_to_active": 0.12423921376466751, "contribution_to_active": 0.48547977209091187, "mix_mean": 0.5044952034950256, "mix_low": 0.0006055831909179688, "mix_high": 0.00030517578125, "read_cosine": 0.545921802520752, "beta_mean": 0.0018709696596488357, "nonfinite": 0.0, "weighted_memory_to_active": 0.06030824035406113, "mix_std": 0.26111820340156555, "beta_std": 0.007557850331068039, "beta_low": 0.9630126953125, "beta_high": 0.0, "orthogonal_to_active": 0.045840565115213394, "evidence_weights": [-0.20177732408046722, -0.2657186686992645, 0.8822858333587646], "agreement_weight": 0.11483296006917953, "orthogonal_weight": -0.9134969115257263}, {"head": 2, "memory_to_active": 0.6396678686141968, "contribution_to_active": 0.46874284744262695, "mix_mean": 0.31347548961639404, "mix_low": 0.026147842407226562, "mix_high": 0.000244140625, "read_cosine": 0.4388639032840729, "beta_mean": 0.08904775977134705, "nonfinite": 0.0, "weighted_memory_to_active": 0.25536414980888367, "mix_std": 0.2431444227695465, "beta_std": 0.06547102332115173, "beta_low": 0.00152587890625, "beta_high": 0.0, "orthogonal_to_active": 0.21759039163589478, "evidence_weights": [-0.2430112212896347, -0.04165932536125183, -0.08153565973043442], "agreement_weight": 0.032329246401786804, "orthogonal_weight": 1.0306979417800903}, {"head": 3, "memory_to_active": 0.027660736814141273, "contribution_to_active": 0.7230291962623596, "mix_mean": 0.6784135103225708, "mix_low": 0.000308990478515625, "mix_high": 0.011453628540039062, "read_cosine": 0.04248398542404175, "beta_mean": 0.00024019811826292425, "nonfinite": 0.0, "weighted_memory_to_active": 0.020665660500526428, "mix_std": 0.24228858947753906, "beta_std": 0.0041913860477507114, "beta_low": 0.99853515625, "beta_high": 0.0, "orthogonal_to_active": 0.01926529034972191, "evidence_weights": [-0.29761892557144165, -0.3323115110397339, 1.2229374647140503], "agreement_weight": 0.2913943827152252, "orthogonal_weight": -0.9531834125518799}, {"head": 4, "memory_to_active": 0.9551621079444885, "contribution_to_active": 0.6797913908958435, "mix_mean": 0.26116007566452026, "mix_low": 0.052641868591308594, "mix_high": 0.0013275146484375, "read_cosine": -0.05793058127164841, "beta_mean": 0.0016664002323523164, "nonfinite": 0.0, "weighted_memory_to_active": 0.6056376695632935, "mix_std": 0.2524157166481018, "beta_std": 0.014612363651394844, "beta_low": 0.96282958984375, "beta_high": 0.0, "orthogonal_to_active": 0.0799541026353836, "evidence_weights": [-0.06066121160984039, -0.19436895847320557, 0.043244749307632446], "agreement_weight": -0.0035871064756065607, "orthogonal_weight": -0.14205366373062134}, {"head": 5, "memory_to_active": 0.051502685993909836, "contribution_to_active": 0.47114992141723633, "mix_mean": 0.5979950428009033, "mix_low": 0.006573677062988281, "mix_high": 0.0024585723876953125, "read_cosine": 0.30943238735198975, "beta_mean": 0.0005526712629944086, "nonfinite": 0.0, "weighted_memory_to_active": 0.030568022280931473, "mix_std": 0.27400922775268555, "beta_std": 0.0019780085422098637, "beta_low": 0.9923095703125, "beta_high": 0.0, "orthogonal_to_active": 0.023581435903906822, "evidence_weights": [-0.26036033034324646, -0.30185994505882263, 1.0494883060455322], "agreement_weight": 0.20898307859897614, "orthogonal_weight": -0.841455340385437}]}, {"layer": 13, "memory_to_active": 0.14765721559524536, "contribution_to_active": 0.5021941661834717, "mix_mean": 0.49061664938926697, "mix_low": 0.005592346657067537, "mix_high": 0.003177325241267681, "read_cosine": 0.411526083946228, "beta_mean": 0.017412638291716576, "nonfinite": 0.0, "weighted_memory_to_active": 0.06473177671432495, "mix_std": 0.25807711482048035, "beta_std": 0.0175547506660223, "beta_low": 0.789703369140625, "beta_high": 0.0, "orthogonal_to_active": 0.06220387667417526, "output_influence": 0.44686442613601685, "residual_influence": 0.1504913866519928, "state_active": 0.09429911524057388, "state_memory": 0.03808046877384186, "state_peak": 93.15370178222656, "growth_active": 0.9550663828849792, "growth_memory": 1.0536421537399292, "heads": [{"head": 0, "memory_to_active": 0.04506567493081093, "contribution_to_active": 0.536558985710144, "mix_mean": 0.536499559879303, "mix_low": 0.00038242340087890625, "mix_high": 0.00115966796875, "read_cosine": 0.401792973279953, "beta_mean": 0.00017128823674283922, "nonfinite": 0.0, "weighted_memory_to_active": 0.025621475651860237, "mix_std": 0.2644560933113098, "beta_std": 0.00038627017056569457, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.02476016990840435, "evidence_weights": [-0.20327478647232056, -0.25736767053604126, 0.7416050434112549], "agreement_weight": 0.07353660464286804, "orthogonal_weight": -1.0745404958724976}, {"head": 1, "memory_to_active": 0.025403080508112907, "contribution_to_active": 0.5615861415863037, "mix_mean": 0.5078245401382446, "mix_low": 0.0021829605102539062, "mix_high": 0.000949859619140625, "read_cosine": 0.23488841950893402, "beta_mean": 9.166634845314547e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014039953239262104, "mix_std": 0.2681806981563568, "beta_std": 0.00016765051987022161, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.014480668120086193, "evidence_weights": [-0.18637575209140778, -0.26587197184562683, 0.6831517815589905], "agreement_weight": 0.240186870098114, "orthogonal_weight": -1.0864983797073364}, {"head": 2, "memory_to_active": 0.13723251223564148, "contribution_to_active": 0.4013829529285431, "mix_mean": 0.3397647440433502, "mix_low": 0.0027866363525390625, "mix_high": 6.771087646484375e-05, "read_cosine": 0.6831579208374023, "beta_mean": 0.012941664084792137, "nonfinite": 0.0, "weighted_memory_to_active": 0.05706271529197693, "mix_std": 0.21653904020786285, "beta_std": 0.035669006407260895, "beta_low": 0.7584228515625, "beta_high": 0.0, "orthogonal_to_active": 0.025788307189941406, "evidence_weights": [-0.21487464010715485, -0.1708899289369583, 0.4632146656513214], "agreement_weight": 0.02184770628809929, "orthogonal_weight": -0.7841843366622925}, {"head": 3, "memory_to_active": 0.1462171971797943, "contribution_to_active": 0.5728667974472046, "mix_mean": 0.5979797840118408, "mix_low": 0.016298294067382812, "mix_high": 0.012788772583007812, "read_cosine": -0.017299670726060867, "beta_mean": 0.0011243035551160574, "nonfinite": 0.0, "weighted_memory_to_active": 0.10342444479465485, "mix_std": 0.2873569130897522, "beta_std": 0.004622742533683777, "beta_low": 0.9891357421875, "beta_high": 0.0, "orthogonal_to_active": 0.10417335480451584, "evidence_weights": [-0.34047895669937134, -0.23609715700149536, 2.0306971073150635], "agreement_weight": 2.8321328163146973, "orthogonal_weight": -1.0649034976959229}, {"head": 4, "memory_to_active": 0.43282008171081543, "contribution_to_active": 0.2867881655693054, "mix_mean": 0.37321558594703674, "mix_low": 0.009258270263671875, "mix_high": 5.53131103515625e-05, "read_cosine": 0.8695211410522461, "beta_mean": 0.08927594125270844, "nonfinite": 0.0, "weighted_memory_to_active": 0.1280929446220398, "mix_std": 0.23231390118598938, "beta_std": 0.05941171199083328, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1565854847431183, "evidence_weights": [-0.16348423063755035, -0.23266752064228058, 0.49819183349609375], "agreement_weight": -0.000406518520321697, "orthogonal_weight": 1.9156694412231445}, {"head": 5, "memory_to_active": 0.09920471161603928, "contribution_to_active": 0.6539817452430725, "mix_mean": 0.5884155631065369, "mix_low": 0.0026454925537109375, "mix_high": 0.004042625427246094, "read_cosine": 0.2970958352088928, "beta_mean": 0.0008709722897037864, "nonfinite": 0.0, "weighted_memory_to_active": 0.060149092227220535, "mix_std": 0.2796160578727722, "beta_std": 0.00507111381739378, "beta_low": 0.99072265625, "beta_high": 0.0, "orthogonal_to_active": 0.04743524268269539, "evidence_weights": [-0.21918879449367523, -0.28877416253089905, 0.6451513767242432], "agreement_weight": 0.08617107570171356, "orthogonal_weight": -0.8706352114677429}]}, {"layer": 14, "memory_to_active": 0.26890814304351807, "contribution_to_active": 0.4255655109882355, "mix_mean": 0.4082348942756653, "mix_low": 0.027082286775112152, "mix_high": 0.004367987625300884, "read_cosine": 0.6076331734657288, "beta_mean": 0.0689099058508873, "nonfinite": 0.0, "weighted_memory_to_active": 0.1212681457400322, "mix_std": 0.24747848510742188, "beta_std": 0.06437438726425171, "beta_low": 0.4342448115348816, "beta_high": 0.0, "orthogonal_to_active": 0.04497441276907921, "output_influence": 0.30601853132247925, "residual_influence": 0.11879011243581772, "state_active": 0.0635291188955307, "state_memory": 0.016126317903399467, "state_peak": 29.351572036743164, "growth_active": 1.1164863109588623, "growth_memory": 1.3188011646270752, "heads": [{"head": 0, "memory_to_active": 0.36185476183891296, "contribution_to_active": 0.21776100993156433, "mix_mean": 0.23139522969722748, "mix_low": 0.012971878051757812, "mix_high": 1.049041748046875e-05, "read_cosine": 0.7373620271682739, "beta_mean": 0.06939659267663956, "nonfinite": 0.0, "weighted_memory_to_active": 0.10521207749843597, "mix_std": 0.20499339699745178, "beta_std": 0.06644302606582642, "beta_low": 0.0203857421875, "beta_high": 0.0, "orthogonal_to_active": 0.06030090153217316, "evidence_weights": [-0.12007411569356918, -0.054682228714227676, 0.6604052186012268], "agreement_weight": 0.02802272140979767, "orthogonal_weight": 0.8662612438201904}, {"head": 1, "memory_to_active": 0.5067089200019836, "contribution_to_active": 0.35175821185112, "mix_mean": 0.23537257313728333, "mix_low": 0.12983226776123047, "mix_high": 5.054473876953125e-05, "read_cosine": 0.7152650356292725, "beta_mean": 0.276899516582489, "nonfinite": 0.0, "weighted_memory_to_active": 0.22738447785377502, "mix_std": 0.2611197531223297, "beta_std": 0.2580510973930359, "beta_low": 0.09576416015625, "beta_high": 0.0, "orthogonal_to_active": 0.01833745837211609, "evidence_weights": [0.007422181777656078, -0.26038187742233276, -0.17422521114349365], "agreement_weight": 0.04794752225279808, "orthogonal_weight": 0.15508566796779633}, {"head": 2, "memory_to_active": 0.1888427436351776, "contribution_to_active": 0.5318512320518494, "mix_mean": 0.5350470542907715, "mix_low": 0.003047943115234375, "mix_high": 0.0018529891967773438, "read_cosine": 0.41874706745147705, "beta_mean": 0.005073530599474907, "nonfinite": 0.0, "weighted_memory_to_active": 0.11225852370262146, "mix_std": 0.26801300048828125, "beta_std": 0.006925922818481922, "beta_low": 0.87567138671875, "beta_high": 0.0, "orthogonal_to_active": 0.08262015879154205, "evidence_weights": [-0.16821855306625366, -0.29007861018180847, 0.8572831749916077], "agreement_weight": 0.07623434066772461, "orthogonal_weight": -0.8438258767127991}, {"head": 3, "memory_to_active": 0.07154978811740875, "contribution_to_active": 0.6668336391448975, "mix_mean": 0.6676019430160522, "mix_low": 0.0043315887451171875, "mix_high": 0.016119003295898438, "read_cosine": 0.24910783767700195, "beta_mean": 0.010465689934790134, "nonfinite": 0.0, "weighted_memory_to_active": 0.048603516072034836, "mix_std": 0.24584902822971344, "beta_std": 0.009490103460848331, "beta_low": 0.62103271484375, "beta_high": 0.0, "orthogonal_to_active": 0.02876262366771698, "evidence_weights": [-0.38129276037216187, -0.3294532001018524, 0.4423224925994873], "agreement_weight": 0.08028897643089294, "orthogonal_weight": -0.6965358853340149}, {"head": 4, "memory_to_active": 0.19046159088611603, "contribution_to_active": 0.47648391127586365, "mix_mean": 0.4686906933784485, "mix_low": 0.004927635192871094, "mix_high": 0.0074005126953125, "read_cosine": 0.8151387572288513, "beta_mean": 0.04933791607618332, "nonfinite": 0.0, "weighted_memory_to_active": 0.10871022939682007, "mix_std": 0.2630845904350281, "beta_std": 0.03555840998888016, "beta_low": 0.03558349609375, "beta_high": 0.0, "orthogonal_to_active": 0.031884096562862396, "evidence_weights": [-0.21780706942081451, -0.2619788348674774, 0.4109192490577698], "agreement_weight": 0.12061618268489838, "orthogonal_weight": -0.5402505397796631}, {"head": 5, "memory_to_active": 0.2940308451652527, "contribution_to_active": 0.3087049722671509, "mix_mean": 0.3113016188144684, "mix_low": 0.007382392883300781, "mix_high": 0.000774383544921875, "read_cosine": 0.7101781964302063, "beta_mean": 0.002286189468577504, "nonfinite": 0.0, "weighted_memory_to_active": 0.12543998658657074, "mix_std": 0.2418111264705658, "beta_std": 0.009777720086276531, "beta_low": 0.95703125, "beta_high": 0.0, "orthogonal_to_active": 0.047941237688064575, "evidence_weights": [-0.18213070929050446, -0.15286654233932495, 0.2068103700876236], "agreement_weight": 0.0875387042760849, "orthogonal_weight": -0.6235451102256775}]}, {"layer": 15, "memory_to_active": 0.07256998121738434, "contribution_to_active": 0.4838354289531708, "mix_mean": 0.3956536650657654, "mix_low": 0.009445826523005962, "mix_high": 0.002769788261502981, "read_cosine": 0.6615641117095947, "beta_mean": 0.04473605379462242, "nonfinite": 0.0, "weighted_memory_to_active": 0.03375295549631119, "mix_std": 0.282631516456604, "beta_std": 0.04925810545682907, "beta_low": 0.7003173828125, "beta_high": 0.0, "orthogonal_to_active": 0.008105375804007053, "output_influence": 0.3053702712059021, "residual_influence": 0.16209083795547485, "state_active": 0.11242765933275223, "state_memory": 0.009293872863054276, "state_peak": 39.43342208862305, "growth_active": 1.2475106716156006, "growth_memory": 1.7917778491973877, "heads": [{"head": 0, "memory_to_active": 0.12082036584615707, "contribution_to_active": 0.4560009837150574, "mix_mean": 0.3532175123691559, "mix_low": 0.008807182312011719, "mix_high": 0.004313468933105469, "read_cosine": 0.6821697354316711, "beta_mean": 0.04416532441973686, "nonfinite": 0.0, "weighted_memory_to_active": 0.054472729563713074, "mix_std": 0.27359575033187866, "beta_std": 0.03225265070796013, "beta_low": 0.083740234375, "beta_high": 0.0, "orthogonal_to_active": 0.021570777520537376, "evidence_weights": [-0.16158856451511383, -0.18133887648582458, 0.34376493096351624], "agreement_weight": 0.08807659149169922, "orthogonal_weight": -0.6230341792106628}, {"head": 1, "memory_to_active": 0.15908217430114746, "contribution_to_active": 0.4184570908546448, "mix_mean": 0.4037192463874817, "mix_low": 0.00431060791015625, "mix_high": 0.0038633346557617188, "read_cosine": 0.8894262313842773, "beta_mean": 0.0016865230863913894, "nonfinite": 0.0, "weighted_memory_to_active": 0.06548710912466049, "mix_std": 0.28249698877334595, "beta_std": 0.004283214919269085, "beta_low": 0.96722412109375, "beta_high": 0.0, "orthogonal_to_active": 0.0022644205018877983, "evidence_weights": [-0.17800134420394897, -0.18551650643348694, 0.26791486144065857], "agreement_weight": 0.09807263314723969, "orthogonal_weight": -0.05905934423208237}, {"head": 2, "memory_to_active": 0.0075113228522241116, "contribution_to_active": 0.47715920209884644, "mix_mean": 0.3486015796661377, "mix_low": 0.01587963104248047, "mix_high": 0.00010204315185546875, "read_cosine": 0.7817093133926392, "beta_mean": 0.00011274335702182725, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034368294291198254, "mix_std": 0.27587783336639404, "beta_std": 0.00022898332099430263, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00021053178352303803, "evidence_weights": [-0.20789141952991486, -0.27854305505752563, 0.3261829614639282], "agreement_weight": 0.016364121809601784, "orthogonal_weight": 0.1249590516090393}, {"head": 3, "memory_to_active": 0.03297720476984978, "contribution_to_active": 0.4469088315963745, "mix_mean": 0.3394833207130432, "mix_low": 0.006960868835449219, "mix_high": 5.245208740234375e-05, "read_cosine": 0.5062488317489624, "beta_mean": 0.00041795108700171113, "nonfinite": 0.0, "weighted_memory_to_active": 0.016623400151729584, "mix_std": 0.2732173800468445, "beta_std": 0.0018129633972421288, "beta_low": 0.99420166015625, "beta_high": 0.0, "orthogonal_to_active": 0.01193928997963667, "evidence_weights": [-0.2025606483221054, -0.2505233585834503, 0.3868500888347626], "agreement_weight": 0.020460344851017, "orthogonal_weight": -0.9672508835792542}, {"head": 4, "memory_to_active": 0.09830072522163391, "contribution_to_active": 0.6100932955741882, "mix_mean": 0.38553857803344727, "mix_low": 0.011908531188964844, "mix_high": 0.0007257461547851562, "read_cosine": 0.7153000831604004, "beta_mean": 0.22097593545913696, "nonfinite": 0.0, "weighted_memory_to_active": 0.051894910633563995, "mix_std": 0.30724576115608215, "beta_std": 0.2487124502658844, "beta_low": 0.17279052734375, "beta_high": 0.0, "orthogonal_to_active": 0.004061403684318066, "evidence_weights": [-0.23021699488162994, -0.2593720257282257, 0.053925223648548126], "agreement_weight": 0.013616643846035004, "orthogonal_weight": 0.14720633625984192}, {"head": 5, "memory_to_active": 0.016728054732084274, "contribution_to_active": 0.49439308047294617, "mix_mean": 0.5433619022369385, "mix_low": 0.008808135986328125, "mix_high": 0.007561683654785156, "read_cosine": 0.3945305049419403, "beta_mean": 0.0010578231886029243, "nonfinite": 0.0, "weighted_memory_to_active": 0.010602733120322227, "mix_std": 0.2833552360534668, "beta_std": 0.008258354850113392, "beta_low": 0.9840087890625, "beta_high": 0.0, "orthogonal_to_active": 0.008585833944380283, "evidence_weights": [-0.34764546155929565, -0.3069404363632202, 0.5350992679595947], "agreement_weight": 0.25960734486579895, "orthogonal_weight": 0.9654804468154907}]}]}, {"step": 27000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8071614360014792, "memory_to_active": 0.9052844047546387, "contribution_to_active": 0.4955737888813019, "mix_mean": 0.35359346866607666, "mix_low": 0.15717731416225433, "mix_high": 0.007867911830544472, "read_cosine": 0.3815837502479553, "beta_mean": 0.08422686904668808, "nonfinite": 0.0, "weighted_memory_to_active": 0.2050684690475464, "mix_std": 0.2153037190437317, "beta_std": 0.06984355300664902, "beta_low": 0.48889225721359253, "beta_high": 1.71661376953125e-05, "orthogonal_to_active": 0.049376294016838074, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1468183994293213, "normal_token_accuracy": 0.4160178005695343, "state_active": 0.06060570850968361, "state_memory": 0.019942423328757286, "state_peak": 114.20907592773438, "growth_active": 1.1448426246643066, "growth_memory": 1.384181261062622, "output_influence": 0.35738053917884827, "residual_influence": 0.11379324644804001, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2120749950408936, "no_memory_loss_delta": 0.33503881096839905, "scrambled_loss_delta": 0.17857222259044647, "memory_only_loss_delta": 2.5871644020080566, "no_evidence_loss_delta": 0.1946152001619339, "no_agreement_loss_delta": 0.00805285107344389, "no_orthogonal_loss_delta": 0.012992668896913528}, {"index": 94, "tokens": 256, "normal_loss": 3.2114531993865967, "no_memory_loss_delta": 0.42567965388298035, "scrambled_loss_delta": 0.34664303064346313, "memory_only_loss_delta": 2.6247198581695557, "no_evidence_loss_delta": 0.2019391506910324, "no_agreement_loss_delta": 0.027767792344093323, "no_orthogonal_loss_delta": 0.025326291099190712}, {"index": 188, "tokens": 256, "normal_loss": 3.566377639770508, "no_memory_loss_delta": 0.35028576850891113, "scrambled_loss_delta": 0.15261241793632507, "memory_only_loss_delta": 2.666888952255249, "no_evidence_loss_delta": 0.20714926719665527, "no_agreement_loss_delta": 0.01510768011212349, "no_orthogonal_loss_delta": 0.016914311796426773}, {"index": 283, "tokens": 131, "normal_loss": 2.0730817317962646, "no_memory_loss_delta": 0.29307717084884644, "scrambled_loss_delta": 0.306212842464447, "memory_only_loss_delta": 2.636134386062622, "no_evidence_loss_delta": 0.10238023847341537, "no_agreement_loss_delta": -0.0006082421750761569, "no_orthogonal_loss_delta": 0.008194604888558388}], "agreement_weight_abs_mean": 0.13329283893108368, "agreement_weight_abs_max": 2.9728341102600098, "agreement_weight_mean": 0.11406735330820084, "orthogonal_weight_abs_mean": 0.634160578250885, "orthogonal_weight_abs_max": 4.92830753326416, "orthogonal_weight_mean": -0.16155855357646942, "no_memory_loss": 3.5058951377868652, "no_memory_loss_delta": 0.35907694697380066, "no_memory_token_accuracy_gain": 0.04338153451681137, "no_memory_token_win_fraction": 0.6351501941680908, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 30.167938437391566, "scrambled_loss": 3.384457588195801, "scrambled_loss_delta": 0.23763936758041382, "scrambled_token_accuracy_gain": 0.023359287530183792, "scrambled_token_win_fraction": 0.628476083278656, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 21.151300620131117, "memory_only_loss": 5.774515628814697, "memory_only_loss_delta": 2.627696990966797, "memory_only_token_accuracy_gain": 0.2358175814151764, "memory_only_token_win_fraction": 0.8765294551849365, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 92.77553491180788, "no_evidence_loss": 3.333648204803467, "no_evidence_loss_delta": 0.18682973086833954, "no_evidence_token_accuracy_gain": 0.0055617354810237885, "no_evidence_token_win_fraction": 0.5984427332878113, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 17.041502292571703, "no_agreement_loss": 3.1612319946289062, "no_agreement_loss_delta": 0.014413760043680668, "no_agreement_token_accuracy_gain": -0.006674082484096289, "no_agreement_token_win_fraction": 0.4627363681793213, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.431037910311122, "no_orthogonal_loss": 3.163740634918213, "no_orthogonal_loss_delta": 0.016922369599342346, "no_orthogonal_token_accuracy_gain": 0.0055617354810237885, "no_orthogonal_token_win_fraction": 0.5550611615180969, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 1.6779990564424019, "short_no_memory_loss_delta": 0.3037145435810089, "long_context_memory_gain": 0.05536240339279175, "normal_context_loss_gain": 0.14678779244422913, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.215588092803955, "contribution_to_active": 0.9811933636665344, "mix_mean": 0.1008460596203804, "mix_low": 0.5309059023857117, "mix_high": 0.004507859703153372, "read_cosine": 0.45442208647727966, "beta_mean": 0.2601592242717743, "nonfinite": 0.0, "weighted_memory_to_active": 1.0522162914276123, "mix_std": 0.08703483641147614, "beta_std": 0.15404152870178223, "beta_low": 0.009531657211482525, "beta_high": 0.0, "orthogonal_to_active": 0.07055774331092834, "output_influence": 0.15833956003189087, "residual_influence": 0.18874143064022064, "state_active": 0.07388795912265778, "state_memory": 0.0491151288151741, "state_peak": 45.86943817138672, "growth_active": 1.2469478845596313, "growth_memory": 1.246734857559204, "heads": [{"head": 0, "memory_to_active": 2.446101427078247, "contribution_to_active": 0.37499919533729553, "mix_mean": 0.017930256202816963, "mix_low": 0.6525211334228516, "mix_high": 0.0, "read_cosine": 0.26706746220588684, "beta_mean": 0.21170151233673096, "nonfinite": 0.0, "weighted_memory_to_active": 0.45410773158073425, "mix_std": 0.05307082086801529, "beta_std": 0.10201363265514374, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03463888540863991, "evidence_weights": [-0.029766419902443886, -0.0762469545006752, -0.1113893985748291], "agreement_weight": -0.10771499574184418, "orthogonal_weight": -0.08788217604160309}, {"head": 1, "memory_to_active": 5.267980575561523, "contribution_to_active": 1.984863042831421, "mix_mean": 0.01524725928902626, "mix_low": 0.6673622131347656, "mix_high": 0.0, "read_cosine": 0.5074347853660583, "beta_mean": 0.4387086033821106, "nonfinite": 0.0, "weighted_memory_to_active": 2.4013559818267822, "mix_std": 0.04988017678260803, "beta_std": 0.18244226276874542, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.18107905983924866, "evidence_weights": [0.022371415048837662, -0.08031894266605377, 0.2868345379829407], "agreement_weight": 0.025498969480395317, "orthogonal_weight": -0.17445456981658936}, {"head": 2, "memory_to_active": 2.591090202331543, "contribution_to_active": 0.5078402757644653, "mix_mean": 0.015938518568873405, "mix_low": 0.6163797378540039, "mix_high": 0.0, "read_cosine": 0.335636168718338, "beta_mean": 0.3112879991531372, "nonfinite": 0.0, "weighted_memory_to_active": 0.5939317941665649, "mix_std": 0.032974258065223694, "beta_std": 0.22161293029785156, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.058548182249069214, "evidence_weights": [0.04223598912358284, 0.05675134435296059, -0.04116392135620117], "agreement_weight": 0.00019048279500566423, "orthogonal_weight": -0.18676427006721497}, {"head": 3, "memory_to_active": 5.9850592613220215, "contribution_to_active": 2.5969762802124023, "mix_mean": 0.042518749833106995, "mix_low": 0.41295337677001953, "mix_high": 0.0, "read_cosine": 0.3541873097419739, "beta_mean": 0.3532158136367798, "nonfinite": 0.0, "weighted_memory_to_active": 2.7613420486450195, "mix_std": 0.08903925120830536, "beta_std": 0.24559924006462097, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1275300234556198, "evidence_weights": [-0.06759274750947952, -0.08946821093559265, -0.13192661106586456], "agreement_weight": -0.014862840063869953, "orthogonal_weight": -0.055213563144207}, {"head": 4, "memory_to_active": 2.8899712562561035, "contribution_to_active": 0.056555166840553284, "mix_mean": 0.006489413790404797, "mix_low": 0.8222980499267578, "mix_high": 0.0, "read_cosine": 0.32341980934143066, "beta_mean": 0.20623314380645752, "nonfinite": 0.0, "weighted_memory_to_active": 0.05320483818650246, "mix_std": 0.012228046543896198, "beta_std": 0.1422654092311859, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006103796884417534, "evidence_weights": [0.11234650015830994, 0.05950719863176346, -0.3461494445800781], "agreement_weight": -0.08883683383464813, "orthogonal_weight": 0.12512628734111786}, {"head": 5, "memory_to_active": 0.11332505941390991, "contribution_to_active": 0.36592596769332886, "mix_mean": 0.506952166557312, "mix_low": 0.013920783996582031, "mix_high": 0.027047157287597656, "read_cosine": 0.938787043094635, "beta_mean": 0.03980826213955879, "nonfinite": 0.0, "weighted_memory_to_active": 0.04935493692755699, "mix_std": 0.2850164771080017, "beta_std": 0.030315693467855453, "beta_low": 0.05718994140625, "beta_high": 0.0, "orthogonal_to_active": 0.015446467325091362, "evidence_weights": [-0.26541849970817566, -0.25367099046707153, 0.24670225381851196], "agreement_weight": 0.23183517158031464, "orthogonal_weight": -0.5839516520500183}]}, {"layer": 1, "memory_to_active": 2.2385354042053223, "contribution_to_active": 0.43136751651763916, "mix_mean": 0.16235646605491638, "mix_low": 0.3762715756893158, "mix_high": 0.0005362828960642219, "read_cosine": 0.3370991349220276, "beta_mean": 0.1986854076385498, "nonfinite": 0.0, "weighted_memory_to_active": 0.3189908266067505, "mix_std": 0.11845740675926208, "beta_std": 0.1332465410232544, "beta_low": 0.1940409392118454, "beta_high": 4.069010537932627e-05, "orthogonal_to_active": 0.04147763177752495, "output_influence": 0.17443962395191193, "residual_influence": 0.041779015213251114, "state_active": 0.040140192955732346, "state_memory": 0.03715500608086586, "state_peak": 29.632753372192383, "growth_active": 1.2126749753952026, "growth_memory": 1.4943596124649048, "heads": [{"head": 0, "memory_to_active": 0.07094600051641464, "contribution_to_active": 0.4158391058444977, "mix_mean": 0.3861003816127777, "mix_low": 0.012749671936035156, "mix_high": 0.0001354217529296875, "read_cosine": -0.03190091624855995, "beta_mean": 0.00043550183181650937, "nonfinite": 0.0, "weighted_memory_to_active": 0.03605590760707855, "mix_std": 0.27413636445999146, "beta_std": 0.0005533488583751023, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.03157854825258255, "evidence_weights": [-0.1732298582792282, -0.2214718908071518, 0.0605786070227623], "agreement_weight": 0.14797694981098175, "orthogonal_weight": -0.9540606141090393}, {"head": 1, "memory_to_active": 3.0219686031341553, "contribution_to_active": 0.19712884724140167, "mix_mean": 0.02063337340950966, "mix_low": 0.49959564208984375, "mix_high": 0.0, "read_cosine": 0.45693886280059814, "beta_mean": 0.31096720695495605, "nonfinite": 0.0, "weighted_memory_to_active": 0.21060556173324585, "mix_std": 0.045576825737953186, "beta_std": 0.19527015089988708, "beta_low": 0.00048828125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.010299211367964745, "evidence_weights": [0.008228108286857605, -0.012612951919436455, -0.022766446694731712], "agreement_weight": -0.01018128264695406, "orthogonal_weight": 0.10001438111066818}, {"head": 2, "memory_to_active": 0.3975958228111267, "contribution_to_active": 0.507581889629364, "mix_mean": 0.5071293711662292, "mix_low": 0.002376556396484375, "mix_high": 0.003078460693359375, "read_cosine": 0.7754039764404297, "beta_mean": 0.03358514606952667, "nonfinite": 0.0, "weighted_memory_to_active": 0.307539701461792, "mix_std": 0.2606576383113861, "beta_std": 0.02563261240720749, "beta_low": 0.1126708984375, "beta_high": 0.0, "orthogonal_to_active": 0.0378214567899704, "evidence_weights": [-0.27442771196365356, -0.378458172082901, 0.2808132469654083], "agreement_weight": 0.024094248190522194, "orthogonal_weight": -0.24056491255760193}, {"head": 3, "memory_to_active": 2.832401990890503, "contribution_to_active": 0.09528205543756485, "mix_mean": 0.015780918300151825, "mix_low": 0.6842422485351562, "mix_high": 0.0, "read_cosine": 0.24440611898899078, "beta_mean": 0.35136905312538147, "nonfinite": 0.0, "weighted_memory_to_active": 0.13283689320087433, "mix_std": 0.04488294571638107, "beta_std": 0.24685312807559967, "beta_low": 0.0234375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.03907792270183563, "evidence_weights": [0.04162071645259857, 0.0004799313610419631, 0.11366412043571472], "agreement_weight": -0.02465754933655262, "orthogonal_weight": -0.32437869906425476}, {"head": 4, "memory_to_active": 3.1988065242767334, "contribution_to_active": 1.3086791038513184, "mix_mean": 0.03512651473283768, "mix_low": 0.29866600036621094, "mix_high": 3.814697265625e-06, "read_cosine": 0.33991438150405884, "beta_mean": 0.20435529947280884, "nonfinite": 0.0, "weighted_memory_to_active": 1.1698006391525269, "mix_std": 0.06361806392669678, "beta_std": 0.11954119056463242, "beta_low": 0.00372314453125, "beta_high": 0.0, "orthogonal_to_active": 0.1261579543352127, "evidence_weights": [-0.036872901022434235, -0.0004978350480087101, -0.017046816647052765], "agreement_weight": -0.020161308348178864, "orthogonal_weight": 0.13682521879673004}, {"head": 5, "memory_to_active": 3.909494400024414, "contribution_to_active": 0.06369400769472122, "mix_mean": 0.00936822034418583, "mix_low": 0.7599992752075195, "mix_high": 0.0, "read_cosine": 0.2378322035074234, "beta_mean": 0.2914002239704132, "nonfinite": 0.0, "weighted_memory_to_active": 0.0571061372756958, "mix_std": 0.021872617304325104, "beta_std": 0.2116287499666214, "beta_low": 0.0240478515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.003930696286261082, "evidence_weights": [0.13980455696582794, 0.06143275275826454, 0.3333621323108673], "agreement_weight": -0.14465822279453278, "orthogonal_weight": 0.06882283836603165}]}, {"layer": 2, "memory_to_active": 1.069101333618164, "contribution_to_active": 0.3250003457069397, "mix_mean": 0.331032931804657, "mix_low": 0.26487797498703003, "mix_high": 0.002700011245906353, "read_cosine": 0.33483847975730896, "beta_mean": 0.10159486532211304, "nonfinite": 0.0, "weighted_memory_to_active": 0.06832151859998703, "mix_std": 0.19605207443237305, "beta_std": 0.0730743408203125, "beta_low": 0.5426025390625, "beta_high": 0.00014241535973269492, "orthogonal_to_active": 0.016678497195243835, "output_influence": 0.31179773807525635, "residual_influence": 0.0774986743927002, "state_active": 0.03941956162452698, "state_memory": 0.019483894109725952, "state_peak": 37.61294937133789, "growth_active": 1.0633245706558228, "growth_memory": 1.574259877204895, "heads": [{"head": 0, "memory_to_active": 0.007241262122988701, "contribution_to_active": 0.2777624726295471, "mix_mean": 0.47638145089149475, "mix_low": 0.0027666091918945312, "mix_high": 0.003192901611328125, "read_cosine": 0.20319338142871857, "beta_mean": 5.854220944456756e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035859448835253716, "mix_std": 0.2920357584953308, "beta_std": 7.126517448341474e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000588097027502954, "evidence_weights": [-0.28170016407966614, -0.17969255149364471, 0.23352710902690887], "agreement_weight": 0.1871567666530609, "orthogonal_weight": -0.17433857917785645}, {"head": 1, "memory_to_active": 3.0702905654907227, "contribution_to_active": 0.052173517644405365, "mix_mean": 0.004980471916496754, "mix_low": 0.8929786682128906, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19479894638061523, "beta_mean": 0.2831422686576843, "nonfinite": 0.0, "weighted_memory_to_active": 0.03805505484342575, "mix_std": 0.01121467724442482, "beta_std": 0.14624910056591034, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.015908630564808846, "evidence_weights": [0.10317341983318329, 0.10963361710309982, 0.43533918261528015], "agreement_weight": 0.03771257400512695, "orthogonal_weight": 0.4309062361717224}, {"head": 2, "memory_to_active": 0.02309126779437065, "contribution_to_active": 0.4258536398410797, "mix_mean": 0.46453648805618286, "mix_low": 0.0019245147705078125, "mix_high": 0.0023851394653320312, "read_cosine": 0.22240430116653442, "beta_mean": 4.085942418896593e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008931638672947884, "mix_std": 0.2810041308403015, "beta_std": 6.904576002852991e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023767214734107256, "evidence_weights": [-0.2268371731042862, -0.26396480202674866, 0.4510449767112732], "agreement_weight": 0.045125991106033325, "orthogonal_weight": -0.31492742896080017}, {"head": 3, "memory_to_active": 0.21065962314605713, "contribution_to_active": 0.48438000679016113, "mix_mean": 0.453210711479187, "mix_low": 0.000732421875, "mix_high": 0.006122589111328125, "read_cosine": 0.7077768445014954, "beta_mean": 0.042742058634757996, "nonfinite": 0.0, "weighted_memory_to_active": 0.11597947776317596, "mix_std": 0.2678406834602356, "beta_std": 0.0526389516890049, "beta_low": 0.233154296875, "beta_high": 0.0, "orthogonal_to_active": 0.06534568220376968, "evidence_weights": [-0.1638365387916565, -0.2550548017024994, 0.8576151132583618], "agreement_weight": 0.06598976254463196, "orthogonal_weight": -0.8601486682891846}, {"head": 4, "memory_to_active": 3.083125591278076, "contribution_to_active": 0.19049134850502014, "mix_mean": 0.016579091548919678, "mix_low": 0.6893329620361328, "mix_high": 0.0, "read_cosine": 0.2210104614496231, "beta_mean": 0.28352588415145874, "nonfinite": 0.0, "weighted_memory_to_active": 0.2305522859096527, "mix_std": 0.04839479923248291, "beta_std": 0.23934000730514526, "beta_low": 0.0224609375, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.01496841013431549, "evidence_weights": [0.03454049304127693, 0.03504396229982376, 0.10566588491201401], "agreement_weight": 0.027867231518030167, "orthogonal_weight": -0.07507498562335968}, {"head": 5, "memory_to_active": 0.02020026184618473, "contribution_to_active": 0.5193409323692322, "mix_mean": 0.5705093145370483, "mix_low": 0.0015325546264648438, "mix_high": 0.004498481750488281, "read_cosine": 0.4598467946052551, "beta_mean": 5.9555961342994124e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012824684381484985, "mix_std": 0.2758224606513977, "beta_std": 7.764157635392621e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008834333275444806, "evidence_weights": [-0.25208258628845215, -0.2772417962551117, 0.8326078057289124], "agreement_weight": 0.06500089913606644, "orthogonal_weight": 0.11911790817975998}]}, {"layer": 3, "memory_to_active": 1.0880446434020996, "contribution_to_active": 0.3822954297065735, "mix_mean": 0.3065668046474457, "mix_low": 0.1872045248746872, "mix_high": 0.004959742538630962, "read_cosine": 0.24159881472587585, "beta_mean": 0.06747718155384064, "nonfinite": 0.0, "weighted_memory_to_active": 0.17223532497882843, "mix_std": 0.2291594296693802, "beta_std": 0.058896254748106, "beta_low": 0.4486287534236908, "beta_high": 0.0, "orthogonal_to_active": 0.06640520691871643, "output_influence": 0.305996298789978, "residual_influence": 0.05929587781429291, "state_active": 0.024874797090888023, "state_memory": 0.014400664716959, "state_peak": 20.157392501831055, "growth_active": 1.0910687446594238, "growth_memory": 1.8510961532592773, "heads": [{"head": 0, "memory_to_active": 0.5711795687675476, "contribution_to_active": 0.47496140003204346, "mix_mean": 0.40394943952560425, "mix_low": 0.025541305541992188, "mix_high": 0.006741523742675781, "read_cosine": 0.2431209236383438, "beta_mean": 0.014925479888916016, "nonfinite": 0.0, "weighted_memory_to_active": 0.27069324254989624, "mix_std": 0.32075557112693787, "beta_std": 0.017304450273513794, "beta_low": 0.49774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.17161579430103302, "evidence_weights": [-0.1589970737695694, -0.2025282382965088, 0.5453459024429321], "agreement_weight": 0.12147368490695953, "orthogonal_weight": -0.6750338673591614}, {"head": 1, "memory_to_active": 0.019351793453097343, "contribution_to_active": 0.33815956115722656, "mix_mean": 0.3901529312133789, "mix_low": 0.021330833435058594, "mix_high": 0.00267791748046875, "read_cosine": 0.21209251880645752, "beta_mean": 9.837823745328933e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007703118957579136, "mix_std": 0.3030620813369751, "beta_std": 0.0002780470240395516, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.002973319496959448, "evidence_weights": [-0.19761575758457184, -0.17063988745212555, 0.5489691495895386], "agreement_weight": 0.31237101554870605, "orthogonal_weight": -0.4269823133945465}, {"head": 2, "memory_to_active": 2.8236165046691895, "contribution_to_active": 0.3318605422973633, "mix_mean": 0.047808852046728134, "mix_low": 0.35514163970947266, "mix_high": 0.0, "read_cosine": 0.18058842420578003, "beta_mean": 0.18514519929885864, "nonfinite": 0.0, "weighted_memory_to_active": 0.4027408957481384, "mix_std": 0.10216528177261353, "beta_std": 0.13961510360240936, "beta_low": 0.0084228515625, "beta_high": 0.0, "orthogonal_to_active": 0.010056066326797009, "evidence_weights": [-0.06356101483106613, 0.013141649775207043, -0.020218420773744583], "agreement_weight": -0.014647599309682846, "orthogonal_weight": -0.031817905604839325}, {"head": 3, "memory_to_active": 0.026094842702150345, "contribution_to_active": 0.471915602684021, "mix_mean": 0.5078176259994507, "mix_low": 0.016000747680664062, "mix_high": 0.011965751647949219, "read_cosine": 0.2850415110588074, "beta_mean": 0.00011230180825805292, "nonfinite": 0.0, "weighted_memory_to_active": 0.016248300671577454, "mix_std": 0.3119143545627594, "beta_std": 0.00023828040866646916, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004651178140193224, "evidence_weights": [-0.20267744362354279, -0.21923550963401794, 1.8442431688308716], "agreement_weight": 1.168944239616394, "orthogonal_weight": -0.37611547112464905}, {"head": 4, "memory_to_active": 2.772972583770752, "contribution_to_active": 0.05031188577413559, "mix_mean": 0.01465810276567936, "mix_low": 0.683323860168457, "mix_high": 0.0, "read_cosine": 0.1991390436887741, "beta_mean": 0.15266487002372742, "nonfinite": 0.0, "weighted_memory_to_active": 0.12792296707630157, "mix_std": 0.033653777092695236, "beta_std": 0.14185377955436707, "beta_low": 0.03466796875, "beta_high": 0.0, "orthogonal_to_active": 0.08072425425052643, "evidence_weights": [0.12894105911254883, 0.08495425432920456, 0.02438526041805744], "agreement_weight": -0.04365610331296921, "orthogonal_weight": -0.6568756103515625}, {"head": 5, "memory_to_active": 0.31505286693573, "contribution_to_active": 0.6265636682510376, "mix_mean": 0.47501376271247864, "mix_low": 0.02188873291015625, "mix_high": 0.008373260498046875, "read_cosine": 0.3296104967594147, "beta_mean": 0.05191687494516373, "nonfinite": 0.0, "weighted_memory_to_active": 0.20810335874557495, "mix_std": 0.3034055233001709, "beta_std": 0.05408784747123718, "beta_low": 0.1510009765625, "beta_high": 0.0, "orthogonal_to_active": 0.12841063737869263, "evidence_weights": [-0.18919000029563904, -0.23473426699638367, 0.41565364599227905], "agreement_weight": 0.1496458351612091, "orthogonal_weight": -0.7280197143554688}]}, {"layer": 4, "memory_to_active": 1.173500418663025, "contribution_to_active": 0.41417962312698364, "mix_mean": 0.33090701699256897, "mix_low": 0.2782685160636902, "mix_high": 0.006426175590604544, "read_cosine": 0.3705598711967468, "beta_mean": 0.1209302768111229, "nonfinite": 0.0, "weighted_memory_to_active": 0.14056997001171112, "mix_std": 0.15913087129592896, "beta_std": 0.10795992612838745, "beta_low": 0.3677673637866974, "beta_high": 0.0, "orthogonal_to_active": 0.04032382369041443, "output_influence": 0.35632169246673584, "residual_influence": 0.09034895896911621, "state_active": 0.035325415432453156, "state_memory": 0.021100232377648354, "state_peak": 42.345333099365234, "growth_active": 1.1380562782287598, "growth_memory": 1.607932686805725, "heads": [{"head": 0, "memory_to_active": 0.270573228597641, "contribution_to_active": 0.6939015984535217, "mix_mean": 0.633463442325592, "mix_low": 0.008057594299316406, "mix_high": 0.021162986755371094, "read_cosine": 0.4886935353279114, "beta_mean": 0.24861061573028564, "nonfinite": 0.0, "weighted_memory_to_active": 0.21211586892604828, "mix_std": 0.33229970932006836, "beta_std": 0.2561003565788269, "beta_low": 0.198974609375, "beta_high": 0.0, "orthogonal_to_active": 0.06232206150889397, "evidence_weights": [-0.26237618923187256, -0.35136979818344116, 0.3360513150691986], "agreement_weight": 0.035796549171209335, "orthogonal_weight": 0.35676121711730957}, {"head": 1, "memory_to_active": 0.11241096258163452, "contribution_to_active": 0.5501511096954346, "mix_mean": 0.5933256149291992, "mix_low": 3.337860107421875e-05, "mix_high": 0.00563812255859375, "read_cosine": 0.7293109893798828, "beta_mean": 0.0031156502664089203, "nonfinite": 0.0, "weighted_memory_to_active": 0.07037953287363052, "mix_std": 0.2581939697265625, "beta_std": 0.008411046117544174, "beta_low": 0.935791015625, "beta_high": 0.0, "orthogonal_to_active": 0.03699376434087753, "evidence_weights": [-0.2347785085439682, -0.2907867431640625, 0.7243255376815796], "agreement_weight": 0.08344294130802155, "orthogonal_weight": -0.806627631187439}, {"head": 2, "memory_to_active": 2.395063638687134, "contribution_to_active": 0.15808232128620148, "mix_mean": 0.017789460718631744, "mix_low": 0.5942659378051758, "mix_high": 0.0, "read_cosine": 0.2796044647693634, "beta_mean": 0.12246088683605194, "nonfinite": 0.0, "weighted_memory_to_active": 0.20058561861515045, "mix_std": 0.03454706445336342, "beta_std": 0.09420207887887955, "beta_low": 0.0098876953125, "beta_high": 0.0, "orthogonal_to_active": 0.013471847400069237, "evidence_weights": [0.05296922102570534, 0.029039176180958748, -0.024609293788671494], "agreement_weight": 0.10751975327730179, "orthogonal_weight": -0.0991223007440567}, {"head": 3, "memory_to_active": 1.692335605621338, "contribution_to_active": 0.12406472116708755, "mix_mean": 0.03151870518922806, "mix_low": 0.41368865966796875, "mix_high": 0.0, "read_cosine": 0.1722021996974945, "beta_mean": 0.1161886528134346, "nonfinite": 0.0, "weighted_memory_to_active": 0.15336185693740845, "mix_std": 0.05884508043527603, "beta_std": 0.11358603835105896, "beta_low": 0.06195068359375, "beta_high": 0.0, "orthogonal_to_active": 0.014213431626558304, "evidence_weights": [-0.02223357744514942, 0.05474305525422096, -0.2553592324256897], "agreement_weight": 0.04231034219264984, "orthogonal_weight": -0.11044243723154068}, {"head": 4, "memory_to_active": 0.029293080791831017, "contribution_to_active": 0.6492096185684204, "mix_mean": 0.6953007578849792, "mix_low": 0.00020313262939453125, "mix_high": 0.011753082275390625, "read_cosine": 0.22059538960456848, "beta_mean": 5.479782703332603e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.021049289032816887, "mix_std": 0.23499412834644318, "beta_std": 0.00015698210336267948, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0033702091313898563, "evidence_weights": [-0.30287763476371765, -0.3321267068386078, 0.7662208676338196], "agreement_weight": 0.15973781049251556, "orthogonal_weight": -0.1789407879114151}, {"head": 5, "memory_to_active": 2.541325807571411, "contribution_to_active": 0.30966824293136597, "mix_mean": 0.014044051989912987, "mix_low": 0.6533622741699219, "mix_high": 2.86102294921875e-06, "read_cosine": 0.332952618598938, "beta_mean": 0.2351510375738144, "nonfinite": 0.0, "weighted_memory_to_active": 0.1859275847673416, "mix_std": 0.03590530529618263, "beta_std": 0.17530308663845062, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11157160997390747, "evidence_weights": [0.04341956973075867, -0.03797858580946922, -0.2110005021095276], "agreement_weight": -0.048071447759866714, "orthogonal_weight": 0.7921304702758789}]}, {"layer": 5, "memory_to_active": 0.6038312911987305, "contribution_to_active": 0.4728982150554657, "mix_mean": 0.4269569516181946, "mix_low": 0.07712936401367188, "mix_high": 0.011384646408259869, "read_cosine": 0.17570801079273224, "beta_mean": 0.053627341985702515, "nonfinite": 0.0, "weighted_memory_to_active": 0.16061809659004211, "mix_std": 0.2516805827617645, "beta_std": 0.0691046193242073, "beta_low": 0.6309102773666382, "beta_high": 0.0, "orthogonal_to_active": 0.056018125265836716, "output_influence": 0.46059003472328186, "residual_influence": 0.11503750085830688, "state_active": 0.04709363356232643, "state_memory": 0.012521996162831783, "state_peak": 62.49024200439453, "growth_active": 1.1077625751495361, "growth_memory": 1.2681723833084106, "heads": [{"head": 0, "memory_to_active": 0.9860318899154663, "contribution_to_active": 0.32987844944000244, "mix_mean": 0.18601971864700317, "mix_low": 0.0876016616821289, "mix_high": 0.0005407333374023438, "read_cosine": -0.1187913790345192, "beta_mean": 0.043330926448106766, "nonfinite": 0.0, "weighted_memory_to_active": 0.15530772507190704, "mix_std": 0.21714863181114197, "beta_std": 0.07325813174247742, "beta_low": 0.2938232421875, "beta_high": 0.0, "orthogonal_to_active": 0.03392370790243149, "evidence_weights": [-0.07600873708724976, -0.11105749756097794, -0.34219321608543396], "agreement_weight": -0.015226798132061958, "orthogonal_weight": -0.20631270110607147}, {"head": 1, "memory_to_active": 0.03369607776403427, "contribution_to_active": 0.5145431756973267, "mix_mean": 0.5852729082107544, "mix_low": 0.00466156005859375, "mix_high": 0.01626300811767578, "read_cosine": 0.1505892425775528, "beta_mean": 8.291733684018254e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01697506755590439, "mix_std": 0.3068966865539551, "beta_std": 0.00015427324979100376, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015308955684304237, "evidence_weights": [-0.23327335715293884, -0.26399344205856323, 0.8336503505706787], "agreement_weight": 0.4087868332862854, "orthogonal_weight": -0.9297763109207153}, {"head": 2, "memory_to_active": 0.03505079820752144, "contribution_to_active": 0.5492458343505859, "mix_mean": 0.6198737621307373, "mix_low": 0.0019283294677734375, "mix_high": 0.014368057250976562, "read_cosine": 0.20021378993988037, "beta_mean": 0.0005005054408684373, "nonfinite": 0.0, "weighted_memory_to_active": 0.023927796632051468, "mix_std": 0.27161869406700134, "beta_std": 0.0008739655604586005, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.0045038629323244095, "evidence_weights": [-0.25570955872535706, -0.3470475375652313, 0.8892439007759094], "agreement_weight": 0.030291331931948662, "orthogonal_weight": -0.2406514585018158}, {"head": 3, "memory_to_active": 1.9859137535095215, "contribution_to_active": 0.2292873114347458, "mix_mean": 0.08834655582904816, "mix_low": 0.1738605499267578, "mix_high": 0.0, "read_cosine": 0.12973633408546448, "beta_mean": 0.07897026091814041, "nonfinite": 0.0, "weighted_memory_to_active": 0.37634947896003723, "mix_std": 0.13427141308784485, "beta_std": 0.1180167943239212, "beta_low": 0.23419189453125, "beta_high": 0.0, "orthogonal_to_active": 0.13871990144252777, "evidence_weights": [-0.05623461678624153, -0.09045357257127762, -0.09648656845092773], "agreement_weight": 0.056622788310050964, "orthogonal_weight": -0.4567579925060272}, {"head": 4, "memory_to_active": 0.45744121074676514, "contribution_to_active": 0.588266909122467, "mix_mean": 0.3227265477180481, "mix_low": 0.19180870056152344, "mix_high": 0.008856773376464844, "read_cosine": 0.5452778339385986, "beta_mean": 0.19845011830329895, "nonfinite": 0.0, "weighted_memory_to_active": 0.30532920360565186, "mix_std": 0.335019588470459, "beta_std": 0.22007672488689423, "beta_low": 0.26580810546875, "beta_high": 0.0, "orthogonal_to_active": 0.0898783877491951, "evidence_weights": [-0.1152711734175682, -0.288427472114563, -0.0662737488746643], "agreement_weight": 0.04905761405825615, "orthogonal_weight": 0.4457715153694153}, {"head": 5, "memory_to_active": 0.12485408782958984, "contribution_to_active": 0.6261675357818604, "mix_mean": 0.7595022916793823, "mix_low": 0.0029153823852539062, "mix_high": 0.02827930450439453, "read_cosine": 0.14722223579883575, "beta_mean": 0.00042930099880322814, "nonfinite": 0.0, "weighted_memory_to_active": 0.08581927418708801, "mix_std": 0.24512843787670135, "beta_std": 0.0022478250321000814, "beta_low": 0.99249267578125, "beta_high": 0.0, "orthogonal_to_active": 0.05377393215894699, "evidence_weights": [-0.3260307312011719, -0.3192950189113617, 1.9781525135040283], "agreement_weight": 0.36227697134017944, "orthogonal_weight": -0.6633415222167969}]}, {"layer": 6, "memory_to_active": 0.37345021963119507, "contribution_to_active": 0.5688507556915283, "mix_mean": 0.5405107736587524, "mix_low": 0.07355451583862305, "mix_high": 0.027760984376072884, "read_cosine": 0.2799419164657593, "beta_mean": 0.07441146671772003, "nonfinite": 0.0, "weighted_memory_to_active": 0.1610504388809204, "mix_std": 0.24236303567886353, "beta_std": 0.08382022380828857, "beta_low": 0.50018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.04481521621346474, "output_influence": 0.48373135924339294, "residual_influence": 0.15666866302490234, "state_active": 0.04909110069274902, "state_memory": 0.009494030848145485, "state_peak": 71.78316497802734, "growth_active": 1.2426871061325073, "growth_memory": 1.1780990362167358, "heads": [{"head": 0, "memory_to_active": 0.2190714180469513, "contribution_to_active": 0.5987188816070557, "mix_mean": 0.7061189413070679, "mix_low": 3.337860107421875e-05, "mix_high": 0.005307197570800781, "read_cosine": -0.08627086132764816, "beta_mean": 0.01413507480174303, "nonfinite": 0.0, "weighted_memory_to_active": 0.15878677368164062, "mix_std": 0.22848522663116455, "beta_std": 0.021886590868234634, "beta_low": 0.53302001953125, "beta_high": 0.0, "orthogonal_to_active": 0.132970929145813, "evidence_weights": [-0.30955734848976135, -0.3274773955345154, 0.4987613260746002], "agreement_weight": 0.10954736918210983, "orthogonal_weight": -0.9367881417274475}, {"head": 1, "memory_to_active": 0.004218993242830038, "contribution_to_active": 0.47683054208755493, "mix_mean": 0.49634307622909546, "mix_low": 0.0015048980712890625, "mix_high": 0.008591651916503906, "read_cosine": 0.6065874695777893, "beta_mean": 9.493237485003192e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0018378989771008492, "mix_std": 0.29028114676475525, "beta_std": 1.918708767334465e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006283954717218876, "evidence_weights": [-0.20385831594467163, -0.27627110481262207, 0.9165706634521484], "agreement_weight": -0.020163489505648613, "orthogonal_weight": -0.4968145489692688}, {"head": 2, "memory_to_active": 1.1823575496673584, "contribution_to_active": 0.07719919085502625, "mix_mean": 0.02854391187429428, "mix_low": 0.4301185607910156, "mix_high": 0.0, "read_cosine": 0.21985746920108795, "beta_mean": 0.06845885515213013, "nonfinite": 0.0, "weighted_memory_to_active": 0.06862428784370422, "mix_std": 0.05137351155281067, "beta_std": 0.06992091238498688, "beta_low": 0.01263427734375, "beta_high": 0.0, "orthogonal_to_active": 0.0016496016178280115, "evidence_weights": [0.08581119030714035, -0.002206734148785472, 0.2661935091018677], "agreement_weight": 0.10935336351394653, "orthogonal_weight": -0.028453700244426727}, {"head": 3, "memory_to_active": 0.14632034301757812, "contribution_to_active": 0.6509392857551575, "mix_mean": 0.6351069211959839, "mix_low": 0.007883071899414062, "mix_high": 0.01482391357421875, "read_cosine": 0.47543197870254517, "beta_mean": 0.22385019063949585, "nonfinite": 0.0, "weighted_memory_to_active": 0.10542906075716019, "mix_std": 0.3314329981803894, "beta_std": 0.25631529092788696, "beta_low": 0.2135009765625, "beta_high": 0.0, "orthogonal_to_active": 0.05561232566833496, "evidence_weights": [-0.17307746410369873, -0.3361521065235138, 0.1524418145418167], "agreement_weight": -0.012756654992699623, "orthogonal_weight": 0.6148625612258911}, {"head": 4, "memory_to_active": 0.636814296245575, "contribution_to_active": 1.0253838300704956, "mix_mean": 0.7650309801101685, "mix_low": 0.0013484954833984375, "mix_high": 0.13574504852294922, "read_cosine": 0.1884443163871765, "beta_mean": 0.1395505964756012, "nonfinite": 0.0, "weighted_memory_to_active": 0.5969262719154358, "mix_std": 0.2866964638233185, "beta_std": 0.1538507640361786, "beta_low": 0.24346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.04281432181596756, "evidence_weights": [-0.2576476037502289, -0.5899091362953186, -0.11536456644535065], "agreement_weight": -0.013322358019649982, "orthogonal_weight": 0.08552796393632889}, {"head": 5, "memory_to_active": 0.05191868543624878, "contribution_to_active": 0.5840326547622681, "mix_mean": 0.6119205951690674, "mix_low": 0.000438690185546875, "mix_high": 0.00209808349609375, "read_cosine": 0.27560096979141235, "beta_mean": 0.00046462472528219223, "nonfinite": 0.0, "weighted_memory_to_active": 0.03469835966825485, "mix_std": 0.2659088373184204, "beta_std": 0.0009286345448344946, "beta_low": 0.99847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.03521573543548584, "evidence_weights": [-0.23902399837970734, -0.29984191060066223, 0.39743393659591675], "agreement_weight": 0.11881193518638611, "orthogonal_weight": -1.083554983139038}]}, {"layer": 7, "memory_to_active": 1.8633713722229004, "contribution_to_active": 0.18475960195064545, "mix_mean": 0.06779855489730835, "mix_low": 0.34217342734336853, "mix_high": 8.074443030636758e-05, "read_cosine": 0.23059020936489105, "beta_mean": 0.13223318755626678, "nonfinite": 0.0, "weighted_memory_to_active": 0.19696743786334991, "mix_std": 0.10019541531801224, "beta_std": 0.10811038315296173, "beta_low": 0.1231689527630806, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.052474986761808395, "output_influence": 0.21349972486495972, "residual_influence": 0.04647291079163551, "state_active": 0.014192628674209118, "state_memory": 0.028410380706191063, "state_peak": 31.220195770263672, "growth_active": 1.0955147743225098, "growth_memory": 1.357072353363037, "heads": [{"head": 0, "memory_to_active": 2.2863895893096924, "contribution_to_active": 0.24196486175060272, "mix_mean": 0.032697129994630814, "mix_low": 0.4544963836669922, "mix_high": 0.0, "read_cosine": 0.3748965263366699, "beta_mean": 0.17009735107421875, "nonfinite": 0.0, "weighted_memory_to_active": 0.2447543740272522, "mix_std": 0.06854169815778732, "beta_std": 0.1278124451637268, "beta_low": 0.00531005859375, "beta_high": 0.00048828125, "orthogonal_to_active": 0.017988013103604317, "evidence_weights": [0.007338705472648144, -0.0892149955034256, 0.09955655038356781], "agreement_weight": -0.0371362678706646, "orthogonal_weight": 0.09590474516153336}, {"head": 1, "memory_to_active": 1.1873799562454224, "contribution_to_active": 0.33384397625923157, "mix_mean": 0.15825840830802917, "mix_low": 0.1827716827392578, "mix_high": 0.0003795623779296875, "read_cosine": 0.2610662579536438, "beta_mean": 0.12013080716133118, "nonfinite": 0.0, "weighted_memory_to_active": 0.3160841166973114, "mix_std": 0.19293449819087982, "beta_std": 0.07762303203344345, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.01017058826982975, "evidence_weights": [-0.17837461829185486, -0.11633283644914627, -0.26023733615875244], "agreement_weight": 0.024969017133116722, "orthogonal_weight": -0.03594662621617317}, {"head": 2, "memory_to_active": 1.0091798305511475, "contribution_to_active": 0.26359811425209045, "mix_mean": 0.08373890817165375, "mix_low": 0.17760276794433594, "mix_high": 0.000102996826171875, "read_cosine": 0.11590933799743652, "beta_mean": 0.07766984403133392, "nonfinite": 0.0, "weighted_memory_to_active": 0.18744195997714996, "mix_std": 0.13585636019706726, "beta_std": 0.05809277296066284, "beta_low": 0.05072021484375, "beta_high": 0.0, "orthogonal_to_active": 0.04114365577697754, "evidence_weights": [-0.030406666919589043, -0.1183159202337265, 0.3207651674747467], "agreement_weight": 0.023428967222571373, "orthogonal_weight": -0.22403261065483093}, {"head": 3, "memory_to_active": 3.911435127258301, "contribution_to_active": 0.07120198756456375, "mix_mean": 0.02839251235127449, "mix_low": 0.3830270767211914, "mix_high": 0.0, "read_cosine": 0.12459738552570343, "beta_mean": 0.24465781450271606, "nonfinite": 0.0, "weighted_memory_to_active": 0.22505387663841248, "mix_std": 0.044033508747816086, "beta_std": 0.2241305410861969, "beta_low": 0.01324462890625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.18455870449543, "evidence_weights": [0.07618200033903122, 0.017014145851135254, 0.5945202708244324], "agreement_weight": 0.058086659759283066, "orthogonal_weight": -0.9039149284362793}, {"head": 4, "memory_to_active": 2.016824245452881, "contribution_to_active": 0.06758320331573486, "mix_mean": 0.013560905121266842, "mix_low": 0.684504508972168, "mix_high": 0.0, "read_cosine": 0.2852366864681244, "beta_mean": 0.16816619038581848, "nonfinite": 0.0, "weighted_memory_to_active": 0.0754406601190567, "mix_std": 0.03155538812279701, "beta_std": 0.1366736888885498, "beta_low": 0.0126953125, "beta_high": 0.0, "orthogonal_to_active": 0.004709477536380291, "evidence_weights": [0.12123067677021027, 0.05110737308859825, 0.1502174437046051], "agreement_weight": 0.052473314106464386, "orthogonal_weight": -0.08324231207370758}, {"head": 5, "memory_to_active": 0.7690195441246033, "contribution_to_active": 0.13036538660526276, "mix_mean": 0.09014344960451126, "mix_low": 0.1706380844116211, "mix_high": 1.9073486328125e-06, "read_cosine": 0.2218349575996399, "beta_mean": 0.01267709955573082, "nonfinite": 0.0, "weighted_memory_to_active": 0.13302956521511078, "mix_std": 0.12825100123882294, "beta_std": 0.024329788982868195, "beta_low": 0.65704345703125, "beta_high": 0.0, "orthogonal_to_active": 0.05627947673201561, "evidence_weights": [-0.006180870346724987, -0.04970899596810341, 0.18143555521965027], "agreement_weight": 0.06193006783723831, "orthogonal_weight": -0.5281645655632019}]}, {"layer": 8, "memory_to_active": 0.5738979578018188, "contribution_to_active": 0.5057417750358582, "mix_mean": 0.39983728528022766, "mix_low": 0.09555864334106445, "mix_high": 0.0007599195232614875, "read_cosine": 0.5138674974441528, "beta_mean": 0.03515634685754776, "nonfinite": 0.0, "weighted_memory_to_active": 0.11180046945810318, "mix_std": 0.22349968552589417, "beta_std": 0.03843431919813156, "beta_low": 0.5457662343978882, "beta_high": 0.0, "orthogonal_to_active": 0.05744551867246628, "output_influence": 0.41028597950935364, "residual_influence": 0.12084714323282242, "state_active": 0.05331232026219368, "state_memory": 0.01162749994546175, "state_peak": 34.24839401245117, "growth_active": 1.1054097414016724, "growth_memory": 1.3901855945587158, "heads": [{"head": 0, "memory_to_active": 0.051122672855854034, "contribution_to_active": 0.5247118473052979, "mix_mean": 0.5690510272979736, "mix_low": 0.00028705596923828125, "mix_high": 0.0005550384521484375, "read_cosine": 0.6398532390594482, "beta_mean": 0.003764834487810731, "nonfinite": 0.0, "weighted_memory_to_active": 0.032378412783145905, "mix_std": 0.24034695327281952, "beta_std": 0.008835714310407639, "beta_low": 0.90472412109375, "beta_high": 0.0, "orthogonal_to_active": 0.019383523613214493, "evidence_weights": [-0.17619101703166962, -0.38128262758255005, 0.7267096042633057], "agreement_weight": 0.08125413954257965, "orthogonal_weight": -0.8175891637802124}, {"head": 1, "memory_to_active": 0.2708367705345154, "contribution_to_active": 0.43779999017715454, "mix_mean": 0.43012624979019165, "mix_low": 0.0025053024291992188, "mix_high": 0.00029659271240234375, "read_cosine": 0.4014895260334015, "beta_mean": 0.009245192632079124, "nonfinite": 0.0, "weighted_memory_to_active": 0.12491374462842941, "mix_std": 0.25282302498817444, "beta_std": 0.015576373785734177, "beta_low": 0.72918701171875, "beta_high": 0.0, "orthogonal_to_active": 0.06840439885854721, "evidence_weights": [-0.18797603249549866, -0.23607216775417328, 0.6562559008598328], "agreement_weight": 0.07551398128271103, "orthogonal_weight": -0.6183120608329773}, {"head": 2, "memory_to_active": 0.03421002998948097, "contribution_to_active": 0.6218177676200867, "mix_mean": 0.5855331420898438, "mix_low": 0.0008411407470703125, "mix_high": 0.0026388168334960938, "read_cosine": 0.3736352324485779, "beta_mean": 0.000318383623380214, "nonfinite": 0.0, "weighted_memory_to_active": 0.023745061829686165, "mix_std": 0.24241583049297333, "beta_std": 0.003544197417795658, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.01662655547261238, "evidence_weights": [-0.21572494506835938, -0.3321826159954071, 0.7311316132545471], "agreement_weight": 0.1554742455482483, "orthogonal_weight": -0.7979025840759277}, {"head": 3, "memory_to_active": 0.14442503452301025, "contribution_to_active": 0.5877538323402405, "mix_mean": 0.508517324924469, "mix_low": 0.0010824203491210938, "mix_high": 0.0002498626708984375, "read_cosine": 0.7805029153823853, "beta_mean": 0.014052015729248524, "nonfinite": 0.0, "weighted_memory_to_active": 0.10283367335796356, "mix_std": 0.25321054458618164, "beta_std": 0.012427552603185177, "beta_low": 0.41131591796875, "beta_high": 0.0, "orthogonal_to_active": 0.030909346416592598, "evidence_weights": [-0.16650089621543884, -0.287373423576355, 0.69959956407547], "agreement_weight": 0.1265149563550949, "orthogonal_weight": -0.5934640765190125}, {"head": 4, "memory_to_active": 2.679419994354248, "contribution_to_active": 0.2315273880958557, "mix_mean": 0.025430161505937576, "mix_low": 0.4449491500854492, "mix_high": 0.0, "read_cosine": 0.3399584889411926, "beta_mean": 0.10890232026576996, "nonfinite": 0.0, "weighted_memory_to_active": 0.22232237458229065, "mix_std": 0.04489102214574814, "beta_std": 0.08774708956480026, "beta_low": 0.0150146484375, "beta_high": 0.0, "orthogonal_to_active": 0.008183320052921772, "evidence_weights": [0.04875271022319794, -0.010238945484161377, -0.13896243274211884], "agreement_weight": 0.03314633294939995, "orthogonal_weight": 0.03837155923247337}, {"head": 5, "memory_to_active": 0.26337307691574097, "contribution_to_active": 0.6308398842811584, "mix_mean": 0.28036558628082275, "mix_low": 0.1236867904663086, "mix_high": 0.0008192062377929688, "read_cosine": 0.5477653741836548, "beta_mean": 0.07465532422065735, "nonfinite": 0.0, "weighted_memory_to_active": 0.16460952162742615, "mix_std": 0.30731067061424255, "beta_std": 0.10247497260570526, "beta_low": 0.21478271484375, "beta_high": 0.0, "orthogonal_to_active": 0.20116597414016724, "evidence_weights": [-0.06527315080165863, -0.292864590883255, -0.40625303983688354], "agreement_weight": 0.02194044180214405, "orthogonal_weight": 1.5087813138961792}]}, {"layer": 9, "memory_to_active": 1.133658528327942, "contribution_to_active": 0.6185160875320435, "mix_mean": 0.29356610774993896, "mix_low": 0.15309128165245056, "mix_high": 0.0022597312927246094, "read_cosine": 0.36394739151000977, "beta_mean": 0.08064119517803192, "nonfinite": 0.0, "weighted_memory_to_active": 0.3596758544445038, "mix_std": 0.20835274457931519, "beta_std": 0.048942357301712036, "beta_low": 0.4988810420036316, "beta_high": 0.0, "orthogonal_to_active": 0.03962339833378792, "output_influence": 0.4642223119735718, "residual_influence": 0.12489835917949677, "state_active": 0.04269636422395706, "state_memory": 0.021530548110604286, "state_peak": 37.67622375488281, "growth_active": 1.2748545408248901, "growth_memory": 1.3703479766845703, "heads": [{"head": 0, "memory_to_active": 0.0041551776230335236, "contribution_to_active": 0.5183825492858887, "mix_mean": 0.5829603672027588, "mix_low": 0.0022420883178710938, "mix_high": 0.009530067443847656, "read_cosine": 0.22736522555351257, "beta_mean": 7.72406201576814e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0025066847447305918, "mix_std": 0.29431670904159546, "beta_std": 3.856412513414398e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0021351072937250137, "evidence_weights": [-0.28742876648902893, -0.29365983605384827, 0.9094440340995789], "agreement_weight": 0.08252804726362228, "orthogonal_weight": -0.8940398693084717}, {"head": 1, "memory_to_active": 0.040257200598716736, "contribution_to_active": 0.579334557056427, "mix_mean": 0.44260430335998535, "mix_low": 0.010244369506835938, "mix_high": 0.0009908676147460938, "read_cosine": 0.4036634862422943, "beta_mean": 0.0001113622565753758, "nonfinite": 0.0, "weighted_memory_to_active": 0.024012403562664986, "mix_std": 0.28933173418045044, "beta_std": 0.00024726850097067654, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01611981727182865, "evidence_weights": [-0.21117955446243286, -0.25367626547813416, 1.1234807968139648], "agreement_weight": 0.322950541973114, "orthogonal_weight": -0.7977723479270935}, {"head": 2, "memory_to_active": 0.22288528084754944, "contribution_to_active": 0.5937227606773376, "mix_mean": 0.5153982639312744, "mix_low": 0.0024766921997070312, "mix_high": 0.0027246475219726562, "read_cosine": 0.4175952672958374, "beta_mean": 0.0009623715886846185, "nonfinite": 0.0, "weighted_memory_to_active": 0.14005157351493835, "mix_std": 0.282146155834198, "beta_std": 0.005760756321251392, "beta_low": 0.99151611328125, "beta_high": 0.0, "orthogonal_to_active": 0.06222113221883774, "evidence_weights": [-0.18106232583522797, -0.23473292589187622, 0.5699344277381897], "agreement_weight": 0.24032418429851532, "orthogonal_weight": -0.5447665452957153}, {"head": 3, "memory_to_active": 1.8076072931289673, "contribution_to_active": 0.20840230584144592, "mix_mean": 0.0399288609623909, "mix_low": 0.3828296661376953, "mix_high": 1.239776611328125e-05, "read_cosine": 0.41303718090057373, "beta_mean": 0.10284610837697983, "nonfinite": 0.0, "weighted_memory_to_active": 0.1806274801492691, "mix_std": 0.0874045267701149, "beta_std": 0.07108578830957413, "beta_low": 0.00177001953125, "beta_high": 0.0, "orthogonal_to_active": 0.03275609388947487, "evidence_weights": [0.06783965975046158, 0.03562984615564346, -0.07797771692276001], "agreement_weight": 0.0728045329451561, "orthogonal_weight": 0.2015162855386734}, {"head": 4, "memory_to_active": 3.0875654220581055, "contribution_to_active": 1.2369492053985596, "mix_mean": 0.11904145777225494, "mix_low": 0.16388893127441406, "mix_high": 0.00030040740966796875, "read_cosine": 0.26398390531539917, "beta_mean": 0.22105883061885834, "nonfinite": 0.0, "weighted_memory_to_active": 1.1644418239593506, "mix_std": 0.18017764389514923, "beta_std": 0.11159567534923553, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.014910177327692509, "evidence_weights": [-0.056452032178640366, -0.14874058961868286, 0.016027797013521194], "agreement_weight": -0.047141969203948975, "orthogonal_weight": -0.012949789874255657}, {"head": 5, "memory_to_active": 1.639480471611023, "contribution_to_active": 0.5743048787117004, "mix_mean": 0.06146334111690521, "mix_low": 0.35686588287353516, "mix_high": 0.0, "read_cosine": 0.45803940296173096, "beta_mean": 0.1588607281446457, "nonfinite": 0.0, "weighted_memory_to_active": 0.6464149951934814, "mix_std": 0.1167396679520607, "beta_std": 0.10492609441280365, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.1095980554819107, "evidence_weights": [0.08511064201593399, -0.056174688041210175, 0.14054925739765167], "agreement_weight": 0.01881217770278454, "orthogonal_weight": 0.23939920961856842}]}, {"layer": 10, "memory_to_active": 0.19285526871681213, "contribution_to_active": 0.5212490558624268, "mix_mean": 0.4794044494628906, "mix_low": 0.01201375387609005, "mix_high": 0.008591970428824425, "read_cosine": 0.3785977363586426, "beta_mean": 0.046377889811992645, "nonfinite": 0.0, "weighted_memory_to_active": 0.08816023170948029, "mix_std": 0.26856404542922974, "beta_std": 0.04745199531316757, "beta_low": 0.6539815664291382, "beta_high": 0.0, "orthogonal_to_active": 0.08022981882095337, "output_influence": 0.40545305609703064, "residual_influence": 0.1344272494316101, "state_active": 0.05463245511054993, "state_memory": 0.013751364313066006, "state_peak": 114.20907592773438, "growth_active": 1.0925520658493042, "growth_memory": 0.9628564715385437, "heads": [{"head": 0, "memory_to_active": 0.6199401617050171, "contribution_to_active": 1.0322885513305664, "mix_mean": 0.6193211078643799, "mix_low": 0.03217887878417969, "mix_high": 0.04033184051513672, "read_cosine": 0.12364818155765533, "beta_mean": 0.20498256385326385, "nonfinite": 0.0, "weighted_memory_to_active": 0.39249518513679504, "mix_std": 0.37713783979415894, "beta_std": 0.21602453291416168, "beta_low": 0.11773681640625, "beta_high": 0.0, "orthogonal_to_active": 0.39812859892845154, "evidence_weights": [-0.14632415771484375, -0.37202635407447815, -0.6537991166114807], "agreement_weight": 0.02358650416135788, "orthogonal_weight": 1.040444016456604}, {"head": 1, "memory_to_active": 0.017089292407035828, "contribution_to_active": 0.5669258236885071, "mix_mean": 0.5809556245803833, "mix_low": 0.0006437301635742188, "mix_high": 0.008149147033691406, "read_cosine": 0.2766798138618469, "beta_mean": 0.00010259165719617158, "nonfinite": 0.0, "weighted_memory_to_active": 0.009594349190592766, "mix_std": 0.2660187780857086, "beta_std": 0.00013467128155753016, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012141085229814053, "evidence_weights": [-0.27310600876808167, -0.20049795508384705, 0.5880346298217773], "agreement_weight": 0.41970276832580566, "orthogonal_weight": -1.3862946033477783}, {"head": 2, "memory_to_active": 0.031159961596131325, "contribution_to_active": 0.5208384990692139, "mix_mean": 0.5232002139091492, "mix_low": 0.0005941390991210938, "mix_high": 0.0025320053100585938, "read_cosine": 0.028580548241734505, "beta_mean": 0.0001016885507851839, "nonfinite": 0.0, "weighted_memory_to_active": 0.011726902797818184, "mix_std": 0.26116278767585754, "beta_std": 0.00039135952829383314, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.004613216035068035, "evidence_weights": [-0.2046205997467041, -0.24814891815185547, 0.5284805297851562], "agreement_weight": -0.08451209217309952, "orthogonal_weight": -0.38509830832481384}, {"head": 3, "memory_to_active": 0.3284880518913269, "contribution_to_active": 0.10017699748277664, "mix_mean": 0.27853745222091675, "mix_low": 0.03468894958496094, "mix_high": 0.00039768218994140625, "read_cosine": 0.9068424701690674, "beta_mean": 0.06605672836303711, "nonfinite": 0.0, "weighted_memory_to_active": 0.038394711911678314, "mix_std": 0.23164524137973785, "beta_std": 0.058245155960321426, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.04479772597551346, "evidence_weights": [-0.12171565741300583, -0.14841963350772858, 0.15144550800323486], "agreement_weight": -0.00021745619596913457, "orthogonal_weight": 1.4888466596603394}, {"head": 4, "memory_to_active": 0.006853384431451559, "contribution_to_active": 0.49695974588394165, "mix_mean": 0.49824702739715576, "mix_low": 0.0015439987182617188, "mix_high": 0.00013065338134765625, "read_cosine": 0.07793940603733063, "beta_mean": 2.4958892026916146e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002225908450782299, "mix_std": 0.25200894474983215, "beta_std": 0.0009751884499564767, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0010253367945551872, "evidence_weights": [-0.18085584044456482, -0.2432435005903244, 0.777906596660614], "agreement_weight": 0.08570893853902817, "orthogonal_weight": -0.462819367647171}, {"head": 5, "memory_to_active": 0.1536007523536682, "contribution_to_active": 0.410304456949234, "mix_mean": 0.3761652708053589, "mix_low": 0.0024328231811523438, "mix_high": 1.049041748046875e-05, "read_cosine": 0.8578959703445435, "beta_mean": 0.006998775992542505, "nonfinite": 0.0, "weighted_memory_to_active": 0.07452432066202164, "mix_std": 0.22341063618659973, "beta_std": 0.008941080421209335, "beta_low": 0.80548095703125, "beta_high": 0.0, "orthogonal_to_active": 0.020672935992479324, "evidence_weights": [-0.14955714344978333, -0.16131512820720673, 0.297164648771286], "agreement_weight": 0.09246014803647995, "orthogonal_weight": -0.6025922894477844}]}, {"layer": 11, "memory_to_active": 0.1589820235967636, "contribution_to_active": 0.5605226159095764, "mix_mean": 0.4449261426925659, "mix_low": 0.06871986389160156, "mix_high": 0.04236428067088127, "read_cosine": 0.44719740748405457, "beta_mean": 0.03580785170197487, "nonfinite": 0.0, "weighted_memory_to_active": 0.06745756417512894, "mix_std": 0.3184656500816345, "beta_std": 0.05136196315288544, "beta_low": 0.5402120351791382, "beta_high": 0.0, "orthogonal_to_active": 0.051715392619371414, "output_influence": 0.4928293228149414, "residual_influence": 0.12292566150426865, "state_active": 0.16963325440883636, "state_memory": 0.00806548073887825, "state_peak": 22.860504150390625, "growth_active": 1.087968111038208, "growth_memory": 1.5074487924575806, "heads": [{"head": 0, "memory_to_active": 0.007025387138128281, "contribution_to_active": 0.562395453453064, "mix_mean": 0.5963923931121826, "mix_low": 0.0012311935424804688, "mix_high": 0.007228851318359375, "read_cosine": 0.3878304064273834, "beta_mean": 9.91966953733936e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004292735364288092, "mix_std": 0.2688794434070587, "beta_std": 0.0038886605761945248, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.002841456327587366, "evidence_weights": [-0.16382691264152527, -0.356341689825058, 0.642900824546814], "agreement_weight": 0.21367022395133972, "orthogonal_weight": -0.755845844745636}, {"head": 1, "memory_to_active": 0.36282050609588623, "contribution_to_active": 0.26384449005126953, "mix_mean": 0.3044619560241699, "mix_low": 0.0186614990234375, "mix_high": 0.00015544891357421875, "read_cosine": 0.620313286781311, "beta_mean": 0.03379976749420166, "nonfinite": 0.0, "weighted_memory_to_active": 0.1429070234298706, "mix_std": 0.2506997585296631, "beta_std": 0.06688369065523148, "beta_low": 0.41815185546875, "beta_high": 0.0, "orthogonal_to_active": 0.032111458480358124, "evidence_weights": [-0.15780718624591827, -0.14300450682640076, 0.2994545102119446], "agreement_weight": 0.08122726529836655, "orthogonal_weight": -0.31163060665130615}, {"head": 2, "memory_to_active": 0.020968038588762283, "contribution_to_active": 0.9470785856246948, "mix_mean": 0.46776342391967773, "mix_low": 0.1361408233642578, "mix_high": 0.13602638244628906, "read_cosine": 0.24984608590602875, "beta_mean": 0.07587852329015732, "nonfinite": 0.0, "weighted_memory_to_active": 0.01647544652223587, "mix_std": 0.4281911849975586, "beta_std": 0.10085983574390411, "beta_low": 0.4412841796875, "beta_high": 0.0, "orthogonal_to_active": 0.0718795657157898, "evidence_weights": [-0.006714276969432831, -0.4572445750236511, -1.2348885536193848], "agreement_weight": -0.0017791815334931016, "orthogonal_weight": 4.603165626525879}, {"head": 3, "memory_to_active": 0.013069269247353077, "contribution_to_active": 0.8571572303771973, "mix_mean": 0.4215558171272278, "mix_low": 0.2406330108642578, "mix_high": 0.10825538635253906, "read_cosine": 0.5165457129478455, "beta_mean": 0.06434530764818192, "nonfinite": 0.0, "weighted_memory_to_active": 0.007892074063420296, "mix_std": 0.4365139603614807, "beta_std": 0.09428580105304718, "beta_low": 0.4739990234375, "beta_high": 0.0, "orthogonal_to_active": 0.030311711132526398, "evidence_weights": [0.05279236286878586, -0.3567860722541809, -0.44773757457733154], "agreement_weight": 0.04016631841659546, "orthogonal_weight": 4.92830753326416}, {"head": 4, "memory_to_active": 0.45256587862968445, "contribution_to_active": 0.3750224709510803, "mix_mean": 0.40115994215011597, "mix_low": 0.014698028564453125, "mix_high": 0.00032711029052734375, "read_cosine": 0.15667876601219177, "beta_mean": 0.031627457588911057, "nonfinite": 0.0, "weighted_memory_to_active": 0.1891031116247177, "mix_std": 0.26613378524780273, "beta_std": 0.02944883331656456, "beta_low": 0.19390869140625, "beta_high": 0.0, "orthogonal_to_active": 0.1517152488231659, "evidence_weights": [-0.19516345858573914, -0.16217301785945892, 1.6463350057601929], "agreement_weight": -0.16421456634998322, "orthogonal_weight": -0.8264164328575134}, {"head": 5, "memory_to_active": 0.09744303673505783, "contribution_to_active": 0.3576371967792511, "mix_mean": 0.47822335362434387, "mix_low": 0.0009546279907226562, "mix_high": 0.0021924972534179688, "read_cosine": 0.7519700527191162, "beta_mean": 0.009096836671233177, "nonfinite": 0.0, "weighted_memory_to_active": 0.04407498985528946, "mix_std": 0.2603757381439209, "beta_std": 0.012804925441741943, "beta_low": 0.7142333984375, "beta_high": 0.0, "orthogonal_to_active": 0.02143291011452675, "evidence_weights": [-0.20721173286437988, -0.26612645387649536, 0.5290786623954773], "agreement_weight": 0.06548110395669937, "orthogonal_weight": -0.6958053708076477}]}, {"layer": 12, "memory_to_active": 0.33422937989234924, "contribution_to_active": 0.5544758439064026, "mix_mean": 0.4686412215232849, "mix_low": 0.01463572308421135, "mix_high": 0.0035805704537779093, "read_cosine": 0.25213003158569336, "beta_mean": 0.01559028122574091, "nonfinite": 0.0, "weighted_memory_to_active": 0.17590467631816864, "mix_std": 0.2571554183959961, "beta_std": 0.01633521541953087, "beta_low": 0.8175048828125, "beta_high": 0.0, "orthogonal_to_active": 0.0670335590839386, "output_influence": 0.42531687021255493, "residual_influence": 0.11573745310306549, "state_active": 0.03903597593307495, "state_memory": 0.009848056361079216, "state_peak": 33.64789962768555, "growth_active": 1.130411148071289, "growth_memory": 1.2132697105407715, "heads": [{"head": 0, "memory_to_active": 0.07305005937814713, "contribution_to_active": 0.46368008852005005, "mix_mean": 0.4404020309448242, "mix_low": 0.0024595260620117188, "mix_high": 0.0007953643798828125, "read_cosine": 0.2686217427253723, "beta_mean": 0.001083827461116016, "nonfinite": 0.0, "weighted_memory_to_active": 0.03341345116496086, "mix_std": 0.2689988911151886, "beta_std": 0.004678011871874332, "beta_low": 0.99072265625, "beta_high": 0.0, "orthogonal_to_active": 0.020450837910175323, "evidence_weights": [-0.17298594117164612, -0.24724169075489044, 0.2997416853904724], "agreement_weight": 0.1125485897064209, "orthogonal_weight": -0.6429566740989685}, {"head": 1, "memory_to_active": 0.10744349658489227, "contribution_to_active": 0.493789404630661, "mix_mean": 0.5049018859863281, "mix_low": 0.0007619857788085938, "mix_high": 0.00038051605224609375, "read_cosine": 0.5527838468551636, "beta_mean": 0.0014351888094097376, "nonfinite": 0.0, "weighted_memory_to_active": 0.05384230986237526, "mix_std": 0.2626988887786865, "beta_std": 0.0060625202022492886, "beta_low": 0.977783203125, "beta_high": 0.0, "orthogonal_to_active": 0.041142888367176056, "evidence_weights": [-0.2025855928659439, -0.2668670415878296, 0.8906395435333252], "agreement_weight": 0.11594000458717346, "orthogonal_weight": -0.92023766040802}, {"head": 2, "memory_to_active": 0.6380394697189331, "contribution_to_active": 0.4262543320655823, "mix_mean": 0.30046629905700684, "mix_low": 0.0303497314453125, "mix_high": 0.0002040863037109375, "read_cosine": 0.4517538249492645, "beta_mean": 0.08774586021900177, "nonfinite": 0.0, "weighted_memory_to_active": 0.22713226079940796, "mix_std": 0.23900926113128662, "beta_std": 0.0641593337059021, "beta_low": 0.00115966796875, "beta_high": 0.0, "orthogonal_to_active": 0.19736170768737793, "evidence_weights": [-0.2413637787103653, -0.0404646210372448, -0.09415265172719955], "agreement_weight": 0.019095849245786667, "orthogonal_weight": 1.0464218854904175}, {"head": 3, "memory_to_active": 0.02675517089664936, "contribution_to_active": 0.728630542755127, "mix_mean": 0.6863417625427246, "mix_low": 0.00026035308837890625, "mix_high": 0.015977859497070312, "read_cosine": 0.042428258806467056, "beta_mean": 0.00025414564879611135, "nonfinite": 0.0, "weighted_memory_to_active": 0.01955006644129753, "mix_std": 0.24261796474456787, "beta_std": 0.00401788717135787, "beta_low": 0.99884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.018300876021385193, "evidence_weights": [-0.2972840964794159, -0.3348117470741272, 1.2257040739059448], "agreement_weight": 0.30099791288375854, "orthogonal_weight": -0.9547428488731384}, {"head": 4, "memory_to_active": 1.0794552564620972, "contribution_to_active": 0.7461885213851929, "mix_mean": 0.2750266194343567, "mix_low": 0.047949790954589844, "mix_high": 0.00162506103515625, "read_cosine": -0.09603171795606613, "beta_mean": 0.0024049642961472273, "nonfinite": 0.0, "weighted_memory_to_active": 0.6704397797584534, "mix_std": 0.25614577531814575, "beta_std": 0.015148203819990158, "beta_low": 0.94818115234375, "beta_high": 0.0, "orthogonal_to_active": 0.0858091488480568, "evidence_weights": [-0.06875521689653397, -0.19950474798679352, 0.03824162855744362], "agreement_weight": -0.004888721276074648, "orthogonal_weight": -0.13781817257404327}, {"head": 5, "memory_to_active": 0.080632783472538, "contribution_to_active": 0.46831172704696655, "mix_mean": 0.6047086119651794, "mix_low": 0.0060329437255859375, "mix_high": 0.0025005340576171875, "read_cosine": 0.293224036693573, "beta_mean": 0.0006176920142024755, "nonfinite": 0.0, "weighted_memory_to_active": 0.05105007439851761, "mix_std": 0.27346184849739075, "beta_std": 0.003945338074117899, "beta_low": 0.98834228515625, "beta_high": 0.0, "orthogonal_to_active": 0.03913591057062149, "evidence_weights": [-0.26067498326301575, -0.30319085717201233, 1.0536158084869385], "agreement_weight": 0.2180357426404953, "orthogonal_weight": -0.8409485816955566}]}, {"layer": 13, "memory_to_active": 0.1419025957584381, "contribution_to_active": 0.4975413978099823, "mix_mean": 0.48960691690444946, "mix_low": 0.006000360008329153, "mix_high": 0.003071943996474147, "read_cosine": 0.415973424911499, "beta_mean": 0.017547793686389923, "nonfinite": 0.0, "weighted_memory_to_active": 0.05933220311999321, "mix_std": 0.25613999366760254, "beta_std": 0.01852065697312355, "beta_low": 0.7913818359375, "beta_high": 0.0, "orthogonal_to_active": 0.05628718435764313, "output_influence": 0.44457247853279114, "residual_influence": 0.14627519249916077, "state_active": 0.09875541925430298, "state_memory": 0.03790571913123131, "state_peak": 96.69913482666016, "growth_active": 0.9896083474159241, "growth_memory": 1.0308119058609009, "heads": [{"head": 0, "memory_to_active": 0.03216034173965454, "contribution_to_active": 0.5416035056114197, "mix_mean": 0.5388245582580566, "mix_low": 0.00054931640625, "mix_high": 0.0011472702026367188, "read_cosine": 0.41880133748054504, "beta_mean": 0.00010771463712444529, "nonfinite": 0.0, "weighted_memory_to_active": 0.018669631332159042, "mix_std": 0.26522573828697205, "beta_std": 0.0002112420479534194, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.017656266689300537, "evidence_weights": [-0.20210811495780945, -0.2580031454563141, 0.732621431350708], "agreement_weight": 0.07701834291219711, "orthogonal_weight": -1.0665632486343384}, {"head": 1, "memory_to_active": 0.02456538751721382, "contribution_to_active": 0.5606470704078674, "mix_mean": 0.5086298584938049, "mix_low": 0.0021228790283203125, "mix_high": 0.000865936279296875, "read_cosine": 0.23768344521522522, "beta_mean": 7.711255602771416e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01343484316021204, "mix_std": 0.26842451095581055, "beta_std": 0.0001191340561490506, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014089465141296387, "evidence_weights": [-0.18542321026325226, -0.2672833800315857, 0.6721588969230652], "agreement_weight": 0.24533917009830475, "orthogonal_weight": -1.1047396659851074}, {"head": 2, "memory_to_active": 0.13943468034267426, "contribution_to_active": 0.412204384803772, "mix_mean": 0.35366398096084595, "mix_low": 0.00286102294921875, "mix_high": 8.96453857421875e-05, "read_cosine": 0.6607046127319336, "beta_mean": 0.01574414037168026, "nonfinite": 0.0, "weighted_memory_to_active": 0.06031332537531853, "mix_std": 0.2201714813709259, "beta_std": 0.044217176735401154, "beta_low": 0.76275634765625, "beta_high": 0.0, "orthogonal_to_active": 0.029182156547904015, "evidence_weights": [-0.21354559063911438, -0.17423953115940094, 0.45348963141441345], "agreement_weight": 0.021151592954993248, "orthogonal_weight": -0.7890916466712952}, {"head": 3, "memory_to_active": 0.13406848907470703, "contribution_to_active": 0.5706982016563416, "mix_mean": 0.6094759702682495, "mix_low": 0.015180587768554688, "mix_high": 0.012128829956054688, "read_cosine": 0.004985364153981209, "beta_mean": 0.0011142758885398507, "nonfinite": 0.0, "weighted_memory_to_active": 0.09727069735527039, "mix_std": 0.28128984570503235, "beta_std": 0.0052217161282896996, "beta_low": 0.9892578125, "beta_high": 0.0, "orthogonal_to_active": 0.09851429611444473, "evidence_weights": [-0.3367483615875244, -0.23476849496364594, 2.0171141624450684], "agreement_weight": 2.9728341102600098, "orthogonal_weight": -1.061314344406128}, {"head": 4, "memory_to_active": 0.43513137102127075, "contribution_to_active": 0.25298011302948, "mix_mean": 0.34463948011398315, "mix_low": 0.012569427490234375, "mix_high": 1.33514404296875e-05, "read_cosine": 0.8907214403152466, "beta_mean": 0.08765743672847748, "nonfinite": 0.0, "weighted_memory_to_active": 0.11346390843391418, "mix_std": 0.22086961567401886, "beta_std": 0.05720803141593933, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.136656254529953, "evidence_weights": [-0.15564236044883728, -0.23074747622013092, 0.496547669172287], "agreement_weight": -0.004014522768557072, "orthogonal_weight": 1.9445074796676636}, {"head": 5, "memory_to_active": 0.08605528622865677, "contribution_to_active": 0.6471149921417236, "mix_mean": 0.5824076533317566, "mix_low": 0.0027189254760742188, "mix_high": 0.0041866302490234375, "read_cosine": 0.2829442024230957, "beta_mean": 0.0005860814126208425, "nonfinite": 0.0, "weighted_memory_to_active": 0.05284081771969795, "mix_std": 0.2808586359024048, "beta_std": 0.004146622959524393, "beta_low": 0.99615478515625, "beta_high": 0.0, "orthogonal_to_active": 0.041624657809734344, "evidence_weights": [-0.21778570115566254, -0.28799888491630554, 0.6593095660209656], "agreement_weight": 0.08496130257844925, "orthogonal_weight": -0.8763796091079712}]}, {"layer": 14, "memory_to_active": 0.26005011796951294, "contribution_to_active": 0.4286840856075287, "mix_mean": 0.41637492179870605, "mix_low": 0.026140054687857628, "mix_high": 0.004594485275447369, "read_cosine": 0.628513514995575, "beta_mean": 0.06709429621696472, "nonfinite": 0.0, "weighted_memory_to_active": 0.11869318038225174, "mix_std": 0.2469223290681839, "beta_std": 0.06306406855583191, "beta_low": 0.4449869990348816, "beta_high": 0.0, "orthogonal_to_active": 0.04248423129320145, "output_influence": 0.3063718378543854, "residual_influence": 0.11831852048635483, "state_active": 0.06663662940263748, "state_memory": 0.01585497334599495, "state_peak": 29.664100646972656, "growth_active": 1.139081597328186, "growth_memory": 1.2847925424575806, "heads": [{"head": 0, "memory_to_active": 0.3443542718887329, "contribution_to_active": 0.20863457024097443, "mix_mean": 0.2135336846113205, "mix_low": 0.014490127563476562, "mix_high": 1.52587890625e-05, "read_cosine": 0.7566767930984497, "beta_mean": 0.06658458709716797, "nonfinite": 0.0, "weighted_memory_to_active": 0.09433731436729431, "mix_std": 0.19367705285549164, "beta_std": 0.06342411041259766, "beta_low": 0.0159912109375, "beta_high": 0.0, "orthogonal_to_active": 0.05401045083999634, "evidence_weights": [-0.11908753216266632, -0.04788526892662048, 0.6377403736114502], "agreement_weight": 0.029769854620099068, "orthogonal_weight": 0.9151615500450134}, {"head": 1, "memory_to_active": 0.5056056976318359, "contribution_to_active": 0.3514665961265564, "mix_mean": 0.24161188304424286, "mix_low": 0.12420082092285156, "mix_high": 4.57763671875e-05, "read_cosine": 0.7546257376670837, "beta_mean": 0.2693388760089874, "nonfinite": 0.0, "weighted_memory_to_active": 0.23659837245941162, "mix_std": 0.26397332549095154, "beta_std": 0.25437891483306885, "beta_low": 0.10638427734375, "beta_high": 0.0, "orthogonal_to_active": 0.01769324205815792, "evidence_weights": [0.002873447258025408, -0.25848597288131714, -0.17832845449447632], "agreement_weight": 0.05344195291399956, "orthogonal_weight": 0.15622223913669586}, {"head": 2, "memory_to_active": 0.1723921298980713, "contribution_to_active": 0.5442752242088318, "mix_mean": 0.5549748539924622, "mix_low": 0.0031538009643554688, "mix_high": 0.0019664764404296875, "read_cosine": 0.41297656297683716, "beta_mean": 0.00384413730353117, "nonfinite": 0.0, "weighted_memory_to_active": 0.10644984245300293, "mix_std": 0.26926061511039734, "beta_std": 0.005188134964555502, "beta_low": 0.92388916015625, "beta_high": 0.0, "orthogonal_to_active": 0.07947764545679092, "evidence_weights": [-0.16590264439582825, -0.29312849044799805, 0.8623844385147095], "agreement_weight": 0.07613914459943771, "orthogonal_weight": -0.8523957133293152}, {"head": 3, "memory_to_active": 0.06779162585735321, "contribution_to_active": 0.6730344891548157, "mix_mean": 0.6790775060653687, "mix_low": 0.004643440246582031, "mix_high": 0.01635456085205078, "read_cosine": 0.2910453677177429, "beta_mean": 0.009514953009784222, "nonfinite": 0.0, "weighted_memory_to_active": 0.047569673508405685, "mix_std": 0.24454151093959808, "beta_std": 0.008798202499747276, "beta_low": 0.65887451171875, "beta_high": 0.0, "orthogonal_to_active": 0.027289338409900665, "evidence_weights": [-0.38123592734336853, -0.3288867771625519, 0.43748316168785095], "agreement_weight": 0.08112701028585434, "orthogonal_weight": -0.6999906897544861}, {"head": 4, "memory_to_active": 0.18052254617214203, "contribution_to_active": 0.4800024926662445, "mix_mean": 0.4847957491874695, "mix_low": 0.003912925720214844, "mix_high": 0.008502960205078125, "read_cosine": 0.8310989141464233, "beta_mean": 0.05064576119184494, "nonfinite": 0.0, "weighted_memory_to_active": 0.104890838265419, "mix_std": 0.26444852352142334, "beta_std": 0.03495414927601814, "beta_low": 0.01806640625, "beta_high": 0.0, "orthogonal_to_active": 0.029901016503572464, "evidence_weights": [-0.21982772648334503, -0.2633490264415741, 0.4169774055480957], "agreement_weight": 0.1235521137714386, "orthogonal_weight": -0.54625403881073}, {"head": 5, "memory_to_active": 0.28963446617126465, "contribution_to_active": 0.3146909475326538, "mix_mean": 0.32425570487976074, "mix_low": 0.006439208984375, "mix_high": 0.0006818771362304688, "read_cosine": 0.7246576547622681, "beta_mean": 0.002637456636875868, "nonfinite": 0.0, "weighted_memory_to_active": 0.1223130077123642, "mix_std": 0.24563293159008026, "beta_std": 0.01164089422672987, "beta_low": 0.94671630859375, "beta_high": 0.0, "orthogonal_to_active": 0.04653368890285492, "evidence_weights": [-0.18241028487682343, -0.15448209643363953, 0.2108014076948166], "agreement_weight": 0.08696863055229187, "orthogonal_weight": -0.6256874799728394}]}, {"layer": 15, "memory_to_active": 0.06355069577693939, "contribution_to_active": 0.48190462589263916, "mix_mean": 0.39816275238990784, "mix_low": 0.008291562087833881, "mix_high": 0.00230725621804595, "read_cosine": 0.6803544759750366, "beta_mean": 0.04029535502195358, "nonfinite": 0.0, "weighted_memory_to_active": 0.02910170890390873, "mix_std": 0.2816861569881439, "beta_std": 0.045132409781217575, "beta_low": 0.7127279043197632, "beta_high": 0.0, "orthogonal_to_active": 0.006450345274060965, "output_influence": 0.30432066321372986, "residual_influence": 0.1614193618297577, "state_active": 0.12096361070871353, "state_memory": 0.008813770487904549, "state_peak": 40.63454055786133, "growth_active": 1.2995591163635254, "growth_memory": 1.8094600439071655, "heads": [{"head": 0, "memory_to_active": 0.09800440073013306, "contribution_to_active": 0.481680303812027, "mix_mean": 0.36273738741874695, "mix_low": 0.007073402404785156, "mix_high": 0.0037832260131835938, "read_cosine": 0.6991417407989502, "beta_mean": 0.03480929508805275, "nonfinite": 0.0, "weighted_memory_to_active": 0.044666316360235214, "mix_std": 0.2740574777126312, "beta_std": 0.02513851597905159, "beta_low": 0.13433837890625, "beta_high": 0.0, "orthogonal_to_active": 0.017433222383260727, "evidence_weights": [-0.16522809863090515, -0.18310768902301788, 0.34258463978767395], "agreement_weight": 0.08826646208763123, "orthogonal_weight": -0.642787754535675}, {"head": 1, "memory_to_active": 0.14434728026390076, "contribution_to_active": 0.41128599643707275, "mix_mean": 0.40299344062805176, "mix_low": 0.004065513610839844, "mix_high": 0.003131866455078125, "read_cosine": 0.8837859034538269, "beta_mean": 0.0012392583303153515, "nonfinite": 0.0, "weighted_memory_to_active": 0.05588463693857193, "mix_std": 0.2801540493965149, "beta_std": 0.0033087884075939655, "beta_low": 0.97869873046875, "beta_high": 0.0, "orthogonal_to_active": 0.0017240772722288966, "evidence_weights": [-0.17528200149536133, -0.18310901522636414, 0.263306200504303], "agreement_weight": 0.09842661768198013, "orthogonal_weight": -0.049115996807813644}, {"head": 2, "memory_to_active": 0.008566983044147491, "contribution_to_active": 0.4687090516090393, "mix_mean": 0.3431214690208435, "mix_low": 0.014628410339355469, "mix_high": 4.9591064453125e-05, "read_cosine": 0.7968093156814575, "beta_mean": 0.00012981198960915208, "nonfinite": 0.0, "weighted_memory_to_active": 0.0039529819041490555, "mix_std": 0.2717420160770416, "beta_std": 0.00026197434635832906, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00026955106295645237, "evidence_weights": [-0.207144632935524, -0.2785557508468628, 0.2897721827030182], "agreement_weight": 0.011867168359458447, "orthogonal_weight": 0.14173874258995056}, {"head": 3, "memory_to_active": 0.02153998799622059, "contribution_to_active": 0.4364532232284546, "mix_mean": 0.3458769917488098, "mix_low": 0.007067680358886719, "mix_high": 0.00011730194091796875, "read_cosine": 0.6289581060409546, "beta_mean": 0.0002372157759964466, "nonfinite": 0.0, "weighted_memory_to_active": 0.010115986689925194, "mix_std": 0.27584099769592285, "beta_std": 0.0008499964606016874, "beta_low": 0.99847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.00687370216473937, "evidence_weights": [-0.2054084837436676, -0.252879798412323, 0.38442713022232056], "agreement_weight": 0.02236790582537651, "orthogonal_weight": -0.9407041668891907}, {"head": 4, "memory_to_active": 0.09289897233247757, "contribution_to_active": 0.5973529815673828, "mix_mean": 0.39285576343536377, "mix_low": 0.007864952087402344, "mix_high": 0.0006542205810546875, "read_cosine": 0.6924846768379211, "beta_mean": 0.2044345885515213, "nonfinite": 0.0, "weighted_memory_to_active": 0.049875520169734955, "mix_std": 0.3075826168060303, "beta_std": 0.2346574366092682, "beta_low": 0.18060302734375, "beta_high": 0.0, "orthogonal_to_active": 0.004985818639397621, "evidence_weights": [-0.227670818567276, -0.25972938537597656, 0.050718240439891815], "agreement_weight": 0.012465380132198334, "orthogonal_weight": 0.1743168979883194}, {"head": 5, "memory_to_active": 0.01594654470682144, "contribution_to_active": 0.4959460496902466, "mix_mean": 0.5413914918899536, "mix_low": 0.009049415588378906, "mix_high": 0.006107330322265625, "read_cosine": 0.380947083234787, "beta_mean": 0.0009219592320732772, "nonfinite": 0.0, "weighted_memory_to_active": 0.010114815086126328, "mix_std": 0.2807397246360779, "beta_std": 0.006577746942639351, "beta_low": 0.98431396484375, "beta_high": 0.0, "orthogonal_to_active": 0.00741569884121418, "evidence_weights": [-0.3422059416770935, -0.3065536618232727, 0.6068933606147766], "agreement_weight": 0.2623794972896576, "orthogonal_weight": 0.885924756526947}]}]}, {"step": 28000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.892762743998901, "memory_to_active": 0.9048328399658203, "contribution_to_active": 0.5093043446540833, "mix_mean": 0.35814550518989563, "mix_low": 0.1555132120847702, "mix_high": 0.007685413584113121, "read_cosine": 0.38245585560798645, "beta_mean": 0.08343949913978577, "nonfinite": 0.0, "weighted_memory_to_active": 0.21357864141464233, "mix_std": 0.21587255597114563, "beta_std": 0.06908015161752701, "beta_low": 0.4953378438949585, "beta_high": 1.7801919966586865e-05, "orthogonal_to_active": 0.05042191594839096, "samples": 4, "scored_tokens": 899, "normal_loss": 3.14886212348938, "normal_token_accuracy": 0.42269188165664673, "state_active": 0.05982435122132301, "state_memory": 0.019584661349654198, "state_peak": 112.4611587524414, "growth_active": 1.1231098175048828, "growth_memory": 1.3925824165344238, "output_influence": 0.360779732465744, "residual_influence": 0.11506930738687515, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.245764970779419, "no_memory_loss_delta": 0.3413738012313843, "scrambled_loss_delta": 0.17238754034042358, "memory_only_loss_delta": 2.547311782836914, "no_evidence_loss_delta": 0.20914506912231445, "no_agreement_loss_delta": 0.01883896067738533, "no_orthogonal_loss_delta": 0.015335330739617348}, {"index": 94, "tokens": 256, "normal_loss": 3.2035276889801025, "no_memory_loss_delta": 0.4216369092464447, "scrambled_loss_delta": 0.38831454515457153, "memory_only_loss_delta": 2.54154109954834, "no_evidence_loss_delta": 0.20231495797634125, "no_agreement_loss_delta": 0.0401180163025856, "no_orthogonal_loss_delta": 0.03372960165143013}, {"index": 188, "tokens": 256, "normal_loss": 3.5449535846710205, "no_memory_loss_delta": 0.394355446100235, "scrambled_loss_delta": 0.1411912590265274, "memory_only_loss_delta": 2.299086093902588, "no_evidence_loss_delta": 0.24560993909835815, "no_agreement_loss_delta": 0.018474552780389786, "no_orthogonal_loss_delta": 0.03596039488911629}, {"index": 283, "tokens": 131, "normal_loss": 2.0786290168762207, "no_memory_loss_delta": 0.19522637128829956, "scrambled_loss_delta": 0.12251997739076614, "memory_only_loss_delta": 2.725653886795044, "no_evidence_loss_delta": 0.07427671551704407, "no_agreement_loss_delta": 0.009080620482563972, "no_orthogonal_loss_delta": -0.015657106414437294}], "agreement_weight_abs_mean": 0.13616590201854706, "agreement_weight_abs_max": 3.1108508110046387, "agreement_weight_mean": 0.11680402606725693, "orthogonal_weight_abs_mean": 0.6420764327049255, "orthogonal_weight_abs_max": 5.099434852600098, "orthogonal_weight_mean": -0.15939216315746307, "no_memory_loss": 3.506882905960083, "no_memory_loss_delta": 0.35802045464515686, "no_memory_token_accuracy_gain": 0.05672970041632652, "no_memory_token_win_fraction": 0.652947723865509, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 30.094122413891984, "scrambled_loss": 3.3665874004364014, "scrambled_loss_delta": 0.21772505342960358, "scrambled_token_accuracy_gain": 0.016685206443071365, "scrambled_token_win_fraction": 0.6206896305084229, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.565343640412056, "memory_only_loss": 5.6498332023620605, "memory_only_loss_delta": 2.5009710788726807, "memory_only_token_accuracy_gain": 0.23692992329597473, "memory_only_token_win_fraction": 0.8943270444869995, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 91.79946736937158, "no_evidence_loss": 3.3467934131622314, "no_evidence_loss_delta": 0.1979312300682068, "no_evidence_token_accuracy_gain": 0.02892102301120758, "no_evidence_token_win_fraction": 0.6084538102149963, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 17.957372814339003, "no_agreement_loss": 3.172234535217285, "no_agreement_loss_delta": 0.023372674360871315, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.486095666885376, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.310164904257002, "no_orthogonal_loss": 3.170793056488037, "no_orthogonal_loss_delta": 0.02193037047982216, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.6095662117004395, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.1691648179163208, "short_no_memory_loss_delta": 0.33384042978286743, "long_context_memory_gain": 0.02418002486228943, "normal_context_loss_gain": 0.1318909376859665, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.2934911251068115, "contribution_to_active": 1.032621145248413, "mix_mean": 0.09958112239837646, "mix_low": 0.5357481837272644, "mix_high": 0.004263401031494141, "read_cosine": 0.44821614027023315, "beta_mean": 0.25991660356521606, "nonfinite": 0.0, "weighted_memory_to_active": 1.1085419654846191, "mix_std": 0.08734025061130524, "beta_std": 0.1514202207326889, "beta_low": 0.0127970390021801, "beta_high": 0.0, "orthogonal_to_active": 0.06788890063762665, "output_influence": 0.1659187376499176, "residual_influence": 0.1975892186164856, "state_active": 0.07304425537586212, "state_memory": 0.047998908907175064, "state_peak": 42.9404411315918, "growth_active": 1.2604423761367798, "growth_memory": 1.248382806777954, "heads": [{"head": 0, "memory_to_active": 2.5981369018554688, "contribution_to_active": 0.47248291969299316, "mix_mean": 0.019576285034418106, "mix_low": 0.6102275848388672, "mix_high": 0.0, "read_cosine": 0.2565780282020569, "beta_mean": 0.21998362243175507, "nonfinite": 0.0, "weighted_memory_to_active": 0.5718613862991333, "mix_std": 0.05552884191274643, "beta_std": 0.10689274221658707, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.044650446623563766, "evidence_weights": [-0.03392898291349411, -0.0811610072851181, -0.10479692369699478], "agreement_weight": -0.09900949895381927, "orthogonal_weight": -0.08921892195940018}, {"head": 1, "memory_to_active": 4.913371562957764, "contribution_to_active": 1.9110991954803467, "mix_mean": 0.01581752672791481, "mix_low": 0.6626787185668945, "mix_high": 0.0, "read_cosine": 0.5151859521865845, "beta_mean": 0.4619652330875397, "nonfinite": 0.0, "weighted_memory_to_active": 2.3222274780273438, "mix_std": 0.05106199160218239, "beta_std": 0.18281416594982147, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16495074331760406, "evidence_weights": [0.025942904874682426, -0.08129299432039261, 0.3011522889137268], "agreement_weight": 0.02478703111410141, "orthogonal_weight": -0.1555185317993164}, {"head": 2, "memory_to_active": 2.7936108112335205, "contribution_to_active": 0.5760191082954407, "mix_mean": 0.014617688953876495, "mix_low": 0.6483573913574219, "mix_high": 0.0, "read_cosine": 0.3080109655857086, "beta_mean": 0.32027819752693176, "nonfinite": 0.0, "weighted_memory_to_active": 0.6726879477500916, "mix_std": 0.03293399512767792, "beta_std": 0.2169460952281952, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06424392014741898, "evidence_weights": [0.050562720745801926, 0.06438171863555908, -0.03048599511384964], "agreement_weight": 0.004182460717856884, "orthogonal_weight": -0.18798907101154327}, {"head": 3, "memory_to_active": 6.44775390625, "contribution_to_active": 2.8181824684143066, "mix_mean": 0.04170168191194534, "mix_low": 0.4196901321411133, "mix_high": 0.0, "read_cosine": 0.3588457703590393, "beta_mean": 0.3177184462547302, "nonfinite": 0.0, "weighted_memory_to_active": 2.9883692264556885, "mix_std": 0.08913657069206238, "beta_std": 0.22407342493534088, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11259360611438751, "evidence_weights": [-0.06526906788349152, -0.08734617382287979, -0.13031147420406342], "agreement_weight": -0.013839475810527802, "orthogonal_weight": -0.04581840708851814}, {"head": 4, "memory_to_active": 2.891606330871582, "contribution_to_active": 0.05022726580500603, "mix_mean": 0.005391671322286129, "mix_low": 0.8592386245727539, "mix_high": 0.0, "read_cosine": 0.31752893328666687, "beta_mean": 0.19666361808776855, "nonfinite": 0.0, "weighted_memory_to_active": 0.04469862952828407, "mix_std": 0.0105983205139637, "beta_std": 0.13689926266670227, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.004664112348109484, "evidence_weights": [0.12127149105072021, 0.06804879009723663, -0.3585110604763031], "agreement_weight": -0.09098006039857864, "orthogonal_weight": 0.10872918367385864}, {"head": 5, "memory_to_active": 0.11646517366170883, "contribution_to_active": 0.36771512031555176, "mix_mean": 0.5003818869590759, "mix_low": 0.014296531677246094, "mix_high": 0.025580406188964844, "read_cosine": 0.9331472516059875, "beta_mean": 0.042890436947345734, "nonfinite": 0.0, "weighted_memory_to_active": 0.05140713229775429, "mix_std": 0.2847817838191986, "beta_std": 0.04089558124542236, "beta_low": 0.0767822265625, "beta_high": 0.0, "orthogonal_to_active": 0.01623055711388588, "evidence_weights": [-0.25641876459121704, -0.24931466579437256, 0.2545355558395386], "agreement_weight": 0.2305082529783249, "orthogonal_weight": -0.5952978134155273}]}, {"layer": 1, "memory_to_active": 2.3167357444763184, "contribution_to_active": 0.4612194895744324, "mix_mean": 0.16844332218170166, "mix_low": 0.37156519293785095, "mix_high": 0.0005787214031443, "read_cosine": 0.3308418393135071, "beta_mean": 0.19976751506328583, "nonfinite": 0.0, "weighted_memory_to_active": 0.3527522087097168, "mix_std": 0.11789058148860931, "beta_std": 0.13345487415790558, "beta_low": 0.2001953274011612, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.04803604260087013, "output_influence": 0.17612341046333313, "residual_influence": 0.04148862138390541, "state_active": 0.036449532955884933, "state_memory": 0.03712739795446396, "state_peak": 28.504175186157227, "growth_active": 1.1444274187088013, "growth_memory": 1.491421103477478, "heads": [{"head": 0, "memory_to_active": 0.08094727993011475, "contribution_to_active": 0.40975049138069153, "mix_mean": 0.39605921506881714, "mix_low": 0.011431694030761719, "mix_high": 0.00026607513427734375, "read_cosine": -0.0369664691388607, "beta_mean": 0.00046745402505621314, "nonfinite": 0.0, "weighted_memory_to_active": 0.04374511167407036, "mix_std": 0.27640846371650696, "beta_std": 0.0006073337281122804, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.039355725049972534, "evidence_weights": [-0.1755574494600296, -0.22499336302280426, 0.01942584104835987], "agreement_weight": 0.12779125571250916, "orthogonal_weight": -0.9720186591148376}, {"head": 1, "memory_to_active": 3.016357421875, "contribution_to_active": 0.1977667212486267, "mix_mean": 0.02067756839096546, "mix_low": 0.4947776794433594, "mix_high": 0.0, "read_cosine": 0.4665321111679077, "beta_mean": 0.33117642998695374, "nonfinite": 0.0, "weighted_memory_to_active": 0.21873648464679718, "mix_std": 0.04354500770568848, "beta_std": 0.2115110456943512, "beta_low": 0.0, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.010300636291503906, "evidence_weights": [0.009393584914505482, -0.012628687545657158, -0.021553581580519676], "agreement_weight": -0.00871544610708952, "orthogonal_weight": 0.09891743212938309}, {"head": 2, "memory_to_active": 0.41123872995376587, "contribution_to_active": 0.5135507583618164, "mix_mean": 0.5332116484642029, "mix_low": 0.0021572113037109375, "mix_high": 0.0031976699829101562, "read_cosine": 0.7590192556381226, "beta_mean": 0.030715227127075195, "nonfinite": 0.0, "weighted_memory_to_active": 0.32853689789772034, "mix_std": 0.25632941722869873, "beta_std": 0.023305818438529968, "beta_low": 0.15911865234375, "beta_high": 0.0, "orthogonal_to_active": 0.04518485069274902, "evidence_weights": [-0.2816123962402344, -0.3782498240470886, 0.2917100191116333], "agreement_weight": 0.028129976242780685, "orthogonal_weight": -0.26039209961891174}, {"head": 3, "memory_to_active": 2.885065793991089, "contribution_to_active": 0.09887532144784927, "mix_mean": 0.016260020434856415, "mix_low": 0.6657419204711914, "mix_high": 0.0, "read_cosine": 0.23180264234542847, "beta_mean": 0.3576020300388336, "nonfinite": 0.0, "weighted_memory_to_active": 0.14108175039291382, "mix_std": 0.04350953921675682, "beta_std": 0.2559741735458374, "beta_low": 0.02557373046875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.03976614773273468, "evidence_weights": [0.04218776151537895, -0.0012697774218395352, 0.10673996806144714], "agreement_weight": -0.03277691826224327, "orthogonal_weight": -0.32328343391418457}, {"head": 4, "memory_to_active": 3.30812668800354, "contribution_to_active": 1.4789055585861206, "mix_mean": 0.03572361171245575, "mix_low": 0.27733898162841797, "mix_high": 8.58306884765625e-06, "read_cosine": 0.339246928691864, "beta_mean": 0.19368092715740204, "nonfinite": 0.0, "weighted_memory_to_active": 1.3220256567001343, "mix_std": 0.06515485793352127, "beta_std": 0.10537727177143097, "beta_low": 0.003173828125, "beta_high": 0.0, "orthogonal_to_active": 0.14911310374736786, "evidence_weights": [-0.038633182644844055, 0.0007966567063704133, 0.0048021902330219746], "agreement_weight": -0.026596011593937874, "orthogonal_weight": 0.13808807730674744}, {"head": 5, "memory_to_active": 4.198678016662598, "contribution_to_active": 0.06846798956394196, "mix_mean": 0.008727899752557278, "mix_low": 0.7779436111450195, "mix_high": 0.0, "read_cosine": 0.2254164218902588, "beta_mean": 0.2849630117416382, "nonfinite": 0.0, "weighted_memory_to_active": 0.062387190759181976, "mix_std": 0.022396182641386986, "beta_std": 0.2039536088705063, "beta_low": 0.013427734375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.004495791159570217, "evidence_weights": [0.14452001452445984, 0.06329121440649033, 0.3055654764175415], "agreement_weight": -0.13171832263469696, "orthogonal_weight": 0.07338117808103561}]}, {"layer": 2, "memory_to_active": 1.0980027914047241, "contribution_to_active": 0.34306371212005615, "mix_mean": 0.33364900946617126, "mix_low": 0.2608412206172943, "mix_high": 0.00295893382281065, "read_cosine": 0.3294474184513092, "beta_mean": 0.1026105284690857, "nonfinite": 0.0, "weighted_memory_to_active": 0.07627037167549133, "mix_std": 0.19613005220890045, "beta_std": 0.0749933123588562, "beta_low": 0.5521138906478882, "beta_high": 0.00014241535973269492, "orthogonal_to_active": 0.01678905263543129, "output_influence": 0.32082533836364746, "residual_influence": 0.07951967418193817, "state_active": 0.03786355257034302, "state_memory": 0.01997474767267704, "state_peak": 44.517059326171875, "growth_active": 1.0250896215438843, "growth_memory": 1.5777090787887573, "heads": [{"head": 0, "memory_to_active": 0.007803581189364195, "contribution_to_active": 0.30941998958587646, "mix_mean": 0.4869690239429474, "mix_low": 0.0026998519897460938, "mix_high": 0.0038919448852539062, "read_cosine": 0.20521122217178345, "beta_mean": 6.243247480597347e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003979026805609465, "mix_std": 0.2944805324077606, "beta_std": 7.367033686023206e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000811352685559541, "evidence_weights": [-0.2862406373023987, -0.181584894657135, 0.19601047039031982], "agreement_weight": 0.2014903426170349, "orthogonal_weight": -0.21636490523815155}, {"head": 1, "memory_to_active": 3.056368827819824, "contribution_to_active": 0.059185996651649475, "mix_mean": 0.005108105484396219, "mix_low": 0.893061637878418, "mix_high": 9.5367431640625e-07, "read_cosine": 0.18674200773239136, "beta_mean": 0.2965763807296753, "nonfinite": 0.0, "weighted_memory_to_active": 0.045014671981334686, "mix_std": 0.011634044349193573, "beta_std": 0.16000989079475403, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.01724296063184738, "evidence_weights": [0.10256539285182953, 0.10866878181695938, 0.438510537147522], "agreement_weight": 0.043417058885097504, "orthogonal_weight": 0.4225044846534729}, {"head": 2, "memory_to_active": 0.01663508638739586, "contribution_to_active": 0.4476366639137268, "mix_mean": 0.4742392301559448, "mix_low": 0.0016078948974609375, "mix_high": 0.0029859542846679688, "read_cosine": 0.2686638832092285, "beta_mean": 3.7968449760228395e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006764460355043411, "mix_std": 0.2819746136665344, "beta_std": 7.671681669307873e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016704198205843568, "evidence_weights": [-0.22628571093082428, -0.2647249400615692, 0.43737977743148804], "agreement_weight": 0.02092565968632698, "orthogonal_weight": -0.29282745718955994}, {"head": 3, "memory_to_active": 0.19747281074523926, "contribution_to_active": 0.4845253527164459, "mix_mean": 0.4387389123439789, "mix_low": 0.0009479522705078125, "mix_high": 0.004464149475097656, "read_cosine": 0.6837272047996521, "beta_mean": 0.038174085319042206, "nonfinite": 0.0, "weighted_memory_to_active": 0.10727792978286743, "mix_std": 0.2663595676422119, "beta_std": 0.05118132010102272, "beta_low": 0.282958984375, "beta_high": 0.0, "orthogonal_to_active": 0.06122180074453354, "evidence_weights": [-0.16192129254341125, -0.25486573576927185, 0.854973554611206], "agreement_weight": 0.06384257972240448, "orthogonal_weight": -0.870897650718689}, {"head": 4, "memory_to_active": 3.290370225906372, "contribution_to_active": 0.243449866771698, "mix_mean": 0.016246428713202477, "mix_low": 0.6650876998901367, "mix_high": 0.0, "read_cosine": 0.1883901208639145, "beta_mean": 0.2807537317276001, "nonfinite": 0.0, "weighted_memory_to_active": 0.282831072807312, "mix_std": 0.04596670717000961, "beta_std": 0.2385362684726715, "beta_low": 0.02960205078125, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.01901397481560707, "evidence_weights": [0.03474416211247444, 0.03338762745261192, 0.0940486416220665], "agreement_weight": 0.03399091958999634, "orthogonal_weight": -0.07782568037509918}, {"head": 5, "memory_to_active": 0.019366035237908363, "contribution_to_active": 0.5141642689704895, "mix_mean": 0.5805922746658325, "mix_low": 0.0016422271728515625, "mix_high": 0.0064105987548828125, "read_cosine": 0.44394996762275696, "beta_mean": 5.8579229516908526e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011755040846765041, "mix_std": 0.27636468410491943, "beta_std": 8.199911826523021e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000773812469560653, "evidence_weights": [-0.25533318519592285, -0.27954408526420593, 0.8318595886230469], "agreement_weight": 0.06692373007535934, "orthogonal_weight": 0.10869432985782623}]}, {"layer": 3, "memory_to_active": 0.9633297324180603, "contribution_to_active": 0.38256311416625977, "mix_mean": 0.3212621808052063, "mix_low": 0.17535211145877838, "mix_high": 0.006781737320125103, "read_cosine": 0.24276059865951538, "beta_mean": 0.06500493735074997, "nonfinite": 0.0, "weighted_memory_to_active": 0.1553870141506195, "mix_std": 0.23327507078647614, "beta_std": 0.055824436247348785, "beta_low": 0.4697469174861908, "beta_high": 0.0, "orthogonal_to_active": 0.06043483316898346, "output_influence": 0.2965339422225952, "residual_influence": 0.05845087394118309, "state_active": 0.023941615596413612, "state_memory": 0.01337459497153759, "state_peak": 19.082332611083984, "growth_active": 1.0510752201080322, "growth_memory": 1.9200903177261353, "heads": [{"head": 0, "memory_to_active": 0.49330824613571167, "contribution_to_active": 0.49308592081069946, "mix_mean": 0.4232097864151001, "mix_low": 0.02371978759765625, "mix_high": 0.012290000915527344, "read_cosine": 0.23450951278209686, "beta_mean": 0.014642316848039627, "nonfinite": 0.0, "weighted_memory_to_active": 0.2402174025774002, "mix_std": 0.32818302512168884, "beta_std": 0.023818524554371834, "beta_low": 0.5601806640625, "beta_high": 0.0, "orthogonal_to_active": 0.15442343056201935, "evidence_weights": [-0.1650172472000122, -0.20793628692626953, 0.603155255317688], "agreement_weight": 0.13169169425964355, "orthogonal_weight": -0.6778298020362854}, {"head": 1, "memory_to_active": 0.008134830743074417, "contribution_to_active": 0.34287768602371216, "mix_mean": 0.41063565015792847, "mix_low": 0.019715309143066406, "mix_high": 0.0034894943237304688, "read_cosine": 0.2012018859386444, "beta_mean": 4.4942935346625745e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034822076559066772, "mix_std": 0.30933722853660583, "beta_std": 0.00010160551028093323, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014782985672354698, "evidence_weights": [-0.20127683877944946, -0.17318671941757202, 0.5817107558250427], "agreement_weight": 0.3082895576953888, "orthogonal_weight": -0.46697258949279785}, {"head": 2, "memory_to_active": 2.683699369430542, "contribution_to_active": 0.30661898851394653, "mix_mean": 0.04745352268218994, "mix_low": 0.3368043899536133, "mix_high": 1.049041748046875e-05, "read_cosine": 0.19688962399959564, "beta_mean": 0.21445710957050323, "nonfinite": 0.0, "weighted_memory_to_active": 0.3684460520744324, "mix_std": 0.09944280236959457, "beta_std": 0.14496935904026031, "beta_low": 0.00518798828125, "beta_high": 0.0, "orthogonal_to_active": 0.01078938040882349, "evidence_weights": [-0.06251206994056702, 0.014433076605200768, -0.00650586886331439], "agreement_weight": -0.01988927461206913, "orthogonal_weight": -0.03691970556974411}, {"head": 3, "memory_to_active": 0.03232218697667122, "contribution_to_active": 0.48317664861679077, "mix_mean": 0.5183376669883728, "mix_low": 0.014208793640136719, "mix_high": 0.011828422546386719, "read_cosine": 0.28897231817245483, "beta_mean": 0.0001355193235212937, "nonfinite": 0.0, "weighted_memory_to_active": 0.02058558724820614, "mix_std": 0.3128536343574524, "beta_std": 0.0003149256226606667, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00552850728854537, "evidence_weights": [-0.20249155163764954, -0.21891114115715027, 1.8813104629516602], "agreement_weight": 1.2189595699310303, "orthogonal_weight": -0.35619521141052246}, {"head": 4, "memory_to_active": 2.251610040664673, "contribution_to_active": 0.03970429301261902, "mix_mean": 0.01798732578754425, "mix_low": 0.6394853591918945, "mix_high": 0.0, "read_cosine": 0.19300715625286102, "beta_mean": 0.11122865974903107, "nonfinite": 0.0, "weighted_memory_to_active": 0.08772198855876923, "mix_std": 0.041776299476623535, "beta_std": 0.11652013659477234, "beta_low": 0.07421875, "beta_high": 0.0, "orthogonal_to_active": 0.056834883987903595, "evidence_weights": [0.11869455128908157, 0.07088736444711685, 0.07016175985336304], "agreement_weight": -0.044318538159132004, "orthogonal_weight": -0.6689181923866272}, {"head": 5, "memory_to_active": 0.3109033703804016, "contribution_to_active": 0.6299150586128235, "mix_mean": 0.5099490284919739, "mix_low": 0.018178939819335938, "mix_high": 0.013072013854980469, "read_cosine": 0.34198296070098877, "beta_mean": 0.04952104017138481, "nonfinite": 0.0, "weighted_memory_to_active": 0.2118687778711319, "mix_std": 0.30805739760398865, "beta_std": 0.04922204837203026, "beta_low": 0.17889404296875, "beta_high": 0.0, "orthogonal_to_active": 0.13355445861816406, "evidence_weights": [-0.19537778198719025, -0.23683157563209534, 0.4254435896873474], "agreement_weight": 0.14512747526168823, "orthogonal_weight": -0.746637225151062}]}, {"layer": 4, "memory_to_active": 1.1937040090560913, "contribution_to_active": 0.4163607656955719, "mix_mean": 0.3362749516963959, "mix_low": 0.26916107535362244, "mix_high": 0.005670229904353619, "read_cosine": 0.37571096420288086, "beta_mean": 0.12303552031517029, "nonfinite": 0.0, "weighted_memory_to_active": 0.14220315217971802, "mix_std": 0.159236878156662, "beta_std": 0.11165288835763931, "beta_low": 0.3680521845817566, "beta_high": 0.0, "orthogonal_to_active": 0.035706471651792526, "output_influence": 0.3587462902069092, "residual_influence": 0.09240010380744934, "state_active": 0.035571929067373276, "state_memory": 0.020648078992962837, "state_peak": 38.95701217651367, "growth_active": 1.1612781286239624, "growth_memory": 1.5660433769226074, "heads": [{"head": 0, "memory_to_active": 0.2379363477230072, "contribution_to_active": 0.6977890729904175, "mix_mean": 0.637786865234375, "mix_low": 0.007641792297363281, "mix_high": 0.013552665710449219, "read_cosine": 0.4972890019416809, "beta_mean": 0.23710085451602936, "nonfinite": 0.0, "weighted_memory_to_active": 0.18722373247146606, "mix_std": 0.3318071961402893, "beta_std": 0.2486167550086975, "beta_low": 0.20294189453125, "beta_high": 0.0, "orthogonal_to_active": 0.05636133253574371, "evidence_weights": [-0.2608564794063568, -0.35257595777511597, 0.32218998670578003], "agreement_weight": 0.03008626401424408, "orthogonal_weight": 0.36657100915908813}, {"head": 1, "memory_to_active": 0.08991454541683197, "contribution_to_active": 0.5637906789779663, "mix_mean": 0.595759928226471, "mix_low": 4.291534423828125e-05, "mix_high": 0.005779266357421875, "read_cosine": 0.7398670315742493, "beta_mean": 0.0021226126700639725, "nonfinite": 0.0, "weighted_memory_to_active": 0.05649678409099579, "mix_std": 0.25703179836273193, "beta_std": 0.006437980569899082, "beta_low": 0.96087646484375, "beta_high": 0.0, "orthogonal_to_active": 0.028923356905579567, "evidence_weights": [-0.23259234428405762, -0.2934209406375885, 0.7271445393562317], "agreement_weight": 0.09097931534051895, "orthogonal_weight": -0.8020490407943726}, {"head": 2, "memory_to_active": 2.4699339866638184, "contribution_to_active": 0.18939292430877686, "mix_mean": 0.01922420784831047, "mix_low": 0.5808811187744141, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3022497892379761, "beta_mean": 0.1406901478767395, "nonfinite": 0.0, "weighted_memory_to_active": 0.23395845293998718, "mix_std": 0.03904973343014717, "beta_std": 0.1077594980597496, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.01549819577485323, "evidence_weights": [0.049435798078775406, 0.02599515952169895, -0.02312314882874489], "agreement_weight": 0.10345304757356644, "orthogonal_weight": -0.09919152408838272}, {"head": 3, "memory_to_active": 1.675796627998352, "contribution_to_active": 0.13603761792182922, "mix_mean": 0.03551292046904564, "mix_low": 0.3799104690551758, "mix_high": 0.0, "read_cosine": 0.1911550760269165, "beta_mean": 0.13470607995986938, "nonfinite": 0.0, "weighted_memory_to_active": 0.19274520874023438, "mix_std": 0.06505702435970306, "beta_std": 0.13216829299926758, "beta_low": 0.043212890625, "beta_high": 0.0, "orthogonal_to_active": 0.013645880855619907, "evidence_weights": [-0.028344178572297096, 0.049641188234090805, -0.25042209029197693], "agreement_weight": 0.05366597697138786, "orthogonal_weight": -0.09513881802558899}, {"head": 4, "memory_to_active": 0.01940787024796009, "contribution_to_active": 0.6455867290496826, "mix_mean": 0.7150332927703857, "mix_low": 0.000385284423828125, "mix_high": 0.014682769775390625, "read_cosine": 0.20142243802547455, "beta_mean": 3.511027171043679e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014083776623010635, "mix_std": 0.22892826795578003, "beta_std": 0.00010504689271328971, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0022776061668992043, "evidence_weights": [-0.30916228890419006, -0.3369807004928589, 0.7733337879180908], "agreement_weight": 0.1415879875421524, "orthogonal_weight": -0.17839315533638}, {"head": 5, "memory_to_active": 2.669234275817871, "contribution_to_active": 0.26556751132011414, "mix_mean": 0.014332436956465244, "mix_low": 0.6461048126220703, "mix_high": 5.7220458984375e-06, "read_cosine": 0.3222825229167938, "beta_mean": 0.22355830669403076, "nonfinite": 0.0, "weighted_memory_to_active": 0.16871097683906555, "mix_std": 0.03354717418551445, "beta_std": 0.17482972145080566, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.09753245115280151, "evidence_weights": [0.044915664941072464, -0.03472697362303734, -0.2301935851573944], "agreement_weight": -0.04511948302388191, "orthogonal_weight": 0.7973572611808777}]}, {"layer": 5, "memory_to_active": 0.6110866069793701, "contribution_to_active": 0.4778204560279846, "mix_mean": 0.43235525488853455, "mix_low": 0.07866319268941879, "mix_high": 0.01261774729937315, "read_cosine": 0.17114382982254028, "beta_mean": 0.057780418545007706, "nonfinite": 0.0, "weighted_memory_to_active": 0.15502579510211945, "mix_std": 0.2501520812511444, "beta_std": 0.07356417179107666, "beta_low": 0.6176961660385132, "beta_high": 0.0, "orthogonal_to_active": 0.054330386221408844, "output_influence": 0.46702703833580017, "residual_influence": 0.11473602801561356, "state_active": 0.04841126129031181, "state_memory": 0.012603888288140297, "state_peak": 62.03499984741211, "growth_active": 1.1227766275405884, "growth_memory": 1.277584195137024, "heads": [{"head": 0, "memory_to_active": 0.9668854475021362, "contribution_to_active": 0.36961135268211365, "mix_mean": 0.1919625699520111, "mix_low": 0.0859823226928711, "mix_high": 0.000560760498046875, "read_cosine": -0.18812081217765808, "beta_mean": 0.048465948551893234, "nonfinite": 0.0, "weighted_memory_to_active": 0.16558192670345306, "mix_std": 0.21706655621528625, "beta_std": 0.07369513064622879, "beta_low": 0.266357421875, "beta_high": 0.0, "orthogonal_to_active": 0.037275027483701706, "evidence_weights": [-0.07633688300848007, -0.11166399717330933, -0.33160150051116943], "agreement_weight": -0.014128698036074638, "orthogonal_weight": -0.21865619719028473}, {"head": 1, "memory_to_active": 0.02373371087014675, "contribution_to_active": 0.5197083950042725, "mix_mean": 0.6073144674301147, "mix_low": 0.0040988922119140625, "mix_high": 0.02137279510498047, "read_cosine": 0.1715497523546219, "beta_mean": 5.738170148106292e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012609757483005524, "mix_std": 0.3041628897190094, "beta_std": 0.00010821520118042827, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011356018483638763, "evidence_weights": [-0.23692554235458374, -0.26943957805633545, 0.8247842788696289], "agreement_weight": 0.41000601649284363, "orthogonal_weight": -0.9359803199768066}, {"head": 2, "memory_to_active": 0.027812160551548004, "contribution_to_active": 0.5447232127189636, "mix_mean": 0.618793249130249, "mix_low": 0.0020780563354492188, "mix_high": 0.013242721557617188, "read_cosine": 0.18848517537117004, "beta_mean": 0.0004580687964335084, "nonfinite": 0.0, "weighted_memory_to_active": 0.018985291942954063, "mix_std": 0.271309494972229, "beta_std": 0.0008981892024166882, "beta_low": 0.99896240234375, "beta_high": 0.0, "orthogonal_to_active": 0.0034107649698853493, "evidence_weights": [-0.25308194756507874, -0.3461448848247528, 0.894343912601471], "agreement_weight": 0.017304178327322006, "orthogonal_weight": -0.23216105997562408}, {"head": 3, "memory_to_active": 2.083982467651367, "contribution_to_active": 0.21117839217185974, "mix_mean": 0.07845551520586014, "mix_low": 0.19187259674072266, "mix_high": 0.0, "read_cosine": 0.12319216132164001, "beta_mean": 0.09835296869277954, "nonfinite": 0.0, "weighted_memory_to_active": 0.35327601432800293, "mix_std": 0.12055499106645584, "beta_std": 0.13920529186725616, "beta_low": 0.20477294921875, "beta_high": 0.0, "orthogonal_to_active": 0.1311565339565277, "evidence_weights": [-0.05095723271369934, -0.08584462106227875, -0.08000507950782776], "agreement_weight": 0.06918054819107056, "orthogonal_weight": -0.45708706974983215}, {"head": 4, "memory_to_active": 0.44218748807907104, "contribution_to_active": 0.6035591959953308, "mix_mean": 0.3358668088912964, "mix_low": 0.1851062774658203, "mix_high": 0.008913040161132812, "read_cosine": 0.552672266960144, "beta_mean": 0.1989443600177765, "nonfinite": 0.0, "weighted_memory_to_active": 0.2964978814125061, "mix_std": 0.3441009223461151, "beta_std": 0.22530405223369598, "beta_low": 0.2427978515625, "beta_high": 0.0, "orthogonal_to_active": 0.0904451385140419, "evidence_weights": [-0.11719067394733429, -0.2926134169101715, -0.06321126222610474], "agreement_weight": 0.03861750289797783, "orthogonal_weight": 0.4597629904747009}, {"head": 5, "memory_to_active": 0.12191826105117798, "contribution_to_active": 0.6181421279907227, "mix_mean": 0.7617389559745789, "mix_low": 0.0028409957885742188, "mix_high": 0.031617164611816406, "read_cosine": 0.1790844202041626, "beta_mean": 0.00040377385448664427, "nonfinite": 0.0, "weighted_memory_to_active": 0.08320387452840805, "mix_std": 0.24371759593486786, "beta_std": 0.0021741464734077454, "beta_low": 0.9932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.05233882740139961, "evidence_weights": [-0.32545217871665955, -0.3186778128147125, 2.0119285583496094], "agreement_weight": 0.3664054870605469, "orthogonal_weight": -0.6720359921455383}]}, {"layer": 6, "memory_to_active": 0.3548745810985565, "contribution_to_active": 0.5805333256721497, "mix_mean": 0.5447827577590942, "mix_low": 0.07365083694458008, "mix_high": 0.02363952249288559, "read_cosine": 0.30310651659965515, "beta_mean": 0.07382945716381073, "nonfinite": 0.0, "weighted_memory_to_active": 0.16454416513442993, "mix_std": 0.24127405881881714, "beta_std": 0.08094713091850281, "beta_low": 0.5100199580192566, "beta_high": 0.0, "orthogonal_to_active": 0.037062279880046844, "output_influence": 0.48529067635536194, "residual_influence": 0.14941871166229248, "state_active": 0.048785652965307236, "state_memory": 0.00932119321078062, "state_peak": 88.23884582519531, "growth_active": 1.2319402694702148, "growth_memory": 1.1479873657226562, "heads": [{"head": 0, "memory_to_active": 0.14105945825576782, "contribution_to_active": 0.5979725122451782, "mix_mean": 0.7244359254837036, "mix_low": 9.34600830078125e-05, "mix_high": 0.007802009582519531, "read_cosine": -0.029315011575818062, "beta_mean": 0.0104176951572299, "nonfinite": 0.0, "weighted_memory_to_active": 0.10363496094942093, "mix_std": 0.2270604372024536, "beta_std": 0.01293929573148489, "beta_low": 0.63470458984375, "beta_high": 0.0, "orthogonal_to_active": 0.08532460033893585, "evidence_weights": [-0.3109433054924011, -0.3302063047885895, 0.5126372575759888], "agreement_weight": 0.12567183375358582, "orthogonal_weight": -0.9316559433937073}, {"head": 1, "memory_to_active": 0.0051210615783929825, "contribution_to_active": 0.49986687302589417, "mix_mean": 0.5054234266281128, "mix_low": 0.0014581680297851562, "mix_high": 0.010534286499023438, "read_cosine": 0.6133156418800354, "beta_mean": 1.1246669600950554e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002224174328148365, "mix_std": 0.28902754187583923, "beta_std": 2.129762651748024e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007975065382197499, "evidence_weights": [-0.20341184735298157, -0.27968987822532654, 0.9423814415931702], "agreement_weight": -0.021298089995980263, "orthogonal_weight": -0.512915849685669}, {"head": 2, "memory_to_active": 1.056307077407837, "contribution_to_active": 0.07718212902545929, "mix_mean": 0.02827020175755024, "mix_low": 0.43072986602783203, "mix_high": 0.0, "read_cosine": 0.22649599611759186, "beta_mean": 0.05748403072357178, "nonfinite": 0.0, "weighted_memory_to_active": 0.06569415330886841, "mix_std": 0.051379624754190445, "beta_std": 0.05748172849416733, "beta_low": 0.03204345703125, "beta_high": 0.0, "orthogonal_to_active": 0.0008059880346991122, "evidence_weights": [0.09039172530174255, 7.5041461968794465e-06, 0.27426427602767944], "agreement_weight": 0.12633907794952393, "orthogonal_weight": -0.014721029438078403}, {"head": 3, "memory_to_active": 0.15874266624450684, "contribution_to_active": 0.6621436476707458, "mix_mean": 0.626001238822937, "mix_low": 0.008149147033691406, "mix_high": 0.011989593505859375, "read_cosine": 0.4959948658943176, "beta_mean": 0.2280086874961853, "nonfinite": 0.0, "weighted_memory_to_active": 0.1122639924287796, "mix_std": 0.32898029685020447, "beta_std": 0.2555884122848511, "beta_low": 0.18756103515625, "beta_high": 0.0, "orthogonal_to_active": 0.058117277920246124, "evidence_weights": [-0.1699916571378708, -0.33242273330688477, 0.15425392985343933], "agreement_weight": -0.01428036205470562, "orthogonal_weight": 0.617924690246582}, {"head": 4, "memory_to_active": 0.7214277982711792, "contribution_to_active": 1.0433028936386108, "mix_mean": 0.7548772096633911, "mix_low": 0.001338958740234375, "mix_high": 0.1083526611328125, "read_cosine": 0.22938528656959534, "beta_mean": 0.14672735333442688, "nonfinite": 0.0, "weighted_memory_to_active": 0.6711971163749695, "mix_std": 0.2900032699108124, "beta_std": 0.15901720523834229, "beta_low": 0.2061767578125, "beta_high": 0.0, "orthogonal_to_active": 0.04477602615952492, "evidence_weights": [-0.25791603326797485, -0.5885952115058899, -0.12814807891845703], "agreement_weight": -0.019280197098851204, "orthogonal_weight": 0.08251824229955673}, {"head": 5, "memory_to_active": 0.0465894490480423, "contribution_to_active": 0.6027316451072693, "mix_mean": 0.6296887993812561, "mix_low": 0.0001354217529296875, "mix_high": 0.0031585693359375, "read_cosine": 0.282762348651886, "beta_mean": 0.0003277241485193372, "nonfinite": 0.0, "weighted_memory_to_active": 0.03225056827068329, "mix_std": 0.26119309663772583, "beta_std": 0.0006348314927890897, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.03255226090550423, "evidence_weights": [-0.23979830741882324, -0.30246248841285706, 0.41003361344337463], "agreement_weight": 0.12425872683525085, "orthogonal_weight": -1.0841642618179321}]}, {"layer": 7, "memory_to_active": 1.8030325174331665, "contribution_to_active": 0.21592479944229126, "mix_mean": 0.07521671056747437, "mix_low": 0.32357966899871826, "mix_high": 9.012222290039062e-05, "read_cosine": 0.24615134298801422, "beta_mean": 0.12551404535770416, "nonfinite": 0.0, "weighted_memory_to_active": 0.22904837131500244, "mix_std": 0.10720353573560715, "beta_std": 0.10460326075553894, "beta_low": 0.1279500424861908, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.05523255467414856, "output_influence": 0.21953217685222626, "residual_influence": 0.048002488911151886, "state_active": 0.014048432931303978, "state_memory": 0.026921194046735764, "state_peak": 27.94837188720703, "growth_active": 1.0869204998016357, "growth_memory": 1.2824690341949463, "heads": [{"head": 0, "memory_to_active": 2.3405768871307373, "contribution_to_active": 0.3829764425754547, "mix_mean": 0.0344272255897522, "mix_low": 0.43135833740234375, "mix_high": 9.5367431640625e-07, "read_cosine": 0.4020175337791443, "beta_mean": 0.1462690532207489, "nonfinite": 0.0, "weighted_memory_to_active": 0.40492236614227295, "mix_std": 0.07078178226947784, "beta_std": 0.11557076871395111, "beta_low": 0.00531005859375, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.02187393605709076, "evidence_weights": [0.005393346771597862, -0.09213738143444061, 0.13531313836574554], "agreement_weight": -0.038822025060653687, "orthogonal_weight": 0.09074517339468002}, {"head": 1, "memory_to_active": 1.1965242624282837, "contribution_to_active": 0.33967891335487366, "mix_mean": 0.16645948588848114, "mix_low": 0.17889881134033203, "mix_high": 0.000457763671875, "read_cosine": 0.29081496596336365, "beta_mean": 0.12575645744800568, "nonfinite": 0.0, "weighted_memory_to_active": 0.32648515701293945, "mix_std": 0.19945091009140015, "beta_std": 0.08072207868099213, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00678630406036973, "evidence_weights": [-0.18184664845466614, -0.11734684556722641, -0.24718032777309418], "agreement_weight": 0.023881610482931137, "orthogonal_weight": -0.02341310679912567}, {"head": 2, "memory_to_active": 0.9681288003921509, "contribution_to_active": 0.26035648584365845, "mix_mean": 0.08778387308120728, "mix_low": 0.16132354736328125, "mix_high": 8.20159912109375e-05, "read_cosine": 0.12902745604515076, "beta_mean": 0.07775337249040604, "nonfinite": 0.0, "weighted_memory_to_active": 0.1906682848930359, "mix_std": 0.13886107504367828, "beta_std": 0.06016470491886139, "beta_low": 0.0389404296875, "beta_high": 0.0, "orthogonal_to_active": 0.044602327048778534, "evidence_weights": [-0.03283756971359253, -0.1213969811797142, 0.33590444922447205], "agreement_weight": 0.019856343045830727, "orthogonal_weight": -0.2388274222612381}, {"head": 3, "memory_to_active": 3.8295326232910156, "contribution_to_active": 0.07312656939029694, "mix_mean": 0.030323751270771027, "mix_low": 0.3769550323486328, "mix_high": 0.0, "read_cosine": 0.12329216301441193, "beta_mean": 0.2565878629684448, "nonfinite": 0.0, "weighted_memory_to_active": 0.2369404286146164, "mix_std": 0.04664193466305733, "beta_std": 0.22780372202396393, "beta_low": 0.01617431640625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.19176602363586426, "evidence_weights": [0.08081197738647461, 0.020084720104932785, 0.6050485968589783], "agreement_weight": 0.04637347534298897, "orthogonal_weight": -0.9127877950668335}, {"head": 4, "memory_to_active": 1.802588701248169, "contribution_to_active": 0.08372367173433304, "mix_mean": 0.01504747848957777, "mix_low": 0.6726083755493164, "mix_high": 0.0, "read_cosine": 0.2950676679611206, "beta_mean": 0.13569584488868713, "nonfinite": 0.0, "weighted_memory_to_active": 0.07414683699607849, "mix_std": 0.037396520376205444, "beta_std": 0.12281288951635361, "beta_low": 0.01507568359375, "beta_high": 0.0, "orthogonal_to_active": 0.00564992893487215, "evidence_weights": [0.12408839166164398, 0.05109972134232521, 0.15002861618995667], "agreement_weight": 0.05249512568116188, "orthogonal_weight": -0.10034999251365662}, {"head": 5, "memory_to_active": 0.6808434724807739, "contribution_to_active": 0.1556866317987442, "mix_mean": 0.11725839972496033, "mix_low": 0.12033367156982422, "mix_high": 0.0, "read_cosine": 0.23668816685676575, "beta_mean": 0.011021707206964493, "nonfinite": 0.0, "weighted_memory_to_active": 0.14112718403339386, "mix_std": 0.15008896589279175, "beta_std": 0.020545385777950287, "beta_low": 0.69219970703125, "beta_high": 0.0, "orthogonal_to_active": 0.06071677431464195, "evidence_weights": [-0.023355724290013313, -0.0693657398223877, 0.2193496823310852], "agreement_weight": 0.07212236523628235, "orthogonal_weight": -0.5420722365379333}]}, {"layer": 8, "memory_to_active": 0.5395795702934265, "contribution_to_active": 0.5174392461776733, "mix_mean": 0.4038039743900299, "mix_low": 0.09744501113891602, "mix_high": 0.000689347623847425, "read_cosine": 0.49750328063964844, "beta_mean": 0.037212222814559937, "nonfinite": 0.0, "weighted_memory_to_active": 0.11256542056798935, "mix_std": 0.22149312496185303, "beta_std": 0.04115654528141022, "beta_low": 0.5736083984375, "beta_high": 0.0, "orthogonal_to_active": 0.05957450717687607, "output_influence": 0.42003312706947327, "residual_influence": 0.1257835179567337, "state_active": 0.0518781840801239, "state_memory": 0.011313660070300102, "state_peak": 36.160682678222656, "growth_active": 1.0787389278411865, "growth_memory": 1.3554894924163818, "heads": [{"head": 0, "memory_to_active": 0.05141853541135788, "contribution_to_active": 0.5351173877716064, "mix_mean": 0.5772324800491333, "mix_low": 0.00027942657470703125, "mix_high": 0.0006923675537109375, "read_cosine": 0.622587263584137, "beta_mean": 0.0038018024060875177, "nonfinite": 0.0, "weighted_memory_to_active": 0.03339926898479462, "mix_std": 0.24182449281215668, "beta_std": 0.009581257589161396, "beta_low": 0.90704345703125, "beta_high": 0.0, "orthogonal_to_active": 0.019783122465014458, "evidence_weights": [-0.17585280537605286, -0.38490739464759827, 0.7212595343589783], "agreement_weight": 0.08311505615711212, "orthogonal_weight": -0.8133968114852905}, {"head": 1, "memory_to_active": 0.21116819977760315, "contribution_to_active": 0.4516884684562683, "mix_mean": 0.43662020564079285, "mix_low": 0.0016984939575195312, "mix_high": 0.00024700164794921875, "read_cosine": 0.3647702932357788, "beta_mean": 0.00579913379624486, "nonfinite": 0.0, "weighted_memory_to_active": 0.10059419274330139, "mix_std": 0.2520616352558136, "beta_std": 0.012220081873238087, "beta_low": 0.85791015625, "beta_high": 0.0, "orthogonal_to_active": 0.057155489921569824, "evidence_weights": [-0.19156892597675323, -0.24104809761047363, 0.6638128757476807], "agreement_weight": 0.06842904537916183, "orthogonal_weight": -0.6360125541687012}, {"head": 2, "memory_to_active": 0.030750859528779984, "contribution_to_active": 0.6290748119354248, "mix_mean": 0.5993496179580688, "mix_low": 0.0010366439819335938, "mix_high": 0.00249481201171875, "read_cosine": 0.39255934953689575, "beta_mean": 0.0002565321046859026, "nonfinite": 0.0, "weighted_memory_to_active": 0.02173992246389389, "mix_std": 0.23832926154136658, "beta_std": 0.0020181413274258375, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.014991354197263718, "evidence_weights": [-0.21707376837730408, -0.33401432633399963, 0.7727718949317932], "agreement_weight": 0.17197464406490326, "orthogonal_weight": -0.7924085855484009}, {"head": 3, "memory_to_active": 0.1397647261619568, "contribution_to_active": 0.5965766310691833, "mix_mean": 0.5170391798019409, "mix_low": 0.0008172988891601562, "mix_high": 0.0003757476806640625, "read_cosine": 0.7845371961593628, "beta_mean": 0.013247953727841377, "nonfinite": 0.0, "weighted_memory_to_active": 0.10030250251293182, "mix_std": 0.25278282165527344, "beta_std": 0.01125793345272541, "beta_low": 0.43804931640625, "beta_high": 0.0, "orthogonal_to_active": 0.030578818172216415, "evidence_weights": [-0.17003096640110016, -0.2912468910217285, 0.6941167712211609], "agreement_weight": 0.13394208252429962, "orthogonal_weight": -0.5873461365699768}, {"head": 4, "memory_to_active": 2.511418342590332, "contribution_to_active": 0.2622252106666565, "mix_mean": 0.025327252224087715, "mix_low": 0.4502105712890625, "mix_high": 0.0, "read_cosine": 0.29431989789009094, "beta_mean": 0.12346585839986801, "nonfinite": 0.0, "weighted_memory_to_active": 0.24343085289001465, "mix_std": 0.04607408493757248, "beta_std": 0.1028146967291832, "beta_low": 0.01068115234375, "beta_high": 0.0, "orthogonal_to_active": 0.01226859726011753, "evidence_weights": [0.05026181787252426, -0.011073550209403038, -0.12307905405759811], "agreement_weight": 0.034674324095249176, "orthogonal_weight": 0.052410438656806946}, {"head": 5, "memory_to_active": 0.2929566204547882, "contribution_to_active": 0.6299528479576111, "mix_mean": 0.26725512742996216, "mix_low": 0.13062763214111328, "mix_high": 0.0003261566162109375, "read_cosine": 0.5262453556060791, "beta_mean": 0.07670202851295471, "nonfinite": 0.0, "weighted_memory_to_active": 0.17592579126358032, "mix_std": 0.29788637161254883, "beta_std": 0.10904712975025177, "beta_low": 0.228271484375, "beta_high": 0.0, "orthogonal_to_active": 0.22266963124275208, "evidence_weights": [-0.06609486043453217, -0.2893916666507721, -0.40659627318382263], "agreement_weight": 0.015889404341578484, "orthogonal_weight": 1.5314745903015137}]}, {"layer": 9, "memory_to_active": 1.1612528562545776, "contribution_to_active": 0.6250625848770142, "mix_mean": 0.29473596811294556, "mix_low": 0.1599416732788086, "mix_high": 0.002386888023465872, "read_cosine": 0.37051552534103394, "beta_mean": 0.073851078748703, "nonfinite": 0.0, "weighted_memory_to_active": 0.36875760555267334, "mix_std": 0.20644447207450867, "beta_std": 0.046310488134622574, "beta_low": 0.4994913935661316, "beta_high": 0.0, "orthogonal_to_active": 0.03876562416553497, "output_influence": 0.4678913950920105, "residual_influence": 0.12514086067676544, "state_active": 0.03836175799369812, "state_memory": 0.021080102771520615, "state_peak": 37.642822265625, "growth_active": 1.1524198055267334, "growth_memory": 1.3831595182418823, "heads": [{"head": 0, "memory_to_active": 0.003995839040726423, "contribution_to_active": 0.5230526328086853, "mix_mean": 0.5877453088760376, "mix_low": 0.0019178390502929688, "mix_high": 0.009890556335449219, "read_cosine": 0.26852816343307495, "beta_mean": 9.152055099548306e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002416386501863599, "mix_std": 0.2926398515701294, "beta_std": 4.1390416299691424e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020515681244432926, "evidence_weights": [-0.2902281880378723, -0.2970705032348633, 0.931489109992981], "agreement_weight": 0.085781991481781, "orthogonal_weight": -0.9012274146080017}, {"head": 1, "memory_to_active": 0.03595167025923729, "contribution_to_active": 0.5974452495574951, "mix_mean": 0.45288386940956116, "mix_low": 0.010575294494628906, "mix_high": 0.0015382766723632812, "read_cosine": 0.3907831013202667, "beta_mean": 9.86323575489223e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.022332631051540375, "mix_std": 0.2898220419883728, "beta_std": 0.00022701291891280562, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015282846987247467, "evidence_weights": [-0.21401436626911163, -0.258633553981781, 1.1373238563537598], "agreement_weight": 0.33167240023612976, "orthogonal_weight": -0.807946503162384}, {"head": 2, "memory_to_active": 0.21127186715602875, "contribution_to_active": 0.6032567620277405, "mix_mean": 0.5132550001144409, "mix_low": 0.0020122528076171875, "mix_high": 0.0028619766235351562, "read_cosine": 0.32205936312675476, "beta_mean": 0.0009079640149138868, "nonfinite": 0.0, "weighted_memory_to_active": 0.1293158382177353, "mix_std": 0.27908262610435486, "beta_std": 0.005407385528087616, "beta_low": 0.99249267578125, "beta_high": 0.0, "orthogonal_to_active": 0.06154070049524307, "evidence_weights": [-0.18065506219863892, -0.23776447772979736, 0.599526584148407], "agreement_weight": 0.24733735620975494, "orthogonal_weight": -0.5538997650146484}, {"head": 3, "memory_to_active": 1.670776605606079, "contribution_to_active": 0.1698639988899231, "mix_mean": 0.03690788894891739, "mix_low": 0.4150218963623047, "mix_high": 1.9073486328125e-06, "read_cosine": 0.4300138056278229, "beta_mean": 0.09343567490577698, "nonfinite": 0.0, "weighted_memory_to_active": 0.14652113616466522, "mix_std": 0.0866178497672081, "beta_std": 0.06372080743312836, "beta_low": 0.004150390625, "beta_high": 0.0, "orthogonal_to_active": 0.025120653212070465, "evidence_weights": [0.07319973409175873, 0.0388692207634449, -0.07407258450984955], "agreement_weight": 0.08450761437416077, "orthogonal_weight": 0.18916529417037964}, {"head": 4, "memory_to_active": 3.334226369857788, "contribution_to_active": 1.276760458946228, "mix_mean": 0.11505758762359619, "mix_low": 0.1724262237548828, "mix_high": 2.86102294921875e-05, "read_cosine": 0.3152163326740265, "beta_mean": 0.19130173325538635, "nonfinite": 0.0, "weighted_memory_to_active": 1.2424540519714355, "mix_std": 0.17498579621315002, "beta_std": 0.10181251168251038, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02096961811184883, "evidence_weights": [-0.0562613345682621, -0.14809560775756836, 0.013301198370754719], "agreement_weight": -0.050449810922145844, "orthogonal_weight": -0.01731102541089058}, {"head": 5, "memory_to_active": 1.7112945318222046, "contribution_to_active": 0.5799960494041443, "mix_mean": 0.06256603449583054, "mix_low": 0.357696533203125, "mix_high": 0.0, "read_cosine": 0.49649226665496826, "beta_mean": 0.15735334157943726, "nonfinite": 0.0, "weighted_memory_to_active": 0.6695053577423096, "mix_std": 0.11551865935325623, "beta_std": 0.10665380209684372, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.10762834548950195, "evidence_weights": [0.0880478173494339, -0.056975413113832474, 0.1504262536764145], "agreement_weight": 0.02477116696536541, "orthogonal_weight": 0.24232500791549683}]}, {"layer": 10, "memory_to_active": 0.22051669657230377, "contribution_to_active": 0.5683534145355225, "mix_mean": 0.48071664571762085, "mix_low": 0.012276491150259972, "mix_high": 0.006739775650203228, "read_cosine": 0.37289607524871826, "beta_mean": 0.04396685212850571, "nonfinite": 0.0, "weighted_memory_to_active": 0.11002590507268906, "mix_std": 0.2675683796405792, "beta_std": 0.04563899338245392, "beta_low": 0.6590678095817566, "beta_high": 0.0, "orthogonal_to_active": 0.10759514570236206, "output_influence": 0.40096694231033325, "residual_influence": 0.1342301219701767, "state_active": 0.052888382226228714, "state_memory": 0.013887308537960052, "state_peak": 112.4611587524414, "growth_active": 1.058113694190979, "growth_memory": 0.9557396769523621, "heads": [{"head": 0, "memory_to_active": 0.7980895638465881, "contribution_to_active": 1.2746754884719849, "mix_mean": 0.6073123812675476, "mix_low": 0.03604602813720703, "mix_high": 0.029784202575683594, "read_cosine": 0.0943479910492897, "beta_mean": 0.19293269515037537, "nonfinite": 0.0, "weighted_memory_to_active": 0.528408408164978, "mix_std": 0.3770299255847931, "beta_std": 0.208720862865448, "beta_low": 0.1142578125, "beta_high": 0.0, "orthogonal_to_active": 0.5630530714988708, "evidence_weights": [-0.14118994772434235, -0.37083372473716736, -0.6859473586082458], "agreement_weight": 0.027349524199962616, "orthogonal_weight": 1.084645390510559}, {"head": 1, "memory_to_active": 0.017040889710187912, "contribution_to_active": 0.5868837833404541, "mix_mean": 0.5830078721046448, "mix_low": 0.000640869140625, "mix_high": 0.0081787109375, "read_cosine": 0.2645607590675354, "beta_mean": 9.833557851379737e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009906219318509102, "mix_std": 0.2664939761161804, "beta_std": 0.00012340664397925138, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012738262303173542, "evidence_weights": [-0.27508485317230225, -0.20224140584468842, 0.5802636742591858], "agreement_weight": 0.409725159406662, "orthogonal_weight": -1.4085365533828735}, {"head": 2, "memory_to_active": 0.044018227607011795, "contribution_to_active": 0.536003053188324, "mix_mean": 0.5405520796775818, "mix_low": 0.00039005279541015625, "mix_high": 0.0022335052490234375, "read_cosine": 0.025840966030955315, "beta_mean": 0.00013982978998683393, "nonfinite": 0.0, "weighted_memory_to_active": 0.01719553954899311, "mix_std": 0.2587602138519287, "beta_std": 0.0006203950033523142, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.006726806052029133, "evidence_weights": [-0.20993171632289886, -0.25314709544181824, 0.5192140340805054], "agreement_weight": -0.09124100208282471, "orthogonal_weight": -0.37987396121025085}, {"head": 3, "memory_to_active": 0.323993057012558, "contribution_to_active": 0.10320956259965897, "mix_mean": 0.268941193819046, "mix_low": 0.03369140625, "mix_high": 0.00012111663818359375, "read_cosine": 0.9054824709892273, "beta_mean": 0.06426919251680374, "nonfinite": 0.0, "weighted_memory_to_active": 0.03955509513616562, "mix_std": 0.22764691710472107, "beta_std": 0.056692324578762054, "beta_low": 0.001953125, "beta_high": 0.0, "orthogonal_to_active": 0.04432058334350586, "evidence_weights": [-0.1226399764418602, -0.14978773891925812, 0.14553941786289215], "agreement_weight": 0.00834461860358715, "orthogonal_weight": 1.4890670776367188}, {"head": 4, "memory_to_active": 0.0037924088537693024, "contribution_to_active": 0.5056798458099365, "mix_mean": 0.5030413866043091, "mix_low": 0.0007581710815429688, "mix_high": 0.0001125335693359375, "read_cosine": 0.08359081298112869, "beta_mean": 2.273109021189157e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0016821317840367556, "mix_std": 0.25018835067749023, "beta_std": 0.0003537547599989921, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0007667762110941112, "evidence_weights": [-0.1795797497034073, -0.24436436593532562, 0.7462722063064575], "agreement_weight": 0.08153318613767624, "orthogonal_weight": -0.47027331590652466}, {"head": 5, "memory_to_active": 0.13616600632667542, "contribution_to_active": 0.4036683440208435, "mix_mean": 0.38144487142562866, "mix_low": 0.002132415771484375, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8635534644126892, "beta_mean": 0.006338338367640972, "nonfinite": 0.0, "weighted_memory_to_active": 0.06340798735618591, "mix_std": 0.22529086470603943, "beta_std": 0.00732320686802268, "beta_low": 0.83868408203125, "beta_high": 0.0, "orthogonal_to_active": 0.01796535775065422, "evidence_weights": [-0.1492779701948166, -0.1609947830438614, 0.29525166749954224], "agreement_weight": 0.09013427048921585, "orthogonal_weight": -0.6070685982704163}]}, {"layer": 11, "memory_to_active": 0.15541210770606995, "contribution_to_active": 0.5649445056915283, "mix_mean": 0.4471111595630646, "mix_low": 0.0754995346069336, "mix_high": 0.041980110108852386, "read_cosine": 0.4434032440185547, "beta_mean": 0.03417389839887619, "nonfinite": 0.0, "weighted_memory_to_active": 0.06832286715507507, "mix_std": 0.31939905881881714, "beta_std": 0.04838109016418457, "beta_low": 0.5509542226791382, "beta_high": 0.0, "orthogonal_to_active": 0.05090121179819107, "output_influence": 0.4874526858329773, "residual_influence": 0.12285622954368591, "state_active": 0.1732349544763565, "state_memory": 0.007937525399029255, "state_peak": 22.685855865478516, "growth_active": 1.076117992401123, "growth_memory": 1.5839033126831055, "heads": [{"head": 0, "memory_to_active": 0.005930883809924126, "contribution_to_active": 0.563384473323822, "mix_mean": 0.586480438709259, "mix_low": 0.0013675689697265625, "mix_high": 0.0071926116943359375, "read_cosine": 0.36558371782302856, "beta_mean": 9.668845450505614e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037175545003265142, "mix_std": 0.26788797974586487, "beta_std": 0.00406346982344985, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.0024666152894496918, "evidence_weights": [-0.16210968792438507, -0.3545539379119873, 0.6518571376800537], "agreement_weight": 0.21495850384235382, "orthogonal_weight": -0.7501612901687622}, {"head": 1, "memory_to_active": 0.3870478868484497, "contribution_to_active": 0.2780255675315857, "mix_mean": 0.3188462257385254, "mix_low": 0.013698577880859375, "mix_high": 0.0001277923583984375, "read_cosine": 0.6221086382865906, "beta_mean": 0.03295568749308586, "nonfinite": 0.0, "weighted_memory_to_active": 0.1583021730184555, "mix_std": 0.25168734788894653, "beta_std": 0.06093596667051315, "beta_low": 0.40484619140625, "beta_high": 0.0, "orthogonal_to_active": 0.03739875182509422, "evidence_weights": [-0.16012679040431976, -0.1484292596578598, 0.3046523928642273], "agreement_weight": 0.07457628101110458, "orthogonal_weight": -0.32553091645240784}, {"head": 2, "memory_to_active": 0.01860992982983589, "contribution_to_active": 0.9553180932998657, "mix_mean": 0.45757532119750977, "mix_low": 0.15978240966796875, "mix_high": 0.1416778564453125, "read_cosine": 0.23434816300868988, "beta_mean": 0.0735003799200058, "nonfinite": 0.0, "weighted_memory_to_active": 0.014258678071200848, "mix_std": 0.43077149987220764, "beta_std": 0.09723057597875595, "beta_low": 0.44183349609375, "beta_high": 0.0, "orthogonal_to_active": 0.06492431461811066, "evidence_weights": [-0.0038441072683781385, -0.4588782787322998, -1.2430497407913208], "agreement_weight": 0.00020455602498259395, "orthogonal_weight": 4.772167682647705}, {"head": 3, "memory_to_active": 0.012891290709376335, "contribution_to_active": 0.8550679683685303, "mix_mean": 0.4137541651725769, "mix_low": 0.2643871307373047, "mix_high": 0.10064983367919922, "read_cosine": 0.5016942024230957, "beta_mean": 0.06358128041028976, "nonfinite": 0.0, "weighted_memory_to_active": 0.00785853248089552, "mix_std": 0.4362127482891083, "beta_std": 0.09124840050935745, "beta_low": 0.46588134765625, "beta_high": 0.0, "orthogonal_to_active": 0.03048625960946083, "evidence_weights": [0.06436333060264587, -0.3513823449611664, -0.43108701705932617], "agreement_weight": 0.041015855967998505, "orthogonal_weight": 5.099434852600098}, {"head": 4, "memory_to_active": 0.4129638671875, "contribution_to_active": 0.37514692544937134, "mix_mean": 0.41855499148368835, "mix_low": 0.013174057006835938, "mix_high": 0.0004367828369140625, "read_cosine": 0.1444764882326126, "beta_mean": 0.027335699647665024, "nonfinite": 0.0, "weighted_memory_to_active": 0.18449857831001282, "mix_std": 0.2679874897003174, "beta_std": 0.025670504197478294, "beta_low": 0.22125244140625, "beta_high": 0.0, "orthogonal_to_active": 0.15127308666706085, "evidence_weights": [-0.2029470056295395, -0.17090003192424774, 1.6702595949172974], "agreement_weight": -0.1651577204465866, "orthogonal_weight": -0.8412390351295471}, {"head": 5, "memory_to_active": 0.09502872824668884, "contribution_to_active": 0.36272385716438293, "mix_mean": 0.4874557554721832, "mix_low": 0.00058746337890625, "mix_high": 0.0017957687377929688, "read_cosine": 0.7922080755233765, "beta_mean": 0.007573658600449562, "nonfinite": 0.0, "weighted_memory_to_active": 0.04130168259143829, "mix_std": 0.26184719800949097, "beta_std": 0.01113760843873024, "beta_low": 0.772216796875, "beta_high": 0.0, "orthogonal_to_active": 0.018858226016163826, "evidence_weights": [-0.21310453116893768, -0.27304935455322266, 0.5286635756492615], "agreement_weight": 0.06660797446966171, "orthogonal_weight": -0.6869929432868958}]}, {"layer": 12, "memory_to_active": 0.3064292371273041, "contribution_to_active": 0.5535403490066528, "mix_mean": 0.4751858115196228, "mix_low": 0.013211727142333984, "mix_high": 0.0034739177208393812, "read_cosine": 0.2722568213939667, "beta_mean": 0.017259974032640457, "nonfinite": 0.0, "weighted_memory_to_active": 0.16759079694747925, "mix_std": 0.2582222819328308, "beta_std": 0.016996879130601883, "beta_low": 0.8216654658317566, "beta_high": 0.0, "orthogonal_to_active": 0.06569403409957886, "output_influence": 0.4313454031944275, "residual_influence": 0.12032577395439148, "state_active": 0.03890695795416832, "state_memory": 0.009783752262592316, "state_peak": 29.78433609008789, "growth_active": 1.124530553817749, "growth_memory": 1.2237156629562378, "heads": [{"head": 0, "memory_to_active": 0.057959891855716705, "contribution_to_active": 0.4605841636657715, "mix_mean": 0.4419930577278137, "mix_low": 0.0023345947265625, "mix_high": 0.0005121231079101562, "read_cosine": 0.3387506902217865, "beta_mean": 0.0007961830124258995, "nonfinite": 0.0, "weighted_memory_to_active": 0.024354474619030952, "mix_std": 0.26777639985084534, "beta_std": 0.00336612481623888, "beta_low": 0.99652099609375, "beta_high": 0.0, "orthogonal_to_active": 0.014975924044847488, "evidence_weights": [-0.17225395143032074, -0.24616791307926178, 0.2938516438007355], "agreement_weight": 0.10616350173950195, "orthogonal_weight": -0.658456027507782}, {"head": 1, "memory_to_active": 0.08192944526672363, "contribution_to_active": 0.49229896068573, "mix_mean": 0.5036062002182007, "mix_low": 0.0007295608520507812, "mix_high": 0.0003795623779296875, "read_cosine": 0.5532264709472656, "beta_mean": 0.0008084975415840745, "nonfinite": 0.0, "weighted_memory_to_active": 0.04062449932098389, "mix_std": 0.26261937618255615, "beta_std": 0.004920640494674444, "beta_low": 0.99383544921875, "beta_high": 0.0, "orthogonal_to_active": 0.03175486624240875, "evidence_weights": [-0.20227015018463135, -0.2674589157104492, 0.8913212418556213], "agreement_weight": 0.11933349072933197, "orthogonal_weight": -0.921448290348053}, {"head": 2, "memory_to_active": 0.6144331097602844, "contribution_to_active": 0.451310932636261, "mix_mean": 0.31539833545684814, "mix_low": 0.02738666534423828, "mix_high": 0.0005769729614257812, "read_cosine": 0.4811502695083618, "beta_mean": 0.09869524091482162, "nonfinite": 0.0, "weighted_memory_to_active": 0.24697048962116241, "mix_std": 0.2438936084508896, "beta_std": 0.06654868274927139, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.21207484602928162, "evidence_weights": [-0.24373403191566467, -0.04430616647005081, -0.09116099029779434], "agreement_weight": 0.017940860241651535, "orthogonal_weight": 1.0716198682785034}, {"head": 3, "memory_to_active": 0.038490794599056244, "contribution_to_active": 0.7285017967224121, "mix_mean": 0.6928385496139526, "mix_low": 0.0002269744873046875, "mix_high": 0.014865875244140625, "read_cosine": 0.08224724233150482, "beta_mean": 0.00033783342223614454, "nonfinite": 0.0, "weighted_memory_to_active": 0.02851628139615059, "mix_std": 0.23806405067443848, "beta_std": 0.005322680808603764, "beta_low": 0.99591064453125, "beta_high": 0.0, "orthogonal_to_active": 0.026278167963027954, "evidence_weights": [-0.2978285253047943, -0.3361763060092926, 1.180423617362976], "agreement_weight": 0.3011572062969208, "orthogonal_weight": -0.9407379627227783}, {"head": 4, "memory_to_active": 0.9794966578483582, "contribution_to_active": 0.7191910743713379, "mix_mean": 0.29600614309310913, "mix_low": 0.04201507568359375, "mix_high": 0.0016384124755859375, "read_cosine": -0.08968095481395721, "beta_mean": 0.002437235787510872, "nonfinite": 0.0, "weighted_memory_to_active": 0.6241112351417542, "mix_std": 0.26295483112335205, "beta_std": 0.019213654100894928, "beta_low": 0.951904296875, "beta_high": 0.0, "orthogonal_to_active": 0.0773906260728836, "evidence_weights": [-0.0772600919008255, -0.20413239300251007, 0.04960362985730171], "agreement_weight": 0.0001421568013029173, "orthogonal_weight": -0.13366520404815674}, {"head": 5, "memory_to_active": 0.06626541912555695, "contribution_to_active": 0.46935516595840454, "mix_mean": 0.601272463798523, "mix_low": 0.006577491760253906, "mix_high": 0.0028705596923828125, "read_cosine": 0.26784706115722656, "beta_mean": 0.0004848490352742374, "nonfinite": 0.0, "weighted_memory_to_active": 0.040967825800180435, "mix_std": 0.2740253210067749, "beta_std": 0.0026094899512827396, "beta_low": 0.99163818359375, "beta_high": 0.0, "orthogonal_to_active": 0.03168979287147522, "evidence_weights": [-0.2602613866329193, -0.3051150441169739, 1.0986648797988892], "agreement_weight": 0.23237578570842743, "orthogonal_weight": -0.8399977684020996}]}, {"layer": 13, "memory_to_active": 0.1436919867992401, "contribution_to_active": 0.5074065923690796, "mix_mean": 0.504279375076294, "mix_low": 0.0051089925691485405, "mix_high": 0.004283746238797903, "read_cosine": 0.41157281398773193, "beta_mean": 0.01745232194662094, "nonfinite": 0.0, "weighted_memory_to_active": 0.06219296157360077, "mix_std": 0.2584940195083618, "beta_std": 0.017582032829523087, "beta_low": 0.7915548086166382, "beta_high": 0.0, "orthogonal_to_active": 0.061195820569992065, "output_influence": 0.4701465964317322, "residual_influence": 0.15328970551490784, "state_active": 0.09523474425077438, "state_memory": 0.037359852343797684, "state_peak": 96.39044952392578, "growth_active": 0.9569942355155945, "growth_memory": 1.027302622795105, "heads": [{"head": 0, "memory_to_active": 0.03619002550840378, "contribution_to_active": 0.5493093132972717, "mix_mean": 0.547195315361023, "mix_low": 0.00037670135498046875, "mix_high": 0.0013589859008789062, "read_cosine": 0.3768932819366455, "beta_mean": 0.00012925005285069346, "nonfinite": 0.0, "weighted_memory_to_active": 0.021243810653686523, "mix_std": 0.26557838916778564, "beta_std": 0.0003129533724859357, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0206151120364666, "evidence_weights": [-0.2031412571668625, -0.25972381234169006, 0.7304168343544006], "agreement_weight": 0.08153397589921951, "orthogonal_weight": -1.0746972560882568}, {"head": 1, "memory_to_active": 0.02850419096648693, "contribution_to_active": 0.5796382427215576, "mix_mean": 0.523378312587738, "mix_low": 0.0018224716186523438, "mix_high": 0.00167083740234375, "read_cosine": 0.19470801949501038, "beta_mean": 9.70320834312588e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01624365523457527, "mix_std": 0.2694430351257324, "beta_std": 0.0001762808533385396, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.017644720152020454, "evidence_weights": [-0.18675559759140015, -0.2682288587093353, 0.6920132040977478], "agreement_weight": 0.25886204838752747, "orthogonal_weight": -1.1395832300186157}, {"head": 2, "memory_to_active": 0.11320473253726959, "contribution_to_active": 0.40958070755004883, "mix_mean": 0.36158013343811035, "mix_low": 0.0027751922607421875, "mix_high": 0.0001068115234375, "read_cosine": 0.6445646286010742, "beta_mean": 0.014097321778535843, "nonfinite": 0.0, "weighted_memory_to_active": 0.047529105097055435, "mix_std": 0.22210069000720978, "beta_std": 0.04056817665696144, "beta_low": 0.76861572265625, "beta_high": 0.0, "orthogonal_to_active": 0.02357708476483822, "evidence_weights": [-0.2120472639799118, -0.1778121143579483, 0.44744548201560974], "agreement_weight": 0.020427582785487175, "orthogonal_weight": -0.7930780053138733}, {"head": 3, "memory_to_active": 0.14270201325416565, "contribution_to_active": 0.578076183795929, "mix_mean": 0.6200664043426514, "mix_low": 0.014753341674804688, "mix_high": 0.015898704528808594, "read_cosine": 0.04600511118769646, "beta_mean": 0.0012184863444417715, "nonfinite": 0.0, "weighted_memory_to_active": 0.10397893190383911, "mix_std": 0.2862849533557892, "beta_std": 0.005208749324083328, "beta_low": 0.98828125, "beta_high": 0.0, "orthogonal_to_active": 0.10407188534736633, "evidence_weights": [-0.33737242221832275, -0.23706907033920288, 2.005675792694092], "agreement_weight": 3.1108508110046387, "orthogonal_weight": -1.0647172927856445}, {"head": 4, "memory_to_active": 0.4349076449871063, "contribution_to_active": 0.2693273425102234, "mix_mean": 0.36671000719070435, "mix_low": 0.008623123168945312, "mix_high": 0.00019359588623046875, "read_cosine": 0.8816352486610413, "beta_mean": 0.08832485973834991, "nonfinite": 0.0, "weighted_memory_to_active": 0.11659971624612808, "mix_std": 0.22988224029541016, "beta_std": 0.05438244342803955, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14789801836013794, "evidence_weights": [-0.15377840399742126, -0.23249505460262299, 0.49735718965530396], "agreement_weight": -0.0017488456796854734, "orthogonal_weight": 1.9729644060134888}, {"head": 5, "memory_to_active": 0.10664329677820206, "contribution_to_active": 0.6585078239440918, "mix_mean": 0.6067458987236023, "mix_low": 0.0023031234741210938, "mix_high": 0.006473541259765625, "read_cosine": 0.32563066482543945, "beta_mean": 0.0008469857275485992, "nonfinite": 0.0, "weighted_memory_to_active": 0.067562535405159, "mix_std": 0.2776748538017273, "beta_std": 0.004843577742576599, "beta_low": 0.992431640625, "beta_high": 0.0, "orthogonal_to_active": 0.05336809903383255, "evidence_weights": [-0.22049108147621155, -0.29194021224975586, 0.6780566573143005], "agreement_weight": 0.09946560114622116, "orthogonal_weight": -0.8860843777656555}]}, {"layer": 14, "memory_to_active": 0.2543014883995056, "contribution_to_active": 0.42298752069473267, "mix_mean": 0.4137495458126068, "mix_low": 0.02723662182688713, "mix_high": 0.004198869224637747, "read_cosine": 0.6292870044708252, "beta_mean": 0.06707219779491425, "nonfinite": 0.0, "weighted_memory_to_active": 0.11606165021657944, "mix_std": 0.24824482202529907, "beta_std": 0.06114339828491211, "beta_low": 0.4545695185661316, "beta_high": 0.0, "orthogonal_to_active": 0.04194304347038269, "output_influence": 0.30686256289482117, "residual_influence": 0.11832398921251297, "state_active": 0.06604838371276855, "state_memory": 0.015523144043982029, "state_peak": 33.1956901550293, "growth_active": 1.1505709886550903, "growth_memory": 1.3150967359542847, "heads": [{"head": 0, "memory_to_active": 0.3429398536682129, "contribution_to_active": 0.20704156160354614, "mix_mean": 0.23233355581760406, "mix_low": 0.011503219604492188, "mix_high": 2.384185791015625e-05, "read_cosine": 0.7794145345687866, "beta_mean": 0.06311871111392975, "nonfinite": 0.0, "weighted_memory_to_active": 0.09403583407402039, "mix_std": 0.20335440337657928, "beta_std": 0.058766596019268036, "beta_low": 0.01568603515625, "beta_high": 0.0, "orthogonal_to_active": 0.057210005819797516, "evidence_weights": [-0.12929829955101013, -0.05172387883067131, 0.6234015226364136], "agreement_weight": 0.02479362115263939, "orthogonal_weight": 0.9634294509887695}, {"head": 1, "memory_to_active": 0.49376416206359863, "contribution_to_active": 0.3491268754005432, "mix_mean": 0.23630867898464203, "mix_low": 0.13441181182861328, "mix_high": 6.961822509765625e-05, "read_cosine": 0.7716538906097412, "beta_mean": 0.2765638530254364, "nonfinite": 0.0, "weighted_memory_to_active": 0.24051128327846527, "mix_std": 0.26192721724510193, "beta_std": 0.2492443323135376, "beta_low": 0.0863037109375, "beta_high": 0.0, "orthogonal_to_active": 0.016225267201662064, "evidence_weights": [0.0037201126106083393, -0.25151902437210083, -0.18340370059013367], "agreement_weight": 0.05494581535458565, "orthogonal_weight": 0.15307791531085968}, {"head": 2, "memory_to_active": 0.17136569321155548, "contribution_to_active": 0.5365654230117798, "mix_mean": 0.5505625605583191, "mix_low": 0.0032148361206054688, "mix_high": 0.0021734237670898438, "read_cosine": 0.4115920066833496, "beta_mean": 0.004265656694769859, "nonfinite": 0.0, "weighted_memory_to_active": 0.10521093755960464, "mix_std": 0.27042946219444275, "beta_std": 0.005807775072753429, "beta_low": 0.906494140625, "beta_high": 0.0, "orthogonal_to_active": 0.07869482785463333, "evidence_weights": [-0.16103456914424896, -0.2928090989589691, 0.8490200042724609], "agreement_weight": 0.07789676636457443, "orthogonal_weight": -0.8582330346107483}, {"head": 3, "memory_to_active": 0.06375374644994736, "contribution_to_active": 0.65076744556427, "mix_mean": 0.6683334112167358, "mix_low": 0.004351615905761719, "mix_high": 0.015299797058105469, "read_cosine": 0.25946351885795593, "beta_mean": 0.008448250591754913, "nonfinite": 0.0, "weighted_memory_to_active": 0.04298066347837448, "mix_std": 0.2450103759765625, "beta_std": 0.008779908530414104, "beta_low": 0.72015380859375, "beta_high": 0.0, "orthogonal_to_active": 0.024798844009637833, "evidence_weights": [-0.3759467601776123, -0.3245164752006531, 0.4454162120819092], "agreement_weight": 0.08146706968545914, "orthogonal_weight": -0.6940356492996216}, {"head": 4, "memory_to_active": 0.16364923119544983, "contribution_to_active": 0.4759102463722229, "mix_mean": 0.4669060707092285, "mix_low": 0.004332542419433594, "mix_high": 0.0068607330322265625, "read_cosine": 0.8256605267524719, "beta_mean": 0.0480487234890461, "nonfinite": 0.0, "weighted_memory_to_active": 0.09212448447942734, "mix_std": 0.262787401676178, "beta_std": 0.035624176263809204, "beta_low": 0.028564453125, "beta_high": 0.0, "orthogonal_to_active": 0.02785923145711422, "evidence_weights": [-0.21495258808135986, -0.2575831413269043, 0.4140195846557617], "agreement_weight": 0.12096571177244186, "orthogonal_weight": -0.5626053214073181}, {"head": 5, "memory_to_active": 0.29033613204956055, "contribution_to_active": 0.3185131251811981, "mix_mean": 0.32805269956588745, "mix_low": 0.0056056976318359375, "mix_high": 0.0007658004760742188, "read_cosine": 0.7279375791549683, "beta_mean": 0.0019879683386534452, "nonfinite": 0.0, "weighted_memory_to_active": 0.12150667607784271, "mix_std": 0.24596007168293, "beta_std": 0.008637606166303158, "beta_low": 0.97021484375, "beta_high": 0.0, "orthogonal_to_active": 0.04687006399035454, "evidence_weights": [-0.18316993117332458, -0.15568912029266357, 0.21072833240032196], "agreement_weight": 0.08399689197540283, "orthogonal_weight": -0.6258331537246704}]}, {"layer": 15, "memory_to_active": 0.06188469007611275, "contribution_to_active": 0.4790281653404236, "mix_mean": 0.39918026328086853, "mix_low": 0.008929888717830181, "mix_high": 0.0026135446969419718, "read_cosine": 0.674480140209198, "beta_mean": 0.03658430650830269, "nonfinite": 0.0, "weighted_memory_to_active": 0.027968335896730423, "mix_std": 0.28159216046333313, "beta_std": 0.04161261394619942, "beta_low": 0.7159220576286316, "beta_high": 0.0, "orthogonal_to_active": 0.005600694566965103, "output_influence": 0.29777926206588745, "residual_influence": 0.15955303609371185, "state_active": 0.12252001464366913, "state_memory": 0.008499233983457088, "state_peak": 38.09563446044922, "growth_active": 1.2883222103118896, "growth_memory": 1.9252240657806396, "heads": [{"head": 0, "memory_to_active": 0.09547232836484909, "contribution_to_active": 0.4523921608924866, "mix_mean": 0.3630996346473694, "mix_low": 0.005620002746582031, "mix_high": 0.00396728515625, "read_cosine": 0.71580970287323, "beta_mean": 0.033066526055336, "nonfinite": 0.0, "weighted_memory_to_active": 0.04332364723086357, "mix_std": 0.2743162214756012, "beta_std": 0.025090094655752182, "beta_low": 0.14019775390625, "beta_high": 0.0, "orthogonal_to_active": 0.01667685993015766, "evidence_weights": [-0.1634899079799652, -0.1794985979795456, 0.3377000689506531], "agreement_weight": 0.08924630284309387, "orthogonal_weight": -0.6423556208610535}, {"head": 1, "memory_to_active": 0.16000837087631226, "contribution_to_active": 0.4175288677215576, "mix_mean": 0.4032750427722931, "mix_low": 0.0041408538818359375, "mix_high": 0.0035905838012695312, "read_cosine": 0.8877587914466858, "beta_mean": 0.001389560755342245, "nonfinite": 0.0, "weighted_memory_to_active": 0.06459792703390121, "mix_std": 0.2807198762893677, "beta_std": 0.0041177221573889256, "beta_low": 0.9737548828125, "beta_high": 0.0, "orthogonal_to_active": 0.0015669383574277163, "evidence_weights": [-0.17408116161823273, -0.18235798180103302, 0.26032859086990356], "agreement_weight": 0.09909127652645111, "orthogonal_weight": -0.04031750187277794}, {"head": 2, "memory_to_active": 0.0054463897831737995, "contribution_to_active": 0.45574408769607544, "mix_mean": 0.34555208683013916, "mix_low": 0.016831398010253906, "mix_high": 3.528594970703125e-05, "read_cosine": 0.7631880044937134, "beta_mean": 8.372778393095359e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002470677951350808, "mix_std": 0.27252835035324097, "beta_std": 0.00012563193740788847, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00019644387066364288, "evidence_weights": [-0.20718823373317719, -0.27908679842948914, 0.2731391489505768], "agreement_weight": 0.007141087204217911, "orthogonal_weight": 0.17121005058288574}, {"head": 3, "memory_to_active": 0.014194106683135033, "contribution_to_active": 0.4337667226791382, "mix_mean": 0.3541460633277893, "mix_low": 0.0063724517822265625, "mix_high": 0.00016117095947265625, "read_cosine": 0.6201721429824829, "beta_mean": 0.0001333835971308872, "nonfinite": 0.0, "weighted_memory_to_active": 0.006503524724394083, "mix_std": 0.2770097255706787, "beta_std": 0.00039854220813140273, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.004358208272606134, "evidence_weights": [-0.20960000157356262, -0.2560645341873169, 0.39215585589408875], "agreement_weight": 0.027110369876027107, "orthogonal_weight": -0.9228944182395935}, {"head": 4, "memory_to_active": 0.08040044456720352, "contribution_to_active": 0.6093284487724304, "mix_mean": 0.38109564781188965, "mix_low": 0.011016845703125, "mix_high": 0.0008039474487304688, "read_cosine": 0.6777153015136719, "beta_mean": 0.18419376015663147, "nonfinite": 0.0, "weighted_memory_to_active": 0.040700241923332214, "mix_std": 0.30600041151046753, "beta_std": 0.21506746113300323, "beta_low": 0.19384765625, "beta_high": 0.0, "orthogonal_to_active": 0.0038579984102398157, "evidence_weights": [-0.22332462668418884, -0.25654497742652893, 0.03453315794467926], "agreement_weight": 0.013217857107520103, "orthogonal_weight": 0.18491442501544952}, {"head": 5, "memory_to_active": 0.015786485746502876, "contribution_to_active": 0.5054084658622742, "mix_mean": 0.547913134098053, "mix_low": 0.0095977783203125, "mix_high": 0.007122993469238281, "read_cosine": 0.38223662972450256, "beta_mean": 0.0006388798356056213, "nonfinite": 0.0, "weighted_memory_to_active": 0.010214011184871197, "mix_std": 0.2789783179759979, "beta_std": 0.004876206628978252, "beta_low": 0.9879150390625, "beta_high": 0.0, "orthogonal_to_active": 0.006947716232389212, "evidence_weights": [-0.33814892172813416, -0.30700618028640747, 0.6399625539779663], "agreement_weight": 0.25554317235946655, "orthogonal_weight": 0.8257458209991455}]}]}, {"step": 29000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.830405950997374, "memory_to_active": 0.8913248181343079, "contribution_to_active": 0.5099402070045471, "mix_mean": 0.36010611057281494, "mix_low": 0.15451255440711975, "mix_high": 0.007807960733771324, "read_cosine": 0.3789002299308777, "beta_mean": 0.08418594300746918, "nonfinite": 0.0, "weighted_memory_to_active": 0.212289959192276, "mix_std": 0.21597948670387268, "beta_std": 0.07008630782365799, "beta_low": 0.493843138217926, "beta_high": 1.4623006791225635e-05, "orthogonal_to_active": 0.05133168026804924, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1439688205718994, "normal_token_accuracy": 0.42602890729904175, "state_active": 0.06064805015921593, "state_memory": 0.019633576273918152, "state_peak": 115.20490264892578, "growth_active": 1.1471309661865234, "growth_memory": 1.366835594177246, "output_influence": 0.3644735813140869, "residual_influence": 0.11668159067630768, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2145137786865234, "no_memory_loss_delta": 0.32532769441604614, "scrambled_loss_delta": 0.20585647225379944, "memory_only_loss_delta": 2.497934341430664, "no_evidence_loss_delta": 0.18646089732646942, "no_agreement_loss_delta": 0.019015047699213028, "no_orthogonal_loss_delta": 0.006962003652006388}, {"index": 94, "tokens": 256, "normal_loss": 3.197521448135376, "no_memory_loss_delta": 0.3942020833492279, "scrambled_loss_delta": 0.37942826747894287, "memory_only_loss_delta": 2.632180690765381, "no_evidence_loss_delta": 0.18885347247123718, "no_agreement_loss_delta": 0.035050343722105026, "no_orthogonal_loss_delta": 0.024794287979602814}, {"index": 188, "tokens": 256, "normal_loss": 3.555269718170166, "no_memory_loss_delta": 0.43511146306991577, "scrambled_loss_delta": 0.15526947379112244, "memory_only_loss_delta": 2.6091465950012207, "no_evidence_loss_delta": 0.23607125878334045, "no_agreement_loss_delta": 0.02471383474767208, "no_orthogonal_loss_delta": 0.01583993434906006}, {"index": 283, "tokens": 131, "normal_loss": 2.097693681716919, "no_memory_loss_delta": 0.3000999987125397, "scrambled_loss_delta": 0.09314822405576706, "memory_only_loss_delta": 2.7769322395324707, "no_evidence_loss_delta": 0.11401532590389252, "no_agreement_loss_delta": -0.018892360851168633, "no_orthogonal_loss_delta": -0.03897910192608833}], "agreement_weight_abs_mean": 0.14050906896591187, "agreement_weight_abs_max": 3.2501022815704346, "agreement_weight_mean": 0.1203974187374115, "orthogonal_weight_abs_mean": 0.6518946886062622, "orthogonal_weight_abs_max": 5.269676685333252, "orthogonal_weight_mean": -0.15854273736476898, "no_memory_loss": 3.5164954662323, "no_memory_loss_delta": 0.3725264370441437, "no_memory_token_accuracy_gain": 0.04783092439174652, "no_memory_token_win_fraction": 0.6340377926826477, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 31.100856357991454, "scrambled_loss": 3.3684232234954834, "scrambled_loss_delta": 0.2244541496038437, "scrambled_token_accuracy_gain": 0.015572858974337578, "scrambled_token_win_fraction": 0.6129032373428345, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.104779186461446, "memory_only_loss": 5.752455711364746, "memory_only_loss_delta": 2.6084864139556885, "memory_only_token_accuracy_gain": 0.256952166557312, "memory_only_token_win_fraction": 0.8954393863677979, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 92.63540711035013, "no_evidence_loss": 3.334681749343872, "no_evidence_loss_delta": 0.19071272015571594, "no_evidence_token_accuracy_gain": 0.02002224698662758, "no_evidence_token_win_fraction": 0.5984427332878113, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 17.363004651401837, "no_agreement_loss": 3.163649082183838, "no_agreement_loss_delta": 0.019680293276906013, "no_agreement_token_accuracy_gain": -0.0055617354810237885, "no_agreement_token_win_fraction": 0.4716351628303528, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.9487900487857863, "no_orthogonal_loss": 3.1518430709838867, "no_orthogonal_loss_delta": 0.00787360593676567, "no_orthogonal_token_accuracy_gain": 0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5317018628120422, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.7842690293948419, "short_no_memory_loss_delta": 0.33580803871154785, "long_context_memory_gain": 0.036718398332595825, "normal_context_loss_gain": 0.14547578990459442, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.247304916381836, "contribution_to_active": 1.0092124938964844, "mix_mean": 0.09827975928783417, "mix_low": 0.5230700373649597, "mix_high": 0.003834724659100175, "read_cosine": 0.4615500271320343, "beta_mean": 0.25926750898361206, "nonfinite": 0.0, "weighted_memory_to_active": 1.0738624334335327, "mix_std": 0.08690702170133591, "beta_std": 0.15491275489330292, "beta_low": 0.0167032890021801, "beta_high": 0.0, "orthogonal_to_active": 0.06228676810860634, "output_influence": 0.15968087315559387, "residual_influence": 0.19089354574680328, "state_active": 0.07425591349601746, "state_memory": 0.04860732704401016, "state_peak": 41.85260009765625, "growth_active": 1.2617533206939697, "growth_memory": 1.2456574440002441, "heads": [{"head": 0, "memory_to_active": 2.458202362060547, "contribution_to_active": 0.40853187441825867, "mix_mean": 0.020286887884140015, "mix_low": 0.5826854705810547, "mix_high": 0.0, "read_cosine": 0.2707412838935852, "beta_mean": 0.209159255027771, "nonfinite": 0.0, "weighted_memory_to_active": 0.48781687021255493, "mix_std": 0.05713675171136856, "beta_std": 0.10028640180826187, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03767414763569832, "evidence_weights": [-0.029925493523478508, -0.07802262902259827, -0.09362576901912689], "agreement_weight": -0.0990971252322197, "orthogonal_weight": -0.08710630983114243}, {"head": 1, "memory_to_active": 5.1972761154174805, "contribution_to_active": 1.988581895828247, "mix_mean": 0.015616076998412609, "mix_low": 0.6480007171630859, "mix_high": 0.0, "read_cosine": 0.5072216987609863, "beta_mean": 0.4158129394054413, "nonfinite": 0.0, "weighted_memory_to_active": 2.372013807296753, "mix_std": 0.048791635781526566, "beta_std": 0.1830715537071228, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14636221528053284, "evidence_weights": [0.02050831913948059, -0.08802676200866699, 0.32341426610946655], "agreement_weight": 0.027591703459620476, "orthogonal_weight": -0.13819539546966553}, {"head": 2, "memory_to_active": 2.6637771129608154, "contribution_to_active": 0.5047829747200012, "mix_mean": 0.014519363641738892, "mix_low": 0.6385040283203125, "mix_high": 0.0, "read_cosine": 0.3475545048713684, "beta_mean": 0.3369015157222748, "nonfinite": 0.0, "weighted_memory_to_active": 0.591526210308075, "mix_std": 0.03148951753973961, "beta_std": 0.23844033479690552, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05304839462041855, "evidence_weights": [0.04805823042988777, 0.06149936839938164, -0.02369493804872036], "agreement_weight": 0.008018478751182556, "orthogonal_weight": -0.18370260298252106}, {"head": 3, "memory_to_active": 6.198537826538086, "contribution_to_active": 2.721254348754883, "mix_mean": 0.04001978412270546, "mix_low": 0.42046165466308594, "mix_high": 0.0, "read_cosine": 0.3834506869316101, "beta_mean": 0.3457260727882385, "nonfinite": 0.0, "weighted_memory_to_active": 2.8909215927124023, "mix_std": 0.08737621456384659, "beta_std": 0.23040428757667542, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11553838849067688, "evidence_weights": [-0.06220419704914093, -0.08367518335580826, -0.11648441851139069], "agreement_weight": -0.01659521833062172, "orthogonal_weight": -0.04821600392460823}, {"head": 4, "memory_to_active": 2.8590307235717773, "contribution_to_active": 0.05965661257505417, "mix_mean": 0.00609398540109396, "mix_low": 0.833765983581543, "mix_high": 0.0, "read_cosine": 0.32666945457458496, "beta_mean": 0.20533761382102966, "nonfinite": 0.0, "weighted_memory_to_active": 0.05323103442788124, "mix_std": 0.012374203652143478, "beta_std": 0.13938036561012268, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005566312000155449, "evidence_weights": [0.11832530051469803, 0.06715528666973114, -0.36576753854751587], "agreement_weight": -0.10386224836111069, "orthogonal_weight": 0.10630939900875092}, {"head": 5, "memory_to_active": 0.10700355470180511, "contribution_to_active": 0.3724668622016907, "mix_mean": 0.4931424558162689, "mix_low": 0.015002250671386719, "mix_high": 0.023008346557617188, "read_cosine": 0.9336625337600708, "beta_mean": 0.042667537927627563, "nonfinite": 0.0, "weighted_memory_to_active": 0.04766528308391571, "mix_std": 0.28427383303642273, "beta_std": 0.03789358213543892, "beta_low": 0.1002197265625, "beta_high": 0.0, "orthogonal_to_active": 0.015531144104897976, "evidence_weights": [-0.24989421665668488, -0.24634088575839996, 0.260427862405777], "agreement_weight": 0.2315426766872406, "orthogonal_weight": -0.616481602191925}]}, {"layer": 1, "memory_to_active": 2.243778944015503, "contribution_to_active": 0.5013806819915771, "mix_mean": 0.17064940929412842, "mix_low": 0.36479032039642334, "mix_high": 0.0007026990642771125, "read_cosine": 0.34271207451820374, "beta_mean": 0.1994064599275589, "nonfinite": 0.0, "weighted_memory_to_active": 0.38021451234817505, "mix_std": 0.12089907377958298, "beta_std": 0.1301494985818863, "beta_low": 0.2054850310087204, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.051707424223423004, "output_influence": 0.1848735809326172, "residual_influence": 0.04563681408762932, "state_active": 0.04168885946273804, "state_memory": 0.036672983318567276, "state_peak": 29.4238338470459, "growth_active": 1.2547252178192139, "growth_memory": 1.4859302043914795, "heads": [{"head": 0, "memory_to_active": 0.04696572571992874, "contribution_to_active": 0.4025351405143738, "mix_mean": 0.405788779258728, "mix_low": 0.011765480041503906, "mix_high": 0.000331878662109375, "read_cosine": 0.053062405437231064, "beta_mean": 0.00033844925928860903, "nonfinite": 0.0, "weighted_memory_to_active": 0.024332590401172638, "mix_std": 0.2790192663669586, "beta_std": 0.00043153573642484844, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.02253313735127449, "evidence_weights": [-0.17911392450332642, -0.22967657446861267, 0.017790766432881355], "agreement_weight": 0.12443525344133377, "orthogonal_weight": -1.0091983079910278}, {"head": 1, "memory_to_active": 2.9150960445404053, "contribution_to_active": 0.225713312625885, "mix_mean": 0.022413033992052078, "mix_low": 0.47604942321777344, "mix_high": 9.5367431640625e-07, "read_cosine": 0.47310006618499756, "beta_mean": 0.3118063807487488, "nonfinite": 0.0, "weighted_memory_to_active": 0.24883760511875153, "mix_std": 0.04665452241897583, "beta_std": 0.1893918216228485, "beta_low": 6.103515625e-05, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.011775960214436054, "evidence_weights": [0.008252655155956745, -0.018625682219862938, -0.009964929893612862], "agreement_weight": -0.009273608215153217, "orthogonal_weight": 0.1009432002902031}, {"head": 2, "memory_to_active": 0.3565582036972046, "contribution_to_active": 0.535408079624176, "mix_mean": 0.5310457944869995, "mix_low": 0.002292633056640625, "mix_high": 0.0038738250732421875, "read_cosine": 0.7569934129714966, "beta_mean": 0.029916908591985703, "nonfinite": 0.0, "weighted_memory_to_active": 0.2883749008178711, "mix_std": 0.2602468729019165, "beta_std": 0.027123233303427696, "beta_low": 0.20452880859375, "beta_high": 0.0, "orthogonal_to_active": 0.0421936996281147, "evidence_weights": [-0.27978673577308655, -0.3724755048751831, 0.30217745900154114], "agreement_weight": 0.028601735830307007, "orthogonal_weight": -0.2813352048397064}, {"head": 3, "memory_to_active": 2.8371710777282715, "contribution_to_active": 0.13226056098937988, "mix_mean": 0.01690678298473358, "mix_low": 0.6543378829956055, "mix_high": 0.0, "read_cosine": 0.23915940523147583, "beta_mean": 0.3583117127418518, "nonfinite": 0.0, "weighted_memory_to_active": 0.19769476354122162, "mix_std": 0.04593782126903534, "beta_std": 0.24319422245025635, "beta_low": 0.01824951171875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.05609307438135147, "evidence_weights": [0.04238018020987511, -0.0007646809099242091, 0.11301673948764801], "agreement_weight": -0.03391589969396591, "orthogonal_weight": -0.3296595513820648}, {"head": 4, "memory_to_active": 3.411471366882324, "contribution_to_active": 1.6382145881652832, "mix_mean": 0.03888554871082306, "mix_low": 0.2589111328125, "mix_high": 4.76837158203125e-06, "read_cosine": 0.3169117271900177, "beta_mean": 0.17359009385108948, "nonfinite": 0.0, "weighted_memory_to_active": 1.4581239223480225, "mix_std": 0.06928643584251404, "beta_std": 0.10511758923530579, "beta_low": 0.00164794921875, "beta_high": 0.0, "orthogonal_to_active": 0.1719495803117752, "evidence_weights": [-0.04254906624555588, 0.00029577757231891155, -0.010577278211712837], "agreement_weight": -0.028370311483740807, "orthogonal_weight": 0.14613932371139526}, {"head": 5, "memory_to_active": 3.8954105377197266, "contribution_to_active": 0.07415225356817245, "mix_mean": 0.008856420405209064, "mix_low": 0.7853851318359375, "mix_high": 4.76837158203125e-06, "read_cosine": 0.21704532206058502, "beta_mean": 0.3224751949310303, "nonfinite": 0.0, "weighted_memory_to_active": 0.06392336636781693, "mix_std": 0.024249553680419922, "beta_std": 0.21563857793807983, "beta_low": 0.00848388671875, "beta_high": 0.0, "orthogonal_to_active": 0.005699096247553825, "evidence_weights": [0.150136336684227, 0.06798485666513443, 0.26527005434036255], "agreement_weight": -0.161690816283226, "orthogonal_weight": 0.08620119839906693}]}, {"layer": 2, "memory_to_active": 1.0175246000289917, "contribution_to_active": 0.34003692865371704, "mix_mean": 0.34394192695617676, "mix_low": 0.24917587637901306, "mix_high": 0.003413359634578228, "read_cosine": 0.33935242891311646, "beta_mean": 0.09790260344743729, "nonfinite": 0.0, "weighted_memory_to_active": 0.07821820676326752, "mix_std": 0.1977453976869583, "beta_std": 0.07197564840316772, "beta_low": 0.5461832880973816, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.01731691136956215, "output_influence": 0.32754701375961304, "residual_influence": 0.08151832222938538, "state_active": 0.040131714195013046, "state_memory": 0.01891530118882656, "state_peak": 39.2055549621582, "growth_active": 1.0921977758407593, "growth_memory": 1.5689817667007446, "heads": [{"head": 0, "memory_to_active": 0.0102305356413126, "contribution_to_active": 0.2946886420249939, "mix_mean": 0.5066738128662109, "mix_low": 0.0019664764404296875, "mix_high": 0.005185127258300781, "read_cosine": 0.19253796339035034, "beta_mean": 7.733788515906781e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0055784559808671474, "mix_std": 0.2937048673629761, "beta_std": 0.00010700224083848298, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013229319592937827, "evidence_weights": [-0.29324671626091003, -0.1862388700246811, 0.1918725073337555], "agreement_weight": 0.21169357001781464, "orthogonal_weight": -0.2527068257331848}, {"head": 1, "memory_to_active": 2.7981600761413574, "contribution_to_active": 0.06293091922998428, "mix_mean": 0.006218015216290951, "mix_low": 0.8638067245483398, "mix_high": 9.5367431640625e-07, "read_cosine": 0.23181799054145813, "beta_mean": 0.2834891080856323, "nonfinite": 0.0, "weighted_memory_to_active": 0.04950060695409775, "mix_std": 0.015155293047428131, "beta_std": 0.14827890694141388, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.019161520525813103, "evidence_weights": [0.09154438972473145, 0.09585481137037277, 0.4535907208919525], "agreement_weight": 0.046973083168268204, "orthogonal_weight": 0.4352375566959381}, {"head": 2, "memory_to_active": 0.013149234466254711, "contribution_to_active": 0.43095189332962036, "mix_mean": 0.47719940543174744, "mix_low": 0.0015897750854492188, "mix_high": 0.0031194686889648438, "read_cosine": 0.2664143443107605, "beta_mean": 2.6709351004683413e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004891821648925543, "mix_std": 0.2806756794452667, "beta_std": 4.36506379628554e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011446003336459398, "evidence_weights": [-0.22699527442455292, -0.26552069187164307, 0.4534214735031128], "agreement_weight": 0.03428874537348747, "orthogonal_weight": -0.2645699381828308}, {"head": 3, "memory_to_active": 0.1986151784658432, "contribution_to_active": 0.4905656576156616, "mix_mean": 0.4608650207519531, "mix_low": 0.0007295608520507812, "mix_high": 0.004695892333984375, "read_cosine": 0.6574710607528687, "beta_mean": 0.04058942198753357, "nonfinite": 0.0, "weighted_memory_to_active": 0.1077408567070961, "mix_std": 0.27121299505233765, "beta_std": 0.05303724855184555, "beta_low": 0.25628662109375, "beta_high": 0.0, "orthogonal_to_active": 0.06349407136440277, "evidence_weights": [-0.16956165432929993, -0.2647828757762909, 0.8564565777778625], "agreement_weight": 0.05935665965080261, "orthogonal_weight": -0.8695545196533203}, {"head": 4, "memory_to_active": 3.0719943046569824, "contribution_to_active": 0.23717021942138672, "mix_mean": 0.01952054351568222, "mix_low": 0.6256275177001953, "mix_high": 0.0, "read_cosine": 0.22600054740905762, "beta_mean": 0.2631932497024536, "nonfinite": 0.0, "weighted_memory_to_active": 0.29360806941986084, "mix_std": 0.05040569603443146, "beta_std": 0.23033282160758972, "beta_low": 0.02069091796875, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.018365807831287384, "evidence_weights": [0.025544166564941406, 0.02231932431459427, 0.08256545662879944], "agreement_weight": 0.023595919832587242, "orthogonal_weight": -0.07553881406784058}, {"head": 5, "memory_to_active": 0.012998124584555626, "contribution_to_active": 0.5239142179489136, "mix_mean": 0.5931746959686279, "mix_low": 0.00133514404296875, "mix_high": 0.0074787139892578125, "read_cosine": 0.461872398853302, "beta_mean": 3.9827187720220536e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007989436388015747, "mix_std": 0.2753177881240845, "beta_std": 5.422838148660958e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00041252258233726025, "evidence_weights": [-0.2592276632785797, -0.2816620171070099, 0.8397546410560608], "agreement_weight": 0.07269109040498734, "orthogonal_weight": 0.08313138037919998}]}, {"layer": 3, "memory_to_active": 0.952257513999939, "contribution_to_active": 0.3867703676223755, "mix_mean": 0.32965755462646484, "mix_low": 0.1740090250968933, "mix_high": 0.0064142546616494656, "read_cosine": 0.22468653321266174, "beta_mean": 0.0647408664226532, "nonfinite": 0.0, "weighted_memory_to_active": 0.1578732579946518, "mix_std": 0.23068219423294067, "beta_std": 0.05754707008600235, "beta_low": 0.4581298828125, "beta_high": 0.0, "orthogonal_to_active": 0.06553785502910614, "output_influence": 0.30476462841033936, "residual_influence": 0.06145041435956955, "state_active": 0.024743616580963135, "state_memory": 0.014215282164514065, "state_peak": 26.335641860961914, "growth_active": 1.0789657831192017, "growth_memory": 1.8907787799835205, "heads": [{"head": 0, "memory_to_active": 0.5416233539581299, "contribution_to_active": 0.5048266649246216, "mix_mean": 0.43295741081237793, "mix_low": 0.01864910125732422, "mix_high": 0.010593414306640625, "read_cosine": 0.2069210708141327, "beta_mean": 0.015099877491593361, "nonfinite": 0.0, "weighted_memory_to_active": 0.28401073813438416, "mix_std": 0.32458066940307617, "beta_std": 0.02363177016377449, "beta_low": 0.543701171875, "beta_high": 0.0, "orthogonal_to_active": 0.1845446228981018, "evidence_weights": [-0.16894115507602692, -0.2100973278284073, 0.6302271485328674], "agreement_weight": 0.12617483735084534, "orthogonal_weight": -0.6890190839767456}, {"head": 1, "memory_to_active": 0.011329777538776398, "contribution_to_active": 0.359845906496048, "mix_mean": 0.4200083613395691, "mix_low": 0.01876068115234375, "mix_high": 0.0034360885620117188, "read_cosine": 0.1854822337627411, "beta_mean": 6.056211714167148e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005250131245702505, "mix_std": 0.3074033856391907, "beta_std": 0.00011749619443435222, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024837679229676723, "evidence_weights": [-0.2042623907327652, -0.17652836441993713, 0.5881774425506592], "agreement_weight": 0.3291829228401184, "orthogonal_weight": -0.5179972052574158}, {"head": 2, "memory_to_active": 2.4519848823547363, "contribution_to_active": 0.2853682041168213, "mix_mean": 0.04498664289712906, "mix_low": 0.36574840545654297, "mix_high": 0.0, "read_cosine": 0.21626147627830505, "beta_mean": 0.2064727544784546, "nonfinite": 0.0, "weighted_memory_to_active": 0.3435686230659485, "mix_std": 0.09585762768983841, "beta_std": 0.15026473999023438, "beta_low": 0.00457763671875, "beta_high": 0.0, "orthogonal_to_active": 0.005789793096482754, "evidence_weights": [-0.05959576740860939, 0.020153887569904327, -0.010510860942304134], "agreement_weight": -0.010527091100811958, "orthogonal_weight": -0.021611861884593964}, {"head": 3, "memory_to_active": 0.02999718300998211, "contribution_to_active": 0.47984403371810913, "mix_mean": 0.52818763256073, "mix_low": 0.013051986694335938, "mix_high": 0.010480880737304688, "read_cosine": 0.2618357837200165, "beta_mean": 0.00011828509741462767, "nonfinite": 0.0, "weighted_memory_to_active": 0.018885768949985504, "mix_std": 0.30905118584632874, "beta_std": 0.0002922458515968174, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005405671428889036, "evidence_weights": [-0.20518897473812103, -0.2205117791891098, 1.9065742492675781], "agreement_weight": 1.2596267461776733, "orthogonal_weight": -0.3609647750854492}, {"head": 4, "memory_to_active": 2.36484432220459, "contribution_to_active": 0.03606930375099182, "mix_mean": 0.019195254892110825, "mix_low": 0.6113090515136719, "mix_high": 0.0, "read_cosine": 0.17960408329963684, "beta_mean": 0.11140454560518265, "nonfinite": 0.0, "weighted_memory_to_active": 0.08342880010604858, "mix_std": 0.0409109964966774, "beta_std": 0.11817190796136856, "beta_low": 0.07415771484375, "beta_high": 0.0, "orthogonal_to_active": 0.05403533950448036, "evidence_weights": [0.11197543889284134, 0.061966508626937866, 0.12268321961164474], "agreement_weight": -0.024782467633485794, "orthogonal_weight": -0.6668674945831299}, {"head": 5, "memory_to_active": 0.3137650489807129, "contribution_to_active": 0.6546680927276611, "mix_mean": 0.5326098203659058, "mix_low": 0.016534805297851562, "mix_high": 0.013975143432617188, "read_cosine": 0.29801446199417114, "beta_mean": 0.0552891381084919, "nonfinite": 0.0, "weighted_memory_to_active": 0.21209542453289032, "mix_std": 0.30628928542137146, "beta_std": 0.05280425772070885, "beta_low": 0.12640380859375, "beta_high": 0.0, "orthogonal_to_active": 0.1409679353237152, "evidence_weights": [-0.2056029736995697, -0.24463289976119995, 0.43330180644989014], "agreement_weight": 0.1468292623758316, "orthogonal_weight": -0.7580087780952454}]}, {"layer": 4, "memory_to_active": 1.1896755695343018, "contribution_to_active": 0.430169939994812, "mix_mean": 0.3347465991973877, "mix_low": 0.27464646100997925, "mix_high": 0.00460863159969449, "read_cosine": 0.3672555685043335, "beta_mean": 0.12804271280765533, "nonfinite": 0.0, "weighted_memory_to_active": 0.1385277360677719, "mix_std": 0.15410065650939941, "beta_std": 0.11567400395870209, "beta_low": 0.3662618398666382, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04510585218667984, "output_influence": 0.36225661635398865, "residual_influence": 0.09561222791671753, "state_active": 0.037308208644390106, "state_memory": 0.021093428134918213, "state_peak": 37.13435363769531, "growth_active": 1.2221715450286865, "growth_memory": 1.5737295150756836, "heads": [{"head": 0, "memory_to_active": 0.23784059286117554, "contribution_to_active": 0.7024858593940735, "mix_mean": 0.6346678733825684, "mix_low": 0.007109642028808594, "mix_high": 0.009455680847167969, "read_cosine": 0.5115321278572083, "beta_mean": 0.25651276111602783, "nonfinite": 0.0, "weighted_memory_to_active": 0.18396174907684326, "mix_std": 0.32941457629203796, "beta_std": 0.26416581869125366, "beta_low": 0.192626953125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.05560583621263504, "evidence_weights": [-0.2640749514102936, -0.3566112816333771, 0.30197346210479736], "agreement_weight": 0.02381339855492115, "orthogonal_weight": 0.37211090326309204}, {"head": 1, "memory_to_active": 0.09520481526851654, "contribution_to_active": 0.5716862678527832, "mix_mean": 0.6030036211013794, "mix_low": 3.337860107421875e-05, "mix_high": 0.005410194396972656, "read_cosine": 0.7438562512397766, "beta_mean": 0.0020415796898305416, "nonfinite": 0.0, "weighted_memory_to_active": 0.06222139298915863, "mix_std": 0.25416263937950134, "beta_std": 0.006547040306031704, "beta_low": 0.9605712890625, "beta_high": 0.0, "orthogonal_to_active": 0.03163781017065048, "evidence_weights": [-0.23415184020996094, -0.29962292313575745, 0.7267856597900391], "agreement_weight": 0.10248184949159622, "orthogonal_weight": -0.8174976110458374}, {"head": 2, "memory_to_active": 2.4649734497070312, "contribution_to_active": 0.17069602012634277, "mix_mean": 0.017299924045801163, "mix_low": 0.5919780731201172, "mix_high": 0.0, "read_cosine": 0.27335435152053833, "beta_mean": 0.14163599908351898, "nonfinite": 0.0, "weighted_memory_to_active": 0.18335771560668945, "mix_std": 0.03390393406152725, "beta_std": 0.10499362647533417, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.013580340892076492, "evidence_weights": [0.05660414695739746, 0.03199498727917671, -0.019502324983477592], "agreement_weight": 0.10660268366336823, "orthogonal_weight": -0.10419222712516785}, {"head": 3, "memory_to_active": 1.8170058727264404, "contribution_to_active": 0.10771214216947556, "mix_mean": 0.02846292033791542, "mix_low": 0.4120512008666992, "mix_high": 0.0, "read_cosine": 0.16345113515853882, "beta_mean": 0.14642998576164246, "nonfinite": 0.0, "weighted_memory_to_active": 0.144839808344841, "mix_std": 0.04832485690712929, "beta_std": 0.13965871930122375, "beta_low": 0.0439453125, "beta_high": 0.0, "orthogonal_to_active": 0.010398505255579948, "evidence_weights": [-0.02005314640700817, 0.05877654254436493, -0.25869089365005493], "agreement_weight": 0.05520924925804138, "orthogonal_weight": -0.09232289344072342}, {"head": 4, "memory_to_active": 0.026268642395734787, "contribution_to_active": 0.637926459312439, "mix_mean": 0.710663378238678, "mix_low": 0.000324249267578125, "mix_high": 0.012772560119628906, "read_cosine": 0.18115895986557007, "beta_mean": 4.609295501722954e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01996714249253273, "mix_std": 0.22750967741012573, "beta_std": 0.00017011433374136686, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0031140604987740517, "evidence_weights": [-0.31065765023231506, -0.3370797038078308, 0.786393940448761], "agreement_weight": 0.15742594003677368, "orthogonal_weight": -0.17198330163955688}, {"head": 5, "memory_to_active": 2.4967598915100098, "contribution_to_active": 0.39051276445388794, "mix_mean": 0.01438183430582285, "mix_low": 0.6363821029663086, "mix_high": 1.33514404296875e-05, "read_cosine": 0.33018043637275696, "beta_mean": 0.2215898334980011, "nonfinite": 0.0, "weighted_memory_to_active": 0.23681852221488953, "mix_std": 0.0312882699072361, "beta_std": 0.1785087287425995, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.15629854798316956, "evidence_weights": [0.04188193753361702, -0.038331832736730576, -0.24359488487243652], "agreement_weight": -0.06311415880918503, "orthogonal_weight": 0.7994539737701416}]}, {"layer": 5, "memory_to_active": 0.6524832248687744, "contribution_to_active": 0.4927753806114197, "mix_mean": 0.43740424513816833, "mix_low": 0.07950735092163086, "mix_high": 0.012695154175162315, "read_cosine": 0.15799599885940552, "beta_mean": 0.06339700520038605, "nonfinite": 0.0, "weighted_memory_to_active": 0.1774025857448578, "mix_std": 0.251463919878006, "beta_std": 0.0794624537229538, "beta_low": 0.6126505732536316, "beta_high": 0.0, "orthogonal_to_active": 0.0630776658654213, "output_influence": 0.48340797424316406, "residual_influence": 0.11792224645614624, "state_active": 0.046804267913103104, "state_memory": 0.012692131102085114, "state_peak": 59.10301971435547, "growth_active": 1.1421722173690796, "growth_memory": 1.2796186208724976, "heads": [{"head": 0, "memory_to_active": 0.9088837504386902, "contribution_to_active": 0.37520286440849304, "mix_mean": 0.19837620854377747, "mix_low": 0.08560371398925781, "mix_high": 0.0007047653198242188, "read_cosine": -0.19124549627304077, "beta_mean": 0.0588727742433548, "nonfinite": 0.0, "weighted_memory_to_active": 0.17789405584335327, "mix_std": 0.22227829694747925, "beta_std": 0.08756700903177261, "beta_low": 0.2525634765625, "beta_high": 0.0, "orthogonal_to_active": 0.03952089697122574, "evidence_weights": [-0.07746754586696625, -0.11200477182865143, -0.3265209496021271], "agreement_weight": -0.023742426186800003, "orthogonal_weight": -0.21875213086605072}, {"head": 1, "memory_to_active": 0.03518856316804886, "contribution_to_active": 0.5282130837440491, "mix_mean": 0.6110597848892212, "mix_low": 0.0041484832763671875, "mix_high": 0.01743793487548828, "read_cosine": 0.1827622354030609, "beta_mean": 9.12300602067262e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.017955955117940903, "mix_std": 0.3000635802745819, "beta_std": 0.00017795548774302006, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.016293713822960854, "evidence_weights": [-0.23816929757595062, -0.2719704508781433, 0.8234876990318298], "agreement_weight": 0.4295628070831299, "orthogonal_weight": -0.9385663866996765}, {"head": 2, "memory_to_active": 0.026805296540260315, "contribution_to_active": 0.5449833869934082, "mix_mean": 0.6243423819541931, "mix_low": 0.0021352767944335938, "mix_high": 0.01648998260498047, "read_cosine": 0.13872899115085602, "beta_mean": 0.00037639011861756444, "nonfinite": 0.0, "weighted_memory_to_active": 0.018071668222546577, "mix_std": 0.2735998332500458, "beta_std": 0.0006318759405985475, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.003269759938120842, "evidence_weights": [-0.25269725918769836, -0.3468030095100403, 0.886114776134491], "agreement_weight": 0.012405581772327423, "orthogonal_weight": -0.2250746190547943}, {"head": 3, "memory_to_active": 2.342649221420288, "contribution_to_active": 0.2796235680580139, "mix_mean": 0.08560862392187119, "mix_low": 0.1883993148803711, "mix_high": 0.0, "read_cosine": 0.11350677162408829, "beta_mean": 0.12020283192396164, "nonfinite": 0.0, "weighted_memory_to_active": 0.44681406021118164, "mix_std": 0.13166268169879913, "beta_std": 0.1586807668209076, "beta_low": 0.1817626953125, "beta_high": 0.0, "orthogonal_to_active": 0.16381078958511353, "evidence_weights": [-0.055426593869924545, -0.08811691403388977, -0.08123476803302765], "agreement_weight": 0.04859168827533722, "orthogonal_weight": -0.4531664252281189}, {"head": 4, "memory_to_active": 0.4696866273880005, "contribution_to_active": 0.5951536893844604, "mix_mean": 0.3311002254486084, "mix_low": 0.19403934478759766, "mix_high": 0.009542465209960938, "read_cosine": 0.5368370413780212, "beta_mean": 0.20042964816093445, "nonfinite": 0.0, "weighted_memory_to_active": 0.312324196100235, "mix_std": 0.3432055413722992, "beta_std": 0.22752335667610168, "beta_low": 0.24853515625, "beta_high": 0.0, "orthogonal_to_active": 0.09568364173173904, "evidence_weights": [-0.11960303783416748, -0.2953193485736847, -0.06169093772768974], "agreement_weight": 0.043907906860113144, "orthogonal_weight": 0.4522998631000519}, {"head": 5, "memory_to_active": 0.13168543577194214, "contribution_to_active": 0.6334755420684814, "mix_mean": 0.7739382982254028, "mix_low": 0.0027179718017578125, "mix_high": 0.03199577331542969, "read_cosine": 0.16738641262054443, "beta_mean": 0.0004091098962817341, "nonfinite": 0.0, "weighted_memory_to_active": 0.09135552495718002, "mix_std": 0.2379734218120575, "beta_std": 0.0021937345154583454, "beta_low": 0.99346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.059887196868658066, "evidence_weights": [-0.3280540406703949, -0.3213152289390564, 2.0123579502105713], "agreement_weight": 0.37347590923309326, "orthogonal_weight": -0.6975705027580261}]}, {"layer": 6, "memory_to_active": 0.3774260878562927, "contribution_to_active": 0.5882381200790405, "mix_mean": 0.5431473255157471, "mix_low": 0.07916005700826645, "mix_high": 0.028866291046142578, "read_cosine": 0.28465819358825684, "beta_mean": 0.06693678349256516, "nonfinite": 0.0, "weighted_memory_to_active": 0.1715560257434845, "mix_std": 0.24173803627490997, "beta_std": 0.07486419379711151, "beta_low": 0.5113322138786316, "beta_high": 0.0, "orthogonal_to_active": 0.03905434533953667, "output_influence": 0.4872477054595947, "residual_influence": 0.15804694592952728, "state_active": 0.04613126069307327, "state_memory": 0.008977397345006466, "state_peak": 87.51515197753906, "growth_active": 1.1704747676849365, "growth_memory": 1.1761554479599, "heads": [{"head": 0, "memory_to_active": 0.1567843109369278, "contribution_to_active": 0.6111165285110474, "mix_mean": 0.7161514759063721, "mix_low": 0.00011920928955078125, "mix_high": 0.007991790771484375, "read_cosine": 0.004029825329780579, "beta_mean": 0.012206404469907284, "nonfinite": 0.0, "weighted_memory_to_active": 0.11596651375293732, "mix_std": 0.2309291660785675, "beta_std": 0.0162850022315979, "beta_low": 0.57989501953125, "beta_high": 0.0, "orthogonal_to_active": 0.09456087648868561, "evidence_weights": [-0.3101298213005066, -0.32966381311416626, 0.5108799338340759], "agreement_weight": 0.1383868008852005, "orthogonal_weight": -0.9266645312309265}, {"head": 1, "memory_to_active": 0.004555557854473591, "contribution_to_active": 0.48188167810440063, "mix_mean": 0.510664701461792, "mix_low": 0.001575469970703125, "mix_high": 0.009768486022949219, "read_cosine": 0.5712637305259705, "beta_mean": 9.030844012158923e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0020570033229887486, "mix_std": 0.2875606417655945, "beta_std": 1.5910885849734768e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007267832988873124, "evidence_weights": [-0.20264282822608948, -0.28184500336647034, 0.9775000810623169], "agreement_weight": -0.007051227614283562, "orthogonal_weight": -0.5119059681892395}, {"head": 2, "memory_to_active": 1.1449600458145142, "contribution_to_active": 0.07344532012939453, "mix_mean": 0.024619724601507187, "mix_low": 0.4632387161254883, "mix_high": 1.621246337890625e-05, "read_cosine": 0.21834619343280792, "beta_mean": 0.06281855702400208, "nonfinite": 0.0, "weighted_memory_to_active": 0.05741018056869507, "mix_std": 0.04475098103284836, "beta_std": 0.05723182484507561, "beta_low": 0.02581787109375, "beta_high": 0.0, "orthogonal_to_active": 0.0004125757550355047, "evidence_weights": [0.09413813054561615, 0.0020688860677182674, 0.2815192639827728], "agreement_weight": 0.1221863403916359, "orthogonal_weight": -0.008500011637806892}, {"head": 3, "memory_to_active": 0.1489175707101822, "contribution_to_active": 0.6706269383430481, "mix_mean": 0.6320388913154602, "mix_low": 0.008434295654296875, "mix_high": 0.0170135498046875, "read_cosine": 0.4771672487258911, "beta_mean": 0.19951452314853668, "nonfinite": 0.0, "weighted_memory_to_active": 0.10489565134048462, "mix_std": 0.33237409591674805, "beta_std": 0.23345541954040527, "beta_low": 0.218505859375, "beta_high": 0.0, "orthogonal_to_active": 0.055799540132284164, "evidence_weights": [-0.1743839532136917, -0.3364720940589905, 0.15954892337322235], "agreement_weight": -0.014641478657722473, "orthogonal_weight": 0.6232048869132996}, {"head": 4, "memory_to_active": 0.7687579393386841, "contribution_to_active": 1.0996137857437134, "mix_mean": 0.7535580396652222, "mix_low": 0.0012941360473632812, "mix_high": 0.136474609375, "read_cosine": 0.14950114488601685, "beta_mean": 0.12675265967845917, "nonfinite": 0.0, "weighted_memory_to_active": 0.7218190431594849, "mix_std": 0.2933245599269867, "beta_std": 0.14156599342823029, "beta_low": 0.2442626953125, "beta_high": 0.0, "orthogonal_to_active": 0.05518161132931709, "evidence_weights": [-0.25758013129234314, -0.5893872976303101, -0.08817633241415024], "agreement_weight": -0.01398015022277832, "orthogonal_weight": 0.08822207152843475}, {"head": 5, "memory_to_active": 0.040581006556749344, "contribution_to_active": 0.5927444696426392, "mix_mean": 0.6218509078025818, "mix_low": 0.00029850006103515625, "mix_high": 0.0019330978393554688, "read_cosine": 0.28764092922210693, "beta_mean": 0.0003195021126884967, "nonfinite": 0.0, "weighted_memory_to_active": 0.027187736704945564, "mix_std": 0.26148879528045654, "beta_std": 0.0006309808813966811, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.02764469012618065, "evidence_weights": [-0.2405308485031128, -0.30364638566970825, 0.41405555605888367], "agreement_weight": 0.12249340862035751, "orthogonal_weight": -1.0906282663345337}]}, {"layer": 7, "memory_to_active": 1.822740912437439, "contribution_to_active": 0.2314719259738922, "mix_mean": 0.07953038066625595, "mix_low": 0.3203342854976654, "mix_high": 7.343292963923886e-05, "read_cosine": 0.2360336035490036, "beta_mean": 0.13410308957099915, "nonfinite": 0.0, "weighted_memory_to_active": 0.23915186524391174, "mix_std": 0.11043788492679596, "beta_std": 0.10882188379764557, "beta_low": 0.1447347104549408, "beta_high": 8.138021075865254e-05, "orthogonal_to_active": 0.058858051896095276, "output_influence": 0.21872282028198242, "residual_influence": 0.0490904226899147, "state_active": 0.014732135459780693, "state_memory": 0.02860262803733349, "state_peak": 45.12694549560547, "growth_active": 1.1443771123886108, "growth_memory": 1.3185970783233643, "heads": [{"head": 0, "memory_to_active": 2.3561596870422363, "contribution_to_active": 0.45690351724624634, "mix_mean": 0.032987479120492935, "mix_low": 0.44683170318603516, "mix_high": 0.0, "read_cosine": 0.36015015840530396, "beta_mean": 0.15998947620391846, "nonfinite": 0.0, "weighted_memory_to_active": 0.4584037661552429, "mix_std": 0.0696234181523323, "beta_std": 0.1210053414106369, "beta_low": 0.00225830078125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.03099757805466652, "evidence_weights": [0.009141157381236553, -0.08687999099493027, 0.12485523521900177], "agreement_weight": -0.039500247687101364, "orthogonal_weight": 0.0918353870511055}, {"head": 1, "memory_to_active": 1.2241228818893433, "contribution_to_active": 0.3493451178073883, "mix_mean": 0.17123621702194214, "mix_low": 0.18630599975585938, "mix_high": 0.00030612945556640625, "read_cosine": 0.30993178486824036, "beta_mean": 0.12308114022016525, "nonfinite": 0.0, "weighted_memory_to_active": 0.3429199159145355, "mix_std": 0.20419886708259583, "beta_std": 0.07894343882799149, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00362263061106205, "evidence_weights": [-0.18579493463039398, -0.11947692185640335, -0.23851124942302704], "agreement_weight": 0.024403508752584457, "orthogonal_weight": -0.011998352594673634}, {"head": 2, "memory_to_active": 0.9296274185180664, "contribution_to_active": 0.2672036588191986, "mix_mean": 0.09376847743988037, "mix_low": 0.14545154571533203, "mix_high": 0.00013446807861328125, "read_cosine": 0.12150317430496216, "beta_mean": 0.06698782742023468, "nonfinite": 0.0, "weighted_memory_to_active": 0.18350502848625183, "mix_std": 0.14249201118946075, "beta_std": 0.0509280301630497, "beta_low": 0.045654296875, "beta_high": 0.0, "orthogonal_to_active": 0.04548720270395279, "evidence_weights": [-0.03580654039978981, -0.12488941103219986, 0.34438762068748474], "agreement_weight": 0.02181645855307579, "orthogonal_weight": -0.25328329205513}, {"head": 3, "memory_to_active": 3.9819183349609375, "contribution_to_active": 0.08185480535030365, "mix_mean": 0.03155944123864174, "mix_low": 0.36566162109375, "mix_high": 0.0, "read_cosine": 0.10607574135065079, "beta_mean": 0.2864139974117279, "nonfinite": 0.0, "weighted_memory_to_active": 0.26305097341537476, "mix_std": 0.04939216747879982, "beta_std": 0.2389870434999466, "beta_low": 0.01336669921875, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.21297596395015717, "evidence_weights": [0.0760723277926445, 0.013245980255305767, 0.6312870979309082], "agreement_weight": 0.05597080662846565, "orthogonal_weight": -0.9181139469146729}, {"head": 4, "memory_to_active": 1.8637491464614868, "contribution_to_active": 0.06880036741495132, "mix_mean": 0.014345702715218067, "mix_low": 0.6765642166137695, "mix_high": 0.0, "read_cosine": 0.2904585003852844, "beta_mean": 0.1605188548564911, "nonfinite": 0.0, "weighted_memory_to_active": 0.06348517537117004, "mix_std": 0.03341672942042351, "beta_std": 0.14789380133152008, "beta_low": 0.016357421875, "beta_high": 0.0, "orthogonal_to_active": 0.005477736704051495, "evidence_weights": [0.12660664319992065, 0.054796479642391205, 0.14765465259552002], "agreement_weight": 0.04368099942803383, "orthogonal_weight": -0.11477115005254745}, {"head": 5, "memory_to_active": 0.5808678269386292, "contribution_to_active": 0.16472406685352325, "mix_mean": 0.13328492641448975, "mix_low": 0.10119056701660156, "mix_high": 0.0, "read_cosine": 0.22808226943016052, "beta_mean": 0.007627234794199467, "nonfinite": 0.0, "weighted_memory_to_active": 0.12354622781276703, "mix_std": 0.16350413858890533, "beta_std": 0.01517360657453537, "beta_low": 0.790771484375, "beta_high": 0.0, "orthogonal_to_active": 0.054587215185165405, "evidence_weights": [-0.036441631615161896, -0.08222347497940063, 0.23298463225364685], "agreement_weight": 0.06848595291376114, "orthogonal_weight": -0.5652005076408386}]}, {"layer": 8, "memory_to_active": 0.528228223323822, "contribution_to_active": 0.5025226473808289, "mix_mean": 0.405270516872406, "mix_low": 0.09886281192302704, "mix_high": 0.0007359187002293766, "read_cosine": 0.5034050941467285, "beta_mean": 0.036418162286281586, "nonfinite": 0.0, "weighted_memory_to_active": 0.10173812508583069, "mix_std": 0.2208520472049713, "beta_std": 0.04027574509382248, "beta_low": 0.5653483271598816, "beta_high": 0.0, "orthogonal_to_active": 0.05421464890241623, "output_influence": 0.4165884852409363, "residual_influence": 0.12593486905097961, "state_active": 0.05139642208814621, "state_memory": 0.011113430373370647, "state_peak": 35.330322265625, "growth_active": 1.0632349252700806, "growth_memory": 1.3723444938659668, "heads": [{"head": 0, "memory_to_active": 0.04449976608157158, "contribution_to_active": 0.5298708081245422, "mix_mean": 0.5746657848358154, "mix_low": 0.00025653839111328125, "mix_high": 0.0006122589111328125, "read_cosine": 0.625299334526062, "beta_mean": 0.003430413780733943, "nonfinite": 0.0, "weighted_memory_to_active": 0.028773456811904907, "mix_std": 0.24079063534736633, "beta_std": 0.008473359979689121, "beta_low": 0.9158935546875, "beta_high": 0.0, "orthogonal_to_active": 0.01781747303903103, "evidence_weights": [-0.1734457165002823, -0.3842425048351288, 0.7185549736022949], "agreement_weight": 0.08553500473499298, "orthogonal_weight": -0.8207775950431824}, {"head": 1, "memory_to_active": 0.2253592610359192, "contribution_to_active": 0.4570215046405792, "mix_mean": 0.43290162086486816, "mix_low": 0.0020914077758789062, "mix_high": 0.00013256072998046875, "read_cosine": 0.30902421474456787, "beta_mean": 0.00593714602291584, "nonfinite": 0.0, "weighted_memory_to_active": 0.10902082175016403, "mix_std": 0.2504248023033142, "beta_std": 0.016209062188863754, "beta_low": 0.86395263671875, "beta_high": 0.0, "orthogonal_to_active": 0.06283801794052124, "evidence_weights": [-0.18912538886070251, -0.24166856706142426, 0.6834881901741028], "agreement_weight": 0.08376599103212357, "orthogonal_weight": -0.6352551579475403}, {"head": 2, "memory_to_active": 0.026375439018011093, "contribution_to_active": 0.623329758644104, "mix_mean": 0.5965628623962402, "mix_low": 0.0008726119995117188, "mix_high": 0.002460479736328125, "read_cosine": 0.38716256618499756, "beta_mean": 0.00022533847368322313, "nonfinite": 0.0, "weighted_memory_to_active": 0.01860504038631916, "mix_std": 0.23771752417087555, "beta_std": 0.0028767313342541456, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.012553436681628227, "evidence_weights": [-0.21684902906417847, -0.3334686756134033, 0.7986131906509399], "agreement_weight": 0.17315301299095154, "orthogonal_weight": -0.7685982584953308}, {"head": 3, "memory_to_active": 0.16023877263069153, "contribution_to_active": 0.5869097709655762, "mix_mean": 0.5374252796173096, "mix_low": 0.0010251998901367188, "mix_high": 0.0007009506225585938, "read_cosine": 0.8113799095153809, "beta_mean": 0.014457998797297478, "nonfinite": 0.0, "weighted_memory_to_active": 0.11495494842529297, "mix_std": 0.25303399562835693, "beta_std": 0.012008649297058582, "beta_low": 0.388916015625, "beta_high": 0.0, "orthogonal_to_active": 0.03422735631465912, "evidence_weights": [-0.17460297048091888, -0.2965962290763855, 0.6918294429779053], "agreement_weight": 0.1356283277273178, "orthogonal_weight": -0.5913590788841248}, {"head": 4, "memory_to_active": 2.4627556800842285, "contribution_to_active": 0.20878753066062927, "mix_mean": 0.025042546913027763, "mix_low": 0.4477357864379883, "mix_high": 0.0, "read_cosine": 0.32812297344207764, "beta_mean": 0.11699019372463226, "nonfinite": 0.0, "weighted_memory_to_active": 0.18824943900108337, "mix_std": 0.042118798941373825, "beta_std": 0.09737540036439896, "beta_low": 0.013671875, "beta_high": 0.0, "orthogonal_to_active": 0.010018979199230671, "evidence_weights": [0.0493016242980957, -0.012583496980369091, -0.11699890345335007], "agreement_weight": 0.03311720862984657, "orthogonal_weight": 0.05562721565365791}, {"head": 5, "memory_to_active": 0.2501402497291565, "contribution_to_active": 0.6092164516448975, "mix_mean": 0.26502490043640137, "mix_low": 0.14119529724121094, "mix_high": 0.0005092620849609375, "read_cosine": 0.5594415068626404, "beta_mean": 0.0774678885936737, "nonfinite": 0.0, "weighted_memory_to_active": 0.15082500874996185, "mix_std": 0.30102652311325073, "beta_std": 0.10471124947071075, "beta_low": 0.2098388671875, "beta_high": 0.0, "orthogonal_to_active": 0.18783262372016907, "evidence_weights": [-0.06503766030073166, -0.2835277020931244, -0.40120428800582886], "agreement_weight": 0.013686944730579853, "orthogonal_weight": 1.5590317249298096}]}, {"layer": 9, "memory_to_active": 1.133713722229004, "contribution_to_active": 0.6156696081161499, "mix_mean": 0.30062663555145264, "mix_low": 0.15864992141723633, "mix_high": 0.002892653224989772, "read_cosine": 0.36882010102272034, "beta_mean": 0.07302725315093994, "nonfinite": 0.0, "weighted_memory_to_active": 0.3546012043952942, "mix_std": 0.20816071331501007, "beta_std": 0.04658229649066925, "beta_low": 0.5010579824447632, "beta_high": 0.0, "orthogonal_to_active": 0.03967529535293579, "output_influence": 0.4650046229362488, "residual_influence": 0.12469606101512909, "state_active": 0.04261422157287598, "state_memory": 0.02083587273955345, "state_peak": 36.75960922241211, "growth_active": 1.2079956531524658, "growth_memory": 1.4001991748809814, "heads": [{"head": 0, "memory_to_active": 0.006230352446436882, "contribution_to_active": 0.5204269886016846, "mix_mean": 0.5919272899627686, "mix_low": 0.0014219284057617188, "mix_high": 0.011999130249023438, "read_cosine": 0.23710709810256958, "beta_mean": 1.1881571481353603e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003732844954356551, "mix_std": 0.2931806147098541, "beta_std": 5.522468200069852e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0032356269657611847, "evidence_weights": [-0.2922550141811371, -0.2992953360080719, 0.9343358278274536], "agreement_weight": 0.09245677292346954, "orthogonal_weight": -0.9137606024742126}, {"head": 1, "memory_to_active": 0.03214489296078682, "contribution_to_active": 0.5962982773780823, "mix_mean": 0.46041423082351685, "mix_low": 0.01064300537109375, "mix_high": 0.00142669677734375, "read_cosine": 0.39942824840545654, "beta_mean": 8.39911590446718e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.019978318363428116, "mix_std": 0.28994178771972656, "beta_std": 0.00019074966257903725, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013713792897760868, "evidence_weights": [-0.2141464352607727, -0.26013731956481934, 1.1324577331542969], "agreement_weight": 0.3235700726509094, "orthogonal_weight": -0.8135209083557129}, {"head": 2, "memory_to_active": 0.1843940168619156, "contribution_to_active": 0.6077531576156616, "mix_mean": 0.5328530669212341, "mix_low": 0.00241851806640625, "mix_high": 0.00379180908203125, "read_cosine": 0.3812851309776306, "beta_mean": 0.0007410317775793374, "nonfinite": 0.0, "weighted_memory_to_active": 0.11439208686351776, "mix_std": 0.27908560633659363, "beta_std": 0.004639751277863979, "beta_low": 0.99395751953125, "beta_high": 0.0, "orthogonal_to_active": 0.054306477308273315, "evidence_weights": [-0.18418152630329132, -0.24289286136627197, 0.6130271553993225], "agreement_weight": 0.24675488471984863, "orthogonal_weight": -0.5635951161384583}, {"head": 3, "memory_to_active": 1.7028276920318604, "contribution_to_active": 0.1628326177597046, "mix_mean": 0.03693311661481857, "mix_low": 0.40929508209228516, "mix_high": 9.5367431640625e-07, "read_cosine": 0.4271494746208191, "beta_mean": 0.09021075069904327, "nonfinite": 0.0, "weighted_memory_to_active": 0.13857698440551758, "mix_std": 0.08812558650970459, "beta_std": 0.06684845685958862, "beta_low": 0.01226806640625, "beta_high": 0.0, "orthogonal_to_active": 0.02426777593791485, "evidence_weights": [0.07413710653781891, 0.03775099292397499, -0.07566823065280914], "agreement_weight": 0.08365077525377274, "orthogonal_weight": 0.19376955926418304}, {"head": 4, "memory_to_active": 3.139115810394287, "contribution_to_active": 1.1780495643615723, "mix_mean": 0.11800573766231537, "mix_low": 0.16898536682128906, "mix_high": 0.0001373291015625, "read_cosine": 0.31654536724090576, "beta_mean": 0.18962518870830536, "nonfinite": 0.0, "weighted_memory_to_active": 1.1583104133605957, "mix_std": 0.17802853882312775, "beta_std": 0.10599552094936371, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.023858942091464996, "evidence_weights": [-0.055942825973033905, -0.1477833092212677, 0.013222835026681423], "agreement_weight": -0.05545947328209877, "orthogonal_weight": -0.02125856652855873}, {"head": 5, "memory_to_active": 1.7375688552856445, "contribution_to_active": 0.6286566853523254, "mix_mean": 0.06362636387348175, "mix_low": 0.35913562774658203, "mix_high": 0.0, "read_cosine": 0.45140528678894043, "beta_mean": 0.15749067068099976, "nonfinite": 0.0, "weighted_memory_to_active": 0.6926164031028748, "mix_std": 0.12060213088989258, "beta_std": 0.10176408290863037, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.11866913735866547, "evidence_weights": [0.09084046632051468, -0.058602314442396164, 0.14521560072898865], "agreement_weight": 0.021687982603907585, "orthogonal_weight": 0.24282938241958618}]}, {"layer": 10, "memory_to_active": 0.19962775707244873, "contribution_to_active": 0.5491731762886047, "mix_mean": 0.47810110449790955, "mix_low": 0.013640881516039371, "mix_high": 0.0077374777756631374, "read_cosine": 0.38281333446502686, "beta_mean": 0.04638712480664253, "nonfinite": 0.0, "weighted_memory_to_active": 0.09719046950340271, "mix_std": 0.26764294505119324, "beta_std": 0.048077136278152466, "beta_low": 0.6520589590072632, "beta_high": 0.0, "orthogonal_to_active": 0.09697839617729187, "output_influence": 0.4046003520488739, "residual_influence": 0.13596530258655548, "state_active": 0.056015171110630035, "state_memory": 0.014156175777316093, "state_peak": 115.20490264892578, "growth_active": 1.0913150310516357, "growth_memory": 0.969628632068634, "heads": [{"head": 0, "memory_to_active": 0.6869843602180481, "contribution_to_active": 1.1729953289031982, "mix_mean": 0.6128876209259033, "mix_low": 0.03535747528076172, "mix_high": 0.036266326904296875, "read_cosine": 0.11430568248033524, "beta_mean": 0.2036474645137787, "nonfinite": 0.0, "weighted_memory_to_active": 0.4514434039592743, "mix_std": 0.3795073330402374, "beta_std": 0.2175150364637375, "beta_low": 0.1224365234375, "beta_high": 0.0, "orthogonal_to_active": 0.5027388334274292, "evidence_weights": [-0.1399090439081192, -0.3728918433189392, -0.7340317964553833], "agreement_weight": 0.02350158430635929, "orthogonal_weight": 1.135556697845459}, {"head": 1, "memory_to_active": 0.014825735241174698, "contribution_to_active": 0.5768353343009949, "mix_mean": 0.5814412832260132, "mix_low": 0.0006666183471679688, "mix_high": 0.00728607177734375, "read_cosine": 0.28537237644195557, "beta_mean": 9.516719728708267e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008303085342049599, "mix_std": 0.2692047655582428, "beta_std": 0.00012486442574299872, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010653665289282799, "evidence_weights": [-0.27224430441856384, -0.20112007856369019, 0.5865682363510132], "agreement_weight": 0.41993409395217896, "orthogonal_weight": -1.4159692525863647}, {"head": 2, "memory_to_active": 0.03411972522735596, "contribution_to_active": 0.5252904295921326, "mix_mean": 0.5410143136978149, "mix_low": 0.0003643035888671875, "mix_high": 0.0025424957275390625, "read_cosine": 0.0317622572183609, "beta_mean": 9.547195077175274e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012759614735841751, "mix_std": 0.2605479955673218, "beta_std": 0.00031790745561011136, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.005225538741797209, "evidence_weights": [-0.21075381338596344, -0.2535199224948883, 0.5165830254554749], "agreement_weight": -0.08843612670898438, "orthogonal_weight": -0.39231202006340027}, {"head": 3, "memory_to_active": 0.3214681148529053, "contribution_to_active": 0.10725125670433044, "mix_mean": 0.25629061460494995, "mix_low": 0.04177093505859375, "mix_high": 0.000171661376953125, "read_cosine": 0.9063149094581604, "beta_mean": 0.06715365499258041, "nonfinite": 0.0, "weighted_memory_to_active": 0.04280233010649681, "mix_std": 0.22238770127296448, "beta_std": 0.06173877790570259, "beta_low": 0.001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.04410618543624878, "evidence_weights": [-0.1174120306968689, -0.14650943875312805, 0.14123502373695374], "agreement_weight": 0.01154446043074131, "orthogonal_weight": 1.4906872510910034}, {"head": 4, "memory_to_active": 0.0024891397915780544, "contribution_to_active": 0.49919673800468445, "mix_mean": 0.5068479776382446, "mix_low": 0.000858306884765625, "mix_high": 0.00015544891357421875, "read_cosine": 0.10392561554908752, "beta_mean": 1.5451205399585888e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0012004036689177155, "mix_std": 0.25149571895599365, "beta_std": 0.00013888528337702155, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0006030173972249031, "evidence_weights": [-0.1813998520374298, -0.24806198477745056, 0.7066488265991211], "agreement_weight": 0.0830528661608696, "orthogonal_weight": -0.5207920670509338}, {"head": 5, "memory_to_active": 0.1378794014453888, "contribution_to_active": 0.4134700298309326, "mix_mean": 0.3701247274875641, "mix_low": 0.0028276443481445312, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8551991581916809, "beta_mean": 0.007315520662814379, "nonfinite": 0.0, "weighted_memory_to_active": 0.06663398444652557, "mix_std": 0.2227141559123993, "beta_std": 0.008627342991530895, "beta_low": 0.78887939453125, "beta_high": 0.0, "orthogonal_to_active": 0.018543098121881485, "evidence_weights": [-0.14794921875, -0.1597195863723755, 0.2943185865879059], "agreement_weight": 0.09089580923318863, "orthogonal_weight": -0.617385983467102}]}, {"layer": 11, "memory_to_active": 0.15000583231449127, "contribution_to_active": 0.5588430762290955, "mix_mean": 0.4510321021080017, "mix_low": 0.08261140435934067, "mix_high": 0.0380299910902977, "read_cosine": 0.4513642191886902, "beta_mean": 0.031916387379169464, "nonfinite": 0.0, "weighted_memory_to_active": 0.06569840013980865, "mix_std": 0.321128785610199, "beta_std": 0.04518058896064758, "beta_low": 0.562286376953125, "beta_high": 0.0, "orthogonal_to_active": 0.05145590752363205, "output_influence": 0.5086068511009216, "residual_influence": 0.12709440290927887, "state_active": 0.1635359525680542, "state_memory": 0.00741229671984911, "state_peak": 25.44392204284668, "growth_active": 1.058609127998352, "growth_memory": 1.516473650932312, "heads": [{"head": 0, "memory_to_active": 0.005863534286618233, "contribution_to_active": 0.5701342225074768, "mix_mean": 0.5905902981758118, "mix_low": 0.0014486312866210938, "mix_high": 0.007948875427246094, "read_cosine": 0.3713207244873047, "beta_mean": 8.091071504168212e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0036536494735628366, "mix_std": 0.27078676223754883, "beta_std": 0.00388080021366477, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0024636234156787395, "evidence_weights": [-0.16327546536922455, -0.3556855618953705, 0.6615806221961975], "agreement_weight": 0.2242717146873474, "orthogonal_weight": -0.7503349781036377}, {"head": 1, "memory_to_active": 0.37129589915275574, "contribution_to_active": 0.2839031517505646, "mix_mean": 0.3358381986618042, "mix_low": 0.011854171752929688, "mix_high": 0.00014781951904296875, "read_cosine": 0.6315246224403381, "beta_mean": 0.03387003391981125, "nonfinite": 0.0, "weighted_memory_to_active": 0.1528395712375641, "mix_std": 0.2558685839176178, "beta_std": 0.05803924426436424, "beta_low": 0.378662109375, "beta_high": 0.0, "orthogonal_to_active": 0.03680645674467087, "evidence_weights": [-0.1611422449350357, -0.1519262194633484, 0.3291614353656769], "agreement_weight": 0.08081914484500885, "orthogonal_weight": -0.33521395921707153}, {"head": 2, "memory_to_active": 0.01802223175764084, "contribution_to_active": 0.9561924934387207, "mix_mean": 0.4468385875225067, "mix_low": 0.18454265594482422, "mix_high": 0.13388633728027344, "read_cosine": 0.24162942171096802, "beta_mean": 0.07017694413661957, "nonfinite": 0.0, "weighted_memory_to_active": 0.013669343665242195, "mix_std": 0.4338599443435669, "beta_std": 0.09334857016801834, "beta_low": 0.43939208984375, "beta_high": 0.0, "orthogonal_to_active": 0.06476224958896637, "evidence_weights": [0.004084836691617966, -0.45517462491989136, -1.2291873693466187], "agreement_weight": 0.0022750436328351498, "orthogonal_weight": 4.953647613525391}, {"head": 3, "memory_to_active": 0.013983436860144138, "contribution_to_active": 0.7970220446586609, "mix_mean": 0.40511777997016907, "mix_low": 0.2839984893798828, "mix_high": 0.08438587188720703, "read_cosine": 0.5691795349121094, "beta_mean": 0.05668066814541817, "nonfinite": 0.0, "weighted_memory_to_active": 0.008094022050499916, "mix_std": 0.4354773461818695, "beta_std": 0.08312232792377472, "beta_low": 0.485107421875, "beta_high": 0.0, "orthogonal_to_active": 0.03395973891019821, "evidence_weights": [0.07204791158437729, -0.34773147106170654, -0.4223324954509735], "agreement_weight": 0.04415462911128998, "orthogonal_weight": 5.269676685333252}, {"head": 4, "memory_to_active": 0.3969373404979706, "contribution_to_active": 0.3818633258342743, "mix_mean": 0.439034104347229, "mix_low": 0.012860298156738281, "mix_high": 0.0002498626708984375, "read_cosine": 0.1185886412858963, "beta_mean": 0.02328510396182537, "nonfinite": 0.0, "weighted_memory_to_active": 0.17450454831123352, "mix_std": 0.269401490688324, "beta_std": 0.021920600906014442, "beta_low": 0.290771484375, "beta_high": 0.0, "orthogonal_to_active": 0.1506364494562149, "evidence_weights": [-0.21046413481235504, -0.18088383972644806, 1.7019201517105103], "agreement_weight": -0.17032183706760406, "orthogonal_weight": -0.8816576600074768}, {"head": 5, "memory_to_active": 0.09393258392810822, "contribution_to_active": 0.3639432191848755, "mix_mean": 0.48877376317977905, "mix_low": 0.0009641647338867188, "mix_high": 0.0015611648559570312, "read_cosine": 0.7759422063827515, "beta_mean": 0.007404658943414688, "nonfinite": 0.0, "weighted_memory_to_active": 0.041429273784160614, "mix_std": 0.2613784074783325, "beta_std": 0.010771999135613441, "beta_low": 0.7799072265625, "beta_high": 0.0, "orthogonal_to_active": 0.02010689117014408, "evidence_weights": [-0.2139492630958557, -0.2752997577190399, 0.5316039323806763], "agreement_weight": 0.06869687139987946, "orthogonal_weight": -0.6811922788619995}]}, {"layer": 12, "memory_to_active": 0.27585500478744507, "contribution_to_active": 0.537656307220459, "mix_mean": 0.4785677194595337, "mix_low": 0.0123113002628088, "mix_high": 0.003458817955106497, "read_cosine": 0.24406525492668152, "beta_mean": 0.014410216361284256, "nonfinite": 0.0, "weighted_memory_to_active": 0.14761188626289368, "mix_std": 0.2580263018608093, "beta_std": 0.015110405161976814, "beta_low": 0.8236389756202698, "beta_high": 0.0, "orthogonal_to_active": 0.06350523978471756, "output_influence": 0.4330819547176361, "residual_influence": 0.11976806074380875, "state_active": 0.04075810685753822, "state_memory": 0.00921331625431776, "state_peak": 32.201053619384766, "growth_active": 1.1127630472183228, "growth_memory": 1.2065191268920898, "heads": [{"head": 0, "memory_to_active": 0.05869761109352112, "contribution_to_active": 0.47435563802719116, "mix_mean": 0.4522815942764282, "mix_low": 0.0018863677978515625, "mix_high": 0.00084686279296875, "read_cosine": 0.3038088083267212, "beta_mean": 0.0007820489699952304, "nonfinite": 0.0, "weighted_memory_to_active": 0.025666048750281334, "mix_std": 0.26875555515289307, "beta_std": 0.004047990310937166, "beta_low": 0.9957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.01686270721256733, "evidence_weights": [-0.1777672916650772, -0.2509904205799103, 0.30063000321388245], "agreement_weight": 0.1083710566163063, "orthogonal_weight": -0.6925662159919739}, {"head": 1, "memory_to_active": 0.09567892551422119, "contribution_to_active": 0.5000305771827698, "mix_mean": 0.51291823387146, "mix_low": 0.000701904296875, "mix_high": 0.0004444122314453125, "read_cosine": 0.5262162685394287, "beta_mean": 0.0009227503323927522, "nonfinite": 0.0, "weighted_memory_to_active": 0.0475023090839386, "mix_std": 0.2617587745189667, "beta_std": 0.00498605752363801, "beta_low": 0.9913330078125, "beta_high": 0.0, "orthogonal_to_active": 0.03733398765325546, "evidence_weights": [-0.2047492265701294, -0.2709958553314209, 0.8950953483581543], "agreement_weight": 0.1252669095993042, "orthogonal_weight": -0.9175158143043518}, {"head": 2, "memory_to_active": 0.5955876111984253, "contribution_to_active": 0.43690627813339233, "mix_mean": 0.3104151487350464, "mix_low": 0.026971817016601562, "mix_high": 0.00012302398681640625, "read_cosine": 0.4351678192615509, "beta_mean": 0.08250126242637634, "nonfinite": 0.0, "weighted_memory_to_active": 0.22789183259010315, "mix_std": 0.24037523567676544, "beta_std": 0.06187403202056885, "beta_low": 0.00140380859375, "beta_high": 0.0, "orthogonal_to_active": 0.20239511132240295, "evidence_weights": [-0.2421531230211258, -0.04313848912715912, -0.09135743230581284], "agreement_weight": 0.029902534559369087, "orthogonal_weight": 1.077430009841919}, {"head": 3, "memory_to_active": 0.025275008752942085, "contribution_to_active": 0.7222341299057007, "mix_mean": 0.6887022852897644, "mix_low": 0.00022220611572265625, "mix_high": 0.014923095703125, "read_cosine": 0.025531284511089325, "beta_mean": 0.00019788173085544258, "nonfinite": 0.0, "weighted_memory_to_active": 0.018820717930793762, "mix_std": 0.23922617733478546, "beta_std": 0.0039404286071658134, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.017474442720413208, "evidence_weights": [-0.29692327976226807, -0.33637434244155884, 1.2240402698516846], "agreement_weight": 0.31719326972961426, "orthogonal_weight": -0.9480723738670349}, {"head": 4, "memory_to_active": 0.7902007102966309, "contribution_to_active": 0.6303817629814148, "mix_mean": 0.29992565512657166, "mix_low": 0.03729248046875, "mix_high": 0.0018444061279296875, "read_cosine": -0.06659016758203506, "beta_mean": 0.0013602644903585315, "nonfinite": 0.0, "weighted_memory_to_active": 0.5098893046379089, "mix_std": 0.2623196244239807, "beta_std": 0.011589054018259048, "beta_low": 0.96990966796875, "beta_high": 0.0, "orthogonal_to_active": 0.06363441795110703, "evidence_weights": [-0.08164482563734055, -0.20498017966747284, 0.04529329016804695], "agreement_weight": 0.0006079005543142557, "orthogonal_weight": -0.13356421887874603}, {"head": 5, "memory_to_active": 0.08969007432460785, "contribution_to_active": 0.46202927827835083, "mix_mean": 0.6071633696556091, "mix_low": 0.006793022155761719, "mix_high": 0.00257110595703125, "read_cosine": 0.2402574121952057, "beta_mean": 0.0006970900576561689, "nonfinite": 0.0, "weighted_memory_to_active": 0.05590103566646576, "mix_std": 0.2757223844528198, "beta_std": 0.004224861040711403, "beta_low": 0.984375, "beta_high": 0.0, "orthogonal_to_active": 0.043330758810043335, "evidence_weights": [-0.25835737586021423, -0.3045221269130707, 1.1161800622940063], "agreement_weight": 0.24253737926483154, "orthogonal_weight": -0.8376126289367676}]}, {"layer": 13, "memory_to_active": 0.1484590321779251, "contribution_to_active": 0.5142816305160522, "mix_mean": 0.5045109391212463, "mix_low": 0.00516064977273345, "mix_high": 0.004764874931424856, "read_cosine": 0.4118272662162781, "beta_mean": 0.01690087839961052, "nonfinite": 0.0, "weighted_memory_to_active": 0.06752854585647583, "mix_std": 0.2594098150730133, "beta_std": 0.019099034368991852, "beta_low": 0.7862345576286316, "beta_high": 0.0, "orthogonal_to_active": 0.06444893032312393, "output_influence": 0.4686466455459595, "residual_influence": 0.15379363298416138, "state_active": 0.09809257835149765, "state_memory": 0.036588214337825775, "state_peak": 96.59244537353516, "growth_active": 0.9812268018722534, "growth_memory": 1.0158835649490356, "heads": [{"head": 0, "memory_to_active": 0.034749533981084824, "contribution_to_active": 0.5384831428527832, "mix_mean": 0.5449331402778625, "mix_low": 0.0005540847778320312, "mix_high": 0.0017461776733398438, "read_cosine": 0.3892388939857483, "beta_mean": 0.0001200652404804714, "nonfinite": 0.0, "weighted_memory_to_active": 0.020120523869991302, "mix_std": 0.2654024660587311, "beta_std": 0.00026714877458289266, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.019574731588363647, "evidence_weights": [-0.20189601182937622, -0.2596419155597687, 0.7398605346679688], "agreement_weight": 0.08693891018629074, "orthogonal_weight": -1.0829321146011353}, {"head": 1, "memory_to_active": 0.025316856801509857, "contribution_to_active": 0.5634980201721191, "mix_mean": 0.5168935060501099, "mix_low": 0.0019073486328125, "mix_high": 0.0013380050659179688, "read_cosine": 0.19450987875461578, "beta_mean": 7.834618736524135e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014128688722848892, "mix_std": 0.26794183254241943, "beta_std": 0.00014832559099886566, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015404135920107365, "evidence_weights": [-0.18729841709136963, -0.26910078525543213, 0.7022435665130615], "agreement_weight": 0.27577877044677734, "orthogonal_weight": -1.1443703174591064}, {"head": 2, "memory_to_active": 0.13034267723560333, "contribution_to_active": 0.4317682981491089, "mix_mean": 0.36625248193740845, "mix_low": 0.0022258758544921875, "mix_high": 0.00014972686767578125, "read_cosine": 0.6862569451332092, "beta_mean": 0.01763962022960186, "nonfinite": 0.0, "weighted_memory_to_active": 0.05783233791589737, "mix_std": 0.22166290879249573, "beta_std": 0.047018371522426605, "beta_low": 0.73651123046875, "beta_high": 0.0, "orthogonal_to_active": 0.026979876682162285, "evidence_weights": [-0.2116154134273529, -0.1821000725030899, 0.4426730275154114], "agreement_weight": 0.01834126189351082, "orthogonal_weight": -0.7930418848991394}, {"head": 3, "memory_to_active": 0.14789694547653198, "contribution_to_active": 0.6110486388206482, "mix_mean": 0.6244841814041138, "mix_low": 0.015923500061035156, "mix_high": 0.01964092254638672, "read_cosine": 0.05466035008430481, "beta_mean": 0.0012017256813123822, "nonfinite": 0.0, "weighted_memory_to_active": 0.1080998033285141, "mix_std": 0.28666800260543823, "beta_std": 0.0058337245136499405, "beta_low": 0.98876953125, "beta_high": 0.0, "orthogonal_to_active": 0.10574370622634888, "evidence_weights": [-0.33803123235702515, -0.2401278167963028, 2.007995128631592], "agreement_weight": 3.2501022815704346, "orthogonal_weight": -1.06032133102417}, {"head": 4, "memory_to_active": 0.43294087052345276, "contribution_to_active": 0.287124902009964, "mix_mean": 0.37412649393081665, "mix_low": 0.00780487060546875, "mix_high": 0.00013256072998046875, "read_cosine": 0.8787901997566223, "beta_mean": 0.08149644732475281, "nonfinite": 0.0, "weighted_memory_to_active": 0.12748026847839355, "mix_std": 0.23440390825271606, "beta_std": 0.05605329945683479, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.15719541907310486, "evidence_weights": [-0.14859265089035034, -0.23085860908031464, 0.4939633011817932], "agreement_weight": -0.0009965607896447182, "orthogonal_weight": 2.0000550746917725}, {"head": 5, "memory_to_active": 0.11950726062059402, "contribution_to_active": 0.6537664532661438, "mix_mean": 0.600375771522522, "mix_low": 0.0025482177734375, "mix_high": 0.005581855773925781, "read_cosine": 0.2675071954727173, "beta_mean": 0.0008690644172020257, "nonfinite": 0.0, "weighted_memory_to_active": 0.07750969380140305, "mix_std": 0.2803795635700226, "beta_std": 0.005273341666907072, "beta_low": 0.9918212890625, "beta_high": 0.0, "orthogonal_to_active": 0.0617956817150116, "evidence_weights": [-0.21823711693286896, -0.2916446626186371, 0.6989343166351318], "agreement_weight": 0.09732133895158768, "orthogonal_weight": -0.8781834840774536}]}, {"layer": 14, "memory_to_active": 0.2553490996360779, "contribution_to_active": 0.4193617105484009, "mix_mean": 0.4119439721107483, "mix_low": 0.026820501312613487, "mix_high": 0.004070281982421875, "read_cosine": 0.6242091059684753, "beta_mean": 0.06933069229125977, "nonfinite": 0.0, "weighted_memory_to_active": 0.11474177241325378, "mix_std": 0.24666057527065277, "beta_std": 0.06363244354724884, "beta_low": 0.4445902705192566, "beta_high": 0.0, "orthogonal_to_active": 0.041197825223207474, "output_influence": 0.30657902359962463, "residual_influence": 0.1184774711728096, "state_active": 0.06568946689367294, "state_memory": 0.015584766864776611, "state_peak": 35.2750358581543, "growth_active": 1.1299434900283813, "growth_memory": 1.2524734735488892, "heads": [{"head": 0, "memory_to_active": 0.34058937430381775, "contribution_to_active": 0.20162415504455566, "mix_mean": 0.2166287899017334, "mix_low": 0.015824317932128906, "mix_high": 1.239776611328125e-05, "read_cosine": 0.7560440301895142, "beta_mean": 0.07102780044078827, "nonfinite": 0.0, "weighted_memory_to_active": 0.09147427976131439, "mix_std": 0.19620870053768158, "beta_std": 0.06760697066783905, "beta_low": 0.0128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.0586920827627182, "evidence_weights": [-0.1287442296743393, -0.04648726060986519, 0.6044226884841919], "agreement_weight": 0.025039685890078545, "orthogonal_weight": 1.005829930305481}, {"head": 1, "memory_to_active": 0.5179305672645569, "contribution_to_active": 0.333743155002594, "mix_mean": 0.2452222853899002, "mix_low": 0.12612628936767578, "mix_high": 0.000217437744140625, "read_cosine": 0.7773654460906982, "beta_mean": 0.28453898429870605, "nonfinite": 0.0, "weighted_memory_to_active": 0.2424566149711609, "mix_std": 0.26736509799957275, "beta_std": 0.258719801902771, "beta_low": 0.09381103515625, "beta_high": 0.0, "orthogonal_to_active": 0.016497604548931122, "evidence_weights": [-0.0012533265398815274, -0.250716894865036, -0.1789066195487976], "agreement_weight": 0.05312100052833557, "orthogonal_weight": 0.1431386023759842}, {"head": 2, "memory_to_active": 0.17126959562301636, "contribution_to_active": 0.5378819704055786, "mix_mean": 0.5441901087760925, "mix_low": 0.0030126571655273438, "mix_high": 0.0013952255249023438, "read_cosine": 0.3833640217781067, "beta_mean": 0.004979035351425409, "nonfinite": 0.0, "weighted_memory_to_active": 0.1019582748413086, "mix_std": 0.2654694616794586, "beta_std": 0.006891828030347824, "beta_low": 0.87554931640625, "beta_high": 0.0, "orthogonal_to_active": 0.07783317565917969, "evidence_weights": [-0.1609869748353958, -0.29712551832199097, 0.8734045624732971], "agreement_weight": 0.08117126673460007, "orthogonal_weight": -0.8682977557182312}, {"head": 3, "memory_to_active": 0.056907400488853455, "contribution_to_active": 0.6546591520309448, "mix_mean": 0.6680243015289307, "mix_low": 0.005076408386230469, "mix_high": 0.014888763427734375, "read_cosine": 0.26022788882255554, "beta_mean": 0.008631697855889797, "nonfinite": 0.0, "weighted_memory_to_active": 0.03867701441049576, "mix_std": 0.24592171609401703, "beta_std": 0.008335135877132416, "beta_low": 0.7032470703125, "beta_high": 0.0, "orthogonal_to_active": 0.022960849106311798, "evidence_weights": [-0.37727653980255127, -0.32681185007095337, 0.4328188896179199], "agreement_weight": 0.08319025486707687, "orthogonal_weight": -0.7057945728302002}, {"head": 4, "memory_to_active": 0.17322315275669098, "contribution_to_active": 0.4737510681152344, "mix_mean": 0.4764668345451355, "mix_low": 0.004795074462890625, "mix_high": 0.007279396057128906, "read_cosine": 0.8332037329673767, "beta_mean": 0.04438900575041771, "nonfinite": 0.0, "weighted_memory_to_active": 0.09794865548610687, "mix_std": 0.26314449310302734, "beta_std": 0.029870709404349327, "beta_low": 0.02984619140625, "beta_high": 0.0, "orthogonal_to_active": 0.027814209461212158, "evidence_weights": [-0.2189253568649292, -0.2603403925895691, 0.41764259338378906], "agreement_weight": 0.12137839198112488, "orthogonal_weight": -0.5541191101074219}, {"head": 5, "memory_to_active": 0.2721744179725647, "contribution_to_active": 0.3145107626914978, "mix_mean": 0.32113128900527954, "mix_low": 0.0060882568359375, "mix_high": 0.0006284713745117188, "read_cosine": 0.7350496053695679, "beta_mean": 0.0024176649749279022, "nonfinite": 0.0, "weighted_memory_to_active": 0.11593575030565262, "mix_std": 0.2418539673089981, "beta_std": 0.010370190255343914, "beta_low": 0.9522705078125, "beta_high": 0.0, "orthogonal_to_active": 0.04338900372385979, "evidence_weights": [-0.18175911903381348, -0.15599308907985687, 0.21586652100086212], "agreement_weight": 0.0863259807229042, "orthogonal_weight": -0.6289488077163696}]}, {"layer": 15, "memory_to_active": 0.06676627695560455, "contribution_to_active": 0.48147913813591003, "mix_mean": 0.3942873477935791, "mix_low": 0.009449958801269531, "mix_high": 0.002628803253173828, "read_cosine": 0.6616549491882324, "beta_mean": 0.04478742554783821, "nonfinite": 0.0, "weighted_memory_to_active": 0.030722498893737793, "mix_std": 0.27981650829315186, "beta_std": 0.05001573637127876, "beta_low": 0.7047933340072632, "beta_high": 0.0, "orthogonal_to_active": 0.006885765120387077, "output_influence": 0.29996874928474426, "residual_influence": 0.1610046774148941, "state_active": 0.1264709085226059, "state_memory": 0.009456688538193703, "state_peak": 33.255401611328125, "growth_active": 1.3421697616577148, "growth_memory": 1.5963997840881348, "heads": [{"head": 0, "memory_to_active": 0.11029887199401855, "contribution_to_active": 0.4648328423500061, "mix_mean": 0.36147433519363403, "mix_low": 0.006213188171386719, "mix_high": 0.0043163299560546875, "read_cosine": 0.6753045916557312, "beta_mean": 0.03828202560544014, "nonfinite": 0.0, "weighted_memory_to_active": 0.051114521920681, "mix_std": 0.27196168899536133, "beta_std": 0.028509562835097313, "beta_low": 0.116455078125, "beta_high": 0.0, "orthogonal_to_active": 0.021250147372484207, "evidence_weights": [-0.16570012271404266, -0.18032953143119812, 0.3444010019302368], "agreement_weight": 0.08777832984924316, "orthogonal_weight": -0.6597874164581299}, {"head": 1, "memory_to_active": 0.16123516857624054, "contribution_to_active": 0.4105432629585266, "mix_mean": 0.40733760595321655, "mix_low": 0.0034971237182617188, "mix_high": 0.0029964447021484375, "read_cosine": 0.8893659114837646, "beta_mean": 0.001441087806597352, "nonfinite": 0.0, "weighted_memory_to_active": 0.06443554908037186, "mix_std": 0.2793789505958557, "beta_std": 0.0038515084888786077, "beta_low": 0.97393798828125, "beta_high": 0.0, "orthogonal_to_active": 0.0016195119824260473, "evidence_weights": [-0.17281830310821533, -0.1810704469680786, 0.2559783458709717], "agreement_weight": 0.10014767199754715, "orthogonal_weight": -0.0412369966506958}, {"head": 2, "memory_to_active": 0.00597409438341856, "contribution_to_active": 0.45353326201438904, "mix_mean": 0.3305005431175232, "mix_low": 0.019468307495117188, "mix_high": 3.337860107421875e-05, "read_cosine": 0.7611171007156372, "beta_mean": 9.449177741771564e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0026894372422248125, "mix_std": 0.26964813470840454, "beta_std": 0.0001895036839414388, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0002225909411208704, "evidence_weights": [-0.20068983733654022, -0.27453190088272095, 0.25652408599853516], "agreement_weight": 0.013334937393665314, "orthogonal_weight": 0.18984776735305786}, {"head": 3, "memory_to_active": 0.019146164879202843, "contribution_to_active": 0.4313734173774719, "mix_mean": 0.3523445427417755, "mix_low": 0.005793571472167969, "mix_high": 0.000102996826171875, "read_cosine": 0.6248315572738647, "beta_mean": 0.00017166277393698692, "nonfinite": 0.0, "weighted_memory_to_active": 0.008358508348464966, "mix_std": 0.2747390866279602, "beta_std": 0.0004983014077879488, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.006169537082314491, "evidence_weights": [-0.2130962759256363, -0.25603511929512024, 0.36896324157714844], "agreement_weight": 0.03211909532546997, "orthogonal_weight": -0.9407201409339905}, {"head": 4, "memory_to_active": 0.08538967370986938, "contribution_to_active": 0.6243085265159607, "mix_mean": 0.375150203704834, "mix_low": 0.012386322021484375, "mix_high": 0.0005474090576171875, "read_cosine": 0.6507654190063477, "beta_mean": 0.2262505441904068, "nonfinite": 0.0, "weighted_memory_to_active": 0.04576452821493149, "mix_std": 0.3059687316417694, "beta_std": 0.24813050031661987, "beta_low": 0.16363525390625, "beta_high": 0.0, "orthogonal_to_active": 0.004550747107714415, "evidence_weights": [-0.22243145108222961, -0.25875934958457947, 0.03337768092751503], "agreement_weight": 0.013711280189454556, "orthogonal_weight": 0.1901141107082367}, {"head": 5, "memory_to_active": 0.018553681671619415, "contribution_to_active": 0.5042833089828491, "mix_mean": 0.5389165878295898, "mix_low": 0.009341239929199219, "mix_high": 0.0077762603759765625, "read_cosine": 0.3685451149940491, "beta_mean": 0.0024847264867275953, "nonfinite": 0.0, "weighted_memory_to_active": 0.011972448788583279, "mix_std": 0.27720245718955994, "beta_std": 0.018915023654699326, "beta_low": 0.9749755859375, "beta_high": 0.0, "orthogonal_to_active": 0.0075020557269454, "evidence_weights": [-0.3368750512599945, -0.3095215857028961, 0.6701026558876038], "agreement_weight": 0.25017374753952026, "orthogonal_weight": 0.7885617017745972}]}]}, {"step": 30000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7992010350026248, "memory_to_active": 0.8886410593986511, "contribution_to_active": 0.5070923566818237, "mix_mean": 0.3622717261314392, "mix_low": 0.1535406857728958, "mix_high": 0.008197407238185406, "read_cosine": 0.3788905739784241, "beta_mean": 0.08155450224876404, "nonfinite": 0.0, "weighted_memory_to_active": 0.20754283666610718, "mix_std": 0.2160848081111908, "beta_std": 0.06818404793739319, "beta_low": 0.49820393323898315, "beta_high": 2.4159751774277538e-05, "orthogonal_to_active": 0.052211206406354904, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1709465980529785, "normal_token_accuracy": 0.44382646679878235, "state_active": 0.05939647555351257, "state_memory": 0.019408343359827995, "state_peak": 110.01144409179688, "growth_active": 1.1236544847488403, "growth_memory": 1.400626301765442, "output_influence": 0.36650586128234863, "residual_influence": 0.11830345541238785, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.280515670776367, "no_memory_loss_delta": 0.35825151205062866, "scrambled_loss_delta": 0.13141421973705292, "memory_only_loss_delta": 2.6098971366882324, "no_evidence_loss_delta": 0.20182643830776215, "no_agreement_loss_delta": 0.016081925481557846, "no_orthogonal_loss_delta": 0.0018706778064370155}, {"index": 94, "tokens": 256, "normal_loss": 3.2554378509521484, "no_memory_loss_delta": 0.4334188401699066, "scrambled_loss_delta": 0.2993297576904297, "memory_only_loss_delta": 2.7641170024871826, "no_evidence_loss_delta": 0.1618872880935669, "no_agreement_loss_delta": 0.010120955295860767, "no_orthogonal_loss_delta": 0.004231950268149376}, {"index": 188, "tokens": 256, "normal_loss": 3.5164308547973633, "no_memory_loss_delta": 0.39819929003715515, "scrambled_loss_delta": 0.14105919003486633, "memory_only_loss_delta": 2.5068912506103516, "no_evidence_loss_delta": 0.22437627613544464, "no_agreement_loss_delta": 0.017658857628703117, "no_orthogonal_loss_delta": 0.017037054523825645}, {"index": 283, "tokens": 131, "normal_loss": 2.116567611694336, "no_memory_loss_delta": 0.22142526507377625, "scrambled_loss_delta": 0.046076297760009766, "memory_only_loss_delta": 2.826946973800659, "no_evidence_loss_delta": 0.0757293850183487, "no_agreement_loss_delta": -0.001651838538236916, "no_orthogonal_loss_delta": -0.04278476908802986}], "agreement_weight_abs_mean": 0.1432131975889206, "agreement_weight_abs_max": 3.381894111633301, "agreement_weight_mean": 0.12370262295007706, "orthogonal_weight_abs_mean": 0.6613499522209167, "orthogonal_weight_abs_max": 5.462330341339111, "orthogonal_weight_mean": -0.15535780787467957, "no_memory_loss": 3.5420405864715576, "no_memory_loss_delta": 0.37109375, "no_memory_token_accuracy_gain": 0.06229143589735031, "no_memory_token_win_fraction": 0.6362625360488892, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 31.00207470284072, "scrambled_loss": 3.3404877185821533, "scrambled_loss_delta": 0.16954128444194794, "scrambled_token_accuracy_gain": 0.021134594455361366, "scrambled_token_win_fraction": 0.6084538102149963, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 15.594809245092886, "memory_only_loss": 5.827054977416992, "memory_only_loss_delta": 2.656108856201172, "memory_only_token_accuracy_gain": 0.2747497260570526, "memory_only_token_win_fraction": 0.8731924295425415, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 92.97790694355376, "no_evidence_loss": 3.3494465351104736, "no_evidence_loss_delta": 0.17850010097026825, "no_evidence_token_accuracy_gain": 0.03337041288614273, "no_evidence_token_win_fraction": 0.5839822292327881, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 16.347602758666167, "no_agreement_loss": 3.1831955909729004, "no_agreement_loss_delta": 0.012249404564499855, "no_agreement_token_accuracy_gain": 0.0, "no_agreement_token_win_fraction": 0.45828697085380554, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 1.2174686005532076, "no_orthogonal_loss": 3.1713013648986816, "no_orthogonal_loss_delta": 0.0003547859378159046, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5283648371696472, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 0.035472300872740725, "short_no_memory_loss_delta": 0.32127436995506287, "long_context_memory_gain": 0.049819380044937134, "normal_context_loss_gain": 0.14198026061058044, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.188373327255249, "contribution_to_active": 0.9858603477478027, "mix_mean": 0.09962542355060577, "mix_low": 0.529883086681366, "mix_high": 0.004451274871826172, "read_cosine": 0.4500172734260559, "beta_mean": 0.25107133388519287, "nonfinite": 0.0, "weighted_memory_to_active": 1.0532639026641846, "mix_std": 0.08695651590824127, "beta_std": 0.14361611008644104, "beta_low": 0.0174051932990551, "beta_high": 0.0, "orthogonal_to_active": 0.06894385814666748, "output_influence": 0.17156843841075897, "residual_influence": 0.20440539717674255, "state_active": 0.07370167225599289, "state_memory": 0.04720614105463028, "state_peak": 42.57006072998047, "growth_active": 1.261872410774231, "growth_memory": 1.2569226026535034, "heads": [{"head": 0, "memory_to_active": 2.253394603729248, "contribution_to_active": 0.42678970098495483, "mix_mean": 0.018506404012441635, "mix_low": 0.6156988143920898, "mix_high": 0.0, "read_cosine": 0.2653917074203491, "beta_mean": 0.21140854060649872, "nonfinite": 0.0, "weighted_memory_to_active": 0.5038440227508545, "mix_std": 0.05494345724582672, "beta_std": 0.098093681037426, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.036489687860012054, "evidence_weights": [-0.027307452633976936, -0.07590952515602112, -0.09259834885597229], "agreement_weight": -0.10069404542446136, "orthogonal_weight": -0.08124839514493942}, {"head": 1, "memory_to_active": 5.356548309326172, "contribution_to_active": 2.008795738220215, "mix_mean": 0.015389163047075272, "mix_low": 0.6548185348510742, "mix_high": 0.0, "read_cosine": 0.4972342252731323, "beta_mean": 0.43026742339134216, "nonfinite": 0.0, "weighted_memory_to_active": 2.391157865524292, "mix_std": 0.048891738057136536, "beta_std": 0.1835457980632782, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.15298789739608765, "evidence_weights": [0.025685476139187813, -0.08831238001585007, 0.3343239724636078], "agreement_weight": 0.02753637544810772, "orthogonal_weight": -0.13331350684165955}, {"head": 2, "memory_to_active": 2.628909111022949, "contribution_to_active": 0.5391205549240112, "mix_mean": 0.014782164245843887, "mix_low": 0.6483840942382812, "mix_high": 0.0, "read_cosine": 0.3144189715385437, "beta_mean": 0.3040500283241272, "nonfinite": 0.0, "weighted_memory_to_active": 0.6348646879196167, "mix_std": 0.03308149427175522, "beta_std": 0.20277753472328186, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05966777727007866, "evidence_weights": [0.047214653342962265, 0.060845620930194855, -0.035689130425453186], "agreement_weight": 0.008176854811608791, "orthogonal_weight": -0.1914971023797989}, {"head": 3, "memory_to_active": 6.2095136642456055, "contribution_to_active": 2.5022029876708984, "mix_mean": 0.039632558822631836, "mix_low": 0.4251575469970703, "mix_high": 0.0, "read_cosine": 0.36125078797340393, "beta_mean": 0.3600310981273651, "nonfinite": 0.0, "weighted_memory_to_active": 2.683090925216675, "mix_std": 0.08570800721645355, "beta_std": 0.23705700039863586, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1413242369890213, "evidence_weights": [-0.06087413802742958, -0.08109664916992188, -0.1313934475183487], "agreement_weight": -0.021894000470638275, "orthogonal_weight": -0.062843456864357}, {"head": 4, "memory_to_active": 2.574686050415039, "contribution_to_active": 0.06675384938716888, "mix_mean": 0.006633885204792023, "mix_low": 0.8214912414550781, "mix_high": 0.0, "read_cosine": 0.3357968330383301, "beta_mean": 0.1574770212173462, "nonfinite": 0.0, "weighted_memory_to_active": 0.05687302350997925, "mix_std": 0.013854487799108028, "beta_std": 0.10053012520074844, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005973258055746555, "evidence_weights": [0.11629194021224976, 0.06554810702800751, -0.3639124035835266], "agreement_weight": -0.10004030168056488, "orthogonal_weight": 0.10581234842538834}, {"head": 5, "memory_to_active": 0.10718726366758347, "contribution_to_active": 0.37149959802627563, "mix_mean": 0.5028083920478821, "mix_low": 0.0137481689453125, "mix_high": 0.02670764923095703, "read_cosine": 0.9260109663009644, "beta_mean": 0.04319380223751068, "nonfinite": 0.0, "weighted_memory_to_active": 0.04975174367427826, "mix_std": 0.2852599322795868, "beta_std": 0.0396924763917923, "beta_low": 0.10443115234375, "beta_high": 0.0, "orthogonal_to_active": 0.01722024753689766, "evidence_weights": [-0.2472444772720337, -0.24592803418636322, 0.2731400728225708], "agreement_weight": 0.23621006309986115, "orthogonal_weight": -0.6251517534255981}]}, {"layer": 1, "memory_to_active": 2.2680742740631104, "contribution_to_active": 0.47414612770080566, "mix_mean": 0.17261378467082977, "mix_low": 0.3686103820800781, "mix_high": 0.0006550153484568, "read_cosine": 0.3392638564109802, "beta_mean": 0.18457894027233124, "nonfinite": 0.0, "weighted_memory_to_active": 0.3600453734397888, "mix_std": 0.11874658614397049, "beta_std": 0.1249874085187912, "beta_low": 0.212310791015625, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.05069667473435402, "output_influence": 0.1795254647731781, "residual_influence": 0.043451227247714996, "state_active": 0.03715706244111061, "state_memory": 0.03606594353914261, "state_peak": 34.50217056274414, "growth_active": 1.1468838453292847, "growth_memory": 1.499661922454834, "heads": [{"head": 0, "memory_to_active": 0.05301746353507042, "contribution_to_active": 0.41355425119400024, "mix_mean": 0.4117521345615387, "mix_low": 0.013016700744628906, "mix_high": 0.00032329559326171875, "read_cosine": 0.008575002662837505, "beta_mean": 0.00035426660906523466, "nonfinite": 0.0, "weighted_memory_to_active": 0.02824111096560955, "mix_std": 0.2801588773727417, "beta_std": 0.0004640783299691975, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.026292361319065094, "evidence_weights": [-0.17898240685462952, -0.23060233891010284, -0.0013671255437657237], "agreement_weight": 0.12076170742511749, "orthogonal_weight": -1.0023443698883057}, {"head": 1, "memory_to_active": 2.830153465270996, "contribution_to_active": 0.2149825394153595, "mix_mean": 0.023211900144815445, "mix_low": 0.47042274475097656, "mix_high": 9.5367431640625e-07, "read_cosine": 0.4654066264629364, "beta_mean": 0.29794150590896606, "nonfinite": 0.0, "weighted_memory_to_active": 0.2401197850704193, "mix_std": 0.051371827721595764, "beta_std": 0.18196824193000793, "beta_low": 0.0, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.011993390507996082, "evidence_weights": [0.0076932646334171295, -0.02206520177423954, -0.007286028005182743], "agreement_weight": -0.006808174774050713, "orthogonal_weight": 0.10757773369550705}, {"head": 2, "memory_to_active": 0.3698529601097107, "contribution_to_active": 0.5245603322982788, "mix_mean": 0.5403077602386475, "mix_low": 0.0026597976684570312, "mix_high": 0.0036039352416992188, "read_cosine": 0.7645758390426636, "beta_mean": 0.02857591211795807, "nonfinite": 0.0, "weighted_memory_to_active": 0.2977152168750763, "mix_std": 0.2567221522331238, "beta_std": 0.022437313571572304, "beta_low": 0.210205078125, "beta_high": 0.0, "orthogonal_to_active": 0.04671287536621094, "evidence_weights": [-0.281948983669281, -0.36733049154281616, 0.32395777106285095], "agreement_weight": 0.02777176909148693, "orthogonal_weight": -0.2962971031665802}, {"head": 3, "memory_to_active": 2.9216978549957275, "contribution_to_active": 0.1341249644756317, "mix_mean": 0.016681233420968056, "mix_low": 0.6543807983398438, "mix_high": 0.0, "read_cosine": 0.25011372566223145, "beta_mean": 0.35898518562316895, "nonfinite": 0.0, "weighted_memory_to_active": 0.2032236009836197, "mix_std": 0.04497538506984711, "beta_std": 0.2534150183200836, "beta_low": 0.04290771484375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.05517217516899109, "evidence_weights": [0.041290685534477234, -0.0021594760473817587, 0.12605053186416626], "agreement_weight": -0.03453398123383522, "orthogonal_weight": -0.34085750579833984}, {"head": 4, "memory_to_active": 3.5131173133850098, "contribution_to_active": 1.4978435039520264, "mix_mean": 0.03578821197152138, "mix_low": 0.2693214416503906, "mix_high": 1.9073486328125e-06, "read_cosine": 0.3347078263759613, "beta_mean": 0.16604742407798767, "nonfinite": 0.0, "weighted_memory_to_active": 1.3384580612182617, "mix_std": 0.061256617307662964, "beta_std": 0.10316051542758942, "beta_low": 0.00811767578125, "beta_high": 0.0, "orthogonal_to_active": 0.15909521281719208, "evidence_weights": [-0.04405447095632553, 0.006222326774150133, -0.0006134724244475365], "agreement_weight": -0.016830163076519966, "orthogonal_weight": 0.14832887053489685}, {"head": 5, "memory_to_active": 3.9206066131591797, "contribution_to_active": 0.05981096625328064, "mix_mean": 0.007941465824842453, "mix_low": 0.8018608093261719, "mix_high": 0.0, "read_cosine": 0.21220412850379944, "beta_mean": 0.2555692493915558, "nonfinite": 0.0, "weighted_memory_to_active": 0.05251437798142433, "mix_std": 0.017994582653045654, "beta_std": 0.1884792000055313, "beta_low": 0.01263427734375, "beta_high": 0.0, "orthogonal_to_active": 0.004914035089313984, "evidence_weights": [0.15481805801391602, 0.07563044130802155, 0.3050328493118286], "agreement_weight": -0.1423647403717041, "orthogonal_weight": 0.09533911198377609}]}, {"layer": 2, "memory_to_active": 1.026466727256775, "contribution_to_active": 0.33927780389785767, "mix_mean": 0.3434470593929291, "mix_low": 0.25608062744140625, "mix_high": 0.0036071143113076687, "read_cosine": 0.32584211230278015, "beta_mean": 0.09623030573129654, "nonfinite": 0.0, "weighted_memory_to_active": 0.07749757170677185, "mix_std": 0.19609268009662628, "beta_std": 0.07470552623271942, "beta_low": 0.5428365468978882, "beta_high": 0.00022379557776730508, "orthogonal_to_active": 0.01910913735628128, "output_influence": 0.31970077753067017, "residual_influence": 0.07971546053886414, "state_active": 0.03836614638566971, "state_memory": 0.018927421420812607, "state_peak": 37.01900100708008, "growth_active": 1.0491873025894165, "growth_memory": 1.5616497993469238, "heads": [{"head": 0, "memory_to_active": 0.009803464636206627, "contribution_to_active": 0.3128716051578522, "mix_mean": 0.5083338022232056, "mix_low": 0.0019969940185546875, "mix_high": 0.005504608154296875, "read_cosine": 0.17068476974964142, "beta_mean": 5.419857188826427e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005574593786150217, "mix_std": 0.29304203391075134, "beta_std": 7.979121437529102e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013642803533002734, "evidence_weights": [-0.29676488041877747, -0.18460790812969208, 0.1751089245080948], "agreement_weight": 0.1882912814617157, "orthogonal_weight": -0.25916969776153564}, {"head": 1, "memory_to_active": 2.881002426147461, "contribution_to_active": 0.05215068161487579, "mix_mean": 0.005811224691569805, "mix_low": 0.8790769577026367, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21149946749210358, "beta_mean": 0.2754085659980774, "nonfinite": 0.0, "weighted_memory_to_active": 0.04012725129723549, "mix_std": 0.014530111104249954, "beta_std": 0.15970809757709503, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.015857461839914322, "evidence_weights": [0.09845414012670517, 0.09962582588195801, 0.48538652062416077], "agreement_weight": 0.04699937626719475, "orthogonal_weight": 0.43925097584724426}, {"head": 2, "memory_to_active": 0.020085396245121956, "contribution_to_active": 0.4223267436027527, "mix_mean": 0.47430241107940674, "mix_low": 0.0021686553955078125, "mix_high": 0.0027446746826171875, "read_cosine": 0.2787957191467285, "beta_mean": 4.252116195857525e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007886541076004505, "mix_std": 0.2803424000740051, "beta_std": 6.246691918931901e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018316918285563588, "evidence_weights": [-0.22540226578712463, -0.26408281922340393, 0.4605437219142914], "agreement_weight": 0.037102289497852325, "orthogonal_weight": -0.2669237554073334}, {"head": 3, "memory_to_active": 0.20548783242702484, "contribution_to_active": 0.47850915789604187, "mix_mean": 0.4589553773403168, "mix_low": 0.0006866455078125, "mix_high": 0.005603790283203125, "read_cosine": 0.6258741617202759, "beta_mean": 0.04227111488580704, "nonfinite": 0.0, "weighted_memory_to_active": 0.11758439987897873, "mix_std": 0.2675972580909729, "beta_std": 0.05070282146334648, "beta_low": 0.22662353515625, "beta_high": 0.0, "orthogonal_to_active": 0.07603222131729126, "evidence_weights": [-0.1692602038383484, -0.265272855758667, 0.8540611863136292], "agreement_weight": 0.06224162131547928, "orthogonal_weight": -0.8774814605712891}, {"head": 4, "memory_to_active": 3.027167558670044, "contribution_to_active": 0.24150612950325012, "mix_mean": 0.017039243131875992, "mix_low": 0.6508626937866211, "mix_high": 0.0, "read_cosine": 0.2133440375328064, "beta_mean": 0.25956064462661743, "nonfinite": 0.0, "weighted_memory_to_active": 0.2849644422531128, "mix_std": 0.04629029706120491, "beta_std": 0.23761960864067078, "beta_low": 0.03021240234375, "beta_high": 0.0013427734375, "orthogonal_to_active": 0.019219225272536278, "evidence_weights": [0.029911406338214874, 0.025326156988739967, 0.07485705614089966], "agreement_weight": 0.021757088601589203, "orthogonal_weight": -0.07661960273981094}, {"head": 5, "memory_to_active": 0.01525395829230547, "contribution_to_active": 0.5283023715019226, "mix_mean": 0.5962401628494263, "mix_low": 0.0016918182373046875, "mix_high": 0.007788658142089844, "read_cosine": 0.4548546075820923, "beta_mean": 4.4776090362574905e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008848171681165695, "mix_std": 0.2747539281845093, "beta_std": 6.038296487531625e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003499406739138067, "evidence_weights": [-0.2601814866065979, -0.2823127508163452, 0.8248477578163147], "agreement_weight": 0.0817386582493782, "orthogonal_weight": 0.06360927224159241}]}, {"layer": 3, "memory_to_active": 1.0368211269378662, "contribution_to_active": 0.3838179111480713, "mix_mean": 0.3293120563030243, "mix_low": 0.1737333983182907, "mix_high": 0.006832440849393606, "read_cosine": 0.23649731278419495, "beta_mean": 0.06057899445295334, "nonfinite": 0.0, "weighted_memory_to_active": 0.16306719183921814, "mix_std": 0.23217618465423584, "beta_std": 0.05471773073077202, "beta_low": 0.467010498046875, "beta_high": 0.0, "orthogonal_to_active": 0.06564129889011383, "output_influence": 0.30751681327819824, "residual_influence": 0.05980009585618973, "state_active": 0.021274954080581665, "state_memory": 0.013860045000910759, "state_peak": 21.382814407348633, "growth_active": 0.9511188268661499, "growth_memory": 1.9291377067565918, "heads": [{"head": 0, "memory_to_active": 0.5457849502563477, "contribution_to_active": 0.47743576765060425, "mix_mean": 0.43174129724502563, "mix_low": 0.019690513610839844, "mix_high": 0.008794784545898438, "read_cosine": 0.24611522257328033, "beta_mean": 0.014983117580413818, "nonfinite": 0.0, "weighted_memory_to_active": 0.26612523198127747, "mix_std": 0.32084590196609497, "beta_std": 0.025571059435606003, "beta_low": 0.55999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.1753244400024414, "evidence_weights": [-0.17290803790092468, -0.21161913871765137, 0.6299054622650146], "agreement_weight": 0.12610113620758057, "orthogonal_weight": -0.694263219833374}, {"head": 1, "memory_to_active": 0.012205749750137329, "contribution_to_active": 0.35632073879241943, "mix_mean": 0.427959680557251, "mix_low": 0.01922893524169922, "mix_high": 0.004633903503417969, "read_cosine": 0.16169120371341705, "beta_mean": 6.292360194493085e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005454829428344965, "mix_std": 0.30915358662605286, "beta_std": 0.0001316736452281475, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026896893978118896, "evidence_weights": [-0.2094925343990326, -0.18132051825523376, 0.5736222267150879], "agreement_weight": 0.3184792697429657, "orthogonal_weight": -0.5353339910507202}, {"head": 2, "memory_to_active": 2.915006637573242, "contribution_to_active": 0.3263576328754425, "mix_mean": 0.04609956592321396, "mix_low": 0.3620176315307617, "mix_high": 0.0, "read_cosine": 0.21088556945323944, "beta_mean": 0.1942596286535263, "nonfinite": 0.0, "weighted_memory_to_active": 0.37628450989723206, "mix_std": 0.10226564109325409, "beta_std": 0.14287112653255463, "beta_low": 0.009521484375, "beta_high": 0.0, "orthogonal_to_active": 0.002818488283082843, "evidence_weights": [-0.06277728825807571, 0.019115662202239037, -0.02580004557967186], "agreement_weight": -0.01122491154819727, "orthogonal_weight": -0.009366724640130997}, {"head": 3, "memory_to_active": 0.02562551014125347, "contribution_to_active": 0.4784640073776245, "mix_mean": 0.5233944654464722, "mix_low": 0.013126373291015625, "mix_high": 0.012400627136230469, "read_cosine": 0.3202891945838928, "beta_mean": 0.00010460442717885599, "nonfinite": 0.0, "weighted_memory_to_active": 0.016589095816016197, "mix_std": 0.3108621835708618, "beta_std": 0.00022364546020980924, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00446767034009099, "evidence_weights": [-0.20694339275360107, -0.2212374210357666, 1.9086329936981201], "agreement_weight": 1.2935497760772705, "orthogonal_weight": -0.3602408468723297}, {"head": 4, "memory_to_active": 2.383180856704712, "contribution_to_active": 0.03935563564300537, "mix_mean": 0.020078692585229874, "mix_low": 0.6115474700927734, "mix_high": 0.0, "read_cosine": 0.16535013914108276, "beta_mean": 0.10320599377155304, "nonfinite": 0.0, "weighted_memory_to_active": 0.08697596937417984, "mix_std": 0.044142842292785645, "beta_std": 0.11239180713891983, "beta_low": 0.093505859375, "beta_high": 0.0, "orthogonal_to_active": 0.05874871090054512, "evidence_weights": [0.11165669560432434, 0.05999986454844475, 0.09196578711271286], "agreement_weight": -0.03723050281405449, "orthogonal_weight": -0.6873961091041565}, {"head": 5, "memory_to_active": 0.33912256360054016, "contribution_to_active": 0.6249735951423645, "mix_mean": 0.5265984535217285, "mix_low": 0.01678943634033203, "mix_high": 0.015165328979492188, "read_cosine": 0.3146524429321289, "beta_mean": 0.05085768550634384, "nonfinite": 0.0, "weighted_memory_to_active": 0.22697347402572632, "mix_std": 0.3057869076728821, "beta_std": 0.04711708426475525, "beta_low": 0.1390380859375, "beta_high": 0.0, "orthogonal_to_active": 0.14979881048202515, "evidence_weights": [-0.2075958549976349, -0.2456568032503128, 0.44714993238449097], "agreement_weight": 0.1415548473596573, "orthogonal_weight": -0.7714973092079163}]}, {"layer": 4, "memory_to_active": 1.245370626449585, "contribution_to_active": 0.4181804656982422, "mix_mean": 0.3391934037208557, "mix_low": 0.2842380404472351, "mix_high": 0.005688031669706106, "read_cosine": 0.3471297025680542, "beta_mean": 0.13590878248214722, "nonfinite": 0.0, "weighted_memory_to_active": 0.145050048828125, "mix_std": 0.1542799472808838, "beta_std": 0.11721138656139374, "beta_low": 0.3643188774585724, "beta_high": 0.0, "orthogonal_to_active": 0.03815440088510513, "output_influence": 0.36845093965530396, "residual_influence": 0.09630197286605835, "state_active": 0.034054454416036606, "state_memory": 0.020919322967529297, "state_peak": 39.643959045410156, "growth_active": 1.1567363739013672, "growth_memory": 1.6084351539611816, "heads": [{"head": 0, "memory_to_active": 0.2573234736919403, "contribution_to_active": 0.706612229347229, "mix_mean": 0.6469322443008423, "mix_low": 0.007840156555175781, "mix_high": 0.011373519897460938, "read_cosine": 0.5050806999206543, "beta_mean": 0.263456255197525, "nonfinite": 0.0, "weighted_memory_to_active": 0.20254284143447876, "mix_std": 0.3312249183654785, "beta_std": 0.2688569724559784, "beta_low": 0.19525146484375, "beta_high": 0.0, "orthogonal_to_active": 0.06008560582995415, "evidence_weights": [-0.26720720529556274, -0.36223921179771423, 0.29365724325180054], "agreement_weight": 0.03358837962150574, "orthogonal_weight": 0.37329062819480896}, {"head": 1, "memory_to_active": 0.08638323843479156, "contribution_to_active": 0.5754843950271606, "mix_mean": 0.6094325184822083, "mix_low": 0.0001087188720703125, "mix_high": 0.006062507629394531, "read_cosine": 0.7203465700149536, "beta_mean": 0.0019166306592524052, "nonfinite": 0.0, "weighted_memory_to_active": 0.05651453882455826, "mix_std": 0.2548311650753021, "beta_std": 0.00603880500420928, "beta_low": 0.961669921875, "beta_high": 0.0, "orthogonal_to_active": 0.031122922897338867, "evidence_weights": [-0.23576609790325165, -0.30588284134864807, 0.7275553345680237], "agreement_weight": 0.10522175580263138, "orthogonal_weight": -0.8314489722251892}, {"head": 2, "memory_to_active": 2.4179086685180664, "contribution_to_active": 0.15667417645454407, "mix_mean": 0.016764627769589424, "mix_low": 0.6038846969604492, "mix_high": 0.0, "read_cosine": 0.2300533652305603, "beta_mean": 0.1358129382133484, "nonfinite": 0.0, "weighted_memory_to_active": 0.18862059712409973, "mix_std": 0.033160120248794556, "beta_std": 0.10356064140796661, "beta_low": 0.0013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.013172941282391548, "evidence_weights": [0.0578206405043602, 0.032658886164426804, -0.011827567592263222], "agreement_weight": 0.11525357514619827, "orthogonal_weight": -0.10611003637313843}, {"head": 3, "memory_to_active": 2.0598411560058594, "contribution_to_active": 0.14470578730106354, "mix_mean": 0.028906190767884254, "mix_low": 0.42664527893066406, "mix_high": 0.0, "read_cosine": 0.16943426430225372, "beta_mean": 0.1731845736503601, "nonfinite": 0.0, "weighted_memory_to_active": 0.2102765440940857, "mix_std": 0.05253523588180542, "beta_std": 0.14428706467151642, "beta_low": 0.02764892578125, "beta_high": 0.0, "orthogonal_to_active": 0.012565241195261478, "evidence_weights": [-0.02094845473766327, 0.05718206241726875, -0.25441688299179077], "agreement_weight": 0.05560353770852089, "orthogonal_weight": -0.08339953422546387}, {"head": 4, "memory_to_active": 0.050645966082811356, "contribution_to_active": 0.6538524031639099, "mix_mean": 0.7200168371200562, "mix_low": 0.00023555755615234375, "mix_high": 0.016686439514160156, "read_cosine": 0.16432282328605652, "beta_mean": 8.345834794454277e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.03868769109249115, "mix_std": 0.22475510835647583, "beta_std": 0.00034725182922556996, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005833037197589874, "evidence_weights": [-0.3145827054977417, -0.3392084538936615, 0.8058043122291565], "agreement_weight": 0.1974402666091919, "orthogonal_weight": -0.16729934513568878}, {"head": 5, "memory_to_active": 2.600120782852173, "contribution_to_active": 0.27175387740135193, "mix_mean": 0.013107981532812119, "mix_low": 0.6667137145996094, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2935406565666199, "beta_mean": 0.24099884927272797, "nonfinite": 0.0, "weighted_memory_to_active": 0.17365804314613342, "mix_std": 0.029173126444220543, "beta_std": 0.18017758429050446, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10614664852619171, "evidence_weights": [0.04642276093363762, -0.03545614704489708, -0.24111470580101013], "agreement_weight": -0.05389497056603432, "orthogonal_weight": 0.8210161924362183}]}, {"layer": 5, "memory_to_active": 0.5565637350082397, "contribution_to_active": 0.4782216250896454, "mix_mean": 0.43367475271224976, "mix_low": 0.07674455642700195, "mix_high": 0.012515386566519737, "read_cosine": 0.18455642461776733, "beta_mean": 0.058915287256240845, "nonfinite": 0.0, "weighted_memory_to_active": 0.15549668669700623, "mix_std": 0.2506473660469055, "beta_std": 0.07541868090629578, "beta_low": 0.6223246455192566, "beta_high": 0.0, "orthogonal_to_active": 0.054827313870191574, "output_influence": 0.47653236985206604, "residual_influence": 0.12042155116796494, "state_active": 0.046957798302173615, "state_memory": 0.01226274948567152, "state_peak": 55.980613708496094, "growth_active": 1.156166672706604, "growth_memory": 1.3014461994171143, "heads": [{"head": 0, "memory_to_active": 0.8321096897125244, "contribution_to_active": 0.34040749073028564, "mix_mean": 0.2014024257659912, "mix_low": 0.07701492309570312, "mix_high": 0.00044918060302734375, "read_cosine": -0.1336020678281784, "beta_mean": 0.05057310312986374, "nonfinite": 0.0, "weighted_memory_to_active": 0.1627575159072876, "mix_std": 0.2227053940296173, "beta_std": 0.0806603729724884, "beta_low": 0.27276611328125, "beta_high": 0.0, "orthogonal_to_active": 0.037335656583309174, "evidence_weights": [-0.08319773524999619, -0.1176823079586029, -0.3128037452697754], "agreement_weight": -0.004182911943644285, "orthogonal_weight": -0.22490155696868896}, {"head": 1, "memory_to_active": 0.021104833111166954, "contribution_to_active": 0.5176432132720947, "mix_mean": 0.6098488569259644, "mix_low": 0.0038690567016601562, "mix_high": 0.01800251007080078, "read_cosine": 0.19857296347618103, "beta_mean": 5.323231016518548e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010861818678677082, "mix_std": 0.2994800806045532, "beta_std": 9.68766980804503e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009876742959022522, "evidence_weights": [-0.23616157472133636, -0.27029210329055786, 0.8189677000045776], "agreement_weight": 0.4388568103313446, "orthogonal_weight": -0.9476385116577148}, {"head": 2, "memory_to_active": 0.026179544627666473, "contribution_to_active": 0.5443030595779419, "mix_mean": 0.6217453479766846, "mix_low": 0.0024213790893554688, "mix_high": 0.015817642211914062, "read_cosine": 0.2186013162136078, "beta_mean": 0.0003626807883847505, "nonfinite": 0.0, "weighted_memory_to_active": 0.017978288233280182, "mix_std": 0.274694561958313, "beta_std": 0.000534711463842541, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.0033477863762527704, "evidence_weights": [-0.2531726062297821, -0.34768250584602356, 0.8701563477516174], "agreement_weight": 0.003611647291108966, "orthogonal_weight": -0.23367609083652496}, {"head": 3, "memory_to_active": 1.8971179723739624, "contribution_to_active": 0.22946614027023315, "mix_mean": 0.08563865721225739, "mix_low": 0.18064212799072266, "mix_high": 0.0, "read_cosine": 0.14726431667804718, "beta_mean": 0.0960971862077713, "nonfinite": 0.0, "weighted_memory_to_active": 0.371060311794281, "mix_std": 0.12838733196258545, "beta_std": 0.13969452679157257, "beta_low": 0.2115478515625, "beta_high": 0.0, "orthogonal_to_active": 0.13585947453975677, "evidence_weights": [-0.05471735820174217, -0.0876200944185257, -0.08139827847480774], "agreement_weight": 0.0545852892100811, "orthogonal_weight": -0.4464392066001892}, {"head": 4, "memory_to_active": 0.44523492455482483, "contribution_to_active": 0.5988377332687378, "mix_mean": 0.32034632563591003, "mix_low": 0.1936969757080078, "mix_high": 0.008731842041015625, "read_cosine": 0.5380764007568359, "beta_mean": 0.20602001249790192, "nonfinite": 0.0, "weighted_memory_to_active": 0.2897321581840515, "mix_std": 0.33494096994400024, "beta_std": 0.22941739857196808, "beta_low": 0.255859375, "beta_high": 0.0, "orthogonal_to_active": 0.08954794704914093, "evidence_weights": [-0.11821787804365158, -0.2913268804550171, -0.06043832004070282], "agreement_weight": 0.04710136726498604, "orthogonal_weight": 0.4584329128265381}, {"head": 5, "memory_to_active": 0.11763520538806915, "contribution_to_active": 0.6386720538139343, "mix_mean": 0.7630667686462402, "mix_low": 0.0028228759765625, "mix_high": 0.03209114074707031, "read_cosine": 0.13842558860778809, "beta_mean": 0.000385503051802516, "nonfinite": 0.0, "weighted_memory_to_active": 0.08059006184339523, "mix_std": 0.24367579817771912, "beta_std": 0.002108239568769932, "beta_low": 0.9940185546875, "beta_high": 0.0, "orthogonal_to_active": 0.052996255457401276, "evidence_weights": [-0.3252074420452118, -0.3199419677257538, 2.028130531311035], "agreement_weight": 0.37445420026779175, "orthogonal_weight": -0.692595362663269}]}, {"layer": 6, "memory_to_active": 0.3799877166748047, "contribution_to_active": 0.5982618927955627, "mix_mean": 0.5469878911972046, "mix_low": 0.0766647681593895, "mix_high": 0.029008230194449425, "read_cosine": 0.29257360100746155, "beta_mean": 0.07186122238636017, "nonfinite": 0.0, "weighted_memory_to_active": 0.1716061383485794, "mix_std": 0.24180886149406433, "beta_std": 0.07852590084075928, "beta_low": 0.4939371943473816, "beta_high": 0.0, "orthogonal_to_active": 0.037133216857910156, "output_influence": 0.5094956159591675, "residual_influence": 0.163210928440094, "state_active": 0.04692097008228302, "state_memory": 0.00919407419860363, "state_peak": 84.39957427978516, "growth_active": 1.1898589134216309, "growth_memory": 1.218542218208313, "heads": [{"head": 0, "memory_to_active": 0.14713488519191742, "contribution_to_active": 0.6366287469863892, "mix_mean": 0.7272568345069885, "mix_low": 0.00027561187744140625, "mix_high": 0.008933067321777344, "read_cosine": 0.018197286874055862, "beta_mean": 0.013843481428921223, "nonfinite": 0.0, "weighted_memory_to_active": 0.11045639961957932, "mix_std": 0.2274036556482315, "beta_std": 0.01635882630944252, "beta_low": 0.5234375, "beta_high": 0.0, "orthogonal_to_active": 0.09050881862640381, "evidence_weights": [-0.3107096552848816, -0.3312731385231018, 0.5314069986343384], "agreement_weight": 0.1382349133491516, "orthogonal_weight": -0.9267919659614563}, {"head": 1, "memory_to_active": 0.003668115008622408, "contribution_to_active": 0.5107669830322266, "mix_mean": 0.5122077465057373, "mix_low": 0.002086639404296875, "mix_high": 0.009545326232910156, "read_cosine": 0.5929312109947205, "beta_mean": 8.193654139176942e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0016444536158815026, "mix_std": 0.29003459215164185, "beta_std": 1.6195614080061205e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005742701468989253, "evidence_weights": [-0.20201410353183746, -0.28276142477989197, 0.9979567527770996], "agreement_weight": 0.003165051806718111, "orthogonal_weight": -0.5098831057548523}, {"head": 2, "memory_to_active": 1.1799395084381104, "contribution_to_active": 0.08174394071102142, "mix_mean": 0.027198409661650658, "mix_low": 0.44470691680908203, "mix_high": 0.0, "read_cosine": 0.20362251996994019, "beta_mean": 0.06578899919986725, "nonfinite": 0.0, "weighted_memory_to_active": 0.06727085262537003, "mix_std": 0.04809097573161125, "beta_std": 0.06140561401844025, "beta_low": 0.0203857421875, "beta_high": 0.0, "orthogonal_to_active": 0.00012577863526530564, "evidence_weights": [0.0920090526342392, -0.0026666561607271433, 0.2902452051639557], "agreement_weight": 0.13346721231937408, "orthogonal_weight": -0.0022335010580718517}, {"head": 3, "memory_to_active": 0.14887385070323944, "contribution_to_active": 0.6665554642677307, "mix_mean": 0.63419508934021, "mix_low": 0.011218070983886719, "mix_high": 0.017304420471191406, "read_cosine": 0.482303261756897, "beta_mean": 0.2101437747478485, "nonfinite": 0.0, "weighted_memory_to_active": 0.10555558651685715, "mix_std": 0.3320036828517914, "beta_std": 0.23884263634681702, "beta_low": 0.198486328125, "beta_high": 0.0, "orthogonal_to_active": 0.05648433044552803, "evidence_weights": [-0.17659473419189453, -0.3385309875011444, 0.15844130516052246], "agreement_weight": -0.008357149548828602, "orthogonal_weight": 0.6252470016479492}, {"head": 4, "memory_to_active": 0.765515923500061, "contribution_to_active": 1.0974985361099243, "mix_mean": 0.7574082016944885, "mix_low": 0.0013027191162109375, "mix_high": 0.13500499725341797, "read_cosine": 0.1688806265592575, "beta_mean": 0.14109551906585693, "nonfinite": 0.0, "weighted_memory_to_active": 0.7207309603691101, "mix_std": 0.289834201335907, "beta_std": 0.15400657057762146, "beta_low": 0.2215576171875, "beta_high": 0.0, "orthogonal_to_active": 0.05081972852349281, "evidence_weights": [-0.25684353709220886, -0.5904775857925415, -0.08561171591281891], "agreement_weight": -0.020336274057626724, "orthogonal_weight": 0.08306574076414108}, {"head": 5, "memory_to_active": 0.0347939096391201, "contribution_to_active": 0.5963775515556335, "mix_mean": 0.6236611604690552, "mix_low": 0.0003986358642578125, "mix_high": 0.003261566162109375, "read_cosine": 0.28950661420822144, "beta_mean": 0.0002874140627682209, "nonfinite": 0.0, "weighted_memory_to_active": 0.023978576064109802, "mix_std": 0.2634860575199127, "beta_std": 0.0005255269352346659, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.02428634651005268, "evidence_weights": [-0.2396785318851471, -0.3039436638355255, 0.41611364483833313], "agreement_weight": 0.13078463077545166, "orthogonal_weight": -1.0852326154708862}]}, {"layer": 7, "memory_to_active": 1.726496934890747, "contribution_to_active": 0.20273497700691223, "mix_mean": 0.08382357656955719, "mix_low": 0.3010786473751068, "mix_high": 0.00010267893958371133, "read_cosine": 0.2457965612411499, "beta_mean": 0.1179918572306633, "nonfinite": 0.0, "weighted_memory_to_active": 0.20473453402519226, "mix_std": 0.11472895741462708, "beta_std": 0.10002660751342773, "beta_low": 0.1701863706111908, "beta_high": 0.00011189779615961015, "orthogonal_to_active": 0.05620657652616501, "output_influence": 0.2136097401380539, "residual_influence": 0.0476541668176651, "state_active": 0.014660954475402832, "state_memory": 0.02716979756951332, "state_peak": 31.631074905395508, "growth_active": 1.1197444200515747, "growth_memory": 1.3260679244995117, "heads": [{"head": 0, "memory_to_active": 2.3003549575805664, "contribution_to_active": 0.280253142118454, "mix_mean": 0.033939436078071594, "mix_low": 0.43253421783447266, "mix_high": 9.5367431640625e-07, "read_cosine": 0.40901803970336914, "beta_mean": 0.16414028406143188, "nonfinite": 0.0, "weighted_memory_to_active": 0.3008699417114258, "mix_std": 0.0700528472661972, "beta_std": 0.11877210438251495, "beta_low": 0.00286865234375, "beta_high": 0.00048828125, "orthogonal_to_active": 0.01598505489528179, "evidence_weights": [0.008120568469166756, -0.08803677558898926, 0.12701515853405], "agreement_weight": -0.03780316933989525, "orthogonal_weight": 0.08082561939954758}, {"head": 1, "memory_to_active": 1.1212480068206787, "contribution_to_active": 0.3318449854850769, "mix_mean": 0.1704254150390625, "mix_low": 0.177947998046875, "mix_high": 0.0005397796630859375, "read_cosine": 0.3108852803707123, "beta_mean": 0.11669361591339111, "nonfinite": 0.0, "weighted_memory_to_active": 0.31216543912887573, "mix_std": 0.20645777881145477, "beta_std": 0.07423892617225647, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 3.121335612377152e-05, "evidence_weights": [-0.18386495113372803, -0.11600401252508163, -0.22659087181091309], "agreement_weight": 0.017803989350795746, "orthogonal_weight": 0.00011381748481653631}, {"head": 2, "memory_to_active": 0.938162624835968, "contribution_to_active": 0.25408557057380676, "mix_mean": 0.09321527928113937, "mix_low": 0.14242267608642578, "mix_high": 7.343292236328125e-05, "read_cosine": 0.14160612225532532, "beta_mean": 0.07080806791782379, "nonfinite": 0.0, "weighted_memory_to_active": 0.1838720589876175, "mix_std": 0.14214596152305603, "beta_std": 0.05705377459526062, "beta_low": 0.04827880859375, "beta_high": 0.0, "orthogonal_to_active": 0.04830223321914673, "evidence_weights": [-0.03342445194721222, -0.12190181761980057, 0.36665230989456177], "agreement_weight": 0.02757214568555355, "orthogonal_weight": -0.2672681510448456}, {"head": 3, "memory_to_active": 3.850902795791626, "contribution_to_active": 0.0883859321475029, "mix_mean": 0.03605511784553528, "mix_low": 0.31850624084472656, "mix_high": 0.0, "read_cosine": 0.08868296444416046, "beta_mean": 0.2367883026599884, "nonfinite": 0.0, "weighted_memory_to_active": 0.272292822599411, "mix_std": 0.0541447289288044, "beta_std": 0.2232433259487152, "beta_low": 0.03314208984375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.22093454003334045, "evidence_weights": [0.06556659936904907, 0.0014275053981691599, 0.6497682929039001], "agreement_weight": 0.06132197007536888, "orthogonal_weight": -0.9196731448173523}, {"head": 4, "memory_to_active": 1.7169976234436035, "contribution_to_active": 0.06472676247358322, "mix_mean": 0.015711169689893723, "mix_low": 0.6541786193847656, "mix_high": 0.0, "read_cosine": 0.2965911626815796, "beta_mean": 0.11472941190004349, "nonfinite": 0.0, "weighted_memory_to_active": 0.05798593536019325, "mix_std": 0.036384161561727524, "beta_std": 0.11659249663352966, "beta_low": 0.05401611328125, "beta_high": 0.0, "orthogonal_to_active": 0.005345572717487812, "evidence_weights": [0.1263090819120407, 0.04974443092942238, 0.15519651770591736], "agreement_weight": 0.052397046238183975, "orthogonal_weight": -0.11611507087945938}, {"head": 5, "memory_to_active": 0.4313144385814667, "contribution_to_active": 0.19711339473724365, "mix_mean": 0.15359503030776978, "mix_low": 0.08088207244873047, "mix_high": 1.9073486328125e-06, "read_cosine": 0.22799570858478546, "beta_mean": 0.004791456274688244, "nonfinite": 0.0, "weighted_memory_to_active": 0.10122102499008179, "mix_std": 0.17918822169303894, "beta_std": 0.010258986614644527, "beta_low": 0.8828125, "beta_high": 0.0, "orthogonal_to_active": 0.04664084315299988, "evidence_weights": [-0.046573229134082794, -0.09320539981126785, 0.2804069221019745], "agreement_weight": 0.07681020349264145, "orthogonal_weight": -0.575023353099823}]}, {"layer": 8, "memory_to_active": 0.530346155166626, "contribution_to_active": 0.5052635669708252, "mix_mean": 0.40810373425483704, "mix_low": 0.09553496539592743, "mix_high": 0.0007799466839060187, "read_cosine": 0.5004758834838867, "beta_mean": 0.03271095082163811, "nonfinite": 0.0, "weighted_memory_to_active": 0.09886515885591507, "mix_std": 0.22126340866088867, "beta_std": 0.03795233741402626, "beta_low": 0.5914815664291382, "beta_high": 0.0, "orthogonal_to_active": 0.05551605671644211, "output_influence": 0.4204614460468292, "residual_influence": 0.12841354310512543, "state_active": 0.053362008184194565, "state_memory": 0.010629535652697086, "state_peak": 32.61547088623047, "growth_active": 1.1211010217666626, "growth_memory": 1.3816592693328857, "heads": [{"head": 0, "memory_to_active": 0.0419868566095829, "contribution_to_active": 0.5323520302772522, "mix_mean": 0.5758864879608154, "mix_low": 0.00030994415283203125, "mix_high": 0.0006275177001953125, "read_cosine": 0.6144546270370483, "beta_mean": 0.002955070463940501, "nonfinite": 0.0, "weighted_memory_to_active": 0.027538100257515907, "mix_std": 0.242645263671875, "beta_std": 0.007489882875233889, "beta_low": 0.9293212890625, "beta_high": 0.0, "orthogonal_to_active": 0.016798552125692368, "evidence_weights": [-0.17414721846580505, -0.3864235281944275, 0.7148277759552002], "agreement_weight": 0.09117875248193741, "orthogonal_weight": -0.828682541847229}, {"head": 1, "memory_to_active": 0.19915637373924255, "contribution_to_active": 0.45466360449790955, "mix_mean": 0.4467298686504364, "mix_low": 0.0015363693237304688, "mix_high": 0.00025463104248046875, "read_cosine": 0.36784517765045166, "beta_mean": 0.005418027751147747, "nonfinite": 0.0, "weighted_memory_to_active": 0.09510014951229095, "mix_std": 0.25337302684783936, "beta_std": 0.014265358448028564, "beta_low": 0.87127685546875, "beta_high": 0.0, "orthogonal_to_active": 0.055639274418354034, "evidence_weights": [-0.1921878457069397, -0.2469133734703064, 0.6915170550346375], "agreement_weight": 0.08231915533542633, "orthogonal_weight": -0.6576395034790039}, {"head": 2, "memory_to_active": 0.03350585326552391, "contribution_to_active": 0.6286234855651855, "mix_mean": 0.6063435673713684, "mix_low": 0.0009212493896484375, "mix_high": 0.0028009414672851562, "read_cosine": 0.3754284381866455, "beta_mean": 0.00028730908525176346, "nonfinite": 0.0, "weighted_memory_to_active": 0.023111669346690178, "mix_std": 0.23618084192276, "beta_std": 0.003342562122270465, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.015768952667713165, "evidence_weights": [-0.21964634954929352, -0.33669546246528625, 0.8201801180839539], "agreement_weight": 0.1931905895471573, "orthogonal_weight": -0.7725198268890381}, {"head": 3, "memory_to_active": 0.1301143765449524, "contribution_to_active": 0.5860882997512817, "mix_mean": 0.5283244252204895, "mix_low": 0.001781463623046875, "mix_high": 0.0005626678466796875, "read_cosine": 0.7871986627578735, "beta_mean": 0.011598891578614712, "nonfinite": 0.0, "weighted_memory_to_active": 0.09152723848819733, "mix_std": 0.2550804316997528, "beta_std": 0.00954810157418251, "beta_low": 0.510498046875, "beta_high": 0.0, "orthogonal_to_active": 0.029033884406089783, "evidence_weights": [-0.17243939638137817, -0.2942248284816742, 0.6820880770683289], "agreement_weight": 0.13989564776420593, "orthogonal_weight": -0.5986025333404541}, {"head": 4, "memory_to_active": 2.5113956928253174, "contribution_to_active": 0.20655807852745056, "mix_mean": 0.025245729833841324, "mix_low": 0.43497753143310547, "mix_high": 0.0, "read_cosine": 0.32149362564086914, "beta_mean": 0.0984644964337349, "nonfinite": 0.0, "weighted_memory_to_active": 0.19537708163261414, "mix_std": 0.04210158437490463, "beta_std": 0.08859708160161972, "beta_low": 0.02001953125, "beta_high": 0.0, "orthogonal_to_active": 0.00894459243863821, "evidence_weights": [0.04731130599975586, -0.015785790979862213, -0.10430887341499329], "agreement_weight": 0.04406289756298065, "orthogonal_weight": 0.04805222153663635}, {"head": 5, "memory_to_active": 0.26591768860816956, "contribution_to_active": 0.6232956647872925, "mix_mean": 0.2660921812057495, "mix_low": 0.1336832046508789, "mix_high": 0.00043392181396484375, "read_cosine": 0.536434531211853, "beta_mean": 0.07754188776016235, "nonfinite": 0.0, "weighted_memory_to_active": 0.16053670644760132, "mix_std": 0.29819923639297485, "beta_std": 0.10447102785110474, "beta_low": 0.21826171875, "beta_high": 0.0, "orthogonal_to_active": 0.20691105723381042, "evidence_weights": [-0.06604813784360886, -0.2805372178554535, -0.39217013120651245], "agreement_weight": 0.01197828259319067, "orthogonal_weight": 1.5780888795852661}]}, {"layer": 9, "memory_to_active": 1.1069961786270142, "contribution_to_active": 0.6076856851577759, "mix_mean": 0.3039162755012512, "mix_low": 0.15183958411216736, "mix_high": 0.0025272369384765625, "read_cosine": 0.38077670335769653, "beta_mean": 0.07674191147089005, "nonfinite": 0.0, "weighted_memory_to_active": 0.3405112028121948, "mix_std": 0.20848751068115234, "beta_std": 0.04757904261350632, "beta_low": 0.500213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.039526138454675674, "output_influence": 0.48104745149612427, "residual_influence": 0.13088437914848328, "state_active": 0.040618591010570526, "state_memory": 0.02081158570945263, "state_peak": 40.571903228759766, "growth_active": 1.1795878410339355, "growth_memory": 1.3725123405456543, "heads": [{"head": 0, "memory_to_active": 0.0022200997918844223, "contribution_to_active": 0.5307374000549316, "mix_mean": 0.5927448272705078, "mix_low": 0.0019807815551757812, "mix_high": 0.0102996826171875, "read_cosine": 0.2736889719963074, "beta_mean": 6.581286470463965e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.001355345593765378, "mix_std": 0.29157960414886475, "beta_std": 2.6113397325389087e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011756190797314048, "evidence_weights": [-0.29116401076316833, -0.29902514815330505, 0.9342694282531738], "agreement_weight": 0.10124687850475311, "orthogonal_weight": -0.9164292216300964}, {"head": 1, "memory_to_active": 0.028818227350711823, "contribution_to_active": 0.6063700318336487, "mix_mean": 0.46295830607414246, "mix_low": 0.009866714477539062, "mix_high": 0.0010662078857421875, "read_cosine": 0.3981214761734009, "beta_mean": 8.193730900529772e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.017581641674041748, "mix_std": 0.290041983127594, "beta_std": 0.00019023465574719012, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012268051505088806, "evidence_weights": [-0.21671028435230255, -0.262185662984848, 1.1271618604660034], "agreement_weight": 0.32881325483322144, "orthogonal_weight": -0.8247350454330444}, {"head": 2, "memory_to_active": 0.1976669430732727, "contribution_to_active": 0.6212977170944214, "mix_mean": 0.5425752401351929, "mix_low": 0.0019550323486328125, "mix_high": 0.0035581588745117188, "read_cosine": 0.38316914439201355, "beta_mean": 0.0007946972618810833, "nonfinite": 0.0, "weighted_memory_to_active": 0.12869758903980255, "mix_std": 0.2801186144351959, "beta_std": 0.00479780463501811, "beta_low": 0.99578857421875, "beta_high": 0.0, "orthogonal_to_active": 0.06148814409971237, "evidence_weights": [-0.1870882213115692, -0.24677884578704834, 0.6204506158828735], "agreement_weight": 0.24887579679489136, "orthogonal_weight": -0.5717341303825378}, {"head": 3, "memory_to_active": 1.695326328277588, "contribution_to_active": 0.17042718827724457, "mix_mean": 0.0383235402405262, "mix_low": 0.38933277130126953, "mix_high": 2.86102294921875e-06, "read_cosine": 0.41998374462127686, "beta_mean": 0.09690146148204803, "nonfinite": 0.0, "weighted_memory_to_active": 0.14073066413402557, "mix_std": 0.08572689443826675, "beta_std": 0.06721602380275726, "beta_low": 0.0052490234375, "beta_high": 0.0, "orthogonal_to_active": 0.025394361466169357, "evidence_weights": [0.07233183830976486, 0.03451918438076973, -0.0770396813750267], "agreement_weight": 0.07391087710857391, "orthogonal_weight": 0.1932474821805954}, {"head": 4, "memory_to_active": 3.030560255050659, "contribution_to_active": 1.1272832155227661, "mix_mean": 0.11791997402906418, "mix_low": 0.16977310180664062, "mix_high": 0.00023651123046875, "read_cosine": 0.3217715620994568, "beta_mean": 0.2041740119457245, "nonfinite": 0.0, "weighted_memory_to_active": 1.082958698272705, "mix_std": 0.17747282981872559, "beta_std": 0.10801777243614197, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.024034744128584862, "evidence_weights": [-0.0573144368827343, -0.1478077620267868, 0.009860719554126263], "agreement_weight": -0.054123181849718094, "orthogonal_weight": -0.02226399816572666}, {"head": 5, "memory_to_active": 1.687384843826294, "contribution_to_active": 0.5899984240531921, "mix_mean": 0.06897581368684769, "mix_low": 0.33812904357910156, "mix_high": 0.0, "read_cosine": 0.48792529106140137, "beta_mean": 0.15849275887012482, "nonfinite": 0.0, "weighted_memory_to_active": 0.6717430353164673, "mix_std": 0.12598516047000885, "beta_std": 0.10522628575563431, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.11279590427875519, "evidence_weights": [0.09064451605081558, -0.06393488496541977, 0.15087948739528656], "agreement_weight": 0.020123392343521118, "orthogonal_weight": 0.2463049590587616}]}, {"layer": 10, "memory_to_active": 0.24115753173828125, "contribution_to_active": 0.5973882675170898, "mix_mean": 0.4862558841705322, "mix_low": 0.012921015731990337, "mix_high": 0.007434686180204153, "read_cosine": 0.3672720491886139, "beta_mean": 0.045727670192718506, "nonfinite": 0.0, "weighted_memory_to_active": 0.11721113324165344, "mix_std": 0.26881688833236694, "beta_std": 0.04771896079182625, "beta_low": 0.6621297597885132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.12553220987319946, "output_influence": 0.41098836064338684, "residual_influence": 0.13907374441623688, "state_active": 0.0535658523440361, "state_memory": 0.01384066790342331, "state_peak": 110.01144409179688, "growth_active": 1.0611506700515747, "growth_memory": 0.9665595889091492, "heads": [{"head": 0, "memory_to_active": 0.9033665060997009, "contribution_to_active": 1.4116051197052002, "mix_mean": 0.6130600571632385, "mix_low": 0.03540611267089844, "mix_high": 0.03239727020263672, "read_cosine": 0.07909881323575974, "beta_mean": 0.20619043707847595, "nonfinite": 0.0, "weighted_memory_to_active": 0.5666711926460266, "mix_std": 0.38056322932243347, "beta_std": 0.22009076178073883, "beta_low": 0.11822509765625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.6684585809707642, "evidence_weights": [-0.13935962319374084, -0.3792993128299713, -0.7708268165588379], "agreement_weight": 0.019725263118743896, "orthogonal_weight": 1.1957967281341553}, {"head": 1, "memory_to_active": 0.01773444376885891, "contribution_to_active": 0.5922830104827881, "mix_mean": 0.5927972793579102, "mix_low": 0.0006380081176757812, "mix_high": 0.009325981140136719, "read_cosine": 0.2680590748786926, "beta_mean": 0.00010434448631713167, "nonfinite": 0.0, "weighted_memory_to_active": 0.010046450421214104, "mix_std": 0.2677479386329651, "beta_std": 0.00013611135364044458, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013349449262022972, "evidence_weights": [-0.2729063928127289, -0.20278967916965485, 0.59608393907547], "agreement_weight": 0.4230203926563263, "orthogonal_weight": -1.4411336183547974}, {"head": 2, "memory_to_active": 0.045679498463869095, "contribution_to_active": 0.5466867685317993, "mix_mean": 0.5586896538734436, "mix_low": 0.00043201446533203125, "mix_high": 0.0025997161865234375, "read_cosine": 0.007151065394282341, "beta_mean": 0.00010863939678529277, "nonfinite": 0.0, "weighted_memory_to_active": 0.01691002957522869, "mix_std": 0.2605559825897217, "beta_std": 0.000491033133585006, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.006991127505898476, "evidence_weights": [-0.21579332649707794, -0.258135050535202, 0.5350120067596436], "agreement_weight": -0.09659924358129501, "orthogonal_weight": -0.39297133684158325}, {"head": 3, "memory_to_active": 0.3390764594078064, "contribution_to_active": 0.10560230165719986, "mix_mean": 0.26579156517982483, "mix_low": 0.03804588317871094, "mix_high": 8.106231689453125e-05, "read_cosine": 0.9097403287887573, "beta_mean": 0.061904177069664, "nonfinite": 0.0, "weighted_memory_to_active": 0.04134559631347656, "mix_std": 0.2258811891078949, "beta_std": 0.057294704020023346, "beta_low": 0.00360107421875, "beta_high": 0.0, "orthogonal_to_active": 0.04470135644078255, "evidence_weights": [-0.12056834995746613, -0.1507796198129654, 0.137477844953537], "agreement_weight": 0.0034472118131816387, "orthogonal_weight": 1.4982162714004517}, {"head": 4, "memory_to_active": 0.00508077209815383, "contribution_to_active": 0.5129566192626953, "mix_mean": 0.511597752571106, "mix_low": 0.00093841552734375, "mix_high": 0.0002002716064453125, "read_cosine": 0.07403147220611572, "beta_mean": 3.082168404944241e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022434673737734556, "mix_std": 0.25363969802856445, "beta_std": 0.0004211494233459234, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0011138856643810868, "evidence_weights": [-0.18051956593990326, -0.24990884959697723, 0.7282477617263794], "agreement_weight": 0.0852029025554657, "orthogonal_weight": -0.5091050267219543}, {"head": 5, "memory_to_active": 0.13600754737854004, "contribution_to_active": 0.4151957035064697, "mix_mean": 0.37559860944747925, "mix_low": 0.0020656585693359375, "mix_high": 3.814697265625e-06, "read_cosine": 0.8655514717102051, "beta_mean": 0.006027596071362495, "nonfinite": 0.0, "weighted_memory_to_active": 0.06605005264282227, "mix_std": 0.22451311349868774, "beta_std": 0.007879987359046936, "beta_low": 0.8514404296875, "beta_high": 0.0, "orthogonal_to_active": 0.018578827381134033, "evidence_weights": [-0.14698190987110138, -0.159512460231781, 0.2942816913127899], "agreement_weight": 0.09040582925081253, "orthogonal_weight": -0.6183090806007385}]}, {"layer": 11, "memory_to_active": 0.14287227392196655, "contribution_to_active": 0.5592124462127686, "mix_mean": 0.4559991955757141, "mix_low": 0.07970619201660156, "mix_high": 0.042471252381801605, "read_cosine": 0.4289473295211792, "beta_mean": 0.0338304340839386, "nonfinite": 0.0, "weighted_memory_to_active": 0.06029372662305832, "mix_std": 0.3195604681968689, "beta_std": 0.047937363386154175, "beta_low": 0.557403564453125, "beta_high": 0.0, "orthogonal_to_active": 0.049880266189575195, "output_influence": 0.5007073283195496, "residual_influence": 0.12509356439113617, "state_active": 0.1601051241159439, "state_memory": 0.007187024224549532, "state_peak": 24.61539649963379, "growth_active": 1.1047154664993286, "growth_memory": 1.6036301851272583, "heads": [{"head": 0, "memory_to_active": 0.0065798088908195496, "contribution_to_active": 0.5775311589241028, "mix_mean": 0.608447790145874, "mix_low": 0.001163482666015625, "mix_high": 0.008745193481445312, "read_cosine": 0.3725672960281372, "beta_mean": 8.312441786983982e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004083532374352217, "mix_std": 0.2651081681251526, "beta_std": 0.003620568662881851, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.002802722854539752, "evidence_weights": [-0.16704754531383514, -0.3594832122325897, 0.6766352653503418], "agreement_weight": 0.23076434433460236, "orthogonal_weight": -0.7593005299568176}, {"head": 1, "memory_to_active": 0.35044988989830017, "contribution_to_active": 0.28148362040519714, "mix_mean": 0.32494083046913147, "mix_low": 0.012628555297851562, "mix_high": 8.487701416015625e-05, "read_cosine": 0.6199721693992615, "beta_mean": 0.030041078105568886, "nonfinite": 0.0, "weighted_memory_to_active": 0.13506276905536652, "mix_std": 0.2502361536026001, "beta_std": 0.05553874000906944, "beta_low": 0.42340087890625, "beta_high": 0.0, "orthogonal_to_active": 0.034381184726953506, "evidence_weights": [-0.15814492106437683, -0.15261191129684448, 0.3506188988685608], "agreement_weight": 0.07725892215967178, "orthogonal_weight": -0.3450252413749695}, {"head": 2, "memory_to_active": 0.016883056610822678, "contribution_to_active": 0.9617006778717041, "mix_mean": 0.4524557590484619, "mix_low": 0.1815013885498047, "mix_high": 0.15069961547851562, "read_cosine": 0.21975892782211304, "beta_mean": 0.07848641276359558, "nonfinite": 0.0, "weighted_memory_to_active": 0.012884557247161865, "mix_std": 0.43581998348236084, "beta_std": 0.10430549085140228, "beta_low": 0.43939208984375, "beta_high": 0.0, "orthogonal_to_active": 0.06356380879878998, "evidence_weights": [0.010350297205150127, -0.45306381583213806, -1.2397809028625488], "agreement_weight": -0.0017742590280249715, "orthogonal_weight": 5.136070728302002}, {"head": 3, "memory_to_active": 0.014171043410897255, "contribution_to_active": 0.783562183380127, "mix_mean": 0.41409730911254883, "mix_low": 0.2704782485961914, "mix_high": 0.09360027313232422, "read_cosine": 0.5557708740234375, "beta_mean": 0.062475498765707016, "nonfinite": 0.0, "weighted_memory_to_active": 0.008171238005161285, "mix_std": 0.43778249621391296, "beta_std": 0.09160183370113373, "beta_low": 0.47216796875, "beta_high": 0.0, "orthogonal_to_active": 0.03778760880231857, "evidence_weights": [0.08134312927722931, -0.34531280398368835, -0.40340501070022583], "agreement_weight": 0.044410958886146545, "orthogonal_weight": 5.462330341339111}, {"head": 4, "memory_to_active": 0.3837333917617798, "contribution_to_active": 0.38682106137275696, "mix_mean": 0.4401063621044159, "mix_low": 0.011084556579589844, "mix_high": 0.0004329681396484375, "read_cosine": 0.08422838151454926, "beta_mean": 0.02420869469642639, "nonfinite": 0.0, "weighted_memory_to_active": 0.16238896548748016, "mix_std": 0.26847344636917114, "beta_std": 0.02210099622607231, "beta_low": 0.2567138671875, "beta_high": 0.0, "orthogonal_to_active": 0.1410677433013916, "evidence_weights": [-0.21017393469810486, -0.1824633628129959, 1.721645474433899], "agreement_weight": -0.17518304288387299, "orthogonal_weight": -0.8848163485527039}, {"head": 5, "memory_to_active": 0.08541649580001831, "contribution_to_active": 0.36417561769485474, "mix_mean": 0.49594712257385254, "mix_low": 0.00138092041015625, "mix_high": 0.0012645721435546875, "read_cosine": 0.7213863134384155, "beta_mean": 0.007687785197049379, "nonfinite": 0.0, "weighted_memory_to_active": 0.03917130082845688, "mix_std": 0.25994256138801575, "beta_std": 0.010456522926688194, "beta_low": 0.75286865234375, "beta_high": 0.0, "orthogonal_to_active": 0.01967853307723999, "evidence_weights": [-0.21514874696731567, -0.2791703939437866, 0.5459338426589966], "agreement_weight": 0.06985287368297577, "orthogonal_weight": -0.6909178495407104}]}, {"layer": 12, "memory_to_active": 0.314054012298584, "contribution_to_active": 0.5561261177062988, "mix_mean": 0.48426419496536255, "mix_low": 0.011150837875902653, "mix_high": 0.003483772510662675, "read_cosine": 0.2593567967414856, "beta_mean": 0.014870719984173775, "nonfinite": 0.0, "weighted_memory_to_active": 0.17116796970367432, "mix_std": 0.25882118940353394, "beta_std": 0.01584821566939354, "beta_low": 0.8209737539291382, "beta_high": 0.0, "orthogonal_to_active": 0.06721578538417816, "output_influence": 0.4354618787765503, "residual_influence": 0.12021949887275696, "state_active": 0.03889436647295952, "state_memory": 0.009575475938618183, "state_peak": 35.245574951171875, "growth_active": 1.1085745096206665, "growth_memory": 1.2563170194625854, "heads": [{"head": 0, "memory_to_active": 0.06655988097190857, "contribution_to_active": 0.4812674820423126, "mix_mean": 0.4672766327857971, "mix_low": 0.0018720626831054688, "mix_high": 0.0008335113525390625, "read_cosine": 0.3230479657649994, "beta_mean": 0.001158040715381503, "nonfinite": 0.0, "weighted_memory_to_active": 0.030030375346541405, "mix_std": 0.26857495307922363, "beta_std": 0.004060101695358753, "beta_low": 0.98968505859375, "beta_high": 0.0, "orthogonal_to_active": 0.019348181784152985, "evidence_weights": [-0.18049654364585876, -0.2539765536785126, 0.31608396768569946], "agreement_weight": 0.11073308438062668, "orthogonal_weight": -0.6948671936988831}, {"head": 1, "memory_to_active": 0.09109754115343094, "contribution_to_active": 0.5028842091560364, "mix_mean": 0.5181291699409485, "mix_low": 0.00067901611328125, "mix_high": 0.0004482269287109375, "read_cosine": 0.5586552619934082, "beta_mean": 0.0009689065627753735, "nonfinite": 0.0, "weighted_memory_to_active": 0.047632090747356415, "mix_std": 0.26089176535606384, "beta_std": 0.005050991661846638, "beta_low": 0.9888916015625, "beta_high": 0.0, "orthogonal_to_active": 0.036231525242328644, "evidence_weights": [-0.2053491473197937, -0.27201011776924133, 0.9063699841499329], "agreement_weight": 0.12171444296836853, "orthogonal_weight": -0.9242624044418335}, {"head": 2, "memory_to_active": 0.604150652885437, "contribution_to_active": 0.443729043006897, "mix_mean": 0.313996285200119, "mix_low": 0.026762962341308594, "mix_high": 0.00025081634521484375, "read_cosine": 0.449520081281662, "beta_mean": 0.08384361118078232, "nonfinite": 0.0, "weighted_memory_to_active": 0.2361832559108734, "mix_std": 0.24184297025203705, "beta_std": 0.06147996336221695, "beta_low": 0.0025634765625, "beta_high": 0.0, "orthogonal_to_active": 0.21136359870433807, "evidence_weights": [-0.2439054697751999, -0.04578523710370064, -0.10049784183502197], "agreement_weight": 0.02785947546362877, "orthogonal_weight": 1.0997185707092285}, {"head": 3, "memory_to_active": 0.022243406623601913, "contribution_to_active": 0.7072782516479492, "mix_mean": 0.6831633448600769, "mix_low": 0.00026798248291015625, "mix_high": 0.013796806335449219, "read_cosine": 0.05119866132736206, "beta_mean": 0.0001955399347934872, "nonfinite": 0.0, "weighted_memory_to_active": 0.016410555690526962, "mix_std": 0.24082091450691223, "beta_std": 0.0033016661182045937, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.015160986222326756, "evidence_weights": [-0.2959679067134857, -0.33582592010498047, 1.2199000120162964], "agreement_weight": 0.3228197991847992, "orthogonal_weight": -0.944534182548523}, {"head": 4, "memory_to_active": 1.0118144750595093, "contribution_to_active": 0.7318764328956604, "mix_mean": 0.3170032203197479, "mix_low": 0.030750274658203125, "mix_high": 0.00194549560546875, "read_cosine": -0.09639037400484085, "beta_mean": 0.0023005856201052666, "nonfinite": 0.0, "weighted_memory_to_active": 0.6431136727333069, "mix_std": 0.2663138210773468, "beta_std": 0.016965322196483612, "beta_low": 0.9617919921875, "beta_high": 0.0, "orthogonal_to_active": 0.07985512912273407, "evidence_weights": [-0.09122743457555771, -0.2113444060087204, 0.05621025711297989], "agreement_weight": -0.003946767654269934, "orthogonal_weight": -0.1327294111251831}, {"head": 5, "memory_to_active": 0.08845814317464828, "contribution_to_active": 0.46972155570983887, "mix_mean": 0.6060164570808411, "mix_low": 0.006572723388671875, "mix_high": 0.003627777099609375, "read_cosine": 0.2701093256473541, "beta_mean": 0.0007576369098387659, "nonfinite": 0.0, "weighted_memory_to_active": 0.05363791435956955, "mix_std": 0.2744826674461365, "beta_std": 0.0042312489822506905, "beta_low": 0.9840087890625, "beta_high": 0.0, "orthogonal_to_active": 0.041335273534059525, "evidence_weights": [-0.25621119141578674, -0.3053736388683319, 1.1304301023483276], "agreement_weight": 0.2540138065814972, "orthogonal_weight": -0.836145281791687}]}, {"layer": 13, "memory_to_active": 0.13432985544204712, "contribution_to_active": 0.5015657544136047, "mix_mean": 0.49998366832733154, "mix_low": 0.005576610565185547, "mix_high": 0.004608790390193462, "read_cosine": 0.4081547260284424, "beta_mean": 0.01603115350008011, "nonfinite": 0.0, "weighted_memory_to_active": 0.05755440890789032, "mix_std": 0.25873899459838867, "beta_std": 0.017110444605350494, "beta_low": 0.7929891347885132, "beta_high": 0.0, "orthogonal_to_active": 0.05763130262494087, "output_influence": 0.4628734886646271, "residual_influence": 0.1534431129693985, "state_active": 0.10053393989801407, "state_memory": 0.03770104795694351, "state_peak": 95.9127197265625, "growth_active": 0.9852697849273682, "growth_memory": 1.0547330379486084, "heads": [{"head": 0, "memory_to_active": 0.037443872541189194, "contribution_to_active": 0.5427379608154297, "mix_mean": 0.5529245138168335, "mix_low": 0.00037670135498046875, "mix_high": 0.001506805419921875, "read_cosine": 0.4038233458995819, "beta_mean": 0.0001305344485444948, "nonfinite": 0.0, "weighted_memory_to_active": 0.02147604152560234, "mix_std": 0.2662104368209839, "beta_std": 0.0003227797569707036, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.020942747592926025, "evidence_weights": [-0.20196528732776642, -0.2613741159439087, 0.7321755290031433], "agreement_weight": 0.08700704574584961, "orthogonal_weight": -1.0862257480621338}, {"head": 1, "memory_to_active": 0.016704924404621124, "contribution_to_active": 0.5687857270240784, "mix_mean": 0.5179725885391235, "mix_low": 0.0021142959594726562, "mix_high": 0.0014629364013671875, "read_cosine": 0.22325065732002258, "beta_mean": 5.0999333325307816e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00933929719030857, "mix_std": 0.2697182893753052, "beta_std": 9.323512495029718e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010388289578258991, "evidence_weights": [-0.18530017137527466, -0.26737475395202637, 0.7118738889694214], "agreement_weight": 0.28588709235191345, "orthogonal_weight": -1.162263035774231}, {"head": 2, "memory_to_active": 0.12091541290283203, "contribution_to_active": 0.41649091243743896, "mix_mean": 0.3493254780769348, "mix_low": 0.002635955810546875, "mix_high": 7.2479248046875e-05, "read_cosine": 0.674572229385376, "beta_mean": 0.013072188012301922, "nonfinite": 0.0, "weighted_memory_to_active": 0.048642728477716446, "mix_std": 0.21872270107269287, "beta_std": 0.03613435477018356, "beta_low": 0.77294921875, "beta_high": 0.0, "orthogonal_to_active": 0.02217591367661953, "evidence_weights": [-0.20524179935455322, -0.1796017587184906, 0.4311436414718628], "agreement_weight": 0.014967470429837704, "orthogonal_weight": -0.7939084768295288}, {"head": 3, "memory_to_active": 0.14143967628479004, "contribution_to_active": 0.5518333911895752, "mix_mean": 0.6061772704124451, "mix_low": 0.01856231689453125, "mix_high": 0.01851367950439453, "read_cosine": -0.02201666310429573, "beta_mean": 0.0010804275516420603, "nonfinite": 0.0, "weighted_memory_to_active": 0.09895198792219162, "mix_std": 0.2940136790275574, "beta_std": 0.006656423676759005, "beta_low": 0.99200439453125, "beta_high": 0.0, "orthogonal_to_active": 0.09914425015449524, "evidence_weights": [-0.33643725514411926, -0.2408606857061386, 2.0013160705566406], "agreement_weight": 3.381894111633301, "orthogonal_weight": -1.070736289024353}, {"head": 4, "memory_to_active": 0.4063675105571747, "contribution_to_active": 0.26583579182624817, "mix_mean": 0.3668113052845001, "mix_low": 0.0077667236328125, "mix_high": 1.621246337890625e-05, "read_cosine": 0.8742884397506714, "beta_mean": 0.08121830224990845, "nonfinite": 0.0, "weighted_memory_to_active": 0.11186692118644714, "mix_std": 0.22324372828006744, "beta_std": 0.05605607479810715, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.14833831787109375, "evidence_weights": [-0.14933136105537415, -0.23346781730651855, 0.49895748496055603], "agreement_weight": -0.008685867302119732, "orthogonal_weight": 2.0251002311706543}, {"head": 5, "memory_to_active": 0.08310773968696594, "contribution_to_active": 0.6637108325958252, "mix_mean": 0.6066908836364746, "mix_low": 0.0020036697387695312, "mix_high": 0.00608062744140625, "read_cosine": 0.29501014947891235, "beta_mean": 0.0006344654830172658, "nonfinite": 0.0, "weighted_memory_to_active": 0.05504946783185005, "mix_std": 0.2805250287055969, "beta_std": 0.003399801906198263, "beta_low": 0.992431640625, "beta_high": 0.0, "orthogonal_to_active": 0.04479827359318733, "evidence_weights": [-0.21817170083522797, -0.29308459162712097, 0.7239382266998291], "agreement_weight": 0.10263504832983017, "orthogonal_weight": -0.9000347852706909}]}, {"layer": 14, "memory_to_active": 0.2570841908454895, "contribution_to_active": 0.4286123812198639, "mix_mean": 0.4149835407733917, "mix_low": 0.02425384521484375, "mix_high": 0.004660129547119141, "read_cosine": 0.6344622373580933, "beta_mean": 0.07168403267860413, "nonfinite": 0.0, "weighted_memory_to_active": 0.11579223722219467, "mix_std": 0.24734759330749512, "beta_std": 0.06562992930412292, "beta_low": 0.4447733759880066, "beta_high": 0.0, "orthogonal_to_active": 0.042504552751779556, "output_influence": 0.3110363781452179, "residual_influence": 0.12018472701311111, "state_active": 0.06650350242853165, "state_memory": 0.015981484204530716, "state_peak": 34.57275390625, "growth_active": 1.139020562171936, "growth_memory": 1.291943907737732, "heads": [{"head": 0, "memory_to_active": 0.34465059638023376, "contribution_to_active": 0.21843065321445465, "mix_mean": 0.21949970722198486, "mix_low": 0.013213157653808594, "mix_high": 2.86102294921875e-06, "read_cosine": 0.7810209393501282, "beta_mean": 0.0688706561923027, "nonfinite": 0.0, "weighted_memory_to_active": 0.09403486549854279, "mix_std": 0.19305390119552612, "beta_std": 0.06525208055973053, "beta_low": 0.01873779296875, "beta_high": 0.0, "orthogonal_to_active": 0.06077789515256882, "evidence_weights": [-0.13386951386928558, -0.04690338298678398, 0.5913822054862976], "agreement_weight": 0.02478809468448162, "orthogonal_weight": 1.0585376024246216}, {"head": 1, "memory_to_active": 0.49460017681121826, "contribution_to_active": 0.3448418974876404, "mix_mean": 0.24885353446006775, "mix_low": 0.11423969268798828, "mix_high": 6.29425048828125e-05, "read_cosine": 0.7926924228668213, "beta_mean": 0.29907500743865967, "nonfinite": 0.0, "weighted_memory_to_active": 0.23069384694099426, "mix_std": 0.2688824534416199, "beta_std": 0.2680751085281372, "beta_low": 0.08642578125, "beta_high": 0.0, "orthogonal_to_active": 0.01590927131474018, "evidence_weights": [-0.005449690390378237, -0.2501269280910492, -0.17911899089813232], "agreement_weight": 0.04897068813443184, "orthogonal_weight": 0.15244828164577484}, {"head": 2, "memory_to_active": 0.1591082513332367, "contribution_to_active": 0.5434768795967102, "mix_mean": 0.5506945252418518, "mix_low": 0.003631591796875, "mix_high": 0.0022678375244140625, "read_cosine": 0.3771347403526306, "beta_mean": 0.004025583621114492, "nonfinite": 0.0, "weighted_memory_to_active": 0.09627240151166916, "mix_std": 0.2687988877296448, "beta_std": 0.005318359471857548, "beta_low": 0.9124755859375, "beta_high": 0.0, "orthogonal_to_active": 0.07461878657341003, "evidence_weights": [-0.16175059974193573, -0.301179438829422, 0.8846690058708191], "agreement_weight": 0.06906892359256744, "orthogonal_weight": -0.8691436648368835}, {"head": 3, "memory_to_active": 0.06994666159152985, "contribution_to_active": 0.6735856533050537, "mix_mean": 0.6748347282409668, "mix_low": 0.0041942596435546875, "mix_high": 0.017600059509277344, "read_cosine": 0.27333536744117737, "beta_mean": 0.009652657434344292, "nonfinite": 0.0, "weighted_memory_to_active": 0.047268062829971313, "mix_std": 0.24681758880615234, "beta_std": 0.009613342583179474, "beta_low": 0.66485595703125, "beta_high": 0.0, "orthogonal_to_active": 0.028663253411650658, "evidence_weights": [-0.3761916756629944, -0.3253304958343506, 0.4236466586589813], "agreement_weight": 0.08092888444662094, "orthogonal_weight": -0.7132388949394226}, {"head": 4, "memory_to_active": 0.17521819472312927, "contribution_to_active": 0.4794090986251831, "mix_mean": 0.4773818254470825, "mix_low": 0.004634857177734375, "mix_high": 0.007210731506347656, "read_cosine": 0.8438056707382202, "beta_mean": 0.04569436237215996, "nonfinite": 0.0, "weighted_memory_to_active": 0.09963306039571762, "mix_std": 0.26352083683013916, "beta_std": 0.033988162875175476, "beta_low": 0.0460205078125, "beta_high": 0.0, "orthogonal_to_active": 0.027756264433264732, "evidence_weights": [-0.2189735621213913, -0.2596634030342102, 0.4233371317386627], "agreement_weight": 0.12202108651399612, "orthogonal_weight": -0.5649445652961731}, {"head": 5, "memory_to_active": 0.2989811599254608, "contribution_to_active": 0.31193000078201294, "mix_mean": 0.318636953830719, "mix_low": 0.0056095123291015625, "mix_high": 0.00081634521484375, "read_cosine": 0.7387837171554565, "beta_mean": 0.0027859609108418226, "nonfinite": 0.0, "weighted_memory_to_active": 0.1268511414527893, "mix_std": 0.24301189184188843, "beta_std": 0.011532464995980263, "beta_low": 0.94012451171875, "beta_high": 0.0, "orthogonal_to_active": 0.04730183258652687, "evidence_weights": [-0.17874442040920258, -0.1541557013988495, 0.21741893887519836], "agreement_weight": 0.08488062024116516, "orthogonal_weight": -0.6314650774002075}]}, {"layer": 15, "memory_to_active": 0.0632612332701683, "contribution_to_active": 0.47712209820747375, "mix_mean": 0.39416319131851196, "mix_low": 0.008634249679744244, "mix_high": 0.0023325285874307156, "read_cosine": 0.6611266136169434, "beta_mean": 0.0361383780837059, "nonfinite": 0.0, "weighted_memory_to_active": 0.028528181836009026, "mix_std": 0.2788839340209961, "beta_std": 0.04195914790034294, "beta_low": 0.710968017578125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.006860567722469568, "output_influence": 0.2951175570487976, "residual_influence": 0.16058188676834106, "state_active": 0.12366615980863571, "state_memory": 0.009201180189847946, "state_peak": 37.603939056396484, "growth_active": 1.2474849224090576, "growth_memory": 1.7808024883270264, "heads": [{"head": 0, "memory_to_active": 0.1131601333618164, "contribution_to_active": 0.46263861656188965, "mix_mean": 0.35971295833587646, "mix_low": 0.005405426025390625, "mix_high": 0.004021644592285156, "read_cosine": 0.7128466367721558, "beta_mean": 0.03797497972846031, "nonfinite": 0.0, "weighted_memory_to_active": 0.05237855389714241, "mix_std": 0.2692306637763977, "beta_std": 0.028149185702204704, "beta_low": 0.1204833984375, "beta_high": 0.0, "orthogonal_to_active": 0.020848438143730164, "evidence_weights": [-0.16422803699970245, -0.17653511464595795, 0.346666544675827], "agreement_weight": 0.08846060186624527, "orthogonal_weight": -0.6688776612281799}, {"head": 1, "memory_to_active": 0.13544830679893494, "contribution_to_active": 0.40315043926239014, "mix_mean": 0.4024279713630676, "mix_low": 0.0030517578125, "mix_high": 0.0031023025512695312, "read_cosine": 0.8930714130401611, "beta_mean": 0.0012079806765541434, "nonfinite": 0.0, "weighted_memory_to_active": 0.0514637753367424, "mix_std": 0.2779362201690674, "beta_std": 0.0034669465385377407, "beta_low": 0.97808837890625, "beta_high": 0.0, "orthogonal_to_active": 0.0010093053570017219, "evidence_weights": [-0.1704915165901184, -0.17911316454410553, 0.2514267861843109], "agreement_weight": 0.10109205543994904, "orthogonal_weight": -0.0312297735363245}, {"head": 2, "memory_to_active": 0.0071370783261954784, "contribution_to_active": 0.45477452874183655, "mix_mean": 0.3317670524120331, "mix_low": 0.017574310302734375, "mix_high": 3.147125244140625e-05, "read_cosine": 0.7609977722167969, "beta_mean": 0.00011721230112016201, "nonfinite": 0.0, "weighted_memory_to_active": 0.003248553490266204, "mix_std": 0.2685564458370209, "beta_std": 0.00027233196306042373, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0003579858457669616, "evidence_weights": [-0.20236429572105408, -0.27733486890792847, 0.253221333026886], "agreement_weight": 0.012657316401600838, "orthogonal_weight": 0.23158110678195953}, {"head": 3, "memory_to_active": 0.02184370346367359, "contribution_to_active": 0.41704732179641724, "mix_mean": 0.3487884998321533, "mix_low": 0.0054931640625, "mix_high": 4.291534423828125e-05, "read_cosine": 0.6160268783569336, "beta_mean": 0.0002056850353255868, "nonfinite": 0.0, "weighted_memory_to_active": 0.00878638494759798, "mix_std": 0.2724080979824066, "beta_std": 0.0006051426753401756, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.006862475536763668, "evidence_weights": [-0.21041104197502136, -0.2519388198852539, 0.35538002848625183], "agreement_weight": 0.022479303181171417, "orthogonal_weight": -0.9356034994125366}, {"head": 4, "memory_to_active": 0.08508143573999405, "contribution_to_active": 0.6252495646476746, "mix_mean": 0.38401442766189575, "mix_low": 0.01056671142578125, "mix_high": 0.0005550384521484375, "read_cosine": 0.6334225535392761, "beta_mean": 0.17614489793777466, "nonfinite": 0.0, "weighted_memory_to_active": 0.04438610374927521, "mix_std": 0.3060915470123291, "beta_std": 0.2111339271068573, "beta_low": 0.188232421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.005464055575430393, "evidence_weights": [-0.2195158451795578, -0.25804367661476135, 0.0267033651471138], "agreement_weight": 0.01277245208621025, "orthogonal_weight": 0.20873358845710754}, {"head": 5, "memory_to_active": 0.01689671166241169, "contribution_to_active": 0.49987202882766724, "mix_mean": 0.5382682085037231, "mix_low": 0.009714126586914062, "mix_high": 0.006241798400878906, "read_cosine": 0.3503945469856262, "beta_mean": 0.0011795097962021828, "nonfinite": 0.0, "weighted_memory_to_active": 0.010905720293521881, "mix_std": 0.27908048033714294, "beta_std": 0.008127347566187382, "beta_low": 0.9796142578125, "beta_high": 0.0, "orthogonal_to_active": 0.006621146108955145, "evidence_weights": [-0.3334868848323822, -0.30952534079551697, 0.7037798762321472], "agreement_weight": 0.249079167842865, "orthogonal_weight": 0.7514840960502625}]}]}, {"step": 31000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 36.7408137350003, "memory_to_active": 0.8855177760124207, "contribution_to_active": 0.5181053280830383, "mix_mean": 0.36612072587013245, "mix_low": 0.15262511372566223, "mix_high": 0.008487194776535034, "read_cosine": 0.3700747489929199, "beta_mean": 0.08151818811893463, "nonfinite": 0.0, "weighted_memory_to_active": 0.21626371145248413, "mix_std": 0.21683382987976074, "beta_std": 0.06879843026399612, "beta_low": 0.502951979637146, "beta_high": 1.4623006791225635e-05, "orthogonal_to_active": 0.053597353398799896, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1658120155334473, "normal_token_accuracy": 0.42380422353744507, "state_active": 0.06020856648683548, "state_memory": 0.019301120191812515, "state_peak": 106.89354705810547, "growth_active": 1.1383390426635742, "growth_memory": 1.3789260387420654, "output_influence": 0.3702850043773651, "residual_influence": 0.12013500928878784, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2657387256622314, "no_memory_loss_delta": 0.4524537920951843, "scrambled_loss_delta": 0.16419708728790283, "memory_only_loss_delta": 2.6092076301574707, "no_evidence_loss_delta": 0.2681024372577667, "no_agreement_loss_delta": 0.017378658056259155, "no_orthogonal_loss_delta": 0.004633218515664339}, {"index": 94, "tokens": 256, "normal_loss": 3.196746349334717, "no_memory_loss_delta": 0.5077837109565735, "scrambled_loss_delta": 0.34512773156166077, "memory_only_loss_delta": 2.904726982116699, "no_evidence_loss_delta": 0.23854397237300873, "no_agreement_loss_delta": 0.02254028059542179, "no_orthogonal_loss_delta": 0.032248008996248245}, {"index": 188, "tokens": 256, "normal_loss": 3.5691399574279785, "no_memory_loss_delta": 0.4266662895679474, "scrambled_loss_delta": 0.14105774462223053, "memory_only_loss_delta": 2.3652377128601074, "no_evidence_loss_delta": 0.23512110114097595, "no_agreement_loss_delta": 0.03851505368947983, "no_orthogonal_loss_delta": 0.03144276142120361}, {"index": 283, "tokens": 131, "normal_loss": 2.12190318107605, "no_memory_loss_delta": 0.15969622135162354, "scrambled_loss_delta": 0.1498456448316574, "memory_only_loss_delta": 2.696610689163208, "no_evidence_loss_delta": 0.06491486728191376, "no_agreement_loss_delta": -0.012272496707737446, "no_orthogonal_loss_delta": -0.017754530534148216}], "agreement_weight_abs_mean": 0.1469375193119049, "agreement_weight_abs_max": 3.508157253265381, "agreement_weight_mean": 0.1266508847475052, "orthogonal_weight_abs_mean": 0.6728119850158691, "orthogonal_weight_abs_max": 5.643202304840088, "orthogonal_weight_mean": -0.15650397539138794, "no_memory_loss": 3.5840187072753906, "no_memory_loss_delta": 0.41820645332336426, "no_memory_token_accuracy_gain": 0.06229143589735031, "no_memory_token_win_fraction": 0.6651835441589355, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 34.177367861549584, "scrambled_loss": 3.3728508949279785, "scrambled_loss_delta": 0.20703862607479095, "scrambled_token_accuracy_gain": 0.01223581749945879, "scrambled_token_win_fraction": 0.6129032373428345, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.70117531637418, "memory_only_loss": 5.802434921264648, "memory_only_loss_delta": 2.636622905731201, "memory_only_token_accuracy_gain": 0.2602892220020294, "memory_only_token_win_fraction": 0.8965517282485962, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 92.83973293214348, "no_evidence_loss": 3.386497974395752, "no_evidence_loss_delta": 0.22068555653095245, "no_evidence_token_accuracy_gain": 0.030033370479941368, "no_evidence_token_win_fraction": 0.6051167845726013, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 19.803118549676014, "no_agreement_loss": 3.186359167098999, "no_agreement_loss_delta": 0.02054661326110363, "no_agreement_token_accuracy_gain": 0.0011123470030725002, "no_agreement_token_win_fraction": 0.4805339276790619, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.0336969878419344, "no_orthogonal_loss": 3.182681083679199, "no_orthogonal_loss_delta": 0.016868852078914642, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5372636318206787, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.67273696594265, "short_no_memory_loss_delta": 0.37381234765052795, "long_context_memory_gain": 0.044394105672836304, "normal_context_loss_gain": 0.12566812336444855, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.36566424369812, "contribution_to_active": 1.0570931434631348, "mix_mean": 0.10193680226802826, "mix_low": 0.5305155515670776, "mix_high": 0.004577795974910259, "read_cosine": 0.43188297748565674, "beta_mean": 0.2641221582889557, "nonfinite": 0.0, "weighted_memory_to_active": 1.1271343231201172, "mix_std": 0.08776184916496277, "beta_std": 0.1543283760547638, "beta_low": 0.0207112655043602, "beta_high": 0.0, "orthogonal_to_active": 0.0692974328994751, "output_influence": 0.1738138645887375, "residual_influence": 0.20792458951473236, "state_active": 0.07015304267406464, "state_memory": 0.047355908900499344, "state_peak": 43.88758087158203, "growth_active": 1.2553266286849976, "growth_memory": 1.2489280700683594, "heads": [{"head": 0, "memory_to_active": 2.572418212890625, "contribution_to_active": 0.46887850761413574, "mix_mean": 0.020980559289455414, "mix_low": 0.5869770050048828, "mix_high": 0.0, "read_cosine": 0.24124270677566528, "beta_mean": 0.20775745809078217, "nonfinite": 0.0, "weighted_memory_to_active": 0.549545168876648, "mix_std": 0.05802652984857559, "beta_std": 0.09823223948478699, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04166780412197113, "evidence_weights": [-0.031152650713920593, -0.0798947662115097, -0.08635912835597992], "agreement_weight": -0.10733630508184433, "orthogonal_weight": -0.08536632359027863}, {"head": 1, "memory_to_active": 5.250485897064209, "contribution_to_active": 2.048191547393799, "mix_mean": 0.01628684438765049, "mix_low": 0.6413297653198242, "mix_high": 0.0, "read_cosine": 0.4976631999015808, "beta_mean": 0.4432661235332489, "nonfinite": 0.0, "weighted_memory_to_active": 2.4254441261291504, "mix_std": 0.05159837752580643, "beta_std": 0.19284141063690186, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12458168715238571, "evidence_weights": [0.023475421592593193, -0.0905102863907814, 0.34597909450531006], "agreement_weight": 0.031314607709646225, "orthogonal_weight": -0.11233626306056976}, {"head": 2, "memory_to_active": 2.9540228843688965, "contribution_to_active": 0.5791417956352234, "mix_mean": 0.01429053582251072, "mix_low": 0.6565732955932617, "mix_high": 0.0, "read_cosine": 0.2832494378089905, "beta_mean": 0.3690215051174164, "nonfinite": 0.0, "weighted_memory_to_active": 0.6829705834388733, "mix_std": 0.03223862871527672, "beta_std": 0.24937810003757477, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07132086902856827, "evidence_weights": [0.049521930515766144, 0.0633426383137703, -0.03510427102446556], "agreement_weight": 0.003986753989011049, "orthogonal_weight": -0.2019263207912445}, {"head": 3, "memory_to_active": 6.658044338226318, "contribution_to_active": 2.798344373703003, "mix_mean": 0.03908952325582504, "mix_low": 0.44631004333496094, "mix_high": 0.0, "read_cosine": 0.3174772262573242, "beta_mean": 0.3603195548057556, "nonfinite": 0.0, "weighted_memory_to_active": 2.9959983825683594, "mix_std": 0.0863615944981575, "beta_std": 0.22202475368976593, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1537986844778061, "evidence_weights": [-0.05856945738196373, -0.07809694856405258, -0.13148103654384613], "agreement_weight": -0.012356971390545368, "orthogonal_weight": -0.06118949502706528}, {"head": 4, "memory_to_active": 2.6439096927642822, "contribution_to_active": 0.06789135932922363, "mix_mean": 0.006158705335110426, "mix_low": 0.8358278274536133, "mix_high": 0.0, "read_cosine": 0.32916560769081116, "beta_mean": 0.161285400390625, "nonfinite": 0.0, "weighted_memory_to_active": 0.05574354901909828, "mix_std": 0.01319303922355175, "beta_std": 0.12334764003753662, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005865645129233599, "evidence_weights": [0.12253138422966003, 0.07211088389158249, -0.3813159763813019], "agreement_weight": -0.11406659334897995, "orthogonal_weight": 0.10375792533159256}, {"head": 5, "memory_to_active": 0.11510411649942398, "contribution_to_active": 0.3801113963127136, "mix_mean": 0.5148146152496338, "mix_low": 0.01607513427734375, "mix_high": 0.027466773986816406, "read_cosine": 0.9224996566772461, "beta_mean": 0.04308280348777771, "nonfinite": 0.0, "weighted_memory_to_active": 0.053103577345609665, "mix_std": 0.2851528823375702, "beta_std": 0.04014609381556511, "beta_low": 0.124267578125, "beta_high": 0.0, "orthogonal_to_active": 0.018549898639321327, "evidence_weights": [-0.24507150053977966, -0.24493566155433655, 0.2804725170135498], "agreement_weight": 0.24071578681468964, "orthogonal_weight": -0.6369118690490723}]}, {"layer": 1, "memory_to_active": 2.209851026535034, "contribution_to_active": 0.5101271867752075, "mix_mean": 0.17232699692249298, "mix_low": 0.36460256576538086, "mix_high": 0.0006562868948094547, "read_cosine": 0.3414214849472046, "beta_mean": 0.18452580273151398, "nonfinite": 0.0, "weighted_memory_to_active": 0.3924614191055298, "mix_std": 0.12072213739156723, "beta_std": 0.1296045184135437, "beta_low": 0.2317301481962204, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.05185019224882126, "output_influence": 0.18354091048240662, "residual_influence": 0.044916436076164246, "state_active": 0.03931977227330208, "state_memory": 0.03650907054543495, "state_peak": 34.04221725463867, "growth_active": 1.2384006977081299, "growth_memory": 1.4898898601531982, "heads": [{"head": 0, "memory_to_active": 0.04148290306329727, "contribution_to_active": 0.4019014239311218, "mix_mean": 0.40304267406463623, "mix_low": 0.014719963073730469, "mix_high": 0.000240325927734375, "read_cosine": 0.030246783047914505, "beta_mean": 0.0002761876385193318, "nonfinite": 0.0, "weighted_memory_to_active": 0.02117891237139702, "mix_std": 0.2781127393245697, "beta_std": 0.0003144824877381325, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.019808579236268997, "evidence_weights": [-0.17736896872520447, -0.2294933944940567, 0.05224870890378952], "agreement_weight": 0.15109625458717346, "orthogonal_weight": -1.0195753574371338}, {"head": 1, "memory_to_active": 2.7422163486480713, "contribution_to_active": 0.19272050261497498, "mix_mean": 0.02196240797638893, "mix_low": 0.4980297088623047, "mix_high": 8.58306884765625e-06, "read_cosine": 0.4983518719673157, "beta_mean": 0.30206793546676636, "nonfinite": 0.0, "weighted_memory_to_active": 0.21978716552257538, "mix_std": 0.04911231994628906, "beta_std": 0.19135722517967224, "beta_low": 0.00042724609375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.011343337595462799, "evidence_weights": [0.013347587548196316, -0.018052183091640472, -0.010661739856004715], "agreement_weight": -0.006354282144457102, "orthogonal_weight": 0.11197766661643982}, {"head": 2, "memory_to_active": 0.33043378591537476, "contribution_to_active": 0.5424621105194092, "mix_mean": 0.5432083606719971, "mix_low": 0.0028467178344726562, "mix_high": 0.003688812255859375, "read_cosine": 0.7506827116012573, "beta_mean": 0.023145897313952446, "nonfinite": 0.0, "weighted_memory_to_active": 0.2677350640296936, "mix_std": 0.2548592984676361, "beta_std": 0.020504837855696678, "beta_low": 0.32403564453125, "beta_high": 0.0, "orthogonal_to_active": 0.04505906626582146, "evidence_weights": [-0.28063809871673584, -0.3627747595310211, 0.34205833077430725], "agreement_weight": 0.027516216039657593, "orthogonal_weight": -0.31697317957878113}, {"head": 3, "memory_to_active": 2.860159158706665, "contribution_to_active": 0.19049102067947388, "mix_mean": 0.020193252712488174, "mix_low": 0.6160268783569336, "mix_high": 0.0, "read_cosine": 0.24021276831626892, "beta_mean": 0.32773274183273315, "nonfinite": 0.0, "weighted_memory_to_active": 0.2844981253147125, "mix_std": 0.05405573174357414, "beta_std": 0.24849888682365417, "beta_low": 0.039306640625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.07207094132900238, "evidence_weights": [0.03335518389940262, -0.010096282698214054, 0.14803244173526764], "agreement_weight": -0.01684373803436756, "orthogonal_weight": -0.3398975729942322}, {"head": 4, "memory_to_active": 3.6267058849334717, "contribution_to_active": 1.6712336540222168, "mix_mean": 0.03678987920284271, "mix_low": 0.26487159729003906, "mix_high": 0.0, "read_cosine": 0.31295105814933777, "beta_mean": 0.19017314910888672, "nonfinite": 0.0, "weighted_memory_to_active": 1.5096848011016846, "mix_std": 0.0640910267829895, "beta_std": 0.1229529082775116, "beta_low": 0.0078125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1568405032157898, "evidence_weights": [-0.041776224970817566, 0.008307164534926414, 7.647011079825461e-05], "agreement_weight": -0.028367558494210243, "orthogonal_weight": 0.13141201436519623}, {"head": 5, "memory_to_active": 3.6581075191497803, "contribution_to_active": 0.06195414438843727, "mix_mean": 0.008765358477830887, "mix_low": 0.7911205291748047, "mix_high": 0.0, "read_cosine": 0.21608367562294006, "beta_mean": 0.26375889778137207, "nonfinite": 0.0, "weighted_memory_to_active": 0.051884330809116364, "mix_std": 0.024101678282022476, "beta_std": 0.19399873912334442, "beta_low": 0.018798828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.005978708155453205, "evidence_weights": [0.1491466462612152, 0.06949353963136673, 0.30054619908332825], "agreement_weight": -0.1415770947933197, "orthogonal_weight": 0.1119779422879219}]}, {"layer": 2, "memory_to_active": 1.0534106492996216, "contribution_to_active": 0.3551800549030304, "mix_mean": 0.3482248783111572, "mix_low": 0.2542724609375, "mix_high": 0.0030034384690225124, "read_cosine": 0.3209768831729889, "beta_mean": 0.0951320081949234, "nonfinite": 0.0, "weighted_memory_to_active": 0.08761642873287201, "mix_std": 0.1964704394340515, "beta_std": 0.06835094839334488, "beta_low": 0.546722412109375, "beta_high": 0.00014241535973269492, "orthogonal_to_active": 0.019245119765400887, "output_influence": 0.32377099990844727, "residual_influence": 0.08099602907896042, "state_active": 0.039636217057704926, "state_memory": 0.01900273934006691, "state_peak": 37.73549270629883, "growth_active": 1.073952317237854, "growth_memory": 1.59909987449646, "heads": [{"head": 0, "memory_to_active": 0.0063010756857693195, "contribution_to_active": 0.3536945581436157, "mix_mean": 0.5169409513473511, "mix_low": 0.002414703369140625, "mix_high": 0.0044765472412109375, "read_cosine": 0.18848749995231628, "beta_mean": 4.120964149478823e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003637969261035323, "mix_std": 0.29308247566223145, "beta_std": 5.9654805227182806e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011491075856611133, "evidence_weights": [-0.3024637997150421, -0.1883133351802826, 0.14729902148246765], "agreement_weight": 0.20041702687740326, "orthogonal_weight": -0.3345160484313965}, {"head": 1, "memory_to_active": 2.801957368850708, "contribution_to_active": 0.07329215109348297, "mix_mean": 0.00510474992915988, "mix_low": 0.8967885971069336, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21650998294353485, "beta_mean": 0.24727506935596466, "nonfinite": 0.0, "weighted_memory_to_active": 0.05179210007190704, "mix_std": 0.013623793609440327, "beta_std": 0.1340707391500473, "beta_low": 0.00018310546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.021980885416269302, "evidence_weights": [0.10410197824239731, 0.10245723277330399, 0.47677209973335266], "agreement_weight": 0.04042937606573105, "orthogonal_weight": 0.43290644884109497}, {"head": 2, "memory_to_active": 0.025215188041329384, "contribution_to_active": 0.4281073212623596, "mix_mean": 0.48003366589546204, "mix_low": 0.0017366409301757812, "mix_high": 0.0021572113037109375, "read_cosine": 0.1922665685415268, "beta_mean": 3.9347080019069836e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009923841804265976, "mix_std": 0.2813154458999634, "beta_std": 7.620595715707168e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023891052696853876, "evidence_weights": [-0.23021288216114044, -0.26886090636253357, 0.4730084538459778], "agreement_weight": 0.043030042201280594, "orthogonal_weight": -0.2722066342830658}, {"head": 3, "memory_to_active": 0.21472080051898956, "contribution_to_active": 0.48281481862068176, "mix_mean": 0.4680173099040985, "mix_low": 0.0007610321044921875, "mix_high": 0.004498481750488281, "read_cosine": 0.6925706267356873, "beta_mean": 0.03682989627122879, "nonfinite": 0.0, "weighted_memory_to_active": 0.11702877283096313, "mix_std": 0.26726651191711426, "beta_std": 0.04369082301855087, "beta_low": 0.26519775390625, "beta_high": 0.0, "orthogonal_to_active": 0.07044361531734467, "evidence_weights": [-0.1699150800704956, -0.26694363355636597, 0.8521342873573303], "agreement_weight": 0.06361737847328186, "orthogonal_weight": -0.879085123538971}, {"head": 4, "memory_to_active": 3.250633478164673, "contribution_to_active": 0.2792758345603943, "mix_mean": 0.01871797814965248, "mix_low": 0.622136116027832, "mix_high": 0.0, "read_cosine": 0.1948923021554947, "beta_mean": 0.28654366731643677, "nonfinite": 0.0, "weighted_memory_to_active": 0.32900163531303406, "mix_std": 0.04987948015332222, "beta_std": 0.23212821781635284, "beta_low": 0.01495361328125, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.018907014280557632, "evidence_weights": [0.024765556678175926, 0.017602654173970222, 0.06768596917390823], "agreement_weight": 0.018442299216985703, "orthogonal_weight": -0.0654296875}, {"head": 5, "memory_to_active": 0.02163591980934143, "contribution_to_active": 0.5138955116271973, "mix_mean": 0.6005346179008484, "mix_low": 0.0017976760864257812, "mix_high": 0.0068874359130859375, "read_cosine": 0.44113433361053467, "beta_mean": 6.284897972363979e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01431421097368002, "mix_std": 0.2736550271511078, "beta_std": 8.005194104043767e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000600983970798552, "evidence_weights": [-0.2634952962398529, -0.28298845887184143, 0.8644580245018005], "agreement_weight": 0.0867505595088005, "orthogonal_weight": 0.06480807811021805}]}, {"layer": 3, "memory_to_active": 0.9353728294372559, "contribution_to_active": 0.39485323429107666, "mix_mean": 0.33893436193466187, "mix_low": 0.16389068961143494, "mix_high": 0.006797314155846834, "read_cosine": 0.22532105445861816, "beta_mean": 0.052289798855781555, "nonfinite": 0.0, "weighted_memory_to_active": 0.16518424451351166, "mix_std": 0.2346285879611969, "beta_std": 0.0487380176782608, "beta_low": 0.4793701171875, "beta_high": 0.0, "orthogonal_to_active": 0.06926438957452774, "output_influence": 0.31802260875701904, "residual_influence": 0.06397796422243118, "state_active": 0.02364991419017315, "state_memory": 0.01223253458738327, "state_peak": 15.783049583435059, "growth_active": 1.0906611680984497, "growth_memory": 1.9747003316879272, "heads": [{"head": 0, "memory_to_active": 0.5196394920349121, "contribution_to_active": 0.5052869915962219, "mix_mean": 0.4441390037536621, "mix_low": 0.01978302001953125, "mix_high": 0.010172843933105469, "read_cosine": 0.2479989230632782, "beta_mean": 0.014946339651942253, "nonfinite": 0.0, "weighted_memory_to_active": 0.2713920474052429, "mix_std": 0.32381030917167664, "beta_std": 0.02442590333521366, "beta_low": 0.541259765625, "beta_high": 0.0, "orthogonal_to_active": 0.17610619962215424, "evidence_weights": [-0.17620405554771423, -0.21296344697475433, 0.667086660861969], "agreement_weight": 0.12024570256471634, "orthogonal_weight": -0.6892744898796082}, {"head": 1, "memory_to_active": 0.020233985036611557, "contribution_to_active": 0.3635769188404083, "mix_mean": 0.43817734718322754, "mix_low": 0.01612567901611328, "mix_high": 0.0037088394165039062, "read_cosine": 0.180383563041687, "beta_mean": 0.00010527441190788522, "nonfinite": 0.0, "weighted_memory_to_active": 0.0090520940721035, "mix_std": 0.30775707960128784, "beta_std": 0.00028441351605579257, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0045161256566643715, "evidence_weights": [-0.21235589683055878, -0.18378308415412903, 0.5658884644508362], "agreement_weight": 0.32848089933395386, "orthogonal_weight": -0.5522786378860474}, {"head": 2, "memory_to_active": 2.653475522994995, "contribution_to_active": 0.31648004055023193, "mix_mean": 0.043923936784267426, "mix_low": 0.3756847381591797, "mix_high": 0.0, "read_cosine": 0.19099128246307373, "beta_mean": 0.1797729879617691, "nonfinite": 0.0, "weighted_memory_to_active": 0.3706943392753601, "mix_std": 0.09794006496667862, "beta_std": 0.13591806590557098, "beta_low": 0.0089111328125, "beta_high": 0.0, "orthogonal_to_active": 0.007785665802657604, "evidence_weights": [-0.0638454258441925, 0.018839478492736816, -0.020838113501667976], "agreement_weight": -0.011513246223330498, "orthogonal_weight": -0.02645622193813324}, {"head": 3, "memory_to_active": 0.031543929129838943, "contribution_to_active": 0.48111051321029663, "mix_mean": 0.5290578603744507, "mix_low": 0.012973785400390625, "mix_high": 0.0109100341796875, "read_cosine": 0.2991636395454407, "beta_mean": 0.00011473981430754066, "nonfinite": 0.0, "weighted_memory_to_active": 0.02101069875061512, "mix_std": 0.30957895517349243, "beta_std": 0.0003082247276324779, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.005927463993430138, "evidence_weights": [-0.20883047580718994, -0.2223343849182129, 1.9202407598495483], "agreement_weight": 1.33992338180542, "orthogonal_weight": -0.36712804436683655}, {"head": 4, "memory_to_active": 2.087512493133545, "contribution_to_active": 0.050692714750766754, "mix_mean": 0.02866065502166748, "mix_low": 0.5449609756469727, "mix_high": 0.0, "read_cosine": 0.162909597158432, "beta_mean": 0.07805308699607849, "nonfinite": 0.0, "weighted_memory_to_active": 0.1124393492937088, "mix_std": 0.06718364357948303, "beta_std": 0.09174028038978577, "beta_low": 0.1519775390625, "beta_high": 0.0, "orthogonal_to_active": 0.07897184789180756, "evidence_weights": [0.09583419561386108, 0.04018982872366905, 0.13450022041797638], "agreement_weight": -0.03640640527009964, "orthogonal_weight": -0.7169902324676514}, {"head": 5, "memory_to_active": 0.2998313009738922, "contribution_to_active": 0.6519721746444702, "mix_mean": 0.5496472716331482, "mix_low": 0.013815879821777344, "mix_high": 0.015992164611816406, "read_cosine": 0.270479291677475, "beta_mean": 0.04074637219309807, "nonfinite": 0.0, "weighted_memory_to_active": 0.20651684701442719, "mix_std": 0.30150139331817627, "beta_std": 0.03975120559334755, "beta_low": 0.17431640625, "beta_high": 0.0, "orthogonal_to_active": 0.1422789841890335, "evidence_weights": [-0.2154017984867096, -0.2525368928909302, 0.46276527643203735], "agreement_weight": 0.1373080611228943, "orthogonal_weight": -0.7894463539123535}]}, {"layer": 4, "memory_to_active": 1.1802977323532104, "contribution_to_active": 0.4322727918624878, "mix_mean": 0.34088394045829773, "mix_low": 0.27974891662597656, "mix_high": 0.0073359813541173935, "read_cosine": 0.36943259835243225, "beta_mean": 0.12574627995491028, "nonfinite": 0.0, "weighted_memory_to_active": 0.15007640421390533, "mix_std": 0.15743309259414673, "beta_std": 0.11072289943695068, "beta_low": 0.3723348081111908, "beta_high": 0.0, "orthogonal_to_active": 0.03875841945409775, "output_influence": 0.37651583552360535, "residual_influence": 0.10084497183561325, "state_active": 0.03623395040631294, "state_memory": 0.02031649276614189, "state_peak": 37.00894546508789, "growth_active": 1.1761963367462158, "growth_memory": 1.581082820892334, "heads": [{"head": 0, "memory_to_active": 0.22656969726085663, "contribution_to_active": 0.7350173592567444, "mix_mean": 0.6577844619750977, "mix_low": 0.009116172790527344, "mix_high": 0.024129867553710938, "read_cosine": 0.5033890604972839, "beta_mean": 0.2463441789150238, "nonfinite": 0.0, "weighted_memory_to_active": 0.18304462730884552, "mix_std": 0.33565956354141235, "beta_std": 0.2585920989513397, "beta_low": 0.224365234375, "beta_high": 0.0, "orthogonal_to_active": 0.055682744830846786, "evidence_weights": [-0.26589906215667725, -0.3631424307823181, 0.2704896926879883], "agreement_weight": 0.018058544024825096, "orthogonal_weight": 0.3823241591453552}, {"head": 1, "memory_to_active": 0.0872262492775917, "contribution_to_active": 0.5798588991165161, "mix_mean": 0.6183269023895264, "mix_low": 0.00014400482177734375, "mix_high": 0.007168769836425781, "read_cosine": 0.7180595397949219, "beta_mean": 0.0018940225709229708, "nonfinite": 0.0, "weighted_memory_to_active": 0.05814339965581894, "mix_std": 0.25295382738113403, "beta_std": 0.007973063737154007, "beta_low": 0.9659423828125, "beta_high": 0.0, "orthogonal_to_active": 0.03171606734395027, "evidence_weights": [-0.2373013198375702, -0.3107990622520447, 0.7249335646629333], "agreement_weight": 0.10767305642366409, "orthogonal_weight": -0.8315988183021545}, {"head": 2, "memory_to_active": 2.4500503540039062, "contribution_to_active": 0.17333129048347473, "mix_mean": 0.018240083009004593, "mix_low": 0.5845575332641602, "mix_high": 2.86102294921875e-06, "read_cosine": 0.25197482109069824, "beta_mean": 0.11964021623134613, "nonfinite": 0.0, "weighted_memory_to_active": 0.22129687666893005, "mix_std": 0.03672930598258972, "beta_std": 0.08950695395469666, "beta_low": 0.00115966796875, "beta_high": 0.0, "orthogonal_to_active": 0.015544540248811245, "evidence_weights": [0.05562124773859978, 0.032366372644901276, -0.016228288412094116], "agreement_weight": 0.11185704171657562, "orthogonal_weight": -0.10749488323926926}, {"head": 3, "memory_to_active": 1.8039911985397339, "contribution_to_active": 0.1662387251853943, "mix_mean": 0.03090471401810646, "mix_low": 0.4129362106323242, "mix_high": 0.0, "read_cosine": 0.2077779471874237, "beta_mean": 0.13471901416778564, "nonfinite": 0.0, "weighted_memory_to_active": 0.23025193810462952, "mix_std": 0.058172907680273056, "beta_std": 0.12958945333957672, "beta_low": 0.04254150390625, "beta_high": 0.0, "orthogonal_to_active": 0.014656925573945045, "evidence_weights": [-0.02111188694834709, 0.059213679283857346, -0.23604372143745422], "agreement_weight": 0.05396701395511627, "orthogonal_weight": -0.0869564488530159}, {"head": 4, "memory_to_active": 0.022313207387924194, "contribution_to_active": 0.6345211267471313, "mix_mean": 0.7068177461624146, "mix_low": 0.00032520294189453125, "mix_high": 0.012703895568847656, "read_cosine": 0.20862339437007904, "beta_mean": 4.551839811028913e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.016053421422839165, "mix_std": 0.22969861328601837, "beta_std": 9.896921983454376e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002888982417061925, "evidence_weights": [-0.31107908487319946, -0.3340663015842438, 0.793125331401825], "agreement_weight": 0.2285173237323761, "orthogonal_weight": -0.19724301993846893}, {"head": 5, "memory_to_active": 2.49163556098938, "contribution_to_active": 0.3046691417694092, "mix_mean": 0.01322980597615242, "mix_low": 0.6714143753051758, "mix_high": 1.049041748046875e-05, "read_cosine": 0.32677069306373596, "beta_mean": 0.2518347203731537, "nonfinite": 0.0, "weighted_memory_to_active": 0.19166816771030426, "mix_std": 0.031384266912937164, "beta_std": 0.17857690155506134, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11206125468015671, "evidence_weights": [0.046000853180885315, -0.03572066128253937, -0.2620948255062103], "agreement_weight": -0.06162523478269577, "orthogonal_weight": 0.8278979659080505}]}, {"layer": 5, "memory_to_active": 0.5739916563034058, "contribution_to_active": 0.4880369305610657, "mix_mean": 0.4441685974597931, "mix_low": 0.0728704184293747, "mix_high": 0.012406985275447369, "read_cosine": 0.14213208854198456, "beta_mean": 0.05680438131093979, "nonfinite": 0.0, "weighted_memory_to_active": 0.16619780659675598, "mix_std": 0.2543998062610626, "beta_std": 0.07408178597688675, "beta_low": 0.623748779296875, "beta_high": 0.0, "orthogonal_to_active": 0.057968754321336746, "output_influence": 0.4901404082775116, "residual_influence": 0.12494602799415588, "state_active": 0.04558844491839409, "state_memory": 0.012158677913248539, "state_peak": 60.106258392333984, "growth_active": 1.1054729223251343, "growth_memory": 1.2927720546722412, "heads": [{"head": 0, "memory_to_active": 0.9593097567558289, "contribution_to_active": 0.38832539319992065, "mix_mean": 0.2207401841878891, "mix_low": 0.0733041763305664, "mix_high": 0.00021839141845703125, "read_cosine": -0.1370285451412201, "beta_mean": 0.047082990407943726, "nonfinite": 0.0, "weighted_memory_to_active": 0.19484619796276093, "mix_std": 0.2305385321378708, "beta_std": 0.07541580498218536, "beta_low": 0.27886962890625, "beta_high": 0.0, "orthogonal_to_active": 0.042990732938051224, "evidence_weights": [-0.08920477330684662, -0.12249699980020523, -0.3055272102355957], "agreement_weight": -0.019799435511231422, "orthogonal_weight": -0.21841786801815033}, {"head": 1, "memory_to_active": 0.02457958087325096, "contribution_to_active": 0.5267504453659058, "mix_mean": 0.6200646162033081, "mix_low": 0.0033245086669921875, "mix_high": 0.0171661376953125, "read_cosine": 0.21230420470237732, "beta_mean": 5.636882997350767e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013219431973993778, "mix_std": 0.29583197832107544, "beta_std": 0.00011643540346994996, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012121867388486862, "evidence_weights": [-0.2399764209985733, -0.27543166279792786, 0.8102353811264038], "agreement_weight": 0.45885705947875977, "orthogonal_weight": -0.9567908048629761}, {"head": 2, "memory_to_active": 0.018209420144557953, "contribution_to_active": 0.5384653806686401, "mix_mean": 0.6234828233718872, "mix_low": 0.0031251907348632812, "mix_high": 0.015619277954101562, "read_cosine": -0.082801952958107, "beta_mean": 0.00022290213382802904, "nonfinite": 0.0, "weighted_memory_to_active": 0.01231109444051981, "mix_std": 0.27635788917541504, "beta_std": 0.0003353790380060673, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028059985488653183, "evidence_weights": [-0.2540675401687622, -0.348753422498703, 0.8326724767684937], "agreement_weight": 0.021211551502346992, "orthogonal_weight": -0.26595979928970337}, {"head": 3, "memory_to_active": 1.8701157569885254, "contribution_to_active": 0.25051578879356384, "mix_mean": 0.09492523223161697, "mix_low": 0.16022300720214844, "mix_high": 0.0, "read_cosine": 0.1325278878211975, "beta_mean": 0.09899753332138062, "nonfinite": 0.0, "weighted_memory_to_active": 0.38999035954475403, "mix_std": 0.137089803814888, "beta_std": 0.14462313055992126, "beta_low": 0.19989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.14336803555488586, "evidence_weights": [-0.05887220427393913, -0.09309855848550797, -0.06042070686817169], "agreement_weight": 0.06341280043125153, "orthogonal_weight": -0.44924497604370117}, {"head": 4, "memory_to_active": 0.4461897015571594, "contribution_to_active": 0.5934123992919922, "mix_mean": 0.3353084325790405, "mix_low": 0.19394969940185547, "mix_high": 0.009054183959960938, "read_cosine": 0.5528064966201782, "beta_mean": 0.1940622627735138, "nonfinite": 0.0, "weighted_memory_to_active": 0.3013126850128174, "mix_std": 0.3442465662956238, "beta_std": 0.22198717296123505, "beta_low": 0.27020263671875, "beta_high": 0.0, "orthogonal_to_active": 0.09199191629886627, "evidence_weights": [-0.11536161601543427, -0.28866198658943176, -0.0571853443980217], "agreement_weight": 0.043494004756212234, "orthogonal_weight": 0.4698112905025482}, {"head": 5, "memory_to_active": 0.1255456656217575, "contribution_to_active": 0.6307520866394043, "mix_mean": 0.770490288734436, "mix_low": 0.0032958984375, "mix_high": 0.03238391876220703, "read_cosine": 0.174984410405159, "beta_mean": 0.0004042237706016749, "nonfinite": 0.0, "weighted_memory_to_active": 0.08550696820020676, "mix_std": 0.24233393371105194, "beta_std": 0.002012774348258972, "beta_low": 0.9935302734375, "beta_high": 0.0, "orthogonal_to_active": 0.054533954709768295, "evidence_weights": [-0.32642486691474915, -0.3220806121826172, 2.031412124633789], "agreement_weight": 0.3820383846759796, "orthogonal_weight": -0.6844239234924316}]}, {"layer": 6, "memory_to_active": 0.35348397493362427, "contribution_to_active": 0.5852044820785522, "mix_mean": 0.5505298972129822, "mix_low": 0.07412879168987274, "mix_high": 0.030287902802228928, "read_cosine": 0.2776026129722595, "beta_mean": 0.06853653490543365, "nonfinite": 0.0, "weighted_memory_to_active": 0.1612955927848816, "mix_std": 0.24111409485340118, "beta_std": 0.07795929908752441, "beta_low": 0.5050048828125, "beta_high": 0.0, "orthogonal_to_active": 0.03775499016046524, "output_influence": 0.49032872915267944, "residual_influence": 0.16149576008319855, "state_active": 0.04658007621765137, "state_memory": 0.008964188396930695, "state_peak": 81.67259979248047, "growth_active": 1.1788479089736938, "growth_memory": 1.1969482898712158, "heads": [{"head": 0, "memory_to_active": 0.15264403820037842, "contribution_to_active": 0.6192516088485718, "mix_mean": 0.7333317399024963, "mix_low": 0.0001506805419921875, "mix_high": 0.010511398315429688, "read_cosine": -0.019602099433541298, "beta_mean": 0.014950687065720558, "nonfinite": 0.0, "weighted_memory_to_active": 0.11320663243532181, "mix_std": 0.22582098841667175, "beta_std": 0.018069522455334663, "beta_low": 0.50537109375, "beta_high": 0.0, "orthogonal_to_active": 0.09107579290866852, "evidence_weights": [-0.31314966082572937, -0.33452245593070984, 0.5449285507202148], "agreement_weight": 0.14148278534412384, "orthogonal_weight": -0.934535801410675}, {"head": 1, "memory_to_active": 0.005373115185648203, "contribution_to_active": 0.49741029739379883, "mix_mean": 0.5130031108856201, "mix_low": 0.001934051513671875, "mix_high": 0.008832931518554688, "read_cosine": 0.5786362886428833, "beta_mean": 1.1956395610468462e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0023398352786898613, "mix_std": 0.2912132441997528, "beta_std": 2.1709123757318594e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000972122186794877, "evidence_weights": [-0.19992974400520325, -0.28249800205230713, 0.9970765709877014], "agreement_weight": 0.014278321526944637, "orthogonal_weight": -0.5768018364906311}, {"head": 2, "memory_to_active": 1.0764658451080322, "contribution_to_active": 0.08378633856773376, "mix_mean": 0.02809496596455574, "mix_low": 0.43296146392822266, "mix_high": 9.5367431640625e-07, "read_cosine": 0.22403673827648163, "beta_mean": 0.06092468649148941, "nonfinite": 0.0, "weighted_memory_to_active": 0.06721240282058716, "mix_std": 0.04912765324115753, "beta_std": 0.057879623025655746, "beta_low": 0.0157470703125, "beta_high": 0.0, "orthogonal_to_active": 0.00024377446970902383, "evidence_weights": [0.08967989683151245, -0.0055073159746825695, 0.3028254508972168], "agreement_weight": 0.13155370950698853, "orthogonal_weight": 0.00449456088244915}, {"head": 3, "memory_to_active": 0.14446592330932617, "contribution_to_active": 0.6628774404525757, "mix_mean": 0.6413329243659973, "mix_low": 0.007994651794433594, "mix_high": 0.017181396484375, "read_cosine": 0.4361454248428345, "beta_mean": 0.2056485116481781, "nonfinite": 0.0, "weighted_memory_to_active": 0.1020103171467781, "mix_std": 0.3307741582393646, "beta_std": 0.2421761006116867, "beta_low": 0.23797607421875, "beta_high": 0.0, "orthogonal_to_active": 0.05606808140873909, "evidence_weights": [-0.1761094033718109, -0.3382229208946228, 0.15882880985736847], "agreement_weight": -0.012183009646832943, "orthogonal_weight": 0.6259916424751282}, {"head": 4, "memory_to_active": 0.6975793838500977, "contribution_to_active": 1.0649402141571045, "mix_mean": 0.7678166627883911, "mix_low": 0.0014019012451171875, "mix_high": 0.14287090301513672, "read_cosine": 0.1751123070716858, "beta_mean": 0.12934648990631104, "nonfinite": 0.0, "weighted_memory_to_active": 0.6529037356376648, "mix_std": 0.28596964478492737, "beta_std": 0.14885888993740082, "beta_low": 0.27178955078125, "beta_high": 0.0, "orthogonal_to_active": 0.04747164994478226, "evidence_weights": [-0.25782284140586853, -0.5969929099082947, -0.0878053605556488], "agreement_weight": -0.0220227912068367, "orthogonal_weight": 0.08630587160587311}, {"head": 5, "memory_to_active": 0.04437548667192459, "contribution_to_active": 0.5829605460166931, "mix_mean": 0.6195998191833496, "mix_low": 0.0003299713134765625, "mix_high": 0.0023298263549804688, "read_cosine": 0.2712870240211487, "beta_mean": 0.00033685422386042774, "nonfinite": 0.0, "weighted_memory_to_active": 0.030100654810667038, "mix_std": 0.2637788951396942, "beta_std": 0.0007499406347051263, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.030698521062731743, "evidence_weights": [-0.23876874148845673, -0.30397361516952515, 0.4282255172729492], "agreement_weight": 0.1317935287952423, "orthogonal_weight": -1.089290976524353}]}, {"layer": 7, "memory_to_active": 1.7451192140579224, "contribution_to_active": 0.2207821160554886, "mix_mean": 0.08847975730895996, "mix_low": 0.30817079544067383, "mix_high": 8.471807086607441e-05, "read_cosine": 0.22983984649181366, "beta_mean": 0.12970706820487976, "nonfinite": 0.0, "weighted_memory_to_active": 0.22135229408740997, "mix_std": 0.11687684059143066, "beta_std": 0.10816048830747604, "beta_low": 0.1687927395105362, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.06009133160114288, "output_influence": 0.21819069981575012, "residual_influence": 0.04961637407541275, "state_active": 0.0140310600399971, "state_memory": 0.02724386751651764, "state_peak": 30.425006866455078, "growth_active": 1.111925721168518, "growth_memory": 1.2811074256896973, "heads": [{"head": 0, "memory_to_active": 2.2721965312957764, "contribution_to_active": 0.3140902817249298, "mix_mean": 0.032920315861701965, "mix_low": 0.44974517822265625, "mix_high": 0.0, "read_cosine": 0.3655542731285095, "beta_mean": 0.1627608835697174, "nonfinite": 0.0, "weighted_memory_to_active": 0.32764261960983276, "mix_std": 0.06840594112873077, "beta_std": 0.1248241439461708, "beta_low": 0.003662109375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.016501903533935547, "evidence_weights": [0.01539666298776865, -0.08105194568634033, 0.10940439999103546], "agreement_weight": -0.03964710608124733, "orthogonal_weight": 0.06818617135286331}, {"head": 1, "memory_to_active": 1.2045559883117676, "contribution_to_active": 0.3707006573677063, "mix_mean": 0.17542903125286102, "mix_low": 0.1744976043701172, "mix_high": 0.00040912628173828125, "read_cosine": 0.30072611570358276, "beta_mean": 0.12133170664310455, "nonfinite": 0.0, "weighted_memory_to_active": 0.3463299572467804, "mix_std": 0.2075345516204834, "beta_std": 0.07530912011861801, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003632360603660345, "evidence_weights": [-0.18608738481998444, -0.11651945114135742, -0.2047698199748993], "agreement_weight": 0.012591755017638206, "orthogonal_weight": 0.00117186747957021}, {"head": 2, "memory_to_active": 0.9169400930404663, "contribution_to_active": 0.27101680636405945, "mix_mean": 0.10230473428964615, "mix_low": 0.12691593170166016, "mix_high": 9.822845458984375e-05, "read_cosine": 0.12069115787744522, "beta_mean": 0.07130241394042969, "nonfinite": 0.0, "weighted_memory_to_active": 0.20768743753433228, "mix_std": 0.14981625974178314, "beta_std": 0.05379791930317879, "beta_low": 0.0499267578125, "beta_high": 0.0, "orthogonal_to_active": 0.05565015971660614, "evidence_weights": [-0.03862687572836876, -0.12595662474632263, 0.38945096731185913], "agreement_weight": 0.0228116475045681, "orthogonal_weight": -0.2753955125808716}, {"head": 3, "memory_to_active": 3.9698033332824707, "contribution_to_active": 0.09785676002502441, "mix_mean": 0.033462248742580414, "mix_low": 0.3537273406982422, "mix_high": 0.0, "read_cosine": 0.0932575985789299, "beta_mean": 0.27167820930480957, "nonfinite": 0.0, "weighted_memory_to_active": 0.2854846715927124, "mix_std": 0.051619164645671844, "beta_std": 0.2414397895336151, "beta_low": 0.02099609375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.234980970621109, "evidence_weights": [0.06573297083377838, 0.002094060182571411, 0.6536521911621094], "agreement_weight": 0.05867866054177284, "orthogonal_weight": -0.9307599067687988}, {"head": 4, "memory_to_active": 1.7141468524932861, "contribution_to_active": 0.0722857192158699, "mix_mean": 0.014255408197641373, "mix_low": 0.6806745529174805, "mix_high": 0.0, "read_cosine": 0.2495238184928894, "beta_mean": 0.14681556820869446, "nonfinite": 0.0, "weighted_memory_to_active": 0.06227733939886093, "mix_std": 0.03317273408174515, "beta_std": 0.14303477108478546, "beta_low": 0.0418701171875, "beta_high": 0.0, "orthogonal_to_active": 0.0053211902268230915, "evidence_weights": [0.13063158094882965, 0.05381852015852928, 0.16299289464950562], "agreement_weight": 0.047562118619680405, "orthogonal_weight": -0.11793064326047897}, {"head": 5, "memory_to_active": 0.3930717706680298, "contribution_to_active": 0.19874240458011627, "mix_mean": 0.17250679433345795, "mix_low": 0.06346416473388672, "mix_high": 9.5367431640625e-07, "read_cosine": 0.24928613007068634, "beta_mean": 0.004353642463684082, "nonfinite": 0.0, "weighted_memory_to_active": 0.09869171679019928, "mix_std": 0.1907123625278473, "beta_std": 0.010557141155004501, "beta_low": 0.89630126953125, "beta_high": 0.0, "orthogonal_to_active": 0.04773053526878357, "evidence_weights": [-0.05826771631836891, -0.10672620683908463, 0.30584198236465454], "agreement_weight": 0.07212834805250168, "orthogonal_weight": -0.5875517129898071}]}, {"layer": 8, "memory_to_active": 0.5155888795852661, "contribution_to_active": 0.5153641700744629, "mix_mean": 0.4104917049407959, "mix_low": 0.09834003448486328, "mix_high": 0.0007947286358103156, "read_cosine": 0.49475282430648804, "beta_mean": 0.03293292596936226, "nonfinite": 0.0, "weighted_memory_to_active": 0.10127423703670502, "mix_std": 0.22183790802955627, "beta_std": 0.03975020721554756, "beta_low": 0.6046549677848816, "beta_high": 0.0, "orthogonal_to_active": 0.05838233232498169, "output_influence": 0.43023329973220825, "residual_influence": 0.131546288728714, "state_active": 0.052371859550476074, "state_memory": 0.010447404347360134, "state_peak": 32.67014694213867, "growth_active": 1.113618016242981, "growth_memory": 1.345633625984192, "heads": [{"head": 0, "memory_to_active": 0.05103560909628868, "contribution_to_active": 0.5338912010192871, "mix_mean": 0.578762412071228, "mix_low": 0.00030612945556640625, "mix_high": 0.0009069442749023438, "read_cosine": 0.6030648946762085, "beta_mean": 0.0034735717345029116, "nonfinite": 0.0, "weighted_memory_to_active": 0.03377435356378555, "mix_std": 0.24157463014125824, "beta_std": 0.007923861034214497, "beta_low": 0.91107177734375, "beta_high": 0.0, "orthogonal_to_active": 0.02088576927781105, "evidence_weights": [-0.17156650125980377, -0.38759925961494446, 0.7118858098983765], "agreement_weight": 0.09842970222234726, "orthogonal_weight": -0.8339956998825073}, {"head": 1, "memory_to_active": 0.1636704057455063, "contribution_to_active": 0.4701705873012543, "mix_mean": 0.4462835192680359, "mix_low": 0.0014925003051757812, "mix_high": 0.00022602081298828125, "read_cosine": 0.3250858187675476, "beta_mean": 0.0041073765605688095, "nonfinite": 0.0, "weighted_memory_to_active": 0.07864037156105042, "mix_std": 0.2521435618400574, "beta_std": 0.011617536656558514, "beta_low": 0.91119384765625, "beta_high": 0.0, "orthogonal_to_active": 0.04670083522796631, "evidence_weights": [-0.19132083654403687, -0.24766336381435394, 0.6870400309562683], "agreement_weight": 0.07713183760643005, "orthogonal_weight": -0.6591715216636658}, {"head": 2, "memory_to_active": 0.027929410338401794, "contribution_to_active": 0.6292174458503723, "mix_mean": 0.6062037944793701, "mix_low": 0.00103759765625, "mix_high": 0.0025081634521484375, "read_cosine": 0.369098424911499, "beta_mean": 0.0002487598976586014, "nonfinite": 0.0, "weighted_memory_to_active": 0.019610639661550522, "mix_std": 0.23643752932548523, "beta_std": 0.0033540152944624424, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.013447706587612629, "evidence_weights": [-0.21815526485443115, -0.33578208088874817, 0.8332400321960449], "agreement_weight": 0.18637202680110931, "orthogonal_weight": -0.778377890586853}, {"head": 3, "memory_to_active": 0.13514135777950287, "contribution_to_active": 0.6043877601623535, "mix_mean": 0.5374351739883423, "mix_low": 0.00151824951171875, "mix_high": 0.0008382797241210938, "read_cosine": 0.8334967494010925, "beta_mean": 0.011070184409618378, "nonfinite": 0.0, "weighted_memory_to_active": 0.09849915653467178, "mix_std": 0.2554202079772949, "beta_std": 0.008547920733690262, "beta_low": 0.5361328125, "beta_high": 0.0, "orthogonal_to_active": 0.028907354921102524, "evidence_weights": [-0.1738189309835434, -0.2963082790374756, 0.6752912402153015], "agreement_weight": 0.14986181259155273, "orthogonal_weight": -0.6167686581611633}, {"head": 4, "memory_to_active": 2.421945571899414, "contribution_to_active": 0.21460629999637604, "mix_mean": 0.026164986193180084, "mix_low": 0.4397287368774414, "mix_high": 0.0, "read_cosine": 0.3342275023460388, "beta_mean": 0.09822609275579453, "nonfinite": 0.0, "weighted_memory_to_active": 0.20435667037963867, "mix_std": 0.04429028183221817, "beta_std": 0.09595697373151779, "beta_low": 0.02685546875, "beta_high": 0.0, "orthogonal_to_active": 0.007020076736807823, "evidence_weights": [0.05059810355305672, -0.015051721595227718, -0.09707716107368469], "agreement_weight": 0.04642794281244278, "orthogonal_weight": 0.03706549108028412}, {"head": 5, "memory_to_active": 0.2938106954097748, "contribution_to_active": 0.6399115920066833, "mix_mean": 0.2681000828742981, "mix_low": 0.14595699310302734, "mix_high": 0.00028896331787109375, "read_cosine": 0.503543496131897, "beta_mean": 0.08047156035900116, "nonfinite": 0.0, "weighted_memory_to_active": 0.17276421189308167, "mix_std": 0.3011612296104431, "beta_std": 0.1111009418964386, "beta_low": 0.2427978515625, "beta_high": 0.0, "orthogonal_to_active": 0.23333224654197693, "evidence_weights": [-0.06855033338069916, -0.2786283493041992, -0.3847900927066803], "agreement_weight": 0.003955836873501539, "orthogonal_weight": 1.6043089628219604}]}, {"layer": 9, "memory_to_active": 1.1026759147644043, "contribution_to_active": 0.6105993986129761, "mix_mean": 0.30712056159973145, "mix_low": 0.1496489942073822, "mix_high": 0.0028540294151753187, "read_cosine": 0.37337201833724976, "beta_mean": 0.07391557097434998, "nonfinite": 0.0, "weighted_memory_to_active": 0.3470042049884796, "mix_std": 0.20881128311157227, "beta_std": 0.04667207971215248, "beta_low": 0.4993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.04187082499265671, "output_influence": 0.48309049010276794, "residual_influence": 0.12912042438983917, "state_active": 0.038596011698246, "state_memory": 0.020365070551633835, "state_peak": 43.195045471191406, "growth_active": 1.1555006504058838, "growth_memory": 1.3442506790161133, "heads": [{"head": 0, "memory_to_active": 0.002698706928640604, "contribution_to_active": 0.531939685344696, "mix_mean": 0.601154088973999, "mix_low": 0.0017642974853515625, "mix_high": 0.012294769287109375, "read_cosine": 0.268140971660614, "beta_mean": 6.625640253332676e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0016508989501744509, "mix_std": 0.2928478717803955, "beta_std": 2.2788721253164113e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014460495440289378, "evidence_weights": [-0.29337114095687866, -0.3019370436668396, 0.9567161798477173], "agreement_weight": 0.09836958348751068, "orthogonal_weight": -0.9307429790496826}, {"head": 1, "memory_to_active": 0.033312443643808365, "contribution_to_active": 0.6049720048904419, "mix_mean": 0.4711296856403351, "mix_low": 0.009848594665527344, "mix_high": 0.0011730194091796875, "read_cosine": 0.4076411724090576, "beta_mean": 9.060010779649019e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.020750202238559723, "mix_std": 0.2901936173439026, "beta_std": 0.00020184025925118476, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01442029606550932, "evidence_weights": [-0.21540340781211853, -0.2629031240940094, 1.1431089639663696], "agreement_weight": 0.34386321902275085, "orthogonal_weight": -0.8250594139099121}, {"head": 2, "memory_to_active": 0.21089395880699158, "contribution_to_active": 0.617997944355011, "mix_mean": 0.5412686467170715, "mix_low": 0.001987457275390625, "mix_high": 0.0033769607543945312, "read_cosine": 0.34544801712036133, "beta_mean": 0.0008981595747172832, "nonfinite": 0.0, "weighted_memory_to_active": 0.1348366141319275, "mix_std": 0.27930793166160583, "beta_std": 0.0053152889013290405, "beta_low": 0.992431640625, "beta_high": 0.0, "orthogonal_to_active": 0.06638850271701813, "evidence_weights": [-0.18833671510219574, -0.2495008260011673, 0.6285573840141296], "agreement_weight": 0.25615113973617554, "orthogonal_weight": -0.5799448490142822}, {"head": 3, "memory_to_active": 1.6352384090423584, "contribution_to_active": 0.16686588525772095, "mix_mean": 0.03982891887426376, "mix_low": 0.37810707092285156, "mix_high": 5.7220458984375e-06, "read_cosine": 0.4220532178878784, "beta_mean": 0.08810168504714966, "nonfinite": 0.0, "weighted_memory_to_active": 0.13549283146858215, "mix_std": 0.08570358902215958, "beta_std": 0.06631047278642654, "beta_low": 0.00177001953125, "beta_high": 0.0, "orthogonal_to_active": 0.023063642904162407, "evidence_weights": [0.07214737683534622, 0.03250693157315254, -0.0847664400935173], "agreement_weight": 0.08006181567907333, "orthogonal_weight": 0.18191465735435486}, {"head": 4, "memory_to_active": 3.020110607147217, "contribution_to_active": 1.121665120124817, "mix_mean": 0.12157633900642395, "mix_low": 0.1654977798461914, "mix_high": 0.00026988983154296875, "read_cosine": 0.31477591395378113, "beta_mean": 0.2184026837348938, "nonfinite": 0.0, "weighted_memory_to_active": 1.103325366973877, "mix_std": 0.18042698502540588, "beta_std": 0.1114034503698349, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.025779124349355698, "evidence_weights": [-0.05927557125687599, -0.1496751606464386, 0.004125010222196579], "agreement_weight": -0.05300190672278404, "orthogonal_weight": -0.023986512795090675}, {"head": 5, "memory_to_active": 1.7138009071350098, "contribution_to_active": 0.6201556921005249, "mix_mean": 0.0677657425403595, "mix_low": 0.34068870544433594, "mix_high": 3.814697265625e-06, "read_cosine": 0.48217281699180603, "beta_mean": 0.13599362969398499, "nonfinite": 0.0, "weighted_memory_to_active": 0.6859691143035889, "mix_std": 0.12438760697841644, "beta_std": 0.09677864611148834, "beta_low": 0.00213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.12012730538845062, "evidence_weights": [0.09660104662179947, -0.06336662918329239, 0.15659305453300476], "agreement_weight": 0.02193489670753479, "orthogonal_weight": 0.255742609500885}]}, {"layer": 10, "memory_to_active": 0.2300955355167389, "contribution_to_active": 0.5963536500930786, "mix_mean": 0.4878930151462555, "mix_low": 0.012458325363695621, "mix_high": 0.007208188530057669, "read_cosine": 0.37042519450187683, "beta_mean": 0.0466596893966198, "nonfinite": 0.0, "weighted_memory_to_active": 0.11588744074106216, "mix_std": 0.2664109170436859, "beta_std": 0.048449546098709106, "beta_low": 0.6572672724723816, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.12811100482940674, "output_influence": 0.415082722902298, "residual_influence": 0.1388503611087799, "state_active": 0.053032927215099335, "state_memory": 0.013962828554213047, "state_peak": 106.89354705810547, "growth_active": 1.0615098476409912, "growth_memory": 0.9939327239990234, "heads": [{"head": 0, "memory_to_active": 0.8445910215377808, "contribution_to_active": 1.4163984060287476, "mix_mean": 0.6169646978378296, "mix_low": 0.030454635620117188, "mix_high": 0.031276702880859375, "read_cosine": 0.09346477687358856, "beta_mean": 0.20839127898216248, "nonfinite": 0.0, "weighted_memory_to_active": 0.5575172901153564, "mix_std": 0.37946927547454834, "beta_std": 0.22247745096683502, "beta_low": 0.1221923828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.6851211190223694, "evidence_weights": [-0.13711528480052948, -0.378520667552948, -0.8427648544311523], "agreement_weight": 0.024251673370599747, "orthogonal_weight": 1.2469512224197388}, {"head": 1, "memory_to_active": 0.016138587146997452, "contribution_to_active": 0.5938313007354736, "mix_mean": 0.5933217406272888, "mix_low": 0.0006237030029296875, "mix_high": 0.00919342041015625, "read_cosine": 0.2648076117038727, "beta_mean": 9.310173481935635e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009392889216542244, "mix_std": 0.2677943706512451, "beta_std": 0.00012074575352016836, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012539929710328579, "evidence_weights": [-0.2723914682865143, -0.2028568983078003, 0.5973079204559326], "agreement_weight": 0.42701202630996704, "orthogonal_weight": -1.4542033672332764}, {"head": 2, "memory_to_active": 0.057130541652441025, "contribution_to_active": 0.5478818416595459, "mix_mean": 0.563380777835846, "mix_low": 0.00040531158447265625, "mix_high": 0.0023851394653320312, "read_cosine": 0.01874927617609501, "beta_mean": 0.00013273216609377414, "nonfinite": 0.0, "weighted_memory_to_active": 0.022784322500228882, "mix_std": 0.2549193501472473, "beta_std": 0.0006657273042947054, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.009890316054224968, "evidence_weights": [-0.21934166550636292, -0.26098182797431946, 0.538600504398346], "agreement_weight": -0.11579225212335587, "orthogonal_weight": -0.41370657086372375}, {"head": 3, "memory_to_active": 0.3191841244697571, "contribution_to_active": 0.09561052918434143, "mix_mean": 0.26187676191329956, "mix_low": 0.040053367614746094, "mix_high": 0.00012874603271484375, "read_cosine": 0.9106415510177612, "beta_mean": 0.06456895917654037, "nonfinite": 0.0, "weighted_memory_to_active": 0.03535504639148712, "mix_std": 0.22135232388973236, "beta_std": 0.05919969454407692, "beta_low": 0.00164794921875, "beta_high": 0.0, "orthogonal_to_active": 0.041440196335315704, "evidence_weights": [-0.11663798987865448, -0.14742188155651093, 0.1409013718366623], "agreement_weight": 0.003962284419685602, "orthogonal_weight": 1.4924989938735962}, {"head": 4, "memory_to_active": 0.002897930331528187, "contribution_to_active": 0.5070796608924866, "mix_mean": 0.5154136419296265, "mix_low": 0.0007419586181640625, "mix_high": 0.00026035308837890625, "read_cosine": 0.06759308278560638, "beta_mean": 1.8607708625495434e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014012743486091495, "mix_std": 0.25134986639022827, "beta_std": 0.00013482046779245138, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0007378313457593322, "evidence_weights": [-0.18084844946861267, -0.25235238671302795, 0.7278900146484375], "agreement_weight": 0.06614620238542557, "orthogonal_weight": -0.5449633598327637}, {"head": 5, "memory_to_active": 0.14063090085983276, "contribution_to_active": 0.41731971502304077, "mix_mean": 0.3764002323150635, "mix_low": 0.0024709701538085938, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8672947883605957, "beta_mean": 0.006753428373485804, "nonfinite": 0.0, "weighted_memory_to_active": 0.0688738226890564, "mix_std": 0.2235802710056305, "beta_std": 0.008098836988210678, "beta_low": 0.82049560546875, "beta_high": 0.0, "orthogonal_to_active": 0.01893659308552742, "evidence_weights": [-0.147536039352417, -0.15946829319000244, 0.2934901714324951], "agreement_weight": 0.09143270552158356, "orthogonal_weight": -0.6191484332084656}]}, {"layer": 11, "memory_to_active": 0.15322236716747284, "contribution_to_active": 0.5669863820075989, "mix_mean": 0.46348220109939575, "mix_low": 0.08044815063476562, "mix_high": 0.04403337091207504, "read_cosine": 0.4031287133693695, "beta_mean": 0.03439535200595856, "nonfinite": 0.0, "weighted_memory_to_active": 0.06975153833627701, "mix_std": 0.32180696725845337, "beta_std": 0.05033539980649948, "beta_low": 0.5578919053077698, "beta_high": 0.0, "orthogonal_to_active": 0.055747684091329575, "output_influence": 0.49993377923965454, "residual_influence": 0.1272881031036377, "state_active": 0.17051902413368225, "state_memory": 0.007431264501065016, "state_peak": 23.968339920043945, "growth_active": 1.0969370603561401, "growth_memory": 1.5921884775161743, "heads": [{"head": 0, "memory_to_active": 0.008485544472932816, "contribution_to_active": 0.5733317732810974, "mix_mean": 0.6055394411087036, "mix_low": 0.0012664794921875, "mix_high": 0.007818222045898438, "read_cosine": 0.2913901209831238, "beta_mean": 0.0001200869883177802, "nonfinite": 0.0, "weighted_memory_to_active": 0.005743345711380243, "mix_std": 0.2660120725631714, "beta_std": 0.0042083594016730785, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0039648148231208324, "evidence_weights": [-0.16587233543395996, -0.3595430254936218, 0.6890513300895691], "agreement_weight": 0.23250052332878113, "orthogonal_weight": -0.7583127021789551}, {"head": 1, "memory_to_active": 0.36604106426239014, "contribution_to_active": 0.2889287769794464, "mix_mean": 0.34605908393859863, "mix_low": 0.011265754699707031, "mix_high": 0.00013446807861328125, "read_cosine": 0.6118761301040649, "beta_mean": 0.03574231266975403, "nonfinite": 0.0, "weighted_memory_to_active": 0.1554853469133377, "mix_std": 0.2583570182323456, "beta_std": 0.07034390419721603, "beta_low": 0.41485595703125, "beta_high": 0.0, "orthogonal_to_active": 0.04175449162721634, "evidence_weights": [-0.16370441019535065, -0.16022048890590668, 0.37028762698173523], "agreement_weight": 0.08086402714252472, "orthogonal_weight": -0.3581879734992981}, {"head": 2, "memory_to_active": 0.01599048264324665, "contribution_to_active": 0.9646340012550354, "mix_mean": 0.4458438456058502, "mix_low": 0.19132614135742188, "mix_high": 0.15104198455810547, "read_cosine": 0.21504420042037964, "beta_mean": 0.07902204245328903, "nonfinite": 0.0, "weighted_memory_to_active": 0.012175324372947216, "mix_std": 0.4368346929550171, "beta_std": 0.10436907410621643, "beta_low": 0.44671630859375, "beta_high": 0.0, "orthogonal_to_active": 0.06210610643029213, "evidence_weights": [0.023079218342900276, -0.44837307929992676, -1.271227478981018], "agreement_weight": 0.01088018249720335, "orthogonal_weight": 5.3142991065979}, {"head": 3, "memory_to_active": 0.012360330671072006, "contribution_to_active": 0.8135373592376709, "mix_mean": 0.4140387773513794, "mix_low": 0.2655611038208008, "mix_high": 0.10233020782470703, "read_cosine": 0.4624784588813782, "beta_mean": 0.0582343265414238, "nonfinite": 0.0, "weighted_memory_to_active": 0.007548805791884661, "mix_std": 0.4387314021587372, "beta_std": 0.0866103321313858, "beta_low": 0.4906005859375, "beta_high": 0.0, "orthogonal_to_active": 0.03416464477777481, "evidence_weights": [0.09089682996273041, -0.34557637572288513, -0.38706016540527344], "agreement_weight": 0.045044396072626114, "orthogonal_weight": 5.643202304840088}, {"head": 4, "memory_to_active": 0.42071741819381714, "contribution_to_active": 0.38966861367225647, "mix_mean": 0.46489137411117554, "mix_low": 0.01201629638671875, "mix_high": 0.000667572021484375, "read_cosine": 0.10336390137672424, "beta_mean": 0.024436689913272858, "nonfinite": 0.0, "weighted_memory_to_active": 0.1935376226902008, "mix_std": 0.2710760235786438, "beta_std": 0.023575276136398315, "beta_low": 0.27178955078125, "beta_high": 0.0, "orthogonal_to_active": 0.17145642638206482, "evidence_weights": [-0.21449686586856842, -0.1898198127746582, 1.7615292072296143], "agreement_weight": -0.1730462610721588, "orthogonal_weight": -0.8994941711425781}, {"head": 5, "memory_to_active": 0.09573933482170105, "contribution_to_active": 0.3718176484107971, "mix_mean": 0.5045204162597656, "mix_low": 0.0012531280517578125, "mix_high": 0.0022077560424804688, "read_cosine": 0.7346192598342896, "beta_mean": 0.008816647343337536, "nonfinite": 0.0, "weighted_memory_to_active": 0.04401875659823418, "mix_std": 0.25983062386512756, "beta_std": 0.01290543656796217, "beta_low": 0.7239990234375, "beta_high": 0.0, "orthogonal_to_active": 0.02103959582746029, "evidence_weights": [-0.21765722334384918, -0.2826603651046753, 0.5483363270759583], "agreement_weight": 0.07078248262405396, "orthogonal_weight": -0.6957650780677795}]}, {"layer": 12, "memory_to_active": 0.2957286238670349, "contribution_to_active": 0.5478706359863281, "mix_mean": 0.48445650935173035, "mix_low": 0.011888345703482628, "mix_high": 0.0034879050217568874, "read_cosine": 0.25688040256500244, "beta_mean": 0.01518746092915535, "nonfinite": 0.0, "weighted_memory_to_active": 0.1594248116016388, "mix_std": 0.2582288384437561, "beta_std": 0.01635226048529148, "beta_low": 0.822021484375, "beta_high": 0.0, "orthogonal_to_active": 0.06427505612373352, "output_influence": 0.43856778740882874, "residual_influence": 0.12217126786708832, "state_active": 0.04295948147773743, "state_memory": 0.009292815811932087, "state_peak": 39.04533767700195, "growth_active": 1.1977434158325195, "growth_memory": 1.21476411819458, "heads": [{"head": 0, "memory_to_active": 0.0495910719037056, "contribution_to_active": 0.49031227827072144, "mix_mean": 0.4698239862918854, "mix_low": 0.0018405914306640625, "mix_high": 0.0011739730834960938, "read_cosine": 0.3265974521636963, "beta_mean": 0.0008348715491592884, "nonfinite": 0.0, "weighted_memory_to_active": 0.022471291944384575, "mix_std": 0.2711789608001709, "beta_std": 0.0034871480893343687, "beta_low": 0.9937744140625, "beta_high": 0.0, "orthogonal_to_active": 0.015092609450221062, "evidence_weights": [-0.18136897683143616, -0.2561910152435303, 0.32003527879714966], "agreement_weight": 0.1167030856013298, "orthogonal_weight": -0.7157731652259827}, {"head": 1, "memory_to_active": 0.08660411089658737, "contribution_to_active": 0.5052918791770935, "mix_mean": 0.5233279466629028, "mix_low": 0.000766754150390625, "mix_high": 0.0006265640258789062, "read_cosine": 0.5464515089988708, "beta_mean": 0.0009718437795527279, "nonfinite": 0.0, "weighted_memory_to_active": 0.044553451240062714, "mix_std": 0.2622944414615631, "beta_std": 0.005293865688145161, "beta_low": 0.9881591796875, "beta_high": 0.0, "orthogonal_to_active": 0.034392185509204865, "evidence_weights": [-0.20616751909255981, -0.27345043420791626, 0.9230968356132507], "agreement_weight": 0.12815967202186584, "orthogonal_weight": -0.9248695969581604}, {"head": 2, "memory_to_active": 0.5899616479873657, "contribution_to_active": 0.41614028811454773, "mix_mean": 0.3033437728881836, "mix_low": 0.028720855712890625, "mix_high": 8.106231689453125e-05, "read_cosine": 0.46665287017822266, "beta_mean": 0.08632980287075043, "nonfinite": 0.0, "weighted_memory_to_active": 0.2125631421804428, "mix_std": 0.23401087522506714, "beta_std": 0.06402841210365295, "beta_low": 0.001953125, "beta_high": 0.0, "orthogonal_to_active": 0.19705766439437866, "evidence_weights": [-0.24087506532669067, -0.043970827013254166, -0.10321633517742157], "agreement_weight": 0.021426409482955933, "orthogonal_weight": 1.115958571434021}, {"head": 3, "memory_to_active": 0.027391910552978516, "contribution_to_active": 0.7134953141212463, "mix_mean": 0.6841568350791931, "mix_low": 0.00024890899658203125, "mix_high": 0.014336585998535156, "read_cosine": 0.08387970924377441, "beta_mean": 0.0002357913472224027, "nonfinite": 0.0, "weighted_memory_to_active": 0.020228121429681778, "mix_std": 0.23920734226703644, "beta_std": 0.0039838883094489574, "beta_low": 0.99847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.018811099231243134, "evidence_weights": [-0.2954132556915283, -0.33747974038124084, 1.254820704460144], "agreement_weight": 0.31246259808540344, "orthogonal_weight": -0.9492716789245605}, {"head": 4, "memory_to_active": 0.9314889907836914, "contribution_to_active": 0.7003359794616699, "mix_mean": 0.3136196732521057, "mix_low": 0.03327751159667969, "mix_high": 0.0019311904907226562, "read_cosine": -0.09516708552837372, "beta_mean": 0.0020570307970046997, "nonfinite": 0.0, "weighted_memory_to_active": 0.6020697951316833, "mix_std": 0.26805123686790466, "beta_std": 0.016360478475689888, "beta_low": 0.9649658203125, "beta_high": 0.0, "orthogonal_to_active": 0.07768618315458298, "evidence_weights": [-0.09320535510778427, -0.2101382464170456, 0.06313861906528473], "agreement_weight": -0.0018183824140578508, "orthogonal_weight": -0.1367943286895752}, {"head": 5, "memory_to_active": 0.08933397382497787, "contribution_to_active": 0.4616478681564331, "mix_mean": 0.6124667525291443, "mix_low": 0.0064754486083984375, "mix_high": 0.0027780532836914062, "read_cosine": 0.21286779642105103, "beta_mean": 0.0006954255513846874, "nonfinite": 0.0, "weighted_memory_to_active": 0.05466301366686821, "mix_std": 0.27463003993034363, "beta_std": 0.004959769546985626, "beta_low": 0.98480224609375, "beta_high": 0.0, "orthogonal_to_active": 0.04261057451367378, "evidence_weights": [-0.2536175847053528, -0.30524834990501404, 1.1476645469665527], "agreement_weight": 0.2580660283565521, "orthogonal_weight": -0.8353849053382874}]}, {"layer": 13, "memory_to_active": 0.13371355831623077, "contribution_to_active": 0.509033203125, "mix_mean": 0.5025172829627991, "mix_low": 0.006015778053551912, "mix_high": 0.004752954002469778, "read_cosine": 0.4019981622695923, "beta_mean": 0.016008755192160606, "nonfinite": 0.0, "weighted_memory_to_active": 0.0565912164747715, "mix_std": 0.25699445605278015, "beta_std": 0.01811668835580349, "beta_low": 0.7971293926239014, "beta_high": 0.0, "orthogonal_to_active": 0.057008296251297, "output_influence": 0.47059378027915955, "residual_influence": 0.15810301899909973, "state_active": 0.10144706070423126, "state_memory": 0.037748754024505615, "state_peak": 96.65151977539062, "growth_active": 0.9474308490753174, "growth_memory": 1.0258358716964722, "heads": [{"head": 0, "memory_to_active": 0.035782866179943085, "contribution_to_active": 0.5492810606956482, "mix_mean": 0.5539324283599854, "mix_low": 0.0004787445068359375, "mix_high": 0.0017070770263671875, "read_cosine": 0.4048841893672943, "beta_mean": 0.00013234137441031635, "nonfinite": 0.0, "weighted_memory_to_active": 0.021117819473147392, "mix_std": 0.268440842628479, "beta_std": 0.0003268996952101588, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.020689208060503006, "evidence_weights": [-0.20055274665355682, -0.26166388392448425, 0.7403502464294434], "agreement_weight": 0.08306962996721268, "orthogonal_weight": -1.0938546657562256}, {"head": 1, "memory_to_active": 0.018788378685712814, "contribution_to_active": 0.5724896788597107, "mix_mean": 0.5263274312019348, "mix_low": 0.0021162033081054688, "mix_high": 0.0018415451049804688, "read_cosine": 0.2237602174282074, "beta_mean": 6.414137897081673e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010430604219436646, "mix_std": 0.27041566371917725, "beta_std": 0.00011168963101226836, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011519213207066059, "evidence_weights": [-0.18570692837238312, -0.2679847478866577, 0.705024242401123], "agreement_weight": 0.29415276646614075, "orthogonal_weight": -1.1553196907043457}, {"head": 2, "memory_to_active": 0.1020599827170372, "contribution_to_active": 0.43701887130737305, "mix_mean": 0.35965120792388916, "mix_low": 0.00262451171875, "mix_high": 0.00011730194091796875, "read_cosine": 0.6297077536582947, "beta_mean": 0.01429308857768774, "nonfinite": 0.0, "weighted_memory_to_active": 0.04330043867230415, "mix_std": 0.22169071435928345, "beta_std": 0.04245098680257797, "beta_low": 0.7957763671875, "beta_high": 0.0, "orthogonal_to_active": 0.020880287513136864, "evidence_weights": [-0.20654715597629547, -0.1851615309715271, 0.43454086780548096], "agreement_weight": 0.0159923005849123, "orthogonal_weight": -0.8008779287338257}, {"head": 3, "memory_to_active": 0.1469920575618744, "contribution_to_active": 0.568977415561676, "mix_mean": 0.6153075098991394, "mix_low": 0.0191650390625, "mix_high": 0.018884658813476562, "read_cosine": -0.012526045553386211, "beta_mean": 0.0010406100191175938, "nonfinite": 0.0, "weighted_memory_to_active": 0.10538676381111145, "mix_std": 0.2877630293369293, "beta_std": 0.007082519121468067, "beta_low": 0.989501953125, "beta_high": 0.0, "orthogonal_to_active": 0.10615846514701843, "evidence_weights": [-0.33389705419540405, -0.24178411066532135, 1.978786826133728], "agreement_weight": 3.508157253265381, "orthogonal_weight": -1.0592925548553467}, {"head": 4, "memory_to_active": 0.4201925992965698, "contribution_to_active": 0.2603798508644104, "mix_mean": 0.3514726459980011, "mix_low": 0.009412765502929688, "mix_high": 1.621246337890625e-05, "read_cosine": 0.8810151815414429, "beta_mean": 0.08003056049346924, "nonfinite": 0.0, "weighted_memory_to_active": 0.10853848606348038, "mix_std": 0.21751518547534943, "beta_std": 0.055330343544483185, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.14182697236537933, "evidence_weights": [-0.1403965950012207, -0.229990616440773, 0.4929305911064148], "agreement_weight": 0.0004994847113266587, "orthogonal_weight": 2.0456185340881348}, {"head": 5, "memory_to_active": 0.07846543192863464, "contribution_to_active": 0.6660523414611816, "mix_mean": 0.6084123849868774, "mix_low": 0.0022974014282226562, "mix_high": 0.005950927734375, "read_cosine": 0.285147488117218, "beta_mean": 0.0004917897749692202, "nonfinite": 0.0, "weighted_memory_to_active": 0.05077317729592323, "mix_std": 0.27614137530326843, "beta_std": 0.0033976847771555185, "beta_low": 0.997314453125, "beta_high": 0.0, "orthogonal_to_active": 0.040975604206323624, "evidence_weights": [-0.21910840272903442, -0.2956056296825409, 0.729097843170166], "agreement_weight": 0.10213145613670349, "orthogonal_weight": -0.8969307541847229}]}, {"layer": 14, "memory_to_active": 0.25608769059181213, "contribution_to_active": 0.4190405607223511, "mix_mean": 0.41328397393226624, "mix_low": 0.025247257202863693, "mix_high": 0.004494031425565481, "read_cosine": 0.6199413537979126, "beta_mean": 0.06875219941139221, "nonfinite": 0.0, "weighted_memory_to_active": 0.11003647744655609, "mix_std": 0.24367305636405945, "beta_std": 0.06394574791193008, "beta_low": 0.4535014033317566, "beta_high": 0.0, "orthogonal_to_active": 0.04120270907878876, "output_influence": 0.3105229139328003, "residual_influence": 0.11959172785282135, "state_active": 0.06237342208623886, "state_memory": 0.016093166545033455, "state_peak": 35.78457260131836, "growth_active": 1.0985915660858154, "growth_memory": 1.273229956626892, "heads": [{"head": 0, "memory_to_active": 0.3764101564884186, "contribution_to_active": 0.1968379020690918, "mix_mean": 0.20135411620140076, "mix_low": 0.017539024353027344, "mix_high": 5.53131103515625e-05, "read_cosine": 0.7699344158172607, "beta_mean": 0.06927543133497238, "nonfinite": 0.0, "weighted_memory_to_active": 0.09238515794277191, "mix_std": 0.17953941226005554, "beta_std": 0.06621333211660385, "beta_low": 0.015869140625, "beta_high": 0.0, "orthogonal_to_active": 0.06416386365890503, "evidence_weights": [-0.13061273097991943, -0.041464779525995255, 0.5692607164382935], "agreement_weight": 0.023886164650321007, "orthogonal_weight": 1.0890294313430786}, {"head": 1, "memory_to_active": 0.4997236728668213, "contribution_to_active": 0.341144859790802, "mix_mean": 0.24097242951393127, "mix_low": 0.11677932739257812, "mix_high": 5.054473876953125e-05, "read_cosine": 0.7596461176872253, "beta_mean": 0.28499794006347656, "nonfinite": 0.0, "weighted_memory_to_active": 0.2274484634399414, "mix_std": 0.262067049741745, "beta_std": 0.25836271047592163, "beta_low": 0.09429931640625, "beta_high": 0.0, "orthogonal_to_active": 0.016197653487324715, "evidence_weights": [-0.008437876589596272, -0.24837230145931244, -0.18057632446289062], "agreement_weight": 0.052236031740903854, "orthogonal_weight": 0.14868389070034027}, {"head": 2, "memory_to_active": 0.15059606730937958, "contribution_to_active": 0.5467177629470825, "mix_mean": 0.560261607170105, "mix_low": 0.0030717849731445312, "mix_high": 0.0021419525146484375, "read_cosine": 0.41018304228782654, "beta_mean": 0.003923054784536362, "nonfinite": 0.0, "weighted_memory_to_active": 0.09097523987293243, "mix_std": 0.266435444355011, "beta_std": 0.005461430177092552, "beta_low": 0.9166259765625, "beta_high": 0.0, "orthogonal_to_active": 0.06933524459600449, "evidence_weights": [-0.1607300490140915, -0.30301693081855774, 0.9005026817321777], "agreement_weight": 0.07779107987880707, "orthogonal_weight": -0.8719256520271301}, {"head": 3, "memory_to_active": 0.06301291286945343, "contribution_to_active": 0.6520195007324219, "mix_mean": 0.6675389409065247, "mix_low": 0.0048809051513671875, "mix_high": 0.015954971313476562, "read_cosine": 0.21021747589111328, "beta_mean": 0.008187438361346722, "nonfinite": 0.0, "weighted_memory_to_active": 0.041564784944057465, "mix_std": 0.24770015478134155, "beta_std": 0.008358469232916832, "beta_low": 0.72100830078125, "beta_high": 0.0, "orthogonal_to_active": 0.0250682532787323, "evidence_weights": [-0.3733474612236023, -0.32277223467826843, 0.41585710644721985], "agreement_weight": 0.07816193997859955, "orthogonal_weight": -0.7115189433097839}, {"head": 4, "memory_to_active": 0.17421303689479828, "contribution_to_active": 0.4677186906337738, "mix_mean": 0.48090314865112305, "mix_low": 0.0045032501220703125, "mix_high": 0.007890701293945312, "read_cosine": 0.8461001515388489, "beta_mean": 0.04327351599931717, "nonfinite": 0.0, "weighted_memory_to_active": 0.09638820588588715, "mix_std": 0.2602843642234802, "beta_std": 0.03162229433655739, "beta_low": 0.0345458984375, "beta_high": 0.0, "orthogonal_to_active": 0.028752218931913376, "evidence_weights": [-0.21943360567092896, -0.25862395763397217, 0.42396894097328186], "agreement_weight": 0.12110870331525803, "orthogonal_weight": -0.5791797041893005}, {"head": 5, "memory_to_active": 0.27257025241851807, "contribution_to_active": 0.3098045289516449, "mix_mean": 0.32867342233657837, "mix_low": 0.0047092437744140625, "mix_high": 0.0008707046508789062, "read_cosine": 0.7235666513442993, "beta_mean": 0.0028557630721479654, "nonfinite": 0.0, "weighted_memory_to_active": 0.1114569827914238, "mix_std": 0.2460118532180786, "beta_std": 0.013656255789101124, "beta_low": 0.93865966796875, "beta_high": 0.0, "orthogonal_to_active": 0.0436989963054657, "evidence_weights": [-0.1804465502500534, -0.15722118318080902, 0.21925996243953705], "agreement_weight": 0.08464089035987854, "orthogonal_weight": -0.6384661793708801}]}, {"layer": 15, "memory_to_active": 0.06398060917854309, "contribution_to_active": 0.48088711500167847, "mix_mean": 0.40320104360580444, "mix_low": 0.009754657745361328, "mix_high": 0.003019491909071803, "read_cosine": 0.6620875597000122, "beta_mean": 0.03957489877939224, "nonfinite": 0.0, "weighted_memory_to_active": 0.028931209817528725, "mix_std": 0.2821708917617798, "beta_std": 0.0452067032456398, "beta_low": 0.7069600820541382, "beta_high": 0.0, "orthogonal_to_active": 0.00672914320603013, "output_influence": 0.30221113562583923, "residual_influence": 0.16077084839344025, "state_active": 0.12684476375579834, "state_memory": 0.009693148545920849, "state_peak": 40.685420989990234, "growth_active": 1.311310052871704, "growth_memory": 1.6084504127502441, "heads": [{"head": 0, "memory_to_active": 0.1074441522359848, "contribution_to_active": 0.4660654067993164, "mix_mean": 0.3653809428215027, "mix_low": 0.0055065155029296875, "mix_high": 0.004669189453125, "read_cosine": 0.6696178913116455, "beta_mean": 0.03895671293139458, "nonfinite": 0.0, "weighted_memory_to_active": 0.048439495265483856, "mix_std": 0.27338123321533203, "beta_std": 0.029462188482284546, "beta_low": 0.11138916015625, "beta_high": 0.0, "orthogonal_to_active": 0.021282508969306946, "evidence_weights": [-0.16405634582042694, -0.17462773621082306, 0.3501396179199219], "agreement_weight": 0.09061405062675476, "orthogonal_weight": -0.6802234649658203}, {"head": 1, "memory_to_active": 0.15774840116500854, "contribution_to_active": 0.4050908088684082, "mix_mean": 0.40886420011520386, "mix_low": 0.0031385421752929688, "mix_high": 0.0035877227783203125, "read_cosine": 0.8942963480949402, "beta_mean": 0.0014304171781986952, "nonfinite": 0.0, "weighted_memory_to_active": 0.06328476965427399, "mix_std": 0.28028520941734314, "beta_std": 0.00368497334420681, "beta_low": 0.9732666015625, "beta_high": 0.0, "orthogonal_to_active": 0.0014349883422255516, "evidence_weights": [-0.17044764757156372, -0.17945288121700287, 0.2500573396682739], "agreement_weight": 0.10511724650859833, "orthogonal_weight": -0.03904474899172783}, {"head": 2, "memory_to_active": 0.009025869891047478, "contribution_to_active": 0.4756799340248108, "mix_mean": 0.3505404591560364, "mix_low": 0.017444610595703125, "mix_high": 0.00011920928955078125, "read_cosine": 0.8070852756500244, "beta_mean": 0.00014210521476343274, "nonfinite": 0.0, "weighted_memory_to_active": 0.004369028843939304, "mix_std": 0.2750232219696045, "beta_std": 0.00028751237550750375, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0004920616047456861, "evidence_weights": [-0.20723514258861542, -0.28101110458374023, 0.2504276931285858], "agreement_weight": 0.010209064930677414, "orthogonal_weight": 0.25738558173179626}, {"head": 3, "memory_to_active": 0.015035007148981094, "contribution_to_active": 0.42068910598754883, "mix_mean": 0.36184799671173096, "mix_low": 0.0061435699462890625, "mix_high": 8.296966552734375e-05, "read_cosine": 0.6461567878723145, "beta_mean": 0.0001155692370957695, "nonfinite": 0.0, "weighted_memory_to_active": 0.00596263725310564, "mix_std": 0.27550768852233887, "beta_std": 0.00031782890437170863, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004490307532250881, "evidence_weights": [-0.2155180275440216, -0.25570639967918396, 0.3482014536857605], "agreement_weight": 0.017240235581994057, "orthogonal_weight": -0.949227511882782}, {"head": 4, "memory_to_active": 0.07360581308603287, "contribution_to_active": 0.5981084704399109, "mix_mean": 0.3738681674003601, "mix_low": 0.016038894653320312, "mix_high": 0.0005474090576171875, "read_cosine": 0.5796619653701782, "beta_mean": 0.19543324410915375, "nonfinite": 0.0, "weighted_memory_to_active": 0.03743409737944603, "mix_std": 0.3070785105228424, "beta_std": 0.22692467272281647, "beta_low": 0.18115234375, "beta_high": 0.0, "orthogonal_to_active": 0.004854615777730942, "evidence_weights": [-0.21546770632266998, -0.25642645359039307, 0.022804569453001022], "agreement_weight": 0.012446051463484764, "orthogonal_weight": 0.21185626089572906}, {"head": 5, "memory_to_active": 0.021024426445364952, "contribution_to_active": 0.5196889638900757, "mix_mean": 0.5587045550346375, "mix_low": 0.010255813598632812, "mix_high": 0.009110450744628906, "read_cosine": 0.3757069706916809, "beta_mean": 0.001371357124298811, "nonfinite": 0.0, "weighted_memory_to_active": 0.014097224920988083, "mix_std": 0.28174927830696106, "beta_std": 0.010563028044998646, "beta_low": 0.97607421875, "beta_high": 0.0, "orthogonal_to_active": 0.007820375263690948, "evidence_weights": [-0.3325823247432709, -0.31197020411491394, 0.7309510707855225], "agreement_weight": 0.24729369580745697, "orthogonal_weight": 0.715247631072998}]}]}, {"step": 32000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8100512650007659, "memory_to_active": 0.8713201284408569, "contribution_to_active": 0.5218620896339417, "mix_mean": 0.36897847056388855, "mix_low": 0.15099643170833588, "mix_high": 0.008291443809866905, "read_cosine": 0.3744574785232544, "beta_mean": 0.082651786506176, "nonfinite": 0.0, "weighted_memory_to_active": 0.21698473393917084, "mix_std": 0.21747490763664246, "beta_std": 0.06924222409725189, "beta_low": 0.5013326406478882, "beta_high": 2.09808349609375e-05, "orthogonal_to_active": 0.0531294010579586, "samples": 4, "scored_tokens": 899, "normal_loss": 3.140662908554077, "normal_token_accuracy": 0.4338153600692749, "state_active": 0.06009383127093315, "state_memory": 0.019306723028421402, "state_peak": 114.2680435180664, "growth_active": 1.121164083480835, "growth_memory": 1.4092209339141846, "output_influence": 0.3740692734718323, "residual_influence": 0.12127171456813812, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2731802463531494, "no_memory_loss_delta": 0.4234900176525116, "scrambled_loss_delta": 0.10261118412017822, "memory_only_loss_delta": 2.9411892890930176, "no_evidence_loss_delta": 0.26431944966316223, "no_agreement_loss_delta": 0.019274674355983734, "no_orthogonal_loss_delta": 0.00110679492354393}, {"index": 94, "tokens": 256, "normal_loss": 3.2382588386535645, "no_memory_loss_delta": 0.47826483845710754, "scrambled_loss_delta": 0.3660105764865875, "memory_only_loss_delta": 3.263322353363037, "no_evidence_loss_delta": 0.23443962633609772, "no_agreement_loss_delta": 0.035538531839847565, "no_orthogonal_loss_delta": 0.024458639323711395}, {"index": 188, "tokens": 256, "normal_loss": 3.465792417526245, "no_memory_loss_delta": 0.523689329624176, "scrambled_loss_delta": 0.13964766263961792, "memory_only_loss_delta": 2.6775667667388916, "no_evidence_loss_delta": 0.29925185441970825, "no_agreement_loss_delta": 0.04001486301422119, "no_orthogonal_loss_delta": 0.021593336015939713}, {"index": 283, "tokens": 131, "normal_loss": 2.055607318878174, "no_memory_loss_delta": 0.23698559403419495, "scrambled_loss_delta": 0.21367894113063812, "memory_only_loss_delta": 2.969529867172241, "no_evidence_loss_delta": 0.11757258325815201, "no_agreement_loss_delta": 0.013016462326049805, "no_orthogonal_loss_delta": -0.02242603898048401}], "agreement_weight_abs_mean": 0.15026338398456573, "agreement_weight_abs_max": 3.6314213275909424, "agreement_weight_mean": 0.13045454025268555, "orthogonal_weight_abs_mean": 0.6822723746299744, "orthogonal_weight_abs_max": 5.827418804168701, "orthogonal_weight_mean": -0.1534673422574997, "no_memory_loss": 3.581106185913086, "no_memory_loss_delta": 0.44044363498687744, "no_memory_token_accuracy_gain": 0.07007786631584167, "no_memory_token_win_fraction": 0.6585094332695007, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 35.62492326383969, "scrambled_loss": 3.345010995864868, "scrambled_loss_delta": 0.20434805750846863, "scrambled_token_accuracy_gain": 0.02002224698662758, "scrambled_token_win_fraction": 0.628476083278656, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.48214072254042, "memory_only_loss": 6.102643013000488, "memory_only_loss_delta": 2.9619805812835693, "memory_only_token_accuracy_gain": 0.26140156388282776, "memory_only_token_win_fraction": 0.9121245741844177, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.82836128367363, "no_evidence_loss": 3.385037660598755, "no_evidence_loss_delta": 0.24437464773654938, "no_evidence_token_accuracy_gain": 0.03337041288614273, "no_evidence_token_win_fraction": 0.5951056480407715, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 21.680584262726942, "no_agreement_loss": 3.169562816619873, "no_agreement_loss_delta": 0.02890004962682724, "no_agreement_token_accuracy_gain": 0.01001112349331379, "no_agreement_token_win_fraction": 0.5061178803443909, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.8486437242841225, "no_orthogonal_loss": 3.1508235931396484, "no_orthogonal_loss_delta": 0.010161105543375015, "no_orthogonal_token_accuracy_gain": 0.0033370412420481443, "no_orthogonal_token_win_fraction": 0.5717464089393616, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.0109655919586613, "short_no_memory_loss_delta": 0.3832646608352661, "long_context_memory_gain": 0.05717897415161133, "normal_context_loss_gain": 0.1284099966287613, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.2773468494415283, "contribution_to_active": 1.0728825330734253, "mix_mean": 0.1004781424999237, "mix_low": 0.5371820330619812, "mix_high": 0.004457950592041016, "read_cosine": 0.4469040334224701, "beta_mean": 0.2568697929382324, "nonfinite": 0.0, "weighted_memory_to_active": 1.142200231552124, "mix_std": 0.08786539733409882, "beta_std": 0.14585717022418976, "beta_low": 0.0222574882209301, "beta_high": 0.0, "orthogonal_to_active": 0.06321736425161362, "output_influence": 0.1777205616235733, "residual_influence": 0.21018406748771667, "state_active": 0.07428819686174393, "state_memory": 0.047596368938684464, "state_peak": 46.19343185424805, "growth_active": 1.3076322078704834, "growth_memory": 1.2450472116470337, "heads": [{"head": 0, "memory_to_active": 2.399505138397217, "contribution_to_active": 0.4237820506095886, "mix_mean": 0.018209183588624, "mix_low": 0.6266622543334961, "mix_high": 0.0, "read_cosine": 0.2579977512359619, "beta_mean": 0.20958147943019867, "nonfinite": 0.0, "weighted_memory_to_active": 0.5135610103607178, "mix_std": 0.056457843631505966, "beta_std": 0.101547010242939, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.038769979029893875, "evidence_weights": [-0.024460548534989357, -0.0729905441403389, -0.11104721575975418], "agreement_weight": -0.10187026113271713, "orthogonal_weight": -0.084153912961483}, {"head": 1, "memory_to_active": 5.5706071853637695, "contribution_to_active": 2.356466293334961, "mix_mean": 0.016368350014090538, "mix_low": 0.6569719314575195, "mix_high": 0.0, "read_cosine": 0.5103495121002197, "beta_mean": 0.4610741436481476, "nonfinite": 0.0, "weighted_memory_to_active": 2.749549627304077, "mix_std": 0.05359238013625145, "beta_std": 0.19899821281433105, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11721288412809372, "evidence_weights": [0.0236025620251894, -0.09703002870082855, 0.3513184189796448], "agreement_weight": 0.02891167439520359, "orthogonal_weight": -0.09506487101316452}, {"head": 2, "memory_to_active": 2.8540632724761963, "contribution_to_active": 0.5818848013877869, "mix_mean": 0.015342256985604763, "mix_low": 0.6324405670166016, "mix_high": 0.0, "read_cosine": 0.3071601390838623, "beta_mean": 0.31095150113105774, "nonfinite": 0.0, "weighted_memory_to_active": 0.6830339431762695, "mix_std": 0.03409209102392197, "beta_std": 0.2021210491657257, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06315258145332336, "evidence_weights": [0.05077733099460602, 0.06487122178077698, -0.01281601469963789], "agreement_weight": 0.005782846827059984, "orthogonal_weight": -0.19235959649085999}, {"head": 3, "memory_to_active": 6.134382247924805, "contribution_to_active": 2.6260948181152344, "mix_mean": 0.03785415366292, "mix_low": 0.46079254150390625, "mix_high": 0.0, "read_cosine": 0.34502825140953064, "beta_mean": 0.3548009991645813, "nonfinite": 0.0, "weighted_memory_to_active": 2.809009075164795, "mix_std": 0.08592630922794342, "beta_std": 0.2286226898431778, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.13770164549350739, "evidence_weights": [-0.054815199226140976, -0.0738922655582428, -0.12351780384778976], "agreement_weight": -0.014037431217730045, "orthogonal_weight": -0.05810008943080902}, {"head": 4, "memory_to_active": 2.601532459259033, "contribution_to_active": 0.058274008333683014, "mix_mean": 0.0060983095318078995, "mix_low": 0.8304424285888672, "mix_high": 0.0, "read_cosine": 0.3364371359348297, "beta_mean": 0.16443100571632385, "nonfinite": 0.0, "weighted_memory_to_active": 0.04896688833832741, "mix_std": 0.012000992894172668, "beta_std": 0.11337525397539139, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0047049978747963905, "evidence_weights": [0.12410233914852142, 0.0739927664399147, -0.3729493319988251], "agreement_weight": -0.11608314514160156, "orthogonal_weight": 0.09721039980649948}, {"head": 5, "memory_to_active": 0.10399043560028076, "contribution_to_active": 0.390793114900589, "mix_mean": 0.5089966058731079, "mix_low": 0.01578235626220703, "mix_high": 0.026747703552246094, "read_cosine": 0.9244513511657715, "beta_mean": 0.04037970304489136, "nonfinite": 0.0, "weighted_memory_to_active": 0.049080848693847656, "mix_std": 0.285122811794281, "beta_std": 0.030478784814476967, "beta_low": 0.133544921875, "beta_high": 0.0, "orthogonal_to_active": 0.017762087285518646, "evidence_weights": [-0.24038586020469666, -0.2432539314031601, 0.2901577949523926], "agreement_weight": 0.24855491518974304, "orthogonal_weight": -0.650486171245575}]}, {"layer": 1, "memory_to_active": 2.307874917984009, "contribution_to_active": 0.532371997833252, "mix_mean": 0.18033798038959503, "mix_low": 0.360066294670105, "mix_high": 0.0011382104130461812, "read_cosine": 0.3216981291770935, "beta_mean": 0.19276930391788483, "nonfinite": 0.0, "weighted_memory_to_active": 0.409559428691864, "mix_std": 0.12151674181222916, "beta_std": 0.13410684466362, "beta_low": 0.2338460385799408, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.055069372057914734, "output_influence": 0.18490973114967346, "residual_influence": 0.04460311308503151, "state_active": 0.03673459589481354, "state_memory": 0.03718439117074013, "state_peak": 30.96052360534668, "growth_active": 1.2077925205230713, "growth_memory": 1.5060838460922241, "heads": [{"head": 0, "memory_to_active": 0.07168148458003998, "contribution_to_active": 0.4313644766807556, "mix_mean": 0.42716389894485474, "mix_low": 0.010223388671875, "mix_high": 0.00047588348388671875, "read_cosine": 0.024996694177389145, "beta_mean": 0.0004730956570710987, "nonfinite": 0.0, "weighted_memory_to_active": 0.04008816182613373, "mix_std": 0.2829812169075012, "beta_std": 0.0007118447683751583, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.03971434012055397, "evidence_weights": [-0.18439975380897522, -0.23766173422336578, 0.04643338918685913], "agreement_weight": 0.16030994057655334, "orthogonal_weight": -1.0551725625991821}, {"head": 1, "memory_to_active": 2.77173113822937, "contribution_to_active": 0.2533525824546814, "mix_mean": 0.022705651819705963, "mix_low": 0.4821968078613281, "mix_high": 9.5367431640625e-06, "read_cosine": 0.4888553023338318, "beta_mean": 0.2967328131198883, "nonfinite": 0.0, "weighted_memory_to_active": 0.2904742360115051, "mix_std": 0.0498056560754776, "beta_std": 0.19919654726982117, "beta_low": 0.000732421875, "beta_high": 0.000244140625, "orthogonal_to_active": 0.012005340307950974, "evidence_weights": [0.013546108268201351, -0.019855951890349388, -0.02321261167526245], "agreement_weight": -0.00536653446033597, "orthogonal_weight": 0.10846520960330963}, {"head": 2, "memory_to_active": 0.33347439765930176, "contribution_to_active": 0.5429497957229614, "mix_mean": 0.5660699605941772, "mix_low": 0.0019140243530273438, "mix_high": 0.0063323974609375, "read_cosine": 0.7097347378730774, "beta_mean": 0.021730434149503708, "nonfinite": 0.0, "weighted_memory_to_active": 0.2663730978965759, "mix_std": 0.25604695081710815, "beta_std": 0.019885467365384102, "beta_low": 0.3553466796875, "beta_high": 0.0, "orthogonal_to_active": 0.05152829736471176, "evidence_weights": [-0.28870347142219543, -0.3646925985813141, 0.3626817762851715], "agreement_weight": 0.027971884235739708, "orthogonal_weight": -0.33560776710510254}, {"head": 3, "memory_to_active": 2.96791934967041, "contribution_to_active": 0.14646804332733154, "mix_mean": 0.018965914845466614, "mix_low": 0.6311101913452148, "mix_high": 0.0, "read_cosine": 0.23602688312530518, "beta_mean": 0.3488563001155853, "nonfinite": 0.0, "weighted_memory_to_active": 0.2241591364145279, "mix_std": 0.049892857670784, "beta_std": 0.254693865776062, "beta_low": 0.0294189453125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0652027577161789, "evidence_weights": [0.03882358595728874, -0.005506971385329962, 0.14030225574970245], "agreement_weight": -0.023525483906269073, "orthogonal_weight": -0.34435418248176575}, {"head": 4, "memory_to_active": 3.6382486820220947, "contribution_to_active": 1.739724040031433, "mix_mean": 0.03747740387916565, "mix_low": 0.2696390151977539, "mix_high": 5.7220458984375e-06, "read_cosine": 0.28938233852386475, "beta_mean": 0.17760883271694183, "nonfinite": 0.0, "weighted_memory_to_active": 1.5686089992523193, "mix_std": 0.06690452992916107, "beta_std": 0.11404009908437729, "beta_low": 0.01123046875, "beta_high": 0.0, "orthogonal_to_active": 0.15433908998966217, "evidence_weights": [-0.04220763221383095, 0.008582343347370625, 0.009774637408554554], "agreement_weight": -0.014970422722399235, "orthogonal_weight": 0.1353880763053894}, {"head": 5, "memory_to_active": 4.0641937255859375, "contribution_to_active": 0.08037284016609192, "mix_mean": 0.00964493677020073, "mix_low": 0.7653141021728516, "mix_high": 5.7220458984375e-06, "read_cosine": 0.1811928153038025, "beta_mean": 0.3112143874168396, "nonfinite": 0.0, "weighted_memory_to_active": 0.06765295565128326, "mix_std": 0.023469163104891777, "beta_std": 0.2161131203174591, "beta_low": 0.007080078125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.00762637984007597, "evidence_weights": [0.1457492709159851, 0.06540016829967499, 0.2945375442504883], "agreement_weight": -0.14797191321849823, "orthogonal_weight": 0.1121891513466835}]}, {"layer": 2, "memory_to_active": 1.0560688972473145, "contribution_to_active": 0.3463256359100342, "mix_mean": 0.347891628742218, "mix_low": 0.25252994894981384, "mix_high": 0.002715428825467825, "read_cosine": 0.32996147871017456, "beta_mean": 0.09552083164453506, "nonfinite": 0.0, "weighted_memory_to_active": 0.07881051301956177, "mix_std": 0.19359618425369263, "beta_std": 0.07227778434753418, "beta_low": 0.5412394404411316, "beta_high": 0.00017293295240961015, "orthogonal_to_active": 0.017995024099946022, "output_influence": 0.3319470286369324, "residual_influence": 0.08238251507282257, "state_active": 0.03558697924017906, "state_memory": 0.018845032900571823, "state_peak": 34.38996124267578, "growth_active": 0.9887022972106934, "growth_memory": 1.5836089849472046, "heads": [{"head": 0, "memory_to_active": 0.009476165287196636, "contribution_to_active": 0.3457455039024353, "mix_mean": 0.5250004529953003, "mix_low": 0.0021514892578125, "mix_high": 0.0038194656372070312, "read_cosine": 0.17807315289974213, "beta_mean": 6.561195914400741e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0055418168194592, "mix_std": 0.2888718545436859, "beta_std": 0.00010477246541995555, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017675842391327024, "evidence_weights": [-0.3105649948120117, -0.19364207983016968, 0.11831436306238174], "agreement_weight": 0.20653888583183289, "orthogonal_weight": -0.33777347207069397}, {"head": 1, "memory_to_active": 2.909886598587036, "contribution_to_active": 0.053770847618579865, "mix_mean": 0.005318986251950264, "mix_low": 0.8835353851318359, "mix_high": 9.5367431640625e-07, "read_cosine": 0.2070736289024353, "beta_mean": 0.26843124628067017, "nonfinite": 0.0, "weighted_memory_to_active": 0.040979936718940735, "mix_std": 0.012664476409554482, "beta_std": 0.14971621334552765, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.01625966653227806, "evidence_weights": [0.10328488051891327, 0.10156666487455368, 0.49275392293930054], "agreement_weight": 0.04136298596858978, "orthogonal_weight": 0.4326480031013489}, {"head": 2, "memory_to_active": 0.01453799195587635, "contribution_to_active": 0.4269954264163971, "mix_mean": 0.47325795888900757, "mix_low": 0.0021648406982421875, "mix_high": 0.001922607421875, "read_cosine": 0.27499547600746155, "beta_mean": 3.309344901936129e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005302996374666691, "mix_std": 0.2788333594799042, "beta_std": 5.3460244089365005e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012194873997941613, "evidence_weights": [-0.2306041419506073, -0.27070602774620056, 0.4808177649974823], "agreement_weight": 0.02774626389145851, "orthogonal_weight": -0.25279295444488525}, {"head": 3, "memory_to_active": 0.20439648628234863, "contribution_to_active": 0.48289310932159424, "mix_mean": 0.4758959412574768, "mix_low": 0.0006961822509765625, "mix_high": 0.004634857177734375, "read_cosine": 0.6803973913192749, "beta_mean": 0.042679715901613235, "nonfinite": 0.0, "weighted_memory_to_active": 0.11534103751182556, "mix_std": 0.2661762833595276, "beta_std": 0.05409594252705574, "beta_low": 0.22955322265625, "beta_high": 0.0, "orthogonal_to_active": 0.07005373388528824, "evidence_weights": [-0.1744900941848755, -0.2723720371723175, 0.8648626804351807], "agreement_weight": 0.062314171344041824, "orthogonal_weight": -0.8781716823577881}, {"head": 4, "memory_to_active": 3.181898355484009, "contribution_to_active": 0.25696495175361633, "mix_mean": 0.016881147399544716, "mix_low": 0.6248064041137695, "mix_high": 0.0, "read_cosine": 0.19494014978408813, "beta_mean": 0.2618616223335266, "nonfinite": 0.0, "weighted_memory_to_active": 0.2954798936843872, "mix_std": 0.041252244263887405, "beta_std": 0.22963322699069977, "beta_low": 0.01776123046875, "beta_high": 0.00103759765625, "orthogonal_to_active": 0.01821610890328884, "evidence_weights": [0.02543056756258011, 0.016173988580703735, 0.06849010288715363], "agreement_weight": 0.022500837221741676, "orthogonal_weight": -0.0670793280005455}, {"head": 5, "memory_to_active": 0.016217801719903946, "contribution_to_active": 0.5115838646888733, "mix_mean": 0.5909952521324158, "mix_low": 0.0018253326416015625, "mix_high": 0.0059146881103515625, "read_cosine": 0.444288969039917, "beta_mean": 5.369212885852903e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010217374190688133, "mix_std": 0.2737788259983063, "beta_std": 6.305710121523589e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004535562184173614, "evidence_weights": [-0.26233798265457153, -0.2827654480934143, 0.9001054167747498], "agreement_weight": 0.0916774719953537, "orthogonal_weight": 0.06716108322143555}]}, {"layer": 3, "memory_to_active": 0.8588494658470154, "contribution_to_active": 0.3950037956237793, "mix_mean": 0.34515345096588135, "mix_low": 0.16354291141033173, "mix_high": 0.006918272003531456, "read_cosine": 0.2343493551015854, "beta_mean": 0.05347050726413727, "nonfinite": 0.0, "weighted_memory_to_active": 0.160854771733284, "mix_std": 0.23391684889793396, "beta_std": 0.04923687502741814, "beta_low": 0.4869486689567566, "beta_high": 0.0, "orthogonal_to_active": 0.06737382709980011, "output_influence": 0.3194189667701721, "residual_influence": 0.06446437537670135, "state_active": 0.021005813032388687, "state_memory": 0.011835797689855099, "state_peak": 19.287410736083984, "growth_active": 0.958139181137085, "growth_memory": 2.0269811153411865, "heads": [{"head": 0, "memory_to_active": 0.4648132920265198, "contribution_to_active": 0.5021193027496338, "mix_mean": 0.4616469442844391, "mix_low": 0.01665496826171875, "mix_high": 0.011889457702636719, "read_cosine": 0.27105841040611267, "beta_mean": 0.014453022740781307, "nonfinite": 0.0, "weighted_memory_to_active": 0.2501583397388458, "mix_std": 0.32438716292381287, "beta_std": 0.022645272314548492, "beta_low": 0.5526123046875, "beta_high": 0.0, "orthogonal_to_active": 0.16197670996189117, "evidence_weights": [-0.1808498054742813, -0.215243399143219, 0.6714277267456055], "agreement_weight": 0.12340638041496277, "orthogonal_weight": -0.7045050263404846}, {"head": 1, "memory_to_active": 0.016552360728383064, "contribution_to_active": 0.36630478501319885, "mix_mean": 0.43976664543151855, "mix_low": 0.014798164367675781, "mix_high": 0.003951072692871094, "read_cosine": 0.1854686439037323, "beta_mean": 8.8251537817996e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008204828947782516, "mix_std": 0.30801457166671753, "beta_std": 0.00016608306032139808, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004349710885435343, "evidence_weights": [-0.21464933454990387, -0.18580619990825653, 0.5665647387504578], "agreement_weight": 0.3418395519256592, "orthogonal_weight": -0.5800386667251587}, {"head": 2, "memory_to_active": 2.530771255493164, "contribution_to_active": 0.3040858209133148, "mix_mean": 0.04477132856845856, "mix_low": 0.38988304138183594, "mix_high": 0.0, "read_cosine": 0.208520770072937, "beta_mean": 0.20161530375480652, "nonfinite": 0.0, "weighted_memory_to_active": 0.36589252948760986, "mix_std": 0.10022882372140884, "beta_std": 0.15559041500091553, "beta_low": 0.011962890625, "beta_high": 0.0, "orthogonal_to_active": 0.003457203507423401, "evidence_weights": [-0.06678644567728043, 0.019230103120207787, -0.01890472136437893], "agreement_weight": -0.014062520116567612, "orthogonal_weight": -0.01241585798561573}, {"head": 3, "memory_to_active": 0.03339897468686104, "contribution_to_active": 0.49318790435791016, "mix_mean": 0.5298567414283752, "mix_low": 0.012715339660644531, "mix_high": 0.011171340942382812, "read_cosine": 0.3107103407382965, "beta_mean": 0.0001262082951143384, "nonfinite": 0.0, "weighted_memory_to_active": 0.021372992545366287, "mix_std": 0.3064928650856018, "beta_std": 0.00033613451523706317, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.005899674259126186, "evidence_weights": [-0.21125835180282593, -0.2229238897562027, 1.9132543802261353], "agreement_weight": 1.3687297105789185, "orthogonal_weight": -0.3648470640182495}, {"head": 4, "memory_to_active": 1.8114662170410156, "contribution_to_active": 0.04708683118224144, "mix_mean": 0.029059909284114838, "mix_low": 0.5347137451171875, "mix_high": 0.0, "read_cosine": 0.1618901491165161, "beta_mean": 0.06888501346111298, "nonfinite": 0.0, "weighted_memory_to_active": 0.11641725897789001, "mix_std": 0.06663814187049866, "beta_std": 0.08230962604284286, "beta_low": 0.151611328125, "beta_high": 0.0, "orthogonal_to_active": 0.08577319979667664, "evidence_weights": [0.0947939082980156, 0.036884088069200516, 0.1621062457561493], "agreement_weight": -0.016150005161762238, "orthogonal_weight": -0.7623319625854492}, {"head": 5, "memory_to_active": 0.29609453678131104, "contribution_to_active": 0.6572380065917969, "mix_mean": 0.5658189654350281, "mix_low": 0.012492179870605469, "mix_high": 0.014497756958007812, "read_cosine": 0.26844775676727295, "beta_mean": 0.03565521538257599, "nonfinite": 0.0, "weighted_memory_to_active": 0.20308265089988708, "mix_std": 0.2977394163608551, "beta_std": 0.034373689442873, "beta_low": 0.20562744140625, "beta_high": 0.0, "orthogonal_to_active": 0.14278647303581238, "evidence_weights": [-0.22051431238651276, -0.2559042274951935, 0.4696987569332123], "agreement_weight": 0.13710252940654755, "orthogonal_weight": -0.7938485145568848}]}, {"layer": 4, "memory_to_active": 1.1760371923446655, "contribution_to_active": 0.42393165826797485, "mix_mean": 0.3433825373649597, "mix_low": 0.2776462435722351, "mix_high": 0.006369750015437603, "read_cosine": 0.36000558733940125, "beta_mean": 0.12979884445667267, "nonfinite": 0.0, "weighted_memory_to_active": 0.13932117819786072, "mix_std": 0.1566539853811264, "beta_std": 0.11722645163536072, "beta_low": 0.3690084218978882, "beta_high": 0.0, "orthogonal_to_active": 0.03895474597811699, "output_influence": 0.38139045238494873, "residual_influence": 0.10157568752765656, "state_active": 0.03181954473257065, "state_memory": 0.01973620243370533, "state_peak": 34.092132568359375, "growth_active": 1.108121395111084, "growth_memory": 1.6181782484054565, "heads": [{"head": 0, "memory_to_active": 0.24810202419757843, "contribution_to_active": 0.7047112584114075, "mix_mean": 0.6421666145324707, "mix_low": 0.0120391845703125, "mix_high": 0.015460968017578125, "read_cosine": 0.4963928163051605, "beta_mean": 0.2582743763923645, "nonfinite": 0.0, "weighted_memory_to_active": 0.19471338391304016, "mix_std": 0.3412204384803772, "beta_std": 0.2654288113117218, "beta_low": 0.20892333984375, "beta_high": 0.0, "orthogonal_to_active": 0.059910137206315994, "evidence_weights": [-0.26411986351013184, -0.3614179790019989, 0.25220033526420593], "agreement_weight": 0.012071584351360798, "orthogonal_weight": 0.38849812746047974}, {"head": 1, "memory_to_active": 0.08543945848941803, "contribution_to_active": 0.5963019728660583, "mix_mean": 0.6339828372001648, "mix_low": 5.626678466796875e-05, "mix_high": 0.007659912109375, "read_cosine": 0.7068078517913818, "beta_mean": 0.0015793025959283113, "nonfinite": 0.0, "weighted_memory_to_active": 0.058134209364652634, "mix_std": 0.2511780261993408, "beta_std": 0.006572366692125797, "beta_low": 0.97210693359375, "beta_high": 0.0, "orthogonal_to_active": 0.03214102238416672, "evidence_weights": [-0.23768529295921326, -0.31495732069015503, 0.731734573841095], "agreement_weight": 0.12413497269153595, "orthogonal_weight": -0.8336590528488159}, {"head": 2, "memory_to_active": 2.365510940551758, "contribution_to_active": 0.17676480114459991, "mix_mean": 0.01690434291958809, "mix_low": 0.6067876815795898, "mix_high": 9.5367431640625e-07, "read_cosine": 0.24193963408470154, "beta_mean": 0.1281585693359375, "nonfinite": 0.0, "weighted_memory_to_active": 0.19803044199943542, "mix_std": 0.03408407047390938, "beta_std": 0.09961915016174316, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.01449163444340229, "evidence_weights": [0.0599806122481823, 0.03695708140730858, -0.018224777653813362], "agreement_weight": 0.12075106799602509, "orthogonal_weight": -0.10804646462202072}, {"head": 3, "memory_to_active": 1.9721611738204956, "contribution_to_active": 0.14484678208827972, "mix_mean": 0.030965842306613922, "mix_low": 0.3968667984008789, "mix_high": 0.0, "read_cosine": 0.19555501639842987, "beta_mean": 0.15519215166568756, "nonfinite": 0.0, "weighted_memory_to_active": 0.2070544958114624, "mix_std": 0.053113289177417755, "beta_std": 0.14201632142066956, "beta_low": 0.03228759765625, "beta_high": 0.0, "orthogonal_to_active": 0.01253964938223362, "evidence_weights": [-0.024038124829530716, 0.05504358932375908, -0.2331545650959015], "agreement_weight": 0.07104500383138657, "orthogonal_weight": -0.08002286404371262}, {"head": 4, "memory_to_active": 0.020220180973410606, "contribution_to_active": 0.645717442035675, "mix_mean": 0.7221217751502991, "mix_low": 0.0002956390380859375, "mix_high": 0.015089988708496094, "read_cosine": 0.19270913302898407, "beta_mean": 3.978304084739648e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01522008515894413, "mix_std": 0.22675640881061554, "beta_std": 0.00011957053357036784, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030734981410205364, "evidence_weights": [-0.3153953552246094, -0.33525440096855164, 0.745142936706543], "agreement_weight": 0.2401498705148697, "orthogonal_weight": -0.22067929804325104}, {"head": 5, "memory_to_active": 2.3647894859313965, "contribution_to_active": 0.275247722864151, "mix_mean": 0.014153750613331795, "mix_low": 0.6498317718505859, "mix_high": 6.67572021484375e-06, "read_cosine": 0.3266289234161377, "beta_mean": 0.23554885387420654, "nonfinite": 0.0, "weighted_memory_to_active": 0.16277439892292023, "mix_std": 0.03357171267271042, "beta_std": 0.18960246443748474, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1115725189447403, "evidence_weights": [0.04047176614403725, -0.04101258143782616, -0.24846580624580383], "agreement_weight": -0.06487032771110535, "orthogonal_weight": 0.823254406452179}]}, {"layer": 5, "memory_to_active": 0.514491081237793, "contribution_to_active": 0.4855443239212036, "mix_mean": 0.4449857473373413, "mix_low": 0.07487520575523376, "mix_high": 0.014215787872672081, "read_cosine": 0.16189610958099365, "beta_mean": 0.05879949405789375, "nonfinite": 0.0, "weighted_memory_to_active": 0.14976412057876587, "mix_std": 0.2525472342967987, "beta_std": 0.0779772400856018, "beta_low": 0.632965087890625, "beta_high": 0.0, "orthogonal_to_active": 0.05344952642917633, "output_influence": 0.4839426875114441, "residual_influence": 0.1247328668832779, "state_active": 0.04478006809949875, "state_memory": 0.01210981048643589, "state_peak": 59.34669876098633, "growth_active": 1.114833116531372, "growth_memory": 1.310795545578003, "heads": [{"head": 0, "memory_to_active": 0.655413031578064, "contribution_to_active": 0.3771638870239258, "mix_mean": 0.2272459715604782, "mix_low": 0.07032489776611328, "mix_high": 0.00022125244140625, "read_cosine": -0.03878316283226013, "beta_mean": 0.04694294556975365, "nonfinite": 0.0, "weighted_memory_to_active": 0.14448794722557068, "mix_std": 0.23430849611759186, "beta_std": 0.08469346910715103, "beta_low": 0.3260498046875, "beta_high": 0.0, "orthogonal_to_active": 0.032209835946559906, "evidence_weights": [-0.09132654964923859, -0.12501850724220276, -0.28221189975738525], "agreement_weight": -0.01055513508617878, "orthogonal_weight": -0.22429625689983368}, {"head": 1, "memory_to_active": 0.02207423560321331, "contribution_to_active": 0.5312643647193909, "mix_mean": 0.6289343237876892, "mix_low": 0.0031328201293945312, "mix_high": 0.022215843200683594, "read_cosine": 0.13936826586723328, "beta_mean": 5.494656215887517e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011565174907445908, "mix_std": 0.29470062255859375, "beta_std": 0.00010763462341856211, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010494116693735123, "evidence_weights": [-0.23862744867801666, -0.2754915654659271, 0.8436371088027954], "agreement_weight": 0.4802279472351074, "orthogonal_weight": -0.949438750743866}, {"head": 2, "memory_to_active": 0.023284895345568657, "contribution_to_active": 0.552068293094635, "mix_mean": 0.6261771321296692, "mix_low": 0.0029497146606445312, "mix_high": 0.017859458923339844, "read_cosine": 0.04165361821651459, "beta_mean": 0.00029905172414146364, "nonfinite": 0.0, "weighted_memory_to_active": 0.01606488786637783, "mix_std": 0.27513399720191956, "beta_std": 0.00048722547944635153, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.003580826334655285, "evidence_weights": [-0.2533620297908783, -0.3487221896648407, 0.8580002188682556], "agreement_weight": 0.004416838753968477, "orthogonal_weight": -0.2636622190475464}, {"head": 3, "memory_to_active": 1.8099267482757568, "contribution_to_active": 0.21174685657024384, "mix_mean": 0.08906089514493942, "mix_low": 0.16626930236816406, "mix_high": 0.0, "read_cosine": 0.13395754992961884, "beta_mean": 0.10387453436851501, "nonfinite": 0.0, "weighted_memory_to_active": 0.3425777852535248, "mix_std": 0.12813465297222137, "beta_std": 0.15355859696865082, "beta_low": 0.2132568359375, "beta_high": 0.0, "orthogonal_to_active": 0.1292380392551422, "evidence_weights": [-0.057859186083078384, -0.09121087193489075, -0.06702429056167603], "agreement_weight": 0.046679429709911346, "orthogonal_weight": -0.4634726345539093}, {"head": 4, "memory_to_active": 0.4707293212413788, "contribution_to_active": 0.6007063984870911, "mix_mean": 0.3254932165145874, "mix_low": 0.2037038803100586, "mix_high": 0.009151458740234375, "read_cosine": 0.5355207324028015, "beta_mean": 0.2012495994567871, "nonfinite": 0.0, "weighted_memory_to_active": 0.3131949007511139, "mix_std": 0.34240952134132385, "beta_std": 0.22742211818695068, "beta_low": 0.26397705078125, "beta_high": 0.0, "orthogonal_to_active": 0.1005711778998375, "evidence_weights": [-0.11586268991231918, -0.2889983057975769, -0.06780403852462769], "agreement_weight": 0.04256250336766243, "orthogonal_weight": 0.4744161069393158}, {"head": 5, "memory_to_active": 0.1055181473493576, "contribution_to_active": 0.6403161287307739, "mix_mean": 0.7730028033256531, "mix_low": 0.0028705596923828125, "mix_high": 0.035846710205078125, "read_cosine": 0.15965966880321503, "beta_mean": 0.00037587975384667516, "nonfinite": 0.0, "weighted_memory_to_active": 0.07069400697946548, "mix_std": 0.24059619009494781, "beta_std": 0.0015943861799314618, "beta_low": 0.99462890625, "beta_high": 0.0, "orthogonal_to_active": 0.0446031391620636, "evidence_weights": [-0.32601478695869446, -0.3220481872558594, 2.034440755844116], "agreement_weight": 0.3720005750656128, "orthogonal_weight": -0.6718804240226746}]}, {"layer": 6, "memory_to_active": 0.346449613571167, "contribution_to_active": 0.5889465808868408, "mix_mean": 0.5506712198257446, "mix_low": 0.0718221664428711, "mix_high": 0.02578338049352169, "read_cosine": 0.26546066999435425, "beta_mean": 0.07283928245306015, "nonfinite": 0.0, "weighted_memory_to_active": 0.16183841228485107, "mix_std": 0.24229523539543152, "beta_std": 0.08230298012495041, "beta_low": 0.5157267451286316, "beta_high": 0.0, "orthogonal_to_active": 0.03615162894129753, "output_influence": 0.49869614839553833, "residual_influence": 0.16157187521457672, "state_active": 0.05024844408035278, "state_memory": 0.009373584762215614, "state_peak": 88.69072723388672, "growth_active": 1.2365000247955322, "growth_memory": 1.2271124124526978, "heads": [{"head": 0, "memory_to_active": 0.1219107061624527, "contribution_to_active": 0.6183303594589233, "mix_mean": 0.7274160981178284, "mix_low": 0.00017642974853515625, "mix_high": 0.009772300720214844, "read_cosine": -0.08216895163059235, "beta_mean": 0.012095562182366848, "nonfinite": 0.0, "weighted_memory_to_active": 0.09175056219100952, "mix_std": 0.23094379901885986, "beta_std": 0.014853238128125668, "beta_low": 0.5904541015625, "beta_high": 0.0, "orthogonal_to_active": 0.07753048837184906, "evidence_weights": [-0.31318581104278564, -0.33475229144096375, 0.5379672646522522], "agreement_weight": 0.14511685073375702, "orthogonal_weight": -0.9331302046775818}, {"head": 1, "memory_to_active": 0.005357756745070219, "contribution_to_active": 0.4972476661205292, "mix_mean": 0.5252304673194885, "mix_low": 0.0013933181762695312, "mix_high": 0.009299278259277344, "read_cosine": 0.5494691729545593, "beta_mean": 1.2219053132866975e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022806511260569096, "mix_std": 0.290260374546051, "beta_std": 2.1250725694699213e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009594880975782871, "evidence_weights": [-0.20315253734588623, -0.2862486243247986, 1.0020772218704224], "agreement_weight": -0.003444898873567581, "orthogonal_weight": -0.5671312212944031}, {"head": 2, "memory_to_active": 1.029421329498291, "contribution_to_active": 0.08251915872097015, "mix_mean": 0.028921566903591156, "mix_low": 0.41886425018310547, "mix_high": 1.9073486328125e-06, "read_cosine": 0.22597870230674744, "beta_mean": 0.06038440018892288, "nonfinite": 0.0, "weighted_memory_to_active": 0.06337770074605942, "mix_std": 0.05060998350381851, "beta_std": 0.06989564746618271, "beta_low": 0.041748046875, "beta_high": 0.0, "orthogonal_to_active": 0.0007022382342256606, "evidence_weights": [0.08915646374225616, -0.008965929970145226, 0.30619949102401733], "agreement_weight": 0.13664953410625458, "orthogonal_weight": 0.013850580900907516}, {"head": 3, "memory_to_active": 0.16158799827098846, "contribution_to_active": 0.6692564487457275, "mix_mean": 0.6379543542861938, "mix_low": 0.008780479431152344, "mix_high": 0.013375282287597656, "read_cosine": 0.46144503355026245, "beta_mean": 0.22615236043930054, "nonfinite": 0.0, "weighted_memory_to_active": 0.1154107004404068, "mix_std": 0.32874220609664917, "beta_std": 0.25572386384010315, "beta_low": 0.2120361328125, "beta_high": 0.0, "orthogonal_to_active": 0.06320521235466003, "evidence_weights": [-0.17414253950119019, -0.337841272354126, 0.1620592623949051], "agreement_weight": -0.009384323842823505, "orthogonal_weight": 0.6340160369873047}, {"head": 4, "memory_to_active": 0.7184525728225708, "contribution_to_active": 1.0679913759231567, "mix_mean": 0.7517451047897339, "mix_low": 0.0012359619140625, "mix_high": 0.11918449401855469, "read_cosine": 0.16846483945846558, "beta_mean": 0.13808421790599823, "nonfinite": 0.0, "weighted_memory_to_active": 0.6689677238464355, "mix_std": 0.29388657212257385, "beta_std": 0.15267740190029144, "beta_low": 0.2508544921875, "beta_high": 0.0, "orthogonal_to_active": 0.0450376495718956, "evidence_weights": [-0.25538191199302673, -0.5973918437957764, -0.098993681371212], "agreement_weight": -0.017347099259495735, "orthogonal_weight": 0.08102443069219589}, {"head": 5, "memory_to_active": 0.0419672429561615, "contribution_to_active": 0.5983341932296753, "mix_mean": 0.6327596306800842, "mix_low": 0.0004825592041015625, "mix_high": 0.0030670166015625, "read_cosine": 0.2695750594139099, "beta_mean": 0.00030693470034748316, "nonfinite": 0.0, "weighted_memory_to_active": 0.029243141412734985, "mix_std": 0.2593285143375397, "beta_std": 0.0006464942125603557, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.029474690556526184, "evidence_weights": [-0.23961639404296875, -0.3052186667919159, 0.4298570454120636], "agreement_weight": 0.15270276367664337, "orthogonal_weight": -1.0798124074935913}]}, {"layer": 7, "memory_to_active": 1.7292096614837646, "contribution_to_active": 0.2204502373933792, "mix_mean": 0.09047143906354904, "mix_low": 0.29415881633758545, "mix_high": 0.00011523565626703203, "read_cosine": 0.2301197648048401, "beta_mean": 0.13527119159698486, "nonfinite": 0.0, "weighted_memory_to_active": 0.21607650816440582, "mix_std": 0.11844243109226227, "beta_std": 0.1059529036283493, "beta_low": 0.178253173828125, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.05686289072036743, "output_influence": 0.22106069326400757, "residual_influence": 0.049072835594415665, "state_active": 0.013336731120944023, "state_memory": 0.027108564972877502, "state_peak": 28.00385856628418, "growth_active": 1.0908864736557007, "growth_memory": 1.320820927619934, "heads": [{"head": 0, "memory_to_active": 2.318511486053467, "contribution_to_active": 0.2966344654560089, "mix_mean": 0.036061905324459076, "mix_low": 0.41315364837646484, "mix_high": 0.0, "read_cosine": 0.36636433005332947, "beta_mean": 0.1736026108264923, "nonfinite": 0.0, "weighted_memory_to_active": 0.3307151794433594, "mix_std": 0.07289592921733856, "beta_std": 0.12300420552492142, "beta_low": 0.000732421875, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.016410015523433685, "evidence_weights": [0.009309252724051476, -0.08543825149536133, 0.10409285873174667], "agreement_weight": -0.038380447775125504, "orthogonal_weight": 0.07391898334026337}, {"head": 1, "memory_to_active": 1.2069846391677856, "contribution_to_active": 0.35832512378692627, "mix_mean": 0.1689361035823822, "mix_low": 0.1736745834350586, "mix_high": 0.000576019287109375, "read_cosine": 0.3009227514266968, "beta_mean": 0.12384802848100662, "nonfinite": 0.0, "weighted_memory_to_active": 0.34252578020095825, "mix_std": 0.20607897639274597, "beta_std": 0.07298637181520462, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0032727262005209923, "evidence_weights": [-0.18757523596286774, -0.11578185856342316, -0.21200533211231232], "agreement_weight": 0.01967599242925644, "orthogonal_weight": 0.010736593045294285}, {"head": 2, "memory_to_active": 0.9521161317825317, "contribution_to_active": 0.27839431166648865, "mix_mean": 0.10437978059053421, "mix_low": 0.12346744537353516, "mix_high": 9.5367431640625e-05, "read_cosine": 0.11877764761447906, "beta_mean": 0.07503950595855713, "nonfinite": 0.0, "weighted_memory_to_active": 0.20584189891815186, "mix_std": 0.1506033092737198, "beta_std": 0.05880079045891762, "beta_low": 0.04559326171875, "beta_high": 0.0, "orthogonal_to_active": 0.05789809674024582, "evidence_weights": [-0.036919452250003815, -0.12620598077774048, 0.42059269547462463], "agreement_weight": 0.026134392246603966, "orthogonal_weight": -0.28818145394325256}, {"head": 3, "memory_to_active": 3.949810266494751, "contribution_to_active": 0.09565657377243042, "mix_mean": 0.03306933492422104, "mix_low": 0.34340763092041016, "mix_high": 0.0, "read_cosine": 0.10751994699239731, "beta_mean": 0.30994340777397156, "nonfinite": 0.0, "weighted_memory_to_active": 0.28085803985595703, "mix_std": 0.05023002624511719, "beta_std": 0.2512298822402954, "beta_low": 0.01220703125, "beta_high": 0.000244140625, "orthogonal_to_active": 0.2278108298778534, "evidence_weights": [0.06776662170886993, 0.002841167151927948, 0.6511316299438477], "agreement_weight": 0.05257824435830116, "orthogonal_weight": -0.9248923659324646}, {"head": 4, "memory_to_active": 1.709340214729309, "contribution_to_active": 0.07698014378547668, "mix_mean": 0.015021159313619137, "mix_low": 0.6604099273681641, "mix_high": 0.0, "read_cosine": 0.22841963171958923, "beta_mean": 0.12715047597885132, "nonfinite": 0.0, "weighted_memory_to_active": 0.07826628535985947, "mix_std": 0.0340736098587513, "beta_std": 0.12354102730751038, "beta_low": 0.04132080078125, "beta_high": 0.0, "orthogonal_to_active": 0.006187109742313623, "evidence_weights": [0.1272353231906891, 0.04877408593893051, 0.1772485226392746], "agreement_weight": 0.05197162181138992, "orthogonal_weight": -0.12206459045410156}, {"head": 5, "memory_to_active": 0.2384943664073944, "contribution_to_active": 0.21671074628829956, "mix_mean": 0.1853603571653366, "mix_low": 0.05083942413330078, "mix_high": 2.002716064453125e-05, "read_cosine": 0.2587141990661621, "beta_mean": 0.0020430628210306168, "nonfinite": 0.0, "weighted_memory_to_active": 0.05825178325176239, "mix_std": 0.1967727541923523, "beta_std": 0.006155157461762428, "beta_low": 0.96966552734375, "beta_high": 0.0, "orthogonal_to_active": 0.02959856018424034, "evidence_weights": [-0.06584638357162476, -0.11555051803588867, 0.3385896682739258], "agreement_weight": 0.0676117017865181, "orthogonal_weight": -0.598554253578186}]}, {"layer": 8, "memory_to_active": 0.46787044405937195, "contribution_to_active": 0.5193533897399902, "mix_mean": 0.4110603630542755, "mix_low": 0.09536807239055634, "mix_high": 0.0008403460378758609, "read_cosine": 0.4876720905303955, "beta_mean": 0.03188253939151764, "nonfinite": 0.0, "weighted_memory_to_active": 0.10010205209255219, "mix_std": 0.2224881649017334, "beta_std": 0.03737516701221466, "beta_low": 0.572906494140625, "beta_high": 0.0, "orthogonal_to_active": 0.05896288901567459, "output_influence": 0.4328676164150238, "residual_influence": 0.13369911909103394, "state_active": 0.052313368767499924, "state_memory": 0.010166211053729057, "state_peak": 38.39422607421875, "growth_active": 1.129490613937378, "growth_memory": 1.4060460329055786, "heads": [{"head": 0, "memory_to_active": 0.046115942299366, "contribution_to_active": 0.539721667766571, "mix_mean": 0.5834042429924011, "mix_low": 0.0002880096435546875, "mix_high": 0.0008211135864257812, "read_cosine": 0.5835975408554077, "beta_mean": 0.003334654029458761, "nonfinite": 0.0, "weighted_memory_to_active": 0.031072452664375305, "mix_std": 0.24181248247623444, "beta_std": 0.008686106652021408, "beta_low": 0.91839599609375, "beta_high": 0.0, "orthogonal_to_active": 0.019234251230955124, "evidence_weights": [-0.17155340313911438, -0.38914602994918823, 0.7144256830215454], "agreement_weight": 0.1013169214129448, "orthogonal_weight": -0.8236814737319946}, {"head": 1, "memory_to_active": 0.14945164322853088, "contribution_to_active": 0.4864141345024109, "mix_mean": 0.45437318086624146, "mix_low": 0.0017242431640625, "mix_high": 0.0004291534423828125, "read_cosine": 0.3231505751609802, "beta_mean": 0.004071752540767193, "nonfinite": 0.0, "weighted_memory_to_active": 0.07126855850219727, "mix_std": 0.2552488148212433, "beta_std": 0.011343698017299175, "beta_low": 0.909423828125, "beta_high": 0.0, "orthogonal_to_active": 0.043554484844207764, "evidence_weights": [-0.19230349361896515, -0.2511306405067444, 0.6799775958061218], "agreement_weight": 0.09596220403909683, "orthogonal_weight": -0.6752452254295349}, {"head": 2, "memory_to_active": 0.0303729809820652, "contribution_to_active": 0.6348395347595215, "mix_mean": 0.6139682531356812, "mix_low": 0.0011758804321289062, "mix_high": 0.0029096603393554688, "read_cosine": 0.39342397451400757, "beta_mean": 0.00027983650215901434, "nonfinite": 0.0, "weighted_memory_to_active": 0.022504571825265884, "mix_std": 0.23517270386219025, "beta_std": 0.002815683837980032, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.014745677821338177, "evidence_weights": [-0.2186499983072281, -0.33681172132492065, 0.8495144844055176], "agreement_weight": 0.20067884027957916, "orthogonal_weight": -0.7535333037376404}, {"head": 3, "memory_to_active": 0.1375061422586441, "contribution_to_active": 0.6042385697364807, "mix_mean": 0.528744101524353, "mix_low": 0.0032434463500976562, "mix_high": 0.0006961822509765625, "read_cosine": 0.7954549789428711, "beta_mean": 0.014633966609835625, "nonfinite": 0.0, "weighted_memory_to_active": 0.10127996653318405, "mix_std": 0.2573363780975342, "beta_std": 0.011206279508769512, "beta_low": 0.3873291015625, "beta_high": 0.0, "orthogonal_to_active": 0.03268067538738251, "evidence_weights": [-0.17251844704151154, -0.29589876532554626, 0.6682196259498596], "agreement_weight": 0.1461130976676941, "orthogonal_weight": -0.6288961172103882}, {"head": 4, "memory_to_active": 2.147448778152466, "contribution_to_active": 0.210178405046463, "mix_mean": 0.028167400509119034, "mix_low": 0.42510414123535156, "mix_high": 0.0, "read_cosine": 0.33115291595458984, "beta_mean": 0.08853290230035782, "nonfinite": 0.0, "weighted_memory_to_active": 0.2010205090045929, "mix_std": 0.050279781222343445, "beta_std": 0.08367108553647995, "beta_low": 0.033447265625, "beta_high": 0.0, "orthogonal_to_active": 0.00611175037920475, "evidence_weights": [0.04878246784210205, -0.0195759404450655, -0.0858430340886116], "agreement_weight": 0.05193769931793213, "orthogonal_weight": 0.03150663152337074}, {"head": 5, "memory_to_active": 0.29632705450057983, "contribution_to_active": 0.6407278180122375, "mix_mean": 0.25770485401153564, "mix_low": 0.1406726837158203, "mix_high": 0.00018596649169921875, "read_cosine": 0.4992523789405823, "beta_mean": 0.08044210076332092, "nonfinite": 0.0, "weighted_memory_to_active": 0.17346620559692383, "mix_std": 0.29507875442504883, "beta_std": 0.1065281331539154, "beta_low": 0.1893310546875, "beta_high": 0.0, "orthogonal_to_active": 0.2374504804611206, "evidence_weights": [-0.0646374374628067, -0.2698306441307068, -0.3816160559654236], "agreement_weight": 0.002569240052253008, "orthogonal_weight": 1.6258835792541504}]}, {"layer": 9, "memory_to_active": 1.0878158807754517, "contribution_to_active": 0.6382472515106201, "mix_mean": 0.311850905418396, "mix_low": 0.14178451895713806, "mix_high": 0.0026289622765034437, "read_cosine": 0.3630819320678711, "beta_mean": 0.06895945966243744, "nonfinite": 0.0, "weighted_memory_to_active": 0.3751858174800873, "mix_std": 0.21007747948169708, "beta_std": 0.04532558470964432, "beta_low": 0.5008036494255066, "beta_high": 0.0, "orthogonal_to_active": 0.042364589869976044, "output_influence": 0.48730528354644775, "residual_influence": 0.13209709525108337, "state_active": 0.04149284213781357, "state_memory": 0.019944867119193077, "state_peak": 43.244510650634766, "growth_active": 1.1585338115692139, "growth_memory": 1.4034548997879028, "heads": [{"head": 0, "memory_to_active": 0.008412305265665054, "contribution_to_active": 0.5364256501197815, "mix_mean": 0.6064528226852417, "mix_low": 0.00145721435546875, "mix_high": 0.01149749755859375, "read_cosine": 0.21943694353103638, "beta_mean": 1.6532057998119853e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0052626184187829494, "mix_std": 0.2906469702720642, "beta_std": 7.778871076880023e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004691722337156534, "evidence_weights": [-0.2952256202697754, -0.30425068736076355, 0.9622249007225037], "agreement_weight": 0.10512170940637589, "orthogonal_weight": -0.9325206279754639}, {"head": 1, "memory_to_active": 0.03488839790225029, "contribution_to_active": 0.5991044640541077, "mix_mean": 0.4874565601348877, "mix_low": 0.009122848510742188, "mix_high": 0.0012359619140625, "read_cosine": 0.40464258193969727, "beta_mean": 9.624502126825973e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.02243279293179512, "mix_std": 0.2901829481124878, "beta_std": 0.00022244243882596493, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015287736430764198, "evidence_weights": [-0.21923503279685974, -0.26762399077415466, 1.1373600959777832], "agreement_weight": 0.34379518032073975, "orthogonal_weight": -0.8224776387214661}, {"head": 2, "memory_to_active": 0.1939411610364914, "contribution_to_active": 0.6191797852516174, "mix_mean": 0.538221001625061, "mix_low": 0.0019817352294921875, "mix_high": 0.0027875900268554688, "read_cosine": 0.3741857707500458, "beta_mean": 0.0007842411869205534, "nonfinite": 0.0, "weighted_memory_to_active": 0.12485933303833008, "mix_std": 0.2781701385974884, "beta_std": 0.004608288407325745, "beta_low": 0.9945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.061087124049663544, "evidence_weights": [-0.187580406665802, -0.24986393749713898, 0.6349917650222778], "agreement_weight": 0.2521023154258728, "orthogonal_weight": -0.586093544960022}, {"head": 3, "memory_to_active": 1.5198588371276855, "contribution_to_active": 0.162113219499588, "mix_mean": 0.043195176869630814, "mix_low": 0.3576688766479492, "mix_high": 6.67572021484375e-06, "read_cosine": 0.4197225570678711, "beta_mean": 0.09790152311325073, "nonfinite": 0.0, "weighted_memory_to_active": 0.13920888304710388, "mix_std": 0.0920504704117775, "beta_std": 0.06637857854366302, "beta_low": 0.0028076171875, "beta_high": 0.0, "orthogonal_to_active": 0.022590043023228645, "evidence_weights": [0.06695996969938278, 0.024318549782037735, -0.05739543214440346], "agreement_weight": 0.09060777723789215, "orthogonal_weight": 0.18087610602378845}, {"head": 4, "memory_to_active": 3.1662986278533936, "contribution_to_active": 1.2899237871170044, "mix_mean": 0.12626568973064423, "mix_low": 0.15308284759521484, "mix_high": 0.00024509429931640625, "read_cosine": 0.2557677924633026, "beta_mean": 0.18627768754959106, "nonfinite": 0.0, "weighted_memory_to_active": 1.266183853149414, "mix_std": 0.1839865744113922, "beta_std": 0.09943849593400955, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02752402052283287, "evidence_weights": [-0.06157686188817024, -0.15035903453826904, 0.005231189075857401], "agreement_weight": -0.04927339032292366, "orthogonal_weight": -0.022766463458538055}, {"head": 5, "memory_to_active": 1.603495717048645, "contribution_to_active": 0.622736394405365, "mix_mean": 0.06951409578323364, "mix_low": 0.3273935317993164, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5047358870506287, "beta_mean": 0.1286805272102356, "nonfinite": 0.0, "weighted_memory_to_active": 0.6931672096252441, "mix_std": 0.12542763352394104, "beta_std": 0.10122792422771454, "beta_low": 0.00750732421875, "beta_high": 0.0, "orthogonal_to_active": 0.1230068951845169, "evidence_weights": [0.09836401790380478, -0.06536847352981567, 0.15520523488521576], "agreement_weight": 0.023921601474285126, "orthogonal_weight": 0.26089024543762207}]}, {"layer": 10, "memory_to_active": 0.21339592337608337, "contribution_to_active": 0.5787353515625, "mix_mean": 0.4921042323112488, "mix_low": 0.013039113022387028, "mix_high": 0.007617473602294922, "read_cosine": 0.37795132398605347, "beta_mean": 0.0448867492377758, "nonfinite": 0.0, "weighted_memory_to_active": 0.1021299809217453, "mix_std": 0.2680581212043762, "beta_std": 0.04627317562699318, "beta_low": 0.6633097529411316, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.11723613739013672, "output_influence": 0.4177226722240448, "residual_influence": 0.14044392108917236, "state_active": 0.05329473316669464, "state_memory": 0.014432207681238651, "state_peak": 114.2680435180664, "growth_active": 1.0602307319641113, "growth_memory": 1.0201376676559448, "heads": [{"head": 0, "memory_to_active": 0.7733016610145569, "contribution_to_active": 1.2951418161392212, "mix_mean": 0.6160809397697449, "mix_low": 0.03642749786376953, "mix_high": 0.03256034851074219, "read_cosine": 0.09894673526287079, "beta_mean": 0.19428326189517975, "nonfinite": 0.0, "weighted_memory_to_active": 0.48541921377182007, "mix_std": 0.3807371258735657, "beta_std": 0.21109764277935028, "beta_low": 0.141845703125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.6244292259216309, "evidence_weights": [-0.1360696256160736, -0.3798181414604187, -0.9287770390510559], "agreement_weight": 0.019552869722247124, "orthogonal_weight": 1.307183861732483}, {"head": 1, "memory_to_active": 0.013493981212377548, "contribution_to_active": 0.6029125452041626, "mix_mean": 0.5991418361663818, "mix_low": 0.0005235671997070312, "mix_high": 0.009776115417480469, "read_cosine": 0.28428348898887634, "beta_mean": 8.060202526394278e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008063194341957569, "mix_std": 0.266568660736084, "beta_std": 0.0001028545739245601, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010790491476655006, "evidence_weights": [-0.2713654339313507, -0.20330512523651123, 0.5863563418388367], "agreement_weight": 0.43242403864860535, "orthogonal_weight": -1.4634276628494263}, {"head": 2, "memory_to_active": 0.020323820412158966, "contribution_to_active": 0.548610270023346, "mix_mean": 0.5748202800750732, "mix_low": 0.00029277801513671875, "mix_high": 0.00299072265625, "read_cosine": 0.05472860112786293, "beta_mean": 5.4569482017541304e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008165189996361732, "mix_std": 0.2587897777557373, "beta_std": 0.00019832466205116361, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003748866729438305, "evidence_weights": [-0.22361628711223602, -0.26426732540130615, 0.534565806388855], "agreement_weight": -0.12773381173610687, "orthogonal_weight": -0.4390105605125427}, {"head": 3, "memory_to_active": 0.32556024193763733, "contribution_to_active": 0.09763060510158539, "mix_mean": 0.2639259397983551, "mix_low": 0.03776264190673828, "mix_high": 0.00014972686767578125, "read_cosine": 0.909920334815979, "beta_mean": 0.06842789053916931, "nonfinite": 0.0, "weighted_memory_to_active": 0.037529557943344116, "mix_std": 0.22836056351661682, "beta_std": 0.058228250592947006, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.04356852546334267, "evidence_weights": [-0.11454350501298904, -0.14486193656921387, 0.13838757574558258], "agreement_weight": 0.004904469940811396, "orthogonal_weight": 1.492835283279419}, {"head": 4, "memory_to_active": 0.0026434913743287325, "contribution_to_active": 0.5104336142539978, "mix_mean": 0.5189579129219055, "mix_low": 0.00058746337890625, "mix_high": 0.00022602081298828125, "read_cosine": 0.05806264653801918, "beta_mean": 1.677541877143085e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0012920490698888898, "mix_std": 0.2494196593761444, "beta_std": 7.931527215987444e-05, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0007281191647052765, "evidence_weights": [-0.1792844533920288, -0.25248396396636963, 0.7279834747314453], "agreement_weight": 0.07249469310045242, "orthogonal_weight": -0.5904761552810669}, {"head": 5, "memory_to_active": 0.14505234360694885, "contribution_to_active": 0.41768312454223633, "mix_mean": 0.37969857454299927, "mix_low": 0.0026407241821289062, "mix_high": 1.9073486328125e-06, "read_cosine": 0.86176598072052, "beta_mean": 0.006457414012402296, "nonfinite": 0.0, "weighted_memory_to_active": 0.0723106786608696, "mix_std": 0.22447293996810913, "beta_std": 0.007932646200060844, "beta_low": 0.83758544921875, "beta_high": 0.0, "orthogonal_to_active": 0.020151503384113312, "evidence_weights": [-0.14601768553256989, -0.15841148793697357, 0.29306459426879883], "agreement_weight": 0.09122125059366226, "orthogonal_weight": -0.6219803690910339}]}, {"layer": 11, "memory_to_active": 0.14893284440040588, "contribution_to_active": 0.5673409104347229, "mix_mean": 0.4644586443901062, "mix_low": 0.08542570471763611, "mix_high": 0.04311529919505119, "read_cosine": 0.44299906492233276, "beta_mean": 0.03405110538005829, "nonfinite": 0.0, "weighted_memory_to_active": 0.0685458779335022, "mix_std": 0.3210197985172272, "beta_std": 0.04698271304368973, "beta_low": 0.5420837998390198, "beta_high": 0.0, "orthogonal_to_active": 0.05662029981613159, "output_influence": 0.5039388537406921, "residual_influence": 0.12588536739349365, "state_active": 0.18057265877723694, "state_memory": 0.007319862954318523, "state_peak": 27.764358520507812, "growth_active": 1.1548972129821777, "growth_memory": 1.642227053642273, "heads": [{"head": 0, "memory_to_active": 0.006551795639097691, "contribution_to_active": 0.558052659034729, "mix_mean": 0.6006302833557129, "mix_low": 0.001483917236328125, "mix_high": 0.007144927978515625, "read_cosine": 0.3380180299282074, "beta_mean": 8.369920396944508e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0040031131356954575, "mix_std": 0.2655690014362335, "beta_std": 0.0036016313824802637, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.002823509043082595, "evidence_weights": [-0.16356009244918823, -0.35911843180656433, 0.6942662000656128], "agreement_weight": 0.24353182315826416, "orthogonal_weight": -0.7567512392997742}, {"head": 1, "memory_to_active": 0.33979249000549316, "contribution_to_active": 0.28883984684944153, "mix_mean": 0.33917737007141113, "mix_low": 0.010403633117675781, "mix_high": 0.0001220703125, "read_cosine": 0.5857030153274536, "beta_mean": 0.029065441340208054, "nonfinite": 0.0, "weighted_memory_to_active": 0.1437871754169464, "mix_std": 0.2546217143535614, "beta_std": 0.053276997059583664, "beta_low": 0.45001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.04132608696818352, "evidence_weights": [-0.1617317646741867, -0.16141225397586823, 0.3789471983909607], "agreement_weight": 0.0811486765742302, "orthogonal_weight": -0.3740486800670624}, {"head": 2, "memory_to_active": 0.01672562211751938, "contribution_to_active": 0.9657273292541504, "mix_mean": 0.4485626518726349, "mix_low": 0.20737648010253906, "mix_high": 0.15526199340820312, "read_cosine": 0.22278663516044617, "beta_mean": 0.07775476574897766, "nonfinite": 0.0, "weighted_memory_to_active": 0.012495665811002254, "mix_std": 0.43737736344337463, "beta_std": 0.09890475869178772, "beta_low": 0.42266845703125, "beta_high": 0.0, "orthogonal_to_active": 0.06575173139572144, "evidence_weights": [0.02721465192735195, -0.45200207829475403, -1.2643108367919922], "agreement_weight": 0.00044064840767532587, "orthogonal_weight": 5.511932849884033}, {"head": 3, "memory_to_active": 0.01241261512041092, "contribution_to_active": 0.8213507533073425, "mix_mean": 0.4135519564151764, "mix_low": 0.28144168853759766, "mix_high": 0.09298324584960938, "read_cosine": 0.6145076751708984, "beta_mean": 0.06076695770025253, "nonfinite": 0.0, "weighted_memory_to_active": 0.007339132949709892, "mix_std": 0.43728846311569214, "beta_std": 0.08671216666698456, "beta_low": 0.46240234375, "beta_high": 0.0, "orthogonal_to_active": 0.031018560752272606, "evidence_weights": [0.10134411603212357, -0.34356606006622314, -0.3752790689468384], "agreement_weight": 0.042920295149087906, "orthogonal_weight": 5.827418804168701}, {"head": 4, "memory_to_active": 0.4047241508960724, "contribution_to_active": 0.3972390592098236, "mix_mean": 0.4732779264450073, "mix_low": 0.010715484619140625, "mix_high": 0.0005578994750976562, "read_cosine": 0.12862129509449005, "beta_mean": 0.02624942734837532, "nonfinite": 0.0, "weighted_memory_to_active": 0.19052785634994507, "mix_std": 0.270790696144104, "beta_std": 0.024550067260861397, "beta_low": 0.2381591796875, "beta_high": 0.0, "orthogonal_to_active": 0.1731378138065338, "evidence_weights": [-0.21705231070518494, -0.19255892932415009, 1.7871214151382446], "agreement_weight": -0.1757977455854416, "orthogonal_weight": -0.9316318035125732}, {"head": 5, "memory_to_active": 0.11339029669761658, "contribution_to_active": 0.37283578515052795, "mix_mean": 0.5115516185760498, "mix_low": 0.001132965087890625, "mix_high": 0.0026216506958007812, "read_cosine": 0.7683576345443726, "beta_mean": 0.01038634404540062, "nonfinite": 0.0, "weighted_memory_to_active": 0.05312233418226242, "mix_std": 0.26047152280807495, "beta_std": 0.014850631356239319, "beta_low": 0.679443359375, "beta_high": 0.0, "orthogonal_to_active": 0.02566409856081009, "evidence_weights": [-0.21671055257320404, -0.28439703583717346, 0.5531689524650574], "agreement_weight": 0.0736081600189209, "orthogonal_weight": -0.7036135196685791}]}, {"layer": 12, "memory_to_active": 0.2897435426712036, "contribution_to_active": 0.5558340549468994, "mix_mean": 0.490134596824646, "mix_low": 0.011390527710318565, "mix_high": 0.003810405731201172, "read_cosine": 0.2502744197845459, "beta_mean": 0.016489705070853233, "nonfinite": 0.0, "weighted_memory_to_active": 0.15689800679683685, "mix_std": 0.2616637945175171, "beta_std": 0.01587456464767456, "beta_low": 0.8209737539291382, "beta_high": 0.0, "orthogonal_to_active": 0.07083266973495483, "output_influence": 0.4409840703010559, "residual_influence": 0.12126164138317108, "state_active": 0.039192866533994675, "state_memory": 0.009657379239797592, "state_peak": 41.89875793457031, "growth_active": 1.1192270517349243, "growth_memory": 1.2964234352111816, "heads": [{"head": 0, "memory_to_active": 0.051106713712215424, "contribution_to_active": 0.49443817138671875, "mix_mean": 0.4752936363220215, "mix_low": 0.0016202926635742188, "mix_high": 0.00148773193359375, "read_cosine": 0.3198295831680298, "beta_mean": 0.0009067553328350186, "nonfinite": 0.0, "weighted_memory_to_active": 0.023208916187286377, "mix_std": 0.2705501317977905, "beta_std": 0.0032093743793666363, "beta_low": 0.9920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.01608668640255928, "evidence_weights": [-0.18274731934070587, -0.25698983669281006, 0.3148926794528961], "agreement_weight": 0.10821129381656647, "orthogonal_weight": -0.7337923049926758}, {"head": 1, "memory_to_active": 0.11562551558017731, "contribution_to_active": 0.5060056447982788, "mix_mean": 0.5235592126846313, "mix_low": 0.000888824462890625, "mix_high": 0.0006656646728515625, "read_cosine": 0.5525278449058533, "beta_mean": 0.0011388789862394333, "nonfinite": 0.0, "weighted_memory_to_active": 0.06031622737646103, "mix_std": 0.2636505663394928, "beta_std": 0.0054575116373598576, "beta_low": 0.9869384765625, "beta_high": 0.0, "orthogonal_to_active": 0.04614840820431709, "evidence_weights": [-0.20486962795257568, -0.2720395028591156, 0.9305648803710938], "agreement_weight": 0.13400182127952576, "orthogonal_weight": -0.9235768914222717}, {"head": 2, "memory_to_active": 0.6384817957878113, "contribution_to_active": 0.5002154111862183, "mix_mean": 0.3219203054904938, "mix_low": 0.02756023406982422, "mix_high": 0.00030422210693359375, "read_cosine": 0.4659598767757416, "beta_mean": 0.09398406744003296, "nonfinite": 0.0, "weighted_memory_to_active": 0.26268628239631653, "mix_std": 0.24540337920188904, "beta_std": 0.06376808881759644, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.2430388331413269, "evidence_weights": [-0.2408459335565567, -0.04413418099284172, -0.10617246478796005], "agreement_weight": 0.01168516930192709, "orthogonal_weight": 1.1264119148254395}, {"head": 3, "memory_to_active": 0.023085705935955048, "contribution_to_active": 0.7205992341041565, "mix_mean": 0.6843112707138062, "mix_low": 0.000286102294921875, "mix_high": 0.014739036560058594, "read_cosine": 0.04279806464910507, "beta_mean": 0.0001978404470719397, "nonfinite": 0.0, "weighted_memory_to_active": 0.01724947616457939, "mix_std": 0.24055247008800507, "beta_std": 0.003960970789194107, "beta_low": 0.998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.01607445999979973, "evidence_weights": [-0.29209136962890625, -0.33538419008255005, 1.2969409227371216], "agreement_weight": 0.32131558656692505, "orthogonal_weight": -0.9512149691581726}, {"head": 4, "memory_to_active": 0.8306063413619995, "contribution_to_active": 0.6597392559051514, "mix_mean": 0.33067020773887634, "mix_low": 0.030517578125, "mix_high": 0.0021638870239257812, "read_cosine": -0.10027457773685455, "beta_mean": 0.00215239729732275, "nonfinite": 0.0, "weighted_memory_to_active": 0.529369592666626, "mix_std": 0.27243486046791077, "beta_std": 0.014458024874329567, "beta_low": 0.95989990234375, "beta_high": 0.0, "orthogonal_to_active": 0.06530644744634628, "evidence_weights": [-0.10022270679473877, -0.2125522345304489, 0.05806746706366539], "agreement_weight": 0.0008334050071425736, "orthogonal_weight": -0.13104236125946045}, {"head": 5, "memory_to_active": 0.07955505698919296, "contribution_to_active": 0.45400646328926086, "mix_mean": 0.6050527691841125, "mix_low": 0.007470130920410156, "mix_high": 0.00350189208984375, "read_cosine": 0.22080576419830322, "beta_mean": 0.0005582921439781785, "nonfinite": 0.0, "weighted_memory_to_active": 0.04855751246213913, "mix_std": 0.27739137411117554, "beta_std": 0.004393423441797495, "beta_low": 0.98760986328125, "beta_high": 0.0, "orthogonal_to_active": 0.03834112361073494, "evidence_weights": [-0.2528490126132965, -0.3057076036930084, 1.1708307266235352], "agreement_weight": 0.2717044949531555, "orthogonal_weight": -0.8404011130332947}]}, {"layer": 13, "memory_to_active": 0.14014612138271332, "contribution_to_active": 0.5156625509262085, "mix_mean": 0.5110248327255249, "mix_low": 0.005683581344783306, "mix_high": 0.005129337310791016, "read_cosine": 0.4101031422615051, "beta_mean": 0.01774592511355877, "nonfinite": 0.0, "weighted_memory_to_active": 0.06112822890281677, "mix_std": 0.2597172260284424, "beta_std": 0.02003292366862297, "beta_low": 0.792388916015625, "beta_high": 0.0, "orthogonal_to_active": 0.06111860275268555, "output_influence": 0.48132920265197754, "residual_influence": 0.16210278868675232, "state_active": 0.09987816214561462, "state_memory": 0.037857990711927414, "state_peak": 92.61577606201172, "growth_active": 0.9615210890769958, "growth_memory": 1.051457166671753, "heads": [{"head": 0, "memory_to_active": 0.04167694225907326, "contribution_to_active": 0.553551971912384, "mix_mean": 0.5619288086891174, "mix_low": 0.000457763671875, "mix_high": 0.0015439987182617188, "read_cosine": 0.42603600025177, "beta_mean": 0.00016759525169618428, "nonfinite": 0.0, "weighted_memory_to_active": 0.0253140889108181, "mix_std": 0.26730257272720337, "beta_std": 0.0004101842059753835, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.024366356432437897, "evidence_weights": [-0.20098061859607697, -0.2629883885383606, 0.7576434016227722], "agreement_weight": 0.08927839249372482, "orthogonal_weight": -1.087958812713623}, {"head": 1, "memory_to_active": 0.01778910867869854, "contribution_to_active": 0.5801650285720825, "mix_mean": 0.530413031578064, "mix_low": 0.0024890899658203125, "mix_high": 0.0022211074829101562, "read_cosine": 0.2180766463279724, "beta_mean": 6.224990647751838e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009931342676281929, "mix_std": 0.2709228992462158, "beta_std": 0.00010565367847448215, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011076821014285088, "evidence_weights": [-0.1876879185438156, -0.2700256407260895, 0.722694456577301], "agreement_weight": 0.30475062131881714, "orthogonal_weight": -1.1758041381835938}, {"head": 2, "memory_to_active": 0.1117512509226799, "contribution_to_active": 0.4292554557323456, "mix_mean": 0.36493492126464844, "mix_low": 0.0020923614501953125, "mix_high": 5.53131103515625e-05, "read_cosine": 0.6304365992546082, "beta_mean": 0.019097773358225822, "nonfinite": 0.0, "weighted_memory_to_active": 0.04636821895837784, "mix_std": 0.22069202363491058, "beta_std": 0.054237280040979385, "beta_low": 0.77093505859375, "beta_high": 0.0, "orthogonal_to_active": 0.024379409849643707, "evidence_weights": [-0.20409846305847168, -0.18553867936134338, 0.4287126362323761], "agreement_weight": 0.013452906161546707, "orthogonal_weight": -0.8052322864532471}, {"head": 3, "memory_to_active": 0.13996157050132751, "contribution_to_active": 0.5747281312942505, "mix_mean": 0.6137428283691406, "mix_low": 0.020064353942871094, "mix_high": 0.020252227783203125, "read_cosine": -0.006533443927764893, "beta_mean": 0.0010165333515033126, "nonfinite": 0.0, "weighted_memory_to_active": 0.10110069066286087, "mix_std": 0.2925773859024048, "beta_std": 0.006715291179716587, "beta_low": 0.9920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.10012990236282349, "evidence_weights": [-0.33343183994293213, -0.2437382936477661, 1.9379550218582153], "agreement_weight": 3.6314213275909424, "orthogonal_weight": -1.0543441772460938}, {"head": 4, "memory_to_active": 0.43524080514907837, "contribution_to_active": 0.2900609076023102, "mix_mean": 0.37529051303863525, "mix_low": 0.006938934326171875, "mix_high": 9.72747802734375e-05, "read_cosine": 0.8797426819801331, "beta_mean": 0.08527577668428421, "nonfinite": 0.0, "weighted_memory_to_active": 0.12510009109973907, "mix_std": 0.23053818941116333, "beta_std": 0.053812023252248764, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.15890932083129883, "evidence_weights": [-0.13892090320587158, -0.23196463286876678, 0.4904870390892029], "agreement_weight": 0.001752114505507052, "orthogonal_weight": 2.0631263256073}, {"head": 5, "memory_to_active": 0.09445700794458389, "contribution_to_active": 0.6662137508392334, "mix_mean": 0.6198387145996094, "mix_low": 0.0020589828491210938, "mix_high": 0.006606101989746094, "read_cosine": 0.31286031007766724, "beta_mean": 0.0008556158863939345, "nonfinite": 0.0, "weighted_memory_to_active": 0.058954887092113495, "mix_std": 0.2762702405452728, "beta_std": 0.004917094483971596, "beta_low": 0.9913330078125, "beta_high": 0.0, "orthogonal_to_active": 0.04784981161355972, "evidence_weights": [-0.22157549858093262, -0.2985703647136688, 0.7576928734779358], "agreement_weight": 0.1041189655661583, "orthogonal_weight": -0.8983699083328247}]}, {"layer": 14, "memory_to_active": 0.26347753405570984, "contribution_to_active": 0.42253899574279785, "mix_mean": 0.4181686341762543, "mix_low": 0.02286640927195549, "mix_high": 0.0050067906267941, "read_cosine": 0.6255847215652466, "beta_mean": 0.06941861659288406, "nonfinite": 0.0, "weighted_memory_to_active": 0.12003310024738312, "mix_std": 0.24822202324867249, "beta_std": 0.0634465366601944, "beta_low": 0.4487101435661316, "beta_high": 0.0, "orthogonal_to_active": 0.04658535122871399, "output_influence": 0.3189396858215332, "residual_influence": 0.12381577491760254, "state_active": 0.06109997257590294, "state_memory": 0.01598302274942398, "state_peak": 34.46077346801758, "growth_active": 1.0869280099868774, "growth_memory": 1.3573888540267944, "heads": [{"head": 0, "memory_to_active": 0.3603513240814209, "contribution_to_active": 0.2012445628643036, "mix_mean": 0.2273058295249939, "mix_low": 0.011116981506347656, "mix_high": 3.814697265625e-06, "read_cosine": 0.7660422325134277, "beta_mean": 0.06846272200345993, "nonfinite": 0.0, "weighted_memory_to_active": 0.09833720326423645, "mix_std": 0.20171955227851868, "beta_std": 0.059196073561906815, "beta_low": 0.01007080078125, "beta_high": 0.0, "orthogonal_to_active": 0.07271187007427216, "evidence_weights": [-0.13401135802268982, -0.04169432446360588, 0.5553330183029175], "agreement_weight": 0.023207001388072968, "orthogonal_weight": 1.1322581768035889}, {"head": 1, "memory_to_active": 0.5136951208114624, "contribution_to_active": 0.33673667907714844, "mix_mean": 0.24523043632507324, "mix_low": 0.1086263656616211, "mix_high": 5.245208740234375e-05, "read_cosine": 0.784180760383606, "beta_mean": 0.283380925655365, "nonfinite": 0.0, "weighted_memory_to_active": 0.23641866445541382, "mix_std": 0.2656710743904114, "beta_std": 0.2603369653224945, "beta_low": 0.0985107421875, "beta_high": 0.0, "orthogonal_to_active": 0.015795301645994186, "evidence_weights": [-0.007691184990108013, -0.24368920922279358, -0.17165496945381165], "agreement_weight": 0.05325588956475258, "orthogonal_weight": 0.14766491949558258}, {"head": 2, "memory_to_active": 0.17994444072246552, "contribution_to_active": 0.544550895690918, "mix_mean": 0.555840015411377, "mix_low": 0.0031023025512695312, "mix_high": 0.0019092559814453125, "read_cosine": 0.4189138412475586, "beta_mean": 0.004062030464410782, "nonfinite": 0.0, "weighted_memory_to_active": 0.1110612004995346, "mix_std": 0.2656872272491455, "beta_std": 0.004928975366055965, "beta_low": 0.91326904296875, "beta_high": 0.0, "orthogonal_to_active": 0.08404151350259781, "evidence_weights": [-0.15616671741008759, -0.30213165283203125, 0.9122664928436279], "agreement_weight": 0.08181006461381912, "orthogonal_weight": -0.8704816102981567}, {"head": 3, "memory_to_active": 0.07570317387580872, "contribution_to_active": 0.6495165824890137, "mix_mean": 0.6709353923797607, "mix_low": 0.0052814483642578125, "mix_high": 0.019568443298339844, "read_cosine": 0.20425614714622498, "beta_mean": 0.009359193034470081, "nonfinite": 0.0, "weighted_memory_to_active": 0.051023513078689575, "mix_std": 0.24956639111042023, "beta_std": 0.009554009884595871, "beta_low": 0.6781005859375, "beta_high": 0.0, "orthogonal_to_active": 0.030809566378593445, "evidence_weights": [-0.3726530075073242, -0.32364869117736816, 0.4131627082824707], "agreement_weight": 0.0812048614025116, "orthogonal_weight": -0.7203394770622253}, {"head": 4, "memory_to_active": 0.17401455342769623, "contribution_to_active": 0.4843656122684479, "mix_mean": 0.48040997982025146, "mix_low": 0.004805564880371094, "mix_high": 0.007762908935546875, "read_cosine": 0.8431510925292969, "beta_mean": 0.04893334209918976, "nonfinite": 0.0, "weighted_memory_to_active": 0.10293566435575485, "mix_std": 0.2625950276851654, "beta_std": 0.03657848387956619, "beta_low": 0.03509521484375, "beta_high": 0.0, "orthogonal_to_active": 0.029859190806746483, "evidence_weights": [-0.22240415215492249, -0.26020386815071106, 0.4254726469516754], "agreement_weight": 0.12399960309267044, "orthogonal_weight": -0.5622624754905701}, {"head": 5, "memory_to_active": 0.2771564722061157, "contribution_to_active": 0.31881949305534363, "mix_mean": 0.32929015159606934, "mix_low": 0.004265785217285156, "mix_high": 0.000743865966796875, "read_cosine": 0.736964225769043, "beta_mean": 0.002313456032425165, "nonfinite": 0.0, "weighted_memory_to_active": 0.12042236328125, "mix_std": 0.24409276247024536, "beta_std": 0.010084668174386024, "beta_low": 0.95721435546875, "beta_high": 0.0, "orthogonal_to_active": 0.046294644474983215, "evidence_weights": [-0.18027520179748535, -0.15805989503860474, 0.22077007591724396], "agreement_weight": 0.08609461784362793, "orthogonal_weight": -0.64873868227005}]}, {"layer": 15, "memory_to_active": 0.06341224163770676, "contribution_to_active": 0.48662450909614563, "mix_mean": 0.40148115158081055, "mix_low": 0.00856145191937685, "mix_high": 0.0028004648629575968, "read_cosine": 0.6832578778266907, "beta_mean": 0.04365517571568489, "nonfinite": 0.0, "weighted_memory_to_active": 0.02930755540728569, "mix_std": 0.28151771426200867, "beta_std": 0.04762675613164902, "beta_low": 0.6999003291130066, "beta_high": 0.0, "orthogonal_to_active": 0.007275505922734737, "output_influence": 0.30293452739715576, "residual_influence": 0.1624542772769928, "state_active": 0.12585628032684326, "state_memory": 0.009756263345479965, "state_peak": 39.417518615722656, "growth_active": 1.255190134048462, "growth_memory": 1.5317715406417847, "heads": [{"head": 0, "memory_to_active": 0.11744362115859985, "contribution_to_active": 0.4612351953983307, "mix_mean": 0.35041481256484985, "mix_low": 0.008113861083984375, "mix_high": 0.004563331604003906, "read_cosine": 0.6758733987808228, "beta_mean": 0.03988135606050491, "nonfinite": 0.0, "weighted_memory_to_active": 0.05221778154373169, "mix_std": 0.271502822637558, "beta_std": 0.028362693265080452, "beta_low": 0.06768798828125, "beta_high": 0.0, "orthogonal_to_active": 0.023299094289541245, "evidence_weights": [-0.16221468150615692, -0.17187964916229248, 0.34761524200439453], "agreement_weight": 0.08955132216215134, "orthogonal_weight": -0.6907983422279358}, {"head": 1, "memory_to_active": 0.13050319254398346, "contribution_to_active": 0.4122934937477112, "mix_mean": 0.41029056906700134, "mix_low": 0.0032510757446289062, "mix_high": 0.0040950775146484375, "read_cosine": 0.8984548449516296, "beta_mean": 0.0010692068608477712, "nonfinite": 0.0, "weighted_memory_to_active": 0.0524769201874733, "mix_std": 0.28358131647109985, "beta_std": 0.002808820456266403, "beta_low": 0.98419189453125, "beta_high": 0.0, "orthogonal_to_active": 0.0011814588215202093, "evidence_weights": [-0.16813498735427856, -0.17680247128009796, 0.2466435730457306], "agreement_weight": 0.10605069249868393, "orthogonal_weight": -0.03921223804354668}, {"head": 2, "memory_to_active": 0.007087308447808027, "contribution_to_active": 0.4580325484275818, "mix_mean": 0.34086835384368896, "mix_low": 0.015527725219726562, "mix_high": 3.147125244140625e-05, "read_cosine": 0.8121393322944641, "beta_mean": 0.00010947441478492692, "nonfinite": 0.0, "weighted_memory_to_active": 0.00317905331030488, "mix_std": 0.2708311676979065, "beta_std": 0.0002333055599592626, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00038870004937052727, "evidence_weights": [-0.20288148522377014, -0.27680402994155884, 0.24455536901950836], "agreement_weight": 0.01118561252951622, "orthogonal_weight": 0.285865843296051}, {"head": 3, "memory_to_active": 0.01581493392586708, "contribution_to_active": 0.43386608362197876, "mix_mean": 0.3686929941177368, "mix_low": 0.005112648010253906, "mix_high": 0.00020122528076171875, "read_cosine": 0.6118702292442322, "beta_mean": 0.00013385737838689238, "nonfinite": 0.0, "weighted_memory_to_active": 0.00653511006385088, "mix_std": 0.2773636281490326, "beta_std": 0.0003454498655628413, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004960240330547094, "evidence_weights": [-0.2177068144083023, -0.25662440061569214, 0.33171939849853516], "agreement_weight": 0.027393003925681114, "orthogonal_weight": -0.9577505588531494}, {"head": 4, "memory_to_active": 0.08648781478404999, "contribution_to_active": 0.6264045834541321, "mix_mean": 0.37923139333724976, "mix_low": 0.009275436401367188, "mix_high": 0.0006074905395507812, "read_cosine": 0.7163479924201965, "beta_mean": 0.21898752450942993, "nonfinite": 0.0, "weighted_memory_to_active": 0.045870207250118256, "mix_std": 0.30613061785697937, "beta_std": 0.24127832055091858, "beta_low": 0.1744384765625, "beta_high": 0.0, "orthogonal_to_active": 0.005384746473282576, "evidence_weights": [-0.21385441720485687, -0.2559700608253479, 0.01819678023457527], "agreement_weight": 0.012360941618680954, "orthogonal_weight": 0.22261668741703033}, {"head": 5, "memory_to_active": 0.02313658967614174, "contribution_to_active": 0.5279150009155273, "mix_mean": 0.5593885183334351, "mix_low": 0.010087966918945312, "mix_high": 0.007304191589355469, "read_cosine": 0.38486120104789734, "beta_mean": 0.0017496284563094378, "nonfinite": 0.0, "weighted_memory_to_active": 0.015566255897283554, "mix_std": 0.2796967029571533, "beta_std": 0.012731922790408134, "beta_low": 0.97314453125, "beta_high": 0.0, "orthogonal_to_active": 0.008438793942332268, "evidence_weights": [-0.3273073732852936, -0.31162720918655396, 0.7178400158882141], "agreement_weight": 0.24821604788303375, "orthogonal_weight": 0.7133906483650208}]}]}, {"step": 33000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8195876279987715, "memory_to_active": 0.866606593132019, "contribution_to_active": 0.5241328477859497, "mix_mean": 0.37136760354042053, "mix_low": 0.15115831792354584, "mix_high": 0.008489708416163921, "read_cosine": 0.3745032846927643, "beta_mean": 0.07895847409963608, "nonfinite": 0.0, "weighted_memory_to_active": 0.21313993632793427, "mix_std": 0.2178342193365097, "beta_std": 0.06745447218418121, "beta_low": 0.507645308971405, "beta_high": 2.47955322265625e-05, "orthogonal_to_active": 0.05524358153343201, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1164369583129883, "normal_token_accuracy": 0.4282536208629608, "state_active": 0.060467515140771866, "state_memory": 0.01901380531489849, "state_peak": 103.63964080810547, "growth_active": 1.1212501525878906, "growth_memory": 1.3777449131011963, "output_influence": 0.37605321407318115, "residual_influence": 0.12193150818347931, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2499027252197266, "no_memory_loss_delta": 0.485848993062973, "scrambled_loss_delta": 0.12513911724090576, "memory_only_loss_delta": 2.4734458923339844, "no_evidence_loss_delta": 0.27885547280311584, "no_agreement_loss_delta": 0.01780274137854576, "no_orthogonal_loss_delta": -0.0005769846029579639}, {"index": 94, "tokens": 256, "normal_loss": 3.15140700340271, "no_memory_loss_delta": 0.48489677906036377, "scrambled_loss_delta": 0.4219083786010742, "memory_only_loss_delta": 2.6595418453216553, "no_evidence_loss_delta": 0.2574446499347687, "no_agreement_loss_delta": 0.031541597098112106, "no_orthogonal_loss_delta": 0.018218882381916046}, {"index": 188, "tokens": 256, "normal_loss": 3.4653801918029785, "no_memory_loss_delta": 0.4527386426925659, "scrambled_loss_delta": 0.1577168107032776, "memory_only_loss_delta": 2.436861276626587, "no_evidence_loss_delta": 0.24980293214321136, "no_agreement_loss_delta": 0.027288828045129776, "no_orthogonal_loss_delta": 0.0169762521982193}, {"index": 283, "tokens": 131, "normal_loss": 2.105377197265625, "no_memory_loss_delta": 0.24776843190193176, "scrambled_loss_delta": 0.14385733008384705, "memory_only_loss_delta": 2.5974857807159424, "no_evidence_loss_delta": 0.057443924248218536, "no_agreement_loss_delta": -0.0014076032675802708, "no_orthogonal_loss_delta": 0.004030105657875538}], "agreement_weight_abs_mean": 0.15469928085803986, "agreement_weight_abs_max": 3.754399538040161, "agreement_weight_mean": 0.1336759328842163, "orthogonal_weight_abs_mean": 0.6935334205627441, "orthogonal_weight_abs_max": 6.014430046081543, "orthogonal_weight_mean": -0.15230655670166016, "no_memory_loss": 3.557894229888916, "no_memory_loss_delta": 0.44145679473876953, "no_memory_token_accuracy_gain": 0.05228031054139137, "no_memory_token_win_fraction": 0.6429365873336792, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 35.690112471496654, "scrambled_loss": 3.3380894660949707, "scrambled_loss_delta": 0.2216518223285675, "scrambled_token_accuracy_gain": 0.01223581749945879, "scrambled_token_win_fraction": 0.6095662117004395, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.880572626657738, "memory_only_loss": 5.6505327224731445, "memory_only_loss_delta": 2.534095525741577, "memory_only_token_accuracy_gain": 0.24916574358940125, "memory_only_token_win_fraction": 0.882091224193573, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 92.0666558119775, "no_evidence_loss": 3.348658800125122, "no_evidence_loss_delta": 0.23222196102142334, "no_evidence_token_accuracy_gain": 0.017797552049160004, "no_evidence_token_win_fraction": 0.5928809642791748, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 20.722986029330684, "no_agreement_loss": 3.138054132461548, "no_agreement_loss_delta": 0.021617010235786438, "no_agreement_token_accuracy_gain": -0.007786429487168789, "no_agreement_token_win_fraction": 0.4749721884727478, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.138503719795746, "no_orthogonal_loss": 3.126882553100586, "no_orthogonal_loss_delta": 0.010445150546729565, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5361512899398804, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.039078939652168, "short_no_memory_loss_delta": 0.4015885591506958, "long_context_memory_gain": 0.03986823558807373, "normal_context_loss_gain": 0.10998531430959702, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.3006303310394287, "contribution_to_active": 1.0684925317764282, "mix_mean": 0.10160423815250397, "mix_low": 0.5412861704826355, "mix_high": 0.0044566793367266655, "read_cosine": 0.4527794122695923, "beta_mean": 0.25345292687416077, "nonfinite": 0.0, "weighted_memory_to_active": 1.1306419372558594, "mix_std": 0.088585264980793, "beta_std": 0.15433014929294586, "beta_low": 0.0239664725959301, "beta_high": 0.0, "orthogonal_to_active": 0.07047531008720398, "output_influence": 0.1822883039712906, "residual_influence": 0.21547940373420715, "state_active": 0.06712035834789276, "state_memory": 0.046634696424007416, "state_peak": 49.01784896850586, "growth_active": 1.212550401687622, "growth_memory": 1.2569524049758911, "heads": [{"head": 0, "memory_to_active": 2.3647303581237793, "contribution_to_active": 0.42357683181762695, "mix_mean": 0.01916372776031494, "mix_low": 0.6126956939697266, "mix_high": 0.0, "read_cosine": 0.2669908106327057, "beta_mean": 0.18737177550792694, "nonfinite": 0.0, "weighted_memory_to_active": 0.5144888758659363, "mix_std": 0.05681142210960388, "beta_std": 0.10021083056926727, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.038363922387361526, "evidence_weights": [-0.023685898631811142, -0.07239720225334167, -0.1149580329656601], "agreement_weight": -0.11407668888568878, "orthogonal_weight": -0.08246151357889175}, {"head": 1, "memory_to_active": 5.343756675720215, "contribution_to_active": 2.0072193145751953, "mix_mean": 0.01598205789923668, "mix_low": 0.662266731262207, "mix_high": 0.0, "read_cosine": 0.5179339051246643, "beta_mean": 0.45153266191482544, "nonfinite": 0.0, "weighted_memory_to_active": 2.35261869430542, "mix_std": 0.05112741142511368, "beta_std": 0.20275288820266724, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11765670776367188, "evidence_weights": [0.026677396148443222, -0.09604921191930771, 0.36358174681663513], "agreement_weight": 0.030132044106721878, "orthogonal_weight": -0.09882714599370956}, {"head": 2, "memory_to_active": 2.6680755615234375, "contribution_to_active": 0.5602567791938782, "mix_mean": 0.014551748521625996, "mix_low": 0.6617021560668945, "mix_high": 0.0, "read_cosine": 0.3140663802623749, "beta_mean": 0.3401365876197815, "nonfinite": 0.0, "weighted_memory_to_active": 0.6594258546829224, "mix_std": 0.03465520963072777, "beta_std": 0.23738110065460205, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06669819355010986, "evidence_weights": [0.054041601717472076, 0.06676910817623138, -0.010973225347697735], "agreement_weight": 0.0005217912257649004, "orthogonal_weight": -0.19586677849292755}, {"head": 3, "memory_to_active": 6.670623779296875, "contribution_to_active": 2.941053628921509, "mix_mean": 0.037298623472452164, "mix_low": 0.47165870666503906, "mix_high": 0.0, "read_cosine": 0.34805893898010254, "beta_mean": 0.3330405354499817, "nonfinite": 0.0, "weighted_memory_to_active": 3.1449990272521973, "mix_std": 0.0857057273387909, "beta_std": 0.24415889382362366, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17542368173599243, "evidence_weights": [-0.05214149132370949, -0.07124365866184235, -0.1362440586090088], "agreement_weight": -0.01612250693142414, "orthogonal_weight": -0.06602640450000763}, {"head": 4, "memory_to_active": 2.6464688777923584, "contribution_to_active": 0.07500802725553513, "mix_mean": 0.006700548809021711, "mix_low": 0.8210020065307617, "mix_high": 0.0, "read_cosine": 0.3437342047691345, "beta_mean": 0.17377471923828125, "nonfinite": 0.0, "weighted_memory_to_active": 0.06001020595431328, "mix_std": 0.014019561931490898, "beta_std": 0.11377277970314026, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006049515679478645, "evidence_weights": [0.12186068296432495, 0.07151337713003159, -0.37324780225753784], "agreement_weight": -0.1158931776881218, "orthogonal_weight": 0.10119810700416565}, {"head": 5, "memory_to_active": 0.11012539267539978, "contribution_to_active": 0.40384015440940857, "mix_mean": 0.5159286260604858, "mix_low": 0.01839160919189453, "mix_high": 0.026740074157714844, "read_cosine": 0.9258922934532166, "beta_mean": 0.034861233085393906, "nonfinite": 0.0, "weighted_memory_to_active": 0.05230836570262909, "mix_std": 0.289192259311676, "beta_std": 0.027704328298568726, "beta_low": 0.143798828125, "beta_high": 0.0, "orthogonal_to_active": 0.01865982636809349, "evidence_weights": [-0.23497967422008514, -0.23993895947933197, 0.2955770790576935], "agreement_weight": 0.2519880533218384, "orthogonal_weight": -0.6677637100219727}]}, {"layer": 1, "memory_to_active": 2.2030222415924072, "contribution_to_active": 0.4905690550804138, "mix_mean": 0.1842649281024933, "mix_low": 0.3688799738883972, "mix_high": 0.0010835330467671156, "read_cosine": 0.3286811113357544, "beta_mean": 0.18557003140449524, "nonfinite": 0.0, "weighted_memory_to_active": 0.35938534140586853, "mix_std": 0.12104949355125427, "beta_std": 0.12828722596168518, "beta_low": 0.2452087551355362, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.048596709966659546, "output_influence": 0.18835102021694183, "residual_influence": 0.046496398746967316, "state_active": 0.034317128360271454, "state_memory": 0.03633595257997513, "state_peak": 31.1519775390625, "growth_active": 1.1254665851593018, "growth_memory": 1.5043705701828003, "heads": [{"head": 0, "memory_to_active": 0.03385460004210472, "contribution_to_active": 0.4377926290035248, "mix_mean": 0.42873334884643555, "mix_low": 0.0119781494140625, "mix_high": 0.0006866455078125, "read_cosine": 0.028781285509467125, "beta_mean": 0.0002509870391804725, "nonfinite": 0.0, "weighted_memory_to_active": 0.018336955457925797, "mix_std": 0.28368711471557617, "beta_std": 0.00032209287746809423, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.017565662041306496, "evidence_weights": [-0.18352611362934113, -0.23884420096874237, 0.06306681036949158], "agreement_weight": 0.15248778462409973, "orthogonal_weight": -1.0379078388214111}, {"head": 1, "memory_to_active": 2.8711295127868652, "contribution_to_active": 0.2536839246749878, "mix_mean": 0.022471243515610695, "mix_low": 0.48763179779052734, "mix_high": 6.67572021484375e-06, "read_cosine": 0.46073147654533386, "beta_mean": 0.3013501763343811, "nonfinite": 0.0, "weighted_memory_to_active": 0.277291864156723, "mix_std": 0.05029991641640663, "beta_std": 0.19822102785110474, "beta_low": 0.000732421875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.015208417549729347, "evidence_weights": [0.010732597671449184, -0.02450128085911274, -0.030208302661776543], "agreement_weight": -0.008008378557860851, "orthogonal_weight": 0.1245763748884201}, {"head": 2, "memory_to_active": 0.31685328483581543, "contribution_to_active": 0.5648603439331055, "mix_mean": 0.589556097984314, "mix_low": 0.001972198486328125, "mix_high": 0.005805015563964844, "read_cosine": 0.7111915946006775, "beta_mean": 0.019532183185219765, "nonfinite": 0.0, "weighted_memory_to_active": 0.2621375322341919, "mix_std": 0.2533210217952728, "beta_std": 0.018163515254855156, "beta_low": 0.41552734375, "beta_high": 0.0, "orthogonal_to_active": 0.053701527416706085, "evidence_weights": [-0.2982053756713867, -0.3665235638618469, 0.3762260377407074], "agreement_weight": 0.03382503241300583, "orthogonal_weight": -0.36037689447402954}, {"head": 3, "memory_to_active": 3.059896945953369, "contribution_to_active": 0.13127464056015015, "mix_mean": 0.017908629029989243, "mix_low": 0.6472177505493164, "mix_high": 0.0, "read_cosine": 0.2505207359790802, "beta_mean": 0.3885645568370819, "nonfinite": 0.0, "weighted_memory_to_active": 0.20478153228759766, "mix_std": 0.04843021184206009, "beta_std": 0.2567794919013977, "beta_low": 0.0264892578125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.06373105943202972, "evidence_weights": [0.045186564326286316, -0.001030282466672361, 0.13572679460048676], "agreement_weight": -0.023874716833233833, "orthogonal_weight": -0.3673974871635437}, {"head": 4, "memory_to_active": 3.3124608993530273, "contribution_to_active": 1.4792826175689697, "mix_mean": 0.03794547915458679, "mix_low": 0.2715110778808594, "mix_high": 2.86102294921875e-06, "read_cosine": 0.3127105236053467, "beta_mean": 0.15468111634254456, "nonfinite": 0.0, "weighted_memory_to_active": 1.3314471244812012, "mix_std": 0.06672308593988419, "beta_std": 0.10101248323917389, "beta_low": 0.00714111328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.13365048170089722, "evidence_weights": [-0.041902393102645874, 0.00863702967762947, 0.016829662024974823], "agreement_weight": -0.02228483185172081, "orthogonal_weight": 0.13125598430633545}, {"head": 5, "memory_to_active": 3.6239380836486816, "contribution_to_active": 0.07652021944522858, "mix_mean": 0.008974799886345863, "mix_low": 0.79296875, "mix_high": 0.0, "read_cosine": 0.2081509381532669, "beta_mean": 0.24904100596904755, "nonfinite": 0.0, "weighted_memory_to_active": 0.06231694668531418, "mix_std": 0.0238355603069067, "beta_std": 0.19522474706172943, "beta_low": 0.0213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.007723087910562754, "evidence_weights": [0.15159988403320312, 0.07133258134126663, 0.2942064702510834], "agreement_weight": -0.14625459909439087, "orthogonal_weight": 0.12293287366628647}]}, {"layer": 2, "memory_to_active": 0.9951757192611694, "contribution_to_active": 0.3288322687149048, "mix_mean": 0.35414695739746094, "mix_low": 0.25304922461509705, "mix_high": 0.0029915175400674343, "read_cosine": 0.3345772624015808, "beta_mean": 0.09023874998092651, "nonfinite": 0.0, "weighted_memory_to_active": 0.06618811190128326, "mix_std": 0.19500568509101868, "beta_std": 0.07053686678409576, "beta_low": 0.5530904531478882, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.016642285510897636, "output_influence": 0.32354316115379333, "residual_influence": 0.08209706842899323, "state_active": 0.0351775586605072, "state_memory": 0.01753327064216137, "state_peak": 34.79658889770508, "growth_active": 1.0035282373428345, "growth_memory": 1.594361662864685, "heads": [{"head": 0, "memory_to_active": 0.007083089090883732, "contribution_to_active": 0.3247763216495514, "mix_mean": 0.5325188040733337, "mix_low": 0.00226593017578125, "mix_high": 0.004666328430175781, "read_cosine": 0.1883787214756012, "beta_mean": 5.274875002214685e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004037927370518446, "mix_std": 0.2902849316596985, "beta_std": 7.74516083765775e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015666119288653135, "evidence_weights": [-0.31270506978034973, -0.19383174180984497, 0.11219403147697449], "agreement_weight": 0.2226106822490692, "orthogonal_weight": -0.408692866563797}, {"head": 1, "memory_to_active": 2.8353376388549805, "contribution_to_active": 0.05026884377002716, "mix_mean": 0.005930324085056782, "mix_low": 0.8731718063354492, "mix_high": 9.5367431640625e-07, "read_cosine": 0.20167796313762665, "beta_mean": 0.24038633704185486, "nonfinite": 0.0, "weighted_memory_to_active": 0.03749625384807587, "mix_std": 0.014646617695689201, "beta_std": 0.1442481279373169, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.015443642623722553, "evidence_weights": [0.10066813975572586, 0.09741070121526718, 0.5124354958534241], "agreement_weight": 0.0400577075779438, "orthogonal_weight": 0.4322204291820526}, {"head": 2, "memory_to_active": 0.014126638881862164, "contribution_to_active": 0.42338305711746216, "mix_mean": 0.4878352880477905, "mix_low": 0.0016574859619140625, "mix_high": 0.0016040802001953125, "read_cosine": 0.2272687405347824, "beta_mean": 2.7951937227044255e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005887303035706282, "mix_std": 0.2811328172683716, "beta_std": 4.405081563163549e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014780319761484861, "evidence_weights": [-0.23262865841388702, -0.27088671922683716, 0.44906654953956604], "agreement_weight": 0.014638189226388931, "orthogonal_weight": -0.29721760749816895}, {"head": 3, "memory_to_active": 0.19700589776039124, "contribution_to_active": 0.4886015057563782, "mix_mean": 0.4822258949279785, "mix_low": 0.0007390975952148438, "mix_high": 0.0050144195556640625, "read_cosine": 0.6948955655097961, "beta_mean": 0.03630220144987106, "nonfinite": 0.0, "weighted_memory_to_active": 0.1076122596859932, "mix_std": 0.26637059450149536, "beta_std": 0.04693484678864479, "beta_low": 0.29229736328125, "beta_high": 0.0, "orthogonal_to_active": 0.06569443643093109, "evidence_weights": [-0.17656521499156952, -0.2747035622596741, 0.8563718795776367], "agreement_weight": 0.061943214386701584, "orthogonal_weight": -0.8830315470695496}, {"head": 4, "memory_to_active": 2.9000566005706787, "contribution_to_active": 0.19369541108608246, "mix_mean": 0.01718284748494625, "mix_low": 0.6387128829956055, "mix_high": 0.0, "read_cosine": 0.21334044635295868, "beta_mean": 0.26461032032966614, "nonfinite": 0.0, "weighted_memory_to_active": 0.23231811821460724, "mix_std": 0.0453348234295845, "beta_std": 0.23185327649116516, "beta_low": 0.0262451171875, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.015221524983644485, "evidence_weights": [0.02839004434645176, 0.020546305924654007, 0.07281754165887833], "agreement_weight": 0.023125289008021355, "orthogonal_weight": -0.07363791763782501}, {"head": 5, "memory_to_active": 0.017444374039769173, "contribution_to_active": 0.49226826429367065, "mix_mean": 0.5991885662078857, "mix_low": 0.0017480850219726562, "mix_high": 0.006663322448730469, "read_cosine": 0.4819021224975586, "beta_mean": 5.289981345413253e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009776812978088856, "mix_std": 0.2722643315792084, "beta_std": 6.348197348415852e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004494699533097446, "evidence_weights": [-0.2640126943588257, -0.28378456830978394, 0.8861091136932373], "agreement_weight": 0.0736459270119667, "orthogonal_weight": 0.06814900040626526}]}, {"layer": 3, "memory_to_active": 0.8409391641616821, "contribution_to_active": 0.4013398587703705, "mix_mean": 0.3517686128616333, "mix_low": 0.1465449333190918, "mix_high": 0.00836690329015255, "read_cosine": 0.23400619626045227, "beta_mean": 0.04828773811459541, "nonfinite": 0.0, "weighted_memory_to_active": 0.17227229475975037, "mix_std": 0.24028170108795166, "beta_std": 0.04575216397643089, "beta_low": 0.50653076171875, "beta_high": 0.0, "orthogonal_to_active": 0.07354271411895752, "output_influence": 0.32029667496681213, "residual_influence": 0.0661945790052414, "state_active": 0.022262107580900192, "state_memory": 0.011444020085036755, "state_peak": 17.866281509399414, "growth_active": 0.9751445055007935, "growth_memory": 1.9712779521942139, "heads": [{"head": 0, "memory_to_active": 0.5168242454528809, "contribution_to_active": 0.5240969061851501, "mix_mean": 0.46086299419403076, "mix_low": 0.018238067626953125, "mix_high": 0.014496803283691406, "read_cosine": 0.22258751094341278, "beta_mean": 0.016545768827199936, "nonfinite": 0.0, "weighted_memory_to_active": 0.27505066990852356, "mix_std": 0.3246346116065979, "beta_std": 0.02827758900821209, "beta_low": 0.54644775390625, "beta_high": 0.0, "orthogonal_to_active": 0.1858745813369751, "evidence_weights": [-0.18019789457321167, -0.21305575966835022, 0.680815577507019], "agreement_weight": 0.14021819829940796, "orthogonal_weight": -0.7084528207778931}, {"head": 1, "memory_to_active": 0.020488832145929337, "contribution_to_active": 0.3763091564178467, "mix_mean": 0.45477163791656494, "mix_low": 0.01311492919921875, "mix_high": 0.0057373046875, "read_cosine": 0.1665276139974594, "beta_mean": 0.00011064104182878509, "nonfinite": 0.0, "weighted_memory_to_active": 0.010014979168772697, "mix_std": 0.31329718232154846, "beta_std": 0.00022990316210780293, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00549551984295249, "evidence_weights": [-0.2157912254333496, -0.18690551817417145, 0.5563092231750488], "agreement_weight": 0.3470161259174347, "orthogonal_weight": -0.5894067883491516}, {"head": 2, "memory_to_active": 2.6017541885375977, "contribution_to_active": 0.328422486782074, "mix_mean": 0.043560005724430084, "mix_low": 0.38416481018066406, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21708166599273682, "beta_mean": 0.19270941615104675, "nonfinite": 0.0, "weighted_memory_to_active": 0.3908607065677643, "mix_std": 0.09848727285861969, "beta_std": 0.14945189654827118, "beta_low": 0.0155029296875, "beta_high": 0.0, "orthogonal_to_active": 0.0016466062515974045, "evidence_weights": [-0.07062973082065582, 0.015997393056750298, -0.021264443174004555], "agreement_weight": -0.023037424311041832, "orthogonal_weight": -0.005446059163659811}, {"head": 3, "memory_to_active": 0.03907007351517677, "contribution_to_active": 0.49928855895996094, "mix_mean": 0.5404412746429443, "mix_low": 0.011948585510253906, "mix_high": 0.0130615234375, "read_cosine": 0.3080707788467407, "beta_mean": 0.00015627863467670977, "nonfinite": 0.0, "weighted_memory_to_active": 0.02524266578257084, "mix_std": 0.30736151337623596, "beta_std": 0.0004595645877998322, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.006714596413075924, "evidence_weights": [-0.21180371940135956, -0.2225559949874878, 1.9418834447860718], "agreement_weight": 1.41290283203125, "orthogonal_weight": -0.3566460907459259}, {"head": 4, "memory_to_active": 1.5735267400741577, "contribution_to_active": 0.057770587503910065, "mix_mean": 0.04586578533053398, "mix_low": 0.43889904022216797, "mix_high": 0.0, "read_cosine": 0.16477717459201813, "beta_mean": 0.049577318131923676, "nonfinite": 0.0, "weighted_memory_to_active": 0.13227137923240662, "mix_std": 0.09578414261341095, "beta_std": 0.06689201295375824, "beta_low": 0.23876953125, "beta_high": 0.0, "orthogonal_to_active": 0.09934914857149124, "evidence_weights": [0.07363667339086533, 0.015576529316604137, 0.16834162175655365], "agreement_weight": -0.024361155927181244, "orthogonal_weight": -0.7655060887336731}, {"head": 5, "memory_to_active": 0.29397064447402954, "contribution_to_active": 0.622151255607605, "mix_mean": 0.56510990858078, "mix_low": 0.012904167175292969, "mix_high": 0.016904830932617188, "read_cosine": 0.3249923288822174, "beta_mean": 0.030626991763710976, "nonfinite": 0.0, "weighted_memory_to_active": 0.2001933455467224, "mix_std": 0.30212557315826416, "beta_std": 0.029202034696936607, "beta_low": 0.23870849609375, "beta_high": 0.0, "orthogonal_to_active": 0.1421757936477661, "evidence_weights": [-0.21939051151275635, -0.2546992301940918, 0.48358333110809326], "agreement_weight": 0.13407273590564728, "orthogonal_weight": -0.8009791374206543}]}, {"layer": 4, "memory_to_active": 1.144368290901184, "contribution_to_active": 0.42519769072532654, "mix_mean": 0.34154707193374634, "mix_low": 0.2755497395992279, "mix_high": 0.00605201767757535, "read_cosine": 0.3589552044868469, "beta_mean": 0.12100189924240112, "nonfinite": 0.0, "weighted_memory_to_active": 0.13122251629829407, "mix_std": 0.1562289446592331, "beta_std": 0.11174538731575012, "beta_low": 0.3711446523666382, "beta_high": 0.0, "orthogonal_to_active": 0.03532803803682327, "output_influence": 0.38314321637153625, "residual_influence": 0.10154804587364197, "state_active": 0.0355755090713501, "state_memory": 0.01873277686536312, "state_peak": 37.78136444091797, "growth_active": 1.1948761940002441, "growth_memory": 1.5269912481307983, "heads": [{"head": 0, "memory_to_active": 0.22064244747161865, "contribution_to_active": 0.7075229287147522, "mix_mean": 0.6345378756523132, "mix_low": 0.010820388793945312, "mix_high": 0.013806343078613281, "read_cosine": 0.47296977043151855, "beta_mean": 0.2473062127828598, "nonfinite": 0.0, "weighted_memory_to_active": 0.17000432312488556, "mix_std": 0.33873459696769714, "beta_std": 0.2616172134876251, "beta_low": 0.2183837890625, "beta_high": 0.0, "orthogonal_to_active": 0.05679919570684433, "evidence_weights": [-0.2635604441165924, -0.3600664734840393, 0.24639594554901123], "agreement_weight": 0.020945986732840538, "orthogonal_weight": 0.39884284138679504}, {"head": 1, "memory_to_active": 0.08135905116796494, "contribution_to_active": 0.5972795486450195, "mix_mean": 0.6302724480628967, "mix_low": 4.38690185546875e-05, "mix_high": 0.006524085998535156, "read_cosine": 0.701770007610321, "beta_mean": 0.0015588400419801474, "nonfinite": 0.0, "weighted_memory_to_active": 0.05604308843612671, "mix_std": 0.25049856305122375, "beta_std": 0.006924258545041084, "beta_low": 0.97222900390625, "beta_high": 0.0, "orthogonal_to_active": 0.031651731580495834, "evidence_weights": [-0.23628108203411102, -0.3164404034614563, 0.733049213886261], "agreement_weight": 0.1399557888507843, "orthogonal_weight": -0.8403816819190979}, {"head": 2, "memory_to_active": 2.287729024887085, "contribution_to_active": 0.19368544220924377, "mix_mean": 0.0181216299533844, "mix_low": 0.5863542556762695, "mix_high": 1.9073486328125e-06, "read_cosine": 0.2568972408771515, "beta_mean": 0.12436710298061371, "nonfinite": 0.0, "weighted_memory_to_active": 0.19363445043563843, "mix_std": 0.03583782538771629, "beta_std": 0.10329803824424744, "beta_low": 0.00360107421875, "beta_high": 0.0, "orthogonal_to_active": 0.012901666574180126, "evidence_weights": [0.05692855268716812, 0.03494451195001602, -0.01631324738264084], "agreement_weight": 0.11491977423429489, "orthogonal_weight": -0.10131645202636719}, {"head": 3, "memory_to_active": 1.908271312713623, "contribution_to_active": 0.15798190236091614, "mix_mean": 0.030390525236725807, "mix_low": 0.422576904296875, "mix_high": 0.0, "read_cosine": 0.22127385437488556, "beta_mean": 0.13844424486160278, "nonfinite": 0.0, "weighted_memory_to_active": 0.21624982357025146, "mix_std": 0.055596232414245605, "beta_std": 0.1280224472284317, "beta_low": 0.03216552734375, "beta_high": 0.0, "orthogonal_to_active": 0.013147346675395966, "evidence_weights": [-0.023762280121445656, 0.059229057282209396, -0.23628628253936768], "agreement_weight": 0.06340871006250381, "orthogonal_weight": -0.08074318617582321}, {"head": 4, "memory_to_active": 0.01730784773826599, "contribution_to_active": 0.6525670289993286, "mix_mean": 0.7214431762695312, "mix_low": 0.0001316070556640625, "mix_high": 0.015976905822753906, "read_cosine": 0.20181533694267273, "beta_mean": 3.2530657335883006e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012901815585792065, "mix_std": 0.224927619099617, "beta_std": 0.00011622685269685462, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0027287788689136505, "evidence_weights": [-0.31490492820739746, -0.33312010765075684, 0.7278534770011902], "agreement_weight": 0.24942539632320404, "orthogonal_weight": -0.23225973546504974}, {"head": 5, "memory_to_active": 2.3508999347686768, "contribution_to_active": 0.24214915931224823, "mix_mean": 0.014516719616949558, "mix_low": 0.6333713531494141, "mix_high": 2.86102294921875e-06, "read_cosine": 0.2990051209926605, "beta_mean": 0.2143024504184723, "nonfinite": 0.0, "weighted_memory_to_active": 0.13850156962871552, "mix_std": 0.03177884593605995, "beta_std": 0.17049412429332733, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.09473951160907745, "evidence_weights": [0.0417252779006958, -0.04028130695223808, -0.23471760749816895], "agreement_weight": -0.05691729858517647, "orthogonal_weight": 0.8374651074409485}]}, {"layer": 5, "memory_to_active": 0.5591192841529846, "contribution_to_active": 0.4957502484321594, "mix_mean": 0.451340913772583, "mix_low": 0.06947676837444305, "mix_high": 0.015635332092642784, "read_cosine": 0.15066543221473694, "beta_mean": 0.05776846781373024, "nonfinite": 0.0, "weighted_memory_to_active": 0.15759462118148804, "mix_std": 0.25528043508529663, "beta_std": 0.07500806450843811, "beta_low": 0.6373189687728882, "beta_high": 0.0, "orthogonal_to_active": 0.05637092888355255, "output_influence": 0.48893558979034424, "residual_influence": 0.12443859875202179, "state_active": 0.04579047113656998, "state_memory": 0.01197320967912674, "state_peak": 61.476192474365234, "growth_active": 1.1234378814697266, "growth_memory": 1.2832573652267456, "heads": [{"head": 0, "memory_to_active": 0.7051578164100647, "contribution_to_active": 0.39458853006362915, "mix_mean": 0.2323094606399536, "mix_low": 0.07106876373291016, "mix_high": 0.000339508056640625, "read_cosine": -0.1238345131278038, "beta_mean": 0.042019568383693695, "nonfinite": 0.0, "weighted_memory_to_active": 0.1656562238931656, "mix_std": 0.24273651838302612, "beta_std": 0.0755997821688652, "beta_low": 0.37451171875, "beta_high": 0.0, "orthogonal_to_active": 0.03607960417866707, "evidence_weights": [-0.08956938982009888, -0.12337435781955719, -0.2615167796611786], "agreement_weight": -0.014725646935403347, "orthogonal_weight": -0.22067119181156158}, {"head": 1, "memory_to_active": 0.020675228908658028, "contribution_to_active": 0.5446124076843262, "mix_mean": 0.6385655403137207, "mix_low": 0.0027713775634765625, "mix_high": 0.02656841278076172, "read_cosine": 0.1842527836561203, "beta_mean": 4.71938619739376e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011297734454274178, "mix_std": 0.2932511866092682, "beta_std": 9.449583012610674e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010581622831523418, "evidence_weights": [-0.23849895596504211, -0.2760782837867737, 0.8550757169723511], "agreement_weight": 0.4938398003578186, "orthogonal_weight": -0.9708394408226013}, {"head": 2, "memory_to_active": 0.019225791096687317, "contribution_to_active": 0.5587372183799744, "mix_mean": 0.6252237558364868, "mix_low": 0.00264739990234375, "mix_high": 0.017154693603515625, "read_cosine": 0.0029644593596458435, "beta_mean": 0.0002283802896272391, "nonfinite": 0.0, "weighted_memory_to_active": 0.012951144017279148, "mix_std": 0.2755547761917114, "beta_std": 0.0003426047333050519, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003033116925507784, "evidence_weights": [-0.2501237392425537, -0.34516850113868713, 0.8832300901412964], "agreement_weight": 0.011920959688723087, "orthogonal_weight": -0.27475714683532715}, {"head": 3, "memory_to_active": 2.0544095039367676, "contribution_to_active": 0.24012084305286407, "mix_mean": 0.09665222465991974, "mix_low": 0.14918041229248047, "mix_high": 0.0, "read_cosine": 0.14262591302394867, "beta_mean": 0.1187916249036789, "nonfinite": 0.0, "weighted_memory_to_active": 0.37959811091423035, "mix_std": 0.13729512691497803, "beta_std": 0.16002409160137177, "beta_low": 0.1783447265625, "beta_high": 0.0, "orthogonal_to_active": 0.14732760190963745, "evidence_weights": [-0.06637060642242432, -0.09720275551080704, -0.07672775536775589], "agreement_weight": 0.04727976769208908, "orthogonal_weight": -0.4555146396160126}, {"head": 4, "memory_to_active": 0.4497646987438202, "contribution_to_active": 0.5988141894340515, "mix_mean": 0.3359149992465973, "mix_low": 0.18842792510986328, "mix_high": 0.009058952331542969, "read_cosine": 0.5474212765693665, "beta_mean": 0.18518948554992676, "nonfinite": 0.0, "weighted_memory_to_active": 0.30582764744758606, "mix_std": 0.34690773487091064, "beta_std": 0.21227367222309113, "beta_low": 0.27618408203125, "beta_high": 0.0, "orthogonal_to_active": 0.09492731094360352, "evidence_weights": [-0.11915204674005508, -0.2929379343986511, -0.07403557747602463], "agreement_weight": 0.04622028023004532, "orthogonal_weight": 0.4724465310573578}, {"head": 5, "memory_to_active": 0.1054825633764267, "contribution_to_active": 0.6376281380653381, "mix_mean": 0.7793793678283691, "mix_low": 0.0027647018432617188, "mix_high": 0.04069042205810547, "read_cosine": 0.1505625993013382, "beta_mean": 0.0003345427685417235, "nonfinite": 0.0, "weighted_memory_to_active": 0.07023680955171585, "mix_std": 0.2359371781349182, "beta_std": 0.0017137294635176659, "beta_low": 0.994873046875, "beta_high": 0.0, "orthogonal_to_active": 0.04627632722258568, "evidence_weights": [-0.3248579204082489, -0.3232128322124481, 2.031329393386841], "agreement_weight": 0.3829251527786255, "orthogonal_weight": -0.6933879852294922}]}, {"layer": 6, "memory_to_active": 0.3559403121471405, "contribution_to_active": 0.5916529297828674, "mix_mean": 0.5517399311065674, "mix_low": 0.07782014459371567, "mix_high": 0.025662105530500412, "read_cosine": 0.28516554832458496, "beta_mean": 0.06934741139411926, "nonfinite": 0.0, "weighted_memory_to_active": 0.16475898027420044, "mix_std": 0.24154753983020782, "beta_std": 0.0774570181965828, "beta_low": 0.5001017451286316, "beta_high": 0.0, "orthogonal_to_active": 0.03645116090774536, "output_influence": 0.4981169104576111, "residual_influence": 0.15910165011882782, "state_active": 0.04892382025718689, "state_memory": 0.009229660965502262, "state_peak": 90.02617645263672, "growth_active": 1.2170993089675903, "growth_memory": 1.187457799911499, "heads": [{"head": 0, "memory_to_active": 0.1222638264298439, "contribution_to_active": 0.6270750164985657, "mix_mean": 0.738269567489624, "mix_low": 0.000171661376953125, "mix_high": 0.01145172119140625, "read_cosine": 0.04769997298717499, "beta_mean": 0.01618489809334278, "nonfinite": 0.0, "weighted_memory_to_active": 0.09103875607252121, "mix_std": 0.2233896106481552, "beta_std": 0.01979774981737137, "beta_low": 0.47076416015625, "beta_high": 0.0, "orthogonal_to_active": 0.07407857477664948, "evidence_weights": [-0.3127216100692749, -0.33466842770576477, 0.5465966463088989], "agreement_weight": 0.1634601503610611, "orthogonal_weight": -0.9391242265701294}, {"head": 1, "memory_to_active": 0.008633481338620186, "contribution_to_active": 0.48921310901641846, "mix_mean": 0.5319495797157288, "mix_low": 0.0011224746704101562, "mix_high": 0.009273529052734375, "read_cosine": 0.5556991696357727, "beta_mean": 1.8043454474536702e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037454930134117603, "mix_std": 0.28854525089263916, "beta_std": 3.1287490855902433e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016439344035461545, "evidence_weights": [-0.20224186778068542, -0.28745660185813904, 1.025942087173462], "agreement_weight": 0.04176348075270653, "orthogonal_weight": -0.6018334627151489}, {"head": 2, "memory_to_active": 1.0665347576141357, "contribution_to_active": 0.07468289881944656, "mix_mean": 0.026038935407996178, "mix_low": 0.4536314010620117, "mix_high": 9.5367431640625e-07, "read_cosine": 0.23067539930343628, "beta_mean": 0.055505067110061646, "nonfinite": 0.0, "weighted_memory_to_active": 0.05858583003282547, "mix_std": 0.04970239847898483, "beta_std": 0.04710130766034126, "beta_low": 0.02691650390625, "beta_high": 0.0, "orthogonal_to_active": 0.0003892759559676051, "evidence_weights": [0.0987435057759285, 0.0005498633254319429, 0.2958424389362335], "agreement_weight": 0.1284862756729126, "orthogonal_weight": 0.007983475923538208}, {"head": 3, "memory_to_active": 0.15396399796009064, "contribution_to_active": 0.6594567894935608, "mix_mean": 0.6313806176185608, "mix_low": 0.01065826416015625, "mix_high": 0.013680458068847656, "read_cosine": 0.4484647810459137, "beta_mean": 0.2147146314382553, "nonfinite": 0.0, "weighted_memory_to_active": 0.1100953221321106, "mix_std": 0.3335517346858978, "beta_std": 0.24940332770347595, "beta_low": 0.22625732421875, "beta_high": 0.0, "orthogonal_to_active": 0.06006135791540146, "evidence_weights": [-0.1725936084985733, -0.33665385842323303, 0.1725991815328598], "agreement_weight": -0.008690660819411278, "orthogonal_weight": 0.6308857202529907}, {"head": 4, "memory_to_active": 0.7439265847206116, "contribution_to_active": 1.1019313335418701, "mix_mean": 0.7476334571838379, "mix_low": 0.0011053085327148438, "mix_high": 0.11699581146240234, "read_cosine": 0.13792502880096436, "beta_mean": 0.1293572187423706, "nonfinite": 0.0, "weighted_memory_to_active": 0.6969783306121826, "mix_std": 0.2958332896232605, "beta_std": 0.14784187078475952, "beta_low": 0.2769775390625, "beta_high": 0.0, "orthogonal_to_active": 0.054366834461688995, "evidence_weights": [-0.2538987100124359, -0.5955680012702942, -0.13409021496772766], "agreement_weight": -0.015591663308441639, "orthogonal_weight": 0.09198197722434998}, {"head": 5, "memory_to_active": 0.04031917452812195, "contribution_to_active": 0.5975581407546997, "mix_mean": 0.6351674795150757, "mix_low": 0.00023174285888671875, "mix_high": 0.0025701522827148438, "read_cosine": 0.29052892327308655, "beta_mean": 0.0003045693156309426, "nonfinite": 0.0, "weighted_memory_to_active": 0.028110019862651825, "mix_std": 0.25826290249824524, "beta_std": 0.0005665782373398542, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.028166960924863815, "evidence_weights": [-0.23904810845851898, -0.3048117458820343, 0.42087697982788086], "agreement_weight": 0.13523520529270172, "orthogonal_weight": -1.079517126083374}]}, {"layer": 7, "memory_to_active": 1.687268853187561, "contribution_to_active": 0.224530890583992, "mix_mean": 0.09739124029874802, "mix_low": 0.2901884913444519, "mix_high": 0.00013415019202511758, "read_cosine": 0.24312691390514374, "beta_mean": 0.11575237661600113, "nonfinite": 0.0, "weighted_memory_to_active": 0.21608489751815796, "mix_std": 0.12275581806898117, "beta_std": 0.09767612814903259, "beta_low": 0.1875508725643158, "beta_high": 0.00019327799964230508, "orthogonal_to_active": 0.061306025832891464, "output_influence": 0.22457098960876465, "residual_influence": 0.05026492848992348, "state_active": 0.014917862601578236, "state_memory": 0.02732422947883606, "state_peak": 32.70708465576172, "growth_active": 1.1382695436477661, "growth_memory": 1.304242730140686, "heads": [{"head": 0, "memory_to_active": 2.2777938842773438, "contribution_to_active": 0.31828099489212036, "mix_mean": 0.0361725352704525, "mix_low": 0.4215536117553711, "mix_high": 0.0, "read_cosine": 0.3841632604598999, "beta_mean": 0.15208017826080322, "nonfinite": 0.0, "weighted_memory_to_active": 0.3433595895767212, "mix_std": 0.07403282076120377, "beta_std": 0.12095284461975098, "beta_low": 0.00677490234375, "beta_high": 0.000732421875, "orthogonal_to_active": 0.014772582799196243, "evidence_weights": [0.010780511423945427, -0.08497312664985657, 0.1283426433801651], "agreement_weight": -0.03604505956172943, "orthogonal_weight": 0.061480652540922165}, {"head": 1, "memory_to_active": 1.017054557800293, "contribution_to_active": 0.33312365412712097, "mix_mean": 0.17848089337348938, "mix_low": 0.16889381408691406, "mix_high": 0.0005817413330078125, "read_cosine": 0.35867393016815186, "beta_mean": 0.09797090291976929, "nonfinite": 0.0, "weighted_memory_to_active": 0.3028351664543152, "mix_std": 0.21398432552814484, "beta_std": 0.0673346072435379, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006245768629014492, "evidence_weights": [-0.19039013981819153, -0.1149250715970993, -0.21618615090847015], "agreement_weight": 0.025269730016589165, "orthogonal_weight": 0.024018844589591026}, {"head": 2, "memory_to_active": 0.930030882358551, "contribution_to_active": 0.27592021226882935, "mix_mean": 0.11151234805583954, "mix_low": 0.10744762420654297, "mix_high": 0.00019741058349609375, "read_cosine": 0.11427804827690125, "beta_mean": 0.06802685558795929, "nonfinite": 0.0, "weighted_memory_to_active": 0.21013247966766357, "mix_std": 0.15496793389320374, "beta_std": 0.05333416163921356, "beta_low": 0.05023193359375, "beta_high": 0.0, "orthogonal_to_active": 0.06449944525957108, "evidence_weights": [-0.04151897504925728, -0.12906619906425476, 0.4361037015914917], "agreement_weight": 0.021358339115977287, "orthogonal_weight": -0.3134004771709442}, {"head": 3, "memory_to_active": 4.061135768890381, "contribution_to_active": 0.09810759872198105, "mix_mean": 0.030942009761929512, "mix_low": 0.3592691421508789, "mix_high": 0.0, "read_cosine": 0.09652987122535706, "beta_mean": 0.28907668590545654, "nonfinite": 0.0, "weighted_memory_to_active": 0.29201826453208923, "mix_std": 0.04672449082136154, "beta_std": 0.24542135000228882, "beta_low": 0.012451171875, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.2358173429965973, "evidence_weights": [0.0681312307715416, 0.0018934601685032248, 0.661710798740387], "agreement_weight": 0.05791463330388069, "orthogonal_weight": -0.9358304738998413}, {"head": 4, "memory_to_active": 1.5779976844787598, "contribution_to_active": 0.08323730528354645, "mix_mean": 0.015861496329307556, "mix_low": 0.6474714279174805, "mix_high": 0.0, "read_cosine": 0.25156670808792114, "beta_mean": 0.08545425534248352, "nonfinite": 0.0, "weighted_memory_to_active": 0.06699328869581223, "mix_std": 0.03673499822616577, "beta_std": 0.09175483882427216, "beta_low": 0.08026123046875, "beta_high": 0.0, "orthogonal_to_active": 0.0066418638452887535, "evidence_weights": [0.12999694049358368, 0.04972691461443901, 0.197387233376503], "agreement_weight": 0.04458577185869217, "orthogonal_weight": -0.13020190596580505}, {"head": 5, "memory_to_active": 0.2596001923084259, "contribution_to_active": 0.23851558566093445, "mix_mean": 0.21137817203998566, "mix_low": 0.036495208740234375, "mix_high": 2.574920654296875e-05, "read_cosine": 0.25354957580566406, "beta_mean": 0.0019054091535508633, "nonfinite": 0.0, "weighted_memory_to_active": 0.08117058873176575, "mix_std": 0.21009030938148499, "beta_std": 0.0072588990442454815, "beta_low": 0.9755859375, "beta_high": 0.0, "orthogonal_to_active": 0.039859138429164886, "evidence_weights": [-0.07854460179805756, -0.12848429381847382, 0.36283230781555176], "agreement_weight": 0.08043137192726135, "orthogonal_weight": -0.6121701002120972}]}, {"layer": 8, "memory_to_active": 0.5010839700698853, "contribution_to_active": 0.5248133540153503, "mix_mean": 0.4160023033618927, "mix_low": 0.09734313189983368, "mix_high": 0.0009271304588764906, "read_cosine": 0.4851454496383667, "beta_mean": 0.03339766710996628, "nonfinite": 0.0, "weighted_memory_to_active": 0.10366109758615494, "mix_std": 0.2221769392490387, "beta_std": 0.038326263427734375, "beta_low": 0.5847371816635132, "beta_high": 0.0, "orthogonal_to_active": 0.05592114478349686, "output_influence": 0.4402223825454712, "residual_influence": 0.13577362895011902, "state_active": 0.04943777620792389, "state_memory": 0.010473744943737984, "state_peak": 34.65732192993164, "growth_active": 1.0720850229263306, "growth_memory": 1.3164114952087402, "heads": [{"head": 0, "memory_to_active": 0.04418330639600754, "contribution_to_active": 0.5414426326751709, "mix_mean": 0.5811759233474731, "mix_low": 0.00026226043701171875, "mix_high": 0.000675201416015625, "read_cosine": 0.5866613388061523, "beta_mean": 0.003315057372674346, "nonfinite": 0.0, "weighted_memory_to_active": 0.030000023543834686, "mix_std": 0.2430468499660492, "beta_std": 0.008435839787125587, "beta_low": 0.9178466796875, "beta_high": 0.0, "orthogonal_to_active": 0.0184509065002203, "evidence_weights": [-0.16940967738628387, -0.3889622092247009, 0.7181150913238525], "agreement_weight": 0.10334229469299316, "orthogonal_weight": -0.8183639049530029}, {"head": 1, "memory_to_active": 0.1522732526063919, "contribution_to_active": 0.4801698625087738, "mix_mean": 0.463059663772583, "mix_low": 0.001552581787109375, "mix_high": 0.000492095947265625, "read_cosine": 0.3274351954460144, "beta_mean": 0.003430983517318964, "nonfinite": 0.0, "weighted_memory_to_active": 0.0736604779958725, "mix_std": 0.2545635402202606, "beta_std": 0.010349161922931671, "beta_low": 0.93206787109375, "beta_high": 0.0, "orthogonal_to_active": 0.04552329331636429, "evidence_weights": [-0.19332727789878845, -0.2535421550273895, 0.6549450755119324], "agreement_weight": 0.08128369599580765, "orthogonal_weight": -0.6863778233528137}, {"head": 2, "memory_to_active": 0.024578005075454712, "contribution_to_active": 0.6392802000045776, "mix_mean": 0.6189558506011963, "mix_low": 0.0010328292846679688, "mix_high": 0.0030727386474609375, "read_cosine": 0.38619697093963623, "beta_mean": 0.00023521811817772686, "nonfinite": 0.0, "weighted_memory_to_active": 0.01726948469877243, "mix_std": 0.2360871583223343, "beta_std": 0.0037235186900943518, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.011370784603059292, "evidence_weights": [-0.22018267214298248, -0.33705058693885803, 0.8748052716255188], "agreement_weight": 0.20603221654891968, "orthogonal_weight": -0.7522480487823486}, {"head": 3, "memory_to_active": 0.134635329246521, "contribution_to_active": 0.6147952079772949, "mix_mean": 0.5446046590805054, "mix_low": 0.0028123855590820312, "mix_high": 0.0010471343994140625, "read_cosine": 0.7895949482917786, "beta_mean": 0.012333070859313011, "nonfinite": 0.0, "weighted_memory_to_active": 0.09951594471931458, "mix_std": 0.25759270787239075, "beta_std": 0.008843556977808475, "beta_low": 0.452880859375, "beta_high": 0.0, "orthogonal_to_active": 0.03163392096757889, "evidence_weights": [-0.17632514238357544, -0.2993719279766083, 0.6641753911972046], "agreement_weight": 0.1530958116054535, "orthogonal_weight": -0.6271811127662659}, {"head": 4, "memory_to_active": 2.3759231567382812, "contribution_to_active": 0.2459280639886856, "mix_mean": 0.02700921893119812, "mix_low": 0.43775081634521484, "mix_high": 0.0, "read_cosine": 0.2985205352306366, "beta_mean": 0.10665661096572876, "nonfinite": 0.0, "weighted_memory_to_active": 0.2374378740787506, "mix_std": 0.04739534109830856, "beta_std": 0.10378901660442352, "beta_low": 0.02093505859375, "beta_high": 0.0, "orthogonal_to_active": 0.007829081267118454, "evidence_weights": [0.04993638023734093, -0.02025609090924263, -0.07902268320322037], "agreement_weight": 0.04603582248091698, "orthogonal_weight": 0.035266369581222534}, {"head": 5, "memory_to_active": 0.2749108374118805, "contribution_to_active": 0.627264142036438, "mix_mean": 0.2612084746360779, "mix_low": 0.14064788818359375, "mix_high": 0.00027561187744140625, "read_cosine": 0.5224636793136597, "beta_mean": 0.07441505789756775, "nonfinite": 0.0, "weighted_memory_to_active": 0.16408270597457886, "mix_std": 0.2943759560585022, "beta_std": 0.09481646120548248, "beta_low": 0.18499755859375, "beta_high": 0.0, "orthogonal_to_active": 0.2207188755273819, "evidence_weights": [-0.06704238802194595, -0.26814335584640503, -0.3768565058708191], "agreement_weight": 0.001784356776624918, "orthogonal_weight": 1.649139165878296}]}, {"layer": 9, "memory_to_active": 1.1479564905166626, "contribution_to_active": 0.6288156509399414, "mix_mean": 0.3095284104347229, "mix_low": 0.15072521567344666, "mix_high": 0.002979119773954153, "read_cosine": 0.35717302560806274, "beta_mean": 0.0692402720451355, "nonfinite": 0.0, "weighted_memory_to_active": 0.365288645029068, "mix_std": 0.20838871598243713, "beta_std": 0.04464716464281082, "beta_low": 0.4994303584098816, "beta_high": 0.0, "orthogonal_to_active": 0.04291484132409096, "output_influence": 0.4850107431411743, "residual_influence": 0.13042056560516357, "state_active": 0.04058784246444702, "state_memory": 0.01963908225297928, "state_peak": 41.2705078125, "growth_active": 1.1684205532073975, "growth_memory": 1.3614094257354736, "heads": [{"head": 0, "memory_to_active": 0.005496742203831673, "contribution_to_active": 0.5303561091423035, "mix_mean": 0.6033815741539001, "mix_low": 0.0017786026000976562, "mix_high": 0.013497352600097656, "read_cosine": 0.2233801782131195, "beta_mean": 1.3320354810275603e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003460447071120143, "mix_std": 0.2930959463119507, "beta_std": 5.438806692836806e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003068220103159547, "evidence_weights": [-0.29262280464172363, -0.3031647801399231, 0.9707167744636536], "agreement_weight": 0.10848871618509293, "orthogonal_weight": -0.9281553626060486}, {"head": 1, "memory_to_active": 0.03661791980266571, "contribution_to_active": 0.60207200050354, "mix_mean": 0.4845147132873535, "mix_low": 0.008962631225585938, "mix_high": 0.0013294219970703125, "read_cosine": 0.4036179184913635, "beta_mean": 0.00010463257785886526, "nonfinite": 0.0, "weighted_memory_to_active": 0.024066181853413582, "mix_std": 0.290017694234848, "beta_std": 0.00021006555471103638, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01648605987429619, "evidence_weights": [-0.22008578479290009, -0.2688230574131012, 1.1503560543060303], "agreement_weight": 0.34668874740600586, "orthogonal_weight": -0.8300406336784363}, {"head": 2, "memory_to_active": 0.20634959638118744, "contribution_to_active": 0.6089268326759338, "mix_mean": 0.5386354327201843, "mix_low": 0.0020656585693359375, "mix_high": 0.0029039382934570312, "read_cosine": 0.34592100977897644, "beta_mean": 0.0008510714396834373, "nonfinite": 0.0, "weighted_memory_to_active": 0.1307421177625656, "mix_std": 0.27613818645477295, "beta_std": 0.004818856716156006, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.06590311229228973, "evidence_weights": [-0.18856091797351837, -0.25257647037506104, 0.6508243083953857], "agreement_weight": 0.257215678691864, "orthogonal_weight": -0.5942299365997314}, {"head": 3, "memory_to_active": 1.6860629320144653, "contribution_to_active": 0.17493820190429688, "mix_mean": 0.03951427340507507, "mix_low": 0.3996572494506836, "mix_high": 0.0, "read_cosine": 0.4014015793800354, "beta_mean": 0.09935929626226425, "nonfinite": 0.0, "weighted_memory_to_active": 0.1440415233373642, "mix_std": 0.0880841612815857, "beta_std": 0.0701601430773735, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.022727735340595245, "evidence_weights": [0.07846768200397491, 0.0337483249604702, -0.03738155588507652], "agreement_weight": 0.08682185411453247, "orthogonal_weight": 0.16637006402015686}, {"head": 4, "memory_to_active": 3.22129487991333, "contribution_to_active": 1.2479827404022217, "mix_mean": 0.12339836359024048, "mix_low": 0.15534114837646484, "mix_high": 0.0001354217529296875, "read_cosine": 0.28103286027908325, "beta_mean": 0.19348227977752686, "nonfinite": 0.0, "weighted_memory_to_active": 1.217834711074829, "mix_std": 0.179742231965065, "beta_std": 0.09967716038227081, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03075304627418518, "evidence_weights": [-0.059820182621479034, -0.14698584377765656, 0.0025362479500472546], "agreement_weight": -0.05492148548364639, "orthogonal_weight": -0.025899739935994148}, {"head": 5, "memory_to_active": 1.731916904449463, "contribution_to_active": 0.6086177229881287, "mix_mean": 0.06772603839635849, "mix_low": 0.3365459442138672, "mix_high": 8.58306884765625e-06, "read_cosine": 0.48768460750579834, "beta_mean": 0.12163098156452179, "nonfinite": 0.0, "weighted_memory_to_active": 0.671586811542511, "mix_std": 0.1232539713382721, "beta_std": 0.0929623693227768, "beta_low": 0.00311279296875, "beta_high": 0.0, "orthogonal_to_active": 0.1185508668422699, "evidence_weights": [0.09999751299619675, -0.067182257771492, 0.15946036577224731], "agreement_weight": 0.020774442702531815, "orthogonal_weight": 0.25945988297462463}]}, {"layer": 10, "memory_to_active": 0.2609919011592865, "contribution_to_active": 0.6405669450759888, "mix_mean": 0.4898281693458557, "mix_low": 0.014466127380728722, "mix_high": 0.006415843963623047, "read_cosine": 0.3824933171272278, "beta_mean": 0.046888209879398346, "nonfinite": 0.0, "weighted_memory_to_active": 0.1306358277797699, "mix_std": 0.2679902911186218, "beta_std": 0.04753778874874115, "beta_low": 0.6557413935661316, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.15989604592323303, "output_influence": 0.419871062040329, "residual_influence": 0.1417146921157837, "state_active": 0.052974291145801544, "state_memory": 0.015045185573399067, "state_peak": 103.63964080810547, "growth_active": 1.0679824352264404, "growth_memory": 0.9377344250679016, "heads": [{"head": 0, "memory_to_active": 1.036431074142456, "contribution_to_active": 1.6618647575378418, "mix_mean": 0.6001291871070862, "mix_low": 0.04358196258544922, "mix_high": 0.024995803833007812, "read_cosine": 0.09076302498579025, "beta_mean": 0.20732109248638153, "nonfinite": 0.0, "weighted_memory_to_active": 0.6478410363197327, "mix_std": 0.3830338716506958, "beta_std": 0.21788331866264343, "beta_low": 0.11663818359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.8761838674545288, "evidence_weights": [-0.13054755330085754, -0.3784457743167877, -0.9885701537132263], "agreement_weight": 0.026813538745045662, "orthogonal_weight": 1.3686141967773438}, {"head": 1, "memory_to_active": 0.016977576538920403, "contribution_to_active": 0.5878118276596069, "mix_mean": 0.5943887233734131, "mix_low": 0.00044536590576171875, "mix_high": 0.009607315063476562, "read_cosine": 0.27473849058151245, "beta_mean": 0.0001045886892825365, "nonfinite": 0.0, "weighted_memory_to_active": 0.009706761687994003, "mix_std": 0.2658529281616211, "beta_std": 0.00014096156519372016, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013064160011708736, "evidence_weights": [-0.2694465219974518, -0.2024964839220047, 0.596983790397644], "agreement_weight": 0.4411774277687073, "orthogonal_weight": -1.4737104177474976}, {"head": 2, "memory_to_active": 0.031195679679512978, "contribution_to_active": 0.5623905062675476, "mix_mean": 0.5745723843574524, "mix_low": 0.000400543212890625, "mix_high": 0.003536224365234375, "read_cosine": 0.04092152789235115, "beta_mean": 7.596641808049753e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012585466727614403, "mix_std": 0.25563377141952515, "beta_std": 0.00024904159363359213, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005695199128240347, "evidence_weights": [-0.22536984086036682, -0.2647436261177063, 0.525844395160675], "agreement_weight": -0.1441287100315094, "orthogonal_weight": -0.4328116476535797}, {"head": 3, "memory_to_active": 0.33251190185546875, "contribution_to_active": 0.09866583347320557, "mix_mean": 0.2657800018787384, "mix_low": 0.03932666778564453, "mix_high": 7.43865966796875e-05, "read_cosine": 0.9075931310653687, "beta_mean": 0.06700758635997772, "nonfinite": 0.0, "weighted_memory_to_active": 0.037723951041698456, "mix_std": 0.22802484035491943, "beta_std": 0.058193616569042206, "beta_low": 0.0008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.04326586052775383, "evidence_weights": [-0.11354856938123703, -0.14411479234695435, 0.1316910684108734], "agreement_weight": 0.001128107076510787, "orthogonal_weight": 1.4961169958114624}, {"head": 4, "memory_to_active": 0.0030102443415671587, "contribution_to_active": 0.5087962746620178, "mix_mean": 0.5251222848892212, "mix_low": 0.00060272216796875, "mix_high": 0.00027370452880859375, "read_cosine": 0.11150810122489929, "beta_mean": 1.9277555111330003e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0015257378108799458, "mix_std": 0.24919816851615906, "beta_std": 9.676449553808197e-05, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0009389971382915974, "evidence_weights": [-0.17884187400341034, -0.2541467249393463, 0.7328479886054993], "agreement_weight": 0.07461351156234741, "orthogonal_weight": -0.6380581855773926}, {"head": 5, "memory_to_active": 0.1458248496055603, "contribution_to_active": 0.42387229204177856, "mix_mean": 0.3789764642715454, "mix_low": 0.0024394989013671875, "mix_high": 7.62939453125e-06, "read_cosine": 0.8694356083869934, "beta_mean": 0.0068007283844053745, "nonfinite": 0.0, "weighted_memory_to_active": 0.0744318962097168, "mix_std": 0.22619812190532684, "beta_std": 0.008663013577461243, "beta_low": 0.81707763671875, "beta_high": 0.0, "orthogonal_to_active": 0.020228158682584763, "evidence_weights": [-0.14525365829467773, -0.15787813067436218, 0.29100149869918823], "agreement_weight": 0.09089662879705429, "orthogonal_weight": -0.6304441094398499}]}, {"layer": 11, "memory_to_active": 0.13520175218582153, "contribution_to_active": 0.5767032504081726, "mix_mean": 0.4671761393547058, "mix_low": 0.07979965209960938, "mix_high": 0.043332576751708984, "read_cosine": 0.4132841229438782, "beta_mean": 0.03309601545333862, "nonfinite": 0.0, "weighted_memory_to_active": 0.061062540858983994, "mix_std": 0.3197077810764313, "beta_std": 0.04672687500715256, "beta_low": 0.5603129267692566, "beta_high": 0.0, "orthogonal_to_active": 0.051979802548885345, "output_influence": 0.5077415108680725, "residual_influence": 0.12717090547084808, "state_active": 0.18619747459888458, "state_memory": 0.006799724884331226, "state_peak": 25.058216094970703, "growth_active": 1.1695878505706787, "growth_memory": 1.5960147380828857, "heads": [{"head": 0, "memory_to_active": 0.008316148072481155, "contribution_to_active": 0.5700639486312866, "mix_mean": 0.6086746454238892, "mix_low": 0.001148223876953125, "mix_high": 0.007925987243652344, "read_cosine": 0.32917076349258423, "beta_mean": 0.00010047744581243023, "nonfinite": 0.0, "weighted_memory_to_active": 0.005444809328764677, "mix_std": 0.265200674533844, "beta_std": 0.0040877400897443295, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.0038127219304442406, "evidence_weights": [-0.166843980550766, -0.3619873821735382, 0.6905545592308044], "agreement_weight": 0.25344130396842957, "orthogonal_weight": -0.7589222192764282}, {"head": 1, "memory_to_active": 0.3115633428096771, "contribution_to_active": 0.2937544882297516, "mix_mean": 0.3420376479625702, "mix_low": 0.010489463806152344, "mix_high": 8.0108642578125e-05, "read_cosine": 0.6077427268028259, "beta_mean": 0.031018121168017387, "nonfinite": 0.0, "weighted_memory_to_active": 0.12657195329666138, "mix_std": 0.2515130043029785, "beta_std": 0.060229942202568054, "beta_low": 0.43780517578125, "beta_high": 0.0, "orthogonal_to_active": 0.0356973372399807, "evidence_weights": [-0.16315416991710663, -0.16355645656585693, 0.3907284438610077], "agreement_weight": 0.07822065055370331, "orthogonal_weight": -0.3751700222492218}, {"head": 2, "memory_to_active": 0.01313887070864439, "contribution_to_active": 0.9719367027282715, "mix_mean": 0.4536968767642975, "mix_low": 0.19075679779052734, "mix_high": 0.15723609924316406, "read_cosine": 0.21767200529575348, "beta_mean": 0.07576793432235718, "nonfinite": 0.0, "weighted_memory_to_active": 0.009675954468548298, "mix_std": 0.43577122688293457, "beta_std": 0.09622686356306076, "beta_low": 0.41644287109375, "beta_high": 0.0, "orthogonal_to_active": 0.05344809964299202, "evidence_weights": [0.029028229415416718, -0.4545440375804901, -1.274590015411377], "agreement_weight": 0.002242608927190304, "orthogonal_weight": 5.694294452667236}, {"head": 3, "memory_to_active": 0.01198512502014637, "contribution_to_active": 0.8586410284042358, "mix_mean": 0.4159195125102997, "mix_low": 0.2650318145751953, "mix_high": 0.09260272979736328, "read_cosine": 0.51200932264328, "beta_mean": 0.06168963760137558, "nonfinite": 0.0, "weighted_memory_to_active": 0.007314437534660101, "mix_std": 0.4356425106525421, "beta_std": 0.08713263273239136, "beta_low": 0.4476318359375, "beta_high": 0.0, "orthogonal_to_active": 0.03308512270450592, "evidence_weights": [0.11024213582277298, -0.3432866036891937, -0.3560706377029419], "agreement_weight": 0.04607842490077019, "orthogonal_weight": 6.014430046081543}, {"head": 4, "memory_to_active": 0.37978118658065796, "contribution_to_active": 0.39965543150901794, "mix_mean": 0.4779047667980194, "mix_low": 0.009642601013183594, "mix_high": 0.0007543563842773438, "read_cosine": 0.09990426898002625, "beta_mean": 0.0217574592679739, "nonfinite": 0.0, "weighted_memory_to_active": 0.177838534116745, "mix_std": 0.27029019594192505, "beta_std": 0.021656418219208717, "beta_low": 0.32562255859375, "beta_high": 0.0, "orthogonal_to_active": 0.16506683826446533, "evidence_weights": [-0.22055070102214813, -0.19570708274841309, 1.8288096189498901], "agreement_weight": -0.1832582652568817, "orthogonal_weight": -0.9448883533477783}, {"head": 5, "memory_to_active": 0.08642586320638657, "contribution_to_active": 0.3661676049232483, "mix_mean": 0.5048234462738037, "mix_low": 0.0017290115356445312, "mix_high": 0.00139617919921875, "read_cosine": 0.7132053971290588, "beta_mean": 0.008242445066571236, "nonfinite": 0.0, "weighted_memory_to_active": 0.03952950984239578, "mix_std": 0.25982895493507385, "beta_std": 0.0110276248306036, "beta_low": 0.73468017578125, "beta_high": 0.0, "orthogonal_to_active": 0.02076869271695614, "evidence_weights": [-0.2173951268196106, -0.2870934009552002, 0.5623090267181396], "agreement_weight": 0.07157158106565475, "orthogonal_weight": -0.711830735206604}]}, {"layer": 12, "memory_to_active": 0.292675644159317, "contribution_to_active": 0.5600360631942749, "mix_mean": 0.4969579577445984, "mix_low": 0.010428428649902344, "mix_high": 0.0040988922119140625, "read_cosine": 0.2512088418006897, "beta_mean": 0.016554001718759537, "nonfinite": 0.0, "weighted_memory_to_active": 0.15781289339065552, "mix_std": 0.25854334235191345, "beta_std": 0.01635407842695713, "beta_low": 0.8225199580192566, "beta_high": 0.0, "orthogonal_to_active": 0.06990280747413635, "output_influence": 0.45011818408966064, "residual_influence": 0.12230078130960464, "state_active": 0.038232769817113876, "state_memory": 0.009496312588453293, "state_peak": 39.47120666503906, "growth_active": 1.1348527669906616, "growth_memory": 1.2476146221160889, "heads": [{"head": 0, "memory_to_active": 0.051169540733098984, "contribution_to_active": 0.5028303861618042, "mix_mean": 0.4881850481033325, "mix_low": 0.0013065338134765625, "mix_high": 0.0014333724975585938, "read_cosine": 0.3024985194206238, "beta_mean": 0.0007799756713211536, "nonfinite": 0.0, "weighted_memory_to_active": 0.023475240916013718, "mix_std": 0.27103886008262634, "beta_std": 0.0032659571152180433, "beta_low": 0.99639892578125, "beta_high": 0.0, "orthogonal_to_active": 0.016652921214699745, "evidence_weights": [-0.18424130976200104, -0.25787580013275146, 0.3112775683403015], "agreement_weight": 0.11177065968513489, "orthogonal_weight": -0.7544779777526855}, {"head": 1, "memory_to_active": 0.0936674252152443, "contribution_to_active": 0.5216960310935974, "mix_mean": 0.5310572981834412, "mix_low": 0.0008554458618164062, "mix_high": 0.0009822845458984375, "read_cosine": 0.5316829681396484, "beta_mean": 0.0010218265233561397, "nonfinite": 0.0, "weighted_memory_to_active": 0.050101686269044876, "mix_std": 0.2626980245113373, "beta_std": 0.004897178150713444, "beta_low": 0.988037109375, "beta_high": 0.0, "orthogonal_to_active": 0.03859788551926613, "evidence_weights": [-0.2060733288526535, -0.27349990606307983, 0.9300215840339661], "agreement_weight": 0.13502630591392517, "orthogonal_weight": -0.9169427752494812}, {"head": 2, "memory_to_active": 0.618836522102356, "contribution_to_active": 0.46416646242141724, "mix_mean": 0.3148597478866577, "mix_low": 0.024381637573242188, "mix_high": 0.00022983551025390625, "read_cosine": 0.46916329860687256, "beta_mean": 0.09473513811826706, "nonfinite": 0.0, "weighted_memory_to_active": 0.24172091484069824, "mix_std": 0.2344001680612564, "beta_std": 0.06458961963653564, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.22669053077697754, "evidence_weights": [-0.24183322489261627, -0.04687102511525154, -0.10188187658786774], "agreement_weight": 0.028725119307637215, "orthogonal_weight": 1.1441775560379028}, {"head": 3, "memory_to_active": 0.03017822466790676, "contribution_to_active": 0.7269212007522583, "mix_mean": 0.6931936740875244, "mix_low": 0.0002689361572265625, "mix_high": 0.014924049377441406, "read_cosine": 0.10238988697528839, "beta_mean": 0.00027038875850848854, "nonfinite": 0.0, "weighted_memory_to_active": 0.02287675067782402, "mix_std": 0.23807057738304138, "beta_std": 0.0053008864633738995, "beta_low": 0.9979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.02153012529015541, "evidence_weights": [-0.2920788526535034, -0.33576762676239014, 1.290871500968933], "agreement_weight": 0.3450550138950348, "orthogonal_weight": -0.9611560702323914}, {"head": 4, "memory_to_active": 0.8620150089263916, "contribution_to_active": 0.6685905456542969, "mix_mean": 0.3308451175689697, "mix_low": 0.029394149780273438, "mix_high": 0.0020933151245117188, "read_cosine": -0.09363266825675964, "beta_mean": 0.0017545524751767516, "nonfinite": 0.0, "weighted_memory_to_active": 0.5440152287483215, "mix_std": 0.26999324560165405, "beta_std": 0.014197787269949913, "beta_low": 0.971435546875, "beta_high": 0.0, "orthogonal_to_active": 0.06487542390823364, "evidence_weights": [-0.10002784430980682, -0.20934592187404633, 0.05892014876008034], "agreement_weight": 0.003327084006741643, "orthogonal_weight": -0.1266210526227951}, {"head": 5, "memory_to_active": 0.10018708556890488, "contribution_to_active": 0.47601163387298584, "mix_mean": 0.6236066818237305, "mix_low": 0.006363868713378906, "mix_high": 0.0049304962158203125, "read_cosine": 0.19515103101730347, "beta_mean": 0.0007621151162311435, "nonfinite": 0.0, "weighted_memory_to_active": 0.06468746066093445, "mix_std": 0.27505916357040405, "beta_std": 0.005873049143701792, "beta_low": 0.9810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.05106988549232483, "evidence_weights": [-0.2552277147769928, -0.3101440668106079, 1.2094855308532715], "agreement_weight": 0.2852419316768646, "orthogonal_weight": -0.8383704423904419}]}, {"layer": 13, "memory_to_active": 0.13357052206993103, "contribution_to_active": 0.5110542178153992, "mix_mean": 0.5080966353416443, "mix_low": 0.005364100448787212, "mix_high": 0.005633036606013775, "read_cosine": 0.41181159019470215, "beta_mean": 0.015864377841353416, "nonfinite": 0.0, "weighted_memory_to_active": 0.056348979473114014, "mix_std": 0.25742000341415405, "beta_std": 0.017647989094257355, "beta_low": 0.7953695058822632, "beta_high": 0.0, "orthogonal_to_active": 0.05748836323618889, "output_influence": 0.4732610583305359, "residual_influence": 0.15698227286338806, "state_active": 0.10222920775413513, "state_memory": 0.03779083117842674, "state_peak": 95.5966567993164, "growth_active": 0.9409541487693787, "growth_memory": 1.0348564386367798, "heads": [{"head": 0, "memory_to_active": 0.03153722360730171, "contribution_to_active": 0.5515909194946289, "mix_mean": 0.56075119972229, "mix_low": 0.000484466552734375, "mix_high": 0.0016603469848632812, "read_cosine": 0.39253878593444824, "beta_mean": 0.00012092737597413361, "nonfinite": 0.0, "weighted_memory_to_active": 0.019261423498392105, "mix_std": 0.26735159754753113, "beta_std": 0.00026060413802042603, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01865282468497753, "evidence_weights": [-0.20185984671115875, -0.263942152261734, 0.7626506686210632], "agreement_weight": 0.10901229083538055, "orthogonal_weight": -1.0837204456329346}, {"head": 1, "memory_to_active": 0.017293784767389297, "contribution_to_active": 0.5710241794586182, "mix_mean": 0.5247445106506348, "mix_low": 0.0021505355834960938, "mix_high": 0.0018024444580078125, "read_cosine": 0.21331703662872314, "beta_mean": 5.613371467916295e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009321246296167374, "mix_std": 0.26876986026763916, "beta_std": 9.11675306269899e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01070439163595438, "evidence_weights": [-0.18670028448104858, -0.270094633102417, 0.7102900743484497], "agreement_weight": 0.29961007833480835, "orthogonal_weight": -1.1986528635025024}, {"head": 2, "memory_to_active": 0.10530634224414825, "contribution_to_active": 0.4375295341014862, "mix_mean": 0.3634972870349884, "mix_low": 0.0023279190063476562, "mix_high": 4.57763671875e-05, "read_cosine": 0.6831179261207581, "beta_mean": 0.013831352815032005, "nonfinite": 0.0, "weighted_memory_to_active": 0.043455906212329865, "mix_std": 0.22158005833625793, "beta_std": 0.039992060512304306, "beta_low": 0.787353515625, "beta_high": 0.0, "orthogonal_to_active": 0.019510477781295776, "evidence_weights": [-0.20389144122600555, -0.18751083314418793, 0.43295353651046753], "agreement_weight": 0.012264815159142017, "orthogonal_weight": -0.8043204545974731}, {"head": 3, "memory_to_active": 0.1562959849834442, "contribution_to_active": 0.5727592706680298, "mix_mean": 0.623340368270874, "mix_low": 0.018898963928222656, "mix_high": 0.023761749267578125, "read_cosine": -0.0053950343281030655, "beta_mean": 0.001290280488319695, "nonfinite": 0.0, "weighted_memory_to_active": 0.11189799755811691, "mix_std": 0.29236266016960144, "beta_std": 0.008708801120519638, "beta_low": 0.987548828125, "beta_high": 0.0, "orthogonal_to_active": 0.11219297349452972, "evidence_weights": [-0.3333827257156372, -0.24523520469665527, 1.9442397356033325], "agreement_weight": 3.754399538040161, "orthogonal_weight": -1.0485725402832031}, {"head": 4, "memory_to_active": 0.41373130679130554, "contribution_to_active": 0.2617398500442505, "mix_mean": 0.359474241733551, "mix_low": 0.0063304901123046875, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8790785074234009, "beta_mean": 0.07936370372772217, "nonfinite": 0.0, "weighted_memory_to_active": 0.10694802552461624, "mix_std": 0.2182816118001938, "beta_std": 0.05292079597711563, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.14515629410743713, "evidence_weights": [-0.1331137716770172, -0.22933484613895416, 0.4873315691947937], "agreement_weight": -0.0009288911824114621, "orthogonal_weight": 2.0838773250579834}, {"head": 5, "memory_to_active": 0.07725843787193298, "contribution_to_active": 0.6716816425323486, "mix_mean": 0.616771936416626, "mix_low": 0.0019922256469726562, "mix_high": 0.006519317626953125, "read_cosine": 0.3082121014595032, "beta_mean": 0.0005238676094450057, "nonfinite": 0.0, "weighted_memory_to_active": 0.04720928147435188, "mix_std": 0.2761741578578949, "beta_std": 0.003914505243301392, "beta_low": 0.9971923828125, "beta_high": 0.0, "orthogonal_to_active": 0.038713227957487106, "evidence_weights": [-0.21961697936058044, -0.29747244715690613, 0.7675719857215881], "agreement_weight": 0.09708669036626816, "orthogonal_weight": -0.9054664373397827}]}, {"layer": 14, "memory_to_active": 0.24896380305290222, "contribution_to_active": 0.4277671277523041, "mix_mean": 0.41814398765563965, "mix_low": 0.02762826532125473, "mix_high": 0.0050983428955078125, "read_cosine": 0.6220052242279053, "beta_mean": 0.06935912370681763, "nonfinite": 0.0, "weighted_memory_to_active": 0.11027668416500092, "mix_std": 0.24684754014015198, "beta_std": 0.06412100791931152, "beta_low": 0.463409423828125, "beta_high": 0.0, "orthogonal_to_active": 0.04114890098571777, "output_influence": 0.32370173931121826, "residual_influence": 0.12590809166431427, "state_active": 0.0620293989777565, "state_memory": 0.016269417479634285, "state_peak": 37.88853073120117, "growth_active": 1.0823736190795898, "growth_memory": 1.2925347089767456, "heads": [{"head": 0, "memory_to_active": 0.3374306559562683, "contribution_to_active": 0.20663709938526154, "mix_mean": 0.20794269442558289, "mix_low": 0.012856483459472656, "mix_high": 9.5367431640625e-07, "read_cosine": 0.7969619631767273, "beta_mean": 0.07359152287244797, "nonfinite": 0.0, "weighted_memory_to_active": 0.08587929606437683, "mix_std": 0.1852489411830902, "beta_std": 0.06554777175188065, "beta_low": 0.00750732421875, "beta_high": 0.0, "orthogonal_to_active": 0.06001415476202965, "evidence_weights": [-0.1348133236169815, -0.040285974740982056, 0.5386552214622498], "agreement_weight": 0.025808019563555717, "orthogonal_weight": 1.1792300939559937}, {"head": 1, "memory_to_active": 0.4908432960510254, "contribution_to_active": 0.33451470732688904, "mix_mean": 0.2339697629213333, "mix_low": 0.13515281677246094, "mix_high": 6.389617919921875e-05, "read_cosine": 0.7825971245765686, "beta_mean": 0.28201642632484436, "nonfinite": 0.0, "weighted_memory_to_active": 0.22073736786842346, "mix_std": 0.26333683729171753, "beta_std": 0.2616628408432007, "beta_low": 0.0919189453125, "beta_high": 0.0, "orthogonal_to_active": 0.014482012018561363, "evidence_weights": [-0.006201969925314188, -0.2377074807882309, -0.16804151237010956], "agreement_weight": 0.05221367999911308, "orthogonal_weight": 0.1421986073255539}, {"head": 2, "memory_to_active": 0.14650492370128632, "contribution_to_active": 0.5589671730995178, "mix_mean": 0.5667121410369873, "mix_low": 0.0027866363525390625, "mix_high": 0.0028285980224609375, "read_cosine": 0.40883487462997437, "beta_mean": 0.0029068593867123127, "nonfinite": 0.0, "weighted_memory_to_active": 0.09199880063533783, "mix_std": 0.26805123686790466, "beta_std": 0.0038717882707715034, "beta_low": 0.94970703125, "beta_high": 0.0, "orthogonal_to_active": 0.06983453035354614, "evidence_weights": [-0.15649183094501495, -0.3050044775009155, 0.9205474853515625], "agreement_weight": 0.08577309548854828, "orthogonal_weight": -0.8641164898872375}, {"head": 3, "memory_to_active": 0.065623439848423, "contribution_to_active": 0.6558753252029419, "mix_mean": 0.6707878708839417, "mix_low": 0.0058307647705078125, "mix_high": 0.01829242706298828, "read_cosine": 0.16298000514507294, "beta_mean": 0.008081890642642975, "nonfinite": 0.0, "weighted_memory_to_active": 0.04193303361535072, "mix_std": 0.25146815180778503, "beta_std": 0.008478047326207161, "beta_low": 0.73291015625, "beta_high": 0.0, "orthogonal_to_active": 0.026443425565958023, "evidence_weights": [-0.3706943988800049, -0.32237204909324646, 0.4074539542198181], "agreement_weight": 0.08032916486263275, "orthogonal_weight": -0.7226620316505432}, {"head": 4, "memory_to_active": 0.17716382443904877, "contribution_to_active": 0.49055230617523193, "mix_mean": 0.4893072843551636, "mix_low": 0.004508018493652344, "mix_high": 0.0084381103515625, "read_cosine": 0.8455316424369812, "beta_mean": 0.04718692600727081, "nonfinite": 0.0, "weighted_memory_to_active": 0.10438023507595062, "mix_std": 0.26471203565597534, "beta_std": 0.03458792343735695, "beta_low": 0.042236328125, "beta_high": 0.0, "orthogonal_to_active": 0.03023276850581169, "evidence_weights": [-0.21949130296707153, -0.25590863823890686, 0.42365148663520813], "agreement_weight": 0.12319792062044144, "orthogonal_weight": -0.5688667297363281}, {"head": 5, "memory_to_active": 0.276216596364975, "contribution_to_active": 0.3200559914112091, "mix_mean": 0.34014397859573364, "mix_low": 0.004634857177734375, "mix_high": 0.0009660720825195312, "read_cosine": 0.7351258993148804, "beta_mean": 0.00237108301371336, "nonfinite": 0.0, "weighted_memory_to_active": 0.11673136800527573, "mix_std": 0.2482679784297943, "beta_std": 0.010577723383903503, "beta_low": 0.9561767578125, "beta_high": 0.0, "orthogonal_to_active": 0.04588651657104492, "evidence_weights": [-0.1807052046060562, -0.15862582623958588, 0.22464945912361145], "agreement_weight": 0.08526599407196045, "orthogonal_weight": -0.647050142288208}]}, {"layer": 15, "memory_to_active": 0.058797936886548996, "contribution_to_active": 0.4900032877922058, "mix_mean": 0.40234434604644775, "mix_low": 0.009982585906982422, "mix_high": 0.002968152519315481, "read_cosine": 0.680974006652832, "beta_mean": 0.037516314536333084, "nonfinite": 0.0, "weighted_memory_to_active": 0.027003800496459007, "mix_std": 0.28353768587112427, "beta_std": 0.04311733692884445, "beta_low": 0.7158915400505066, "beta_high": 0.0, "orthogonal_to_active": 0.005932222586125135, "output_influence": 0.30767884850502014, "residual_influence": 0.16501252353191376, "state_active": 0.13170666992664337, "state_memory": 0.009498747996985912, "state_peak": 44.0522575378418, "growth_active": 1.313372015953064, "growth_memory": 1.628430962562561, "heads": [{"head": 0, "memory_to_active": 0.09612078964710236, "contribution_to_active": 0.47624504566192627, "mix_mean": 0.3701607584953308, "mix_low": 0.005991935729980469, "mix_high": 0.005234718322753906, "read_cosine": 0.7250279784202576, "beta_mean": 0.035432375967502594, "nonfinite": 0.0, "weighted_memory_to_active": 0.0427125059068203, "mix_std": 0.27800557017326355, "beta_std": 0.02910526841878891, "beta_low": 0.1395263671875, "beta_high": 0.0, "orthogonal_to_active": 0.01863868720829487, "evidence_weights": [-0.1658095419406891, -0.1743048131465912, 0.35573336482048035], "agreement_weight": 0.09164705872535706, "orthogonal_weight": -0.6915213465690613}, {"head": 1, "memory_to_active": 0.14187541604042053, "contribution_to_active": 0.4157821536064148, "mix_mean": 0.41306474804878235, "mix_low": 0.0028486251831054688, "mix_high": 0.004866600036621094, "read_cosine": 0.8963273763656616, "beta_mean": 0.0010024033254012465, "nonfinite": 0.0, "weighted_memory_to_active": 0.05736411362886429, "mix_std": 0.28350263833999634, "beta_std": 0.002633697586134076, "beta_low": 0.9859619140625, "beta_high": 0.0, "orthogonal_to_active": 0.000579100102186203, "evidence_weights": [-0.16768714785575867, -0.1759462207555771, 0.24438464641571045], "agreement_weight": 0.10698109120130539, "orthogonal_weight": -0.017251165583729744}, {"head": 2, "memory_to_active": 0.006628627888858318, "contribution_to_active": 0.46706315875053406, "mix_mean": 0.33653587102890015, "mix_low": 0.021330833435058594, "mix_high": 3.147125244140625e-05, "read_cosine": 0.7831518650054932, "beta_mean": 0.00011418075155233964, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031306836754083633, "mix_std": 0.273379921913147, "beta_std": 0.00023353191500063986, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.000414023146731779, "evidence_weights": [-0.20390678942203522, -0.27732428908348083, 0.2292993813753128], "agreement_weight": 0.010375979356467724, "orthogonal_weight": 0.31034713983535767}, {"head": 3, "memory_to_active": 0.009497791528701782, "contribution_to_active": 0.431739866733551, "mix_mean": 0.3622390925884247, "mix_low": 0.0074176788330078125, "mix_high": 6.103515625e-05, "read_cosine": 0.6060996055603027, "beta_mean": 7.51879415474832e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038957686629146338, "mix_std": 0.2783844470977783, "beta_std": 0.00019606461864896119, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0031315810047090054, "evidence_weights": [-0.21826040744781494, -0.25322723388671875, 0.3234580457210541], "agreement_weight": 0.027856456115841866, "orthogonal_weight": -0.9746246337890625}, {"head": 4, "memory_to_active": 0.07614240050315857, "contribution_to_active": 0.6227999329566956, "mix_mean": 0.379016637802124, "mix_low": 0.012282371520996094, "mix_high": 0.0007753372192382812, "read_cosine": 0.6950669288635254, "beta_mean": 0.18715545535087585, "nonfinite": 0.0, "weighted_memory_to_active": 0.040145181119441986, "mix_std": 0.3063276410102844, "beta_std": 0.21710315346717834, "beta_low": 0.19378662109375, "beta_high": 0.0, "orthogonal_to_active": 0.004922363441437483, "evidence_weights": [-0.21491366624832153, -0.2578239440917969, 0.023442011326551437], "agreement_weight": 0.012700239196419716, "orthogonal_weight": 0.22356769442558289}, {"head": 5, "memory_to_active": 0.02252260409295559, "contribution_to_active": 0.526389479637146, "mix_mean": 0.5530487895011902, "mix_low": 0.010024070739746094, "mix_high": 0.006839752197265625, "read_cosine": 0.3801700174808502, "beta_mean": 0.0013182791881263256, "nonfinite": 0.0, "weighted_memory_to_active": 0.014774547889828682, "mix_std": 0.28162580728530884, "beta_std": 0.009432308375835419, "beta_low": 0.97613525390625, "beta_high": 0.0, "orthogonal_to_active": 0.00790757592767477, "evidence_weights": [-0.32167819142341614, -0.31146174669265747, 0.7390219569206238], "agreement_weight": 0.25206297636032104, "orthogonal_weight": 0.7063608169555664}]}]}, {"step": 34000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9237502810010483, "memory_to_active": 0.8712753653526306, "contribution_to_active": 0.5261766910552979, "mix_mean": 0.37390440702438354, "mix_low": 0.15015588700771332, "mix_high": 0.009028077125549316, "read_cosine": 0.3705575168132782, "beta_mean": 0.07691328227519989, "nonfinite": 0.0, "weighted_memory_to_active": 0.21294370293617249, "mix_std": 0.21837440133094788, "beta_std": 0.06645756214857101, "beta_low": 0.5143076777458191, "beta_high": 1.9073486328125e-05, "orthogonal_to_active": 0.055415716022253036, "samples": 4, "scored_tokens": 899, "normal_loss": 3.114621639251709, "normal_token_accuracy": 0.4371523857116699, "state_active": 0.06062474474310875, "state_memory": 0.018935859203338623, "state_peak": 114.97332763671875, "growth_active": 1.1159772872924805, "growth_memory": 1.36830735206604, "output_influence": 0.3793230354785919, "residual_influence": 0.12324805557727814, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1831181049346924, "no_memory_loss_delta": 0.5005216598510742, "scrambled_loss_delta": 0.1715511679649353, "memory_only_loss_delta": 3.2653839588165283, "no_evidence_loss_delta": 0.2739276885986328, "no_agreement_loss_delta": 0.034453339874744415, "no_orthogonal_loss_delta": -3.1029339879751205e-05}, {"index": 94, "tokens": 256, "normal_loss": 3.1743807792663574, "no_memory_loss_delta": 0.495961993932724, "scrambled_loss_delta": 0.35161226987838745, "memory_only_loss_delta": 3.051866292953491, "no_evidence_loss_delta": 0.26312336325645447, "no_agreement_loss_delta": 0.04200344160199165, "no_orthogonal_loss_delta": 0.0364440456032753}, {"index": 188, "tokens": 256, "normal_loss": 3.527728796005249, "no_memory_loss_delta": 0.42395415902137756, "scrambled_loss_delta": 0.16002419590950012, "memory_only_loss_delta": 2.7733042240142822, "no_evidence_loss_delta": 0.19577868282794952, "no_agreement_loss_delta": 0.022115027531981468, "no_orthogonal_loss_delta": 0.021171042695641518}, {"index": 283, "tokens": 131, "normal_loss": 2.0566914081573486, "no_memory_loss_delta": 0.269334077835083, "scrambled_loss_delta": 0.2219277024269104, "memory_only_loss_delta": 2.697580337524414, "no_evidence_loss_delta": 0.05136454850435257, "no_agreement_loss_delta": -0.007325415499508381, "no_orthogonal_loss_delta": 0.009289698675274849}], "agreement_weight_abs_mean": 0.1582588404417038, "agreement_weight_abs_max": 3.8671371936798096, "agreement_weight_mean": 0.13727806508541107, "orthogonal_weight_abs_mean": 0.7021495699882507, "orthogonal_weight_abs_max": 6.172863483428955, "orthogonal_weight_mean": -0.14893530309200287, "no_memory_loss": 3.5583531856536865, "no_memory_loss_delta": 0.4437316954135895, "no_memory_token_accuracy_gain": 0.055617351084947586, "no_memory_token_win_fraction": 0.6729699373245239, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 35.83624479674851, "scrambled_loss": 3.341505765914917, "scrambled_loss_delta": 0.22688382863998413, "scrambled_token_accuracy_gain": 0.01890989951789379, "scrambled_token_win_fraction": 0.6184649467468262, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.29866329610128, "memory_only_loss": 6.096340656280518, "memory_only_loss_delta": 2.9817183017730713, "memory_only_token_accuracy_gain": 0.2513904273509979, "memory_only_token_win_fraction": 0.9010010957717896, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.92943686100513, "no_evidence_loss": 3.3307878971099854, "no_evidence_loss_delta": 0.2161659151315689, "no_evidence_token_accuracy_gain": 0.017797552049160004, "no_evidence_token_win_fraction": 0.6140155792236328, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 19.439837071568114, "no_agreement_loss": 3.1416237354278564, "no_agreement_loss_delta": 0.02700195088982582, "no_agreement_token_accuracy_gain": -0.011123470962047577, "no_agreement_token_win_fraction": 0.4905450642108917, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.6640657394353857, "no_orthogonal_loss": 3.132373094558716, "no_orthogonal_loss_delta": 0.017751356586813927, "no_orthogonal_token_accuracy_gain": -0.0033370412420481443, "no_orthogonal_token_win_fraction": 0.5606229305267334, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.7594729407459566, "short_no_memory_loss_delta": 0.3933247923851013, "long_context_memory_gain": 0.05040690302848816, "normal_context_loss_gain": 0.1492883265018463, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.2953505516052246, "contribution_to_active": 1.0671123266220093, "mix_mean": 0.10285359621047974, "mix_low": 0.5437641143798828, "mix_high": 0.004503726959228516, "read_cosine": 0.4460183382034302, "beta_mean": 0.2465166449546814, "nonfinite": 0.0, "weighted_memory_to_active": 1.1241281032562256, "mix_std": 0.08859401941299438, "beta_std": 0.146683007478714, "beta_low": 0.02236938662827015, "beta_high": 0.0, "orthogonal_to_active": 0.06625448912382126, "output_influence": 0.18203690648078918, "residual_influence": 0.21764598786830902, "state_active": 0.07012695074081421, "state_memory": 0.04732127860188484, "state_peak": 52.60742950439453, "growth_active": 1.2456316947937012, "growth_memory": 1.2432582378387451, "heads": [{"head": 0, "memory_to_active": 2.4466354846954346, "contribution_to_active": 0.47477495670318604, "mix_mean": 0.018761852756142616, "mix_low": 0.6234922409057617, "mix_high": 0.0, "read_cosine": 0.2560027241706848, "beta_mean": 0.1911165714263916, "nonfinite": 0.0, "weighted_memory_to_active": 0.5465884208679199, "mix_std": 0.057631295174360275, "beta_std": 0.09478668123483658, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03979264199733734, "evidence_weights": [-0.022756095975637436, -0.07146722823381424, -0.11816179752349854], "agreement_weight": -0.1161404475569725, "orthogonal_weight": -0.07957068085670471}, {"head": 1, "memory_to_active": 5.362915992736816, "contribution_to_active": 2.046428680419922, "mix_mean": 0.015637755393981934, "mix_low": 0.6837663650512695, "mix_high": 0.0, "read_cosine": 0.5142655968666077, "beta_mean": 0.4601934552192688, "nonfinite": 0.0, "weighted_memory_to_active": 2.3844919204711914, "mix_std": 0.050109319388866425, "beta_std": 0.19499289989471436, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10097255557775497, "evidence_weights": [0.030049068853259087, -0.09759221225976944, 0.37436679005622864], "agreement_weight": 0.02819621004164219, "orthogonal_weight": -0.0875636413693428}, {"head": 2, "memory_to_active": 2.6938910484313965, "contribution_to_active": 0.5356326699256897, "mix_mean": 0.014723083935678005, "mix_low": 0.654789924621582, "mix_high": 0.0, "read_cosine": 0.32092803716659546, "beta_mean": 0.29946357011795044, "nonfinite": 0.0, "weighted_memory_to_active": 0.627263069152832, "mix_std": 0.033499713987112045, "beta_std": 0.2099071890115738, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.059433773159980774, "evidence_weights": [0.053259436041116714, 0.06466031819581985, 0.01194656640291214], "agreement_weight": 0.003905290039256215, "orthogonal_weight": -0.1912521868944168}, {"head": 3, "memory_to_active": 6.7958784103393555, "contribution_to_active": 2.864922046661377, "mix_mean": 0.03825407102704048, "mix_low": 0.45332908630371094, "mix_high": 0.0, "read_cosine": 0.3256213665008545, "beta_mean": 0.3492235243320465, "nonfinite": 0.0, "weighted_memory_to_active": 3.0731916427612305, "mix_std": 0.08584968000650406, "beta_std": 0.2494206428527832, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17237786948680878, "evidence_weights": [-0.05203847214579582, -0.06973963230848312, -0.13074028491973877], "agreement_weight": -0.01574031449854374, "orthogonal_weight": -0.06648039072751999}, {"head": 4, "memory_to_active": 2.36787748336792, "contribution_to_active": 0.07720290124416351, "mix_mean": 0.006607222370803356, "mix_low": 0.8283958435058594, "mix_high": 0.0, "read_cosine": 0.3375806212425232, "beta_mean": 0.1429585963487625, "nonfinite": 0.0, "weighted_memory_to_active": 0.06108543276786804, "mix_std": 0.015436174347996712, "beta_std": 0.10021113604307175, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006107877008616924, "evidence_weights": [0.1247376799583435, 0.0751354768872261, -0.3784328103065491], "agreement_weight": -0.12964695692062378, "orthogonal_weight": 0.09746117144823074}, {"head": 5, "memory_to_active": 0.10490594059228897, "contribution_to_active": 0.40371209383010864, "mix_mean": 0.5231375694274902, "mix_low": 0.01881122589111328, "mix_high": 0.027022361755371094, "read_cosine": 0.9217116832733154, "beta_mean": 0.03614410385489464, "nonfinite": 0.0, "weighted_memory_to_active": 0.05214807018637657, "mix_std": 0.28903788328170776, "beta_std": 0.030779412016272545, "beta_low": 0.13421630859375, "beta_high": 0.0, "orthogonal_to_active": 0.018842210993170738, "evidence_weights": [-0.23329059779644012, -0.24061264097690582, 0.30733293294906616], "agreement_weight": 0.25615406036376953, "orthogonal_weight": -0.6673365235328674}]}, {"layer": 1, "memory_to_active": 2.323760986328125, "contribution_to_active": 0.476761132478714, "mix_mean": 0.1807234287261963, "mix_low": 0.37047624588012695, "mix_high": 0.0007818540325388312, "read_cosine": 0.32841742038726807, "beta_mean": 0.1813449263572693, "nonfinite": 0.0, "weighted_memory_to_active": 0.35606199502944946, "mix_std": 0.11770199239253998, "beta_std": 0.1272929310798645, "beta_low": 0.2602437436580658, "beta_high": 8.138020348269492e-05, "orthogonal_to_active": 0.0511382594704628, "output_influence": 0.1806507408618927, "residual_influence": 0.04388539493083954, "state_active": 0.03621496632695198, "state_memory": 0.03660813719034195, "state_peak": 35.89352035522461, "growth_active": 1.1726335287094116, "growth_memory": 1.5039750337600708, "heads": [{"head": 0, "memory_to_active": 0.07206323742866516, "contribution_to_active": 0.4090234637260437, "mix_mean": 0.4228886663913727, "mix_low": 0.011469841003417969, "mix_high": 0.00036907196044921875, "read_cosine": 0.02381887473165989, "beta_mean": 0.0005254948046058416, "nonfinite": 0.0, "weighted_memory_to_active": 0.0363892987370491, "mix_std": 0.28161585330963135, "beta_std": 0.0006986205698922276, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.03516102954745293, "evidence_weights": [-0.18261316418647766, -0.23895542323589325, 0.07411565631628036], "agreement_weight": 0.17315267026424408, "orthogonal_weight": -1.0282220840454102}, {"head": 1, "memory_to_active": 3.1506083011627197, "contribution_to_active": 0.2500506043434143, "mix_mean": 0.022572264075279236, "mix_low": 0.472137451171875, "mix_high": 9.5367431640625e-07, "read_cosine": 0.47587576508522034, "beta_mean": 0.2871476709842682, "nonfinite": 0.0, "weighted_memory_to_active": 0.28234660625457764, "mix_std": 0.04861628636717796, "beta_std": 0.18716225028038025, "beta_low": 0.00079345703125, "beta_high": 0.000244140625, "orthogonal_to_active": 0.0161823071539402, "evidence_weights": [0.012171282432973385, -0.026701297610998154, -0.013966301456093788], "agreement_weight": -0.00380527856759727, "orthogonal_weight": 0.1449252814054489}, {"head": 2, "memory_to_active": 0.29652348160743713, "contribution_to_active": 0.5605944991111755, "mix_mean": 0.5797110199928284, "mix_low": 0.0016164779663085938, "mix_high": 0.0043182373046875, "read_cosine": 0.7139779329299927, "beta_mean": 0.01675151288509369, "nonfinite": 0.0, "weighted_memory_to_active": 0.24390584230422974, "mix_std": 0.2514212131500244, "beta_std": 0.019209209829568863, "beta_low": 0.5086669921875, "beta_high": 0.0, "orthogonal_to_active": 0.05284804478287697, "evidence_weights": [-0.29793861508369446, -0.3580479621887207, 0.3825588822364807], "agreement_weight": 0.029648786410689354, "orthogonal_weight": -0.3760276734828949}, {"head": 3, "memory_to_active": 2.8988590240478516, "contribution_to_active": 0.1341119110584259, "mix_mean": 0.016240809112787247, "mix_low": 0.6572589874267578, "mix_high": 0.0, "read_cosine": 0.24679815769195557, "beta_mean": 0.32658326625823975, "nonfinite": 0.0, "weighted_memory_to_active": 0.20689180493354797, "mix_std": 0.04315817356109619, "beta_std": 0.24948234856128693, "beta_low": 0.0289306640625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.063006691634655, "evidence_weights": [0.048427339643239975, 0.0012335452483966947, 0.11532266438007355], "agreement_weight": -0.0227463748306036, "orthogonal_weight": -0.37865906953811646}, {"head": 4, "memory_to_active": 3.3975846767425537, "contribution_to_active": 1.4203686714172363, "mix_mean": 0.03446159511804581, "mix_low": 0.29041194915771484, "mix_high": 2.86102294921875e-06, "read_cosine": 0.3385414779186249, "beta_mean": 0.14444077014923096, "nonfinite": 0.0, "weighted_memory_to_active": 1.2950470447540283, "mix_std": 0.060404080897569656, "beta_std": 0.09142811596393585, "beta_low": 0.003662109375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1306924670934677, "evidence_weights": [-0.03715573623776436, 0.011194163002073765, 0.0017488023731857538], "agreement_weight": -0.028767243027687073, "orthogonal_weight": 0.13009974360466003}, {"head": 5, "memory_to_active": 4.126927375793457, "contribution_to_active": 0.086417555809021, "mix_mean": 0.008466183207929134, "mix_low": 0.7899627685546875, "mix_high": 0.0, "read_cosine": 0.17149221897125244, "beta_mean": 0.31262078881263733, "nonfinite": 0.0, "weighted_memory_to_active": 0.07179127633571625, "mix_std": 0.020996391773223877, "beta_std": 0.21577700972557068, "beta_low": 0.01983642578125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.008939004503190517, "evidence_weights": [0.15080317854881287, 0.07185742259025574, 0.256758451461792], "agreement_weight": -0.15012654662132263, "orthogonal_weight": 0.12814296782016754}]}, {"layer": 2, "memory_to_active": 1.054426908493042, "contribution_to_active": 0.3332582116127014, "mix_mean": 0.35388273000717163, "mix_low": 0.25068825483322144, "mix_high": 0.0033895177766680717, "read_cosine": 0.3321308493614197, "beta_mean": 0.08618856221437454, "nonfinite": 0.0, "weighted_memory_to_active": 0.06324853748083115, "mix_std": 0.19635891914367676, "beta_std": 0.0683872252702713, "beta_low": 0.5512288808822632, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.015749163925647736, "output_influence": 0.34062743186950684, "residual_influence": 0.08488039672374725, "state_active": 0.038157518953084946, "state_memory": 0.017305422574281693, "state_peak": 35.32686233520508, "growth_active": 1.0260117053985596, "growth_memory": 1.5867607593536377, "heads": [{"head": 0, "memory_to_active": 0.006356460507959127, "contribution_to_active": 0.324303537607193, "mix_mean": 0.536733865737915, "mix_low": 0.0022039413452148438, "mix_high": 0.0049877166748046875, "read_cosine": 0.19119028747081757, "beta_mean": 5.068269092589617e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00374582433141768, "mix_std": 0.2905277609825134, "beta_std": 7.013531285338104e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016380778979510069, "evidence_weights": [-0.31572791934013367, -0.1966099590063095, 0.11382037401199341], "agreement_weight": 0.21884037554264069, "orthogonal_weight": -0.4626815617084503}, {"head": 1, "memory_to_active": 2.879323959350586, "contribution_to_active": 0.05560310184955597, "mix_mean": 0.0053024026565253735, "mix_low": 0.8852930068969727, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21178238093852997, "beta_mean": 0.2553778290748596, "nonfinite": 0.0, "weighted_memory_to_active": 0.04036559909582138, "mix_std": 0.012381977401673794, "beta_std": 0.14780443906784058, "beta_low": 0.000244140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.017039157450199127, "evidence_weights": [0.10784812271595001, 0.10308227688074112, 0.504460871219635], "agreement_weight": 0.029044561088085175, "orthogonal_weight": 0.43496838212013245}, {"head": 2, "memory_to_active": 0.023485712707042694, "contribution_to_active": 0.4068930447101593, "mix_mean": 0.481851190328598, "mix_low": 0.0019941329956054688, "mix_high": 0.001888275146484375, "read_cosine": 0.22979846596717834, "beta_mean": 4.339408042142168e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008496659807860851, "mix_std": 0.28119802474975586, "beta_std": 7.423113856930286e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019412297988310456, "evidence_weights": [-0.2325880229473114, -0.2712409794330597, 0.4579172134399414], "agreement_weight": 0.035134561359882355, "orthogonal_weight": -0.25611555576324463}, {"head": 3, "memory_to_active": 0.17223940789699554, "contribution_to_active": 0.5037592649459839, "mix_mean": 0.48570364713668823, "mix_low": 0.0006036758422851562, "mix_high": 0.0063934326171875, "read_cosine": 0.6574166417121887, "beta_mean": 0.03596532344818115, "nonfinite": 0.0, "weighted_memory_to_active": 0.09562062472105026, "mix_std": 0.2684963345527649, "beta_std": 0.04504359886050224, "beta_low": 0.2659912109375, "beta_high": 0.0, "orthogonal_to_active": 0.05912845581769943, "evidence_weights": [-0.17684084177017212, -0.27782851457595825, 0.8641185760498047], "agreement_weight": 0.06397561728954315, "orthogonal_weight": -0.8870900273323059}, {"head": 4, "memory_to_active": 3.2323970794677734, "contribution_to_active": 0.19660750031471252, "mix_mean": 0.01882607862353325, "mix_low": 0.6120262145996094, "mix_high": 0.0, "read_cosine": 0.20764079689979553, "beta_mean": 0.22565405070781708, "nonfinite": 0.0, "weighted_memory_to_active": 0.22321102023124695, "mix_std": 0.04993336647748947, "beta_std": 0.21727541089057922, "beta_low": 0.0411376953125, "beta_high": 0.000732421875, "orthogonal_to_active": 0.014496924355626106, "evidence_weights": [0.025358401238918304, 0.018113719299435616, 0.06700645387172699], "agreement_weight": 0.013863102532923222, "orthogonal_weight": -0.06913826614618301}, {"head": 5, "memory_to_active": 0.012758955359458923, "contribution_to_active": 0.5123827457427979, "mix_mean": 0.5948792099952698, "mix_low": 0.0020084381103515625, "mix_high": 0.0070667266845703125, "read_cosine": 0.4949564039707184, "beta_mean": 4.011070996057242e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008051506243646145, "mix_std": 0.2756160795688629, "beta_std": 5.551766662392765e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000251126941293478, "evidence_weights": [-0.2630932629108429, -0.2826702296733856, 0.9229468107223511], "agreement_weight": 0.07560374587774277, "orthogonal_weight": 0.052790068089962006}]}, {"layer": 3, "memory_to_active": 0.7529647350311279, "contribution_to_active": 0.4101267158985138, "mix_mean": 0.36155790090560913, "mix_low": 0.1314188688993454, "mix_high": 0.008383911103010178, "read_cosine": 0.21593675017356873, "beta_mean": 0.044926390051841736, "nonfinite": 0.0, "weighted_memory_to_active": 0.16159027814865112, "mix_std": 0.24206605553627014, "beta_std": 0.04008134827017784, "beta_low": 0.5384420156478882, "beta_high": 0.0, "orthogonal_to_active": 0.07066351920366287, "output_influence": 0.3270922601222992, "residual_influence": 0.06838546693325043, "state_active": 0.024571113288402557, "state_memory": 0.010724430903792381, "state_peak": 17.306915283203125, "growth_active": 1.07619309425354, "growth_memory": 1.952318549156189, "heads": [{"head": 0, "memory_to_active": 0.44712772965431213, "contribution_to_active": 0.5464104413986206, "mix_mean": 0.47135794162750244, "mix_low": 0.014821052551269531, "mix_high": 0.014440536499023438, "read_cosine": 0.1930619478225708, "beta_mean": 0.011994358152151108, "nonfinite": 0.0, "weighted_memory_to_active": 0.24825914204120636, "mix_std": 0.3224602937698364, "beta_std": 0.013008447363972664, "beta_low": 0.59222412109375, "beta_high": 0.0, "orthogonal_to_active": 0.16916149854660034, "evidence_weights": [-0.18628454208374023, -0.2190963476896286, 0.728122889995575], "agreement_weight": 0.12764686346054077, "orthogonal_weight": -0.7134333252906799}, {"head": 1, "memory_to_active": 0.02843574434518814, "contribution_to_active": 0.3665159344673157, "mix_mean": 0.4562499523162842, "mix_low": 0.012827873229980469, "mix_high": 0.0054168701171875, "read_cosine": 0.1911344975233078, "beta_mean": 0.00016354801482520998, "nonfinite": 0.0, "weighted_memory_to_active": 0.013429023325443268, "mix_std": 0.31090179085731506, "beta_std": 0.0003819451085291803, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0075655728578567505, "evidence_weights": [-0.21993498504161835, -0.1887407898902893, 0.5657382607460022], "agreement_weight": 0.36337071657180786, "orthogonal_weight": -0.6124911308288574}, {"head": 2, "memory_to_active": 2.4442429542541504, "contribution_to_active": 0.3208215832710266, "mix_mean": 0.04909244924783707, "mix_low": 0.36243152618408203, "mix_high": 3.814697265625e-06, "read_cosine": 0.20337960124015808, "beta_mean": 0.19096365571022034, "nonfinite": 0.0, "weighted_memory_to_active": 0.3698709011077881, "mix_std": 0.10314654558897018, "beta_std": 0.1474006175994873, "beta_low": 0.00909423828125, "beta_high": 0.0, "orthogonal_to_active": 0.0012138503370806575, "evidence_weights": [-0.07122212648391724, 0.015364092774689198, -0.02621946856379509], "agreement_weight": -0.01945260725915432, "orthogonal_weight": -0.004145455546677113}, {"head": 3, "memory_to_active": 0.026196518912911415, "contribution_to_active": 0.5080056190490723, "mix_mean": 0.5430307388305664, "mix_low": 0.01242828369140625, "mix_high": 0.012056350708007812, "read_cosine": 0.3105514645576477, "beta_mean": 0.00010003812349168584, "nonfinite": 0.0, "weighted_memory_to_active": 0.01741081476211548, "mix_std": 0.31006675958633423, "beta_std": 0.0002482897834852338, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004475947469472885, "evidence_weights": [-0.21267209947109222, -0.22212786972522736, 1.9746272563934326], "agreement_weight": 1.4574030637741089, "orthogonal_weight": -0.3419922888278961}, {"head": 4, "memory_to_active": 1.28811776638031, "contribution_to_active": 0.07464893162250519, "mix_mean": 0.05740935355424881, "mix_low": 0.3756370544433594, "mix_high": 9.5367431640625e-07, "read_cosine": 0.1523824781179428, "beta_mean": 0.027987608686089516, "nonfinite": 0.0, "weighted_memory_to_active": 0.1339205801486969, "mix_std": 0.11062595248222351, "beta_std": 0.043293796479701996, "beta_low": 0.4346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.10042642056941986, "evidence_weights": [0.057529106736183167, -0.0010293852537870407, 0.20433951914310455], "agreement_weight": -0.008755018003284931, "orthogonal_weight": -0.7753857970237732}, {"head": 5, "memory_to_active": 0.28366735577583313, "contribution_to_active": 0.6443575024604797, "mix_mean": 0.5922069549560547, "mix_low": 0.010367393493652344, "mix_high": 0.018384933471679688, "read_cosine": 0.2451103925704956, "beta_mean": 0.038349151611328125, "nonfinite": 0.0, "weighted_memory_to_active": 0.18665120005607605, "mix_std": 0.29519495368003845, "beta_std": 0.03615499660372734, "beta_low": 0.19482421875, "beta_high": 0.0, "orthogonal_to_active": 0.14113782346248627, "evidence_weights": [-0.22622260451316833, -0.2612910866737366, 0.48965689539909363], "agreement_weight": 0.13503816723823547, "orthogonal_weight": -0.806918203830719}]}, {"layer": 4, "memory_to_active": 1.1702649593353271, "contribution_to_active": 0.4357461631298065, "mix_mean": 0.3456474542617798, "mix_low": 0.2760343551635742, "mix_high": 0.006689390167593956, "read_cosine": 0.3505236506462097, "beta_mean": 0.12408585846424103, "nonfinite": 0.0, "weighted_memory_to_active": 0.13869547843933105, "mix_std": 0.15455420315265656, "beta_std": 0.11440403014421463, "beta_low": 0.3685608208179474, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04132567718625069, "output_influence": 0.38824546337127686, "residual_influence": 0.10483640432357788, "state_active": 0.03743525221943855, "state_memory": 0.01932746358215809, "state_peak": 35.303321838378906, "growth_active": 1.2290040254592896, "growth_memory": 1.5189646482467651, "heads": [{"head": 0, "memory_to_active": 0.21997955441474915, "contribution_to_active": 0.7168194055557251, "mix_mean": 0.6561486721038818, "mix_low": 0.010703086853027344, "mix_high": 0.017910003662109375, "read_cosine": 0.5147156715393066, "beta_mean": 0.24953137338161469, "nonfinite": 0.0, "weighted_memory_to_active": 0.1740947663784027, "mix_std": 0.3361116051673889, "beta_std": 0.26170963048934937, "beta_low": 0.19427490234375, "beta_high": 0.0, "orthogonal_to_active": 0.057847339659929276, "evidence_weights": [-0.26816651225090027, -0.36546626687049866, 0.24156850576400757], "agreement_weight": 0.023397693410515785, "orthogonal_weight": 0.4176974296569824}, {"head": 1, "memory_to_active": 0.0881461501121521, "contribution_to_active": 0.5979115962982178, "mix_mean": 0.6356297135353088, "mix_low": 5.14984130859375e-05, "mix_high": 0.006855010986328125, "read_cosine": 0.6587062478065491, "beta_mean": 0.001465728273615241, "nonfinite": 0.0, "weighted_memory_to_active": 0.060744836926460266, "mix_std": 0.24687018990516663, "beta_std": 0.007760814391076565, "beta_low": 0.974853515625, "beta_high": 0.0, "orthogonal_to_active": 0.03585602343082428, "evidence_weights": [-0.23804113268852234, -0.3216322362422943, 0.7354291677474976], "agreement_weight": 0.14149917662143707, "orthogonal_weight": -0.8399784564971924}, {"head": 2, "memory_to_active": 2.3656864166259766, "contribution_to_active": 0.21222618222236633, "mix_mean": 0.01809627003967762, "mix_low": 0.5941190719604492, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2678736448287964, "beta_mean": 0.1233057975769043, "nonfinite": 0.0, "weighted_memory_to_active": 0.22791510820388794, "mix_std": 0.03716384246945381, "beta_std": 0.09921807050704956, "beta_low": 0.00146484375, "beta_high": 0.0, "orthogonal_to_active": 0.016016865149140358, "evidence_weights": [0.056471578776836395, 0.032864104956388474, -0.0060880775563418865], "agreement_weight": 0.12120630592107773, "orthogonal_weight": -0.11195973306894302}, {"head": 3, "memory_to_active": 1.9181843996047974, "contribution_to_active": 0.1382884830236435, "mix_mean": 0.029831960797309875, "mix_low": 0.4243030548095703, "mix_high": 0.0, "read_cosine": 0.19512240588665009, "beta_mean": 0.1369924396276474, "nonfinite": 0.0, "weighted_memory_to_active": 0.17483243346214294, "mix_std": 0.05470004677772522, "beta_std": 0.13113680481910706, "beta_low": 0.04058837890625, "beta_high": 0.0, "orthogonal_to_active": 0.009911524131894112, "evidence_weights": [-0.020877689123153687, 0.06272018700838089, -0.23178768157958984], "agreement_weight": 0.06411715596914291, "orthogonal_weight": -0.07143872231245041}, {"head": 4, "memory_to_active": 0.028780370950698853, "contribution_to_active": 0.6509973406791687, "mix_mean": 0.7195776700973511, "mix_low": 0.00016880035400390625, "mix_high": 0.015360832214355469, "read_cosine": 0.175202876329422, "beta_mean": 5.029997919336893e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.020830564200878143, "mix_std": 0.2214735448360443, "beta_std": 0.00016984569083433598, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004576084204018116, "evidence_weights": [-0.32084885239601135, -0.33484697341918945, 0.6833421587944031], "agreement_weight": 0.2635843753814697, "orthogonal_weight": -0.23902663588523865}, {"head": 5, "memory_to_active": 2.4008126258850098, "contribution_to_active": 0.2982339560985565, "mix_mean": 0.014600234106183052, "mix_low": 0.6268606185913086, "mix_high": 4.76837158203125e-06, "read_cosine": 0.29152095317840576, "beta_mean": 0.23316945135593414, "nonfinite": 0.0, "weighted_memory_to_active": 0.17375510931015015, "mix_std": 0.03100600838661194, "beta_std": 0.1864290088415146, "beta_low": 0.00018310546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12374621629714966, "evidence_weights": [0.04320717230439186, -0.03901718929409981, -0.23591376841068268], "agreement_weight": -0.053026262670755386, "orthogonal_weight": 0.8549534678459167}]}, {"layer": 5, "memory_to_active": 0.5141514539718628, "contribution_to_active": 0.48770463466644287, "mix_mean": 0.4519544243812561, "mix_low": 0.07439994812011719, "mix_high": 0.014536859467625618, "read_cosine": 0.14979347586631775, "beta_mean": 0.05974837765097618, "nonfinite": 0.0, "weighted_memory_to_active": 0.14700230956077576, "mix_std": 0.2540483772754669, "beta_std": 0.07701073586940765, "beta_low": 0.6254476308822632, "beta_high": 0.0, "orthogonal_to_active": 0.052008870989084244, "output_influence": 0.48303353786468506, "residual_influence": 0.12743310630321503, "state_active": 0.04602556303143501, "state_memory": 0.011831437237560749, "state_peak": 61.318748474121094, "growth_active": 1.1325289011001587, "growth_memory": 1.2697099447250366, "heads": [{"head": 0, "memory_to_active": 0.6778383255004883, "contribution_to_active": 0.41004303097724915, "mix_mean": 0.254141241312027, "mix_low": 0.06703948974609375, "mix_high": 0.00046825408935546875, "read_cosine": -0.11446607112884521, "beta_mean": 0.04278973489999771, "nonfinite": 0.0, "weighted_memory_to_active": 0.16097509860992432, "mix_std": 0.2500901222229004, "beta_std": 0.07208054512739182, "beta_low": 0.32305908203125, "beta_high": 0.0, "orthogonal_to_active": 0.03557521849870682, "evidence_weights": [-0.0963728129863739, -0.13004203140735626, -0.252278208732605], "agreement_weight": -0.01660045236349106, "orthogonal_weight": -0.2211642861366272}, {"head": 1, "memory_to_active": 0.022309591993689537, "contribution_to_active": 0.5222395658493042, "mix_mean": 0.641049325466156, "mix_low": 0.0027027130126953125, "mix_high": 0.0276947021484375, "read_cosine": 0.2027793824672699, "beta_mean": 5.375999171519652e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01204640418291092, "mix_std": 0.2960563898086548, "beta_std": 0.00011093351349700242, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011131584644317627, "evidence_weights": [-0.23968786001205444, -0.2772786617279053, 0.8592960834503174], "agreement_weight": 0.5180383324623108, "orthogonal_weight": -0.9619178771972656}, {"head": 2, "memory_to_active": 0.014769704081118107, "contribution_to_active": 0.5571373701095581, "mix_mean": 0.634669840335846, "mix_low": 0.0024538040161132812, "mix_high": 0.017299652099609375, "read_cosine": -0.008022725582122803, "beta_mean": 0.0001814806746551767, "nonfinite": 0.0, "weighted_memory_to_active": 0.009831753559410572, "mix_std": 0.2735365033149719, "beta_std": 0.0003299223608337343, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0021831123158335686, "evidence_weights": [-0.25368058681488037, -0.34839004278182983, 0.8727849721908569], "agreement_weight": 0.005090869031846523, "orthogonal_weight": -0.2605125606060028}, {"head": 3, "memory_to_active": 1.8210852146148682, "contribution_to_active": 0.21963520348072052, "mix_mean": 0.09112276881933212, "mix_low": 0.16275691986083984, "mix_high": 1.9073486328125e-06, "read_cosine": 0.12809599936008453, "beta_mean": 0.11413772404193878, "nonfinite": 0.0, "weighted_memory_to_active": 0.3473306894302368, "mix_std": 0.1299930363893509, "beta_std": 0.16029512882232666, "beta_low": 0.18768310546875, "beta_high": 0.0, "orthogonal_to_active": 0.1292104423046112, "evidence_weights": [-0.06150441616773605, -0.09237110614776611, -0.06575395911931992], "agreement_weight": 0.05231042951345444, "orthogonal_weight": -0.4512365758419037}, {"head": 4, "memory_to_active": 0.44120317697525024, "contribution_to_active": 0.5705246925354004, "mix_mean": 0.31643420457839966, "mix_low": 0.20842266082763672, "mix_high": 0.006491661071777344, "read_cosine": 0.5415153503417969, "beta_mean": 0.2009691596031189, "nonfinite": 0.0, "weighted_memory_to_active": 0.2801056504249573, "mix_std": 0.3369790315628052, "beta_std": 0.22758647799491882, "beta_low": 0.2476806640625, "beta_high": 0.0, "orthogonal_to_active": 0.08738741278648376, "evidence_weights": [-0.11226285994052887, -0.28359147906303406, -0.08701787143945694], "agreement_weight": 0.035894691944122314, "orthogonal_weight": 0.47213050723075867}, {"head": 5, "memory_to_active": 0.10770244151353836, "contribution_to_active": 0.6466479301452637, "mix_mean": 0.7743090391159058, "mix_low": 0.0030241012573242188, "mix_high": 0.03526496887207031, "read_cosine": 0.14885897934436798, "beta_mean": 0.00035838241456076503, "nonfinite": 0.0, "weighted_memory_to_active": 0.07172428071498871, "mix_std": 0.23763510584831238, "beta_std": 0.0016614370979368687, "beta_low": 0.9942626953125, "beta_high": 0.0, "orthogonal_to_active": 0.046565450727939606, "evidence_weights": [-0.3222513198852539, -0.3221464157104492, 2.0042800903320312], "agreement_weight": 0.3848097026348114, "orthogonal_weight": -0.6829912662506104}]}, {"layer": 6, "memory_to_active": 0.381374329328537, "contribution_to_active": 0.6067118644714355, "mix_mean": 0.5532153248786926, "mix_low": 0.06886880099773407, "mix_high": 0.029562633484601974, "read_cosine": 0.24668729305267334, "beta_mean": 0.06981346011161804, "nonfinite": 0.0, "weighted_memory_to_active": 0.1800810843706131, "mix_std": 0.23867426812648773, "beta_std": 0.07982797920703888, "beta_low": 0.5064290761947632, "beta_high": 0.0, "orthogonal_to_active": 0.044285863637924194, "output_influence": 0.5075242519378662, "residual_influence": 0.15760548412799835, "state_active": 0.04510270059108734, "state_memory": 0.009089875966310501, "state_peak": 89.20703887939453, "growth_active": 1.1584585905075073, "growth_memory": 1.1753770112991333, "heads": [{"head": 0, "memory_to_active": 0.16930900514125824, "contribution_to_active": 0.6441622376441956, "mix_mean": 0.7326151728630066, "mix_low": 0.00014591217041015625, "mix_high": 0.010045051574707031, "read_cosine": -0.12417248636484146, "beta_mean": 0.015307574532926083, "nonfinite": 0.0, "weighted_memory_to_active": 0.1309768408536911, "mix_std": 0.22654058039188385, "beta_std": 0.01857258565723896, "beta_low": 0.5025634765625, "beta_high": 0.0, "orthogonal_to_active": 0.11359492689371109, "evidence_weights": [-0.31185182929039, -0.33458951115608215, 0.5974186062812805], "agreement_weight": 0.17753544449806213, "orthogonal_weight": -0.9371790289878845}, {"head": 1, "memory_to_active": 0.004188741557300091, "contribution_to_active": 0.49967342615127563, "mix_mean": 0.5193482041358948, "mix_low": 0.0012912750244140625, "mix_high": 0.008734703063964844, "read_cosine": 0.5762640833854675, "beta_mean": 7.701361028011888e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.001838227966800332, "mix_std": 0.285522997379303, "beta_std": 1.25377355288947e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007721530855633318, "evidence_weights": [-0.19910076260566711, -0.28594571352005005, 1.0391258001327515], "agreement_weight": 0.039289772510528564, "orthogonal_weight": -0.5874087810516357}, {"head": 2, "memory_to_active": 1.1221818923950195, "contribution_to_active": 0.08049845695495605, "mix_mean": 0.029043614864349365, "mix_low": 0.39941883087158203, "mix_high": 9.5367431640625e-07, "read_cosine": 0.22230088710784912, "beta_mean": 0.05794009938836098, "nonfinite": 0.0, "weighted_memory_to_active": 0.06586991250514984, "mix_std": 0.048770781606435776, "beta_std": 0.05970101058483124, "beta_low": 0.03021240234375, "beta_high": 0.0, "orthogonal_to_active": 0.0012502146419137716, "evidence_weights": [0.09087734669446945, -0.008115173317492008, 0.29120954871177673], "agreement_weight": 0.1289040893316269, "orthogonal_weight": 0.022506816312670708}, {"head": 3, "memory_to_active": 0.15882807970046997, "contribution_to_active": 0.677756130695343, "mix_mean": 0.6389724016189575, "mix_low": 0.010790824890136719, "mix_high": 0.014876365661621094, "read_cosine": 0.4200576841831207, "beta_mean": 0.21398189663887024, "nonfinite": 0.0, "weighted_memory_to_active": 0.113326296210289, "mix_std": 0.3290627598762512, "beta_std": 0.25005853176116943, "beta_low": 0.23529052734375, "beta_high": 0.0, "orthogonal_to_active": 0.0633157342672348, "evidence_weights": [-0.1728668212890625, -0.33687418699264526, 0.17354118824005127], "agreement_weight": -0.00924394279718399, "orthogonal_weight": 0.6404772400856018}, {"head": 4, "memory_to_active": 0.7864567637443542, "contribution_to_active": 1.139417290687561, "mix_mean": 0.7637017965316772, "mix_low": 0.0010499954223632812, "mix_high": 0.14108943939208984, "read_cosine": 0.10792073607444763, "beta_mean": 0.13129901885986328, "nonfinite": 0.0, "weighted_memory_to_active": 0.7358479499816895, "mix_std": 0.28559717535972595, "beta_std": 0.14991024136543274, "beta_low": 0.27130126953125, "beta_high": 0.0, "orthogonal_to_active": 0.05432039871811867, "evidence_weights": [-0.2551306486129761, -0.6014131903648376, -0.13948285579681396], "agreement_weight": -0.016035791486501694, "orthogonal_weight": 0.08541680127382278}, {"head": 5, "memory_to_active": 0.04728120565414429, "contribution_to_active": 0.5987638235092163, "mix_mean": 0.6356102228164673, "mix_low": 0.0005159378051757812, "mix_high": 0.0026292800903320312, "read_cosine": 0.2777528166770935, "beta_mean": 0.00034444883931428194, "nonfinite": 0.0, "weighted_memory_to_active": 0.032627250999212265, "mix_std": 0.2565512955188751, "beta_std": 0.0007129772566258907, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.0324617400765419, "evidence_weights": [-0.2411598116159439, -0.30628830194473267, 0.41501474380493164], "agreement_weight": 0.14652183651924133, "orthogonal_weight": -1.0678285360336304}]}, {"layer": 7, "memory_to_active": 1.693023443222046, "contribution_to_active": 0.2407185137271881, "mix_mean": 0.10072161257266998, "mix_low": 0.28639715909957886, "mix_high": 0.00018103918409906328, "read_cosine": 0.23378220200538635, "beta_mean": 0.11329882591962814, "nonfinite": 0.0, "weighted_memory_to_active": 0.22412973642349243, "mix_std": 0.12486855685710907, "beta_std": 0.0972554013133049, "beta_low": 0.190338134765625, "beta_high": 7.120768714230508e-05, "orthogonal_to_active": 0.0592319518327713, "output_influence": 0.24372950196266174, "residual_influence": 0.05453610420227051, "state_active": 0.013744625262916088, "state_memory": 0.026555290445685387, "state_peak": 39.20564270019531, "growth_active": 1.1118831634521484, "growth_memory": 1.2510879039764404, "heads": [{"head": 0, "memory_to_active": 2.3643698692321777, "contribution_to_active": 0.37741830945014954, "mix_mean": 0.03740042448043823, "mix_low": 0.41498279571533203, "mix_high": 0.0, "read_cosine": 0.36461129784584045, "beta_mean": 0.1435953676700592, "nonfinite": 0.0, "weighted_memory_to_active": 0.42550116777420044, "mix_std": 0.07585267722606659, "beta_std": 0.11771675944328308, "beta_low": 0.01104736328125, "beta_high": 0.000244140625, "orthogonal_to_active": 0.01440029963850975, "evidence_weights": [0.009384852834045887, -0.08702705055475235, 0.11435998231172562], "agreement_weight": -0.0323416031897068, "orthogonal_weight": 0.057336512953042984}, {"head": 1, "memory_to_active": 1.084913730621338, "contribution_to_active": 0.37018489837646484, "mix_mean": 0.18750570714473724, "mix_low": 0.16909027099609375, "mix_high": 0.0009441375732421875, "read_cosine": 0.3237585425376892, "beta_mean": 0.09646796435117722, "nonfinite": 0.0, "weighted_memory_to_active": 0.33874940872192383, "mix_std": 0.22279521822929382, "beta_std": 0.06966114044189453, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.010833255015313625, "evidence_weights": [-0.19495701789855957, -0.11716480553150177, -0.21022869646549225], "agreement_weight": 0.030611500144004822, "orthogonal_weight": 0.0364682711660862}, {"head": 2, "memory_to_active": 0.937211275100708, "contribution_to_active": 0.2819406986236572, "mix_mean": 0.10755132883787155, "mix_low": 0.115081787109375, "mix_high": 0.00013637542724609375, "read_cosine": 0.11549156159162521, "beta_mean": 0.06606872379779816, "nonfinite": 0.0, "weighted_memory_to_active": 0.19961829483509064, "mix_std": 0.15202844142913818, "beta_std": 0.053047969937324524, "beta_low": 0.04693603515625, "beta_high": 0.0, "orthogonal_to_active": 0.06176356226205826, "evidence_weights": [-0.040724050253629684, -0.12790784239768982, 0.46544358134269714], "agreement_weight": 0.025369035080075264, "orthogonal_weight": -0.31604981422424316}, {"head": 3, "memory_to_active": 3.900333881378174, "contribution_to_active": 0.08768810331821442, "mix_mean": 0.03420107066631317, "mix_low": 0.33199310302734375, "mix_high": 0.0, "read_cosine": 0.09039053320884705, "beta_mean": 0.26764118671417236, "nonfinite": 0.0, "weighted_memory_to_active": 0.2809840738773346, "mix_std": 0.05052386224269867, "beta_std": 0.23757493495941162, "beta_low": 0.0159912109375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.23628371953964233, "evidence_weights": [0.06385777145624161, -0.005357363726943731, 0.6633360981941223], "agreement_weight": 0.0553201399743557, "orthogonal_weight": -0.9371887445449829}, {"head": 4, "memory_to_active": 1.71659517288208, "contribution_to_active": 0.06816303730010986, "mix_mean": 0.015436233021318913, "mix_low": 0.657414436340332, "mix_high": 0.0, "read_cosine": 0.24768832325935364, "beta_mean": 0.1049303263425827, "nonfinite": 0.0, "weighted_memory_to_active": 0.053401656448841095, "mix_std": 0.03520623967051506, "beta_std": 0.10201311856508255, "beta_low": 0.07647705078125, "beta_high": 0.0, "orthogonal_to_active": 0.006360840518027544, "evidence_weights": [0.13165585696697235, 0.05040457472205162, 0.2088950276374817], "agreement_weight": 0.04067486897110939, "orthogonal_weight": -0.13091301918029785}, {"head": 5, "memory_to_active": 0.15471607446670532, "contribution_to_active": 0.2589160203933716, "mix_mean": 0.22223490476608276, "mix_low": 0.02982044219970703, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2607528865337372, "beta_mean": 0.0010893483413383365, "nonfinite": 0.0, "weighted_memory_to_active": 0.046523723751306534, "mix_std": 0.2128048986196518, "beta_std": 0.0035184912849217653, "beta_low": 0.9912109375, "beta_high": 0.0, "orthogonal_to_active": 0.025750024244189262, "evidence_weights": [-0.08773297071456909, -0.13865087926387787, 0.3773127794265747], "agreement_weight": 0.08973849564790726, "orthogonal_weight": -0.6379131078720093}]}, {"layer": 8, "memory_to_active": 0.49608391523361206, "contribution_to_active": 0.522300124168396, "mix_mean": 0.41824808716773987, "mix_low": 0.10140880197286606, "mix_high": 0.0009085338097065687, "read_cosine": 0.4899289608001709, "beta_mean": 0.02780602127313614, "nonfinite": 0.0, "weighted_memory_to_active": 0.09865391254425049, "mix_std": 0.2214462012052536, "beta_std": 0.0351041778922081, "beta_low": 0.6031901240348816, "beta_high": 0.0, "orthogonal_to_active": 0.060109831392765045, "output_influence": 0.4424719512462616, "residual_influence": 0.13833895325660706, "state_active": 0.049737006425857544, "state_memory": 0.009698919020593166, "state_peak": 39.351661682128906, "growth_active": 1.070408582687378, "growth_memory": 1.3555313348770142, "heads": [{"head": 0, "memory_to_active": 0.051895879209041595, "contribution_to_active": 0.5522111654281616, "mix_mean": 0.59409499168396, "mix_low": 0.00026798248291015625, "mix_high": 0.0010633468627929688, "read_cosine": 0.5924574136734009, "beta_mean": 0.00314038316719234, "nonfinite": 0.0, "weighted_memory_to_active": 0.03549101576209068, "mix_std": 0.24375955760478973, "beta_std": 0.008589702658355236, "beta_low": 0.92559814453125, "beta_high": 0.0, "orthogonal_to_active": 0.021486012265086174, "evidence_weights": [-0.17270511388778687, -0.3932018578052521, 0.7189750671386719], "agreement_weight": 0.11414186656475067, "orthogonal_weight": -0.8249266147613525}, {"head": 1, "memory_to_active": 0.15675190091133118, "contribution_to_active": 0.48786720633506775, "mix_mean": 0.46586093306541443, "mix_low": 0.0011997222900390625, "mix_high": 0.0003871917724609375, "read_cosine": 0.3607396185398102, "beta_mean": 0.00355127127841115, "nonfinite": 0.0, "weighted_memory_to_active": 0.07654428482055664, "mix_std": 0.25225502252578735, "beta_std": 0.011911083944141865, "beta_low": 0.9327392578125, "beta_high": 0.0, "orthogonal_to_active": 0.046915680170059204, "evidence_weights": [-0.1934177726507187, -0.2571505010128021, 0.6856679916381836], "agreement_weight": 0.09343336522579193, "orthogonal_weight": -0.6925978660583496}, {"head": 2, "memory_to_active": 0.02493729628622532, "contribution_to_active": 0.6419965028762817, "mix_mean": 0.6201121807098389, "mix_low": 0.0009107589721679688, "mix_high": 0.002880096435546875, "read_cosine": 0.39112943410873413, "beta_mean": 0.0002539296692702919, "nonfinite": 0.0, "weighted_memory_to_active": 0.018284041434526443, "mix_std": 0.23345836997032166, "beta_std": 0.0038496730849146843, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.012275224551558495, "evidence_weights": [-0.21976634860038757, -0.33818304538726807, 0.884411633014679], "agreement_weight": 0.20881840586662292, "orthogonal_weight": -0.7658832669258118}, {"head": 3, "memory_to_active": 0.1297765076160431, "contribution_to_active": 0.6014026999473572, "mix_mean": 0.5440244078636169, "mix_low": 0.00341796875, "mix_high": 0.0008077621459960938, "read_cosine": 0.808120846748352, "beta_mean": 0.011950662359595299, "nonfinite": 0.0, "weighted_memory_to_active": 0.09505445510149002, "mix_std": 0.2561190724372864, "beta_std": 0.008297150023281574, "beta_low": 0.48199462890625, "beta_high": 0.0, "orthogonal_to_active": 0.030334368348121643, "evidence_weights": [-0.17516107857227325, -0.29906368255615234, 0.658573567867279], "agreement_weight": 0.16245241463184357, "orthogonal_weight": -0.643753170967102}, {"head": 4, "memory_to_active": 2.3226308822631836, "contribution_to_active": 0.2008129060268402, "mix_mean": 0.026640815660357475, "mix_low": 0.4536294937133789, "mix_high": 0.0, "read_cosine": 0.323493093252182, "beta_mean": 0.07860830426216125, "nonfinite": 0.0, "weighted_memory_to_active": 0.19394639134407043, "mix_std": 0.04726129770278931, "beta_std": 0.0833592340350151, "beta_low": 0.0574951171875, "beta_high": 0.0, "orthogonal_to_active": 0.004414016846567392, "evidence_weights": [0.055237382650375366, -0.017940092831850052, -0.06812943518161774], "agreement_weight": 0.05666319280862808, "orthogonal_weight": 0.024131326004862785}, {"head": 5, "memory_to_active": 0.2905108332633972, "contribution_to_active": 0.6495101451873779, "mix_mean": 0.2587552070617676, "mix_low": 0.14902687072753906, "mix_high": 0.00031280517578125, "read_cosine": 0.46363329887390137, "beta_mean": 0.06933155655860901, "nonfinite": 0.0, "weighted_memory_to_active": 0.1726033091545105, "mix_std": 0.295823872089386, "beta_std": 0.09461823105812073, "beta_low": 0.2215576171875, "beta_high": 0.0, "orthogonal_to_active": 0.2452336698770523, "evidence_weights": [-0.06699280440807343, -0.262152761220932, -0.3674941658973694], "agreement_weight": 0.00500167440623045, "orthogonal_weight": 1.6741851568222046}]}, {"layer": 9, "memory_to_active": 1.155273199081421, "contribution_to_active": 0.6374623775482178, "mix_mean": 0.3159255087375641, "mix_low": 0.14858373999595642, "mix_high": 0.003274122951552272, "read_cosine": 0.3790603578090668, "beta_mean": 0.06823042780160904, "nonfinite": 0.0, "weighted_memory_to_active": 0.3744385540485382, "mix_std": 0.21195855736732483, "beta_std": 0.04683379456400871, "beta_low": 0.500579833984375, "beta_high": 0.0, "orthogonal_to_active": 0.04217846691608429, "output_influence": 0.48745957016944885, "residual_influence": 0.13348694145679474, "state_active": 0.043914102017879486, "state_memory": 0.020097404718399048, "state_peak": 47.09174728393555, "growth_active": 1.2069636583328247, "growth_memory": 1.3667683601379395, "heads": [{"head": 0, "memory_to_active": 0.004540239926427603, "contribution_to_active": 0.5465057492256165, "mix_mean": 0.6123417019844055, "mix_low": 0.00165557861328125, "mix_high": 0.015184402465820312, "read_cosine": 0.23671497404575348, "beta_mean": 8.959425031207502e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002867917064577341, "mix_std": 0.29204127192497253, "beta_std": 3.9708414988126606e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025353129021823406, "evidence_weights": [-0.2951383888721466, -0.3056171238422394, 0.9345892667770386], "agreement_weight": 0.0993606373667717, "orthogonal_weight": -0.932593584060669}, {"head": 1, "memory_to_active": 0.025775916874408722, "contribution_to_active": 0.608025848865509, "mix_mean": 0.48930537700653076, "mix_low": 0.008636474609375, "mix_high": 0.0009441375732421875, "read_cosine": 0.4175434708595276, "beta_mean": 7.33196793589741e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01633237488567829, "mix_std": 0.28986048698425293, "beta_std": 0.00015566876390948892, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01116093061864376, "evidence_weights": [-0.21979370713233948, -0.26812466979026794, 1.1344175338745117], "agreement_weight": 0.34975194931030273, "orthogonal_weight": -0.8306976556777954}, {"head": 2, "memory_to_active": 0.18469616770744324, "contribution_to_active": 0.6265693306922913, "mix_mean": 0.5540463924407959, "mix_low": 0.0023317337036132812, "mix_high": 0.00321197509765625, "read_cosine": 0.38199156522750854, "beta_mean": 0.0007365875062532723, "nonfinite": 0.0, "weighted_memory_to_active": 0.12117797136306763, "mix_std": 0.278860867023468, "beta_std": 0.0047531733289361, "beta_low": 0.99462890625, "beta_high": 0.0, "orthogonal_to_active": 0.060417309403419495, "evidence_weights": [-0.19016139209270477, -0.2556224763393402, 0.6773275136947632], "agreement_weight": 0.26113876700401306, "orthogonal_weight": -0.5962483882904053}, {"head": 3, "memory_to_active": 1.5928229093551636, "contribution_to_active": 0.180117666721344, "mix_mean": 0.042592357844114304, "mix_low": 0.3853578567504883, "mix_high": 9.5367431640625e-07, "read_cosine": 0.41122522950172424, "beta_mean": 0.08040094375610352, "nonfinite": 0.0, "weighted_memory_to_active": 0.15193969011306763, "mix_std": 0.09595328569412231, "beta_std": 0.057677507400512695, "beta_low": 0.0037841796875, "beta_high": 0.0, "orthogonal_to_active": 0.022164365276694298, "evidence_weights": [0.0764210894703865, 0.030148256570100784, -0.02830105647444725], "agreement_weight": 0.08713886141777039, "orthogonal_weight": 0.1545737385749817}, {"head": 4, "memory_to_active": 3.3507237434387207, "contribution_to_active": 1.217362880706787, "mix_mean": 0.12447693943977356, "mix_low": 0.162139892578125, "mix_high": 0.000278472900390625, "read_cosine": 0.34564274549484253, "beta_mean": 0.18306726217269897, "nonfinite": 0.0, "weighted_memory_to_active": 1.2491605281829834, "mix_std": 0.18276529014110565, "beta_std": 0.10831699520349503, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03035564161837101, "evidence_weights": [-0.06219314783811569, -0.14881357550621033, -0.009941869415342808], "agreement_weight": -0.05594587326049805, "orthogonal_weight": -0.025880809873342514}, {"head": 5, "memory_to_active": 1.773079514503479, "contribution_to_active": 0.6461925506591797, "mix_mean": 0.07279014587402344, "mix_low": 0.33138084411621094, "mix_high": 2.47955322265625e-05, "read_cosine": 0.4812440872192383, "beta_mean": 0.14509548246860504, "nonfinite": 0.0, "weighted_memory_to_active": 0.7051527500152588, "mix_std": 0.1322701871395111, "beta_std": 0.1100597158074379, "beta_low": 0.00506591796875, "beta_high": 0.0, "orthogonal_to_active": 0.1264372169971466, "evidence_weights": [0.10050012916326523, -0.06976131349802017, 0.1609230488538742], "agreement_weight": 0.023955048993229866, "orthogonal_weight": 0.2609076201915741}]}, {"layer": 10, "memory_to_active": 0.23696565628051758, "contribution_to_active": 0.638214111328125, "mix_mean": 0.4920145571231842, "mix_low": 0.015860557556152344, "mix_high": 0.009425640106201172, "read_cosine": 0.3766384720802307, "beta_mean": 0.04237569868564606, "nonfinite": 0.0, "weighted_memory_to_active": 0.12399548292160034, "mix_std": 0.2686028480529785, "beta_std": 0.045156460255384445, "beta_low": 0.6687215566635132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.15928354859352112, "output_influence": 0.41831907629966736, "residual_influence": 0.1419822871685028, "state_active": 0.054112065583467484, "state_memory": 0.01526840403676033, "state_peak": 114.97332763671875, "growth_active": 1.0496375560760498, "growth_memory": 0.9545641541481018, "heads": [{"head": 0, "memory_to_active": 0.9362236261367798, "contribution_to_active": 1.6648603677749634, "mix_mean": 0.6153006553649902, "mix_low": 0.043229103088378906, "mix_high": 0.04175567626953125, "read_cosine": 0.09344936907291412, "beta_mean": 0.192405104637146, "nonfinite": 0.0, "weighted_memory_to_active": 0.6315224766731262, "mix_std": 0.38692209124565125, "beta_std": 0.20907807350158691, "beta_low": 0.1400146484375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.8874709606170654, "evidence_weights": [-0.131717249751091, -0.38480645418167114, -1.0296845436096191], "agreement_weight": 0.026781126856803894, "orthogonal_weight": 1.4227068424224854}, {"head": 1, "memory_to_active": 0.011301357299089432, "contribution_to_active": 0.5983487963676453, "mix_mean": 0.6044840216636658, "mix_low": 0.0005502700805664062, "mix_high": 0.010465621948242188, "read_cosine": 0.2647949159145355, "beta_mean": 7.290927896974608e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006576914805918932, "mix_std": 0.26598525047302246, "beta_std": 8.850359154166654e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00895039364695549, "evidence_weights": [-0.2699369490146637, -0.20329700410366058, 0.6022191643714905], "agreement_weight": 0.44862842559814453, "orthogonal_weight": -1.4947903156280518}, {"head": 2, "memory_to_active": 0.016869019716978073, "contribution_to_active": 0.5521882772445679, "mix_mean": 0.5778709053993225, "mix_low": 0.00038623809814453125, "mix_high": 0.003914833068847656, "read_cosine": 0.034256454557180405, "beta_mean": 4.4639280531555414e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00714810099452734, "mix_std": 0.25823527574539185, "beta_std": 0.00015280485968105495, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0032408086117357016, "evidence_weights": [-0.22950716316699982, -0.26714542508125305, 0.5322771668434143], "agreement_weight": -0.14411485195159912, "orthogonal_weight": -0.4402081072330475}, {"head": 3, "memory_to_active": 0.32239896059036255, "contribution_to_active": 0.08690909296274185, "mix_mean": 0.24544182419776917, "mix_low": 0.04797554016113281, "mix_high": 0.000209808349609375, "read_cosine": 0.9158967733383179, "beta_mean": 0.0560525506734848, "nonfinite": 0.0, "weighted_memory_to_active": 0.03209606558084488, "mix_std": 0.22256480157375336, "beta_std": 0.05430108681321144, "beta_low": 0.0069580078125, "beta_high": 0.0, "orthogonal_to_active": 0.036868155002593994, "evidence_weights": [-0.11500956863164902, -0.14625056087970734, 0.13803622126579285], "agreement_weight": 0.0031102767679840326, "orthogonal_weight": 1.494862675666809}, {"head": 4, "memory_to_active": 0.002593982731923461, "contribution_to_active": 0.5074928998947144, "mix_mean": 0.5262611508369446, "mix_low": 0.0007009506225585938, "mix_high": 0.00020313262939453125, "read_cosine": 0.08202475309371948, "beta_mean": 1.6610578313702717e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0013199407840147614, "mix_std": 0.2503109574317932, "beta_std": 5.949995102128014e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008613279205746949, "evidence_weights": [-0.17964579164981842, -0.25600412487983704, 0.7253797650337219], "agreement_weight": 0.0733039453625679, "orthogonal_weight": -0.6768772602081299}, {"head": 5, "memory_to_active": 0.1324070543050766, "contribution_to_active": 0.41948485374450684, "mix_mean": 0.3827286958694458, "mix_low": 0.0023212432861328125, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8694084882736206, "beta_mean": 0.005662384908646345, "nonfinite": 0.0, "weighted_memory_to_active": 0.06530943512916565, "mix_std": 0.22759872674942017, "beta_std": 0.007258770056068897, "beta_low": 0.8653564453125, "beta_high": 0.0, "orthogonal_to_active": 0.018309619277715683, "evidence_weights": [-0.14609861373901367, -0.1590552031993866, 0.2894132435321808], "agreement_weight": 0.09188632667064667, "orthogonal_weight": -0.6272691488265991}]}, {"layer": 11, "memory_to_active": 0.13049528002738953, "contribution_to_active": 0.5789964199066162, "mix_mean": 0.47859087586402893, "mix_low": 0.08169396966695786, "mix_high": 0.04541302099823952, "read_cosine": 0.4051889479160309, "beta_mean": 0.030609378591179848, "nonfinite": 0.0, "weighted_memory_to_active": 0.0602625235915184, "mix_std": 0.3215692341327667, "beta_std": 0.04450440779328346, "beta_low": 0.5987142324447632, "beta_high": 0.0, "orthogonal_to_active": 0.04958384856581688, "output_influence": 0.5204936861991882, "residual_influence": 0.1304415762424469, "state_active": 0.1849127560853958, "state_memory": 0.006763821467757225, "state_peak": 24.556041717529297, "growth_active": 1.0108431577682495, "growth_memory": 1.607280969619751, "heads": [{"head": 0, "memory_to_active": 0.007787601090967655, "contribution_to_active": 0.5761626958847046, "mix_mean": 0.6206987500190735, "mix_low": 0.0011034011840820312, "mix_high": 0.008832931518554688, "read_cosine": 0.30161619186401367, "beta_mean": 0.00010275686508975923, "nonfinite": 0.0, "weighted_memory_to_active": 0.005246648099273443, "mix_std": 0.2622697055339813, "beta_std": 0.003963235765695572, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.0037811785005033016, "evidence_weights": [-0.16750961542129517, -0.3636319041252136, 0.6992988586425781], "agreement_weight": 0.25608131289482117, "orthogonal_weight": -0.7723082304000854}, {"head": 1, "memory_to_active": 0.327228844165802, "contribution_to_active": 0.3048306107521057, "mix_mean": 0.3524142801761627, "mix_low": 0.011330604553222656, "mix_high": 0.00018310546875, "read_cosine": 0.5753223299980164, "beta_mean": 0.02841649204492569, "nonfinite": 0.0, "weighted_memory_to_active": 0.1422596126794815, "mix_std": 0.2589598596096039, "beta_std": 0.0551285520195961, "beta_low": 0.4786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.0409257635474205, "evidence_weights": [-0.16504332423210144, -0.168204665184021, 0.4027777314186096], "agreement_weight": 0.07328851521015167, "orthogonal_weight": -0.3791849613189697}, {"head": 2, "memory_to_active": 0.012760954909026623, "contribution_to_active": 0.9756621718406677, "mix_mean": 0.4594872295856476, "mix_low": 0.1961984634399414, "mix_high": 0.16256046295166016, "read_cosine": 0.19744479656219482, "beta_mean": 0.07464194297790527, "nonfinite": 0.0, "weighted_memory_to_active": 0.00973578542470932, "mix_std": 0.43725281953811646, "beta_std": 0.09742734581232071, "beta_low": 0.44647216796875, "beta_high": 0.0, "orthogonal_to_active": 0.05514908581972122, "evidence_weights": [0.03691976144909859, -0.4449733793735504, -1.2675780057907104], "agreement_weight": 0.006350056733936071, "orthogonal_weight": 5.854552745819092}, {"head": 3, "memory_to_active": 0.01137570757418871, "contribution_to_active": 0.842033863067627, "mix_mean": 0.42377912998199463, "mix_low": 0.2695960998535156, "mix_high": 0.097198486328125, "read_cosine": 0.5556305646896362, "beta_mean": 0.05514278635382652, "nonfinite": 0.0, "weighted_memory_to_active": 0.006753726862370968, "mix_std": 0.4385993778705597, "beta_std": 0.08124189078807831, "beta_low": 0.485595703125, "beta_high": 0.0, "orthogonal_to_active": 0.03204496204853058, "evidence_weights": [0.1169603168964386, -0.33863207697868347, -0.3421080410480499], "agreement_weight": 0.04358038306236267, "orthogonal_weight": 6.172863483428955}, {"head": 4, "memory_to_active": 0.3290066421031952, "contribution_to_active": 0.4104328155517578, "mix_mean": 0.49532240629196167, "mix_low": 0.010233879089355469, "mix_high": 0.0008392333984375, "read_cosine": 0.10232967138290405, "beta_mean": 0.017782755196094513, "nonfinite": 0.0, "weighted_memory_to_active": 0.1540248990058899, "mix_std": 0.27135390043258667, "beta_std": 0.018150661140680313, "beta_low": 0.4168701171875, "beta_high": 0.0, "orthogonal_to_active": 0.1430848240852356, "evidence_weights": [-0.2258700579404831, -0.20239095389842987, 1.8488826751708984], "agreement_weight": -0.1845831423997879, "orthogonal_weight": -0.9445429444313049}, {"head": 5, "memory_to_active": 0.09481192380189896, "contribution_to_active": 0.364856094121933, "mix_mean": 0.5198432803153992, "mix_low": 0.00170135498046875, "mix_high": 0.0028638839721679688, "read_cosine": 0.6987900733947754, "beta_mean": 0.007569537498056889, "nonfinite": 0.0, "weighted_memory_to_active": 0.04355446249246597, "mix_std": 0.26097968220710754, "beta_std": 0.011114741675555706, "beta_low": 0.76507568359375, "beta_high": 0.0, "orthogonal_to_active": 0.02251727506518364, "evidence_weights": [-0.2179640680551529, -0.2890932559967041, 0.5735815167427063], "agreement_weight": 0.07330523431301117, "orthogonal_weight": -0.7175489664077759}]}, {"layer": 12, "memory_to_active": 0.2849196791648865, "contribution_to_active": 0.5612533092498779, "mix_mean": 0.49784570932388306, "mix_low": 0.010616779327392578, "mix_high": 0.0038855874445289373, "read_cosine": 0.24873989820480347, "beta_mean": 0.013972517102956772, "nonfinite": 0.0, "weighted_memory_to_active": 0.15418511629104614, "mix_std": 0.2611687481403351, "beta_std": 0.014617260545492172, "beta_low": 0.8239034414291382, "beta_high": 0.0, "orthogonal_to_active": 0.06599981337785721, "output_influence": 0.4455711841583252, "residual_influence": 0.1238471195101738, "state_active": 0.03891729936003685, "state_memory": 0.008875039406120777, "state_peak": 39.438087463378906, "growth_active": 1.1276178359985352, "growth_memory": 1.246849536895752, "heads": [{"head": 0, "memory_to_active": 0.05693085491657257, "contribution_to_active": 0.5104194283485413, "mix_mean": 0.49244630336761475, "mix_low": 0.0013484954833984375, "mix_high": 0.002079010009765625, "read_cosine": 0.29185807704925537, "beta_mean": 0.0009719132212921977, "nonfinite": 0.0, "weighted_memory_to_active": 0.027606535702943802, "mix_std": 0.27385571599006653, "beta_std": 0.004106980282813311, "beta_low": 0.99212646484375, "beta_high": 0.0, "orthogonal_to_active": 0.020294714719057083, "evidence_weights": [-0.18679210543632507, -0.2606310248374939, 0.30121707916259766], "agreement_weight": 0.10776828229427338, "orthogonal_weight": -0.7898338437080383}, {"head": 1, "memory_to_active": 0.10053971409797668, "contribution_to_active": 0.5033740997314453, "mix_mean": 0.5290782451629639, "mix_low": 0.0007982254028320312, "mix_high": 0.0007886886596679688, "read_cosine": 0.542035698890686, "beta_mean": 0.0009172058198601007, "nonfinite": 0.0, "weighted_memory_to_active": 0.05179879069328308, "mix_std": 0.2628828287124634, "beta_std": 0.005358065478503704, "beta_low": 0.9903564453125, "beta_high": 0.0, "orthogonal_to_active": 0.03999130800366402, "evidence_weights": [-0.20583753287792206, -0.2733106017112732, 0.9341117739677429], "agreement_weight": 0.13332056999206543, "orthogonal_weight": -0.9231017231941223}, {"head": 2, "memory_to_active": 0.5791370272636414, "contribution_to_active": 0.4466816782951355, "mix_mean": 0.3141198754310608, "mix_low": 0.028244972229003906, "mix_high": 0.0002536773681640625, "read_cosine": 0.43421807885169983, "beta_mean": 0.07982048392295837, "nonfinite": 0.0, "weighted_memory_to_active": 0.22242319583892822, "mix_std": 0.24640144407749176, "beta_std": 0.05943937227129936, "beta_low": 0.00189208984375, "beta_high": 0.0, "orthogonal_to_active": 0.21639162302017212, "evidence_weights": [-0.24232524633407593, -0.04720484837889671, -0.10006832331418991], "agreement_weight": 0.015963660553097725, "orthogonal_weight": 1.1642946004867554}, {"head": 3, "memory_to_active": 0.032174643129110336, "contribution_to_active": 0.7327665090560913, "mix_mean": 0.6928582787513733, "mix_low": 0.00019931793212890625, "mix_high": 0.013311386108398438, "read_cosine": 0.07264401018619537, "beta_mean": 0.0002657620352692902, "nonfinite": 0.0, "weighted_memory_to_active": 0.024742502719163895, "mix_std": 0.23507192730903625, "beta_std": 0.0048218946903944016, "beta_low": 0.997802734375, "beta_high": 0.0, "orthogonal_to_active": 0.023251118138432503, "evidence_weights": [-0.2927842438220978, -0.3373309075832367, 1.3161897659301758], "agreement_weight": 0.3598951995372772, "orthogonal_weight": -0.9595314264297485}, {"head": 4, "memory_to_active": 0.8752198219299316, "contribution_to_active": 0.6863037347793579, "mix_mean": 0.33576834201812744, "mix_low": 0.02701568603515625, "mix_high": 0.0023403167724609375, "read_cosine": -0.11614315956830978, "beta_mean": 0.0014258554438129067, "nonfinite": 0.0, "weighted_memory_to_active": 0.5574291944503784, "mix_std": 0.27294957637786865, "beta_std": 0.011479323729872704, "beta_low": 0.97198486328125, "beta_high": 0.0, "orthogonal_to_active": 0.06435459852218628, "evidence_weights": [-0.10470087826251984, -0.21134652197360992, 0.053390275686979294], "agreement_weight": -4.125831765122712e-06, "orthogonal_weight": -0.12258382886648178}, {"head": 5, "memory_to_active": 0.0655161663889885, "contribution_to_active": 0.48797407746315, "mix_mean": 0.6228031516075134, "mix_low": 0.0060939788818359375, "mix_high": 0.004540443420410156, "read_cosine": 0.2678266763687134, "beta_mean": 0.0004338761791586876, "nonfinite": 0.0, "weighted_memory_to_active": 0.04111047834157944, "mix_std": 0.27585095167160034, "beta_std": 0.0024979240261018276, "beta_low": 0.9892578125, "beta_high": 0.0, "orthogonal_to_active": 0.0317155197262764, "evidence_weights": [-0.2539937198162079, -0.31051889061927795, 1.2207705974578857], "agreement_weight": 0.29292288422584534, "orthogonal_weight": -0.8413736820220947}]}, {"layer": 13, "memory_to_active": 0.13377556204795837, "contribution_to_active": 0.5141269564628601, "mix_mean": 0.5073413252830505, "mix_low": 0.005504290573298931, "mix_high": 0.005872249603271484, "read_cosine": 0.427396297454834, "beta_mean": 0.01598316803574562, "nonfinite": 0.0, "weighted_memory_to_active": 0.05761130899190903, "mix_std": 0.2584802210330963, "beta_std": 0.018770117312669754, "beta_low": 0.7965495586395264, "beta_high": 0.0, "orthogonal_to_active": 0.05768401920795441, "output_influence": 0.4774937033653259, "residual_influence": 0.15892864763736725, "state_active": 0.10144783556461334, "state_memory": 0.037781015038490295, "state_peak": 94.39356994628906, "growth_active": 0.9325438737869263, "growth_memory": 1.017132043838501, "heads": [{"head": 0, "memory_to_active": 0.027630828320980072, "contribution_to_active": 0.5551049113273621, "mix_mean": 0.5663343667984009, "mix_low": 0.0005168914794921875, "mix_high": 0.001720428466796875, "read_cosine": 0.4078071117401123, "beta_mean": 0.00010190052853431553, "nonfinite": 0.0, "weighted_memory_to_active": 0.016541045159101486, "mix_std": 0.26837387681007385, "beta_std": 0.00023343726934399456, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.016206134110689163, "evidence_weights": [-0.203794926404953, -0.26595979928970337, 0.7653452754020691], "agreement_weight": 0.10780567675828934, "orthogonal_weight": -1.0904271602630615}, {"head": 1, "memory_to_active": 0.01819775626063347, "contribution_to_active": 0.582904577255249, "mix_mean": 0.535469114780426, "mix_low": 0.0019369125366210938, "mix_high": 0.0019884109497070312, "read_cosine": 0.22404108941555023, "beta_mean": 6.324613787000999e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010222104378044605, "mix_std": 0.2698521018028259, "beta_std": 0.0001049048951244913, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011912068352103233, "evidence_weights": [-0.18646787106990814, -0.27017369866371155, 0.7000441551208496], "agreement_weight": 0.2989331781864166, "orthogonal_weight": -1.2221014499664307}, {"head": 2, "memory_to_active": 0.11006619781255722, "contribution_to_active": 0.4322594404220581, "mix_mean": 0.36817821860313416, "mix_low": 0.0023899078369140625, "mix_high": 8.296966552734375e-05, "read_cosine": 0.6999861001968384, "beta_mean": 0.01513286679983139, "nonfinite": 0.0, "weighted_memory_to_active": 0.04802665859460831, "mix_std": 0.22514724731445312, "beta_std": 0.04470634460449219, "beta_low": 0.7952880859375, "beta_high": 0.0, "orthogonal_to_active": 0.02264036424458027, "evidence_weights": [-0.20052862167358398, -0.18663688004016876, 0.42666321992874146], "agreement_weight": 0.014151602983474731, "orthogonal_weight": -0.8066082000732422}, {"head": 3, "memory_to_active": 0.15482598543167114, "contribution_to_active": 0.5955151319503784, "mix_mean": 0.6323858499526978, "mix_low": 0.018312454223632812, "mix_high": 0.024878501892089844, "read_cosine": 0.03377878665924072, "beta_mean": 0.0011226353235542774, "nonfinite": 0.0, "weighted_memory_to_active": 0.11373081803321838, "mix_std": 0.29051491618156433, "beta_std": 0.008029577322304249, "beta_low": 0.9888916015625, "beta_high": 0.0, "orthogonal_to_active": 0.10970136523246765, "evidence_weights": [-0.33151915669441223, -0.245615154504776, 1.931357502937317], "agreement_weight": 3.8671371936798096, "orthogonal_weight": -1.0457035303115845}, {"head": 4, "memory_to_active": 0.4053373336791992, "contribution_to_active": 0.2483927309513092, "mix_mean": 0.32675012946128845, "mix_low": 0.007374763488769531, "mix_high": 1.621246337890625e-05, "read_cosine": 0.8742833733558655, "beta_mean": 0.07882994413375854, "nonfinite": 0.0, "weighted_memory_to_active": 0.10244116187095642, "mix_std": 0.2179650366306305, "beta_std": 0.054567426443099976, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.14068186283111572, "evidence_weights": [-0.1298365294933319, -0.23137669265270233, 0.4861127734184265], "agreement_weight": 0.00010965023102471605, "orthogonal_weight": 2.104783296585083}, {"head": 5, "memory_to_active": 0.08659522235393524, "contribution_to_active": 0.6705847382545471, "mix_mean": 0.6149301528930664, "mix_low": 0.00249481201171875, "mix_high": 0.006546974182128906, "read_cosine": 0.32448121905326843, "beta_mean": 0.0006484061013907194, "nonfinite": 0.0, "weighted_memory_to_active": 0.05470607429742813, "mix_std": 0.279028058052063, "beta_std": 0.004979004617780447, "beta_low": 0.99493408203125, "beta_high": 0.0, "orthogonal_to_active": 0.044962331652641296, "evidence_weights": [-0.22100822627544403, -0.29888391494750977, 0.7786829471588135], "agreement_weight": 0.1119488924741745, "orthogonal_weight": -0.9184108972549438}]}, {"layer": 14, "memory_to_active": 0.25651562213897705, "contribution_to_active": 0.4246276915073395, "mix_mean": 0.42087769508361816, "mix_low": 0.028413614258170128, "mix_high": 0.005379676818847656, "read_cosine": 0.6187096238136292, "beta_mean": 0.06770863384008408, "nonfinite": 0.0, "weighted_memory_to_active": 0.11462166905403137, "mix_std": 0.25073859095573425, "beta_std": 0.06370307505130768, "beta_low": 0.4553121030330658, "beta_high": 0.0, "orthogonal_to_active": 0.04467929154634476, "output_influence": 0.32242220640182495, "residual_influence": 0.12419852614402771, "state_active": 0.057862307876348495, "state_memory": 0.016014613211154938, "state_peak": 37.305599212646484, "growth_active": 1.0625500679016113, "growth_memory": 1.3008114099502563, "heads": [{"head": 0, "memory_to_active": 0.36181652545928955, "contribution_to_active": 0.21055564284324646, "mix_mean": 0.2124098837375641, "mix_low": 0.019214630126953125, "mix_high": 5.340576171875e-05, "read_cosine": 0.7805254459381104, "beta_mean": 0.06113428249955177, "nonfinite": 0.0, "weighted_memory_to_active": 0.09515755623579025, "mix_std": 0.20276881754398346, "beta_std": 0.05759091302752495, "beta_low": 0.01971435546875, "beta_high": 0.0, "orthogonal_to_active": 0.07230360805988312, "evidence_weights": [-0.13618521392345428, -0.04170644283294678, 0.5229302644729614], "agreement_weight": 0.026013826951384544, "orthogonal_weight": 1.210864543914795}, {"head": 1, "memory_to_active": 0.5051157474517822, "contribution_to_active": 0.3286161720752716, "mix_mean": 0.2346496284008026, "mix_low": 0.1335296630859375, "mix_high": 0.000141143798828125, "read_cosine": 0.7903891801834106, "beta_mean": 0.27694448828697205, "nonfinite": 0.0, "weighted_memory_to_active": 0.23194251954555511, "mix_std": 0.26616305112838745, "beta_std": 0.2632608413696289, "beta_low": 0.102294921875, "beta_high": 0.0, "orthogonal_to_active": 0.015267783775925636, "evidence_weights": [-0.008378575555980206, -0.23706217110157013, -0.14781731367111206], "agreement_weight": 0.052222952246665955, "orthogonal_weight": 0.14632444083690643}, {"head": 2, "memory_to_active": 0.16354021430015564, "contribution_to_active": 0.5626968145370483, "mix_mean": 0.5726546049118042, "mix_low": 0.0029630661010742188, "mix_high": 0.0037050247192382812, "read_cosine": 0.4040716886520386, "beta_mean": 0.0032968209125101566, "nonfinite": 0.0, "weighted_memory_to_active": 0.10285764932632446, "mix_std": 0.26954472064971924, "beta_std": 0.0044684577733278275, "beta_low": 0.93975830078125, "beta_high": 0.0, "orthogonal_to_active": 0.07803957164287567, "evidence_weights": [-0.154938206076622, -0.30707690119743347, 0.9159047603607178], "agreement_weight": 0.08347209542989731, "orthogonal_weight": -0.8599600195884705}, {"head": 3, "memory_to_active": 0.06178252398967743, "contribution_to_active": 0.6499892473220825, "mix_mean": 0.672768771648407, "mix_low": 0.0056171417236328125, "mix_high": 0.018976211547851562, "read_cosine": 0.17023251950740814, "beta_mean": 0.009212326258420944, "nonfinite": 0.0, "weighted_memory_to_active": 0.04012411832809448, "mix_std": 0.25071051716804504, "beta_std": 0.009737815707921982, "beta_low": 0.6832275390625, "beta_high": 0.0, "orthogonal_to_active": 0.02548319101333618, "evidence_weights": [-0.370309442281723, -0.322659432888031, 0.4067336916923523], "agreement_weight": 0.08318185061216354, "orthogonal_weight": -0.7320902347564697}, {"head": 4, "memory_to_active": 0.19079738855361938, "contribution_to_active": 0.4727252721786499, "mix_mean": 0.4882469177246094, "mix_low": 0.0045623779296875, "mix_high": 0.008477210998535156, "read_cosine": 0.8305201530456543, "beta_mean": 0.053677089512348175, "nonfinite": 0.0, "weighted_memory_to_active": 0.10991578549146652, "mix_std": 0.2646845877170563, "beta_std": 0.03798612579703331, "beta_low": 0.0174560546875, "beta_high": 0.0, "orthogonal_to_active": 0.03375404328107834, "evidence_weights": [-0.22234703600406647, -0.25712230801582336, 0.42665553092956543], "agreement_weight": 0.12360213696956635, "orthogonal_weight": -0.5778607726097107}, {"head": 5, "memory_to_active": 0.2560412287712097, "contribution_to_active": 0.32318294048309326, "mix_mean": 0.34453633427619934, "mix_low": 0.0045948028564453125, "mix_high": 0.0009250640869140625, "read_cosine": 0.7365185022354126, "beta_mean": 0.0019867485389113426, "nonfinite": 0.0, "weighted_memory_to_active": 0.10773233324289322, "mix_std": 0.25055992603302, "beta_std": 0.009174313396215439, "beta_low": 0.96942138671875, "beta_high": 0.0, "orthogonal_to_active": 0.043227531015872955, "evidence_weights": [-0.18176132440567017, -0.15978634357452393, 0.2318195402622223], "agreement_weight": 0.08353415131568909, "orthogonal_weight": -0.6502602696418762}]}, {"layer": 15, "memory_to_active": 0.0610598661005497, "contribution_to_active": 0.4837062954902649, "mix_mean": 0.40107008814811707, "mix_low": 0.00836451817303896, "mix_high": 0.00226147985085845, "read_cosine": 0.6799676418304443, "beta_mean": 0.038003675639629364, "nonfinite": 0.0, "weighted_memory_to_active": 0.028393274173140526, "mix_std": 0.28315967321395874, "beta_std": 0.043689046055078506, "beta_low": 0.7188924551010132, "beta_high": 0.0, "orthogonal_to_active": 0.006474833004176617, "output_influence": 0.3019970953464508, "residual_influence": 0.16153652966022491, "state_active": 0.12771379947662354, "state_memory": 0.009711192920804024, "state_peak": 39.70970153808594, "growth_active": 1.2427290678024292, "growth_memory": 1.5425300598144531, "heads": [{"head": 0, "memory_to_active": 0.10318693518638611, "contribution_to_active": 0.47523072361946106, "mix_mean": 0.3646185100078583, "mix_low": 0.006319999694824219, "mix_high": 0.003930091857910156, "read_cosine": 0.6839725971221924, "beta_mean": 0.030192837119102478, "nonfinite": 0.0, "weighted_memory_to_active": 0.04709591716527939, "mix_std": 0.27560994029045105, "beta_std": 0.02371710166335106, "beta_low": 0.16650390625, "beta_high": 0.0, "orthogonal_to_active": 0.020815229043364525, "evidence_weights": [-0.1662217676639557, -0.17400814592838287, 0.35436418652534485], "agreement_weight": 0.0922684445977211, "orthogonal_weight": -0.692758321762085}, {"head": 1, "memory_to_active": 0.1359444111585617, "contribution_to_active": 0.4133736193180084, "mix_mean": 0.4091625213623047, "mix_low": 0.0033178329467773438, "mix_high": 0.003520965576171875, "read_cosine": 0.8994718790054321, "beta_mean": 0.000827689771540463, "nonfinite": 0.0, "weighted_memory_to_active": 0.05443699657917023, "mix_std": 0.2848115563392639, "beta_std": 0.0022680668625980616, "beta_low": 0.98992919921875, "beta_high": 0.0, "orthogonal_to_active": 0.0008266259683296084, "evidence_weights": [-0.1644797921180725, -0.173026904463768, 0.24072453379631042], "agreement_weight": 0.10808458924293518, "orthogonal_weight": -0.026054061949253082}, {"head": 2, "memory_to_active": 0.007986259646713734, "contribution_to_active": 0.4804667532444, "mix_mean": 0.33984312415122986, "mix_low": 0.01598358154296875, "mix_high": 2.86102294921875e-05, "read_cosine": 0.8093968629837036, "beta_mean": 0.0001345831115031615, "nonfinite": 0.0, "weighted_memory_to_active": 0.003926502075046301, "mix_std": 0.2722359001636505, "beta_std": 0.00028554105665534735, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0005617857095785439, "evidence_weights": [-0.2032807469367981, -0.2766827642917633, 0.24077953398227692], "agreement_weight": 0.011329380795359612, "orthogonal_weight": 0.35252442955970764}, {"head": 3, "memory_to_active": 0.008928262628614902, "contribution_to_active": 0.40082475543022156, "mix_mean": 0.363533616065979, "mix_low": 0.006663322448730469, "mix_high": 0.0001239776611328125, "read_cosine": 0.5830547213554382, "beta_mean": 6.9188725319691e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0033545661717653275, "mix_std": 0.2812846899032593, "beta_std": 0.00015735006309114397, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00281751761212945, "evidence_weights": [-0.2209969311952591, -0.25230440497398376, 0.3191167414188385], "agreement_weight": 0.027164733037352562, "orthogonal_weight": -0.9688920974731445}, {"head": 4, "memory_to_active": 0.08637073636054993, "contribution_to_active": 0.6173142194747925, "mix_mean": 0.3747856020927429, "mix_low": 0.008380889892578125, "mix_high": 0.000873565673828125, "read_cosine": 0.7386190891265869, "beta_mean": 0.1948939859867096, "nonfinite": 0.0, "weighted_memory_to_active": 0.04580682888627052, "mix_std": 0.3066422939300537, "beta_std": 0.2218582183122635, "beta_low": 0.18206787109375, "beta_high": 0.0, "orthogonal_to_active": 0.005163693800568581, "evidence_weights": [-0.21893975138664246, -0.260349303483963, 0.008649960160255432], "agreement_weight": 0.013353320769965649, "orthogonal_weight": 0.2371475100517273}, {"head": 5, "memory_to_active": 0.02394258975982666, "contribution_to_active": 0.5150277614593506, "mix_mean": 0.5544770359992981, "mix_low": 0.009521484375, "mix_high": 0.005091667175292969, "read_cosine": 0.36529070138931274, "beta_mean": 0.0019037649035453796, "nonfinite": 0.0, "weighted_memory_to_active": 0.015738841146230698, "mix_std": 0.2783735394477844, "beta_std": 0.013848010450601578, "beta_low": 0.9749755859375, "beta_high": 0.0, "orthogonal_to_active": 0.008664142340421677, "evidence_weights": [-0.3210391700267792, -0.3150617778301239, 0.7410833835601807], "agreement_weight": 0.25852257013320923, "orthogonal_weight": 0.704189121723175}]}]}, {"step": 35000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9050334690000454, "memory_to_active": 0.8651511669158936, "contribution_to_active": 0.5332424640655518, "mix_mean": 0.37684357166290283, "mix_low": 0.15012292563915253, "mix_high": 0.009043365716934204, "read_cosine": 0.37022897601127625, "beta_mean": 0.07943886518478394, "nonfinite": 0.0, "weighted_memory_to_active": 0.21734455227851868, "mix_std": 0.21822845935821533, "beta_std": 0.06792989373207092, "beta_low": 0.5157839059829712, "beta_high": 2.288818359375e-05, "orthogonal_to_active": 0.05746623873710632, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1297903060913086, "normal_token_accuracy": 0.42936596274375916, "state_active": 0.061223819851875305, "state_memory": 0.019073711708188057, "state_peak": 112.90807342529297, "growth_active": 1.1338391304016113, "growth_memory": 1.366588830947876, "output_influence": 0.38416340947151184, "residual_influence": 0.12481598556041718, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2450735569000244, "no_memory_loss_delta": 0.438440203666687, "scrambled_loss_delta": 0.12690021097660065, "memory_only_loss_delta": 3.2840049266815186, "no_evidence_loss_delta": 0.24455232918262482, "no_agreement_loss_delta": 0.023581575602293015, "no_orthogonal_loss_delta": 0.006765021942555904}, {"index": 94, "tokens": 256, "normal_loss": 3.1980388164520264, "no_memory_loss_delta": 0.5967282652854919, "scrambled_loss_delta": 0.39363521337509155, "memory_only_loss_delta": 3.278982400894165, "no_evidence_loss_delta": 0.27592504024505615, "no_agreement_loss_delta": 0.033739589154720306, "no_orthogonal_loss_delta": 0.04615422338247299}, {"index": 188, "tokens": 256, "normal_loss": 3.4834930896759033, "no_memory_loss_delta": 0.5529905557632446, "scrambled_loss_delta": 0.12504352629184723, "memory_only_loss_delta": 2.5353517532348633, "no_evidence_loss_delta": 0.2944486737251282, "no_agreement_loss_delta": 0.035091184079647064, "no_orthogonal_loss_delta": 0.04045972600579262}, {"index": 283, "tokens": 131, "normal_loss": 2.079927682876587, "no_memory_loss_delta": 0.2610457241535187, "scrambled_loss_delta": 0.19087032973766327, "memory_only_loss_delta": 2.665069341659546, "no_evidence_loss_delta": 0.07331469655036926, "no_agreement_loss_delta": -0.007183216046541929, "no_orthogonal_loss_delta": -0.007143401075154543}], "agreement_weight_abs_mean": 0.16122868657112122, "agreement_weight_abs_max": 3.9835567474365234, "agreement_weight_mean": 0.1401677280664444, "orthogonal_weight_abs_mean": 0.711780846118927, "orthogonal_weight_abs_max": 6.335473537445068, "orthogonal_weight_mean": -0.14716500043869019, "no_memory_loss": 3.62007474899292, "no_memory_loss_delta": 0.49028441309928894, "no_memory_token_accuracy_gain": 0.07007786631584167, "no_memory_token_win_fraction": 0.6618465185165405, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 38.754782001148605, "scrambled_loss": 3.3414390087127686, "scrambled_loss_delta": 0.21164877712726593, "scrambled_token_accuracy_gain": 0.021134594455361366, "scrambled_token_win_fraction": 0.5984427332878113, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.075112562670444, "memory_only_loss": 6.108988285064697, "memory_only_loss_delta": 2.9791977405548096, "memory_only_token_accuracy_gain": 0.25583982467651367, "memory_only_token_win_fraction": 0.8943270444869995, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.9166400754366, "no_evidence_loss": 3.372532606124878, "no_evidence_loss_delta": 0.24274224042892456, "no_evidence_token_accuracy_gain": 0.026696329936385155, "no_evidence_token_win_fraction": 0.6173526048660278, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 21.552630668371016, "no_agreement_loss": 3.1550588607788086, "no_agreement_loss_delta": 0.02526869811117649, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.47608453035354614, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.495211669795263, "no_orthogonal_loss": 3.1553401947021484, "no_orthogonal_loss_delta": 0.02554975636303425, "no_orthogonal_token_accuracy_gain": 0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5539488196372986, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.5226123443778885, "short_no_memory_loss_delta": 0.3976423144340515, "long_context_memory_gain": 0.09264209866523743, "normal_context_loss_gain": 0.14941291511058807, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.221641778945923, "contribution_to_active": 1.0495061874389648, "mix_mean": 0.10229576379060745, "mix_low": 0.5508807897567749, "mix_high": 0.00428009033203125, "read_cosine": 0.44142770767211914, "beta_mean": 0.2421611249446869, "nonfinite": 0.0, "weighted_memory_to_active": 1.1046125888824463, "mix_std": 0.0886615514755249, "beta_std": 0.14466392993927002, "beta_low": 0.0242411307990551, "beta_high": 0.0, "orthogonal_to_active": 0.06026493012905121, "output_influence": 0.1848861575126648, "residual_influence": 0.2210649996995926, "state_active": 0.06619841605424881, "state_memory": 0.0468049980700016, "state_peak": 50.8215446472168, "growth_active": 1.1873953342437744, "growth_memory": 1.25178861618042, "heads": [{"head": 0, "memory_to_active": 2.435366153717041, "contribution_to_active": 0.438609778881073, "mix_mean": 0.019498534500598907, "mix_low": 0.615574836730957, "mix_high": 0.0, "read_cosine": 0.24294613301753998, "beta_mean": 0.21647565066814423, "nonfinite": 0.0, "weighted_memory_to_active": 0.522087812423706, "mix_std": 0.06033904105424881, "beta_std": 0.09954402595758438, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.038962144404649734, "evidence_weights": [-0.022359080612659454, -0.07063020765781403, -0.11671742796897888], "agreement_weight": -0.12271834909915924, "orthogonal_weight": -0.08282431960105896}, {"head": 1, "memory_to_active": 5.335635185241699, "contribution_to_active": 2.197422981262207, "mix_mean": 0.015467841178178787, "mix_low": 0.6991071701049805, "mix_high": 0.0, "read_cosine": 0.4961090087890625, "beta_mean": 0.44396108388900757, "nonfinite": 0.0, "weighted_memory_to_active": 2.557079792022705, "mix_std": 0.0528125986456871, "beta_std": 0.19620837271213531, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10711844265460968, "evidence_weights": [0.03137611225247383, -0.09947358071804047, 0.39812949299812317], "agreement_weight": 0.027690133079886436, "orthogonal_weight": -0.08553308993577957}, {"head": 2, "memory_to_active": 2.631946086883545, "contribution_to_active": 0.46756282448768616, "mix_mean": 0.013588540256023407, "mix_low": 0.6758928298950195, "mix_high": 0.0, "read_cosine": 0.2980424165725708, "beta_mean": 0.27711182832717896, "nonfinite": 0.0, "weighted_memory_to_active": 0.5493308305740356, "mix_std": 0.030285075306892395, "beta_std": 0.2021632194519043, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05222081393003464, "evidence_weights": [0.061024103313684464, 0.07045584172010422, 0.02899968810379505], "agreement_weight": 0.004397804848849773, "orthogonal_weight": -0.19247916340827942}, {"head": 3, "memory_to_active": 6.445923805236816, "contribution_to_active": 2.71359920501709, "mix_mean": 0.03733647242188454, "mix_low": 0.4648628234863281, "mix_high": 0.0, "read_cosine": 0.34665441513061523, "beta_mean": 0.33277297019958496, "nonfinite": 0.0, "weighted_memory_to_active": 2.8882858753204346, "mix_std": 0.08518753200769424, "beta_std": 0.23114094138145447, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.13836728036403656, "evidence_weights": [-0.05135564133524895, -0.06746336817741394, -0.131367489695549], "agreement_weight": -0.012399398721754551, "orthogonal_weight": -0.057264551520347595}, {"head": 4, "memory_to_active": 2.369422435760498, "contribution_to_active": 0.0701630711555481, "mix_mean": 0.006406478583812714, "mix_low": 0.8310213088989258, "mix_high": 0.0, "read_cosine": 0.33475881814956665, "beta_mean": 0.1482561081647873, "nonfinite": 0.0, "weighted_memory_to_active": 0.05566994100809097, "mix_std": 0.013781649060547352, "beta_std": 0.10976341366767883, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005549327470362186, "evidence_weights": [0.12831339240074158, 0.07982099801301956, -0.3650430142879486], "agreement_weight": -0.133021742105484, "orthogonal_weight": 0.09667535871267319}, {"head": 5, "memory_to_active": 0.11155682802200317, "contribution_to_active": 0.4096795320510864, "mix_mean": 0.5214767456054688, "mix_low": 0.018825531005859375, "mix_high": 0.0256805419921875, "read_cosine": 0.930055558681488, "beta_mean": 0.034388989210128784, "nonfinite": 0.0, "weighted_memory_to_active": 0.05522116273641586, "mix_std": 0.2895633578300476, "beta_std": 0.029163528233766556, "beta_low": 0.14544677734375, "beta_high": 0.0, "orthogonal_to_active": 0.01937154307961464, "evidence_weights": [-0.2272605448961258, -0.23722365498542786, 0.3112000524997711], "agreement_weight": 0.26237785816192627, "orthogonal_weight": -0.6828757524490356}]}, {"layer": 1, "memory_to_active": 2.3010520935058594, "contribution_to_active": 0.5267131924629211, "mix_mean": 0.1841079592704773, "mix_low": 0.37562960386276245, "mix_high": 0.0008029937744140625, "read_cosine": 0.3292590379714966, "beta_mean": 0.18703311681747437, "nonfinite": 0.0, "weighted_memory_to_active": 0.400856077671051, "mix_std": 0.11886435747146606, "beta_std": 0.13310691714286804, "beta_low": 0.2685648798942566, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.05846808850765228, "output_influence": 0.1926499456167221, "residual_influence": 0.048087626695632935, "state_active": 0.039165887981653214, "state_memory": 0.03671783581376076, "state_peak": 32.81772994995117, "growth_active": 1.274200439453125, "growth_memory": 1.501732349395752, "heads": [{"head": 0, "memory_to_active": 0.04896199703216553, "contribution_to_active": 0.4294736683368683, "mix_mean": 0.4386524558067322, "mix_low": 0.011149406433105469, "mix_high": 0.0008296966552734375, "read_cosine": 0.056846313178539276, "beta_mean": 0.0003559262549970299, "nonfinite": 0.0, "weighted_memory_to_active": 0.026115139946341515, "mix_std": 0.28389331698417664, "beta_std": 0.0004913508309982717, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.025335172191262245, "evidence_weights": [-0.18862764537334442, -0.24443092942237854, 0.10958084464073181], "agreement_weight": 0.16640934348106384, "orthogonal_weight": -1.0247540473937988}, {"head": 1, "memory_to_active": 2.8306078910827637, "contribution_to_active": 0.23093964159488678, "mix_mean": 0.021053995937108994, "mix_low": 0.4833965301513672, "mix_high": 9.5367431640625e-07, "read_cosine": 0.47975432872772217, "beta_mean": 0.3194012939929962, "nonfinite": 0.0, "weighted_memory_to_active": 0.24782253801822662, "mix_std": 0.04246341437101364, "beta_std": 0.1957576721906662, "beta_low": 0.00140380859375, "beta_high": 0.000244140625, "orthogonal_to_active": 0.015896987169981003, "evidence_weights": [0.012381091713905334, -0.02722153440117836, 0.0006249513826332986], "agreement_weight": -0.013212474063038826, "orthogonal_weight": 0.13792657852172852}, {"head": 2, "memory_to_active": 0.28737857937812805, "contribution_to_active": 0.5743580460548401, "mix_mean": 0.5842255353927612, "mix_low": 0.0015621185302734375, "mix_high": 0.0039806365966796875, "read_cosine": 0.69830721616745, "beta_mean": 0.01829717494547367, "nonfinite": 0.0, "weighted_memory_to_active": 0.235512375831604, "mix_std": 0.25018590688705444, "beta_std": 0.022201284766197205, "beta_low": 0.53155517578125, "beta_high": 0.0, "orthogonal_to_active": 0.05452148616313934, "evidence_weights": [-0.2958656847476959, -0.3533683717250824, 0.40901699662208557], "agreement_weight": 0.026303162798285484, "orthogonal_weight": -0.40080487728118896}, {"head": 3, "memory_to_active": 2.8218321800231934, "contribution_to_active": 0.24346861243247986, "mix_mean": 0.018318386748433113, "mix_low": 0.6505022048950195, "mix_high": 4.76837158203125e-06, "read_cosine": 0.2366349995136261, "beta_mean": 0.33041664958000183, "nonfinite": 0.0, "weighted_memory_to_active": 0.3751941919326782, "mix_std": 0.05303352326154709, "beta_std": 0.248529314994812, "beta_low": 0.03643798828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.09285740554332733, "evidence_weights": [0.04461248219013214, -0.002545506926253438, 0.11890139430761337], "agreement_weight": -0.024845192208886147, "orthogonal_weight": -0.3718247413635254}, {"head": 4, "memory_to_active": 3.6138017177581787, "contribution_to_active": 1.603119134902954, "mix_mean": 0.034328337758779526, "mix_low": 0.3031320571899414, "mix_high": 0.0, "read_cosine": 0.335629403591156, "beta_mean": 0.1541808843612671, "nonfinite": 0.0, "weighted_memory_to_active": 1.4548842906951904, "mix_std": 0.06351029872894287, "beta_std": 0.10938235372304916, "beta_low": 0.0103759765625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.15360048413276672, "evidence_weights": [-0.04043397307395935, 0.01158562395721674, 0.005322892684489489], "agreement_weight": -0.023405637592077255, "orthogonal_weight": 0.13550230860710144}, {"head": 5, "memory_to_active": 4.203729152679443, "contribution_to_active": 0.07891994714736938, "mix_mean": 0.008069013245403767, "mix_low": 0.8040351867675781, "mix_high": 1.9073486328125e-06, "read_cosine": 0.16838189959526062, "beta_mean": 0.29954662919044495, "nonfinite": 0.0, "weighted_memory_to_active": 0.06560775637626648, "mix_std": 0.020099621266126633, "beta_std": 0.22227951884269714, "beta_low": 0.03173828125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.008596984669566154, "evidence_weights": [0.1595766693353653, 0.07801073789596558, 0.24857300519943237], "agreement_weight": -0.1383664309978485, "orthogonal_weight": 0.12782447040081024}]}, {"layer": 2, "memory_to_active": 1.022869348526001, "contribution_to_active": 0.33325108885765076, "mix_mean": 0.3579547703266144, "mix_low": 0.24987603724002838, "mix_high": 0.0033372244797647, "read_cosine": 0.33652037382125854, "beta_mean": 0.0924285501241684, "nonfinite": 0.0, "weighted_memory_to_active": 0.06909441202878952, "mix_std": 0.1936754435300827, "beta_std": 0.07229230552911758, "beta_low": 0.552154541015625, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.01614367589354515, "output_influence": 0.33711811900138855, "residual_influence": 0.08568251132965088, "state_active": 0.03654531016945839, "state_memory": 0.01761571876704693, "state_peak": 33.237831115722656, "growth_active": 0.9977858662605286, "growth_memory": 1.6063169240951538, "heads": [{"head": 0, "memory_to_active": 0.0060396878980100155, "contribution_to_active": 0.3236548900604248, "mix_mean": 0.5492610931396484, "mix_low": 0.0019941329956054688, "mix_high": 0.005829811096191406, "read_cosine": 0.18480831384658813, "beta_mean": 4.8879646783461794e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0033429020550101995, "mix_std": 0.29012006521224976, "beta_std": 6.409655907191336e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015484127216041088, "evidence_weights": [-0.3191852867603302, -0.19924519956111908, 0.1088951826095581], "agreement_weight": 0.2212379425764084, "orthogonal_weight": -0.4872492253780365}, {"head": 1, "memory_to_active": 3.07894229888916, "contribution_to_active": 0.0651995912194252, "mix_mean": 0.005472605116665363, "mix_low": 0.8817262649536133, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21037402749061584, "beta_mean": 0.28017523884773254, "nonfinite": 0.0, "weighted_memory_to_active": 0.04950934648513794, "mix_std": 0.012837127782404423, "beta_std": 0.16031289100646973, "beta_low": 0.00018310546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.020253315567970276, "evidence_weights": [0.10249578952789307, 0.09750895947217941, 0.5087276697158813], "agreement_weight": 0.04758162423968315, "orthogonal_weight": 0.4429757297039032}, {"head": 2, "memory_to_active": 0.01668291911482811, "contribution_to_active": 0.4161608815193176, "mix_mean": 0.4835032522678375, "mix_low": 0.00176239013671875, "mix_high": 0.0016145706176757812, "read_cosine": 0.26157134771347046, "beta_mean": 3.289732558187097e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006099929101765156, "mix_std": 0.27990400791168213, "beta_std": 6.403746374417096e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014003159012645483, "evidence_weights": [-0.2336590737104416, -0.27192509174346924, 0.4480755031108856], "agreement_weight": 0.03282075747847557, "orthogonal_weight": -0.2534007728099823}, {"head": 3, "memory_to_active": 0.17380616068840027, "contribution_to_active": 0.4894537031650543, "mix_mean": 0.48583149909973145, "mix_low": 0.0007295608520507812, "mix_high": 0.0055694580078125, "read_cosine": 0.6624947786331177, "beta_mean": 0.034291673451662064, "nonfinite": 0.0, "weighted_memory_to_active": 0.09599071741104126, "mix_std": 0.26679715514183044, "beta_std": 0.04287382587790489, "beta_low": 0.28466796875, "beta_high": 0.0, "orthogonal_to_active": 0.059236086905002594, "evidence_weights": [-0.17658846080303192, -0.279441773891449, 0.8652951121330261], "agreement_weight": 0.060516417026519775, "orthogonal_weight": -0.8871506452560425}, {"head": 4, "memory_to_active": 2.848724126815796, "contribution_to_active": 0.19875994324684143, "mix_mean": 0.01711880974471569, "mix_low": 0.6112070083618164, "mix_high": 0.0, "read_cosine": 0.21943633258342743, "beta_mean": 0.23998206853866577, "nonfinite": 0.0, "weighted_memory_to_active": 0.2515302300453186, "mix_std": 0.041195958852767944, "beta_std": 0.23038071393966675, "beta_low": 0.028076171875, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.014149881899356842, "evidence_weights": [0.02264063060283661, 0.01451214961707592, 0.0791119709610939], "agreement_weight": 0.017594635486602783, "orthogonal_weight": -0.06715109199285507}, {"head": 5, "memory_to_active": 0.01302002090960741, "contribution_to_active": 0.5062774419784546, "mix_mean": 0.6065413951873779, "mix_low": 0.0018367767333984375, "mix_high": 0.007008552551269531, "read_cosine": 0.4804374575614929, "beta_mean": 4.045980313094333e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008093303069472313, "mix_std": 0.2711982727050781, "beta_std": 5.8228491980116814e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00027403858257457614, "evidence_weights": [-0.26744580268859863, -0.2862392067909241, 0.9267728328704834], "agreement_weight": 0.08275625854730606, "orthogonal_weight": 0.052937086671590805}]}, {"layer": 3, "memory_to_active": 0.7035432457923889, "contribution_to_active": 0.4028621315956116, "mix_mean": 0.3661731481552124, "mix_low": 0.12668800354003906, "mix_high": 0.007216453552246094, "read_cosine": 0.22900380194187164, "beta_mean": 0.03947756066918373, "nonfinite": 0.0, "weighted_memory_to_active": 0.16839566826820374, "mix_std": 0.24247577786445618, "beta_std": 0.03669264167547226, "beta_low": 0.571197509765625, "beta_high": 0.0, "orthogonal_to_active": 0.07187183946371078, "output_influence": 0.3305860757827759, "residual_influence": 0.06672751158475876, "state_active": 0.022901155054569244, "state_memory": 0.009974841959774494, "state_peak": 17.622297286987305, "growth_active": 1.0266327857971191, "growth_memory": 1.9834799766540527, "heads": [{"head": 0, "memory_to_active": 0.4693482518196106, "contribution_to_active": 0.5228900909423828, "mix_mean": 0.4747118353843689, "mix_low": 0.014514923095703125, "mix_high": 0.012853622436523438, "read_cosine": 0.23381417989730835, "beta_mean": 0.013925105333328247, "nonfinite": 0.0, "weighted_memory_to_active": 0.2625907361507416, "mix_std": 0.3202262818813324, "beta_std": 0.02104390412569046, "beta_low": 0.584716796875, "beta_high": 0.0, "orthogonal_to_active": 0.1785237193107605, "evidence_weights": [-0.18809813261032104, -0.2193789780139923, 0.7588688731193542], "agreement_weight": 0.1258513331413269, "orthogonal_weight": -0.7138631343841553}, {"head": 1, "memory_to_active": 0.01771301031112671, "contribution_to_active": 0.36681699752807617, "mix_mean": 0.4565763473510742, "mix_low": 0.01177978515625, "mix_high": 0.004245758056640625, "read_cosine": 0.19100350141525269, "beta_mean": 9.725992276798934e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008714718744158745, "mix_std": 0.3077083230018616, "beta_std": 0.00018525136692915112, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005346095655113459, "evidence_weights": [-0.22080767154693604, -0.1888570636510849, 0.5879459381103516], "agreement_weight": 0.38127240538597107, "orthogonal_weight": -0.670354425907135}, {"head": 2, "memory_to_active": 2.4637415409088135, "contribution_to_active": 0.33233949542045593, "mix_mean": 0.04379495978355408, "mix_low": 0.39080333709716797, "mix_high": 2.86102294921875e-06, "read_cosine": 0.21249324083328247, "beta_mean": 0.17514389753341675, "nonfinite": 0.0, "weighted_memory_to_active": 0.4018532335758209, "mix_std": 0.09942550212144852, "beta_std": 0.14029580354690552, "beta_low": 0.0113525390625, "beta_high": 0.0, "orthogonal_to_active": 0.00011003336840076372, "evidence_weights": [-0.06923264265060425, 0.017103897407650948, -0.009878458455204964], "agreement_weight": -0.0044366661459207535, "orthogonal_weight": -0.00036084005841985345}, {"head": 3, "memory_to_active": 0.022560732439160347, "contribution_to_active": 0.4906270205974579, "mix_mean": 0.5470075607299805, "mix_low": 0.010433197021484375, "mix_high": 0.010024070739746094, "read_cosine": 0.29389292001724243, "beta_mean": 9.504525951342657e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014458317309617996, "mix_std": 0.30325958132743835, "beta_std": 0.00023372145369648933, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0037335678935050964, "evidence_weights": [-0.216211199760437, -0.2244931310415268, 1.9722743034362793], "agreement_weight": 1.4817978143692017, "orthogonal_weight": -0.33766642212867737}, {"head": 4, "memory_to_active": 0.9561964273452759, "contribution_to_active": 0.08600983023643494, "mix_mean": 0.07089492678642273, "mix_low": 0.32257938385009766, "mix_high": 1.621246337890625e-05, "read_cosine": 0.1655709594488144, "beta_mean": 0.018170826137065887, "nonfinite": 0.0, "weighted_memory_to_active": 0.11934021860361099, "mix_std": 0.1297326385974884, "beta_std": 0.030781500041484833, "beta_low": 0.57489013671875, "beta_high": 0.0, "orthogonal_to_active": 0.09318259358406067, "evidence_weights": [0.038544636219739914, -0.019103853031992912, 0.23740406334400177], "agreement_weight": -0.004621066618710756, "orthogonal_weight": -0.8010364770889282}, {"head": 5, "memory_to_active": 0.291699081659317, "contribution_to_active": 0.6184892654418945, "mix_mean": 0.6040531396865845, "mix_low": 0.01001739501953125, "mix_high": 0.01615619659423828, "read_cosine": 0.27724796533584595, "beta_mean": 0.02943320758640766, "nonfinite": 0.0, "weighted_memory_to_active": 0.20341673493385315, "mix_std": 0.2945021986961365, "beta_std": 0.027615638449788094, "beta_low": 0.2562255859375, "beta_high": 0.0, "orthogonal_to_active": 0.15033502876758575, "evidence_weights": [-0.22999705374240875, -0.26386958360671997, 0.494878351688385], "agreement_weight": 0.13048814237117767, "orthogonal_weight": -0.8114894032478333}]}, {"layer": 4, "memory_to_active": 1.1522600650787354, "contribution_to_active": 0.44423162937164307, "mix_mean": 0.3483884334564209, "mix_low": 0.27296894788742065, "mix_high": 0.006629308685660362, "read_cosine": 0.3578040599822998, "beta_mean": 0.12374521791934967, "nonfinite": 0.0, "weighted_memory_to_active": 0.13914403319358826, "mix_std": 0.1558580994606018, "beta_std": 0.11318845301866531, "beta_low": 0.3716024160385132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04464797303080559, "output_influence": 0.39563071727752686, "residual_influence": 0.10574404150247574, "state_active": 0.036435991525650024, "state_memory": 0.019713282585144043, "state_peak": 35.508724212646484, "growth_active": 1.1871395111083984, "growth_memory": 1.5755298137664795, "heads": [{"head": 0, "memory_to_active": 0.21854044497013092, "contribution_to_active": 0.7179919481277466, "mix_mean": 0.6594772338867188, "mix_low": 0.010756492614746094, "mix_high": 0.016874313354492188, "read_cosine": 0.4930504560470581, "beta_mean": 0.2610524296760559, "nonfinite": 0.0, "weighted_memory_to_active": 0.17017194628715515, "mix_std": 0.3375874161720276, "beta_std": 0.27089953422546387, "beta_low": 0.2159423828125, "beta_high": 0.0, "orthogonal_to_active": 0.058167874813079834, "evidence_weights": [-0.26762905716896057, -0.36572515964508057, 0.21471744775772095], "agreement_weight": 0.01672501675784588, "orthogonal_weight": 0.4195868968963623}, {"head": 1, "memory_to_active": 0.072850801050663, "contribution_to_active": 0.5970805883407593, "mix_mean": 0.6353024244308472, "mix_low": 0.00011539459228515625, "mix_high": 0.007105827331542969, "read_cosine": 0.6793668270111084, "beta_mean": 0.0011999134439975023, "nonfinite": 0.0, "weighted_memory_to_active": 0.05037928372621536, "mix_std": 0.24879205226898193, "beta_std": 0.004716954194009304, "beta_low": 0.97882080078125, "beta_high": 0.0, "orthogonal_to_active": 0.029426876455545425, "evidence_weights": [-0.23707234859466553, -0.3250104486942291, 0.7349041700363159], "agreement_weight": 0.1437585949897766, "orthogonal_weight": -0.8397192358970642}, {"head": 2, "memory_to_active": 2.3530001640319824, "contribution_to_active": 0.2002231627702713, "mix_mean": 0.019244033843278885, "mix_low": 0.5662746429443359, "mix_high": 1.1444091796875e-05, "read_cosine": 0.26508188247680664, "beta_mean": 0.1154920682311058, "nonfinite": 0.0, "weighted_memory_to_active": 0.2130189836025238, "mix_std": 0.03827914968132973, "beta_std": 0.09233679622411728, "beta_low": 0.009521484375, "beta_high": 0.0, "orthogonal_to_active": 0.01631893962621689, "evidence_weights": [0.05290195718407631, 0.0282980315387249, 0.008257308043539524], "agreement_weight": 0.12316033244132996, "orthogonal_weight": -0.11440399289131165}, {"head": 3, "memory_to_active": 1.8046307563781738, "contribution_to_active": 0.13005727529525757, "mix_mean": 0.03062838688492775, "mix_low": 0.4256095886230469, "mix_high": 0.0, "read_cosine": 0.22398695349693298, "beta_mean": 0.157675102353096, "nonfinite": 0.0, "weighted_memory_to_active": 0.18003708124160767, "mix_std": 0.05811255797743797, "beta_std": 0.13762125372886658, "beta_low": 0.02520751953125, "beta_high": 0.0, "orthogonal_to_active": 0.011097004637122154, "evidence_weights": [-0.019383667036890984, 0.06292469799518585, -0.2211373746395111], "agreement_weight": 0.058350030332803726, "orthogonal_weight": -0.08149529248476028}, {"head": 4, "memory_to_active": 0.013472049497067928, "contribution_to_active": 0.6536760330200195, "mix_mean": 0.7308464050292969, "mix_low": 9.441375732421875e-05, "mix_high": 0.015779495239257812, "read_cosine": 0.177704855799675, "beta_mean": 2.5533696316415444e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009848047979176044, "mix_std": 0.21876493096351624, "beta_std": 8.646516653243452e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020803376100957394, "evidence_weights": [-0.32351434230804443, -0.33549749851226807, 0.6647017002105713], "agreement_weight": 0.24342145025730133, "orthogonal_weight": -0.23010317981243134}, {"head": 5, "memory_to_active": 2.451066255569458, "contribution_to_active": 0.36636072397232056, "mix_mean": 0.014832048676908016, "mix_low": 0.6349630355834961, "mix_high": 4.76837158203125e-06, "read_cosine": 0.30763310194015503, "beta_mean": 0.20702624320983887, "nonfinite": 0.0, "weighted_memory_to_active": 0.2114088386297226, "mix_std": 0.03361250460147858, "beta_std": 0.1734696924686432, "beta_low": 0.0001220703125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1507968008518219, "evidence_weights": [0.04191553220152855, -0.03912058100104332, -0.2647872865200043], "agreement_weight": -0.05886251851916313, "orthogonal_weight": 0.8555752635002136}]}, {"layer": 5, "memory_to_active": 0.5198957920074463, "contribution_to_active": 0.5012582540512085, "mix_mean": 0.45577532052993774, "mix_low": 0.07175111770629883, "mix_high": 0.014163970947265625, "read_cosine": 0.13705219328403473, "beta_mean": 0.057131245732307434, "nonfinite": 0.0, "weighted_memory_to_active": 0.14975234866142273, "mix_std": 0.2546042501926422, "beta_std": 0.07484862208366394, "beta_low": 0.6441752314567566, "beta_high": 0.0, "orthogonal_to_active": 0.05250468850135803, "output_influence": 0.4936606287956238, "residual_influence": 0.12608659267425537, "state_active": 0.044924862682819366, "state_memory": 0.011498192325234413, "state_peak": 60.525577545166016, "growth_active": 1.117021083831787, "growth_memory": 1.2438933849334717, "heads": [{"head": 0, "memory_to_active": 0.7382626533508301, "contribution_to_active": 0.41724640130996704, "mix_mean": 0.2591702342033386, "mix_low": 0.05852222442626953, "mix_high": 0.00037479400634765625, "read_cosine": -0.10116931796073914, "beta_mean": 0.03422752767801285, "nonfinite": 0.0, "weighted_memory_to_active": 0.1752297580242157, "mix_std": 0.25060737133026123, "beta_std": 0.06472529470920563, "beta_low": 0.41070556640625, "beta_high": 0.0, "orthogonal_to_active": 0.04017893970012665, "evidence_weights": [-0.09894546121358871, -0.1319766789674759, -0.24510152637958527], "agreement_weight": -0.015822093933820724, "orthogonal_weight": -0.23052597045898438}, {"head": 1, "memory_to_active": 0.025449471548199654, "contribution_to_active": 0.5362262725830078, "mix_mean": 0.6430667638778687, "mix_low": 0.002361297607421875, "mix_high": 0.022968292236328125, "read_cosine": 0.20425370335578918, "beta_mean": 6.36074910289608e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013595456257462502, "mix_std": 0.29186880588531494, "beta_std": 0.0001306949561694637, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012550009414553642, "evidence_weights": [-0.2390444278717041, -0.2775057554244995, 0.8635669946670532], "agreement_weight": 0.5253006815910339, "orthogonal_weight": -0.9644603729248047}, {"head": 2, "memory_to_active": 0.015136378817260265, "contribution_to_active": 0.5807247757911682, "mix_mean": 0.6312936544418335, "mix_low": 0.0027799606323242188, "mix_high": 0.01635265350341797, "read_cosine": -0.14715790748596191, "beta_mean": 0.00016589174629189074, "nonfinite": 0.0, "weighted_memory_to_active": 0.010502464137971401, "mix_std": 0.2752440273761749, "beta_std": 0.00025464274222031236, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002391087356954813, "evidence_weights": [-0.2519439160823822, -0.3484831750392914, 0.8725315928459167], "agreement_weight": -0.008562923409044743, "orthogonal_weight": -0.2732698917388916}, {"head": 3, "memory_to_active": 1.811123251914978, "contribution_to_active": 0.2219107747077942, "mix_mean": 0.09268658608198166, "mix_low": 0.1688823699951172, "mix_high": 9.5367431640625e-07, "read_cosine": 0.15205274522304535, "beta_mean": 0.11023930460214615, "nonfinite": 0.0, "weighted_memory_to_active": 0.3414408266544342, "mix_std": 0.13421112298965454, "beta_std": 0.15664196014404297, "beta_low": 0.189453125, "beta_high": 0.0, "orthogonal_to_active": 0.1300325095653534, "evidence_weights": [-0.062352947890758514, -0.0912487655878067, -0.052835796028375626], "agreement_weight": 0.05173705518245697, "orthogonal_weight": -0.44502153992652893}, {"head": 4, "memory_to_active": 0.44320815801620483, "contribution_to_active": 0.6010462641716003, "mix_mean": 0.32701215147972107, "mix_low": 0.19541454315185547, "mix_high": 0.007689476013183594, "read_cosine": 0.5431997776031494, "beta_mean": 0.19779294729232788, "nonfinite": 0.0, "weighted_memory_to_active": 0.29965969920158386, "mix_std": 0.3405033349990845, "beta_std": 0.22599470615386963, "beta_low": 0.2684326171875, "beta_high": 0.0, "orthogonal_to_active": 0.09234952926635742, "evidence_weights": [-0.11626141518354416, -0.28641578555107117, -0.07575967907905579], "agreement_weight": 0.042181968688964844, "orthogonal_weight": 0.47990891337394714}, {"head": 5, "memory_to_active": 0.08619491010904312, "contribution_to_active": 0.6503950357437134, "mix_mean": 0.7814225554466248, "mix_low": 0.0025463104248046875, "mix_high": 0.03759765625, "read_cosine": 0.17113414406776428, "beta_mean": 0.00029818288749083877, "nonfinite": 0.0, "weighted_memory_to_active": 0.05808589607477188, "mix_std": 0.23519068956375122, "beta_std": 0.001344401971437037, "beta_low": 0.9964599609375, "beta_high": 0.0, "orthogonal_to_active": 0.03752606362104416, "evidence_weights": [-0.3241400718688965, -0.3248600959777832, 2.005120277404785], "agreement_weight": 0.3939674198627472, "orthogonal_weight": -0.6799378991127014}]}, {"layer": 6, "memory_to_active": 0.34086281061172485, "contribution_to_active": 0.5866274237632751, "mix_mean": 0.554920494556427, "mix_low": 0.07533026486635208, "mix_high": 0.03092273138463497, "read_cosine": 0.2821458578109741, "beta_mean": 0.07128702104091644, "nonfinite": 0.0, "weighted_memory_to_active": 0.15634769201278687, "mix_std": 0.24065177142620087, "beta_std": 0.08090466260910034, "beta_low": 0.5167439579963684, "beta_high": 0.0, "orthogonal_to_active": 0.03436949849128723, "output_influence": 0.5001404285430908, "residual_influence": 0.15906542539596558, "state_active": 0.04797005653381348, "state_memory": 0.008700195699930191, "state_peak": 85.77964782714844, "growth_active": 1.206589937210083, "growth_memory": 1.1930664777755737, "heads": [{"head": 0, "memory_to_active": 0.10090994089841843, "contribution_to_active": 0.6333953142166138, "mix_mean": 0.7471805810928345, "mix_low": 0.000263214111328125, "mix_high": 0.015000343322753906, "read_cosine": -0.020416811108589172, "beta_mean": 0.014374702237546444, "nonfinite": 0.0, "weighted_memory_to_active": 0.07660915702581406, "mix_std": 0.2254888415336609, "beta_std": 0.017325755208730698, "beta_low": 0.54107666015625, "beta_high": 0.0, "orthogonal_to_active": 0.066270612180233, "evidence_weights": [-0.3150385022163391, -0.33667895197868347, 0.643673837184906], "agreement_weight": 0.18898338079452515, "orthogonal_weight": -0.9284789562225342}, {"head": 1, "memory_to_active": 0.0039829714223742485, "contribution_to_active": 0.4832867980003357, "mix_mean": 0.5265294313430786, "mix_low": 0.0010480880737304688, "mix_high": 0.010552406311035156, "read_cosine": 0.5630685687065125, "beta_mean": 7.7325057645794e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0016852770932018757, "mix_std": 0.28682994842529297, "beta_std": 1.246726060344372e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007552253664471209, "evidence_weights": [-0.20217598974704742, -0.29029273986816406, 1.050133466720581], "agreement_weight": 0.04144003987312317, "orthogonal_weight": -0.6163113713264465}, {"head": 2, "memory_to_active": 1.054800271987915, "contribution_to_active": 0.08366359770298004, "mix_mean": 0.027585649862885475, "mix_low": 0.43781185150146484, "mix_high": 0.0, "read_cosine": 0.23860348761081696, "beta_mean": 0.05437985062599182, "nonfinite": 0.0, "weighted_memory_to_active": 0.07117465138435364, "mix_std": 0.049859896302223206, "beta_std": 0.0514761283993721, "beta_low": 0.03472900390625, "beta_high": 0.0, "orthogonal_to_active": 0.0010760682635009289, "evidence_weights": [0.09494011104106903, -0.004587979055941105, 0.28896379470825195], "agreement_weight": 0.1353713423013687, "orthogonal_weight": 0.018154781311750412}, {"head": 3, "memory_to_active": 0.14516544342041016, "contribution_to_active": 0.6754961013793945, "mix_mean": 0.632914662361145, "mix_low": 0.011461257934570312, "mix_high": 0.018171310424804688, "read_cosine": 0.4390231966972351, "beta_mean": 0.22780939936637878, "nonfinite": 0.0, "weighted_memory_to_active": 0.10688909888267517, "mix_std": 0.3343897759914398, "beta_std": 0.2658095955848694, "beta_low": 0.232421875, "beta_high": 0.0, "orthogonal_to_active": 0.0595666840672493, "evidence_weights": [-0.1706046760082245, -0.33428335189819336, 0.17940454185009003], "agreement_weight": -0.007158766034990549, "orthogonal_weight": 0.6443338990211487}, {"head": 4, "memory_to_active": 0.7008780241012573, "contribution_to_active": 1.0530346632003784, "mix_mean": 0.7644094824790955, "mix_low": 0.0008640289306640625, "mix_high": 0.13863563537597656, "read_cosine": 0.16769541800022125, "beta_mean": 0.13084906339645386, "nonfinite": 0.0, "weighted_memory_to_active": 0.6547916531562805, "mix_std": 0.28874555230140686, "beta_std": 0.150202676653862, "beta_low": 0.292724609375, "beta_high": 0.0, "orthogonal_to_active": 0.05174272879958153, "evidence_weights": [-0.25354525446891785, -0.6010113954544067, -0.15482324361801147], "agreement_weight": -0.01804090477526188, "orthogonal_weight": 0.09138993173837662}, {"head": 5, "memory_to_active": 0.039439935237169266, "contribution_to_active": 0.5908879041671753, "mix_mean": 0.6309031844139099, "mix_low": 0.0005331039428710938, "mix_high": 0.0031766891479492188, "read_cosine": 0.304901123046875, "beta_mean": 0.0003013732493855059, "nonfinite": 0.0, "weighted_memory_to_active": 0.026936188340187073, "mix_std": 0.2585965692996979, "beta_std": 0.0006013554520905018, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.026805654168128967, "evidence_weights": [-0.24005410075187683, -0.30649232864379883, 0.3935154378414154], "agreement_weight": 0.1414303183555603, "orthogonal_weight": -1.0748611688613892}]}, {"layer": 7, "memory_to_active": 1.7957559823989868, "contribution_to_active": 0.24701209366321564, "mix_mean": 0.10187247395515442, "mix_low": 0.29114770889282227, "mix_high": 0.00016260147094726562, "read_cosine": 0.23264990746974945, "beta_mean": 0.13760258257389069, "nonfinite": 0.0, "weighted_memory_to_active": 0.23424093425273895, "mix_std": 0.12359771132469177, "beta_std": 0.10548867285251617, "beta_low": 0.1845601499080658, "beta_high": 0.00010172526526730508, "orthogonal_to_active": 0.06342732161283493, "output_influence": 0.23340658843517303, "residual_influence": 0.0530858114361763, "state_active": 0.015030557289719582, "state_memory": 0.028031688183546066, "state_peak": 39.56454849243164, "growth_active": 1.1709887981414795, "growth_memory": 1.3087263107299805, "heads": [{"head": 0, "memory_to_active": 2.483144521713257, "contribution_to_active": 0.43413209915161133, "mix_mean": 0.0338619202375412, "mix_low": 0.4427938461303711, "mix_high": 0.0, "read_cosine": 0.3533206582069397, "beta_mean": 0.16235671937465668, "nonfinite": 0.0, "weighted_memory_to_active": 0.5013530850410461, "mix_std": 0.07170537114143372, "beta_std": 0.1244722232222557, "beta_low": 0.00714111328125, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.019584588706493378, "evidence_weights": [0.017706314101815224, -0.08178194612264633, 0.1086440309882164], "agreement_weight": -0.035507965832948685, "orthogonal_weight": 0.06798961013555527}, {"head": 1, "memory_to_active": 1.0374294519424438, "contribution_to_active": 0.3301161229610443, "mix_mean": 0.17471645772457123, "mix_low": 0.16027259826660156, "mix_high": 0.0007381439208984375, "read_cosine": 0.3656872510910034, "beta_mean": 0.10278946906328201, "nonfinite": 0.0, "weighted_memory_to_active": 0.2981630265712738, "mix_std": 0.21022748947143555, "beta_std": 0.06743533909320831, "beta_low": 0.0006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.013919958844780922, "evidence_weights": [-0.19176551699638367, -0.1119396984577179, -0.19952306151390076], "agreement_weight": 0.020003000274300575, "orthogonal_weight": 0.05418749898672104}, {"head": 2, "memory_to_active": 0.8922813534736633, "contribution_to_active": 0.28833889961242676, "mix_mean": 0.11470311880111694, "mix_low": 0.10667133331298828, "mix_high": 0.00020503997802734375, "read_cosine": 0.11058466881513596, "beta_mean": 0.06406080722808838, "nonfinite": 0.0, "weighted_memory_to_active": 0.207651749253273, "mix_std": 0.15500329434871674, "beta_std": 0.052019473165273666, "beta_low": 0.04644775390625, "beta_high": 0.0, "orthogonal_to_active": 0.06864338368177414, "evidence_weights": [-0.04379134625196457, -0.1307760328054428, 0.4642162621021271], "agreement_weight": 0.025290295481681824, "orthogonal_weight": -0.33723029494285583}, {"head": 3, "memory_to_active": 4.3107829093933105, "contribution_to_active": 0.08814337104558945, "mix_mean": 0.03293014317750931, "mix_low": 0.3365287780761719, "mix_high": 0.0, "read_cosine": 0.09833210706710815, "beta_mean": 0.356931209564209, "nonfinite": 0.0, "weighted_memory_to_active": 0.2929343581199646, "mix_std": 0.048531822860240936, "beta_std": 0.2584752142429352, "beta_low": 0.0030517578125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.2479942888021469, "evidence_weights": [0.06270868331193924, -0.00970769114792347, 0.6707517504692078], "agreement_weight": 0.051535531878471375, "orthogonal_weight": -0.9368453621864319}, {"head": 4, "memory_to_active": 1.913720726966858, "contribution_to_active": 0.06305690109729767, "mix_mean": 0.014503349550068378, "mix_low": 0.6759099960327148, "mix_high": 0.0, "read_cosine": 0.2188669741153717, "beta_mean": 0.1385141909122467, "nonfinite": 0.0, "weighted_memory_to_active": 0.06374210119247437, "mix_std": 0.03367207571864128, "beta_std": 0.12783804535865784, "beta_low": 0.05712890625, "beta_high": 0.0, "orthogonal_to_active": 0.007400496397167444, "evidence_weights": [0.13557486236095428, 0.05174512788653374, 0.22200115025043488], "agreement_weight": 0.048954110592603683, "orthogonal_weight": -0.15251997113227844}, {"head": 5, "memory_to_active": 0.13717621564865112, "contribution_to_active": 0.27828508615493774, "mix_mean": 0.24051988124847412, "mix_low": 0.024709701538085938, "mix_high": 3.24249267578125e-05, "read_cosine": 0.24910777807235718, "beta_mean": 0.0009630277054384351, "nonfinite": 0.0, "weighted_memory_to_active": 0.041601281613111496, "mix_std": 0.22244618833065033, "beta_std": 0.00269178650341928, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.023021193221211433, "evidence_weights": [-0.09553702920675278, -0.14808110892772675, 0.40312275290489197], "agreement_weight": 0.07784634828567505, "orthogonal_weight": -0.631413996219635}]}, {"layer": 8, "memory_to_active": 0.478584885597229, "contribution_to_active": 0.5236573219299316, "mix_mean": 0.4149152636528015, "mix_low": 0.09662406146526337, "mix_high": 0.0009719531517475843, "read_cosine": 0.4837072789669037, "beta_mean": 0.03194757550954819, "nonfinite": 0.0, "weighted_memory_to_active": 0.10082727670669556, "mix_std": 0.21981549263000488, "beta_std": 0.03862260282039642, "beta_low": 0.58673095703125, "beta_high": 0.0, "orthogonal_to_active": 0.05830749496817589, "output_influence": 0.44038310647010803, "residual_influence": 0.13905669748783112, "state_active": 0.05464759096503258, "state_memory": 0.01049683429300785, "state_peak": 32.802085876464844, "growth_active": 1.1691434383392334, "growth_memory": 1.370045781135559, "heads": [{"head": 0, "memory_to_active": 0.04872066527605057, "contribution_to_active": 0.5413112640380859, "mix_mean": 0.583889901638031, "mix_low": 0.0003147125244140625, "mix_high": 0.0011358261108398438, "read_cosine": 0.5672667026519775, "beta_mean": 0.00287408335134387, "nonfinite": 0.0, "weighted_memory_to_active": 0.033429261296987534, "mix_std": 0.24409091472625732, "beta_std": 0.007972465828061104, "beta_low": 0.9315185546875, "beta_high": 0.0, "orthogonal_to_active": 0.0204436257481575, "evidence_weights": [-0.1683373898267746, -0.3919335901737213, 0.713275671005249], "agreement_weight": 0.121274933218956, "orthogonal_weight": -0.8197592496871948}, {"head": 1, "memory_to_active": 0.1489906907081604, "contribution_to_active": 0.4921807050704956, "mix_mean": 0.4720679521560669, "mix_low": 0.0009469985961914062, "mix_high": 0.0005741119384765625, "read_cosine": 0.34867334365844727, "beta_mean": 0.0037628612481057644, "nonfinite": 0.0, "weighted_memory_to_active": 0.07156829535961151, "mix_std": 0.2525334358215332, "beta_std": 0.013372697867453098, "beta_low": 0.931640625, "beta_high": 0.0, "orthogonal_to_active": 0.04478967934846878, "evidence_weights": [-0.1940772831439972, -0.2597306966781616, 0.6759060621261597], "agreement_weight": 0.09468580782413483, "orthogonal_weight": -0.7025723457336426}, {"head": 2, "memory_to_active": 0.019982200115919113, "contribution_to_active": 0.647773027420044, "mix_mean": 0.6213306188583374, "mix_low": 0.0012407302856445312, "mix_high": 0.0030336380004882812, "read_cosine": 0.4016976058483124, "beta_mean": 0.00018238135089632124, "nonfinite": 0.0, "weighted_memory_to_active": 0.014661436900496483, "mix_std": 0.23458707332611084, "beta_std": 0.0031396052800118923, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00975771527737379, "evidence_weights": [-0.22001177072525024, -0.3383292257785797, 0.883304238319397], "agreement_weight": 0.2050824761390686, "orthogonal_weight": -0.76726895570755}, {"head": 3, "memory_to_active": 0.12806379795074463, "contribution_to_active": 0.6148125529289246, "mix_mean": 0.5414043664932251, "mix_low": 0.0034837722778320312, "mix_high": 0.0009765625, "read_cosine": 0.7840958833694458, "beta_mean": 0.012924679554998875, "nonfinite": 0.0, "weighted_memory_to_active": 0.094851553440094, "mix_std": 0.2575678527355194, "beta_std": 0.009295358322560787, "beta_low": 0.45098876953125, "beta_high": 0.0, "orthogonal_to_active": 0.03164469823241234, "evidence_weights": [-0.17489197850227356, -0.2984500825405121, 0.6532301306724548], "agreement_weight": 0.1691564917564392, "orthogonal_weight": -0.6345144510269165}, {"head": 4, "memory_to_active": 2.217283248901367, "contribution_to_active": 0.2292959839105606, "mix_mean": 0.028385287150740623, "mix_low": 0.4330434799194336, "mix_high": 0.0, "read_cosine": 0.3090711832046509, "beta_mean": 0.09017448127269745, "nonfinite": 0.0, "weighted_memory_to_active": 0.2214367687702179, "mix_std": 0.05141513794660568, "beta_std": 0.08989180624485016, "beta_low": 0.0325927734375, "beta_high": 0.0, "orthogonal_to_active": 0.002870935248211026, "evidence_weights": [0.054633576422929764, -0.02281961776316166, -0.07876111567020416], "agreement_weight": 0.04988986253738403, "orthogonal_weight": 0.01331575121730566}, {"head": 5, "memory_to_active": 0.3084685504436493, "contribution_to_active": 0.6165701150894165, "mix_mean": 0.24241343140602112, "mix_low": 0.1407146453857422, "mix_high": 0.00011157989501953125, "read_cosine": 0.49143868684768677, "beta_mean": 0.08176694810390472, "nonfinite": 0.0, "weighted_memory_to_active": 0.1690163016319275, "mix_std": 0.27869850397109985, "beta_std": 0.10806368291378021, "beta_low": 0.1737060546875, "beta_high": 0.0, "orthogonal_to_active": 0.24033834040164948, "evidence_weights": [-0.06770313531160355, -0.25535884499549866, -0.35165977478027344], "agreement_weight": -0.002845308044925332, "orthogonal_weight": 1.6939581632614136}]}, {"layer": 9, "memory_to_active": 1.1477200984954834, "contribution_to_active": 0.6606297492980957, "mix_mean": 0.3186584711074829, "mix_low": 0.14472246170043945, "mix_high": 0.003568649524822831, "read_cosine": 0.3692070245742798, "beta_mean": 0.0695391371846199, "nonfinite": 0.0, "weighted_memory_to_active": 0.3886581361293793, "mix_std": 0.2118263840675354, "beta_std": 0.04815198481082916, "beta_low": 0.500396728515625, "beta_high": 0.0, "orthogonal_to_active": 0.04504195228219032, "output_influence": 0.4989550709724426, "residual_influence": 0.13513129949569702, "state_active": 0.040433209389448166, "state_memory": 0.020106760784983635, "state_peak": 42.86101150512695, "growth_active": 1.1633776426315308, "growth_memory": 1.3674389123916626, "heads": [{"head": 0, "memory_to_active": 0.001854062546044588, "contribution_to_active": 0.5402276515960693, "mix_mean": 0.6165233254432678, "mix_low": 0.0017518997192382812, "mix_high": 0.015535354614257812, "read_cosine": 0.28561756014823914, "beta_mean": 4.707542302639922e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0011469550663605332, "mix_std": 0.2916467785835266, "beta_std": 1.4906844626239035e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009930318919941783, "evidence_weights": [-0.2949833869934082, -0.30570101737976074, 0.9346362948417664], "agreement_weight": 0.1075383871793747, "orthogonal_weight": -0.9228057265281677}, {"head": 1, "memory_to_active": 0.029103204607963562, "contribution_to_active": 0.6177493333816528, "mix_mean": 0.49649855494499207, "mix_low": 0.009489059448242188, "mix_high": 0.001621246337890625, "read_cosine": 0.41843533515930176, "beta_mean": 8.032281039049849e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.018994083628058434, "mix_std": 0.290188729763031, "beta_std": 0.0001573809568071738, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013297065161168575, "evidence_weights": [-0.2194659560918808, -0.2684459686279297, 1.134985089302063], "agreement_weight": 0.36023780703544617, "orthogonal_weight": -0.8359476923942566}, {"head": 2, "memory_to_active": 0.17943227291107178, "contribution_to_active": 0.6283078193664551, "mix_mean": 0.5524249076843262, "mix_low": 0.0019845962524414062, "mix_high": 0.0038499832153320312, "read_cosine": 0.35939112305641174, "beta_mean": 0.0006780052208341658, "nonfinite": 0.0, "weighted_memory_to_active": 0.11419232934713364, "mix_std": 0.2775310277938843, "beta_std": 0.004397259559482336, "beta_low": 0.99658203125, "beta_high": 0.0, "orthogonal_to_active": 0.05861584097146988, "evidence_weights": [-0.18989922106266022, -0.25623056292533875, 0.6812002062797546], "agreement_weight": 0.26523369550704956, "orthogonal_weight": -0.604748010635376}, {"head": 3, "memory_to_active": 1.6732981204986572, "contribution_to_active": 0.19753514230251312, "mix_mean": 0.042138539254665375, "mix_low": 0.3877439498901367, "mix_high": 5.817413330078125e-05, "read_cosine": 0.4122634828090668, "beta_mean": 0.08619004487991333, "nonfinite": 0.0, "weighted_memory_to_active": 0.17310024797916412, "mix_std": 0.0921221449971199, "beta_std": 0.06137288361787796, "beta_low": 0.00390625, "beta_high": 0.0, "orthogonal_to_active": 0.025364048779010773, "evidence_weights": [0.07886331528425217, 0.02913752943277359, -0.024099212139844894], "agreement_weight": 0.09324300289154053, "orthogonal_weight": 0.16115567088127136}, {"head": 4, "memory_to_active": 3.1313581466674805, "contribution_to_active": 1.255786657333374, "mix_mean": 0.1282127946615219, "mix_low": 0.15195655822753906, "mix_high": 0.0002841949462890625, "read_cosine": 0.2444070726633072, "beta_mean": 0.17450223863124847, "nonfinite": 0.0, "weighted_memory_to_active": 1.2525736093521118, "mix_std": 0.18578749895095825, "beta_std": 0.10515299439430237, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.030559593811631203, "evidence_weights": [-0.06122589856386185, -0.14706647396087646, -0.00041969111771322787], "agreement_weight": -0.05333050340414047, "orthogonal_weight": -0.0256559569388628}, {"head": 5, "memory_to_active": 1.871274709701538, "contribution_to_active": 0.7241716980934143, "mix_mean": 0.07615284621715546, "mix_low": 0.31540870666503906, "mix_high": 6.29425048828125e-05, "read_cosine": 0.49512729048728943, "beta_mean": 0.15577951073646545, "nonfinite": 0.0, "weighted_memory_to_active": 0.7719414830207825, "mix_std": 0.13368210196495056, "beta_std": 0.11781645566225052, "beta_low": 0.00189208984375, "beta_high": 0.0, "orthogonal_to_active": 0.1414221227169037, "evidence_weights": [0.10008460283279419, -0.07433206588029861, 0.16027110815048218], "agreement_weight": 0.01804431714117527, "orthogonal_weight": 0.2674543559551239}]}, {"layer": 10, "memory_to_active": 0.264373779296875, "contribution_to_active": 0.6655443906784058, "mix_mean": 0.49974483251571655, "mix_low": 0.013518334366381168, "mix_high": 0.006611665245145559, "read_cosine": 0.3668960928916931, "beta_mean": 0.049419865012168884, "nonfinite": 0.0, "weighted_memory_to_active": 0.13162334263324738, "mix_std": 0.26761394739151, "beta_std": 0.04963097348809242, "beta_low": 0.65509033203125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.17608891427516937, "output_influence": 0.428231805562973, "residual_influence": 0.1437065154314041, "state_active": 0.05369818955659866, "state_memory": 0.015037836506962776, "state_peak": 112.90807342529297, "growth_active": 1.0890337228775024, "growth_memory": 0.9417806267738342, "heads": [{"head": 0, "memory_to_active": 1.0631771087646484, "contribution_to_active": 1.7823643684387207, "mix_mean": 0.6034769415855408, "mix_low": 0.04311943054199219, "mix_high": 0.024484634399414062, "read_cosine": 0.06777375936508179, "beta_mean": 0.21990470588207245, "nonfinite": 0.0, "weighted_memory_to_active": 0.6639316082000732, "mix_std": 0.3840022683143616, "beta_std": 0.22661611437797546, "beta_low": 0.12359619140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.9771157503128052, "evidence_weights": [-0.13241805136203766, -0.3895469605922699, -1.099936842918396], "agreement_weight": 0.02992083691060543, "orthogonal_weight": 1.485801100730896}, {"head": 1, "memory_to_active": 0.014572370797395706, "contribution_to_active": 0.5843846201896667, "mix_mean": 0.5996917486190796, "mix_low": 0.0007715225219726562, "mix_high": 0.010157585144042969, "read_cosine": 0.2866211235523224, "beta_mean": 8.602951129432768e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008225986734032631, "mix_std": 0.2671680450439453, "beta_std": 0.00010504890087759122, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011352092027664185, "evidence_weights": [-0.2679111957550049, -0.2025635987520218, 0.6028211712837219], "agreement_weight": 0.4609032869338989, "orthogonal_weight": -1.5127604007720947}, {"head": 2, "memory_to_active": 0.02093564160168171, "contribution_to_active": 0.5895513892173767, "mix_mean": 0.6001112461090088, "mix_low": 0.00028228759765625, "mix_high": 0.0045032501220703125, "read_cosine": 0.026282446458935738, "beta_mean": 5.243371560936794e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008867232128977776, "mix_std": 0.2542082965373993, "beta_std": 0.0002028706658165902, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0038044145330786705, "evidence_weights": [-0.23324422538280487, -0.27012404799461365, 0.5401471257209778], "agreement_weight": -0.14332248270511627, "orthogonal_weight": -0.41530561447143555}, {"head": 3, "memory_to_active": 0.3388392925262451, "contribution_to_active": 0.09542359411716461, "mix_mean": 0.2675590515136719, "mix_low": 0.03396129608154297, "mix_high": 0.00018024444580078125, "read_cosine": 0.9124059081077576, "beta_mean": 0.06943724304437637, "nonfinite": 0.0, "weighted_memory_to_active": 0.03705490753054619, "mix_std": 0.22276245057582855, "beta_std": 0.06217222660779953, "beta_low": 0.00213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.04254323989152908, "evidence_weights": [-0.1139792650938034, -0.1456630975008011, 0.1351112723350525], "agreement_weight": 0.0059585487470030785, "orthogonal_weight": 1.4934114217758179}, {"head": 4, "memory_to_active": 0.0023644836619496346, "contribution_to_active": 0.5314549803733826, "mix_mean": 0.5388062000274658, "mix_low": 0.000537872314453125, "mix_high": 0.000339508056640625, "read_cosine": 0.046672023832798004, "beta_mean": 1.4983081200625747e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0012489655055105686, "mix_std": 0.25034523010253906, "beta_std": 3.8049307477194816e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008565992466174066, "evidence_weights": [-0.18108558654785156, -0.258625328540802, 0.742775022983551], "agreement_weight": 0.07638943940401077, "orthogonal_weight": -0.7142097353935242}, {"head": 5, "memory_to_active": 0.14635369181632996, "contribution_to_active": 0.4100874066352844, "mix_mean": 0.3888237476348877, "mix_low": 0.002437591552734375, "mix_high": 4.76837158203125e-06, "read_cosine": 0.861621081829071, "beta_mean": 0.007023787125945091, "nonfinite": 0.0, "weighted_memory_to_active": 0.07041127979755402, "mix_std": 0.22719720005989075, "beta_std": 0.008651524782180786, "beta_low": 0.80487060546875, "beta_high": 0.0, "orthogonal_to_active": 0.020861271768808365, "evidence_weights": [-0.14691820740699768, -0.1593577265739441, 0.28772473335266113], "agreement_weight": 0.09110485762357712, "orthogonal_weight": -0.6292778849601746}]}, {"layer": 11, "memory_to_active": 0.1264030933380127, "contribution_to_active": 0.5809008479118347, "mix_mean": 0.4729815721511841, "mix_low": 0.08392493426799774, "mix_high": 0.04797633737325668, "read_cosine": 0.4102974236011505, "beta_mean": 0.031834762543439865, "nonfinite": 0.0, "weighted_memory_to_active": 0.05888056755065918, "mix_std": 0.3227153420448303, "beta_std": 0.04704464226961136, "beta_low": 0.595794677734375, "beta_high": 0.0, "orthogonal_to_active": 0.04868224635720253, "output_influence": 0.5197606086730957, "residual_influence": 0.12891142070293427, "state_active": 0.1967075616121292, "state_memory": 0.006591431796550751, "state_peak": 27.22345542907715, "growth_active": 1.0993901491165161, "growth_memory": 1.5890929698944092, "heads": [{"head": 0, "memory_to_active": 0.0066276174038648605, "contribution_to_active": 0.5700222253799438, "mix_mean": 0.604703426361084, "mix_low": 0.001140594482421875, "mix_high": 0.0069580078125, "read_cosine": 0.36228853464126587, "beta_mean": 8.649339724797755e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004278057254850864, "mix_std": 0.26659518480300903, "beta_std": 0.0037549505941569805, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0030187892261892557, "evidence_weights": [-0.16692779958248138, -0.3626771569252014, 0.6900705695152283], "agreement_weight": 0.2640114426612854, "orthogonal_weight": -0.7745153903961182}, {"head": 1, "memory_to_active": 0.32330840826034546, "contribution_to_active": 0.3173195719718933, "mix_mean": 0.35545772314071655, "mix_low": 0.01059722900390625, "mix_high": 0.000179290771484375, "read_cosine": 0.5980829000473022, "beta_mean": 0.030202221125364304, "nonfinite": 0.0, "weighted_memory_to_active": 0.14035366475582123, "mix_std": 0.2569526135921478, "beta_std": 0.058794114738702774, "beta_low": 0.4493408203125, "beta_high": 0.0, "orthogonal_to_active": 0.04058142378926277, "evidence_weights": [-0.16162194311618805, -0.168887659907341, 0.42358607053756714], "agreement_weight": 0.08310935646295547, "orthogonal_weight": -0.39783766865730286}, {"head": 2, "memory_to_active": 0.010960700921714306, "contribution_to_active": 0.9765206575393677, "mix_mean": 0.448814332485199, "mix_low": 0.2056407928466797, "mix_high": 0.17759990692138672, "read_cosine": 0.18312843143939972, "beta_mean": 0.07696351408958435, "nonfinite": 0.0, "weighted_memory_to_active": 0.008475624024868011, "mix_std": 0.4410091042518616, "beta_std": 0.10406749695539474, "beta_low": 0.45794677734375, "beta_high": 0.0, "orthogonal_to_active": 0.04889119789004326, "evidence_weights": [0.03530419245362282, -0.4436718821525574, -1.2654626369476318], "agreement_weight": 0.011724279262125492, "orthogonal_weight": 6.028775691986084}, {"head": 3, "memory_to_active": 0.011375367641448975, "contribution_to_active": 0.8363096117973328, "mix_mean": 0.4148159623146057, "mix_low": 0.2748126983642578, "mix_high": 0.10048866271972656, "read_cosine": 0.567080020904541, "beta_mean": 0.05943569913506508, "nonfinite": 0.0, "weighted_memory_to_active": 0.006784430705010891, "mix_std": 0.43904873728752136, "beta_std": 0.08792394399642944, "beta_low": 0.48907470703125, "beta_high": 0.0, "orthogonal_to_active": 0.0332479402422905, "evidence_weights": [0.12447084486484528, -0.33371320366859436, -0.329374760389328], "agreement_weight": 0.04744327440857887, "orthogonal_weight": 6.335473537445068}, {"head": 4, "memory_to_active": 0.3179251253604889, "contribution_to_active": 0.40997666120529175, "mix_mean": 0.500213086605072, "mix_low": 0.009432792663574219, "mix_high": 0.0005655288696289062, "read_cosine": 0.08341577649116516, "beta_mean": 0.016382304951548576, "nonfinite": 0.0, "weighted_memory_to_active": 0.15162454545497894, "mix_std": 0.27124637365341187, "beta_std": 0.01628050021827221, "beta_low": 0.42474365234375, "beta_high": 0.0, "orthogonal_to_active": 0.14246642589569092, "evidence_weights": [-0.22716796398162842, -0.206089586019516, 1.8656760454177856], "agreement_weight": -0.18558642268180847, "orthogonal_weight": -0.9535551071166992}, {"head": 5, "memory_to_active": 0.08822138607501984, "contribution_to_active": 0.3752560317516327, "mix_mean": 0.5138847827911377, "mix_low": 0.0019254684448242188, "mix_high": 0.0020666122436523438, "read_cosine": 0.6677886843681335, "beta_mean": 0.007938348688185215, "nonfinite": 0.0, "weighted_memory_to_active": 0.04176708310842514, "mix_std": 0.26144012808799744, "beta_std": 0.011446841061115265, "beta_low": 0.7537841796875, "beta_high": 0.0, "orthogonal_to_active": 0.023887690156698227, "evidence_weights": [-0.21749216318130493, -0.29011455178260803, 0.5734089016914368], "agreement_weight": 0.0720355436205864, "orthogonal_weight": -0.7177237272262573}]}, {"layer": 12, "memory_to_active": 0.3070164620876312, "contribution_to_active": 0.577299177646637, "mix_mean": 0.5061066150665283, "mix_low": 0.009984811767935753, "mix_high": 0.004093647003173828, "read_cosine": 0.23383381962776184, "beta_mean": 0.016907718032598495, "nonfinite": 0.0, "weighted_memory_to_active": 0.1698356717824936, "mix_std": 0.26152747869491577, "beta_std": 0.01738257333636284, "beta_low": 0.821380615234375, "beta_high": 0.0, "orthogonal_to_active": 0.07552556693553925, "output_influence": 0.4563826322555542, "residual_influence": 0.1265886425971985, "state_active": 0.0394703634083271, "state_memory": 0.009324097074568272, "state_peak": 46.572853088378906, "growth_active": 1.1466346979141235, "growth_memory": 1.172852873802185, "heads": [{"head": 0, "memory_to_active": 0.06739753484725952, "contribution_to_active": 0.5000563859939575, "mix_mean": 0.5005174875259399, "mix_low": 0.0011968612670898438, "mix_high": 0.0017299652099609375, "read_cosine": 0.28950339555740356, "beta_mean": 0.0010367928771302104, "nonfinite": 0.0, "weighted_memory_to_active": 0.03170325607061386, "mix_std": 0.27157914638519287, "beta_std": 0.003873407607898116, "beta_low": 0.9913330078125, "beta_high": 0.0, "orthogonal_to_active": 0.023249130696058273, "evidence_weights": [-0.18996486067771912, -0.26305899024009705, 0.30682483315467834], "agreement_weight": 0.10791676491498947, "orthogonal_weight": -0.7831124663352966}, {"head": 1, "memory_to_active": 0.08421941101551056, "contribution_to_active": 0.5073536038398743, "mix_mean": 0.5284114480018616, "mix_low": 0.0008516311645507812, "mix_high": 0.0006475448608398438, "read_cosine": 0.49992895126342773, "beta_mean": 0.000928147288504988, "nonfinite": 0.0, "weighted_memory_to_active": 0.044102612882852554, "mix_std": 0.26283353567123413, "beta_std": 0.004979309160262346, "beta_low": 0.98956298828125, "beta_high": 0.0, "orthogonal_to_active": 0.035281628370285034, "evidence_weights": [-0.20703156292438507, -0.2741174101829529, 0.9345971345901489], "agreement_weight": 0.13560466468334198, "orthogonal_weight": -0.9228953719139099}, {"head": 2, "memory_to_active": 0.6276460886001587, "contribution_to_active": 0.5129284858703613, "mix_mean": 0.3402351140975952, "mix_low": 0.02489757537841797, "mix_high": 0.000125885009765625, "read_cosine": 0.4402976334095001, "beta_mean": 0.09622394293546677, "nonfinite": 0.0, "weighted_memory_to_active": 0.2624858617782593, "mix_std": 0.24735407531261444, "beta_std": 0.06704479455947876, "beta_low": 0.00115966796875, "beta_high": 0.0, "orthogonal_to_active": 0.254509836435318, "evidence_weights": [-0.23964229226112366, -0.04647863283753395, -0.09284535050392151], "agreement_weight": 0.023386571556329727, "orthogonal_weight": 1.169959306716919}, {"head": 3, "memory_to_active": 0.01889335922896862, "contribution_to_active": 0.7333139777183533, "mix_mean": 0.6999142169952393, "mix_low": 0.00025272369384765625, "mix_high": 0.014540672302246094, "read_cosine": 0.036537617444992065, "beta_mean": 0.0001890380517579615, "nonfinite": 0.0, "weighted_memory_to_active": 0.01449192501604557, "mix_std": 0.23680958151817322, "beta_std": 0.004127076826989651, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.013690716587007046, "evidence_weights": [-0.2936875820159912, -0.33879080414772034, 1.353708028793335], "agreement_weight": 0.3697296380996704, "orthogonal_weight": -0.9640430212020874}, {"head": 4, "memory_to_active": 0.9408340454101562, "contribution_to_active": 0.7163930535316467, "mix_mean": 0.3424075245857239, "mix_low": 0.02635955810546875, "mix_high": 0.0022153854370117188, "read_cosine": -0.10028967261314392, "beta_mean": 0.0022311131469905376, "nonfinite": 0.0, "weighted_memory_to_active": 0.6002091765403748, "mix_std": 0.2734389305114746, "beta_std": 0.01739206537604332, "beta_low": 0.9656982421875, "beta_high": 0.0, "orthogonal_to_active": 0.07497821748256683, "evidence_weights": [-0.10870800912380219, -0.21262015402317047, 0.05090164765715599], "agreement_weight": 0.0018727942369878292, "orthogonal_weight": -0.13204078376293182}, {"head": 5, "memory_to_active": 0.10310831665992737, "contribution_to_active": 0.49374938011169434, "mix_mean": 0.6251537203788757, "mix_low": 0.006350517272949219, "mix_high": 0.00530242919921875, "read_cosine": 0.23702502250671387, "beta_mean": 0.000837263185530901, "nonfinite": 0.0, "weighted_memory_to_active": 0.06602117419242859, "mix_std": 0.2771495580673218, "beta_std": 0.006878780201077461, "beta_low": 0.9814453125, "beta_high": 0.0, "orthogonal_to_active": 0.051443830132484436, "evidence_weights": [-0.25379788875579834, -0.31203383207321167, 1.239080548286438], "agreement_weight": 0.30792731046676636, "orthogonal_weight": -0.8445795774459839}]}, {"layer": 13, "memory_to_active": 0.13720767199993134, "contribution_to_active": 0.5236459374427795, "mix_mean": 0.5186112523078918, "mix_low": 0.005672772880643606, "mix_high": 0.005966822616755962, "read_cosine": 0.4162652790546417, "beta_mean": 0.01755860261619091, "nonfinite": 0.0, "weighted_memory_to_active": 0.05982823669910431, "mix_std": 0.25881481170654297, "beta_std": 0.018556639552116394, "beta_low": 0.7964579463005066, "beta_high": 0.0, "orthogonal_to_active": 0.06124961003661156, "output_influence": 0.4960463047027588, "residual_influence": 0.16758762300014496, "state_active": 0.09981172531843185, "state_memory": 0.03866688534617424, "state_peak": 94.14469909667969, "growth_active": 0.9451419115066528, "growth_memory": 1.0111944675445557, "heads": [{"head": 0, "memory_to_active": 0.03965790569782257, "contribution_to_active": 0.554112434387207, "mix_mean": 0.5694926977157593, "mix_low": 0.00035953521728515625, "mix_high": 0.0013990402221679688, "read_cosine": 0.40499311685562134, "beta_mean": 0.00013490015408024192, "nonfinite": 0.0, "weighted_memory_to_active": 0.02447289228439331, "mix_std": 0.26609861850738525, "beta_std": 0.00028858144651167095, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.024011872708797455, "evidence_weights": [-0.201960951089859, -0.2648628056049347, 0.758906364440918], "agreement_weight": 0.10919374227523804, "orthogonal_weight": -1.098636507987976}, {"head": 1, "memory_to_active": 0.012912476435303688, "contribution_to_active": 0.5837904214859009, "mix_mean": 0.5471692681312561, "mix_low": 0.00160980224609375, "mix_high": 0.0026292800903320312, "read_cosine": 0.21597537398338318, "beta_mean": 4.383145278552547e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007482488173991442, "mix_std": 0.2707354426383972, "beta_std": 7.649774488527328e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008789056912064552, "evidence_weights": [-0.1881941556930542, -0.2716352045536041, 0.693037748336792], "agreement_weight": 0.3149735927581787, "orthogonal_weight": -1.2303731441497803}, {"head": 2, "memory_to_active": 0.10969051718711853, "contribution_to_active": 0.4373990595340729, "mix_mean": 0.3696434795856476, "mix_low": 0.002239227294921875, "mix_high": 3.719329833984375e-05, "read_cosine": 0.6657694578170776, "beta_mean": 0.01515943557024002, "nonfinite": 0.0, "weighted_memory_to_active": 0.0471879206597805, "mix_std": 0.2255341112613678, "beta_std": 0.04364641010761261, "beta_low": 0.7923583984375, "beta_high": 0.0, "orthogonal_to_active": 0.022564470767974854, "evidence_weights": [-0.20090366899967194, -0.18994149565696716, 0.43075770139694214], "agreement_weight": 0.012715485878288746, "orthogonal_weight": -0.814750075340271}, {"head": 3, "memory_to_active": 0.13984543085098267, "contribution_to_active": 0.6012188196182251, "mix_mean": 0.6297906041145325, "mix_low": 0.021074295043945312, "mix_high": 0.02471637725830078, "read_cosine": 0.027293581515550613, "beta_mean": 0.0010910844430327415, "nonfinite": 0.0, "weighted_memory_to_active": 0.10274358093738556, "mix_std": 0.29062458872795105, "beta_std": 0.006858333479613066, "beta_low": 0.9908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.10002728551626205, "evidence_weights": [-0.33048778772354126, -0.24592143297195435, 1.9355069398880005], "agreement_weight": 3.9835567474365234, "orthogonal_weight": -1.0450143814086914}, {"head": 4, "memory_to_active": 0.43714624643325806, "contribution_to_active": 0.28939008712768555, "mix_mean": 0.36970406770706177, "mix_low": 0.006336212158203125, "mix_high": 1.239776611328125e-05, "read_cosine": 0.8726921081542969, "beta_mean": 0.0883222222328186, "nonfinite": 0.0, "weighted_memory_to_active": 0.12169862538576126, "mix_std": 0.22354871034622192, "beta_std": 0.056770555675029755, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.1665700525045395, "evidence_weights": [-0.1264423280954361, -0.2308642864227295, 0.4820514917373657], "agreement_weight": -0.004859402310103178, "orthogonal_weight": 2.1156485080718994}, {"head": 5, "memory_to_active": 0.08399344235658646, "contribution_to_active": 0.6759648323059082, "mix_mean": 0.6258674263954163, "mix_low": 0.0024175643920898438, "mix_high": 0.007006645202636719, "read_cosine": 0.3108677864074707, "beta_mean": 0.0006001396686770022, "nonfinite": 0.0, "weighted_memory_to_active": 0.05538389831781387, "mix_std": 0.276347279548645, "beta_std": 0.00369945727288723, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.04553491249680519, "evidence_weights": [-0.221149742603302, -0.3006443679332733, 0.7911811470985413], "agreement_weight": 0.11831597238779068, "orthogonal_weight": -0.9125303626060486}]}, {"layer": 14, "memory_to_active": 0.25928354263305664, "contribution_to_active": 0.4236539304256439, "mix_mean": 0.4249076545238495, "mix_low": 0.024418672546744347, "mix_high": 0.0052577657625079155, "read_cosine": 0.6302188038825989, "beta_mean": 0.06302118301391602, "nonfinite": 0.0, "weighted_memory_to_active": 0.11525292694568634, "mix_std": 0.24804824590682983, "beta_std": 0.05995913967490196, "beta_low": 0.4536946713924408, "beta_high": 0.0, "orthogonal_to_active": 0.0458107590675354, "output_influence": 0.33258315920829773, "residual_influence": 0.12783528864383698, "state_active": 0.058311931788921356, "state_memory": 0.015909021720290184, "state_peak": 35.834712982177734, "growth_active": 1.0831013917922974, "growth_memory": 1.2960810661315918, "heads": [{"head": 0, "memory_to_active": 0.379484623670578, "contribution_to_active": 0.2157181203365326, "mix_mean": 0.23591136932373047, "mix_low": 0.009713172912597656, "mix_high": 2.002716064453125e-05, "read_cosine": 0.7937029004096985, "beta_mean": 0.06809025257825851, "nonfinite": 0.0, "weighted_memory_to_active": 0.10375592857599258, "mix_std": 0.20185419917106628, "beta_std": 0.06250570714473724, "beta_low": 0.0146484375, "beta_high": 0.0, "orthogonal_to_active": 0.08120116591453552, "evidence_weights": [-0.13430878520011902, -0.038621317595243454, 0.5137870907783508], "agreement_weight": 0.024012980982661247, "orthogonal_weight": 1.257347583770752}, {"head": 1, "memory_to_active": 0.5012574791908264, "contribution_to_active": 0.3073846697807312, "mix_mean": 0.22987090051174164, "mix_low": 0.1197652816772461, "mix_high": 8.0108642578125e-05, "read_cosine": 0.8165350556373596, "beta_mean": 0.2495407909154892, "nonfinite": 0.0, "weighted_memory_to_active": 0.22195439040660858, "mix_std": 0.2572094202041626, "beta_std": 0.24220003187656403, "beta_low": 0.11968994140625, "beta_high": 0.0, "orthogonal_to_active": 0.0130948880687356, "evidence_weights": [-0.006095180753618479, -0.23233425617218018, -0.15350273251533508], "agreement_weight": 0.05156895890831947, "orthogonal_weight": 0.13978610932826996}, {"head": 2, "memory_to_active": 0.13640034198760986, "contribution_to_active": 0.5634117722511292, "mix_mean": 0.5697497725486755, "mix_low": 0.002574920654296875, "mix_high": 0.0028619766235351562, "read_cosine": 0.40698593854904175, "beta_mean": 0.0028898504097014666, "nonfinite": 0.0, "weighted_memory_to_active": 0.08694133162498474, "mix_std": 0.26756346225738525, "beta_std": 0.0041262311860919, "beta_low": 0.9544677734375, "beta_high": 0.0, "orthogonal_to_active": 0.06733016669750214, "evidence_weights": [-0.1545298993587494, -0.30810829997062683, 0.8970789909362793], "agreement_weight": 0.0831572562456131, "orthogonal_weight": -0.8813464045524597}, {"head": 3, "memory_to_active": 0.08398057520389557, "contribution_to_active": 0.648594081401825, "mix_mean": 0.6766343116760254, "mix_low": 0.0057010650634765625, "mix_high": 0.019433021545410156, "read_cosine": 0.17163525521755219, "beta_mean": 0.010189266875386238, "nonfinite": 0.0, "weighted_memory_to_active": 0.05633530020713806, "mix_std": 0.250181645154953, "beta_std": 0.009937024675309658, "beta_low": 0.63629150390625, "beta_high": 0.0, "orthogonal_to_active": 0.03541422635316849, "evidence_weights": [-0.36884868144989014, -0.3217960000038147, 0.4164385199546814], "agreement_weight": 0.08217111974954605, "orthogonal_weight": -0.7444891929626465}, {"head": 4, "memory_to_active": 0.17632608115673065, "contribution_to_active": 0.4794671833515167, "mix_mean": 0.49065977334976196, "mix_low": 0.0044040679931640625, "mix_high": 0.00830841064453125, "read_cosine": 0.847143828868866, "beta_mean": 0.04524289444088936, "nonfinite": 0.0, "weighted_memory_to_active": 0.1014256551861763, "mix_std": 0.2623984217643738, "beta_std": 0.031784094870090485, "beta_low": 0.03509521484375, "beta_high": 0.0, "orthogonal_to_active": 0.030065370723605156, "evidence_weights": [-0.2215288281440735, -0.25570037961006165, 0.42688316106796265], "agreement_weight": 0.12402691692113876, "orthogonal_weight": -0.5808357000350952}, {"head": 5, "memory_to_active": 0.2782520651817322, "contribution_to_active": 0.3273478150367737, "mix_mean": 0.3466196358203888, "mix_low": 0.004353523254394531, "mix_high": 0.000843048095703125, "read_cosine": 0.7453097105026245, "beta_mean": 0.002174019580706954, "nonfinite": 0.0, "weighted_memory_to_active": 0.12110491842031479, "mix_std": 0.2490822821855545, "beta_std": 0.009201748296618462, "beta_low": 0.96197509765625, "beta_high": 0.0, "orthogonal_to_active": 0.04775872081518173, "evidence_weights": [-0.18295077979564667, -0.16253870725631714, 0.23530571162700653], "agreement_weight": 0.0849146842956543, "orthogonal_weight": -0.6583443880081177}]}, {"layer": 15, "memory_to_active": 0.06394824385643005, "contribution_to_active": 0.4850862920284271, "mix_mean": 0.402082622051239, "mix_low": 0.008828322403132915, "mix_high": 0.002731641288846731, "read_cosine": 0.6673749685287476, "beta_mean": 0.03992663323879242, "nonfinite": 0.0, "weighted_memory_to_active": 0.03016280010342598, "mix_std": 0.2829046845436096, "beta_std": 0.04634355381131172, "beta_low": 0.7097575068473816, "beta_high": 0.0, "orthogonal_to_active": 0.007055299356579781, "output_influence": 0.3061930537223816, "residual_influence": 0.1626937985420227, "state_active": 0.1273282766342163, "state_memory": 0.009989766404032707, "state_peak": 39.15190505981445, "growth_active": 1.277849793434143, "growth_memory": 1.4524019956588745, "heads": [{"head": 0, "memory_to_active": 0.11214526742696762, "contribution_to_active": 0.4774859547615051, "mix_mean": 0.3736376166343689, "mix_low": 0.005501747131347656, "mix_high": 0.0050754547119140625, "read_cosine": 0.6636499166488647, "beta_mean": 0.03323371708393097, "nonfinite": 0.0, "weighted_memory_to_active": 0.05239687114953995, "mix_std": 0.278067946434021, "beta_std": 0.024765614420175552, "beta_low": 0.13055419921875, "beta_high": 0.0, "orthogonal_to_active": 0.023474857211112976, "evidence_weights": [-0.17001427710056305, -0.17698393762111664, 0.359224408864975], "agreement_weight": 0.09411398321390152, "orthogonal_weight": -0.6963010430335999}, {"head": 1, "memory_to_active": 0.14100128412246704, "contribution_to_active": 0.41650378704071045, "mix_mean": 0.4142751693725586, "mix_low": 0.0029191970825195312, "mix_high": 0.004906654357910156, "read_cosine": 0.9035167694091797, "beta_mean": 0.0012407532194629312, "nonfinite": 0.0, "weighted_memory_to_active": 0.05734067037701607, "mix_std": 0.2836276888847351, "beta_std": 0.0031797613482922316, "beta_low": 0.97821044921875, "beta_high": 0.0, "orthogonal_to_active": 0.0006754851783625782, "evidence_weights": [-0.1639210283756256, -0.1726129949092865, 0.2371622920036316], "agreement_weight": 0.10977485775947571, "orthogonal_weight": -0.020659292116761208}, {"head": 2, "memory_to_active": 0.008526465855538845, "contribution_to_active": 0.4677233397960663, "mix_mean": 0.33439144492149353, "mix_low": 0.017000198364257812, "mix_high": 4.291534423828125e-05, "read_cosine": 0.7785540223121643, "beta_mean": 0.0001348527439404279, "nonfinite": 0.0, "weighted_memory_to_active": 0.004176177084445953, "mix_std": 0.2690044939517975, "beta_std": 0.00024667292018420994, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0006573573919013143, "evidence_weights": [-0.19968943297863007, -0.27391287684440613, 0.2277582436800003], "agreement_weight": 0.010343195870518684, "orthogonal_weight": 0.3652501702308655}, {"head": 3, "memory_to_active": 0.008874873630702496, "contribution_to_active": 0.4137343466281891, "mix_mean": 0.3639194369316101, "mix_low": 0.006534576416015625, "mix_high": 0.00014972686767578125, "read_cosine": 0.6369476318359375, "beta_mean": 5.949076512479223e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034049060195684433, "mix_std": 0.2801090180873871, "beta_std": 0.00012192597205284983, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0027699547354131937, "evidence_weights": [-0.22582793235778809, -0.25562745332717896, 0.3049992322921753], "agreement_weight": 0.02600005827844143, "orthogonal_weight": -1.0063081979751587}, {"head": 4, "memory_to_active": 0.08918564021587372, "contribution_to_active": 0.6108882427215576, "mix_mean": 0.37017351388931274, "mix_low": 0.011404037475585938, "mix_high": 0.000743865966796875, "read_cosine": 0.6709312200546265, "beta_mean": 0.20294825732707977, "nonfinite": 0.0, "weighted_memory_to_active": 0.04774995520710945, "mix_std": 0.3056620955467224, "beta_std": 0.23380157351493835, "beta_low": 0.17572021484375, "beta_high": 0.0, "orthogonal_to_active": 0.006051366683095694, "evidence_weights": [-0.21550945937633514, -0.2572891414165497, 0.007299909833818674], "agreement_weight": 0.013253491371870041, "orthogonal_weight": 0.24364544451236725}, {"head": 5, "memory_to_active": 0.023955941200256348, "contribution_to_active": 0.5241819024085999, "mix_mean": 0.5560982823371887, "mix_low": 0.009610176086425781, "mix_high": 0.005471229553222656, "read_cosine": 0.35065022110939026, "beta_mean": 0.0019427203806117177, "nonfinite": 0.0, "weighted_memory_to_active": 0.01590823382139206, "mix_std": 0.28095677495002747, "beta_std": 0.015945792198181152, "beta_low": 0.97412109375, "beta_high": 0.0, "orthogonal_to_active": 0.008702773600816727, "evidence_weights": [-0.3161851465702057, -0.3144553601741791, 0.7452351450920105], "agreement_weight": 0.25295671820640564, "orthogonal_weight": 0.7056093215942383}]}]}, {"step": 36000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.914285958999244, "memory_to_active": 0.8659799098968506, "contribution_to_active": 0.5277349352836609, "mix_mean": 0.3781443238258362, "mix_low": 0.14877338707447052, "mix_high": 0.008544247597455978, "read_cosine": 0.36475396156311035, "beta_mean": 0.07767299562692642, "nonfinite": 0.0, "weighted_memory_to_active": 0.2118372619152069, "mix_std": 0.21857236325740814, "beta_std": 0.0665644183754921, "beta_low": 0.5197455286979675, "beta_high": 1.589457315276377e-05, "orthogonal_to_active": 0.05601360648870468, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1292991638183594, "normal_token_accuracy": 0.4516128897666931, "state_active": 0.060127634555101395, "state_memory": 0.01888403668999672, "state_peak": 114.09674835205078, "growth_active": 1.1177611351013184, "growth_memory": 1.3910731077194214, "output_influence": 0.3852333724498749, "residual_influence": 0.12562771141529083, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1818795204162598, "no_memory_loss_delta": 0.4576376974582672, "scrambled_loss_delta": 0.15600496530532837, "memory_only_loss_delta": 3.2052953243255615, "no_evidence_loss_delta": 0.284740686416626, "no_agreement_loss_delta": 0.023621227592229843, "no_orthogonal_loss_delta": 0.009955699555575848}, {"index": 94, "tokens": 256, "normal_loss": 3.259676933288574, "no_memory_loss_delta": 0.5457953810691833, "scrambled_loss_delta": 0.37566208839416504, "memory_only_loss_delta": 3.1121535301208496, "no_evidence_loss_delta": 0.2474675476551056, "no_agreement_loss_delta": 0.029890574514865875, "no_orthogonal_loss_delta": 0.05660109221935272}, {"index": 188, "tokens": 256, "normal_loss": 3.482177257537842, "no_memory_loss_delta": 0.4593113660812378, "scrambled_loss_delta": 0.1454331874847412, "memory_only_loss_delta": 2.543437957763672, "no_evidence_loss_delta": 0.2186645269393921, "no_agreement_loss_delta": 0.03974270075559616, "no_orthogonal_loss_delta": 0.01818062551319599}, {"index": 283, "tokens": 131, "normal_loss": 2.082170009613037, "no_memory_loss_delta": 0.32469722628593445, "scrambled_loss_delta": 0.22631017863750458, "memory_only_loss_delta": 2.541949510574341, "no_evidence_loss_delta": 0.1011514738202095, "no_agreement_loss_delta": 0.001762121682986617, "no_orthogonal_loss_delta": 0.009347173385322094}], "agreement_weight_abs_mean": 0.16529807448387146, "agreement_weight_abs_max": 4.09531831741333, "agreement_weight_mean": 0.14318154752254486, "orthogonal_weight_abs_mean": 0.7203064560890198, "orthogonal_weight_abs_max": 6.518043041229248, "orthogonal_weight_mean": -0.1450052112340927, "no_memory_loss": 3.593146324157715, "no_memory_loss_delta": 0.4638464152812958, "no_memory_token_accuracy_gain": 0.07786429673433304, "no_memory_token_win_fraction": 0.6607341766357422, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 37.11398696431567, "scrambled_loss": 3.355088472366333, "scrambled_loss_delta": 0.22578896582126617, "scrambled_token_accuracy_gain": 0.032258063554763794, "scrambled_token_win_fraction": 0.6129032373428345, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.21135347852728, "memory_only_loss": 6.022939205169678, "memory_only_loss_delta": 2.8936402797698975, "memory_only_token_accuracy_gain": 0.27919912338256836, "memory_only_token_win_fraction": 0.909899890422821, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.46257325595052, "no_evidence_loss": 3.3578579425811768, "no_evidence_loss_delta": 0.22855868935585022, "no_evidence_token_accuracy_gain": 0.030033370479941368, "no_evidence_token_win_fraction": 0.5873192548751831, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 20.432040208894225, "no_agreement_loss": 3.156111478805542, "no_agreement_loss_delta": 0.026812000200152397, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.46718576550483704, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.6455749554795993, "no_orthogonal_loss": 3.1547913551330566, "no_orthogonal_loss_delta": 0.025491945445537567, "no_orthogonal_token_accuracy_gain": -0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5850945711135864, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 2.5169769242695037, "short_no_memory_loss_delta": 0.3716745376586914, "long_context_memory_gain": 0.09217187762260437, "normal_context_loss_gain": 0.12478768080472946, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.333073616027832, "contribution_to_active": 1.076486349105835, "mix_mean": 0.10150852054357529, "mix_low": 0.5414400100708008, "mix_high": 0.004119396209716797, "read_cosine": 0.4358757734298706, "beta_mean": 0.250118613243103, "nonfinite": 0.0, "weighted_memory_to_active": 1.1223933696746826, "mix_std": 0.08777599781751633, "beta_std": 0.14673519134521484, "beta_low": 0.03060913272202015, "beta_high": 0.0, "orthogonal_to_active": 0.05707366019487381, "output_influence": 0.18577107787132263, "residual_influence": 0.22221055626869202, "state_active": 0.071072518825531, "state_memory": 0.0472249761223793, "state_peak": 44.91571044921875, "growth_active": 1.2595937252044678, "growth_memory": 1.254753589630127, "heads": [{"head": 0, "memory_to_active": 2.443143606185913, "contribution_to_active": 0.4447918236255646, "mix_mean": 0.020723670721054077, "mix_low": 0.5740299224853516, "mix_high": 0.0, "read_cosine": 0.21490973234176636, "beta_mean": 0.21138866245746613, "nonfinite": 0.0, "weighted_memory_to_active": 0.5220872163772583, "mix_std": 0.05946579948067665, "beta_std": 0.10538816452026367, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0394306406378746, "evidence_weights": [-0.024982290342450142, -0.07296925038099289, -0.12912020087242126], "agreement_weight": -0.12563401460647583, "orthogonal_weight": -0.08280818164348602}, {"head": 1, "memory_to_active": 5.3086466789245605, "contribution_to_active": 2.0675768852233887, "mix_mean": 0.015120443888008595, "mix_low": 0.7077732086181641, "mix_high": 0.0, "read_cosine": 0.523646354675293, "beta_mean": 0.46779537200927734, "nonfinite": 0.0, "weighted_memory_to_active": 2.393171787261963, "mix_std": 0.0514385849237442, "beta_std": 0.2079668492078781, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08395576477050781, "evidence_weights": [0.03431442752480507, -0.099212646484375, 0.4313810169696808], "agreement_weight": 0.03095574676990509, "orthogonal_weight": -0.07221163809299469}, {"head": 2, "memory_to_active": 2.6782031059265137, "contribution_to_active": 0.5080375671386719, "mix_mean": 0.013513009995222092, "mix_low": 0.6731967926025391, "mix_high": 0.0, "read_cosine": 0.28650689125061035, "beta_mean": 0.30230894684791565, "nonfinite": 0.0, "weighted_memory_to_active": 0.5954904556274414, "mix_std": 0.031438883394002914, "beta_std": 0.19899792969226837, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05519912764430046, "evidence_weights": [0.061953917145729065, 0.06725570559501648, 0.03497587516903877], "agreement_weight": 0.008225888013839722, "orthogonal_weight": -0.19303764402866364}, {"head": 3, "memory_to_active": 6.832870960235596, "contribution_to_active": 2.9527742862701416, "mix_mean": 0.03720078989863396, "mix_low": 0.44747257232666016, "mix_high": 0.0, "read_cosine": 0.3273236155509949, "beta_mean": 0.32596588134765625, "nonfinite": 0.0, "weighted_memory_to_active": 3.1193244457244873, "mix_std": 0.08291785418987274, "beta_std": 0.22949932515621185, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14170318841934204, "evidence_weights": [-0.0493118055164814, -0.06608644127845764, -0.15602710843086243], "agreement_weight": -0.021104522049427032, "orthogonal_weight": -0.052773453295230865}, {"head": 4, "memory_to_active": 2.6417300701141357, "contribution_to_active": 0.06717411428689957, "mix_mean": 0.006441964767873287, "mix_low": 0.826533317565918, "mix_high": 0.0, "read_cosine": 0.3353610038757324, "beta_mean": 0.16613274812698364, "nonfinite": 0.0, "weighted_memory_to_active": 0.05836035683751106, "mix_std": 0.012704968452453613, "beta_std": 0.11924627423286438, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0057065170258283615, "evidence_weights": [0.12839287519454956, 0.08046180754899979, -0.36027926206588745], "agreement_weight": -0.12158247828483582, "orthogonal_weight": 0.1004602462053299}, {"head": 5, "memory_to_active": 0.09384608268737793, "contribution_to_active": 0.41856348514556885, "mix_mean": 0.5160512328147888, "mix_low": 0.019634246826171875, "mix_high": 0.02471637725830078, "read_cosine": 0.9275069236755371, "beta_mean": 0.027119994163513184, "nonfinite": 0.0, "weighted_memory_to_active": 0.045925039798021317, "mix_std": 0.28868988156318665, "beta_std": 0.019312553107738495, "beta_low": 0.18365478515625, "beta_high": 0.0, "orthogonal_to_active": 0.016446702182292938, "evidence_weights": [-0.22248071432113647, -0.23409128189086914, 0.3166637420654297], "agreement_weight": 0.2719650864601135, "orthogonal_weight": -0.6773572564125061}]}, {"layer": 1, "memory_to_active": 2.334752082824707, "contribution_to_active": 0.510511577129364, "mix_mean": 0.1860661804676056, "mix_low": 0.3717314600944519, "mix_high": 0.0009872118243947625, "read_cosine": 0.32689720392227173, "beta_mean": 0.17866036295890808, "nonfinite": 0.0, "weighted_memory_to_active": 0.37891536951065063, "mix_std": 0.11965475976467133, "beta_std": 0.1285877376794815, "beta_low": 0.27288818359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.05257945507764816, "output_influence": 0.19230051338672638, "residual_influence": 0.0481041818857193, "state_active": 0.03858867287635803, "state_memory": 0.03598889335989952, "state_peak": 37.25618362426758, "growth_active": 1.2968382835388184, "growth_memory": 1.5135784149169922, "heads": [{"head": 0, "memory_to_active": 0.04019354283809662, "contribution_to_active": 0.4308696389198303, "mix_mean": 0.4420226812362671, "mix_low": 0.009860992431640625, "mix_high": 0.0012197494506835938, "read_cosine": 0.019414052367210388, "beta_mean": 0.0002999505668412894, "nonfinite": 0.0, "weighted_memory_to_active": 0.020731789991259575, "mix_std": 0.2867804169654846, "beta_std": 0.0003397726686671376, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.019521309062838554, "evidence_weights": [-0.1869097650051117, -0.24345797300338745, 0.1446780562400818], "agreement_weight": 0.1756221354007721, "orthogonal_weight": -1.0156593322753906}, {"head": 1, "memory_to_active": 2.8919363021850586, "contribution_to_active": 0.2249087691307068, "mix_mean": 0.021933570504188538, "mix_low": 0.4723367691040039, "mix_high": 4.76837158203125e-06, "read_cosine": 0.48562198877334595, "beta_mean": 0.2463758885860443, "nonfinite": 0.0, "weighted_memory_to_active": 0.2530476152896881, "mix_std": 0.04609938710927963, "beta_std": 0.18008705973625183, "beta_low": 0.00274658203125, "beta_high": 0.000244140625, "orthogonal_to_active": 0.014641324058175087, "evidence_weights": [0.014565708115696907, -0.02816290408372879, 0.013941014185547829], "agreement_weight": -0.004600373096764088, "orthogonal_weight": 0.1339534968137741}, {"head": 2, "memory_to_active": 0.2622206509113312, "contribution_to_active": 0.5813749432563782, "mix_mean": 0.5899291634559631, "mix_low": 0.0011444091796875, "mix_high": 0.004694938659667969, "read_cosine": 0.7067870497703552, "beta_mean": 0.01575000397861004, "nonfinite": 0.0, "weighted_memory_to_active": 0.21266721189022064, "mix_std": 0.2509852349758148, "beta_std": 0.019740348681807518, "beta_low": 0.56072998046875, "beta_high": 0.0, "orthogonal_to_active": 0.050172604620456696, "evidence_weights": [-0.3007453382015228, -0.3499578833580017, 0.42949238419532776], "agreement_weight": 0.02779662422835827, "orthogonal_weight": -0.4077003300189972}, {"head": 3, "memory_to_active": 2.9577836990356445, "contribution_to_active": 0.20516838133335114, "mix_mean": 0.01918160915374756, "mix_low": 0.639592170715332, "mix_high": 0.0, "read_cosine": 0.22876742482185364, "beta_mean": 0.35700562596321106, "nonfinite": 0.0, "weighted_memory_to_active": 0.3127756118774414, "mix_std": 0.050889577716588974, "beta_std": 0.2522161602973938, "beta_low": 0.03759765625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0854589194059372, "evidence_weights": [0.04499393701553345, -0.004701989237219095, 0.1280115693807602], "agreement_weight": -0.02193400450050831, "orthogonal_weight": -0.3719077706336975}, {"head": 4, "memory_to_active": 3.7031235694885254, "contribution_to_active": 1.53764009475708, "mix_mean": 0.03594088926911354, "mix_low": 0.2812032699584961, "mix_high": 1.9073486328125e-06, "read_cosine": 0.3333907723426819, "beta_mean": 0.15760891139507294, "nonfinite": 0.0, "weighted_memory_to_active": 1.4026241302490234, "mix_std": 0.06290976703166962, "beta_std": 0.10734891891479492, "beta_low": 0.00921630859375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.13646608591079712, "evidence_weights": [-0.04469562694430351, 0.0018001897260546684, 0.024770144373178482], "agreement_weight": -0.02693939208984375, "orthogonal_weight": 0.12254171818494797}, {"head": 5, "memory_to_active": 4.153253555297852, "contribution_to_active": 0.08310748636722565, "mix_mean": 0.007389053236693144, "mix_low": 0.8262510299682617, "mix_high": 1.9073486328125e-06, "read_cosine": 0.1874018907546997, "beta_mean": 0.29492175579071045, "nonfinite": 0.0, "weighted_memory_to_active": 0.07164593040943146, "mix_std": 0.020264137536287308, "beta_std": 0.2117941826581955, "beta_low": 0.02703857421875, "beta_high": 0.0, "orthogonal_to_active": 0.009216470643877983, "evidence_weights": [0.1632082164287567, 0.08203420042991638, 0.25578194856643677], "agreement_weight": -0.1531100571155548, "orthogonal_weight": 0.13588520884513855}]}, {"layer": 2, "memory_to_active": 1.0894960165023804, "contribution_to_active": 0.33318138122558594, "mix_mean": 0.3593234419822693, "mix_low": 0.25522851943969727, "mix_high": 0.0030730566941201687, "read_cosine": 0.3265179395675659, "beta_mean": 0.09044308960437775, "nonfinite": 0.0, "weighted_memory_to_active": 0.06960965692996979, "mix_std": 0.19395104050636292, "beta_std": 0.07150088250637054, "beta_low": 0.5446268916130066, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.017314210534095764, "output_influence": 0.3319208323955536, "residual_influence": 0.08365582674741745, "state_active": 0.03806713595986366, "state_memory": 0.01739007607102394, "state_peak": 35.949501037597656, "growth_active": 1.029395580291748, "growth_memory": 1.5933982133865356, "heads": [{"head": 0, "memory_to_active": 0.0070193312130868435, "contribution_to_active": 0.3284327983856201, "mix_mean": 0.5429085493087769, "mix_low": 0.0021524429321289062, "mix_high": 0.004412651062011719, "read_cosine": 0.1878628134727478, "beta_mean": 5.721590423490852e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00396377919241786, "mix_std": 0.28789830207824707, "beta_std": 8.599470311310142e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020814472809433937, "evidence_weights": [-0.31888923048973083, -0.1981634646654129, 0.08865247666835785], "agreement_weight": 0.24015840888023376, "orthogonal_weight": -0.5507217049598694}, {"head": 1, "memory_to_active": 3.0441064834594727, "contribution_to_active": 0.06628082692623138, "mix_mean": 0.0049314480274915695, "mix_low": 0.8971376419067383, "mix_high": 0.0, "read_cosine": 0.18799440562725067, "beta_mean": 0.28042685985565186, "nonfinite": 0.0, "weighted_memory_to_active": 0.04819358140230179, "mix_std": 0.011887817643582821, "beta_std": 0.1536928117275238, "beta_low": 0.00018310546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0197999756783247, "evidence_weights": [0.11387823522090912, 0.10829311609268188, 0.5044063329696655], "agreement_weight": 0.02869643270969391, "orthogonal_weight": 0.42599308490753174}, {"head": 2, "memory_to_active": 0.020045805722475052, "contribution_to_active": 0.4129112660884857, "mix_mean": 0.49349427223205566, "mix_low": 0.0019321441650390625, "mix_high": 0.0015802383422851562, "read_cosine": 0.291874498128891, "beta_mean": 4.242137219989672e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0073602222837507725, "mix_std": 0.28152352571487427, "beta_std": 7.076685142237693e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001368225901387632, "evidence_weights": [-0.23839087784290314, -0.27625760436058044, 0.4610070288181305], "agreement_weight": 0.023919731378555298, "orthogonal_weight": -0.20658214390277863}, {"head": 3, "memory_to_active": 0.18940401077270508, "contribution_to_active": 0.49014896154403687, "mix_mean": 0.4952138364315033, "mix_low": 0.0005254745483398438, "mix_high": 0.00566864013671875, "read_cosine": 0.6714425086975098, "beta_mean": 0.04077515751123428, "nonfinite": 0.0, "weighted_memory_to_active": 0.10928795486688614, "mix_std": 0.26702815294265747, "beta_std": 0.04647863283753395, "beta_low": 0.22784423828125, "beta_high": 0.0, "orthogonal_to_active": 0.06473471969366074, "evidence_weights": [-0.1782616674900055, -0.28132861852645874, 0.8639254570007324], "agreement_weight": 0.061722129583358765, "orthogonal_weight": -0.886437714099884}, {"head": 4, "memory_to_active": 3.262593984603882, "contribution_to_active": 0.20725594460964203, "mix_mean": 0.01646416261792183, "mix_low": 0.6280241012573242, "mix_high": 0.0, "read_cosine": 0.18487942218780518, "beta_mean": 0.22131343185901642, "nonfinite": 0.0, "weighted_memory_to_active": 0.24063733220100403, "mix_std": 0.04161061719059944, "beta_std": 0.22860842943191528, "beta_low": 0.03973388671875, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.015575427561998367, "evidence_weights": [0.025290222838521004, 0.017736338078975677, 0.06481675803661346], "agreement_weight": 0.012425852939486504, "orthogonal_weight": -0.07067321240901947}, {"head": 5, "memory_to_active": 0.013806231319904327, "contribution_to_active": 0.49405843019485474, "mix_mean": 0.6029282808303833, "mix_low": 0.0015993118286132812, "mix_high": 0.0067768096923828125, "read_cosine": 0.4350539445877075, "beta_mean": 4.341906605986878e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0082150399684906, "mix_std": 0.27375784516334534, "beta_std": 6.86410057824105e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003254532930441201, "evidence_weights": [-0.26510393619537354, -0.2834455668926239, 0.9025618433952332], "agreement_weight": 0.0951380580663681, "orthogonal_weight": 0.05884220078587532}]}, {"layer": 3, "memory_to_active": 0.6812915205955505, "contribution_to_active": 0.400017112493515, "mix_mean": 0.3753869831562042, "mix_low": 0.12059768289327621, "mix_high": 0.007493337616324425, "read_cosine": 0.2010612189769745, "beta_mean": 0.03608781099319458, "nonfinite": 0.0, "weighted_memory_to_active": 0.16024871170520782, "mix_std": 0.24548012018203735, "beta_std": 0.03184682875871658, "beta_low": 0.597412109375, "beta_high": 0.0, "orthogonal_to_active": 0.07730373740196228, "output_influence": 0.3339853882789612, "residual_influence": 0.06822414696216583, "state_active": 0.022169921547174454, "state_memory": 0.009564235806465149, "state_peak": 16.85773277282715, "growth_active": 1.002298355102539, "growth_memory": 2.0071041584014893, "heads": [{"head": 0, "memory_to_active": 0.5108532905578613, "contribution_to_active": 0.5434988737106323, "mix_mean": 0.4929317831993103, "mix_low": 0.012932777404785156, "mix_high": 0.01631450653076172, "read_cosine": 0.18052217364311218, "beta_mean": 0.013440337963402271, "nonfinite": 0.0, "weighted_memory_to_active": 0.2917602062225342, "mix_std": 0.32073837518692017, "beta_std": 0.01582600548863411, "beta_low": 0.56524658203125, "beta_high": 0.0, "orthogonal_to_active": 0.19929896295070648, "evidence_weights": [-0.19038821756839752, -0.22134357690811157, 0.7544675469398499], "agreement_weight": 0.12581999599933624, "orthogonal_weight": -0.7113668918609619}, {"head": 1, "memory_to_active": 0.011676022782921791, "contribution_to_active": 0.3981018364429474, "mix_mean": 0.46442902088165283, "mix_low": 0.012511253356933594, "mix_high": 0.006496429443359375, "read_cosine": 0.18367010354995728, "beta_mean": 5.9725218306994066e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005904180929064751, "mix_std": 0.30992162227630615, "beta_std": 0.00010732482769526541, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003946712240576744, "evidence_weights": [-0.22145266830921173, -0.1880701184272766, 0.6103354692459106], "agreement_weight": 0.3742223381996155, "orthogonal_weight": -0.7248823642730713}, {"head": 2, "memory_to_active": 2.4897139072418213, "contribution_to_active": 0.27201586961746216, "mix_mean": 0.04151923209428787, "mix_low": 0.4161806106567383, "mix_high": 0.0, "read_cosine": 0.21699143946170807, "beta_mean": 0.1656826287508011, "nonfinite": 0.0, "weighted_memory_to_active": 0.31745970249176025, "mix_std": 0.09539727121591568, "beta_std": 0.13068732619285583, "beta_low": 0.01263427734375, "beta_high": 0.0, "orthogonal_to_active": 0.0014580838615074754, "evidence_weights": [-0.06486064195632935, 0.02011922374367714, 0.005071109160780907], "agreement_weight": -0.018927117809653282, "orthogonal_weight": -0.00592091865837574}, {"head": 3, "memory_to_active": 0.02946198172867298, "contribution_to_active": 0.4781912565231323, "mix_mean": 0.5471911430358887, "mix_low": 0.01007843017578125, "mix_high": 0.008362770080566406, "read_cosine": 0.2914983332157135, "beta_mean": 0.00011556576646398753, "nonfinite": 0.0, "weighted_memory_to_active": 0.01964840665459633, "mix_std": 0.30109408497810364, "beta_std": 0.00030295061878859997, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005055845715105534, "evidence_weights": [-0.21968474984169006, -0.22626955807209015, 1.9990445375442505], "agreement_weight": 1.5201547145843506, "orthogonal_weight": -0.324775755405426}, {"head": 4, "memory_to_active": 0.7242419719696045, "contribution_to_active": 0.09742067754268646, "mix_mean": 0.09436551481485367, "mix_low": 0.262054443359375, "mix_high": 2.6702880859375e-05, "read_cosine": 0.14403551816940308, "beta_mean": 0.009206421673297882, "nonfinite": 0.0, "weighted_memory_to_active": 0.10571548342704773, "mix_std": 0.1543002426624298, "beta_std": 0.01596144586801529, "beta_low": 0.7410888671875, "beta_high": 0.0, "orthogonal_to_active": 0.08621951937675476, "evidence_weights": [0.01879240944981575, -0.036441195756196976, 0.23880401253700256], "agreement_weight": -0.017125392332673073, "orthogonal_weight": -0.8328197002410889}, {"head": 5, "memory_to_active": 0.3218018710613251, "contribution_to_active": 0.6108741164207458, "mix_mean": 0.6118850708007812, "mix_low": 0.009828567504882812, "mix_high": 0.013759613037109375, "read_cosine": 0.18964973092079163, "beta_mean": 0.028022196143865585, "nonfinite": 0.0, "weighted_memory_to_active": 0.22100432217121124, "mix_std": 0.2914291024208069, "beta_std": 0.02819589152932167, "beta_low": 0.26556396484375, "beta_high": 0.0, "orthogonal_to_active": 0.1678433120250702, "evidence_weights": [-0.2341872900724411, -0.269622802734375, 0.517256498336792], "agreement_weight": 0.1329202950000763, "orthogonal_weight": -0.8086580038070679}]}, {"layer": 4, "memory_to_active": 1.145514965057373, "contribution_to_active": 0.4201412796974182, "mix_mean": 0.3485760986804962, "mix_low": 0.27895212173461914, "mix_high": 0.005947908386588097, "read_cosine": 0.3489580452442169, "beta_mean": 0.12462118268013, "nonfinite": 0.0, "weighted_memory_to_active": 0.12358322739601135, "mix_std": 0.1543898731470108, "beta_std": 0.11509484052658081, "beta_low": 0.3724365234375, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03657282143831253, "output_influence": 0.3951389193534851, "residual_influence": 0.10955554991960526, "state_active": 0.03806313872337341, "state_memory": 0.019800882786512375, "state_peak": 40.031063079833984, "growth_active": 1.235377550125122, "growth_memory": 1.5664646625518799, "heads": [{"head": 0, "memory_to_active": 0.2361992597579956, "contribution_to_active": 0.6927874684333801, "mix_mean": 0.6506176590919495, "mix_low": 0.010287284851074219, "mix_high": 0.01383209228515625, "read_cosine": 0.5035158395767212, "beta_mean": 0.2689429521560669, "nonfinite": 0.0, "weighted_memory_to_active": 0.17327781021595, "mix_std": 0.3385395407676697, "beta_std": 0.2795974910259247, "beta_low": 0.21826171875, "beta_high": 0.0, "orthogonal_to_active": 0.0611322745680809, "evidence_weights": [-0.2651386857032776, -0.3628186881542206, 0.2034471333026886], "agreement_weight": 0.01829211227595806, "orthogonal_weight": 0.4334130883216858}, {"head": 1, "memory_to_active": 0.07584524154663086, "contribution_to_active": 0.6101130843162537, "mix_mean": 0.6491367816925049, "mix_low": 6.103515625e-05, "mix_high": 0.006430625915527344, "read_cosine": 0.7001021504402161, "beta_mean": 0.001329523976892233, "nonfinite": 0.0, "weighted_memory_to_active": 0.05330085754394531, "mix_std": 0.24377615749835968, "beta_std": 0.005493303295224905, "beta_low": 0.97406005859375, "beta_high": 0.0, "orthogonal_to_active": 0.02939274162054062, "evidence_weights": [-0.23483407497406006, -0.32481518387794495, 0.7363205552101135], "agreement_weight": 0.15483590960502625, "orthogonal_weight": -0.834821879863739}, {"head": 2, "memory_to_active": 2.3631253242492676, "contribution_to_active": 0.18991553783416748, "mix_mean": 0.017628170549869537, "mix_low": 0.6020851135253906, "mix_high": 1.239776611328125e-05, "read_cosine": 0.2373153269290924, "beta_mean": 0.11550018936395645, "nonfinite": 0.0, "weighted_memory_to_active": 0.1942560374736786, "mix_std": 0.036548275500535965, "beta_std": 0.09323183447122574, "beta_low": 0.00634765625, "beta_high": 0.0, "orthogonal_to_active": 0.018663639202713966, "evidence_weights": [0.061144109815359116, 0.03451167419552803, 0.010625410825014114], "agreement_weight": 0.1234559640288353, "orthogonal_weight": -0.13209019601345062}, {"head": 3, "memory_to_active": 1.7798376083374023, "contribution_to_active": 0.1245153620839119, "mix_mean": 0.03142533451318741, "mix_low": 0.41701221466064453, "mix_high": 0.0, "read_cosine": 0.2037290632724762, "beta_mean": 0.13481588661670685, "nonfinite": 0.0, "weighted_memory_to_active": 0.163190558552742, "mix_std": 0.05593663454055786, "beta_std": 0.1290871798992157, "beta_low": 0.0355224609375, "beta_high": 0.0, "orthogonal_to_active": 0.009629756212234497, "evidence_weights": [-0.02360159158706665, 0.05734188109636307, -0.22573956847190857], "agreement_weight": 0.05512818694114685, "orthogonal_weight": -0.07446177303791046}, {"head": 4, "memory_to_active": 0.020666684955358505, "contribution_to_active": 0.6652405858039856, "mix_mean": 0.7287105321884155, "mix_low": 8.20159912109375e-05, "mix_high": 0.015406608581542969, "read_cosine": 0.15052802860736847, "beta_mean": 3.353338615852408e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015275988727807999, "mix_std": 0.2203022986650467, "beta_std": 0.0001611886254977435, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0034741221461445093, "evidence_weights": [-0.322604238986969, -0.3317147493362427, 0.6666354537010193], "agreement_weight": 0.2632887363433838, "orthogonal_weight": -0.24387498199939728}, {"head": 5, "memory_to_active": 2.39741587638855, "contribution_to_active": 0.23827561736106873, "mix_mean": 0.013937962241470814, "mix_low": 0.6441850662231445, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2985578179359436, "beta_mean": 0.2271050065755844, "nonfinite": 0.0, "weighted_memory_to_active": 0.1421981155872345, "mix_std": 0.031236324459314346, "beta_std": 0.18299803137779236, "beta_low": 0.00042724609375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.09714438766241074, "evidence_weights": [0.0452270433306694, -0.03631173074245453, -0.26869621872901917], "agreement_weight": -0.0608370304107666, "orthogonal_weight": 0.8649201393127441}]}, {"layer": 5, "memory_to_active": 0.5044571161270142, "contribution_to_active": 0.49594324827194214, "mix_mean": 0.4546271860599518, "mix_low": 0.07223877310752869, "mix_high": 0.013897418975830078, "read_cosine": 0.12392878532409668, "beta_mean": 0.05834057182073593, "nonfinite": 0.0, "weighted_memory_to_active": 0.14761444926261902, "mix_std": 0.254807710647583, "beta_std": 0.07585780322551727, "beta_low": 0.6357218623161316, "beta_high": 0.0, "orthogonal_to_active": 0.05344056710600853, "output_influence": 0.5038866400718689, "residual_influence": 0.1319332867860794, "state_active": 0.04463471099734306, "state_memory": 0.011861502192914486, "state_peak": 64.67781829833984, "growth_active": 1.0940803289413452, "growth_memory": 1.2823548316955566, "heads": [{"head": 0, "memory_to_active": 0.6393718123435974, "contribution_to_active": 0.4111480116844177, "mix_mean": 0.26235848665237427, "mix_low": 0.058803558349609375, "mix_high": 0.000217437744140625, "read_cosine": -0.09405574202537537, "beta_mean": 0.03673114255070686, "nonfinite": 0.0, "weighted_memory_to_active": 0.1541345715522766, "mix_std": 0.2522633969783783, "beta_std": 0.06089112162590027, "beta_low": 0.35357666015625, "beta_high": 0.0, "orthogonal_to_active": 0.03824716433882713, "evidence_weights": [-0.10187781602144241, -0.13398362696170807, -0.2371152937412262], "agreement_weight": -0.012864457443356514, "orthogonal_weight": -0.2471853792667389}, {"head": 1, "memory_to_active": 0.02239290624856949, "contribution_to_active": 0.528499960899353, "mix_mean": 0.6438162922859192, "mix_low": 0.0024805068969726562, "mix_high": 0.021709442138671875, "read_cosine": 0.1712808907032013, "beta_mean": 5.3944342653267086e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011842021718621254, "mix_std": 0.2938522696495056, "beta_std": 9.916500857798383e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010977668687701225, "evidence_weights": [-0.23792612552642822, -0.2768256962299347, 0.9045356512069702], "agreement_weight": 0.5361891984939575, "orthogonal_weight": -0.9594339728355408}, {"head": 2, "memory_to_active": 0.01294834166765213, "contribution_to_active": 0.5663644671440125, "mix_mean": 0.6310817003250122, "mix_low": 0.0033712387084960938, "mix_high": 0.017375946044921875, "read_cosine": -0.16730299592018127, "beta_mean": 0.00013705799938179553, "nonfinite": 0.0, "weighted_memory_to_active": 0.008327886462211609, "mix_std": 0.27494731545448303, "beta_std": 0.0002110116183757782, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019031948177143931, "evidence_weights": [-0.2514582872390747, -0.3480110764503479, 0.8656762838363647], "agreement_weight": -0.014357070438563824, "orthogonal_weight": -0.2775154411792755}, {"head": 3, "memory_to_active": 1.7952685356140137, "contribution_to_active": 0.21193161606788635, "mix_mean": 0.08871890604496002, "mix_low": 0.17040729522705078, "mix_high": 9.5367431640625e-07, "read_cosine": 0.13556838035583496, "beta_mean": 0.10593418776988983, "nonfinite": 0.0, "weighted_memory_to_active": 0.33083051443099976, "mix_std": 0.12803882360458374, "beta_std": 0.1537865847349167, "beta_low": 0.1983642578125, "beta_high": 0.0, "orthogonal_to_active": 0.1269896924495697, "evidence_weights": [-0.06334272772073746, -0.09197782725095749, -0.06284414231777191], "agreement_weight": 0.04964088276028633, "orthogonal_weight": -0.4537851810455322}, {"head": 4, "memory_to_active": 0.4447701871395111, "contribution_to_active": 0.5974349975585938, "mix_mean": 0.3244321048259735, "mix_low": 0.19574451446533203, "mix_high": 0.008214950561523438, "read_cosine": 0.5474939346313477, "beta_mean": 0.20681598782539368, "nonfinite": 0.0, "weighted_memory_to_active": 0.3043641448020935, "mix_std": 0.34196099638938904, "beta_std": 0.2383982241153717, "beta_low": 0.26751708984375, "beta_high": 0.0, "orthogonal_to_active": 0.0932367667555809, "evidence_weights": [-0.11048103868961334, -0.28024959564208984, -0.0780186876654625], "agreement_weight": 0.04392717778682709, "orthogonal_weight": 0.48092979192733765}, {"head": 5, "memory_to_active": 0.11199046671390533, "contribution_to_active": 0.6602804660797119, "mix_mean": 0.7773555517196655, "mix_low": 0.0026254653930664062, "mix_high": 0.03586578369140625, "read_cosine": 0.1505882292985916, "beta_mean": 0.0003711326280608773, "nonfinite": 0.0, "weighted_memory_to_active": 0.07618754357099533, "mix_std": 0.2377834916114807, "beta_std": 0.0017606532201170921, "beta_low": 0.994873046875, "beta_high": 0.0, "orthogonal_to_active": 0.04928891360759735, "evidence_weights": [-0.3255954682826996, -0.3271072804927826, 2.012408971786499], "agreement_weight": 0.3970857262611389, "orthogonal_weight": -0.6860396862030029}]}, {"layer": 6, "memory_to_active": 0.34503182768821716, "contribution_to_active": 0.5907291173934937, "mix_mean": 0.5519827604293823, "mix_low": 0.07049306482076645, "mix_high": 0.025121372193098068, "read_cosine": 0.25905704498291016, "beta_mean": 0.06801771372556686, "nonfinite": 0.0, "weighted_memory_to_active": 0.16493380069732666, "mix_std": 0.2397836297750473, "beta_std": 0.07778355479240417, "beta_low": 0.5103251338005066, "beta_high": 0.0, "orthogonal_to_active": 0.03581959754228592, "output_influence": 0.5095570683479309, "residual_influence": 0.16419890522956848, "state_active": 0.04554833844304085, "state_memory": 0.008439222350716591, "state_peak": 87.30615997314453, "growth_active": 1.1607681512832642, "growth_memory": 1.1788101196289062, "heads": [{"head": 0, "memory_to_active": 0.10678017884492874, "contribution_to_active": 0.6201450228691101, "mix_mean": 0.7446359395980835, "mix_low": 0.00015544891357421875, "mix_high": 0.013926506042480469, "read_cosine": -0.0022064968943595886, "beta_mean": 0.01730497181415558, "nonfinite": 0.0, "weighted_memory_to_active": 0.08045150339603424, "mix_std": 0.22439564764499664, "beta_std": 0.021357908844947815, "beta_low": 0.4757080078125, "beta_high": 0.0, "orthogonal_to_active": 0.06794898957014084, "evidence_weights": [-0.3134324252605438, -0.33475518226623535, 0.7011968493461609], "agreement_weight": 0.21381480991840363, "orthogonal_weight": -0.9239336252212524}, {"head": 1, "memory_to_active": 0.0060650138184428215, "contribution_to_active": 0.4943680763244629, "mix_mean": 0.5348948836326599, "mix_low": 0.0011692047119140625, "mix_high": 0.0076732635498046875, "read_cosine": 0.5091461539268494, "beta_mean": 1.0705862223403528e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0027216854505240917, "mix_std": 0.2841586768627167, "beta_std": 1.685476308921352e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013774315593764186, "evidence_weights": [-0.20244336128234863, -0.2928757965564728, 1.0827971696853638], "agreement_weight": 0.05412856489419937, "orthogonal_weight": -0.6873871684074402}, {"head": 2, "memory_to_active": 0.9904204607009888, "contribution_to_active": 0.0790349617600441, "mix_mean": 0.029112813994288445, "mix_low": 0.40979480743408203, "mix_high": 0.0, "read_cosine": 0.21832099556922913, "beta_mean": 0.04438436031341553, "nonfinite": 0.0, "weighted_memory_to_active": 0.057072386145591736, "mix_std": 0.049754343926906586, "beta_std": 0.04738973081111908, "beta_low": 0.0697021484375, "beta_high": 0.0, "orthogonal_to_active": 0.0012945488560944796, "evidence_weights": [0.08607501536607742, -0.013882648199796677, 0.2648274004459381], "agreement_weight": 0.14073635637760162, "orthogonal_weight": 0.027736397460103035}, {"head": 3, "memory_to_active": 0.16511546075344086, "contribution_to_active": 0.6619477868080139, "mix_mean": 0.6204077005386353, "mix_low": 0.009990692138671875, "mix_high": 0.011857986450195312, "read_cosine": 0.42848658561706543, "beta_mean": 0.21391664445400238, "nonfinite": 0.0, "weighted_memory_to_active": 0.11751888692378998, "mix_std": 0.3325974941253662, "beta_std": 0.24827684462070465, "beta_low": 0.234375, "beta_high": 0.0, "orthogonal_to_active": 0.06478967517614365, "evidence_weights": [-0.16933202743530273, -0.33143988251686096, 0.18714018166065216], "agreement_weight": -0.011961364187300205, "orthogonal_weight": 0.6412099599838257}, {"head": 4, "memory_to_active": 0.7678647041320801, "contribution_to_active": 1.1013383865356445, "mix_mean": 0.7501022219657898, "mix_low": 0.001125335693359375, "mix_high": 0.11363983154296875, "read_cosine": 0.1109810322523117, "beta_mean": 0.13222603499889374, "nonfinite": 0.0, "weighted_memory_to_active": 0.7086777091026306, "mix_std": 0.29024258255958557, "beta_std": 0.149116650223732, "beta_low": 0.282470703125, "beta_high": 0.0, "orthogonal_to_active": 0.05634389817714691, "evidence_weights": [-0.2487650364637375, -0.5972070693969727, -0.13714969158172607], "agreement_weight": -0.011739435605704784, "orthogonal_weight": 0.09103098511695862}, {"head": 5, "memory_to_active": 0.03394503891468048, "contribution_to_active": 0.5875402092933655, "mix_mean": 0.6327430009841919, "mix_low": 0.0007228851318359375, "mix_high": 0.0036306381225585938, "read_cosine": 0.2896139919757843, "beta_mean": 0.00026355666341260076, "nonfinite": 0.0, "weighted_memory_to_active": 0.023160655051469803, "mix_std": 0.25755295157432556, "beta_std": 0.000543345115147531, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.02316301316022873, "evidence_weights": [-0.23988652229309082, -0.3074493408203125, 0.4233277142047882], "agreement_weight": 0.14942555129528046, "orthogonal_weight": -1.075619101524353}]}, {"layer": 7, "memory_to_active": 1.745804786682129, "contribution_to_active": 0.22369469702243805, "mix_mean": 0.11182177811861038, "mix_low": 0.27435845136642456, "mix_high": 0.0001932780141942203, "read_cosine": 0.22721277177333832, "beta_mean": 0.12389123439788818, "nonfinite": 0.0, "weighted_memory_to_active": 0.20595434308052063, "mix_std": 0.12995266914367676, "beta_std": 0.10056209564208984, "beta_low": 0.1906534880399704, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.06810424476861954, "output_influence": 0.2367628663778305, "residual_influence": 0.0532747320830822, "state_active": 0.013757435604929924, "state_memory": 0.026285476982593536, "state_peak": 31.36770248413086, "growth_active": 1.0795978307724, "growth_memory": 1.301565170288086, "heads": [{"head": 0, "memory_to_active": 2.346191167831421, "contribution_to_active": 0.2582232356071472, "mix_mean": 0.03794991225004196, "mix_low": 0.40647315979003906, "mix_high": 0.0, "read_cosine": 0.37977713346481323, "beta_mean": 0.14042875170707703, "nonfinite": 0.0, "weighted_memory_to_active": 0.2847878038883209, "mix_std": 0.0755854994058609, "beta_std": 0.10838049650192261, "beta_low": 0.01263427734375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.009680259972810745, "evidence_weights": [0.017605898901820183, -0.08220028877258301, 0.0868171900510788], "agreement_weight": -0.037438586354255676, "orthogonal_weight": 0.04691826552152634}, {"head": 1, "memory_to_active": 1.0122705698013306, "contribution_to_active": 0.3525712490081787, "mix_mean": 0.19229939579963684, "mix_low": 0.14619159698486328, "mix_high": 0.0009603500366210938, "read_cosine": 0.34370720386505127, "beta_mean": 0.10288010537624359, "nonfinite": 0.0, "weighted_memory_to_active": 0.31292176246643066, "mix_std": 0.22235789895057678, "beta_std": 0.0661846250295639, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.016163982450962067, "evidence_weights": [-0.19552387297153473, -0.11358468979597092, -0.1980348378419876], "agreement_weight": 0.022200003266334534, "orthogonal_weight": 0.06071845442056656}, {"head": 2, "memory_to_active": 0.8493674993515015, "contribution_to_active": 0.29200083017349243, "mix_mean": 0.1268189400434494, "mix_low": 0.0890665054321289, "mix_high": 0.0001583099365234375, "read_cosine": 0.1022629514336586, "beta_mean": 0.06360927224159241, "nonfinite": 0.0, "weighted_memory_to_active": 0.22008323669433594, "mix_std": 0.16287581622600555, "beta_std": 0.05083382874727249, "beta_low": 0.05877685546875, "beta_high": 0.0, "orthogonal_to_active": 0.07489880174398422, "evidence_weights": [-0.050884850323200226, -0.1377730518579483, 0.4800631105899811], "agreement_weight": 0.023714860901236534, "orthogonal_weight": -0.348595529794693}, {"head": 3, "memory_to_active": 4.305296897888184, "contribution_to_active": 0.09273020178079605, "mix_mean": 0.03758528083562851, "mix_low": 0.3010578155517578, "mix_high": 0.0, "read_cosine": 0.09938955307006836, "beta_mean": 0.296794593334198, "nonfinite": 0.0, "weighted_memory_to_active": 0.3274995684623718, "mix_std": 0.05354766547679901, "beta_std": 0.24508529901504517, "beta_low": 0.01153564453125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.27821487188339233, "evidence_weights": [0.061708200722932816, -0.013072967529296875, 0.6730529069900513], "agreement_weight": 0.055092450231313705, "orthogonal_weight": -0.9469476342201233}, {"head": 4, "memory_to_active": 1.8404626846313477, "contribution_to_active": 0.062104254961013794, "mix_mean": 0.013984667137265205, "mix_low": 0.6868047714233398, "mix_high": 0.0, "read_cosine": 0.18005917966365814, "beta_mean": 0.13875839114189148, "nonfinite": 0.0, "weighted_memory_to_active": 0.0514143705368042, "mix_std": 0.033694732934236526, "beta_std": 0.1305789351463318, "beta_low": 0.0672607421875, "beta_high": 0.0, "orthogonal_to_active": 0.007424675393849611, "evidence_weights": [0.14514599740505219, 0.05808805674314499, 0.2196900099515915], "agreement_weight": 0.042121533304452896, "orthogonal_weight": -0.1634596288204193}, {"head": 5, "memory_to_active": 0.12123902887105942, "contribution_to_active": 0.2845383882522583, "mix_mean": 0.26229244470596313, "mix_low": 0.016556739807128906, "mix_high": 4.100799560546875e-05, "read_cosine": 0.2580806016921997, "beta_mean": 0.0008763234363868833, "nonfinite": 0.0, "weighted_memory_to_active": 0.03901930898427963, "mix_std": 0.23165442049503326, "beta_std": 0.0023093533236533403, "beta_low": 0.99365234375, "beta_high": 0.0, "orthogonal_to_active": 0.022242851555347443, "evidence_weights": [-0.10085496306419373, -0.15508118271827698, 0.4427427649497986], "agreement_weight": 0.09526224434375763, "orthogonal_weight": -0.6507994532585144}]}, {"layer": 8, "memory_to_active": 0.44744277000427246, "contribution_to_active": 0.520423173904419, "mix_mean": 0.4204300045967102, "mix_low": 0.09375779330730438, "mix_high": 0.0009314219933003187, "read_cosine": 0.47943001985549927, "beta_mean": 0.030305832624435425, "nonfinite": 0.0, "weighted_memory_to_active": 0.09301093220710754, "mix_std": 0.21800397336483002, "beta_std": 0.03724838048219681, "beta_low": 0.6078695058822632, "beta_high": 0.0, "orthogonal_to_active": 0.059444621205329895, "output_influence": 0.44809409976005554, "residual_influence": 0.13961684703826904, "state_active": 0.04721931740641594, "state_memory": 0.009708449244499207, "state_peak": 31.352035522460938, "growth_active": 1.076634168624878, "growth_memory": 1.309097409248352, "heads": [{"head": 0, "memory_to_active": 0.05505870282649994, "contribution_to_active": 0.5504615306854248, "mix_mean": 0.5968651175498962, "mix_low": 0.00028896331787109375, "mix_high": 0.001071929931640625, "read_cosine": 0.549416184425354, "beta_mean": 0.0034167927224189043, "nonfinite": 0.0, "weighted_memory_to_active": 0.03797540068626404, "mix_std": 0.2410365343093872, "beta_std": 0.009945282712578773, "beta_low": 0.9171142578125, "beta_high": 0.0, "orthogonal_to_active": 0.023511797189712524, "evidence_weights": [-0.17079830169677734, -0.39365842938423157, 0.7054033279418945], "agreement_weight": 0.12406634539365768, "orthogonal_weight": -0.8110897541046143}, {"head": 1, "memory_to_active": 0.15649093687534332, "contribution_to_active": 0.49614831805229187, "mix_mean": 0.4660714566707611, "mix_low": 0.0005064010620117188, "mix_high": 0.00035858154296875, "read_cosine": 0.3419730067253113, "beta_mean": 0.004159563221037388, "nonfinite": 0.0, "weighted_memory_to_active": 0.07406534999608994, "mix_std": 0.24776968359947205, "beta_std": 0.01531269308179617, "beta_low": 0.9234619140625, "beta_high": 0.0, "orthogonal_to_active": 0.04671570658683777, "evidence_weights": [-0.19019977748394012, -0.2576996088027954, 0.6817171573638916], "agreement_weight": 0.08635558933019638, "orthogonal_weight": -0.7046000361442566}, {"head": 2, "memory_to_active": 0.02077803947031498, "contribution_to_active": 0.6447002291679382, "mix_mean": 0.6288003325462341, "mix_low": 0.0009851455688476562, "mix_high": 0.0027523040771484375, "read_cosine": 0.37770986557006836, "beta_mean": 0.00018656751490198076, "nonfinite": 0.0, "weighted_memory_to_active": 0.01524362526834011, "mix_std": 0.23197241127490997, "beta_std": 0.002587739611044526, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.010210610926151276, "evidence_weights": [-0.21999500691890717, -0.3386996388435364, 0.93515944480896], "agreement_weight": 0.21147499978542328, "orthogonal_weight": -0.7630569934844971}, {"head": 3, "memory_to_active": 0.1225900948047638, "contribution_to_active": 0.6353371739387512, "mix_mean": 0.5528978705406189, "mix_low": 0.0027971267700195312, "mix_high": 0.00130462646484375, "read_cosine": 0.8055674433708191, "beta_mean": 0.01004956103861332, "nonfinite": 0.0, "weighted_memory_to_active": 0.09053706377744675, "mix_std": 0.2573629319667816, "beta_std": 0.007266221102327108, "beta_low": 0.58807373046875, "beta_high": 0.0, "orthogonal_to_active": 0.029020188376307487, "evidence_weights": [-0.17974546551704407, -0.30274057388305664, 0.6490486264228821], "agreement_weight": 0.1735302358865738, "orthogonal_weight": -0.6361348628997803}, {"head": 4, "memory_to_active": 2.0228869915008545, "contribution_to_active": 0.1808483898639679, "mix_mean": 0.028663987293839455, "mix_low": 0.4205904006958008, "mix_high": 0.0, "read_cosine": 0.3392215669155121, "beta_mean": 0.08642959594726562, "nonfinite": 0.0, "weighted_memory_to_active": 0.1739853024482727, "mix_std": 0.04859832674264908, "beta_std": 0.08579472452402115, "beta_low": 0.04815673828125, "beta_high": 0.0, "orthogonal_to_active": 0.002861542860046029, "evidence_weights": [0.05629529058933258, -0.025569606572389603, -0.0822572186589241], "agreement_weight": 0.059655189514160156, "orthogonal_weight": 0.0173597764223814}, {"head": 5, "memory_to_active": 0.3068518042564392, "contribution_to_active": 0.615043580532074, "mix_mean": 0.24928125739097595, "mix_low": 0.13737869262695312, "mix_high": 0.0001010894775390625, "read_cosine": 0.4626920819282532, "beta_mean": 0.07759290933609009, "nonfinite": 0.0, "weighted_memory_to_active": 0.1662588268518448, "mix_std": 0.2812839150428772, "beta_std": 0.1025836169719696, "beta_low": 0.1705322265625, "beta_high": 0.0, "orthogonal_to_active": 0.24434784054756165, "evidence_weights": [-0.06840487569570541, -0.24916459619998932, -0.3236261010169983], "agreement_weight": -0.006606514565646648, "orthogonal_weight": 1.7154598236083984}]}, {"layer": 9, "memory_to_active": 1.1317315101623535, "contribution_to_active": 0.6577227711677551, "mix_mean": 0.32033827900886536, "mix_low": 0.15085665881633759, "mix_high": 0.004065036773681641, "read_cosine": 0.36516159772872925, "beta_mean": 0.06387130916118622, "nonfinite": 0.0, "weighted_memory_to_active": 0.39618319272994995, "mix_std": 0.21232719719409943, "beta_std": 0.04265594109892845, "beta_low": 0.5010172724723816, "beta_high": 0.0, "orthogonal_to_active": 0.04261142015457153, "output_influence": 0.49850329756736755, "residual_influence": 0.13687671720981598, "state_active": 0.04038914293050766, "state_memory": 0.019734736531972885, "state_peak": 46.88402557373047, "growth_active": 1.176074504852295, "growth_memory": 1.369876503944397, "heads": [{"head": 0, "memory_to_active": 0.003140623215585947, "contribution_to_active": 0.549752950668335, "mix_mean": 0.6157467365264893, "mix_low": 0.0016031265258789062, "mix_high": 0.017961502075195312, "read_cosine": 0.22365465760231018, "beta_mean": 6.097309778851923e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.001964099472388625, "mix_std": 0.2919325530529022, "beta_std": 2.904892244259827e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017624527681618929, "evidence_weights": [-0.29569828510284424, -0.3063872754573822, 0.9225870370864868], "agreement_weight": 0.09781572967767715, "orthogonal_weight": -0.9412341713905334}, {"head": 1, "memory_to_active": 0.022330431267619133, "contribution_to_active": 0.629040002822876, "mix_mean": 0.5047669410705566, "mix_low": 0.008615493774414062, "mix_high": 0.0012693405151367188, "read_cosine": 0.40295353531837463, "beta_mean": 6.147751992102712e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014834035187959671, "mix_std": 0.28835567831993103, "beta_std": 0.00013044624938629568, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010376867838203907, "evidence_weights": [-0.22195614874362946, -0.27119559049606323, 1.129798173904419], "agreement_weight": 0.354446679353714, "orthogonal_weight": -0.8388988971710205}, {"head": 2, "memory_to_active": 0.20943060517311096, "contribution_to_active": 0.6228476762771606, "mix_mean": 0.5565381646156311, "mix_low": 0.0027875900268554688, "mix_high": 0.004590034484863281, "read_cosine": 0.3506598174571991, "beta_mean": 0.000818758097011596, "nonfinite": 0.0, "weighted_memory_to_active": 0.1380029022693634, "mix_std": 0.2777208089828491, "beta_std": 0.004635191988199949, "beta_low": 0.993408203125, "beta_high": 0.0, "orthogonal_to_active": 0.07131235301494598, "evidence_weights": [-0.1915549635887146, -0.2594924569129944, 0.6897081732749939], "agreement_weight": 0.2683359980583191, "orthogonal_weight": -0.6096915006637573}, {"head": 3, "memory_to_active": 1.5630912780761719, "contribution_to_active": 0.16762173175811768, "mix_mean": 0.039444416761398315, "mix_low": 0.41753101348876953, "mix_high": 5.91278076171875e-05, "read_cosine": 0.4020211100578308, "beta_mean": 0.09254388511180878, "nonfinite": 0.0, "weighted_memory_to_active": 0.13826556503772736, "mix_std": 0.09099198877811432, "beta_std": 0.061490289866924286, "beta_low": 0.002685546875, "beta_high": 0.0, "orthogonal_to_active": 0.019278082996606827, "evidence_weights": [0.08465050905942917, 0.03243396431207657, -0.0394321046769619], "agreement_weight": 0.08535096049308777, "orthogonal_weight": 0.14784228801727295}, {"head": 4, "memory_to_active": 3.3850033283233643, "contribution_to_active": 1.369940996170044, "mix_mean": 0.12936142086982727, "mix_low": 0.1544198989868164, "mix_high": 0.00045680999755859375, "read_cosine": 0.3097902834415436, "beta_mean": 0.16826298832893372, "nonfinite": 0.0, "weighted_memory_to_active": 1.3991644382476807, "mix_std": 0.19038069248199463, "beta_std": 0.09277433156967163, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03325103223323822, "evidence_weights": [-0.06271137297153473, -0.1463005393743515, -0.004814410116523504], "agreement_weight": -0.05612262710928917, "orthogonal_weight": -0.0252598375082016}, {"head": 5, "memory_to_active": 1.6073929071426392, "contribution_to_active": 0.6071330904960632, "mix_mean": 0.07617191970348358, "mix_low": 0.32018280029296875, "mix_high": 5.340576171875e-05, "read_cosine": 0.5018900036811829, "beta_mean": 0.12153462320566177, "nonfinite": 0.0, "weighted_memory_to_active": 0.6848680973052979, "mix_std": 0.13458135724067688, "beta_std": 0.09687632322311401, "beta_low": 0.010009765625, "beta_high": 0.0, "orthogonal_to_active": 0.11968772113323212, "evidence_weights": [0.10350000858306885, -0.07382481545209885, 0.14895960688591003], "agreement_weight": 0.02232741378247738, "orthogonal_weight": 0.26183217763900757}]}, {"layer": 10, "memory_to_active": 0.2467687427997589, "contribution_to_active": 0.6399502754211426, "mix_mean": 0.5019424557685852, "mix_low": 0.014736334793269634, "mix_high": 0.007232984062284231, "read_cosine": 0.36681488156318665, "beta_mean": 0.04829467087984085, "nonfinite": 0.0, "weighted_memory_to_active": 0.12014225125312805, "mix_std": 0.27061063051223755, "beta_std": 0.048931024968624115, "beta_low": 0.6569010615348816, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.16349247097969055, "output_influence": 0.4269680380821228, "residual_influence": 0.1446799337863922, "state_active": 0.048594120889902115, "state_memory": 0.015489598736166954, "state_peak": 114.09674835205078, "growth_active": 1.032371997833252, "growth_memory": 0.9230367541313171, "heads": [{"head": 0, "memory_to_active": 0.910364031791687, "contribution_to_active": 1.6267189979553223, "mix_mean": 0.6065590977668762, "mix_low": 0.05099296569824219, "mix_high": 0.027383804321289062, "read_cosine": 0.10772916674613953, "beta_mean": 0.21082955598831177, "nonfinite": 0.0, "weighted_memory_to_active": 0.5853264331817627, "mix_std": 0.38718146085739136, "beta_std": 0.22194476425647736, "beta_low": 0.12109375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.8896570205688477, "evidence_weights": [-0.12712393701076508, -0.38705870509147644, -1.133139967918396], "agreement_weight": 0.03891107067465782, "orthogonal_weight": 1.5424238443374634}, {"head": 1, "memory_to_active": 0.0125355776399374, "contribution_to_active": 0.5990907549858093, "mix_mean": 0.6017505526542664, "mix_low": 0.0007028579711914062, "mix_high": 0.010112762451171875, "read_cosine": 0.25421571731567383, "beta_mean": 7.715290848864242e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007176428101956844, "mix_std": 0.26646023988723755, "beta_std": 0.00011021078535122797, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010148022323846817, "evidence_weights": [-0.26775258779525757, -0.20286689698696136, 0.6194108128547668], "agreement_weight": 0.4590061604976654, "orthogonal_weight": -1.5274423360824585}, {"head": 2, "memory_to_active": 0.022824294865131378, "contribution_to_active": 0.5688616037368774, "mix_mean": 0.5959345698356628, "mix_low": 0.00036525726318359375, "mix_high": 0.0052204132080078125, "read_cosine": 0.03279989957809448, "beta_mean": 6.424701132345945e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008599825203418732, "mix_std": 0.2586139440536499, "beta_std": 0.00026671934756450355, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0038542496040463448, "evidence_weights": [-0.23457807302474976, -0.2714342772960663, 0.5468243956565857], "agreement_weight": -0.14411784708499908, "orthogonal_weight": -0.42597252130508423}, {"head": 3, "memory_to_active": 0.38269147276878357, "contribution_to_active": 0.11717981100082397, "mix_mean": 0.2840955853462219, "mix_low": 0.03287220001220703, "mix_high": 0.00021266937255859375, "read_cosine": 0.9012892842292786, "beta_mean": 0.07199358195066452, "nonfinite": 0.0, "weighted_memory_to_active": 0.04660428315401077, "mix_std": 0.23029857873916626, "beta_std": 0.06300670653581619, "beta_low": 0.0010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.05463158339262009, "evidence_weights": [-0.11116569489240646, -0.14378152787685394, 0.13190674781799316], "agreement_weight": 0.005046727601438761, "orthogonal_weight": 1.487079381942749}, {"head": 4, "memory_to_active": 0.002968207001686096, "contribution_to_active": 0.5234223008155823, "mix_mean": 0.5369551181793213, "mix_low": 0.0006160736083984375, "mix_high": 0.00046253204345703125, "read_cosine": 0.04343708977103233, "beta_mean": 2.00301619770471e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0015964999329298735, "mix_std": 0.2520228922367096, "beta_std": 4.964321124134585e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011279599275439978, "evidence_weights": [-0.17966385185718536, -0.25809964537620544, 0.7648477554321289], "agreement_weight": 0.07789617031812668, "orthogonal_weight": -0.7352066040039062}, {"head": 5, "memory_to_active": 0.1492287963628769, "contribution_to_active": 0.4044281542301178, "mix_mean": 0.38635963201522827, "mix_low": 0.00286865234375, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8614178895950317, "beta_mean": 0.006783428136259317, "nonfinite": 0.0, "weighted_memory_to_active": 0.07155008614063263, "mix_std": 0.22908660769462585, "beta_std": 0.008208069019019604, "beta_low": 0.8193359375, "beta_high": 0.0, "orthogonal_to_active": 0.021535934880375862, "evidence_weights": [-0.1459798961877823, -0.1586591750383377, 0.2865813076496124], "agreement_weight": 0.08955978602170944, "orthogonal_weight": -0.6308590769767761}]}, {"layer": 11, "memory_to_active": 0.12954583764076233, "contribution_to_active": 0.5812920331954956, "mix_mean": 0.4788092374801636, "mix_low": 0.08638016879558563, "mix_high": 0.04461924359202385, "read_cosine": 0.4088960886001587, "beta_mean": 0.03173787146806717, "nonfinite": 0.0, "weighted_memory_to_active": 0.061254605650901794, "mix_std": 0.3217090964317322, "beta_std": 0.04710416868329048, "beta_low": 0.6067301630973816, "beta_high": 0.0, "orthogonal_to_active": 0.05074075236916542, "output_influence": 0.525126039981842, "residual_influence": 0.12849345803260803, "state_active": 0.18878288567066193, "state_memory": 0.006575867533683777, "state_peak": 26.72281265258789, "growth_active": 1.0792076587677002, "growth_memory": 1.6790083646774292, "heads": [{"head": 0, "memory_to_active": 0.009584159590303898, "contribution_to_active": 0.5773463249206543, "mix_mean": 0.6205062866210938, "mix_low": 0.0010061264038085938, "mix_high": 0.009035110473632812, "read_cosine": 0.3233528733253479, "beta_mean": 7.589287270093337e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006376510951668024, "mix_std": 0.2633185088634491, "beta_std": 0.003590512555092573, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004586505703628063, "evidence_weights": [-0.16860918700695038, -0.36370253562927246, 0.6927567720413208], "agreement_weight": 0.2703269422054291, "orthogonal_weight": -0.7649455666542053}, {"head": 1, "memory_to_active": 0.3299810290336609, "contribution_to_active": 0.3134521543979645, "mix_mean": 0.3624485433101654, "mix_low": 0.008387565612792969, "mix_high": 0.00018596649169921875, "read_cosine": 0.5767650008201599, "beta_mean": 0.028210921213030815, "nonfinite": 0.0, "weighted_memory_to_active": 0.14277499914169312, "mix_std": 0.2583950161933899, "beta_std": 0.05870712548494339, "beta_low": 0.4954833984375, "beta_high": 0.0, "orthogonal_to_active": 0.045278992503881454, "evidence_weights": [-0.163882777094841, -0.17371754348278046, 0.44276896119117737], "agreement_weight": 0.07714476436376572, "orthogonal_weight": -0.40143635869026184}, {"head": 2, "memory_to_active": 0.011319257318973541, "contribution_to_active": 0.9801385402679443, "mix_mean": 0.4506545960903168, "mix_low": 0.21610164642333984, "mix_high": 0.1638946533203125, "read_cosine": 0.19914118945598602, "beta_mean": 0.0773809477686882, "nonfinite": 0.0, "weighted_memory_to_active": 0.008625191636383533, "mix_std": 0.43906164169311523, "beta_std": 0.10138128697872162, "beta_low": 0.44659423828125, "beta_high": 0.0, "orthogonal_to_active": 0.05156867206096649, "evidence_weights": [0.03686419129371643, -0.44289785623550415, -1.2632118463516235], "agreement_weight": 0.0082608163356781, "orthogonal_weight": 6.189324855804443}, {"head": 3, "memory_to_active": 0.01093260943889618, "contribution_to_active": 0.8466016054153442, "mix_mean": 0.4129979610443115, "mix_low": 0.2810029983520508, "mix_high": 0.09103775024414062, "read_cosine": 0.5379812121391296, "beta_mean": 0.06074943393468857, "nonfinite": 0.0, "weighted_memory_to_active": 0.00687654921784997, "mix_std": 0.436555415391922, "beta_std": 0.09029867500066757, "beta_low": 0.4959716796875, "beta_high": 0.0, "orthogonal_to_active": 0.031449127942323685, "evidence_weights": [0.13503484427928925, -0.32956042885780334, -0.308393657207489], "agreement_weight": 0.045603230595588684, "orthogonal_weight": 6.518043041229248}, {"head": 4, "memory_to_active": 0.3177155554294586, "contribution_to_active": 0.4118587374687195, "mix_mean": 0.5035759210586548, "mix_low": 0.010569572448730469, "mix_high": 0.0005502700805664062, "read_cosine": 0.07976410537958145, "beta_mean": 0.01631443202495575, "nonfinite": 0.0, "weighted_memory_to_active": 0.15722474455833435, "mix_std": 0.2726869583129883, "beta_std": 0.016785215586423874, "beta_low": 0.43304443359375, "beta_high": 0.0, "orthogonal_to_active": 0.1478620320558548, "evidence_weights": [-0.22824712097644806, -0.2065502107143402, 1.8730701208114624], "agreement_weight": -0.1917927861213684, "orthogonal_weight": -0.9530560970306396}, {"head": 5, "memory_to_active": 0.09774239361286163, "contribution_to_active": 0.35835444927215576, "mix_mean": 0.522672176361084, "mix_low": 0.00121307373046875, "mix_high": 0.0030117034912109375, "read_cosine": 0.7363720536231995, "beta_mean": 0.007695602253079414, "nonfinite": 0.0, "weighted_memory_to_active": 0.045649655163288116, "mix_std": 0.2602368891239166, "beta_std": 0.011862186715006828, "beta_low": 0.76934814453125, "beta_high": 0.0, "orthogonal_to_active": 0.023699162527918816, "evidence_weights": [-0.21663233637809753, -0.290365070104599, 0.5841741561889648], "agreement_weight": 0.08040324598550797, "orthogonal_weight": -0.7205937504768372}]}, {"layer": 12, "memory_to_active": 0.2742454707622528, "contribution_to_active": 0.563567578792572, "mix_mean": 0.5023707151412964, "mix_low": 0.010040760040283203, "mix_high": 0.0040186247788369656, "read_cosine": 0.2504428029060364, "beta_mean": 0.015778210014104843, "nonfinite": 0.0, "weighted_memory_to_active": 0.1483188271522522, "mix_std": 0.2596897482872009, "beta_std": 0.015206294134259224, "beta_low": 0.822845458984375, "beta_high": 0.0, "orthogonal_to_active": 0.07203223556280136, "output_influence": 0.45650923252105713, "residual_influence": 0.126231849193573, "state_active": 0.03552698716521263, "state_memory": 0.009053422138094902, "state_peak": 47.18743896484375, "growth_active": 1.0901868343353271, "growth_memory": 1.2679740190505981, "heads": [{"head": 0, "memory_to_active": 0.06625757366418839, "contribution_to_active": 0.5189573764801025, "mix_mean": 0.5074261426925659, "mix_low": 0.0011415481567382812, "mix_high": 0.0024089813232421875, "read_cosine": 0.2893514931201935, "beta_mean": 0.00104902358725667, "nonfinite": 0.0, "weighted_memory_to_active": 0.03337554633617401, "mix_std": 0.2701281011104584, "beta_std": 0.0046606725081801414, "beta_low": 0.991943359375, "beta_high": 0.0, "orthogonal_to_active": 0.025221403688192368, "evidence_weights": [-0.192217156291008, -0.26455122232437134, 0.2876111567020416], "agreement_weight": 0.10780167579650879, "orthogonal_weight": -0.8121144771575928}, {"head": 1, "memory_to_active": 0.09240008145570755, "contribution_to_active": 0.5167251229286194, "mix_mean": 0.5316487550735474, "mix_low": 0.0006809234619140625, "mix_high": 0.0010080337524414062, "read_cosine": 0.5230625867843628, "beta_mean": 0.0011662153992801905, "nonfinite": 0.0, "weighted_memory_to_active": 0.05052095651626587, "mix_std": 0.2630619406700134, "beta_std": 0.006271016784012318, "beta_low": 0.98297119140625, "beta_high": 0.0, "orthogonal_to_active": 0.03946337848901749, "evidence_weights": [-0.2076415866613388, -0.27485138177871704, 0.9534546136856079], "agreement_weight": 0.14054863154888153, "orthogonal_weight": -0.9209476113319397}, {"head": 2, "memory_to_active": 0.6204874515533447, "contribution_to_active": 0.4974514842033386, "mix_mean": 0.31696292757987976, "mix_low": 0.026178359985351562, "mix_high": 0.0001888275146484375, "read_cosine": 0.45969101786613464, "beta_mean": 0.090624138712883, "nonfinite": 0.0, "weighted_memory_to_active": 0.2563900053501129, "mix_std": 0.23901543021202087, "beta_std": 0.06358261406421661, "beta_low": 0.0008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.24542978405952454, "evidence_weights": [-0.23952627182006836, -0.04771188274025917, -0.08928637206554413], "agreement_weight": 0.020908521488308907, "orthogonal_weight": 1.182111144065857}, {"head": 3, "memory_to_active": 0.03704032301902771, "contribution_to_active": 0.7348464727401733, "mix_mean": 0.693175196647644, "mix_low": 0.000240325927734375, "mix_high": 0.013711929321289062, "read_cosine": 0.12771067023277283, "beta_mean": 0.00024131125246640295, "nonfinite": 0.0, "weighted_memory_to_active": 0.025654371827840805, "mix_std": 0.23509271442890167, "beta_std": 0.004520175978541374, "beta_low": 0.998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.024232618510723114, "evidence_weights": [-0.29089951515197754, -0.33801180124282837, 1.367310643196106], "agreement_weight": 0.36990487575531006, "orthogonal_weight": -0.9704267978668213}, {"head": 4, "memory_to_active": 0.7529172897338867, "contribution_to_active": 0.6307322978973389, "mix_mean": 0.3497887849807739, "mix_low": 0.025122642517089844, "mix_high": 0.0025072097778320312, "read_cosine": -0.11519628018140793, "beta_mean": 0.0011199420550838113, "nonfinite": 0.0, "weighted_memory_to_active": 0.473444402217865, "mix_std": 0.2734583616256714, "beta_std": 0.009561780840158463, "beta_low": 0.975830078125, "beta_high": 0.0, "orthogonal_to_active": 0.05818390101194382, "evidence_weights": [-0.11372710764408112, -0.21421869099140167, 0.03202996030449867], "agreement_weight": -0.0009023842867463827, "orthogonal_weight": -0.13004723191261292}, {"head": 5, "memory_to_active": 0.07637010514736176, "contribution_to_active": 0.48269256949424744, "mix_mean": 0.6152223348617554, "mix_low": 0.006880760192871094, "mix_high": 0.004286766052246094, "read_cosine": 0.2180372029542923, "beta_mean": 0.00046861154260113835, "nonfinite": 0.0, "weighted_memory_to_active": 0.050527699291706085, "mix_std": 0.27738186717033386, "beta_std": 0.0026415041647851467, "beta_low": 0.9871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.039662305265665054, "evidence_weights": [-0.24924592673778534, -0.30964407324790955, 1.2364568710327148], "agreement_weight": 0.31403136253356934, "orthogonal_weight": -0.8450238704681396}]}, {"layer": 13, "memory_to_active": 0.1374937891960144, "contribution_to_active": 0.517383337020874, "mix_mean": 0.5138773918151855, "mix_low": 0.006067912094295025, "mix_high": 0.007296244613826275, "read_cosine": 0.42205384373664856, "beta_mean": 0.01637999154627323, "nonfinite": 0.0, "weighted_memory_to_active": 0.06102859228849411, "mix_std": 0.2581360340118408, "beta_std": 0.018323348835110664, "beta_low": 0.7960612177848816, "beta_high": 0.0, "orthogonal_to_active": 0.06088295206427574, "output_influence": 0.48634225130081177, "residual_influence": 0.16261571645736694, "state_active": 0.10714954137802124, "state_memory": 0.03860878944396973, "state_peak": 94.03446960449219, "growth_active": 0.9563798308372498, "growth_memory": 1.0617715120315552, "heads": [{"head": 0, "memory_to_active": 0.029326755553483963, "contribution_to_active": 0.5620180368423462, "mix_mean": 0.5709795951843262, "mix_low": 0.0003681182861328125, "mix_high": 0.002025604248046875, "read_cosine": 0.4343828558921814, "beta_mean": 0.00010515653411857784, "nonfinite": 0.0, "weighted_memory_to_active": 0.018064996227622032, "mix_std": 0.26770836114883423, "beta_std": 0.00022669939789921045, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01780252903699875, "evidence_weights": [-0.20219217240810394, -0.26532164216041565, 0.7535935640335083], "agreement_weight": 0.11525727808475494, "orthogonal_weight": -1.1116060018539429}, {"head": 1, "memory_to_active": 0.018260058015584946, "contribution_to_active": 0.578788697719574, "mix_mean": 0.5329117178916931, "mix_low": 0.0023241043090820312, "mix_high": 0.0020503997802734375, "read_cosine": 0.2275635004043579, "beta_mean": 5.8048412029165775e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009993043728172779, "mix_std": 0.2702769637107849, "beta_std": 0.00011713580170180649, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01185954175889492, "evidence_weights": [-0.18686829507350922, -0.2706744372844696, 0.7017050981521606], "agreement_weight": 0.33295202255249023, "orthogonal_weight": -1.238887906074524}, {"head": 2, "memory_to_active": 0.11345754563808441, "contribution_to_active": 0.4415585994720459, "mix_mean": 0.3689584732055664, "mix_low": 0.0025539398193359375, "mix_high": 0.00014209747314453125, "read_cosine": 0.6510263681411743, "beta_mean": 0.013842940330505371, "nonfinite": 0.0, "weighted_memory_to_active": 0.04910688102245331, "mix_std": 0.22618989646434784, "beta_std": 0.04314710944890976, "beta_low": 0.79388427734375, "beta_high": 0.0, "orthogonal_to_active": 0.022860221564769745, "evidence_weights": [-0.19995349645614624, -0.19092455506324768, 0.42816412448883057], "agreement_weight": 0.010029847733676434, "orthogonal_weight": -0.8091208934783936}, {"head": 3, "memory_to_active": 0.1873832494020462, "contribution_to_active": 0.6031968593597412, "mix_mean": 0.6537019610404968, "mix_low": 0.020177841186523438, "mix_high": 0.03168773651123047, "read_cosine": 0.055571846663951874, "beta_mean": 0.0011782534420490265, "nonfinite": 0.0, "weighted_memory_to_active": 0.14321696758270264, "mix_std": 0.28938573598861694, "beta_std": 0.009114330634474754, "beta_low": 0.986328125, "beta_high": 0.0, "orthogonal_to_active": 0.13505978882312775, "evidence_weights": [-0.33061477541923523, -0.24681690335273743, 1.8847153186798096], "agreement_weight": 4.09531831741333, "orthogonal_weight": -1.0392159223556519}, {"head": 4, "memory_to_active": 0.3896854817867279, "contribution_to_active": 0.2410106211900711, "mix_mean": 0.3344111442565918, "mix_low": 0.008427619934082031, "mix_high": 6.29425048828125e-05, "read_cosine": 0.871544599533081, "beta_mean": 0.08255346119403839, "nonfinite": 0.0, "weighted_memory_to_active": 0.09352432191371918, "mix_std": 0.21871039271354675, "beta_std": 0.05306576192378998, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.13383135199546814, "evidence_weights": [-0.1235773041844368, -0.23039329051971436, 0.4796238839626312], "agreement_weight": -0.0018973456462845206, "orthogonal_weight": 2.1293344497680664}, {"head": 5, "memory_to_active": 0.08684967458248138, "contribution_to_active": 0.6777269840240479, "mix_mean": 0.6223012804985046, "mix_low": 0.00255584716796875, "mix_high": 0.007808685302734375, "read_cosine": 0.2922336757183075, "beta_mean": 0.0005420910310931504, "nonfinite": 0.0, "weighted_memory_to_active": 0.05226535350084305, "mix_std": 0.27654460072517395, "beta_std": 0.004269050434231758, "beta_low": 0.99609375, "beta_high": 0.0, "orthogonal_to_active": 0.043884266167879105, "evidence_weights": [-0.2218313068151474, -0.3010312020778656, 0.7998451590538025], "agreement_weight": 0.11799155920743942, "orthogonal_weight": -0.9134421348571777}]}, {"layer": 14, "memory_to_active": 0.24786105751991272, "contribution_to_active": 0.42270416021347046, "mix_mean": 0.41820570826530457, "mix_low": 0.024358591064810753, "mix_high": 0.005268414970487356, "read_cosine": 0.6069716811180115, "beta_mean": 0.06848236918449402, "nonfinite": 0.0, "weighted_memory_to_active": 0.10810138285160065, "mix_std": 0.2455243319272995, "beta_std": 0.06365747004747391, "beta_low": 0.4612630307674408, "beta_high": 0.0, "orthogonal_to_active": 0.04253864288330078, "output_influence": 0.3262330889701843, "residual_influence": 0.12569105625152588, "state_active": 0.059721626341342926, "state_memory": 0.01642339676618576, "state_peak": 35.15165710449219, "growth_active": 1.1027013063430786, "growth_memory": 1.3304469585418701, "heads": [{"head": 0, "memory_to_active": 0.37069088220596313, "contribution_to_active": 0.1980654001235962, "mix_mean": 0.19753633439540863, "mix_low": 0.019356727600097656, "mix_high": 5.7220458984375e-06, "read_cosine": 0.781205415725708, "beta_mean": 0.07462388277053833, "nonfinite": 0.0, "weighted_memory_to_active": 0.09077230095863342, "mix_std": 0.18164199590682983, "beta_std": 0.06459342688322067, "beta_low": 0.00604248046875, "beta_high": 0.0, "orthogonal_to_active": 0.07126999646425247, "evidence_weights": [-0.13384725153446198, -0.036670319736003876, 0.508818507194519], "agreement_weight": 0.02677699364721775, "orthogonal_weight": 1.2932052612304688}, {"head": 1, "memory_to_active": 0.49255579710006714, "contribution_to_active": 0.3315257132053375, "mix_mean": 0.23904263973236084, "mix_low": 0.10991573333740234, "mix_high": 1.33514404296875e-05, "read_cosine": 0.8088971972465515, "beta_mean": 0.27325284481048584, "nonfinite": 0.0, "weighted_memory_to_active": 0.2247871458530426, "mix_std": 0.26229920983314514, "beta_std": 0.26050856709480286, "beta_low": 0.10382080078125, "beta_high": 0.0, "orthogonal_to_active": 0.014152726158499718, "evidence_weights": [-0.01010554563254118, -0.2322000414133072, -0.1539473533630371], "agreement_weight": 0.05408777669072151, "orthogonal_weight": 0.1451469212770462}, {"head": 2, "memory_to_active": 0.1500898003578186, "contribution_to_active": 0.5593775510787964, "mix_mean": 0.5663143396377563, "mix_low": 0.0027141571044921875, "mix_high": 0.0026845932006835938, "read_cosine": 0.39805465936660767, "beta_mean": 0.003165427828207612, "nonfinite": 0.0, "weighted_memory_to_active": 0.09590521454811096, "mix_std": 0.26746946573257446, "beta_std": 0.004240643233060837, "beta_low": 0.94500732421875, "beta_high": 0.0, "orthogonal_to_active": 0.07373262941837311, "evidence_weights": [-0.15289027988910675, -0.30921804904937744, 0.9045795798301697], "agreement_weight": 0.08494173735380173, "orthogonal_weight": -0.8690241575241089}, {"head": 3, "memory_to_active": 0.05865221470594406, "contribution_to_active": 0.636703372001648, "mix_mean": 0.6764537692070007, "mix_low": 0.005560874938964844, "mix_high": 0.01942920684814453, "read_cosine": 0.08263832330703735, "beta_mean": 0.00830872543156147, "nonfinite": 0.0, "weighted_memory_to_active": 0.03815118968486786, "mix_std": 0.2505154311656952, "beta_std": 0.009361657314002514, "beta_low": 0.72381591796875, "beta_high": 0.0, "orthogonal_to_active": 0.024996917694807053, "evidence_weights": [-0.36761876940727234, -0.32064518332481384, 0.39824560284614563], "agreement_weight": 0.07958517223596573, "orthogonal_weight": -0.7397702932357788}, {"head": 4, "memory_to_active": 0.15492472052574158, "contribution_to_active": 0.4793696403503418, "mix_mean": 0.4850350618362427, "mix_low": 0.004588127136230469, "mix_high": 0.008541107177734375, "read_cosine": 0.8503753542900085, "beta_mean": 0.04966212809085846, "nonfinite": 0.0, "weighted_memory_to_active": 0.08842737972736359, "mix_std": 0.26270565390586853, "beta_std": 0.03394998982548714, "beta_low": 0.02154541015625, "beta_high": 0.0, "orthogonal_to_active": 0.025381864979863167, "evidence_weights": [-0.21927863359451294, -0.2524462044239044, 0.4255361557006836], "agreement_weight": 0.12354924529790878, "orthogonal_weight": -0.5727368593215942}, {"head": 5, "memory_to_active": 0.26025283336639404, "contribution_to_active": 0.33118316531181335, "mix_mean": 0.3448520302772522, "mix_low": 0.004015922546386719, "mix_high": 0.0009365081787109375, "read_cosine": 0.7206590175628662, "beta_mean": 0.001881165662780404, "nonfinite": 0.0, "weighted_memory_to_active": 0.11056506633758545, "mix_std": 0.24851422011852264, "beta_std": 0.009290565736591816, "beta_low": 0.96734619140625, "beta_high": 0.0, "orthogonal_to_active": 0.04569773003458977, "evidence_weights": [-0.18094241619110107, -0.16202525794506073, 0.23738278448581696], "agreement_weight": 0.08482286334037781, "orthogonal_weight": -0.6655741930007935}]}, {"layer": 15, "memory_to_active": 0.06116835027933121, "contribution_to_active": 0.4900110065937042, "mix_mean": 0.4050425887107849, "mix_low": 0.009135882370173931, "mix_high": 0.002442995784804225, "read_cosine": 0.6867838501930237, "beta_mean": 0.037737101316452026, "nonfinite": 0.0, "weighted_memory_to_active": 0.0281035415828228, "mix_std": 0.285361111164093, "beta_std": 0.04393522068858147, "beta_low": 0.7085673213005066, "beta_high": 0.0, "orthogonal_to_active": 0.006266306154429913, "output_influence": 0.3066348433494568, "residual_influence": 0.16468071937561035, "state_active": 0.12275665253400803, "state_memory": 0.009995072148740292, "state_peak": 41.6431999206543, "growth_active": 1.2126712799072266, "growth_memory": 1.617929458618164, "heads": [{"head": 0, "memory_to_active": 0.10772022604942322, "contribution_to_active": 0.4715486764907837, "mix_mean": 0.36927592754364014, "mix_low": 0.007197380065917969, "mix_high": 0.004099845886230469, "read_cosine": 0.6807449460029602, "beta_mean": 0.04254277050495148, "nonfinite": 0.0, "weighted_memory_to_active": 0.04747752100229263, "mix_std": 0.2783788740634918, "beta_std": 0.03337763622403145, "beta_low": 0.10076904296875, "beta_high": 0.0, "orthogonal_to_active": 0.021528612822294235, "evidence_weights": [-0.16902408003807068, -0.17574840784072876, 0.3635324239730835], "agreement_weight": 0.09473862498998642, "orthogonal_weight": -0.6951543092727661}, {"head": 1, "memory_to_active": 0.13511742651462555, "contribution_to_active": 0.410669207572937, "mix_mean": 0.41505926847457886, "mix_low": 0.0022020339965820312, "mix_high": 0.004492759704589844, "read_cosine": 0.9019798040390015, "beta_mean": 0.0009983066702261567, "nonfinite": 0.0, "weighted_memory_to_active": 0.05383116006851196, "mix_std": 0.2828846871852875, "beta_std": 0.002685044426470995, "beta_low": 0.98504638671875, "beta_high": 0.0, "orthogonal_to_active": 0.00042679053149186075, "evidence_weights": [-0.1616310030221939, -0.1703345626592636, 0.2311459183692932], "agreement_weight": 0.11172114312648773, "orthogonal_weight": -0.013687524944543839}, {"head": 2, "memory_to_active": 0.006015982013195753, "contribution_to_active": 0.4889925420284271, "mix_mean": 0.3419952988624573, "mix_low": 0.017465591430664062, "mix_high": 6.008148193359375e-05, "read_cosine": 0.795232355594635, "beta_mean": 9.842446161201224e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029377189930528402, "mix_std": 0.27508407831192017, "beta_std": 0.0001966806303244084, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004829919198527932, "evidence_weights": [-0.20534884929656982, -0.27806487679481506, 0.21613194048404694], "agreement_weight": 0.011597002856433392, "orthogonal_weight": 0.38699930906295776}, {"head": 3, "memory_to_active": 0.0034568083938211203, "contribution_to_active": 0.41819125413894653, "mix_mean": 0.3769519329071045, "mix_low": 0.0063076019287109375, "mix_high": 0.00017452239990234375, "read_cosine": 0.6397000551223755, "beta_mean": 2.3308770323637873e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0012870189966633916, "mix_std": 0.28490474820137024, "beta_std": 5.291896377457306e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010963575914502144, "evidence_weights": [-0.23056448996067047, -0.25561708211898804, 0.3082740008831024], "agreement_weight": 0.03450363501906395, "orthogonal_weight": -0.996457576751709}, {"head": 4, "memory_to_active": 0.09136604517698288, "contribution_to_active": 0.618630588054657, "mix_mean": 0.37751317024230957, "mix_low": 0.011643409729003906, "mix_high": 0.0013704299926757812, "read_cosine": 0.7359641194343567, "beta_mean": 0.18118739128112793, "nonfinite": 0.0, "weighted_memory_to_active": 0.048086848109960556, "mix_std": 0.3102014362812042, "beta_std": 0.21635012328624725, "beta_low": 0.19281005859375, "beta_high": 0.0, "orthogonal_to_active": 0.00595520855858922, "evidence_weights": [-0.21726442873477936, -0.25848597288131714, 0.010336496867239475], "agreement_weight": 0.01251384150236845, "orthogonal_weight": 0.25787806510925293}, {"head": 5, "memory_to_active": 0.023333590477705002, "contribution_to_active": 0.5320335626602173, "mix_mean": 0.5494599342346191, "mix_low": 0.009999275207519531, "mix_high": 0.004460334777832031, "read_cosine": 0.3670816123485565, "beta_mean": 0.0015724089462310076, "nonfinite": 0.0, "weighted_memory_to_active": 0.015000964514911175, "mix_std": 0.2807128429412842, "beta_std": 0.01094892993569374, "beta_low": 0.9727783203125, "beta_high": 0.0, "orthogonal_to_active": 0.008107874542474747, "evidence_weights": [-0.3128049373626709, -0.31418365240097046, 0.7425457239151001], "agreement_weight": 0.25983694195747375, "orthogonal_weight": 0.7158665657043457}]}]}, {"step": 37000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8265105540012883, "memory_to_active": 0.8538752794265747, "contribution_to_active": 0.5307231545448303, "mix_mean": 0.38324981927871704, "mix_low": 0.14942872524261475, "mix_high": 0.009274881333112717, "read_cosine": 0.3670172691345215, "beta_mean": 0.07929258793592453, "nonfinite": 0.0, "weighted_memory_to_active": 0.20940996706485748, "mix_std": 0.21916496753692627, "beta_std": 0.06702502071857452, "beta_low": 0.5225849151611328, "beta_high": 2.352396768401377e-05, "orthogonal_to_active": 0.05566147342324257, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1261563301086426, "normal_token_accuracy": 0.4505005478858948, "state_active": 0.06156020611524582, "state_memory": 0.019051678478717804, "state_peak": 120.49251556396484, "growth_active": 1.1258761882781982, "growth_memory": 1.3523696660995483, "output_influence": 0.3912699222564697, "residual_influence": 0.12709908187389374, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2432117462158203, "no_memory_loss_delta": 0.408448725938797, "scrambled_loss_delta": 0.19642263650894165, "memory_only_loss_delta": 3.181915521621704, "no_evidence_loss_delta": 0.23806807398796082, "no_agreement_loss_delta": 0.03246232494711876, "no_orthogonal_loss_delta": 0.009603481739759445}, {"index": 94, "tokens": 256, "normal_loss": 3.232966661453247, "no_memory_loss_delta": 0.5206824541091919, "scrambled_loss_delta": 0.36692243814468384, "memory_only_loss_delta": 3.0262610912323, "no_evidence_loss_delta": 0.2580797076225281, "no_agreement_loss_delta": 0.029480665922164917, "no_orthogonal_loss_delta": 0.03875209391117096}, {"index": 188, "tokens": 256, "normal_loss": 3.4792916774749756, "no_memory_loss_delta": 0.5972918272018433, "scrambled_loss_delta": 0.16343288123607635, "memory_only_loss_delta": 3.1198678016662598, "no_evidence_loss_delta": 0.31857404112815857, "no_agreement_loss_delta": 0.04455246403813362, "no_orthogonal_loss_delta": 0.03311397135257721}, {"index": 283, "tokens": 131, "normal_loss": 1.9985827207565308, "no_memory_loss_delta": 0.27977845072746277, "scrambled_loss_delta": 0.23830096423625946, "memory_only_loss_delta": 2.8942928314208984, "no_evidence_loss_delta": 0.1531732678413391, "no_agreement_loss_delta": 0.0006214462337084115, "no_orthogonal_loss_delta": 0.003237182041630149}], "agreement_weight_abs_mean": 0.16750721633434296, "agreement_weight_abs_max": 4.189348220825195, "agreement_weight_mean": 0.1465805023908615, "orthogonal_weight_abs_mean": 0.7297077178955078, "orthogonal_weight_abs_max": 6.685608863830566, "orthogonal_weight_mean": -0.14288967847824097, "no_memory_loss": 3.601590156555176, "no_memory_loss_delta": 0.4754341244697571, "no_memory_token_accuracy_gain": 0.07341490685939789, "no_memory_token_win_fraction": 0.656284749507904, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 37.8384860462357, "scrambled_loss": 3.3678386211395264, "scrambled_loss_delta": 0.2416824996471405, "scrambled_token_accuracy_gain": 0.03448275849223137, "scrambled_token_win_fraction": 0.6295884251594543, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 21.469452726122256, "memory_only_loss": 6.20416784286499, "memory_only_loss_delta": 3.0780110359191895, "memory_only_token_accuracy_gain": 0.2714126706123352, "memory_only_token_win_fraction": 0.8932147026062012, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.39492410730932, "no_evidence_loss": 3.3804776668548584, "no_evidence_loss_delta": 0.25432088971138, "no_evidence_token_accuracy_gain": 0.045606229454278946, "no_evidence_token_win_fraction": 0.5984427332878113, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 22.45570695397012, "no_agreement_loss": 3.1565728187561035, "no_agreement_loss_delta": 0.03041629120707512, "no_agreement_token_accuracy_gain": 0.0022246940061450005, "no_agreement_token_win_fraction": 0.507230281829834, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.9958370317406455, "no_orthogonal_loss": 3.1498279571533203, "no_orthogonal_loss_delta": 0.023671049624681473, "no_orthogonal_token_accuracy_gain": 0.007786429487168789, "no_orthogonal_token_win_fraction": 0.5194661021232605, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 2.339308786452278, "short_no_memory_loss_delta": 0.4202413558959961, "long_context_memory_gain": 0.055192768573760986, "normal_context_loss_gain": 0.13460996747016907, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.345473289489746, "contribution_to_active": 1.0915645360946655, "mix_mean": 0.10288634151220322, "mix_low": 0.542681872844696, "mix_high": 0.004739920608699322, "read_cosine": 0.437636137008667, "beta_mean": 0.2516590356826782, "nonfinite": 0.0, "weighted_memory_to_active": 1.1413638591766357, "mix_std": 0.08862237632274628, "beta_std": 0.14678864181041718, "beta_low": 0.025115966796875, "beta_high": 0.0, "orthogonal_to_active": 0.06008844077587128, "output_influence": 0.18928290903568268, "residual_influence": 0.22453200817108154, "state_active": 0.06312070786952972, "state_memory": 0.04719657450914383, "state_peak": 43.61664581298828, "growth_active": 1.1858383417129517, "growth_memory": 1.2464998960494995, "heads": [{"head": 0, "memory_to_active": 2.439291477203369, "contribution_to_active": 0.4216127097606659, "mix_mean": 0.020066028460860252, "mix_low": 0.608942985534668, "mix_high": 0.0, "read_cosine": 0.23594607412815094, "beta_mean": 0.22045478224754333, "nonfinite": 0.0, "weighted_memory_to_active": 0.5096379518508911, "mix_std": 0.06085432320833206, "beta_std": 0.1010727807879448, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.039238907396793365, "evidence_weights": [-0.019037622958421707, -0.0667865052819252, -0.11752761155366898], "agreement_weight": -0.12884117662906647, "orthogonal_weight": -0.08517928421497345}, {"head": 1, "memory_to_active": 5.4534735679626465, "contribution_to_active": 2.1141886711120605, "mix_mean": 0.014876311644911766, "mix_low": 0.7093935012817383, "mix_high": 0.0, "read_cosine": 0.49973753094673157, "beta_mean": 0.47054189443588257, "nonfinite": 0.0, "weighted_memory_to_active": 2.442340612411499, "mix_std": 0.05079197138547897, "beta_std": 0.2038298398256302, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.09022781252861023, "evidence_weights": [0.03411544859409332, -0.09777858108282089, 0.41712531447410583], "agreement_weight": 0.028114018961787224, "orthogonal_weight": -0.07470668107271194}, {"head": 2, "memory_to_active": 2.5843276977539062, "contribution_to_active": 0.5505092144012451, "mix_mean": 0.013880346901714802, "mix_low": 0.6472721099853516, "mix_high": 0.0, "read_cosine": 0.30314674973487854, "beta_mean": 0.284242182970047, "nonfinite": 0.0, "weighted_memory_to_active": 0.6481252908706665, "mix_std": 0.03279050439596176, "beta_std": 0.20110198855400085, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06663914024829865, "evidence_weights": [0.05950573831796646, 0.0660572499036789, 0.04002079367637634], "agreement_weight": 0.003692169673740864, "orthogonal_weight": -0.2097730040550232}, {"head": 3, "memory_to_active": 7.076174736022949, "contribution_to_active": 2.9685683250427246, "mix_mean": 0.036948323249816895, "mix_low": 0.4647951126098633, "mix_high": 0.0, "read_cosine": 0.31335198879241943, "beta_mean": 0.36840611696243286, "nonfinite": 0.0, "weighted_memory_to_active": 3.136979341506958, "mix_std": 0.08408332616090775, "beta_std": 0.2343391329050064, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.13984838128089905, "evidence_weights": [-0.04910643398761749, -0.06540682166814804, -0.15285614132881165], "agreement_weight": -0.020163895562291145, "orthogonal_weight": -0.05226394161581993}, {"head": 4, "memory_to_active": 2.405148506164551, "contribution_to_active": 0.06614945828914642, "mix_mean": 0.006560785695910454, "mix_low": 0.8075971603393555, "mix_high": 0.0, "read_cosine": 0.33906692266464233, "beta_mean": 0.1339721977710724, "nonfinite": 0.0, "weighted_memory_to_active": 0.052887868136167526, "mix_std": 0.013340489938855171, "beta_std": 0.1149630919098854, "beta_low": 0.0006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.005581893026828766, "evidence_weights": [0.13041654229164124, 0.08202522248029709, -0.341670960187912], "agreement_weight": -0.11111186444759369, "orthogonal_weight": 0.10297451913356781}, {"head": 5, "memory_to_active": 0.11442282795906067, "contribution_to_active": 0.42835885286331177, "mix_mean": 0.5249862670898438, "mix_low": 0.018090248107910156, "mix_high": 0.02843952178955078, "read_cosine": 0.9345674514770508, "beta_mean": 0.032337166368961334, "nonfinite": 0.0, "weighted_memory_to_active": 0.058212585747241974, "mix_std": 0.2898736298084259, "beta_std": 0.025425001978874207, "beta_low": 0.15008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.018994484096765518, "evidence_weights": [-0.2204185426235199, -0.23387591540813446, 0.32195353507995605], "agreement_weight": 0.27333950996398926, "orthogonal_weight": -0.666327714920044}]}, {"layer": 1, "memory_to_active": 2.3140053749084473, "contribution_to_active": 0.49373000860214233, "mix_mean": 0.19029709696769714, "mix_low": 0.3661065399646759, "mix_high": 0.0009935697307810187, "read_cosine": 0.32633456587791443, "beta_mean": 0.1814504861831665, "nonfinite": 0.0, "weighted_memory_to_active": 0.3406173288822174, "mix_std": 0.11931278556585312, "beta_std": 0.128627210855484, "beta_low": 0.2860310971736908, "beta_high": 9.155274165095761e-05, "orthogonal_to_active": 0.045538343489170074, "output_influence": 0.19831687211990356, "residual_influence": 0.048489756882190704, "state_active": 0.038006167858839035, "state_memory": 0.03605090454220772, "state_peak": 34.35771942138672, "growth_active": 1.2816123962402344, "growth_memory": 1.524450421333313, "heads": [{"head": 0, "memory_to_active": 0.04560909420251846, "contribution_to_active": 0.4368273615837097, "mix_mean": 0.44524598121643066, "mix_low": 0.010178565979003906, "mix_high": 0.000820159912109375, "read_cosine": 0.048311591148376465, "beta_mean": 0.0003260766388848424, "nonfinite": 0.0, "weighted_memory_to_active": 0.023216918110847473, "mix_std": 0.28635793924331665, "beta_std": 0.0005754551966674626, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.02103276550769806, "evidence_weights": [-0.18778984248638153, -0.24432054162025452, 0.20066456496715546], "agreement_weight": 0.1665254384279251, "orthogonal_weight": -0.959945559501648}, {"head": 1, "memory_to_active": 2.9645206928253174, "contribution_to_active": 0.20680086314678192, "mix_mean": 0.02122741937637329, "mix_low": 0.48115062713623047, "mix_high": 9.5367431640625e-07, "read_cosine": 0.47847050428390503, "beta_mean": 0.25002768635749817, "nonfinite": 0.0, "weighted_memory_to_active": 0.2196529507637024, "mix_std": 0.04281311854720116, "beta_std": 0.17086568474769592, "beta_low": 0.00067138671875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.014986637979745865, "evidence_weights": [0.01752333529293537, -0.02625402808189392, 0.007806099485605955], "agreement_weight": -0.011249527335166931, "orthogonal_weight": 0.13873358070850372}, {"head": 2, "memory_to_active": 0.20255185663700104, "contribution_to_active": 0.6162917613983154, "mix_mean": 0.6102453470230103, "mix_low": 0.0013227462768554688, "mix_high": 0.005125999450683594, "read_cosine": 0.6881685256958008, "beta_mean": 0.012115910649299622, "nonfinite": 0.0, "weighted_memory_to_active": 0.16540245711803436, "mix_std": 0.2487063854932785, "beta_std": 0.01416870765388012, "beta_low": 0.64263916015625, "beta_high": 0.0, "orthogonal_to_active": 0.04403508082032204, "evidence_weights": [-0.30519723892211914, -0.3522597551345825, 0.45759546756744385], "agreement_weight": 0.027140425518155098, "orthogonal_weight": -0.44524645805358887}, {"head": 3, "memory_to_active": 2.905958652496338, "contribution_to_active": 0.1501186490058899, "mix_mean": 0.02075769752264023, "mix_low": 0.6083173751831055, "mix_high": 0.0, "read_cosine": 0.22711846232414246, "beta_mean": 0.3627643883228302, "nonfinite": 0.0, "weighted_memory_to_active": 0.22072213888168335, "mix_std": 0.05556807667016983, "beta_std": 0.25688114762306213, "beta_low": 0.03826904296875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.06550579518079758, "evidence_weights": [0.04058767110109329, -0.009294006042182446, 0.11536136269569397], "agreement_weight": -0.01874708943068981, "orthogonal_weight": -0.3733237683773041}, {"head": 4, "memory_to_active": 3.5985755920410156, "contribution_to_active": 1.485642910003662, "mix_mean": 0.03687556833028793, "mix_low": 0.27136707305908203, "mix_high": 1.430511474609375e-05, "read_cosine": 0.31999605894088745, "beta_mean": 0.16403105854988098, "nonfinite": 0.0, "weighted_memory_to_active": 1.3588941097259521, "mix_std": 0.06358630955219269, "beta_std": 0.11143572628498077, "beta_low": 0.00537109375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.11955676972866058, "evidence_weights": [-0.04186907783150673, 0.0014828123385086656, 0.02101045660674572], "agreement_weight": -0.01845371723175049, "orthogonal_weight": 0.11692483723163605}, {"head": 5, "memory_to_active": 4.166815757751465, "contribution_to_active": 0.06669816374778748, "mix_mean": 0.007430511526763439, "mix_low": 0.8243026733398438, "mix_high": 0.0, "read_cosine": 0.1959422528743744, "beta_mean": 0.29943788051605225, "nonfinite": 0.0, "weighted_memory_to_active": 0.05581531301140785, "mix_std": 0.0188448503613472, "beta_std": 0.21783654391765594, "beta_low": 0.02923583984375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.008113005198538303, "evidence_weights": [0.16362649202346802, 0.08206287771463394, 0.24770930409431458], "agreement_weight": -0.14829768240451813, "orthogonal_weight": 0.1473667174577713}]}, {"layer": 2, "memory_to_active": 1.0261727571487427, "contribution_to_active": 0.3459947407245636, "mix_mean": 0.37068426609039307, "mix_low": 0.2554323077201843, "mix_high": 0.004024188034236431, "read_cosine": 0.3314690589904785, "beta_mean": 0.0887666568160057, "nonfinite": 0.0, "weighted_memory_to_active": 0.06608270108699799, "mix_std": 0.1925504058599472, "beta_std": 0.06938637793064117, "beta_low": 0.5504049062728882, "beta_high": 0.00010172525799134746, "orthogonal_to_active": 0.01508861593902111, "output_influence": 0.3488619923591614, "residual_influence": 0.0905189961194992, "state_active": 0.0398818701505661, "state_memory": 0.01740715280175209, "state_peak": 34.4367790222168, "growth_active": 1.0839347839355469, "growth_memory": 1.5937587022781372, "heads": [{"head": 0, "memory_to_active": 0.008144700899720192, "contribution_to_active": 0.37325751781463623, "mix_mean": 0.5819998979568481, "mix_low": 0.001865386962890625, "mix_high": 0.007428169250488281, "read_cosine": 0.16212452948093414, "beta_mean": 6.315156497294083e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004820398520678282, "mix_std": 0.28634220361709595, "beta_std": 9.181936911772937e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029378144536167383, "evidence_weights": [-0.33072149753570557, -0.20726118981838226, 0.08065631240606308], "agreement_weight": 0.23752492666244507, "orthogonal_weight": -0.6325323581695557}, {"head": 1, "memory_to_active": 2.9117345809936523, "contribution_to_active": 0.04660571366548538, "mix_mean": 0.00515028927475214, "mix_low": 0.8929128646850586, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19177918136119843, "beta_mean": 0.26158493757247925, "nonfinite": 0.0, "weighted_memory_to_active": 0.03465820103883743, "mix_std": 0.011654654517769814, "beta_std": 0.15130406618118286, "beta_low": 0.0003662109375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.013894076459109783, "evidence_weights": [0.11145349591970444, 0.10408546775579453, 0.5050509572029114], "agreement_weight": 0.03561874106526375, "orthogonal_weight": 0.42518350481987}, {"head": 2, "memory_to_active": 0.012797203846275806, "contribution_to_active": 0.4188464283943176, "mix_mean": 0.5008921027183533, "mix_low": 0.001873016357421875, "mix_high": 0.0019121170043945312, "read_cosine": 0.3429622948169708, "beta_mean": 3.1306251912610605e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004892838187515736, "mix_std": 0.28045958280563354, "beta_std": 5.795891775051132e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008926909067668021, "evidence_weights": [-0.2378511130809784, -0.2757836878299713, 0.45826035737991333], "agreement_weight": 0.02548821270465851, "orthogonal_weight": -0.21531471610069275}, {"head": 3, "memory_to_active": 0.15795300900936127, "contribution_to_active": 0.49797528982162476, "mix_mean": 0.4980107545852661, "mix_low": 0.000560760498046875, "mix_high": 0.006893157958984375, "read_cosine": 0.6356291174888611, "beta_mean": 0.036940522491931915, "nonfinite": 0.0, "weighted_memory_to_active": 0.08727573603391647, "mix_std": 0.26824551820755005, "beta_std": 0.04648435860872269, "beta_low": 0.26251220703125, "beta_high": 0.0, "orthogonal_to_active": 0.056442782282829285, "evidence_weights": [-0.18284548819065094, -0.2858837842941284, 0.8615676760673523], "agreement_weight": 0.05947331339120865, "orthogonal_weight": -0.8885208964347839}, {"head": 4, "memory_to_active": 3.0521998405456543, "contribution_to_active": 0.2174031287431717, "mix_mean": 0.01567813940346241, "mix_low": 0.6340980529785156, "mix_high": 0.0, "read_cosine": 0.2066047042608261, "beta_mean": 0.23393432796001434, "nonfinite": 0.0, "weighted_memory_to_active": 0.2560352683067322, "mix_std": 0.04005686566233635, "beta_std": 0.21830908954143524, "beta_low": 0.03955078125, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.016003010794520378, "evidence_weights": [0.026658080518245697, 0.019164204597473145, 0.05035840719938278], "agreement_weight": 0.021135520190000534, "orthogonal_weight": -0.07121982425451279}, {"head": 5, "memory_to_active": 0.014207214117050171, "contribution_to_active": 0.5218802690505981, "mix_mean": 0.6223742961883545, "mix_low": 0.0012836456298828125, "mix_high": 0.007910728454589844, "read_cosine": 0.44971439242362976, "beta_mean": 4.5709075493505225e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008813759312033653, "mix_std": 0.268543541431427, "beta_std": 7.097417983459309e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003613232111092657, "evidence_weights": [-0.2700308859348297, -0.28769272565841675, 0.9251440167427063], "agreement_weight": 0.09906601905822754, "orthogonal_weight": 0.06283723562955856}]}, {"layer": 3, "memory_to_active": 0.6705678701400757, "contribution_to_active": 0.41179126501083374, "mix_mean": 0.3911009430885315, "mix_low": 0.10441287606954575, "mix_high": 0.008994261734187603, "read_cosine": 0.2155725508928299, "beta_mean": 0.04019229859113693, "nonfinite": 0.0, "weighted_memory_to_active": 0.155495285987854, "mix_std": 0.2509055435657501, "beta_std": 0.03485552966594696, "beta_low": 0.624603271484375, "beta_high": 0.0, "orthogonal_to_active": 0.0719556212425232, "output_influence": 0.34242504835128784, "residual_influence": 0.07076499611139297, "state_active": 0.022527970373630524, "state_memory": 0.009177442640066147, "state_peak": 19.03848648071289, "growth_active": 1.0353621244430542, "growth_memory": 1.8106107711791992, "heads": [{"head": 0, "memory_to_active": 0.4468863308429718, "contribution_to_active": 0.5357732772827148, "mix_mean": 0.5045990943908691, "mix_low": 0.011224746704101562, "mix_high": 0.01758861541748047, "read_cosine": 0.21621501445770264, "beta_mean": 0.013565641827881336, "nonfinite": 0.0, "weighted_memory_to_active": 0.25948718190193176, "mix_std": 0.32257357239723206, "beta_std": 0.02156246453523636, "beta_low": 0.60272216796875, "beta_high": 0.0, "orthogonal_to_active": 0.17736661434173584, "evidence_weights": [-0.1921844184398651, -0.22312311828136444, 0.7750032544136047], "agreement_weight": 0.11177133768796921, "orthogonal_weight": -0.7160468101501465}, {"head": 1, "memory_to_active": 0.014970116317272186, "contribution_to_active": 0.40043243765830994, "mix_mean": 0.4786989390850067, "mix_low": 0.011896133422851562, "mix_high": 0.0052623748779296875, "read_cosine": 0.1901862472295761, "beta_mean": 8.399271609960124e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007886427454650402, "mix_std": 0.3092629909515381, "beta_std": 0.00016878789756447077, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005395638290792704, "evidence_weights": [-0.22725096344947815, -0.19285383820533752, 0.6019163727760315], "agreement_weight": 0.381585955619812, "orthogonal_weight": -0.7378520965576172}, {"head": 2, "memory_to_active": 2.7139430046081543, "contribution_to_active": 0.28115975856781006, "mix_mean": 0.04247918725013733, "mix_low": 0.38675880432128906, "mix_high": 0.0, "read_cosine": 0.1961308866739273, "beta_mean": 0.19424936175346375, "nonfinite": 0.0, "weighted_memory_to_active": 0.33878830075263977, "mix_std": 0.09475494921207428, "beta_std": 0.14862321317195892, "beta_low": 0.010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.0018593044951558113, "evidence_weights": [-0.07337907701730728, 0.013874434866011143, -0.011632119305431843], "agreement_weight": -0.012073746882379055, "orthogonal_weight": -0.007168911397457123}, {"head": 3, "memory_to_active": 0.028791408985853195, "contribution_to_active": 0.5103678107261658, "mix_mean": 0.5577229261398315, "mix_low": 0.010486602783203125, "mix_high": 0.010569572448730469, "read_cosine": 0.32227927446365356, "beta_mean": 0.00011295326112303883, "nonfinite": 0.0, "weighted_memory_to_active": 0.01899934560060501, "mix_std": 0.3030973970890045, "beta_std": 0.00028512364951893687, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004642102401703596, "evidence_weights": [-0.2188136726617813, -0.2247679978609085, 2.02006196975708], "agreement_weight": 1.5646082162857056, "orthogonal_weight": -0.32101356983184814}, {"head": 4, "memory_to_active": 0.5116774439811707, "contribution_to_active": 0.11097738146781921, "mix_mean": 0.125579372048378, "mix_low": 0.19733142852783203, "mix_high": 6.103515625e-05, "read_cosine": 0.155470609664917, "beta_mean": 0.005874156020581722, "nonfinite": 0.0, "weighted_memory_to_active": 0.095314159989357, "mix_std": 0.1864049881696701, "beta_std": 0.010444610379636288, "beta_low": 0.8382568359375, "beta_high": 0.0, "orthogonal_to_active": 0.07869423925876617, "evidence_weights": [-0.007020761724561453, -0.058431606739759445, 0.23053881525993347], "agreement_weight": -0.004997272044420242, "orthogonal_weight": -0.8464183211326599}, {"head": 5, "memory_to_active": 0.3071385622024536, "contribution_to_active": 0.6320369243621826, "mix_mean": 0.6375260949134827, "mix_low": 0.008779525756835938, "mix_high": 0.020483970642089844, "read_cosine": 0.21315321326255798, "beta_mean": 0.027267690747976303, "nonfinite": 0.0, "weighted_memory_to_active": 0.2124963104724884, "mix_std": 0.2893393635749817, "beta_std": 0.028048953041434288, "beta_low": 0.295654296875, "beta_high": 0.0, "orthogonal_to_active": 0.16377580165863037, "evidence_weights": [-0.2427102029323578, -0.2803630530834198, 0.4993526041507721], "agreement_weight": 0.1277020275592804, "orthogonal_weight": -0.823777437210083}]}, {"layer": 4, "memory_to_active": 1.1283650398254395, "contribution_to_active": 0.4382767379283905, "mix_mean": 0.3522055149078369, "mix_low": 0.2916228175163269, "mix_high": 0.008090972900390625, "read_cosine": 0.36074644327163696, "beta_mean": 0.1259709894657135, "nonfinite": 0.0, "weighted_memory_to_active": 0.1452028453350067, "mix_std": 0.15628598630428314, "beta_std": 0.11183736473321915, "beta_low": 0.3714294731616974, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04256930947303772, "output_influence": 0.4161861538887024, "residual_influence": 0.11225974559783936, "state_active": 0.03694027289748192, "state_memory": 0.019310055300593376, "state_peak": 34.95930099487305, "growth_active": 1.1993675231933594, "growth_memory": 1.5140482187271118, "heads": [{"head": 0, "memory_to_active": 0.22987042367458344, "contribution_to_active": 0.711222767829895, "mix_mean": 0.6594324111938477, "mix_low": 0.012827873229980469, "mix_high": 0.018922805786132812, "read_cosine": 0.49895867705345154, "beta_mean": 0.2556721866130829, "nonfinite": 0.0, "weighted_memory_to_active": 0.1750333607196808, "mix_std": 0.34175658226013184, "beta_std": 0.2659134268760681, "beta_low": 0.2205810546875, "beta_high": 0.0, "orthogonal_to_active": 0.0623459666967392, "evidence_weights": [-0.27040836215019226, -0.36879202723503113, 0.1868545264005661], "agreement_weight": 0.028529316186904907, "orthogonal_weight": 0.43454909324645996}, {"head": 1, "memory_to_active": 0.0746016800403595, "contribution_to_active": 0.6153329014778137, "mix_mean": 0.6569077968597412, "mix_low": 8.106231689453125e-05, "mix_high": 0.010080337524414062, "read_cosine": 0.7230381965637207, "beta_mean": 0.0011949785985052586, "nonfinite": 0.0, "weighted_memory_to_active": 0.05251361057162285, "mix_std": 0.24814462661743164, "beta_std": 0.004749372601509094, "beta_low": 0.9781494140625, "beta_high": 0.0, "orthogonal_to_active": 0.028066249564290047, "evidence_weights": [-0.2382451742887497, -0.3306773602962494, 0.7206326127052307], "agreement_weight": 0.16520076990127563, "orthogonal_weight": -0.8323122262954712}, {"head": 2, "memory_to_active": 2.3324596881866455, "contribution_to_active": 0.22187042236328125, "mix_mean": 0.01759432628750801, "mix_low": 0.6174821853637695, "mix_high": 3.814697265625e-06, "read_cosine": 0.28543806076049805, "beta_mean": 0.1434313952922821, "nonfinite": 0.0, "weighted_memory_to_active": 0.25271955132484436, "mix_std": 0.03828674554824829, "beta_std": 0.10274805128574371, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.02272779867053032, "evidence_weights": [0.06389935314655304, 0.03685122728347778, 0.006557142827659845], "agreement_weight": 0.12213163822889328, "orthogonal_weight": -0.14136260747909546}, {"head": 3, "memory_to_active": 1.7950108051300049, "contribution_to_active": 0.12204774469137192, "mix_mean": 0.029410097748041153, "mix_low": 0.44440746307373047, "mix_high": 0.0, "read_cosine": 0.21477152407169342, "beta_mean": 0.1578761339187622, "nonfinite": 0.0, "weighted_memory_to_active": 0.19372043013572693, "mix_std": 0.0537877231836319, "beta_std": 0.13924241065979004, "beta_low": 0.02874755859375, "beta_high": 0.0, "orthogonal_to_active": 0.00949659664183855, "evidence_weights": [-0.01761060580611229, 0.06355782598257065, -0.23473450541496277], "agreement_weight": 0.07228396832942963, "orthogonal_weight": -0.07737884670495987}, {"head": 4, "memory_to_active": 0.019742369651794434, "contribution_to_active": 0.6531572937965393, "mix_mean": 0.7361429929733276, "mix_low": 0.00032329559326171875, "mix_high": 0.01953411102294922, "read_cosine": 0.17708319425582886, "beta_mean": 3.6290104617364705e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014649360440671444, "mix_std": 0.22153589129447937, "beta_std": 0.00011739577166736126, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0032748309895396233, "evidence_weights": [-0.32591310143470764, -0.33305448293685913, 0.6411712765693665], "agreement_weight": 0.25847145915031433, "orthogonal_weight": -0.24383880198001862}, {"head": 5, "memory_to_active": 2.31850528717041, "contribution_to_active": 0.3060292601585388, "mix_mean": 0.013745453208684921, "mix_low": 0.6746149063110352, "mix_high": 4.76837158203125e-06, "read_cosine": 0.2651888132095337, "beta_mean": 0.19761492311954498, "nonfinite": 0.0, "weighted_memory_to_active": 0.18258070945739746, "mix_std": 0.034204330295324326, "beta_std": 0.1582535058259964, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12950444221496582, "evidence_weights": [0.04800841212272644, -0.033318474888801575, -0.2861310839653015], "agreement_weight": -0.05898918956518173, "orthogonal_weight": 0.8654832243919373}]}, {"layer": 5, "memory_to_active": 0.5080212354660034, "contribution_to_active": 0.49609509110450745, "mix_mean": 0.4582524299621582, "mix_low": 0.06924676895141602, "mix_high": 0.015160560607910156, "read_cosine": 0.13617904484272003, "beta_mean": 0.05720466747879982, "nonfinite": 0.0, "weighted_memory_to_active": 0.14955314993858337, "mix_std": 0.25317075848579407, "beta_std": 0.07483583688735962, "beta_low": 0.6381429433822632, "beta_high": 0.0, "orthogonal_to_active": 0.054427243769168854, "output_influence": 0.5002051591873169, "residual_influence": 0.12854041159152985, "state_active": 0.04535498470067978, "state_memory": 0.011826533824205399, "state_peak": 64.74568939208984, "growth_active": 1.1161231994628906, "growth_memory": 1.2688350677490234, "heads": [{"head": 0, "memory_to_active": 0.7515897750854492, "contribution_to_active": 0.4272260069847107, "mix_mean": 0.2682513892650604, "mix_low": 0.04918098449707031, "mix_high": 0.0004177093505859375, "read_cosine": -0.07602760940790176, "beta_mean": 0.041860684752464294, "nonfinite": 0.0, "weighted_memory_to_active": 0.1718076467514038, "mix_std": 0.25094887614250183, "beta_std": 0.0725070983171463, "beta_low": 0.36834716796875, "beta_high": 0.0, "orthogonal_to_active": 0.04153948649764061, "evidence_weights": [-0.10632608085870743, -0.13814139366149902, -0.19540074467658997], "agreement_weight": -0.013936793431639671, "orthogonal_weight": -0.24037708342075348}, {"head": 1, "memory_to_active": 0.026945695281028748, "contribution_to_active": 0.5559004545211792, "mix_mean": 0.6571269035339355, "mix_low": 0.0020122528076171875, "mix_high": 0.02472400665283203, "read_cosine": 0.1628221869468689, "beta_mean": 5.925586447119713e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015146555379033089, "mix_std": 0.2878348231315613, "beta_std": 0.00011288592941127717, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014184209518134594, "evidence_weights": [-0.23960399627685547, -0.27777937054634094, 0.9117690324783325], "agreement_weight": 0.5473474860191345, "orthogonal_weight": -0.9672918319702148}, {"head": 2, "memory_to_active": 0.015265039168298244, "contribution_to_active": 0.5515196323394775, "mix_mean": 0.6263170838356018, "mix_low": 0.0032835006713867188, "mix_high": 0.01775836944580078, "read_cosine": -0.1053735613822937, "beta_mean": 0.000169665741850622, "nonfinite": 0.0, "weighted_memory_to_active": 0.00977612379938364, "mix_std": 0.2779310941696167, "beta_std": 0.0002876552753150463, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0021102591417729855, "evidence_weights": [-0.24798767268657684, -0.3423990309238434, 0.8374657034873962], "agreement_weight": 0.004260126501321793, "orthogonal_weight": -0.26097121834754944}, {"head": 3, "memory_to_active": 1.6845195293426514, "contribution_to_active": 0.2079741656780243, "mix_mean": 0.09195359796285629, "mix_low": 0.16074657440185547, "mix_high": 0.0, "read_cosine": 0.12461864948272705, "beta_mean": 0.0994691550731659, "nonfinite": 0.0, "weighted_memory_to_active": 0.3161104619503021, "mix_std": 0.1313026249408722, "beta_std": 0.14911673963069916, "beta_low": 0.2027587890625, "beta_high": 0.0, "orthogonal_to_active": 0.12381018698215485, "evidence_weights": [-0.06548146903514862, -0.09283480048179626, -0.034419573843479156], "agreement_weight": 0.047133881598711014, "orthogonal_weight": -0.4541615843772888}, {"head": 4, "memory_to_active": 0.44718122482299805, "contribution_to_active": 0.5849364399909973, "mix_mean": 0.3195297420024872, "mix_low": 0.19773387908935547, "mix_high": 0.00656890869140625, "read_cosine": 0.5548394918441772, "beta_mean": 0.20130684971809387, "nonfinite": 0.0, "weighted_memory_to_active": 0.30040663480758667, "mix_std": 0.33791840076446533, "beta_std": 0.22515708208084106, "beta_low": 0.26275634765625, "beta_high": 0.0, "orthogonal_to_active": 0.09156140685081482, "evidence_weights": [-0.11124025285243988, -0.27978581190109253, -0.06421209871768951], "agreement_weight": 0.04510579630732536, "orthogonal_weight": 0.48373350501060486}, {"head": 5, "memory_to_active": 0.12262635678052902, "contribution_to_active": 0.6490137577056885, "mix_mean": 0.786335825920105, "mix_low": 0.0025234222412109375, "mix_high": 0.04149436950683594, "read_cosine": 0.15619513392448425, "beta_mean": 0.00036240159533917904, "nonfinite": 0.0, "weighted_memory_to_active": 0.08407142758369446, "mix_std": 0.23308879137039185, "beta_std": 0.0018335115164518356, "beta_low": 0.9949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.05335792154073715, "evidence_weights": [-0.3255020081996918, -0.3277556598186493, 1.9955483675003052], "agreement_weight": 0.3983611464500427, "orthogonal_weight": -0.6733924150466919}]}, {"layer": 6, "memory_to_active": 0.34819817543029785, "contribution_to_active": 0.5954148173332214, "mix_mean": 0.5657004117965698, "mix_low": 0.07196378707885742, "mix_high": 0.0316116027534008, "read_cosine": 0.27283650636672974, "beta_mean": 0.07191271334886551, "nonfinite": 0.0, "weighted_memory_to_active": 0.1666385382413864, "mix_std": 0.2380845844745636, "beta_std": 0.0809752345085144, "beta_low": 0.4972330927848816, "beta_high": 0.0, "orthogonal_to_active": 0.042457398027181625, "output_influence": 0.5125676393508911, "residual_influence": 0.1649470031261444, "state_active": 0.0476074256002903, "state_memory": 0.008832547813653946, "state_peak": 84.35713195800781, "growth_active": 1.2232353687286377, "growth_memory": 1.1841927766799927, "heads": [{"head": 0, "memory_to_active": 0.169986292719841, "contribution_to_active": 0.6608999967575073, "mix_mean": 0.7555565237998962, "mix_low": 0.0001277923583984375, "mix_high": 0.016531944274902344, "read_cosine": 0.00672744307667017, "beta_mean": 0.01903432607650757, "nonfinite": 0.0, "weighted_memory_to_active": 0.13530726730823517, "mix_std": 0.22196468710899353, "beta_std": 0.024826480075716972, "beta_low": 0.4569091796875, "beta_high": 0.0, "orthogonal_to_active": 0.10753796994686127, "evidence_weights": [-0.31540748476982117, -0.336276650428772, 0.7433478832244873], "agreement_weight": 0.22808010876178741, "orthogonal_weight": -0.912356436252594}, {"head": 1, "memory_to_active": 0.006545691750943661, "contribution_to_active": 0.4970811605453491, "mix_mean": 0.5447373986244202, "mix_low": 0.00121307373046875, "mix_high": 0.009724617004394531, "read_cosine": 0.5411849021911621, "beta_mean": 1.3180882888264023e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029590718913823366, "mix_std": 0.2874563932418823, "beta_std": 2.153148307115771e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015226311516016722, "evidence_weights": [-0.20127250254154205, -0.2938482165336609, 1.0902575254440308], "agreement_weight": 0.059278711676597595, "orthogonal_weight": -0.711696982383728}, {"head": 2, "memory_to_active": 1.0112519264221191, "contribution_to_active": 0.07957812398672104, "mix_mean": 0.028441833332180977, "mix_low": 0.41962242126464844, "mix_high": 0.0, "read_cosine": 0.22595112025737762, "beta_mean": 0.049659229815006256, "nonfinite": 0.0, "weighted_memory_to_active": 0.06271432340145111, "mix_std": 0.050845228135585785, "beta_std": 0.05259401723742485, "beta_low": 0.063232421875, "beta_high": 0.0, "orthogonal_to_active": 0.0009618077892810106, "evidence_weights": [0.09063751995563507, -0.010752331465482712, 0.24424463510513306], "agreement_weight": 0.1412213146686554, "orthogonal_weight": 0.0193864144384861}, {"head": 3, "memory_to_active": 0.165053591132164, "contribution_to_active": 0.6664844155311584, "mix_mean": 0.6462194323539734, "mix_low": 0.009192466735839844, "mix_high": 0.021047592163085938, "read_cosine": 0.45503559708595276, "beta_mean": 0.22415050864219666, "nonfinite": 0.0, "weighted_memory_to_active": 0.11717721074819565, "mix_std": 0.32829010486602783, "beta_std": 0.2536162734031677, "beta_low": 0.21112060546875, "beta_high": 0.0, "orthogonal_to_active": 0.0656806230545044, "evidence_weights": [-0.17359818518161774, -0.3351205289363861, 0.19971558451652527], "agreement_weight": -0.01607886515557766, "orthogonal_weight": 0.6500734686851501}, {"head": 4, "memory_to_active": 0.6959462761878967, "contribution_to_active": 1.0599764585494995, "mix_mean": 0.769136369228363, "mix_low": 0.0008754730224609375, "mix_high": 0.1384286880493164, "read_cosine": 0.14956876635551453, "beta_mean": 0.13834106922149658, "nonfinite": 0.0, "weighted_memory_to_active": 0.6528834104537964, "mix_std": 0.284869909286499, "beta_std": 0.1540941596031189, "beta_low": 0.25299072265625, "beta_high": 0.0, "orthogonal_to_active": 0.04990226402878761, "evidence_weights": [-0.24947725236415863, -0.5985190272331238, -0.1514274626970291], "agreement_weight": -0.015766991302371025, "orthogonal_weight": 0.08958674967288971}, {"head": 5, "memory_to_active": 0.040405262261629105, "contribution_to_active": 0.6084684729576111, "mix_mean": 0.6501107215881348, "mix_low": 0.000751495361328125, "mix_high": 0.003936767578125, "read_cosine": 0.2585510015487671, "beta_mean": 0.0002779330825433135, "nonfinite": 0.0, "weighted_memory_to_active": 0.02878998965024948, "mix_std": 0.2550812065601349, "beta_std": 0.0006989357643760741, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.0291390810161829, "evidence_weights": [-0.24344344437122345, -0.3111351728439331, 0.4434438943862915], "agreement_weight": 0.14972829818725586, "orthogonal_weight": -1.0801690816879272}]}, {"layer": 7, "memory_to_active": 1.746344804763794, "contribution_to_active": 0.23320162296295166, "mix_mean": 0.11209990084171295, "mix_low": 0.29521578550338745, "mix_high": 0.0002104441518895328, "read_cosine": 0.23612722754478455, "beta_mean": 0.13457095623016357, "nonfinite": 0.0, "weighted_memory_to_active": 0.20498958230018616, "mix_std": 0.12913525104522705, "beta_std": 0.1028536781668663, "beta_low": 0.1995747983455658, "beta_high": 0.0001627604360692203, "orthogonal_to_active": 0.06283722817897797, "output_influence": 0.243959441781044, "residual_influence": 0.05506732687354088, "state_active": 0.013893911615014076, "state_memory": 0.027271660044789314, "state_peak": 31.940336227416992, "growth_active": 1.099812626838684, "growth_memory": 1.2992446422576904, "heads": [{"head": 0, "memory_to_active": 2.437194585800171, "contribution_to_active": 0.28621208667755127, "mix_mean": 0.03381733596324921, "mix_low": 0.4615468978881836, "mix_high": 0.0, "read_cosine": 0.3679235875606537, "beta_mean": 0.18651878833770752, "nonfinite": 0.0, "weighted_memory_to_active": 0.3124135136604309, "mix_std": 0.07373380661010742, "beta_std": 0.12241930514574051, "beta_low": 0.00164794921875, "beta_high": 0.00048828125, "orthogonal_to_active": 0.009645300917327404, "evidence_weights": [0.018810678273439407, -0.08081334829330444, 0.09244389832019806], "agreement_weight": -0.03210172802209854, "orthogonal_weight": 0.043360598385334015}, {"head": 1, "memory_to_active": 1.0821161270141602, "contribution_to_active": 0.3575367331504822, "mix_mean": 0.18883363902568817, "mix_low": 0.15114784240722656, "mix_high": 0.00101470947265625, "read_cosine": 0.3579086363315582, "beta_mean": 0.1137104332447052, "nonfinite": 0.0, "weighted_memory_to_active": 0.3264746069908142, "mix_std": 0.22063899040222168, "beta_std": 0.07286900281906128, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.018312500789761543, "evidence_weights": [-0.19678200781345367, -0.11277025192975998, -0.19676020741462708], "agreement_weight": 0.027134526520967484, "orthogonal_weight": 0.06573832035064697}, {"head": 2, "memory_to_active": 0.79224693775177, "contribution_to_active": 0.28818023204803467, "mix_mean": 0.12395021319389343, "mix_low": 0.08903980255126953, "mix_high": 0.00021266937255859375, "read_cosine": 0.1237824559211731, "beta_mean": 0.05673529952764511, "nonfinite": 0.0, "weighted_memory_to_active": 0.1968674659729004, "mix_std": 0.16174231469631195, "beta_std": 0.04600266367197037, "beta_low": 0.07415771484375, "beta_high": 0.0, "orthogonal_to_active": 0.07193097472190857, "evidence_weights": [-0.04805077239871025, -0.13507768511772156, 0.48988795280456543], "agreement_weight": 0.021638525649905205, "orthogonal_weight": -0.37397336959838867}, {"head": 3, "memory_to_active": 4.326118469238281, "contribution_to_active": 0.09323130548000336, "mix_mean": 0.03192637115716934, "mix_low": 0.34587669372558594, "mix_high": 0.0, "read_cosine": 0.08774659037590027, "beta_mean": 0.35390058159828186, "nonfinite": 0.0, "weighted_memory_to_active": 0.29861122369766235, "mix_std": 0.04685128852725029, "beta_std": 0.2686707377433777, "beta_low": 0.00665283203125, "beta_high": 0.00048828125, "orthogonal_to_active": 0.2442498803138733, "evidence_weights": [0.06588469445705414, -0.0075757429003715515, 0.6833994388580322], "agreement_weight": 0.05630303546786308, "orthogonal_weight": -0.9264236688613892}, {"head": 4, "memory_to_active": 1.7103307247161865, "contribution_to_active": 0.062353890389204025, "mix_mean": 0.01282355934381485, "mix_low": 0.7091274261474609, "mix_high": 0.0, "read_cosine": 0.22142843902111053, "beta_mean": 0.09560491144657135, "nonfinite": 0.0, "weighted_memory_to_active": 0.04998200759291649, "mix_std": 0.03196671977639198, "beta_std": 0.10510333627462387, "beta_low": 0.1197509765625, "beta_high": 0.0, "orthogonal_to_active": 0.006676473654806614, "evidence_weights": [0.14475849270820618, 0.055046603083610535, 0.22666311264038086], "agreement_weight": 0.04483344405889511, "orthogonal_weight": -0.16838490962982178}, {"head": 5, "memory_to_active": 0.13006217777729034, "contribution_to_active": 0.3116954565048218, "mix_mean": 0.2812482714653015, "mix_low": 0.014555931091308594, "mix_high": 3.528594970703125e-05, "read_cosine": 0.2579735517501831, "beta_mean": 0.0009557216544635594, "nonfinite": 0.0, "weighted_memory_to_active": 0.04558863490819931, "mix_std": 0.23987837135791779, "beta_std": 0.0020570100750774145, "beta_low": 0.9952392578125, "beta_high": 0.0, "orthogonal_to_active": 0.026208238676190376, "evidence_weights": [-0.10859309136867523, -0.16410131752490997, 0.4735332727432251], "agreement_weight": 0.10966263711452484, "orthogonal_weight": -0.6624901294708252}]}, {"layer": 8, "memory_to_active": 0.44618842005729675, "contribution_to_active": 0.5217572450637817, "mix_mean": 0.41865912079811096, "mix_low": 0.09470526874065399, "mix_high": 0.000986258266493678, "read_cosine": 0.4811236262321472, "beta_mean": 0.03090887889266014, "nonfinite": 0.0, "weighted_memory_to_active": 0.09413839876651764, "mix_std": 0.21789947152137756, "beta_std": 0.03689751774072647, "beta_low": 0.6050618886947632, "beta_high": 0.0, "orthogonal_to_active": 0.05958530306816101, "output_influence": 0.4522778391838074, "residual_influence": 0.14245115220546722, "state_active": 0.0473068431019783, "state_memory": 0.009557314217090607, "state_peak": 34.689849853515625, "growth_active": 1.059098720550537, "growth_memory": 1.3141562938690186, "heads": [{"head": 0, "memory_to_active": 0.055401843041181564, "contribution_to_active": 0.5488486886024475, "mix_mean": 0.5930963754653931, "mix_low": 0.0002651214599609375, "mix_high": 0.0010499954223632812, "read_cosine": 0.5589613318443298, "beta_mean": 0.0025646421127021313, "nonfinite": 0.0, "weighted_memory_to_active": 0.03846082091331482, "mix_std": 0.24317282438278198, "beta_std": 0.0070035746321082115, "beta_low": 0.9407958984375, "beta_high": 0.0, "orthogonal_to_active": 0.023314988240599632, "evidence_weights": [-0.16812020540237427, -0.3913678824901581, 0.6940016746520996], "agreement_weight": 0.1297391951084137, "orthogonal_weight": -0.8091613054275513}, {"head": 1, "memory_to_active": 0.1253109574317932, "contribution_to_active": 0.4962414801120758, "mix_mean": 0.46144330501556396, "mix_low": 0.0006504058837890625, "mix_high": 0.0002803802490234375, "read_cosine": 0.3116529583930969, "beta_mean": 0.002610042691230774, "nonfinite": 0.0, "weighted_memory_to_active": 0.05613294243812561, "mix_std": 0.24929071962833405, "beta_std": 0.010394429787993431, "beta_low": 0.9608154296875, "beta_high": 0.0, "orthogonal_to_active": 0.03626122325658798, "evidence_weights": [-0.18732750415802002, -0.2572723627090454, 0.6807726621627808], "agreement_weight": 0.08591349422931671, "orthogonal_weight": -0.723750650882721}, {"head": 2, "memory_to_active": 0.021629322320222855, "contribution_to_active": 0.6530264616012573, "mix_mean": 0.6309161186218262, "mix_low": 0.0009641647338867188, "mix_high": 0.0033597946166992188, "read_cosine": 0.4131755232810974, "beta_mean": 0.00019107067782897502, "nonfinite": 0.0, "weighted_memory_to_active": 0.016173429787158966, "mix_std": 0.23210950195789337, "beta_std": 0.0028479634784162045, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.010743782855570316, "evidence_weights": [-0.22046568989753723, -0.3398580849170685, 0.9491032958030701], "agreement_weight": 0.22646820545196533, "orthogonal_weight": -0.766288161277771}, {"head": 3, "memory_to_active": 0.1406741887331009, "contribution_to_active": 0.6368440389633179, "mix_mean": 0.5550846457481384, "mix_low": 0.0028591156005859375, "mix_high": 0.0011243820190429688, "read_cosine": 0.809877872467041, "beta_mean": 0.011760186403989792, "nonfinite": 0.0, "weighted_memory_to_active": 0.10824225842952728, "mix_std": 0.25646844506263733, "beta_std": 0.00887894257903099, "beta_low": 0.51202392578125, "beta_high": 0.0, "orthogonal_to_active": 0.034001994878053665, "evidence_weights": [-0.1794644594192505, -0.3016315996646881, 0.643229067325592], "agreement_weight": 0.1775330752134323, "orthogonal_weight": -0.6376438140869141}, {"head": 4, "memory_to_active": 2.020206928253174, "contribution_to_active": 0.1770162284374237, "mix_mean": 0.027548611164093018, "mix_low": 0.42818641662597656, "mix_high": 0.0, "read_cosine": 0.325891375541687, "beta_mean": 0.09082823991775513, "nonfinite": 0.0, "weighted_memory_to_active": 0.1738550364971161, "mix_std": 0.046697162091732025, "beta_std": 0.09451240301132202, "beta_low": 0.06268310546875, "beta_high": 0.0, "orthogonal_to_active": 0.0003112459380645305, "evidence_weights": [0.061381030827760696, -0.0255753006786108, -0.07906389981508255], "agreement_weight": 0.07133272290229797, "orthogonal_weight": 0.0019504977390170097}, {"head": 5, "memory_to_active": 0.3139071762561798, "contribution_to_active": 0.618566632270813, "mix_mean": 0.24386562407016754, "mix_low": 0.13530635833740234, "mix_high": 0.000102996826171875, "read_cosine": 0.46718257665634155, "beta_mean": 0.07749909162521362, "nonfinite": 0.0, "weighted_memory_to_active": 0.17196589708328247, "mix_std": 0.27965813875198364, "beta_std": 0.0977477878332138, "beta_low": 0.15411376953125, "beta_high": 0.0, "orthogonal_to_active": 0.2528785765171051, "evidence_weights": [-0.07081338763237, -0.24490886926651, -0.3105926215648651], "agreement_weight": 0.0072059412486851215, "orthogonal_weight": 1.73289954662323}]}, {"layer": 9, "memory_to_active": 1.0491299629211426, "contribution_to_active": 0.6343193054199219, "mix_mean": 0.32094550132751465, "mix_low": 0.14653047919273376, "mix_high": 0.003489653579890728, "read_cosine": 0.3551133871078491, "beta_mean": 0.06746700406074524, "nonfinite": 0.0, "weighted_memory_to_active": 0.356920063495636, "mix_std": 0.20970869064331055, "beta_std": 0.04611776024103165, "beta_low": 0.5003865957260132, "beta_high": 0.0, "orthogonal_to_active": 0.039894744753837585, "output_influence": 0.5004627108573914, "residual_influence": 0.13646286725997925, "state_active": 0.04139949381351471, "state_memory": 0.019535472616553307, "state_peak": 45.7150993347168, "growth_active": 1.1523146629333496, "growth_memory": 1.3696926832199097, "heads": [{"head": 0, "memory_to_active": 0.0013163189869374037, "contribution_to_active": 0.5508336424827576, "mix_mean": 0.6119109988212585, "mix_low": 0.0017557144165039062, "mix_high": 0.015000343322753906, "read_cosine": 0.27626505494117737, "beta_mean": 3.066899807890877e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0008286124793812633, "mix_std": 0.2910466194152832, "beta_std": 1.1543516848178115e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000726423051673919, "evidence_weights": [-0.29514873027801514, -0.30519014596939087, 0.9043564200401306], "agreement_weight": 0.09790028631687164, "orthogonal_weight": -0.9319908618927002}, {"head": 1, "memory_to_active": 0.024817468598484993, "contribution_to_active": 0.6290504932403564, "mix_mean": 0.5061656832695007, "mix_low": 0.008481025695800781, "mix_high": 0.0015201568603515625, "read_cosine": 0.3833529055118561, "beta_mean": 7.195107900770381e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01654018461704254, "mix_std": 0.2860383093357086, "beta_std": 0.0001555932976771146, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011753457598388195, "evidence_weights": [-0.2232958823442459, -0.2734290659427643, 1.1288267374038696], "agreement_weight": 0.3630799353122711, "orthogonal_weight": -0.8483737707138062}, {"head": 2, "memory_to_active": 0.18903779983520508, "contribution_to_active": 0.6300012469291687, "mix_mean": 0.5662631988525391, "mix_low": 0.001796722412109375, "mix_high": 0.004082679748535156, "read_cosine": 0.37887001037597656, "beta_mean": 0.0007990136509761214, "nonfinite": 0.0, "weighted_memory_to_active": 0.12336210906505585, "mix_std": 0.27332016825675964, "beta_std": 0.004810616374015808, "beta_low": 0.99432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.063590869307518, "evidence_weights": [-0.19563288986682892, -0.26433879137039185, 0.7031940817832947], "agreement_weight": 0.2743431329727173, "orthogonal_weight": -0.6140720248222351}, {"head": 3, "memory_to_active": 1.5191996097564697, "contribution_to_active": 0.17633190751075745, "mix_mean": 0.04329522326588631, "mix_low": 0.3885488510131836, "mix_high": 7.62939453125e-06, "read_cosine": 0.41759994626045227, "beta_mean": 0.08580672740936279, "nonfinite": 0.0, "weighted_memory_to_active": 0.1495189070701599, "mix_std": 0.09629253298044205, "beta_std": 0.05775177478790283, "beta_low": 0.0030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.018726984038949013, "evidence_weights": [0.08104413747787476, 0.02690977416932583, -0.026070818305015564], "agreement_weight": 0.0867963656783104, "orthogonal_weight": 0.1328565776348114}, {"head": 4, "memory_to_active": 2.9141509532928467, "contribution_to_active": 1.2248367071151733, "mix_mean": 0.12523941695690155, "mix_low": 0.15293216705322266, "mix_high": 0.0002727508544921875, "read_cosine": 0.19707328081130981, "beta_mean": 0.1798829734325409, "nonfinite": 0.0, "weighted_memory_to_active": 1.2056840658187866, "mix_std": 0.1842423975467682, "beta_std": 0.10545693337917328, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02522289752960205, "evidence_weights": [-0.06333627551794052, -0.14713189005851746, -0.020183203741908073], "agreement_weight": -0.05459500104188919, "orthogonal_weight": -0.021798402070999146}, {"head": 5, "memory_to_active": 1.646256923675537, "contribution_to_active": 0.5948615670204163, "mix_mean": 0.07279840111732483, "mix_low": 0.3256683349609375, "mix_high": 5.435943603515625e-05, "read_cosine": 0.4775192141532898, "beta_mean": 0.1382383108139038, "nonfinite": 0.0, "weighted_memory_to_active": 0.6455862522125244, "mix_std": 0.1273120939731598, "beta_std": 0.10852008312940598, "beta_low": 0.00494384765625, "beta_high": 0.0, "orthogonal_to_active": 0.11934781819581985, "evidence_weights": [0.1046474352478981, -0.0754338875412941, 0.14585599303245544], "agreement_weight": 0.01770440861582756, "orthogonal_weight": 0.2661198675632477}]}, {"layer": 10, "memory_to_active": 0.23052978515625, "contribution_to_active": 0.6379093527793884, "mix_mean": 0.5065236687660217, "mix_low": 0.014490446075797081, "mix_high": 0.007286707870662212, "read_cosine": 0.3794882893562317, "beta_mean": 0.04485785961151123, "nonfinite": 0.0, "weighted_memory_to_active": 0.11421440541744232, "mix_std": 0.271435022354126, "beta_std": 0.04662356153130531, "beta_low": 0.6701864004135132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.16096553206443787, "output_influence": 0.4331011176109314, "residual_influence": 0.1450987011194229, "state_active": 0.05076805129647255, "state_memory": 0.015579942613840103, "state_peak": 120.49251556396484, "growth_active": 1.0325554609298706, "growth_memory": 0.9199438095092773, "heads": [{"head": 0, "memory_to_active": 0.8501865863800049, "contribution_to_active": 1.5962111949920654, "mix_mean": 0.6035821437835693, "mix_low": 0.051522254943847656, "mix_high": 0.028738021850585938, "read_cosine": 0.1109866201877594, "beta_mean": 0.19453389942646027, "nonfinite": 0.0, "weighted_memory_to_active": 0.5554981827735901, "mix_std": 0.3882843255996704, "beta_std": 0.21098867058753967, "beta_low": 0.1490478515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.8812305927276611, "evidence_weights": [-0.12842616438865662, -0.3911866247653961, -1.183335542678833], "agreement_weight": 0.026459865272045135, "orthogonal_weight": 1.61021089553833}, {"head": 1, "memory_to_active": 0.011855866760015488, "contribution_to_active": 0.598031759262085, "mix_mean": 0.5992671251296997, "mix_low": 0.0008058547973632812, "mix_high": 0.009784698486328125, "read_cosine": 0.27497780323028564, "beta_mean": 7.588032895000651e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006959396414458752, "mix_std": 0.2666628956794739, "beta_std": 0.00010699661652324721, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009739001281559467, "evidence_weights": [-0.26728010177612305, -0.20308932662010193, 0.6011114120483398], "agreement_weight": 0.45751953125, "orthogonal_weight": -1.5313210487365723}, {"head": 2, "memory_to_active": 0.01557532325387001, "contribution_to_active": 0.5831820964813232, "mix_mean": 0.6012839674949646, "mix_low": 0.00037097930908203125, "mix_high": 0.004344940185546875, "read_cosine": 0.029607078060507774, "beta_mean": 4.258628177922219e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006260670255869627, "mix_std": 0.25609463453292847, "beta_std": 0.00017722637858241796, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0028067664243280888, "evidence_weights": [-0.23572255671024323, -0.2715132236480713, 0.5509507656097412], "agreement_weight": -0.1466933786869049, "orthogonal_weight": -0.4319840967655182}, {"head": 3, "memory_to_active": 0.3589407205581665, "contribution_to_active": 0.1129881739616394, "mix_mean": 0.31181618571281433, "mix_low": 0.031084060668945312, "mix_high": 0.000522613525390625, "read_cosine": 0.9076344966888428, "beta_mean": 0.0687975063920021, "nonfinite": 0.0, "weighted_memory_to_active": 0.045009795576334, "mix_std": 0.24062880873680115, "beta_std": 0.060647327452898026, "beta_low": 0.00323486328125, "beta_high": 0.0, "orthogonal_to_active": 0.05054570361971855, "evidence_weights": [-0.11154507845640182, -0.14583465456962585, 0.13467533886432648], "agreement_weight": 0.011704664677381516, "orthogonal_weight": 1.4888331890106201}, {"head": 4, "memory_to_active": 0.002088724635541439, "contribution_to_active": 0.5234828591346741, "mix_mean": 0.5356640219688416, "mix_low": 0.0005598068237304688, "mix_high": 0.00032138824462890625, "read_cosine": 0.08581139147281647, "beta_mean": 1.2226444596308284e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0011383637320250273, "mix_std": 0.24974273145198822, "beta_std": 3.335054498165846e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008655792917124927, "evidence_weights": [-0.1798316240310669, -0.2594074308872223, 0.7708303332328796], "agreement_weight": 0.0717402771115303, "orthogonal_weight": -0.7933631539344788}, {"head": 5, "memory_to_active": 0.1445314586162567, "contribution_to_active": 0.41355979442596436, "mix_mean": 0.3875282108783722, "mix_low": 0.0025997161865234375, "mix_high": 8.58306884765625e-06, "read_cosine": 0.867912232875824, "beta_mean": 0.005685028620064259, "nonfinite": 0.0, "weighted_memory_to_active": 0.07041999697685242, "mix_std": 0.22719670832157135, "beta_std": 0.007787787355482578, "beta_low": 0.868896484375, "beta_high": 0.0, "orthogonal_to_active": 0.020605430006980896, "evidence_weights": [-0.14740388095378876, -0.15975166857242584, 0.28632020950317383], "agreement_weight": 0.09104175120592117, "orthogonal_weight": -0.6365290284156799}]}, {"layer": 11, "memory_to_active": 0.11913120746612549, "contribution_to_active": 0.5854887962341309, "mix_mean": 0.47935137152671814, "mix_low": 0.08843867480754852, "mix_high": 0.04403495788574219, "read_cosine": 0.39128467440605164, "beta_mean": 0.033038180321455, "nonfinite": 0.0, "weighted_memory_to_active": 0.055572282522916794, "mix_std": 0.32070618867874146, "beta_std": 0.05086896941065788, "beta_low": 0.6034546494483948, "beta_high": 0.0, "orthogonal_to_active": 0.04797670990228653, "output_influence": 0.5301588773727417, "residual_influence": 0.13234221935272217, "state_active": 0.20706644654273987, "state_memory": 0.006760105490684509, "state_peak": 28.675405502319336, "growth_active": 1.1154803037643433, "growth_memory": 1.619042158126831, "heads": [{"head": 0, "memory_to_active": 0.006340636871755123, "contribution_to_active": 0.5767644047737122, "mix_mean": 0.6203128099441528, "mix_low": 0.0010852813720703125, "mix_high": 0.008392333984375, "read_cosine": 0.3327469527721405, "beta_mean": 7.67327583162114e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004038055427372456, "mix_std": 0.2617068588733673, "beta_std": 0.003798890393227339, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0029536609072238207, "evidence_weights": [-0.1662389189004898, -0.3640403151512146, 0.7100090980529785], "agreement_weight": 0.27614688873291016, "orthogonal_weight": -0.7753065228462219}, {"head": 1, "memory_to_active": 0.29013708233833313, "contribution_to_active": 0.31810811161994934, "mix_mean": 0.3658629059791565, "mix_low": 0.009511947631835938, "mix_high": 0.00014591217041015625, "read_cosine": 0.5454481840133667, "beta_mean": 0.03289889544248581, "nonfinite": 0.0, "weighted_memory_to_active": 0.12407226860523224, "mix_std": 0.2576560974121094, "beta_std": 0.06975536048412323, "beta_low": 0.44110107421875, "beta_high": 0.0, "orthogonal_to_active": 0.0420135036110878, "evidence_weights": [-0.16271883249282837, -0.1768941581249237, 0.452411413192749], "agreement_weight": 0.07476107031106949, "orthogonal_weight": -0.4211519956588745}, {"head": 2, "memory_to_active": 0.00968428049236536, "contribution_to_active": 0.9799667000770569, "mix_mean": 0.450096070766449, "mix_low": 0.2137298583984375, "mix_high": 0.16335773468017578, "read_cosine": 0.19689801335334778, "beta_mean": 0.0773620754480362, "nonfinite": 0.0, "weighted_memory_to_active": 0.007217176258563995, "mix_std": 0.43893370032310486, "beta_std": 0.10397680103778839, "beta_low": 0.45361328125, "beta_high": 0.0, "orthogonal_to_active": 0.043839242309331894, "evidence_weights": [0.04355132207274437, -0.44123905897140503, -1.2710680961608887], "agreement_weight": 0.00903466995805502, "orthogonal_weight": 6.3632941246032715}, {"head": 3, "memory_to_active": 0.011978218331933022, "contribution_to_active": 0.8515772819519043, "mix_mean": 0.4070121943950653, "mix_low": 0.2942314147949219, "mix_high": 0.08842086791992188, "read_cosine": 0.5342413187026978, "beta_mean": 0.06306079030036926, "nonfinite": 0.0, "weighted_memory_to_active": 0.007341711316257715, "mix_std": 0.4358055591583252, "beta_std": 0.09614555537700653, "beta_low": 0.49591064453125, "beta_high": 0.0, "orthogonal_to_active": 0.03575420379638672, "evidence_weights": [0.1446772813796997, -0.3264502286911011, -0.29822924733161926], "agreement_weight": 0.04706215485930443, "orthogonal_weight": 6.685608863830566}, {"head": 4, "memory_to_active": 0.3046111762523651, "contribution_to_active": 0.41141030192375183, "mix_mean": 0.5088930130004883, "mix_low": 0.009987831115722656, "mix_high": 0.0008087158203125, "read_cosine": 0.09720046073198318, "beta_mean": 0.016184329986572266, "nonfinite": 0.0, "weighted_memory_to_active": 0.14534538984298706, "mix_std": 0.27114084362983704, "beta_std": 0.018561463803052902, "beta_low": 0.48052978515625, "beta_high": 0.0, "orthogonal_to_active": 0.1384819746017456, "evidence_weights": [-0.22804085910320282, -0.20577609539031982, 1.8882375955581665], "agreement_weight": -0.18973110616207123, "orthogonal_weight": -0.9628735780715942}, {"head": 5, "memory_to_active": 0.09203588962554932, "contribution_to_active": 0.37510567903518677, "mix_mean": 0.5239312648773193, "mix_low": 0.0020856857299804688, "mix_high": 0.0030841827392578125, "read_cosine": 0.6411730647087097, "beta_mean": 0.008646264672279358, "nonfinite": 0.0, "weighted_memory_to_active": 0.04541908577084541, "mix_std": 0.25899389386177063, "beta_std": 0.012975731864571571, "beta_low": 0.749755859375, "beta_high": 0.0, "orthogonal_to_active": 0.02481767348945141, "evidence_weights": [-0.21864622831344604, -0.29446956515312195, 0.5965282320976257], "agreement_weight": 0.07785940915346146, "orthogonal_weight": -0.7221012711524963}]}, {"layer": 12, "memory_to_active": 0.28392910957336426, "contribution_to_active": 0.5723767280578613, "mix_mean": 0.5128587484359741, "mix_low": 0.00954183004796505, "mix_high": 0.004497210495173931, "read_cosine": 0.2547086775302887, "beta_mean": 0.016937443986535072, "nonfinite": 0.0, "weighted_memory_to_active": 0.16056038439273834, "mix_std": 0.2629329264163971, "beta_std": 0.01662151888012886, "beta_low": 0.82342529296875, "beta_high": 0.0, "orthogonal_to_active": 0.07509157061576843, "output_influence": 0.4593448042869568, "residual_influence": 0.12777560949325562, "state_active": 0.037639766931533813, "state_memory": 0.009657814167439938, "state_peak": 49.063114166259766, "growth_active": 1.1429269313812256, "growth_memory": 1.2253358364105225, "heads": [{"head": 0, "memory_to_active": 0.05871497094631195, "contribution_to_active": 0.5252896547317505, "mix_mean": 0.5148698687553406, "mix_low": 0.0011873245239257812, "mix_high": 0.0023193359375, "read_cosine": 0.313862681388855, "beta_mean": 0.0008675697608850896, "nonfinite": 0.0, "weighted_memory_to_active": 0.027717258781194687, "mix_std": 0.27170372009277344, "beta_std": 0.003785329405218363, "beta_low": 0.99505615234375, "beta_high": 0.0, "orthogonal_to_active": 0.021683942526578903, "evidence_weights": [-0.19505853950977325, -0.26690739393234253, 0.2677944302558899], "agreement_weight": 0.10948998481035233, "orthogonal_weight": -0.8339254856109619}, {"head": 1, "memory_to_active": 0.07517409324645996, "contribution_to_active": 0.5140836238861084, "mix_mean": 0.5303762555122375, "mix_low": 0.0005216598510742188, "mix_high": 0.000957489013671875, "read_cosine": 0.5066519379615784, "beta_mean": 0.0008015226339921355, "nonfinite": 0.0, "weighted_memory_to_active": 0.039341263473033905, "mix_std": 0.26215463876724243, "beta_std": 0.005320079158991575, "beta_low": 0.99261474609375, "beta_high": 0.0, "orthogonal_to_active": 0.03138095885515213, "evidence_weights": [-0.20813778042793274, -0.2742999494075775, 0.9655706882476807], "agreement_weight": 0.1456076204776764, "orthogonal_weight": -0.9234678149223328}, {"head": 2, "memory_to_active": 0.6155511140823364, "contribution_to_active": 0.5332488417625427, "mix_mean": 0.35379013419151306, "mix_low": 0.023252487182617188, "mix_high": 0.0007658004760742188, "read_cosine": 0.490756630897522, "beta_mean": 0.09718634188175201, "nonfinite": 0.0, "weighted_memory_to_active": 0.2808384895324707, "mix_std": 0.252108097076416, "beta_std": 0.06593801081180573, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.26330843567848206, "evidence_weights": [-0.2390124797821045, -0.047815367579460144, -0.08733907341957092], "agreement_weight": 0.021636269986629486, "orthogonal_weight": 1.191076636314392}, {"head": 3, "memory_to_active": 0.02935698814690113, "contribution_to_active": 0.7276920676231384, "mix_mean": 0.698615550994873, "mix_low": 0.00032138824462890625, "mix_high": 0.015234947204589844, "read_cosine": 0.08466077595949173, "beta_mean": 0.00023224388132803142, "nonfinite": 0.0, "weighted_memory_to_active": 0.02230767533183098, "mix_std": 0.23722800612449646, "beta_std": 0.004542862065136433, "beta_low": 0.99835205078125, "beta_high": 0.0, "orthogonal_to_active": 0.02109421417117119, "evidence_weights": [-0.2922155559062958, -0.34123653173446655, 1.3736815452575684], "agreement_weight": 0.3706948161125183, "orthogonal_weight": -0.9674574136734009}, {"head": 4, "memory_to_active": 0.8318927884101868, "contribution_to_active": 0.6701996922492981, "mix_mean": 0.35770028829574585, "mix_low": 0.02423381805419922, "mix_high": 0.0026397705078125, "read_cosine": -0.08719585835933685, "beta_mean": 0.0019630035385489464, "nonfinite": 0.0, "weighted_memory_to_active": 0.5299485325813293, "mix_std": 0.27659159898757935, "beta_std": 0.015832440927624702, "beta_low": 0.96990966796875, "beta_high": 0.0, "orthogonal_to_active": 0.06403706222772598, "evidence_weights": [-0.11709178239107132, -0.21504226326942444, 0.024050146341323853], "agreement_weight": -0.0026536760851740837, "orthogonal_weight": -0.12691716849803925}, {"head": 5, "memory_to_active": 0.09288465231657028, "contribution_to_active": 0.4637463688850403, "mix_mean": 0.621800422668457, "mix_low": 0.0077342987060546875, "mix_high": 0.00506591796875, "read_cosine": 0.21951591968536377, "beta_mean": 0.00057396967895329, "nonfinite": 0.0, "weighted_memory_to_active": 0.06320906430482864, "mix_std": 0.2778114676475525, "beta_std": 0.0043103923089802265, "beta_low": 0.98406982421875, "beta_high": 0.0, "orthogonal_to_active": 0.04904481768608093, "evidence_weights": [-0.24850596487522125, -0.3109820485115051, 1.2395505905151367], "agreement_weight": 0.3199950158596039, "orthogonal_weight": -0.841443657875061}]}, {"layer": 13, "memory_to_active": 0.13503779470920563, "contribution_to_active": 0.5264223217964172, "mix_mean": 0.5233677625656128, "mix_low": 0.005702018737792969, "mix_high": 0.006189028732478619, "read_cosine": 0.42055848240852356, "beta_mean": 0.017538240179419518, "nonfinite": 0.0, "weighted_memory_to_active": 0.06028936803340912, "mix_std": 0.26199212670326233, "beta_std": 0.018830306828022003, "beta_low": 0.7982890009880066, "beta_high": 0.0, "orthogonal_to_active": 0.0617118701338768, "output_influence": 0.5006997585296631, "residual_influence": 0.16852641105651855, "state_active": 0.10694460570812225, "state_memory": 0.039923086762428284, "state_peak": 96.68193817138672, "growth_active": 0.960407018661499, "growth_memory": 0.998482346534729, "heads": [{"head": 0, "memory_to_active": 0.026887858286499977, "contribution_to_active": 0.5634261965751648, "mix_mean": 0.5773905515670776, "mix_low": 0.00039196014404296875, "mix_high": 0.0014801025390625, "read_cosine": 0.4281015396118164, "beta_mean": 0.00010544742690399289, "nonfinite": 0.0, "weighted_memory_to_active": 0.016526464372873306, "mix_std": 0.26797550916671753, "beta_std": 0.00022988310956861824, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.016327902674674988, "evidence_weights": [-0.20297539234161377, -0.2663688659667969, 0.7504529356956482], "agreement_weight": 0.11186198145151138, "orthogonal_weight": -1.1000133752822876}, {"head": 1, "memory_to_active": 0.016145769506692886, "contribution_to_active": 0.5810816884040833, "mix_mean": 0.5409443378448486, "mix_low": 0.0018224716186523438, "mix_high": 0.0018892288208007812, "read_cosine": 0.22208355367183685, "beta_mean": 6.0254238633206114e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00891962368041277, "mix_std": 0.2714303731918335, "beta_std": 0.000101831006759312, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010680359788239002, "evidence_weights": [-0.1868266463279724, -0.2703644931316376, 0.7024874091148376], "agreement_weight": 0.33694374561309814, "orthogonal_weight": -1.2524973154067993}, {"head": 2, "memory_to_active": 0.10810763388872147, "contribution_to_active": 0.43666917085647583, "mix_mean": 0.38134610652923584, "mix_low": 0.0021371841430664062, "mix_high": 0.00015926361083984375, "read_cosine": 0.6610666513442993, "beta_mean": 0.014422341249883175, "nonfinite": 0.0, "weighted_memory_to_active": 0.04846695065498352, "mix_std": 0.22865012288093567, "beta_std": 0.04341467469930649, "beta_low": 0.7991943359375, "beta_high": 0.0, "orthogonal_to_active": 0.025256197899580002, "evidence_weights": [-0.2013690322637558, -0.196151003241539, 0.43266549706459045], "agreement_weight": 0.009960460476577282, "orthogonal_weight": -0.8099679946899414}, {"head": 3, "memory_to_active": 0.16357170045375824, "contribution_to_active": 0.6127901673316956, "mix_mean": 0.6305621266365051, "mix_low": 0.02072620391845703, "mix_high": 0.025017738342285156, "read_cosine": 0.008601789362728596, "beta_mean": 0.0010097497142851353, "nonfinite": 0.0, "weighted_memory_to_active": 0.119701087474823, "mix_std": 0.29345107078552246, "beta_std": 0.008756007067859173, "beta_low": 0.99383544921875, "beta_high": 0.0, "orthogonal_to_active": 0.11310151219367981, "evidence_weights": [-0.32891520857810974, -0.24677877128124237, 1.8566009998321533], "agreement_weight": 4.189348220825195, "orthogonal_weight": -1.0285062789916992}, {"head": 4, "memory_to_active": 0.4156053960323334, "contribution_to_active": 0.2874690592288971, "mix_mean": 0.38555580377578735, "mix_low": 0.006404876708984375, "mix_high": 0.0001163482666015625, "read_cosine": 0.8778218030929565, "beta_mean": 0.08909487724304199, "nonfinite": 0.0, "weighted_memory_to_active": 0.11808940768241882, "mix_std": 0.23330366611480713, "beta_std": 0.056980084627866745, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.163385808467865, "evidence_weights": [-0.1250762790441513, -0.23426207900047302, 0.48127588629722595], "agreement_weight": 0.0022077395115047693, "orthogonal_weight": 2.148402690887451}, {"head": 5, "memory_to_active": 0.0799083337187767, "contribution_to_active": 0.6770973205566406, "mix_mean": 0.6244074106216431, "mix_low": 0.0027294158935546875, "mix_high": 0.008471488952636719, "read_cosine": 0.32567542791366577, "beta_mean": 0.0005367568228393793, "nonfinite": 0.0, "weighted_memory_to_active": 0.05003266781568527, "mix_std": 0.2771420478820801, "beta_std": 0.0034993526060134172, "beta_low": 0.9964599609375, "beta_high": 0.0, "orthogonal_to_active": 0.0415194109082222, "evidence_weights": [-0.22081869840621948, -0.301002562046051, 0.8062264919281006], "agreement_weight": 0.11863770335912704, "orthogonal_weight": -0.9128381609916687}]}, {"layer": 14, "memory_to_active": 0.24521780014038086, "contribution_to_active": 0.4253033399581909, "mix_mean": 0.42164450883865356, "mix_low": 0.024675529450178146, "mix_high": 0.005506674759089947, "read_cosine": 0.6132146120071411, "beta_mean": 0.06705625355243683, "nonfinite": 0.0, "weighted_memory_to_active": 0.10802081227302551, "mix_std": 0.24778372049331665, "beta_std": 0.06104402244091034, "beta_low": 0.4671732783317566, "beta_high": 0.0, "orthogonal_to_active": 0.042905814945697784, "output_influence": 0.3319240212440491, "residual_influence": 0.12626539170742035, "state_active": 0.05954359471797943, "state_memory": 0.016517896205186844, "state_peak": 34.875892639160156, "growth_active": 1.1054275035858154, "growth_memory": 1.3106632232666016, "heads": [{"head": 0, "memory_to_active": 0.3846800923347473, "contribution_to_active": 0.21039196848869324, "mix_mean": 0.2183612883090973, "mix_low": 0.01419830322265625, "mix_high": 9.5367431640625e-06, "read_cosine": 0.791058361530304, "beta_mean": 0.07627858221530914, "nonfinite": 0.0, "weighted_memory_to_active": 0.10022401809692383, "mix_std": 0.19418632984161377, "beta_std": 0.0642833262681961, "beta_low": 0.009033203125, "beta_high": 0.0, "orthogonal_to_active": 0.0793503150343895, "evidence_weights": [-0.13798817992210388, -0.03939823433756828, 0.5021708011627197], "agreement_weight": 0.024173032492399216, "orthogonal_weight": 1.3203610181808472}, {"head": 1, "memory_to_active": 0.4660945534706116, "contribution_to_active": 0.33409589529037476, "mix_mean": 0.23828750848770142, "mix_low": 0.11801910400390625, "mix_high": 2.6702880859375e-05, "read_cosine": 0.8001242876052856, "beta_mean": 0.2537898123264313, "nonfinite": 0.0, "weighted_memory_to_active": 0.21602685749530792, "mix_std": 0.26272040605545044, "beta_std": 0.2425513118505478, "beta_low": 0.09686279296875, "beta_high": 0.0, "orthogonal_to_active": 0.014371351338922977, "evidence_weights": [-0.011268546804785728, -0.22780399024486542, -0.14782913029193878], "agreement_weight": 0.05360237509012222, "orthogonal_weight": 0.15512914955615997}, {"head": 2, "memory_to_active": 0.14123034477233887, "contribution_to_active": 0.5592389702796936, "mix_mean": 0.5653596520423889, "mix_low": 0.0022325515747070312, "mix_high": 0.0027589797973632812, "read_cosine": 0.42355453968048096, "beta_mean": 0.003014977555721998, "nonfinite": 0.0, "weighted_memory_to_active": 0.08944091945886612, "mix_std": 0.2658192813396454, "beta_std": 0.0045337434858083725, "beta_low": 0.94647216796875, "beta_high": 0.0, "orthogonal_to_active": 0.06818126142024994, "evidence_weights": [-0.15194737911224365, -0.31023257970809937, 0.9124003052711487], "agreement_weight": 0.08289770036935806, "orthogonal_weight": -0.8775666952133179}, {"head": 3, "memory_to_active": 0.0555584654211998, "contribution_to_active": 0.6334711313247681, "mix_mean": 0.6768985986709595, "mix_low": 0.005393028259277344, "mix_high": 0.020613670349121094, "read_cosine": 0.07206287235021591, "beta_mean": 0.0072957295924425125, "nonfinite": 0.0, "weighted_memory_to_active": 0.036359403282403946, "mix_std": 0.2500419020652771, "beta_std": 0.0075850654393434525, "beta_low": 0.7662353515625, "beta_high": 0.0, "orthogonal_to_active": 0.023649193346500397, "evidence_weights": [-0.3673175573348999, -0.3213813900947571, 0.38445809483528137], "agreement_weight": 0.07977015525102615, "orthogonal_weight": -0.7427241206169128}, {"head": 4, "memory_to_active": 0.17281973361968994, "contribution_to_active": 0.48216480016708374, "mix_mean": 0.480953574180603, "mix_low": 0.004741668701171875, "mix_high": 0.008347511291503906, "read_cosine": 0.8559341430664062, "beta_mean": 0.06029459089040756, "nonfinite": 0.0, "weighted_memory_to_active": 0.0995483249425888, "mix_std": 0.26249316334724426, "beta_std": 0.03922709822654724, "beta_low": 0.00775146484375, "beta_high": 0.0, "orthogonal_to_active": 0.028274457901716232, "evidence_weights": [-0.21788792312145233, -0.25089600682258606, 0.42394939064979553], "agreement_weight": 0.1253214329481125, "orthogonal_weight": -0.5908977389335632}, {"head": 5, "memory_to_active": 0.25092360377311707, "contribution_to_active": 0.33245721459388733, "mix_mean": 0.3500061631202698, "mix_low": 0.0034685134887695312, "mix_high": 0.0012836456298828125, "read_cosine": 0.7365535497665405, "beta_mean": 0.0016637772787362337, "nonfinite": 0.0, "weighted_memory_to_active": 0.10652532428503036, "mix_std": 0.251441091299057, "beta_std": 0.008083557710051537, "beta_low": 0.9766845703125, "beta_high": 0.0, "orthogonal_to_active": 0.04360828176140785, "evidence_weights": [-0.18163838982582092, -0.16389364004135132, 0.24524858593940735], "agreement_weight": 0.08350802212953568, "orthogonal_weight": -0.6662483811378479}]}, {"layer": 15, "memory_to_active": 0.06569255888462067, "contribution_to_active": 0.48192423582077026, "mix_mean": 0.4054194688796997, "mix_low": 0.010092576034367085, "mix_high": 0.002582073211669922, "read_cosine": 0.6598833203315735, "beta_mean": 0.03914976865053177, "nonfinite": 0.0, "weighted_memory_to_active": 0.030900441110134125, "mix_std": 0.2861133813858032, "beta_std": 0.04523678123950958, "beta_low": 0.7008463740348816, "beta_high": 0.0, "orthogonal_to_active": 0.0074898311868309975, "output_influence": 0.3005445599555969, "residual_influence": 0.1595427393913269, "state_active": 0.12696123123168945, "state_memory": 0.010222363285720348, "state_peak": 37.91092300415039, "growth_active": 1.2205225229263306, "growth_memory": 1.4389592409133911, "heads": [{"head": 0, "memory_to_active": 0.11430931836366653, "contribution_to_active": 0.45884740352630615, "mix_mean": 0.37146469950675964, "mix_low": 0.006621360778808594, "mix_high": 0.005030632019042969, "read_cosine": 0.6457449197769165, "beta_mean": 0.040792711079120636, "nonfinite": 0.0, "weighted_memory_to_active": 0.0518081896007061, "mix_std": 0.27968406677246094, "beta_std": 0.02911207266151905, "beta_low": 0.07110595703125, "beta_high": 0.0, "orthogonal_to_active": 0.024966858327388763, "evidence_weights": [-0.17409172654151917, -0.18047334253787994, 0.36790040135383606], "agreement_weight": 0.09572000801563263, "orthogonal_weight": -0.709880530834198}, {"head": 1, "memory_to_active": 0.14643582701683044, "contribution_to_active": 0.4106440842151642, "mix_mean": 0.41553786396980286, "mix_low": 0.0026035308837890625, "mix_high": 0.005108833312988281, "read_cosine": 0.908549427986145, "beta_mean": 0.001149952644482255, "nonfinite": 0.0, "weighted_memory_to_active": 0.06044157221913338, "mix_std": 0.2848275601863861, "beta_std": 0.0032992204651236534, "beta_low": 0.98095703125, "beta_high": 0.0, "orthogonal_to_active": 0.0005931005580350757, "evidence_weights": [-0.16112151741981506, -0.17040672898292542, 0.22882507741451263], "agreement_weight": 0.11216907203197479, "orthogonal_weight": -0.018238291144371033}, {"head": 2, "memory_to_active": 0.009725509211421013, "contribution_to_active": 0.47546494007110596, "mix_mean": 0.34095895290374756, "mix_low": 0.020371437072753906, "mix_high": 7.43865966796875e-05, "read_cosine": 0.7644506692886353, "beta_mean": 0.00015714476467110217, "nonfinite": 0.0, "weighted_memory_to_active": 0.004719856195151806, "mix_std": 0.27536869049072266, "beta_std": 0.0002888987655751407, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0009428106131963432, "evidence_weights": [-0.20248626172542572, -0.27344802021980286, 0.19969496130943298], "agreement_weight": 0.006775439251214266, "orthogonal_weight": 0.4392586350440979}, {"head": 3, "memory_to_active": 0.008128901943564415, "contribution_to_active": 0.40805110335350037, "mix_mean": 0.37405282258987427, "mix_low": 0.0074367523193359375, "mix_high": 0.000316619873046875, "read_cosine": 0.5988905429840088, "beta_mean": 5.907432932872325e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003234333824366331, "mix_std": 0.28542566299438477, "beta_std": 0.00013284063606988639, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002696622395887971, "evidence_weights": [-0.23057396709918976, -0.25339531898498535, 0.29771679639816284], "agreement_weight": 0.034670133143663406, "orthogonal_weight": -1.0011600255966187}, {"head": 4, "memory_to_active": 0.08936553448438644, "contribution_to_active": 0.615532398223877, "mix_mean": 0.3789842128753662, "mix_low": 0.013535499572753906, "mix_high": 0.0011320114135742188, "read_cosine": 0.6680175065994263, "beta_mean": 0.19092091917991638, "nonfinite": 0.0, "weighted_memory_to_active": 0.04811572656035423, "mix_std": 0.3101734220981598, "beta_std": 0.22551560401916504, "beta_low": 0.18121337890625, "beta_high": 0.0, "orthogonal_to_active": 0.006816597189754248, "evidence_weights": [-0.21922478079795837, -0.2599981129169464, 0.016313299536705017], "agreement_weight": 0.013747374527156353, "orthogonal_weight": 0.27620962262153625}, {"head": 5, "memory_to_active": 0.02619023807346821, "contribution_to_active": 0.5230055451393127, "mix_mean": 0.5515181422233582, "mix_low": 0.00998687744140625, "mix_high": 0.0038299560546875, "read_cosine": 0.3736470937728882, "beta_mean": 0.0018187920795753598, "nonfinite": 0.0, "weighted_memory_to_active": 0.017082957550883293, "mix_std": 0.2812007665634155, "beta_std": 0.013072041794657707, "beta_low": 0.97186279296875, "beta_high": 0.0, "orthogonal_to_active": 0.008922996930778027, "evidence_weights": [-0.3073638081550598, -0.3103784918785095, 0.7542737126350403], "agreement_weight": 0.2566344141960144, "orthogonal_weight": 0.7091204524040222}]}]}, {"step": 38000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.901224376000755, "memory_to_active": 0.8511969447135925, "contribution_to_active": 0.529978334903717, "mix_mean": 0.3824542760848999, "mix_low": 0.14662866294384003, "mix_high": 0.00917784497141838, "read_cosine": 0.3649039566516876, "beta_mean": 0.07810556888580322, "nonfinite": 0.0, "weighted_memory_to_active": 0.20651520788669586, "mix_std": 0.2189742922782898, "beta_std": 0.06700645387172699, "beta_low": 0.5239677429199219, "beta_high": 2.6702880859375e-05, "orthogonal_to_active": 0.05626443773508072, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1032869815826416, "normal_token_accuracy": 0.44938820600509644, "state_active": 0.06011828035116196, "state_memory": 0.018704330548644066, "state_peak": 115.84151458740234, "growth_active": 1.139728307723999, "growth_memory": 1.3743146657943726, "output_influence": 0.3887857496738434, "residual_influence": 0.12751996517181396, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2296061515808105, "no_memory_loss_delta": 0.44881463050842285, "scrambled_loss_delta": 0.17147812247276306, "memory_only_loss_delta": 2.8805315494537354, "no_evidence_loss_delta": 0.24365553259849548, "no_agreement_loss_delta": 0.02161845937371254, "no_orthogonal_loss_delta": 0.004271085374057293}, {"index": 94, "tokens": 256, "normal_loss": 3.2085988521575928, "no_memory_loss_delta": 0.5340186357498169, "scrambled_loss_delta": 0.34925150871276855, "memory_only_loss_delta": 3.21498703956604, "no_evidence_loss_delta": 0.23684947192668915, "no_agreement_loss_delta": 0.026722002774477005, "no_orthogonal_loss_delta": 0.038094934076070786}, {"index": 188, "tokens": 256, "normal_loss": 3.4578917026519775, "no_memory_loss_delta": 0.4993349313735962, "scrambled_loss_delta": 0.14072182774543762, "memory_only_loss_delta": 2.8408937454223633, "no_evidence_loss_delta": 0.2525021433830261, "no_agreement_loss_delta": 0.03612423688173294, "no_orthogonal_loss_delta": 0.030266204848885536}, {"index": 283, "tokens": 131, "normal_loss": 1.9576642513275146, "no_memory_loss_delta": 0.23225274682044983, "scrambled_loss_delta": 0.2369932234287262, "memory_only_loss_delta": 2.729728937149048, "no_evidence_loss_delta": 0.06458175927400589, "no_agreement_loss_delta": 0.005754261743277311, "no_orthogonal_loss_delta": 0.017673319205641747}], "agreement_weight_abs_mean": 0.17050330340862274, "agreement_weight_abs_max": 4.293066501617432, "agreement_weight_mean": 0.14960215985774994, "orthogonal_weight_abs_mean": 0.7375991940498352, "orthogonal_weight_abs_max": 6.850977420806885, "orthogonal_weight_mean": -0.14102888107299805, "no_memory_loss": 3.5591933727264404, "no_memory_loss_delta": 0.4559067487716675, "no_memory_token_accuracy_gain": 0.0767519474029541, "no_memory_token_win_fraction": 0.6418242454528809, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 36.61270562267652, "scrambled_loss": 3.326176404953003, "scrambled_loss_delta": 0.22288952767848969, "scrambled_token_accuracy_gain": 0.036707453429698944, "scrambled_token_win_fraction": 0.6329254508018494, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.979675527803526, "memory_only_loss": 6.0457963943481445, "memory_only_loss_delta": 2.942509412765503, "memory_only_token_accuracy_gain": 0.28142380714416504, "memory_only_token_win_fraction": 0.8998887538909912, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.7266766883572, "no_evidence_loss": 3.3214285373687744, "no_evidence_loss_delta": 0.21814243495464325, "no_evidence_token_accuracy_gain": 0.03337041288614273, "no_evidence_token_win_fraction": 0.600667417049408, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 19.598908574770448, "no_agreement_loss": 3.1281771659851074, "no_agreement_loss_delta": 0.024890735745429993, "no_agreement_token_accuracy_gain": 0.0033370412420481443, "no_agreement_token_win_fraction": 0.4983314871788025, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.45835156389876, "no_orthogonal_loss": 3.126544713973999, "no_orthogonal_loss_delta": 0.02325812354683876, "no_orthogonal_token_accuracy_gain": 0.0055617354810237885, "no_orthogonal_token_win_fraction": 0.5684093236923218, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 2.2989738131620188, "short_no_memory_loss_delta": 0.4101814329624176, "long_context_memory_gain": 0.04572531580924988, "normal_context_loss_gain": 0.14852988719940186, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.2920875549316406, "contribution_to_active": 1.00570547580719, "mix_mean": 0.10015561431646347, "mix_low": 0.5492873787879944, "mix_high": 0.0044337911531329155, "read_cosine": 0.4515659213066101, "beta_mean": 0.24237331748008728, "nonfinite": 0.0, "weighted_memory_to_active": 1.0479518175125122, "mix_std": 0.0863259881734848, "beta_std": 0.15037499368190765, "beta_low": 0.026275634765625, "beta_high": 0.0, "orthogonal_to_active": 0.048815757036209106, "output_influence": 0.18616537749767303, "residual_influence": 0.22313754260540009, "state_active": 0.0699978843331337, "state_memory": 0.046560078859329224, "state_peak": 43.973567962646484, "growth_active": 1.2658154964447021, "growth_memory": 1.2394318580627441, "heads": [{"head": 0, "memory_to_active": 2.325161933898926, "contribution_to_active": 0.4036158323287964, "mix_mean": 0.019787762314081192, "mix_low": 0.614100456237793, "mix_high": 0.0, "read_cosine": 0.2434549629688263, "beta_mean": 0.20174159109592438, "nonfinite": 0.0, "weighted_memory_to_active": 0.5035626292228699, "mix_std": 0.05727162957191467, "beta_std": 0.0991230309009552, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03897032514214516, "evidence_weights": [-0.019860688596963882, -0.06704091280698776, -0.09802507609128952], "agreement_weight": -0.13791030645370483, "orthogonal_weight": -0.0883866548538208}, {"head": 1, "memory_to_active": 5.535592079162598, "contribution_to_active": 2.0831074714660645, "mix_mean": 0.015158731490373611, "mix_low": 0.6985321044921875, "mix_high": 0.0, "read_cosine": 0.5229402780532837, "beta_mean": 0.41609030961990356, "nonfinite": 0.0, "weighted_memory_to_active": 2.390087842941284, "mix_std": 0.048996880650520325, "beta_std": 0.20752927660942078, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07636313885450363, "evidence_weights": [0.03451545163989067, -0.0955309048295021, 0.4433300495147705], "agreement_weight": 0.025858143344521523, "orthogonal_weight": -0.0672711506485939}, {"head": 2, "memory_to_active": 2.572981595993042, "contribution_to_active": 0.4099045395851135, "mix_mean": 0.012384278699755669, "mix_low": 0.6970224380493164, "mix_high": 0.0, "read_cosine": 0.3366837203502655, "beta_mean": 0.2925417423248291, "nonfinite": 0.0, "weighted_memory_to_active": 0.48991501331329346, "mix_std": 0.027820037677884102, "beta_std": 0.2133566290140152, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04910105839371681, "evidence_weights": [0.06574521958827972, 0.07092102617025375, 0.049086228013038635], "agreement_weight": 0.005977022461593151, "orthogonal_weight": -0.21987393498420715}, {"head": 3, "memory_to_active": 6.753559112548828, "contribution_to_active": 2.654575824737549, "mix_mean": 0.03548798710107803, "mix_low": 0.4680347442626953, "mix_high": 0.0, "read_cosine": 0.3550501763820648, "beta_mean": 0.3469659090042114, "nonfinite": 0.0, "weighted_memory_to_active": 2.7983181476593018, "mix_std": 0.08012602478265762, "beta_std": 0.24259638786315918, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1033501997590065, "evidence_weights": [-0.04606234282255173, -0.06185314804315567, -0.16539770364761353], "agreement_weight": -0.015977023169398308, "orthogonal_weight": -0.043482597917318344}, {"head": 4, "memory_to_active": 2.462470531463623, "contribution_to_active": 0.06300149857997894, "mix_mean": 0.006987493950873613, "mix_low": 0.7980384826660156, "mix_high": 0.0, "read_cosine": 0.3330489993095398, "beta_mean": 0.1659114956855774, "nonfinite": 0.0, "weighted_memory_to_active": 0.05288655310869217, "mix_std": 0.013168946839869022, "beta_std": 0.1141141876578331, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.005499459803104401, "evidence_weights": [0.1268921196460724, 0.07992648333311081, -0.3338336646556854], "agreement_weight": -0.11964912712574005, "orthogonal_weight": 0.10682462900876999}, {"head": 5, "memory_to_active": 0.10275913774967194, "contribution_to_active": 0.4200276732444763, "mix_mean": 0.5111274123191833, "mix_low": 0.019995689392089844, "mix_high": 0.026602745056152344, "read_cosine": 0.9182171821594238, "beta_mean": 0.030988769605755806, "nonfinite": 0.0, "weighted_memory_to_active": 0.05294016748666763, "mix_std": 0.2905723452568054, "beta_std": 0.025530453771352768, "beta_low": 0.15765380859375, "beta_high": 0.0, "orthogonal_to_active": 0.019610339775681496, "evidence_weights": [-0.2160395383834839, -0.2313728928565979, 0.32903316617012024], "agreement_weight": 0.2739730775356293, "orthogonal_weight": -0.67377769947052}]}, {"layer": 1, "memory_to_active": 2.271169900894165, "contribution_to_active": 0.5073416233062744, "mix_mean": 0.19228756427764893, "mix_low": 0.3649093508720398, "mix_high": 0.0008489291067235172, "read_cosine": 0.34477922320365906, "beta_mean": 0.17907115817070007, "nonfinite": 0.0, "weighted_memory_to_active": 0.3556627631187439, "mix_std": 0.11888153851032257, "beta_std": 0.1315554678440094, "beta_low": 0.28289794921875, "beta_high": 7.120767986634746e-05, "orthogonal_to_active": 0.05089757591485977, "output_influence": 0.2005588412284851, "residual_influence": 0.050599440932273865, "state_active": 0.042080219835042953, "state_memory": 0.03596332669258118, "state_peak": 35.298362731933594, "growth_active": 1.4131768941879272, "growth_memory": 1.5159295797348022, "heads": [{"head": 0, "memory_to_active": 0.05722460895776749, "contribution_to_active": 0.47192275524139404, "mix_mean": 0.45673465728759766, "mix_low": 0.008287429809570312, "mix_high": 0.0013751983642578125, "read_cosine": 0.057853154838085175, "beta_mean": 0.00041678003617562354, "nonfinite": 0.0, "weighted_memory_to_active": 0.031530268490314484, "mix_std": 0.2858324646949768, "beta_std": 0.0006674114265479147, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.028414450585842133, "evidence_weights": [-0.19312669336795807, -0.2497195303440094, 0.21149949729442596], "agreement_weight": 0.1691264510154724, "orthogonal_weight": -0.9739406704902649}, {"head": 1, "memory_to_active": 2.8285176753997803, "contribution_to_active": 0.1966908574104309, "mix_mean": 0.02278502844274044, "mix_low": 0.47518348693847656, "mix_high": 1.9073486328125e-06, "read_cosine": 0.5198565721511841, "beta_mean": 0.2656695246696472, "nonfinite": 0.0, "weighted_memory_to_active": 0.2233351618051529, "mix_std": 0.05051596462726593, "beta_std": 0.18333187699317932, "beta_low": 0.0013427734375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.013460892252624035, "evidence_weights": [0.01425192691385746, -0.031984589993953705, 0.006804622244089842], "agreement_weight": -0.008277847431600094, "orthogonal_weight": 0.13633479177951813}, {"head": 2, "memory_to_active": 0.19903643429279327, "contribution_to_active": 0.6062941551208496, "mix_mean": 0.6102579832077026, "mix_low": 0.0011014938354492188, "mix_high": 0.0037126541137695312, "read_cosine": 0.6874598264694214, "beta_mean": 0.011373667046427727, "nonfinite": 0.0, "weighted_memory_to_active": 0.16432753205299377, "mix_std": 0.24459706246852875, "beta_std": 0.01200883649289608, "beta_low": 0.62835693359375, "beta_high": 0.0, "orthogonal_to_active": 0.04713664948940277, "evidence_weights": [-0.3021264672279358, -0.34745386242866516, 0.47466549277305603], "agreement_weight": 0.028128551319241524, "orthogonal_weight": -0.4713875651359558}, {"head": 3, "memory_to_active": 2.722038745880127, "contribution_to_active": 0.16120494902133942, "mix_mean": 0.02047136425971985, "mix_low": 0.6125068664550781, "mix_high": 0.0, "read_cosine": 0.25066399574279785, "beta_mean": 0.2983344495296478, "nonfinite": 0.0, "weighted_memory_to_active": 0.2465321123600006, "mix_std": 0.05550223961472511, "beta_std": 0.2424643635749817, "beta_low": 0.03448486328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.07513546943664551, "evidence_weights": [0.042853131890296936, -0.007945787161588669, 0.11287281662225723], "agreement_weight": -0.01607554964721203, "orthogonal_weight": -0.3708740770816803}, {"head": 4, "memory_to_active": 3.710735321044922, "contribution_to_active": 1.5352704524993896, "mix_mean": 0.03654167428612709, "mix_low": 0.2660350799560547, "mix_high": 3.814697265625e-06, "read_cosine": 0.33112987875938416, "beta_mean": 0.17806288599967957, "nonfinite": 0.0, "weighted_memory_to_active": 1.4103362560272217, "mix_std": 0.06160677969455719, "beta_std": 0.12807545065879822, "beta_low": 0.011474609375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.13214410841464996, "evidence_weights": [-0.0420939177274704, -0.002333990065380931, 0.030128024518489838], "agreement_weight": -0.017798621207475662, "orthogonal_weight": 0.12181883305311203}, {"head": 5, "memory_to_active": 4.109467506408691, "contribution_to_active": 0.07266660779714584, "mix_mean": 0.006934651639312506, "mix_low": 0.8263416290283203, "mix_high": 0.0, "read_cosine": 0.2217116355895996, "beta_mean": 0.32056963443756104, "nonfinite": 0.0, "weighted_memory_to_active": 0.057915087789297104, "mix_std": 0.01523474883288145, "beta_std": 0.22278471291065216, "beta_low": 0.021728515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.00909388530999422, "evidence_weights": [0.1652776449918747, 0.0838971734046936, 0.24685131013393402], "agreement_weight": -0.1491841822862625, "orthogonal_weight": 0.15270169079303741}]}, {"layer": 2, "memory_to_active": 1.0608655214309692, "contribution_to_active": 0.3639676570892334, "mix_mean": 0.3649917244911194, "mix_low": 0.25528019666671753, "mix_high": 0.0032660167198628187, "read_cosine": 0.32584255933761597, "beta_mean": 0.09527407586574554, "nonfinite": 0.0, "weighted_memory_to_active": 0.08896704018115997, "mix_std": 0.1938425898551941, "beta_std": 0.07211791723966599, "beta_low": 0.5566508769989014, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.017298348248004913, "output_influence": 0.3518763780593872, "residual_influence": 0.08793854713439941, "state_active": 0.040380675345659256, "state_memory": 0.017669226974248886, "state_peak": 35.374881744384766, "growth_active": 1.1082831621170044, "growth_memory": 1.5909194946289062, "heads": [{"head": 0, "memory_to_active": 0.009344201534986496, "contribution_to_active": 0.3564267158508301, "mix_mean": 0.5591086149215698, "mix_low": 0.0022745132446289062, "mix_high": 0.005423545837402344, "read_cosine": 0.17177902162075043, "beta_mean": 6.577444582944736e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005399099085479975, "mix_std": 0.28900936245918274, "beta_std": 8.307643292937428e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0034991914872080088, "evidence_weights": [-0.32890963554382324, -0.20129066705703735, 0.08010705560445786], "agreement_weight": 0.2269626259803772, "orthogonal_weight": -0.6795478463172913}, {"head": 1, "memory_to_active": 2.9583959579467773, "contribution_to_active": 0.0482628308236599, "mix_mean": 0.005507409106940031, "mix_low": 0.8881540298461914, "mix_high": 0.0, "read_cosine": 0.1920054852962494, "beta_mean": 0.301451176404953, "nonfinite": 0.0, "weighted_memory_to_active": 0.035663530230522156, "mix_std": 0.0134386345744133, "beta_std": 0.16760988533496857, "beta_low": 0.00048828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.014869875274598598, "evidence_weights": [0.10999059677124023, 0.10020524263381958, 0.5048280954360962], "agreement_weight": 0.036837510764598846, "orthogonal_weight": 0.4272790253162384}, {"head": 2, "memory_to_active": 0.01794394478201866, "contribution_to_active": 0.42744511365890503, "mix_mean": 0.4961712956428528, "mix_low": 0.0018987655639648438, "mix_high": 0.0018863677978515625, "read_cosine": 0.3068111538887024, "beta_mean": 4.0372120565734804e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006990061141550541, "mix_std": 0.2782321274280548, "beta_std": 9.807680908124894e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011923573911190033, "evidence_weights": [-0.2394724041223526, -0.2758447527885437, 0.45984217524528503], "agreement_weight": 0.031065475195646286, "orthogonal_weight": -0.18736015260219574}, {"head": 3, "memory_to_active": 0.1594906449317932, "contribution_to_active": 0.5039581060409546, "mix_mean": 0.500768780708313, "mix_low": 0.00067901611328125, "mix_high": 0.005341529846191406, "read_cosine": 0.6696119904518127, "beta_mean": 0.03146887570619583, "nonfinite": 0.0, "weighted_memory_to_active": 0.08985155075788498, "mix_std": 0.26511090993881226, "beta_std": 0.038093678653240204, "beta_low": 0.308837890625, "beta_high": 0.0, "orthogonal_to_active": 0.05628738924860954, "evidence_weights": [-0.18182314932346344, -0.2858830690383911, 0.8573352098464966], "agreement_weight": 0.05871497839689255, "orthogonal_weight": -0.8923636078834534}, {"head": 4, "memory_to_active": 3.2059688568115234, "contribution_to_active": 0.3234005868434906, "mix_mean": 0.017104800790548325, "mix_low": 0.6372461318969727, "mix_high": 0.0, "read_cosine": 0.18942902982234955, "beta_mean": 0.23857668042182922, "nonfinite": 0.0, "weighted_memory_to_active": 0.3871783912181854, "mix_std": 0.04665140062570572, "beta_std": 0.22674402594566345, "beta_low": 0.03057861328125, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.027672626078128815, "evidence_weights": [0.022438092157244682, 0.014864344149827957, 0.04945335164666176], "agreement_weight": 0.019724344834685326, "orthogonal_weight": -0.08140918612480164}, {"head": 5, "memory_to_active": 0.014049333520233631, "contribution_to_active": 0.524312436580658, "mix_mean": 0.6112891435623169, "mix_low": 0.0014286041259765625, "mix_high": 0.0069446563720703125, "read_cosine": 0.4254188537597656, "beta_mean": 4.156588693149388e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008719626814126968, "mix_std": 0.2706129848957062, "beta_std": 7.874108268879354e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00026864654500968754, "evidence_weights": [-0.27155083417892456, -0.2883792221546173, 0.9146555662155151], "agreement_weight": 0.11967261880636215, "orthogonal_weight": 0.046841099858284}]}, {"layer": 3, "memory_to_active": 0.6202760934829712, "contribution_to_active": 0.42268869280815125, "mix_mean": 0.40326064825057983, "mix_low": 0.09108670800924301, "mix_high": 0.00857400894165039, "read_cosine": 0.2141648232936859, "beta_mean": 0.038965579122304916, "nonfinite": 0.0, "weighted_memory_to_active": 0.16102206707000732, "mix_std": 0.2573099136352539, "beta_std": 0.03280781954526901, "beta_low": 0.6423238515853882, "beta_high": 0.0, "orthogonal_to_active": 0.072896808385849, "output_influence": 0.3576303720474243, "residual_influence": 0.0729084238409996, "state_active": 0.023587267845869064, "state_memory": 0.009003766812384129, "state_peak": 18.186357498168945, "growth_active": 1.0240609645843506, "growth_memory": 1.8172626495361328, "heads": [{"head": 0, "memory_to_active": 0.48673734068870544, "contribution_to_active": 0.5453349947929382, "mix_mean": 0.5119202136993408, "mix_low": 0.007891654968261719, "mix_high": 0.015557289123535156, "read_cosine": 0.23915405571460724, "beta_mean": 0.015080476179718971, "nonfinite": 0.0, "weighted_memory_to_active": 0.29139646887779236, "mix_std": 0.31906601786613464, "beta_std": 0.02702321484684944, "beta_low": 0.59033203125, "beta_high": 0.0, "orthogonal_to_active": 0.1976621448993683, "evidence_weights": [-0.19516155123710632, -0.22594986855983734, 0.8033377528190613], "agreement_weight": 0.12046324461698532, "orthogonal_weight": -0.7164761424064636}, {"head": 1, "memory_to_active": 0.01959903910756111, "contribution_to_active": 0.3971734344959259, "mix_mean": 0.49580812454223633, "mix_low": 0.009677886962890625, "mix_high": 0.007860183715820312, "read_cosine": 0.18884006142616272, "beta_mean": 0.00010143198596779257, "nonfinite": 0.0, "weighted_memory_to_active": 0.01097662840038538, "mix_std": 0.31427085399627686, "beta_std": 0.00018114586418960243, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007906284183263779, "evidence_weights": [-0.23096522688865662, -0.19583410024642944, 0.612154483795166], "agreement_weight": 0.3943314254283905, "orthogonal_weight": -0.7715931534767151}, {"head": 2, "memory_to_active": 2.626429319381714, "contribution_to_active": 0.30566808581352234, "mix_mean": 0.04675398766994476, "mix_low": 0.3640298843383789, "mix_high": 0.0, "read_cosine": 0.18906185030937195, "beta_mean": 0.19286632537841797, "nonfinite": 0.0, "weighted_memory_to_active": 0.3620581030845642, "mix_std": 0.1020832359790802, "beta_std": 0.14378702640533447, "beta_low": 0.0089111328125, "beta_high": 0.0, "orthogonal_to_active": 0.0034156145993620157, "evidence_weights": [-0.07488102465867996, 0.013772345148026943, -0.022615808993577957], "agreement_weight": -0.009980873204767704, "orthogonal_weight": -0.012256642803549767}, {"head": 3, "memory_to_active": 0.02389460615813732, "contribution_to_active": 0.5057034492492676, "mix_mean": 0.5613972544670105, "mix_low": 0.008642196655273438, "mix_high": 0.010271072387695312, "read_cosine": 0.2863582372665405, "beta_mean": 9.140811016550288e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015814246609807014, "mix_std": 0.3023760914802551, "beta_std": 0.00024066015612334013, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003856044728308916, "evidence_weights": [-0.22347195446491241, -0.22779732942581177, 2.0343801975250244], "agreement_weight": 1.5891185998916626, "orthogonal_weight": -0.31626853346824646}, {"head": 4, "memory_to_active": 0.2667008638381958, "contribution_to_active": 0.15059402585029602, "mix_mean": 0.16249984502792358, "mix_low": 0.1489696502685547, "mix_high": 0.00020122528076171875, "read_cosine": 0.17212702333927155, "beta_mean": 0.003010007319971919, "nonfinite": 0.0, "weighted_memory_to_active": 0.07746873795986176, "mix_std": 0.21669739484786987, "beta_std": 0.004439796321094036, "beta_low": 0.9320068359375, "beta_high": 0.0, "orthogonal_to_active": 0.06483591347932816, "evidence_weights": [-0.027475031092762947, -0.0767945647239685, 0.2510969936847687], "agreement_weight": 0.00798592809587717, "orthogonal_weight": -0.8653674721717834}, {"head": 5, "memory_to_active": 0.2982955873012543, "contribution_to_active": 0.6316580176353455, "mix_mean": 0.6411845684051514, "mix_low": 0.0073089599609375, "mix_high": 0.017554283142089844, "read_cosine": 0.2094477117061615, "beta_mean": 0.02264382503926754, "nonfinite": 0.0, "weighted_memory_to_active": 0.20841820538043976, "mix_std": 0.28936567902565, "beta_std": 0.021175067871809006, "beta_low": 0.32269287109375, "beta_high": 0.0, "orthogonal_to_active": 0.15970486402511597, "evidence_weights": [-0.24584253132343292, -0.2846413850784302, 0.5132623314857483], "agreement_weight": 0.127552330493927, "orthogonal_weight": -0.81817626953125}]}, {"layer": 4, "memory_to_active": 1.168236255645752, "contribution_to_active": 0.43211811780929565, "mix_mean": 0.348240464925766, "mix_low": 0.2931615710258484, "mix_high": 0.006112734787166119, "read_cosine": 0.3500879406929016, "beta_mean": 0.12368591129779816, "nonfinite": 0.0, "weighted_memory_to_active": 0.12587159872055054, "mix_std": 0.15156251192092896, "beta_std": 0.11070872843265533, "beta_low": 0.3715311884880066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03911445662379265, "output_influence": 0.40823835134506226, "residual_influence": 0.11118277907371521, "state_active": 0.03729365020990372, "state_memory": 0.019282080233097076, "state_peak": 42.818138122558594, "growth_active": 1.2057263851165771, "growth_memory": 1.5564990043640137, "heads": [{"head": 0, "memory_to_active": 0.2160191833972931, "contribution_to_active": 0.7005118727684021, "mix_mean": 0.6538003087043762, "mix_low": 0.010896682739257812, "mix_high": 0.013516426086425781, "read_cosine": 0.49946194887161255, "beta_mean": 0.25544166564941406, "nonfinite": 0.0, "weighted_memory_to_active": 0.16238363087177277, "mix_std": 0.33410942554473877, "beta_std": 0.2638639211654663, "beta_low": 0.2127685546875, "beta_high": 0.0, "orthogonal_to_active": 0.0579545721411705, "evidence_weights": [-0.2667955160140991, -0.36656174063682556, 0.1756685972213745], "agreement_weight": 0.017079195007681847, "orthogonal_weight": 0.43903449177742004}, {"head": 1, "memory_to_active": 0.06752301007509232, "contribution_to_active": 0.6138108968734741, "mix_mean": 0.649985671043396, "mix_low": 5.245208740234375e-05, "mix_high": 0.008553504943847656, "read_cosine": 0.6823487281799316, "beta_mean": 0.0011324459919705987, "nonfinite": 0.0, "weighted_memory_to_active": 0.047395966947078705, "mix_std": 0.2465268224477768, "beta_std": 0.004759883042424917, "beta_low": 0.9820556640625, "beta_high": 0.0, "orthogonal_to_active": 0.026737041771411896, "evidence_weights": [-0.23708383738994598, -0.33215034008026123, 0.7179110050201416], "agreement_weight": 0.1720726042985916, "orthogonal_weight": -0.8327091932296753}, {"head": 2, "memory_to_active": 2.287492275238037, "contribution_to_active": 0.21611902117729187, "mix_mean": 0.017710017040371895, "mix_low": 0.6109466552734375, "mix_high": 1.9073486328125e-05, "read_cosine": 0.24194401502609253, "beta_mean": 0.10894818603992462, "nonfinite": 0.0, "weighted_memory_to_active": 0.19925081729888916, "mix_std": 0.038281626999378204, "beta_std": 0.09142806380987167, "beta_low": 0.00439453125, "beta_high": 0.0, "orthogonal_to_active": 0.019649183377623558, "evidence_weights": [0.06089719384908676, 0.03159743547439575, 0.013165363110601902], "agreement_weight": 0.12912729382514954, "orthogonal_weight": -0.14631348848342896}, {"head": 3, "memory_to_active": 2.005816698074341, "contribution_to_active": 0.1054607480764389, "mix_mean": 0.02588304877281189, "mix_low": 0.45644187927246094, "mix_high": 0.0, "read_cosine": 0.22012834250926971, "beta_mean": 0.15661180019378662, "nonfinite": 0.0, "weighted_memory_to_active": 0.15197798609733582, "mix_std": 0.04407500848174095, "beta_std": 0.13358135521411896, "beta_low": 0.02996826171875, "beta_high": 0.0, "orthogonal_to_active": 0.007000780664384365, "evidence_weights": [-0.01795320026576519, 0.0627298429608345, -0.2522062659263611], "agreement_weight": 0.06926780194044113, "orthogonal_weight": -0.06520925462245941}, {"head": 4, "memory_to_active": 0.023930495604872704, "contribution_to_active": 0.6521636247634888, "mix_mean": 0.7300324440002441, "mix_low": 0.00013446807861328125, "mix_high": 0.014585494995117188, "read_cosine": 0.17762070894241333, "beta_mean": 4.525567783275619e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.018022892996668816, "mix_std": 0.21996508538722992, "beta_std": 0.00013926232350058854, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004232125822454691, "evidence_weights": [-0.3263184130191803, -0.3316356837749481, 0.6473341584205627], "agreement_weight": 0.2824046313762665, "orthogonal_weight": -0.2558706998825073}, {"head": 5, "memory_to_active": 2.4086356163024902, "contribution_to_active": 0.3046424984931946, "mix_mean": 0.012031307443976402, "mix_low": 0.6804971694946289, "mix_high": 1.9073486328125e-06, "read_cosine": 0.2790239155292511, "beta_mean": 0.21993610262870789, "nonfinite": 0.0, "weighted_memory_to_active": 0.17619825899600983, "mix_std": 0.0264170840382576, "beta_std": 0.17047986388206482, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.119113028049469, "evidence_weights": [0.051705457270145416, -0.030657894909381866, -0.2689005434513092], "agreement_weight": -0.05985710397362709, "orthogonal_weight": 0.8682177066802979}]}, {"layer": 5, "memory_to_active": 0.46163564920425415, "contribution_to_active": 0.49210357666015625, "mix_mean": 0.4567207396030426, "mix_low": 0.07011286914348602, "mix_high": 0.015309016220271587, "read_cosine": 0.09858384728431702, "beta_mean": 0.05766909942030907, "nonfinite": 0.0, "weighted_memory_to_active": 0.14557510614395142, "mix_std": 0.25192415714263916, "beta_std": 0.07278132438659668, "beta_low": 0.628143310546875, "beta_high": 0.0, "orthogonal_to_active": 0.051676057279109955, "output_influence": 0.4916248917579651, "residual_influence": 0.13012003898620605, "state_active": 0.0432337187230587, "state_memory": 0.011233782395720482, "state_peak": 63.96430969238281, "growth_active": 1.081746220588684, "growth_memory": 1.261224389076233, "heads": [{"head": 0, "memory_to_active": 0.6730164885520935, "contribution_to_active": 0.4116172790527344, "mix_mean": 0.271446168422699, "mix_low": 0.04798412322998047, "mix_high": 0.0003204345703125, "read_cosine": -0.11141877621412277, "beta_mean": 0.036950886249542236, "nonfinite": 0.0, "weighted_memory_to_active": 0.16678519546985626, "mix_std": 0.2496044933795929, "beta_std": 0.05750643461942673, "beta_low": 0.33074951171875, "beta_high": 0.0, "orthogonal_to_active": 0.04082740470767021, "evidence_weights": [-0.10769189894199371, -0.13882237672805786, -0.17579223215579987], "agreement_weight": -0.01460378710180521, "orthogonal_weight": -0.2440025359392166}, {"head": 1, "memory_to_active": 0.021929796785116196, "contribution_to_active": 0.5319439768791199, "mix_mean": 0.652793288230896, "mix_low": 0.0023927688598632812, "mix_high": 0.024926185607910156, "read_cosine": 0.16762977838516235, "beta_mean": 4.926943074679002e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012425393797457218, "mix_std": 0.2886609435081482, "beta_std": 0.00010409186506876722, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01155396830290556, "evidence_weights": [-0.23993030190467834, -0.27880579233169556, 0.9192178249359131], "agreement_weight": 0.562497079372406, "orthogonal_weight": -0.9596037864685059}, {"head": 2, "memory_to_active": 0.017081961035728455, "contribution_to_active": 0.5558366775512695, "mix_mean": 0.6345885992050171, "mix_low": 0.0029230117797851562, "mix_high": 0.019527435302734375, "read_cosine": -0.32765379548072815, "beta_mean": 0.0001369531819364056, "nonfinite": 0.0, "weighted_memory_to_active": 0.010978550650179386, "mix_std": 0.27771398425102234, "beta_std": 0.0002760301867965609, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025314295198768377, "evidence_weights": [-0.2508457899093628, -0.3447057008743286, 0.8119235038757324], "agreement_weight": 0.0027817392256110907, "orthogonal_weight": -0.27333372831344604}, {"head": 3, "memory_to_active": 1.5318604707717896, "contribution_to_active": 0.2260887771844864, "mix_mean": 0.09267193078994751, "mix_low": 0.16119670867919922, "mix_high": 0.0, "read_cosine": 0.1616428643465042, "beta_mean": 0.11055225878953934, "nonfinite": 0.0, "weighted_memory_to_active": 0.33920618891716003, "mix_std": 0.13096211850643158, "beta_std": 0.1523619443178177, "beta_low": 0.17431640625, "beta_high": 0.0, "orthogonal_to_active": 0.12732625007629395, "evidence_weights": [-0.06255479902029037, -0.08987537771463394, -0.024335244670510292], "agreement_weight": 0.06195451319217682, "orthogonal_weight": -0.46250292658805847}, {"head": 4, "memory_to_active": 0.4182772636413574, "contribution_to_active": 0.5719276070594788, "mix_mean": 0.30435046553611755, "mix_low": 0.20330429077148438, "mix_high": 0.0061664581298828125, "read_cosine": 0.5559894442558289, "beta_mean": 0.1980002224445343, "nonfinite": 0.0, "weighted_memory_to_active": 0.2719860374927521, "mix_std": 0.3294624388217926, "beta_std": 0.2248319536447525, "beta_low": 0.26763916015625, "beta_high": 0.0, "orthogonal_to_active": 0.0813145637512207, "evidence_weights": [-0.1092657744884491, -0.2775202989578247, -0.06164773553609848], "agreement_weight": 0.03763487562537193, "orthogonal_weight": 0.494497686624527}, {"head": 5, "memory_to_active": 0.10764792561531067, "contribution_to_active": 0.6552072763442993, "mix_mean": 0.7844740152359009, "mix_low": 0.00287628173828125, "mix_high": 0.04091358184814453, "read_cosine": 0.14531353116035461, "beta_mean": 0.00032498021028004587, "nonfinite": 0.0, "weighted_memory_to_active": 0.07206928730010986, "mix_std": 0.23514100909233093, "beta_std": 0.0016074569430202246, "beta_low": 0.99615478515625, "beta_high": 0.0, "orthogonal_to_active": 0.046502675861120224, "evidence_weights": [-0.32371827960014343, -0.32686513662338257, 1.9808781147003174], "agreement_weight": 0.4134708344936371, "orthogonal_weight": -0.6797657012939453}]}, {"layer": 6, "memory_to_active": 0.33752989768981934, "contribution_to_active": 0.5892418622970581, "mix_mean": 0.5612134337425232, "mix_low": 0.06965367496013641, "mix_high": 0.029136817902326584, "read_cosine": 0.28805011510849, "beta_mean": 0.072180837392807, "nonfinite": 0.0, "weighted_memory_to_active": 0.15395522117614746, "mix_std": 0.2394898235797882, "beta_std": 0.08130320906639099, "beta_low": 0.5006510615348816, "beta_high": 0.0, "orthogonal_to_active": 0.03549104928970337, "output_influence": 0.5067489147186279, "residual_influence": 0.16731825470924377, "state_active": 0.046926990151405334, "state_memory": 0.00846666470170021, "state_peak": 85.33049774169922, "growth_active": 1.178452968597412, "growth_memory": 1.1966334581375122, "heads": [{"head": 0, "memory_to_active": 0.10396143049001694, "contribution_to_active": 0.6351038813591003, "mix_mean": 0.7534290552139282, "mix_low": 0.000179290771484375, "mix_high": 0.016844749450683594, "read_cosine": 0.13344493508338928, "beta_mean": 0.01785111241042614, "nonfinite": 0.0, "weighted_memory_to_active": 0.08028657734394073, "mix_std": 0.22203421592712402, "beta_std": 0.02218274585902691, "beta_low": 0.4683837890625, "beta_high": 0.0, "orthogonal_to_active": 0.06859595328569412, "evidence_weights": [-0.31618961691856384, -0.3366028666496277, 0.7976839542388916], "agreement_weight": 0.23891328275203705, "orthogonal_weight": -0.9128066301345825}, {"head": 1, "memory_to_active": 0.007067929022014141, "contribution_to_active": 0.4692230820655823, "mix_mean": 0.5362741351127625, "mix_low": 0.0019617080688476562, "mix_high": 0.009955406188964844, "read_cosine": 0.5559695363044739, "beta_mean": 1.3947862498753238e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0030804516281932592, "mix_std": 0.2888372540473938, "beta_std": 2.1824225768796168e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015456941910088062, "evidence_weights": [-0.19945380091667175, -0.29270538687705994, 1.0935500860214233], "agreement_weight": 0.05770478770136833, "orthogonal_weight": -0.7068275213241577}, {"head": 2, "memory_to_active": 1.034482479095459, "contribution_to_active": 0.08223626017570496, "mix_mean": 0.03054143860936165, "mix_low": 0.4046030044555664, "mix_high": 0.0, "read_cosine": 0.2301752269268036, "beta_mean": 0.054305002093315125, "nonfinite": 0.0, "weighted_memory_to_active": 0.06113925948739052, "mix_std": 0.05074986815452576, "beta_std": 0.05157984420657158, "beta_low": 0.040771484375, "beta_high": 0.0, "orthogonal_to_active": 0.0005611516535282135, "evidence_weights": [0.0892440527677536, -0.013791786506772041, 0.2417808473110199], "agreement_weight": 0.14147545397281647, "orthogonal_weight": 0.010963506996631622}, {"head": 3, "memory_to_active": 0.16020095348358154, "contribution_to_active": 0.6831056475639343, "mix_mean": 0.6444168090820312, "mix_low": 0.009790420532226562, "mix_high": 0.017120361328125, "read_cosine": 0.42238718271255493, "beta_mean": 0.22212286293506622, "nonfinite": 0.0, "weighted_memory_to_active": 0.11412474513053894, "mix_std": 0.32909563183784485, "beta_std": 0.2541409432888031, "beta_low": 0.22137451171875, "beta_high": 0.0, "orthogonal_to_active": 0.06408185511827469, "evidence_weights": [-0.17428548634052277, -0.3351747393608093, 0.20064063370227814], "agreement_weight": -0.009070378728210926, "orthogonal_weight": 0.6432653665542603}, {"head": 4, "memory_to_active": 0.6839849948883057, "contribution_to_active": 1.0638201236724854, "mix_mean": 0.7597558498382568, "mix_low": 0.0009326934814453125, "mix_high": 0.12712478637695312, "read_cosine": 0.10948209464550018, "beta_mean": 0.1385534107685089, "nonfinite": 0.0, "weighted_memory_to_active": 0.6403247117996216, "mix_std": 0.28829503059387207, "beta_std": 0.15942545235157013, "beta_low": 0.27349853515625, "beta_high": 0.0, "orthogonal_to_active": 0.05349995195865631, "evidence_weights": [-0.24439775943756104, -0.5923915505409241, -0.1727733612060547], "agreement_weight": -0.015908073633909225, "orthogonal_weight": 0.09655871242284775}, {"head": 5, "memory_to_active": 0.035481683909893036, "contribution_to_active": 0.6019622087478638, "mix_mean": 0.6428632736206055, "mix_low": 0.00045490264892578125, "mix_high": 0.0037755966186523438, "read_cosine": 0.27684175968170166, "beta_mean": 0.0002386240812484175, "nonfinite": 0.0, "weighted_memory_to_active": 0.024775542318820953, "mix_std": 0.25792697072029114, "beta_std": 0.0004684261803049594, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.02466166950762272, "evidence_weights": [-0.24313490092754364, -0.3110847771167755, 0.44576388597488403], "agreement_weight": 0.14529307186603546, "orthogonal_weight": -1.0673366785049438}]}, {"layer": 7, "memory_to_active": 1.797821044921875, "contribution_to_active": 0.25129979848861694, "mix_mean": 0.11566727608442307, "mix_low": 0.27243566513061523, "mix_high": 0.00024525326443836093, "read_cosine": 0.2385433465242386, "beta_mean": 0.13281714916229248, "nonfinite": 0.0, "weighted_memory_to_active": 0.22780153155326843, "mix_std": 0.13064604997634888, "beta_std": 0.10196280479431152, "beta_low": 0.1918843686580658, "beta_high": 0.00019327799964230508, "orthogonal_to_active": 0.06442727148532867, "output_influence": 0.2439841628074646, "residual_influence": 0.05525604635477066, "state_active": 0.01399835105985403, "state_memory": 0.02704571560025215, "state_peak": 29.544309616088867, "growth_active": 1.137067198753357, "growth_memory": 1.2961320877075195, "heads": [{"head": 0, "memory_to_active": 2.4668877124786377, "contribution_to_active": 0.3737739324569702, "mix_mean": 0.037431150674819946, "mix_low": 0.41031932830810547, "mix_high": 0.0, "read_cosine": 0.36600199341773987, "beta_mean": 0.15074148774147034, "nonfinite": 0.0, "weighted_memory_to_active": 0.429806649684906, "mix_std": 0.07762573659420013, "beta_std": 0.11175168305635452, "beta_low": 0.00091552734375, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.008625729009509087, "evidence_weights": [0.015705544501543045, -0.0819944366812706, 0.09902320057153702], "agreement_weight": -0.03450903296470642, "orthogonal_weight": 0.03304470330476761}, {"head": 1, "memory_to_active": 1.1122068166732788, "contribution_to_active": 0.38955751061439514, "mix_mean": 0.19584912061691284, "mix_low": 0.13967037200927734, "mix_high": 0.00122833251953125, "read_cosine": 0.35322892665863037, "beta_mean": 0.11044902354478836, "nonfinite": 0.0, "weighted_memory_to_active": 0.3566928803920746, "mix_std": 0.22328396141529083, "beta_std": 0.06827758997678757, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.021850710734725, "evidence_weights": [-0.2007661610841751, -0.11588616669178009, -0.1927947849035263], "agreement_weight": 0.029206784442067146, "orthogonal_weight": 0.07235365360975266}, {"head": 2, "memory_to_active": 0.8262636661529541, "contribution_to_active": 0.2885463833808899, "mix_mean": 0.12631024420261383, "mix_low": 0.08941173553466797, "mix_high": 0.000179290771484375, "read_cosine": 0.10175036638975143, "beta_mean": 0.05669328570365906, "nonfinite": 0.0, "weighted_memory_to_active": 0.19652989506721497, "mix_std": 0.16130328178405762, "beta_std": 0.0448833666741848, "beta_low": 0.0538330078125, "beta_high": 0.0, "orthogonal_to_active": 0.0733836218714714, "evidence_weights": [-0.04837988689541817, -0.13401208817958832, 0.5221453905105591], "agreement_weight": 0.0250154547393322, "orthogonal_weight": -0.3832252621650696}, {"head": 3, "memory_to_active": 4.528727054595947, "contribution_to_active": 0.08803407847881317, "mix_mean": 0.0338774248957634, "mix_low": 0.33170604705810547, "mix_high": 0.0, "read_cosine": 0.10941646993160248, "beta_mean": 0.373451828956604, "nonfinite": 0.0, "weighted_memory_to_active": 0.297075092792511, "mix_std": 0.04907328262925148, "beta_std": 0.27731627225875854, "beta_low": 0.00506591796875, "beta_high": 0.000732421875, "orthogonal_to_active": 0.2534128725528717, "evidence_weights": [0.06387800723314285, -0.01025637798011303, 0.67891925573349], "agreement_weight": 0.05150364711880684, "orthogonal_weight": -0.9349563121795654}, {"head": 4, "memory_to_active": 1.7424730062484741, "contribution_to_active": 0.05805947631597519, "mix_mean": 0.01567487232387066, "mix_low": 0.651580810546875, "mix_high": 0.0, "read_cosine": 0.21584251523017883, "beta_mean": 0.10486096143722534, "nonfinite": 0.0, "weighted_memory_to_active": 0.04844146966934204, "mix_std": 0.035935573279857635, "beta_std": 0.10786111652851105, "beta_low": 0.09344482421875, "beta_high": 0.0, "orthogonal_to_active": 0.006901380605995655, "evidence_weights": [0.1415826678276062, 0.046747125685214996, 0.25670698285102844], "agreement_weight": 0.04589548334479332, "orthogonal_weight": -0.15234160423278809}, {"head": 5, "memory_to_active": 0.11036734282970428, "contribution_to_active": 0.3098272681236267, "mix_mean": 0.2848607897758484, "mix_low": 0.011925697326660156, "mix_high": 6.389617919921875e-05, "read_cosine": 0.28501975536346436, "beta_mean": 0.0007063375087454915, "nonfinite": 0.0, "weighted_memory_to_active": 0.03826320171356201, "mix_std": 0.2366543710231781, "beta_std": 0.0016867534723132849, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.02238933928310871, "evidence_weights": [-0.11267741024494171, -0.16999809443950653, 0.4992465376853943], "agreement_weight": 0.11415054649114609, "orthogonal_weight": -0.6727739572525024}]}, {"layer": 8, "memory_to_active": 0.4135218858718872, "contribution_to_active": 0.5217777490615845, "mix_mean": 0.4220237135887146, "mix_low": 0.08625650405883789, "mix_high": 0.0009795825462788343, "read_cosine": 0.4668480157852173, "beta_mean": 0.0285121388733387, "nonfinite": 0.0, "weighted_memory_to_active": 0.0896734744310379, "mix_std": 0.22007325291633606, "beta_std": 0.03517132252454758, "beta_low": 0.6257120966911316, "beta_high": 0.0, "orthogonal_to_active": 0.06261712312698364, "output_influence": 0.458224892616272, "residual_influence": 0.14471837878227234, "state_active": 0.04895613715052605, "state_memory": 0.00903925858438015, "state_peak": 32.644615173339844, "growth_active": 1.0993667840957642, "growth_memory": 1.357776403427124, "heads": [{"head": 0, "memory_to_active": 0.05098356306552887, "contribution_to_active": 0.5483807325363159, "mix_mean": 0.5927484631538391, "mix_low": 0.0003509521484375, "mix_high": 0.0010089874267578125, "read_cosine": 0.5243660807609558, "beta_mean": 0.0027582556940615177, "nonfinite": 0.0, "weighted_memory_to_active": 0.035260625183582306, "mix_std": 0.24404853582382202, "beta_std": 0.008338067680597305, "beta_low": 0.9354248046875, "beta_high": 0.0, "orthogonal_to_active": 0.02159142680466175, "evidence_weights": [-0.17036177217960358, -0.39358100295066833, 0.6981624364852905], "agreement_weight": 0.13404393196105957, "orthogonal_weight": -0.7961737513542175}, {"head": 1, "memory_to_active": 0.13535264134407043, "contribution_to_active": 0.5042406320571899, "mix_mean": 0.4683563709259033, "mix_low": 0.0006656646728515625, "mix_high": 0.0004558563232421875, "read_cosine": 0.26981616020202637, "beta_mean": 0.003096991451457143, "nonfinite": 0.0, "weighted_memory_to_active": 0.06328658759593964, "mix_std": 0.24893374741077423, "beta_std": 0.013138111680746078, "beta_low": 0.95013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.04239680618047714, "evidence_weights": [-0.18982185423374176, -0.26091328263282776, 0.6756545305252075], "agreement_weight": 0.08722621202468872, "orthogonal_weight": -0.7263615131378174}, {"head": 2, "memory_to_active": 0.02359040081501007, "contribution_to_active": 0.6473653316497803, "mix_mean": 0.6317038536071777, "mix_low": 0.0011491775512695312, "mix_high": 0.0029935836791992188, "read_cosine": 0.4054005742073059, "beta_mean": 0.0002393478062003851, "nonfinite": 0.0, "weighted_memory_to_active": 0.01774844527244568, "mix_std": 0.2313327044248581, "beta_std": 0.0030649174004793167, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.011655948124825954, "evidence_weights": [-0.22115272283554077, -0.34106165170669556, 0.9565926194190979], "agreement_weight": 0.2128150761127472, "orthogonal_weight": -0.7569263577461243}, {"head": 3, "memory_to_active": 0.11323272436857224, "contribution_to_active": 0.6212077140808105, "mix_mean": 0.5545830726623535, "mix_low": 0.00201416015625, "mix_high": 0.0011730194091796875, "read_cosine": 0.7944667339324951, "beta_mean": 0.00932970643043518, "nonfinite": 0.0, "weighted_memory_to_active": 0.08396856486797333, "mix_std": 0.25608134269714355, "beta_std": 0.006966856308281422, "beta_low": 0.6353759765625, "beta_high": 0.0, "orthogonal_to_active": 0.028109371662139893, "evidence_weights": [-0.18143698573112488, -0.30369624495506287, 0.6371053457260132], "agreement_weight": 0.1733827441930771, "orthogonal_weight": -0.6448606848716736}, {"head": 4, "memory_to_active": 1.8294754028320312, "contribution_to_active": 0.16350312530994415, "mix_mean": 0.032539159059524536, "mix_low": 0.38845062255859375, "mix_high": 0.0, "read_cosine": 0.33251750469207764, "beta_mean": 0.0792376771569252, "nonfinite": 0.0, "weighted_memory_to_active": 0.1551118791103363, "mix_std": 0.05487605929374695, "beta_std": 0.08348555862903595, "beta_low": 0.06756591796875, "beta_high": 0.0, "orthogonal_to_active": 0.0006270332960411906, "evidence_weights": [0.058004915714263916, -0.030913950875401497, -0.05548201501369476], "agreement_weight": 0.06192058324813843, "orthogonal_weight": 0.004387206397950649}, {"head": 5, "memory_to_active": 0.3284965455532074, "contribution_to_active": 0.645969033241272, "mix_mean": 0.2522112727165222, "mix_low": 0.124908447265625, "mix_high": 0.0002460479736328125, "read_cosine": 0.47452107071876526, "beta_mean": 0.07641084492206573, "nonfinite": 0.0, "weighted_memory_to_active": 0.18266472220420837, "mix_std": 0.28516703844070435, "beta_std": 0.09603440761566162, "beta_low": 0.16607666015625, "beta_high": 0.0, "orthogonal_to_active": 0.27132219076156616, "evidence_weights": [-0.07507000118494034, -0.2414228916168213, -0.2953847646713257], "agreement_weight": 0.003045981517061591, "orthogonal_weight": 1.7514017820358276}]}, {"layer": 9, "memory_to_active": 1.1204265356063843, "contribution_to_active": 0.6642706394195557, "mix_mean": 0.3223338723182678, "mix_low": 0.1427626609802246, "mix_high": 0.0033477148972451687, "read_cosine": 0.3542782962322235, "beta_mean": 0.06124797835946083, "nonfinite": 0.0, "weighted_memory_to_active": 0.38925111293792725, "mix_std": 0.21101593971252441, "beta_std": 0.04099813103675842, "beta_low": 0.5024515986442566, "beta_high": 0.0, "orthogonal_to_active": 0.04233977198600769, "output_influence": 0.49386245012283325, "residual_influence": 0.13615399599075317, "state_active": 0.041075848042964935, "state_memory": 0.019351135939359665, "state_peak": 43.82331466674805, "growth_active": 1.1670242547988892, "growth_memory": 1.3844558000564575, "heads": [{"head": 0, "memory_to_active": 0.004281016532331705, "contribution_to_active": 0.5394982695579529, "mix_mean": 0.6156202554702759, "mix_low": 0.0018129348754882812, "mix_high": 0.014214515686035156, "read_cosine": 0.2227039933204651, "beta_mean": 7.4654508352978155e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002734807785600424, "mix_std": 0.28926220536231995, "beta_std": 3.9056096284184605e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025176783092319965, "evidence_weights": [-0.29655367136001587, -0.3050517141819, 0.8825774192810059], "agreement_weight": 0.10447199642658234, "orthogonal_weight": -0.9605591297149658}, {"head": 1, "memory_to_active": 0.026241634041070938, "contribution_to_active": 0.6394897699356079, "mix_mean": 0.5100879669189453, "mix_low": 0.008329391479492188, "mix_high": 0.001705169677734375, "read_cosine": 0.4034358859062195, "beta_mean": 7.700995774939656e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.017499936744570732, "mix_std": 0.287695050239563, "beta_std": 0.00014338447363115847, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012168947607278824, "evidence_weights": [-0.22454720735549927, -0.2735882103443146, 1.1394708156585693], "agreement_weight": 0.35970425605773926, "orthogonal_weight": -0.8342913389205933}, {"head": 2, "memory_to_active": 0.17678669095039368, "contribution_to_active": 0.6222288608551025, "mix_mean": 0.5594151020050049, "mix_low": 0.001842498779296875, "mix_high": 0.0037889480590820312, "read_cosine": 0.35521960258483887, "beta_mean": 0.0006780526018701494, "nonfinite": 0.0, "weighted_memory_to_active": 0.11682270467281342, "mix_std": 0.27428537607192993, "beta_std": 0.003669177647680044, "beta_low": 0.99560546875, "beta_high": 0.0, "orthogonal_to_active": 0.061832867562770844, "evidence_weights": [-0.19415342807769775, -0.26299864053726196, 0.7130005955696106], "agreement_weight": 0.272829532623291, "orthogonal_weight": -0.6194132566452026}, {"head": 3, "memory_to_active": 1.420103907585144, "contribution_to_active": 0.16449572145938873, "mix_mean": 0.04415920749306679, "mix_low": 0.38177490234375, "mix_high": 0.0, "read_cosine": 0.413028359413147, "beta_mean": 0.07896662503480911, "nonfinite": 0.0, "weighted_memory_to_active": 0.13621315360069275, "mix_std": 0.09605449438095093, "beta_std": 0.053086306899785995, "beta_low": 0.00555419921875, "beta_high": 0.0, "orthogonal_to_active": 0.014024212025105953, "evidence_weights": [0.0787273570895195, 0.02261756733059883, -0.012849414721131325], "agreement_weight": 0.08884389698505402, "orthogonal_weight": 0.11374795436859131}, {"head": 4, "memory_to_active": 3.3630142211914062, "contribution_to_active": 1.3819949626922607, "mix_mean": 0.12855131924152374, "mix_low": 0.14873123168945312, "mix_high": 0.0002651214599609375, "read_cosine": 0.240697979927063, "beta_mean": 0.1822046935558319, "nonfinite": 0.0, "weighted_memory_to_active": 1.3908286094665527, "mix_std": 0.1846681535243988, "beta_std": 0.10255150496959686, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03438940644264221, "evidence_weights": [-0.06576350331306458, -0.15015727281570435, -0.01915542222559452], "agreement_weight": -0.05314934253692627, "orthogonal_weight": -0.026134271174669266}, {"head": 5, "memory_to_active": 1.7321314811706543, "contribution_to_active": 0.6379159092903137, "mix_mean": 0.07616937160491943, "mix_low": 0.3140850067138672, "mix_high": 0.0001125335693359375, "read_cosine": 0.4905838966369629, "beta_mean": 0.1055540069937706, "nonfinite": 0.0, "weighted_memory_to_active": 0.6714073419570923, "mix_std": 0.13413020968437195, "beta_std": 0.08649933338165283, "beta_low": 0.0135498046875, "beta_high": 0.0, "orthogonal_to_active": 0.12910552322864532, "evidence_weights": [0.10383833199739456, -0.07967143505811691, 0.14840862154960632], "agreement_weight": 0.01677561178803444, "orthogonal_weight": 0.27459320425987244}]}, {"layer": 10, "memory_to_active": 0.2500241994857788, "contribution_to_active": 0.659207820892334, "mix_mean": 0.4970617890357971, "mix_low": 0.01588519476354122, "mix_high": 0.005656401626765728, "read_cosine": 0.37417834997177124, "beta_mean": 0.04647953808307648, "nonfinite": 0.0, "weighted_memory_to_active": 0.12151908874511719, "mix_std": 0.2678508460521698, "beta_std": 0.04805155098438263, "beta_low": 0.6613057851791382, "beta_high": 0.0, "orthogonal_to_active": 0.18167895078659058, "output_influence": 0.42284610867500305, "residual_influence": 0.14183229207992554, "state_active": 0.051598742604255676, "state_memory": 0.015461144968867302, "state_peak": 115.84151458740234, "growth_active": 1.0750186443328857, "growth_memory": 0.9480681419372559, "heads": [{"head": 0, "memory_to_active": 0.9889730215072632, "contribution_to_active": 1.7558071613311768, "mix_mean": 0.5962041616439819, "mix_low": 0.053879737854003906, "mix_high": 0.020015716552734375, "read_cosine": 0.08501775562763214, "beta_mean": 0.20323967933654785, "nonfinite": 0.0, "weighted_memory_to_active": 0.6072312593460083, "mix_std": 0.38789981603622437, "beta_std": 0.21895132958889008, "beta_low": 0.13836669921875, "beta_high": 0.0, "orthogonal_to_active": 1.0057876110076904, "evidence_weights": [-0.12343911826610565, -0.3896854817867279, -1.2258068323135376], "agreement_weight": 0.03249264508485794, "orthogonal_weight": 1.6765453815460205}, {"head": 1, "memory_to_active": 0.019548872485756874, "contribution_to_active": 0.5967301726341248, "mix_mean": 0.5996116995811462, "mix_low": 0.0008707046508789062, "mix_high": 0.009290695190429688, "read_cosine": 0.2748929262161255, "beta_mean": 0.00012500966840889305, "nonfinite": 0.0, "weighted_memory_to_active": 0.010889616794884205, "mix_std": 0.26607781648635864, "beta_std": 0.00017276109429076314, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015530141070485115, "evidence_weights": [-0.26670870184898376, -0.2030211091041565, 0.6020493507385254], "agreement_weight": 0.47448039054870605, "orthogonal_weight": -1.5545467138290405}, {"head": 2, "memory_to_active": 0.010822346433997154, "contribution_to_active": 0.5720088481903076, "mix_mean": 0.6026724576950073, "mix_low": 0.000408172607421875, "mix_high": 0.00402069091796875, "read_cosine": 0.05217656120657921, "beta_mean": 2.6539235477684997e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00436296546831727, "mix_std": 0.2576209306716919, "beta_std": 9.159521869150922e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019941728096455336, "evidence_weights": [-0.23702549934387207, -0.27246350049972534, 0.5783809423446655], "agreement_weight": -0.14863497018814087, "orthogonal_weight": -0.435741662979126}, {"head": 3, "memory_to_active": 0.3409022092819214, "contribution_to_active": 0.10126068443059921, "mix_mean": 0.2549075186252594, "mix_low": 0.03722858428955078, "mix_high": 0.00016021728515625, "read_cosine": 0.9023007750511169, "beta_mean": 0.06901034712791443, "nonfinite": 0.0, "weighted_memory_to_active": 0.0395347885787487, "mix_std": 0.2186863124370575, "beta_std": 0.061160244047641754, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.04549955949187279, "evidence_weights": [-0.10999054461717606, -0.14387553930282593, 0.1353703737258911], "agreement_weight": 0.0036149772349745035, "orthogonal_weight": 1.4782553911209106}, {"head": 4, "memory_to_active": 0.0019328626804053783, "contribution_to_active": 0.5219753384590149, "mix_mean": 0.5448657274246216, "mix_low": 0.0005168914794921875, "mix_high": 0.000446319580078125, "read_cosine": 0.07535895705223083, "beta_mean": 1.0221390766673721e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.001065347925759852, "mix_std": 0.25107863545417786, "beta_std": 2.4564827981521375e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008624041802249849, "evidence_weights": [-0.1818092316389084, -0.2616136372089386, 0.7816243767738342], "agreement_weight": 0.07656117528676987, "orthogonal_weight": -0.8469306230545044}, {"head": 5, "memory_to_active": 0.13796588778495789, "contribution_to_active": 0.4074646532535553, "mix_mean": 0.3841092586517334, "mix_low": 0.002407073974609375, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8553229570388794, "beta_mean": 0.0064653996378183365, "nonfinite": 0.0, "weighted_memory_to_active": 0.06603051722049713, "mix_std": 0.2257414609193802, "beta_std": 0.007908796891570091, "beta_low": 0.8287353515625, "beta_high": 0.0, "orthogonal_to_active": 0.02039984054863453, "evidence_weights": [-0.14815741777420044, -0.1617623269557953, 0.2860252559185028], "agreement_weight": 0.09147485345602036, "orthogonal_weight": -0.6379426717758179}]}, {"layer": 11, "memory_to_active": 0.11754211038351059, "contribution_to_active": 0.5859525799751282, "mix_mean": 0.48343825340270996, "mix_low": 0.0858505591750145, "mix_high": 0.049574218690395355, "read_cosine": 0.38314253091812134, "beta_mean": 0.03369089588522911, "nonfinite": 0.0, "weighted_memory_to_active": 0.05582698434591293, "mix_std": 0.322569876909256, "beta_std": 0.05071578919887543, "beta_low": 0.5959269404411316, "beta_high": 0.0, "orthogonal_to_active": 0.05118653178215027, "output_influence": 0.5199341177940369, "residual_influence": 0.1324542909860611, "state_active": 0.17343521118164062, "state_memory": 0.006399262696504593, "state_peak": 24.938135147094727, "growth_active": 1.0792359113693237, "growth_memory": 1.7022916078567505, "heads": [{"head": 0, "memory_to_active": 0.009258302859961987, "contribution_to_active": 0.5878574252128601, "mix_mean": 0.6213604211807251, "mix_low": 0.0010595321655273438, "mix_high": 0.009512901306152344, "read_cosine": 0.29808393120765686, "beta_mean": 9.032603702507913e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006001257337629795, "mix_std": 0.2642468810081482, "beta_std": 0.0034341481514275074, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.004437302239239216, "evidence_weights": [-0.1656576693058014, -0.3647342920303345, 0.7140905857086182], "agreement_weight": 0.2828759253025055, "orthogonal_weight": -0.7862024307250977}, {"head": 1, "memory_to_active": 0.27462875843048096, "contribution_to_active": 0.32775774598121643, "mix_mean": 0.36750760674476624, "mix_low": 0.010789871215820312, "mix_high": 0.00013446807861328125, "read_cosine": 0.5552823543548584, "beta_mean": 0.03094809502363205, "nonfinite": 0.0, "weighted_memory_to_active": 0.11670481413602829, "mix_std": 0.25903433561325073, "beta_std": 0.07113753259181976, "beta_low": 0.49835205078125, "beta_high": 0.0, "orthogonal_to_active": 0.03899586945772171, "evidence_weights": [-0.16401474177837372, -0.1800323873758316, 0.4729764461517334], "agreement_weight": 0.07434563338756561, "orthogonal_weight": -0.423417866230011}, {"head": 2, "memory_to_active": 0.011440178379416466, "contribution_to_active": 0.9800601005554199, "mix_mean": 0.451722115278244, "mix_low": 0.2120685577392578, "mix_high": 0.17366313934326172, "read_cosine": 0.18966186046600342, "beta_mean": 0.08232555538415909, "nonfinite": 0.0, "weighted_memory_to_active": 0.00848398543894291, "mix_std": 0.44031184911727905, "beta_std": 0.10653683543205261, "beta_low": 0.43231201171875, "beta_high": 0.0, "orthogonal_to_active": 0.05340484157204628, "evidence_weights": [0.042800113558769226, -0.437540203332901, -1.2552967071533203], "agreement_weight": 0.011544513516128063, "orthogonal_weight": 6.516524314880371}, {"head": 3, "memory_to_active": 0.012268472462892532, "contribution_to_active": 0.8249893188476562, "mix_mean": 0.4170949161052704, "mix_low": 0.27800846099853516, "mix_high": 0.11040115356445312, "read_cosine": 0.524819552898407, "beta_mean": 0.06226782500743866, "nonfinite": 0.0, "weighted_memory_to_active": 0.007187246344983578, "mix_std": 0.4393663704395294, "beta_std": 0.09156724810600281, "beta_low": 0.4859619140625, "beta_high": 0.0, "orthogonal_to_active": 0.040623340755701065, "evidence_weights": [0.1449519544839859, -0.3289842903614044, -0.2834988832473755], "agreement_weight": 0.04641985148191452, "orthogonal_weight": 6.850977420806885}, {"head": 4, "memory_to_active": 0.3088909685611725, "contribution_to_active": 0.410150408744812, "mix_mean": 0.5133690237998962, "mix_low": 0.0107269287109375, "mix_high": 0.0006608963012695312, "read_cosine": 0.061899133026599884, "beta_mean": 0.01788249984383583, "nonfinite": 0.0, "weighted_memory_to_active": 0.15385815501213074, "mix_std": 0.27200764417648315, "beta_std": 0.01900395378470421, "beta_low": 0.41729736328125, "beta_high": 0.0, "orthogonal_to_active": 0.1461390256881714, "evidence_weights": [-0.23056674003601074, -0.20858150720596313, 1.8707025051116943], "agreement_weight": -0.1902918517589569, "orthogonal_weight": -0.9640931487083435}, {"head": 5, "memory_to_active": 0.08876597136259079, "contribution_to_active": 0.38490045070648193, "mix_mean": 0.5295752882957458, "mix_low": 0.0024499893188476562, "mix_high": 0.0030727386474609375, "read_cosine": 0.6691082715988159, "beta_mean": 0.008631068281829357, "nonfinite": 0.0, "weighted_memory_to_active": 0.042726412415504456, "mix_std": 0.26045215129852295, "beta_std": 0.012615014798939228, "beta_low": 0.74200439453125, "beta_high": 0.0, "orthogonal_to_active": 0.023518815636634827, "evidence_weights": [-0.21924681961536407, -0.2973034679889679, 0.6089760065078735], "agreement_weight": 0.08261509239673615, "orthogonal_weight": -0.7242181301116943}]}, {"layer": 12, "memory_to_active": 0.2633299231529236, "contribution_to_active": 0.5557341575622559, "mix_mean": 0.5062999725341797, "mix_low": 0.009967486374080181, "mix_high": 0.004488945472985506, "read_cosine": 0.24215880036354065, "beta_mean": 0.01637316308915615, "nonfinite": 0.0, "weighted_memory_to_active": 0.14326684176921844, "mix_std": 0.26092106103897095, "beta_std": 0.016350679099559784, "beta_low": 0.823699951171875, "beta_high": 0.0, "orthogonal_to_active": 0.07203774899244308, "output_influence": 0.45783817768096924, "residual_influence": 0.12853240966796875, "state_active": 0.03421155735850334, "state_memory": 0.009181542322039604, "state_peak": 48.48774719238281, "growth_active": 1.0845706462860107, "growth_memory": 1.23867666721344, "heads": [{"head": 0, "memory_to_active": 0.055983200669288635, "contribution_to_active": 0.5240412950515747, "mix_mean": 0.5142072439193726, "mix_low": 0.0014982223510742188, "mix_high": 0.0024585723876953125, "read_cosine": 0.29329949617385864, "beta_mean": 0.0009165379451587796, "nonfinite": 0.0, "weighted_memory_to_active": 0.02727483958005905, "mix_std": 0.2712237238883972, "beta_std": 0.0033825403079390526, "beta_low": 0.99468994140625, "beta_high": 0.0, "orthogonal_to_active": 0.022005828097462654, "evidence_weights": [-0.19522503018379211, -0.2672072649002075, 0.24491028487682343], "agreement_weight": 0.10686126351356506, "orthogonal_weight": -0.8612203001976013}, {"head": 1, "memory_to_active": 0.11427120864391327, "contribution_to_active": 0.516938328742981, "mix_mean": 0.5328788757324219, "mix_low": 0.0007495880126953125, "mix_high": 0.0008087158203125, "read_cosine": 0.4716980755329132, "beta_mean": 0.0011196734849363565, "nonfinite": 0.0, "weighted_memory_to_active": 0.06108585000038147, "mix_std": 0.2631044387817383, "beta_std": 0.008704962208867073, "beta_low": 0.9866943359375, "beta_high": 0.0, "orthogonal_to_active": 0.0497397780418396, "evidence_weights": [-0.20754015445709229, -0.27332887053489685, 0.9716359376907349], "agreement_weight": 0.14887601137161255, "orthogonal_weight": -0.9336935877799988}, {"head": 2, "memory_to_active": 0.6113714575767517, "contribution_to_active": 0.4817779064178467, "mix_mean": 0.312258780002594, "mix_low": 0.02625274658203125, "mix_high": 9.918212890625e-05, "read_cosine": 0.4607447385787964, "beta_mean": 0.09422656148672104, "nonfinite": 0.0, "weighted_memory_to_active": 0.24906986951828003, "mix_std": 0.23944856226444244, "beta_std": 0.06534750759601593, "beta_low": 0.00115966796875, "beta_high": 0.0, "orthogonal_to_active": 0.24052263796329498, "evidence_weights": [-0.23725171387195587, -0.046395402401685715, -0.09037254750728607], "agreement_weight": 0.022832199931144714, "orthogonal_weight": 1.201316475868225}, {"head": 3, "memory_to_active": 0.03981096297502518, "contribution_to_active": 0.730918824672699, "mix_mean": 0.6993960738182068, "mix_low": 0.0002307891845703125, "mix_high": 0.01598358154296875, "read_cosine": 0.10333684086799622, "beta_mean": 0.00029534450732171535, "nonfinite": 0.0, "weighted_memory_to_active": 0.02829478122293949, "mix_std": 0.23663540184497833, "beta_std": 0.005661461502313614, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.02667282149195671, "evidence_weights": [-0.29324135184288025, -0.3432979881763458, 1.3728410005569458], "agreement_weight": 0.3854944407939911, "orthogonal_weight": -0.9676567912101746}, {"head": 4, "memory_to_active": 0.6826329231262207, "contribution_to_active": 0.6106433868408203, "mix_mean": 0.3640441298484802, "mix_low": 0.023244857788085938, "mix_high": 0.0029287338256835938, "read_cosine": -0.10358932614326477, "beta_mean": 0.0012405194574967027, "nonfinite": 0.0, "weighted_memory_to_active": 0.44490739703178406, "mix_std": 0.27645841240882874, "beta_std": 0.011529998853802681, "beta_low": 0.972412109375, "beta_high": 0.0, "orthogonal_to_active": 0.05516577884554863, "evidence_weights": [-0.12031804770231247, -0.2146410048007965, 0.019680488854646683], "agreement_weight": 0.0015912381932139397, "orthogonal_weight": -0.13035805523395538}, {"head": 5, "memory_to_active": 0.0759098157286644, "contribution_to_active": 0.47008541226387024, "mix_mean": 0.6150147914886475, "mix_low": 0.007828712463378906, "mix_high": 0.004654884338378906, "read_cosine": 0.2274629771709442, "beta_mean": 0.00044033635640516877, "nonfinite": 0.0, "weighted_memory_to_active": 0.04896827042102814, "mix_std": 0.2786557674407959, "beta_std": 0.0034776071552187204, "beta_low": 0.98919677734375, "beta_high": 0.0, "orthogonal_to_active": 0.03811965137720108, "evidence_weights": [-0.2487751990556717, -0.3126889169216156, 1.2577614784240723], "agreement_weight": 0.3220853805541992, "orthogonal_weight": -0.8393438458442688}]}, {"layer": 13, "memory_to_active": 0.13080580532550812, "contribution_to_active": 0.518699586391449, "mix_mean": 0.5146555304527283, "mix_low": 0.006006558891385794, "mix_high": 0.0067423186264932156, "read_cosine": 0.40377235412597656, "beta_mean": 0.015547873452305794, "nonfinite": 0.0, "weighted_memory_to_active": 0.05680478736758232, "mix_std": 0.2583366632461548, "beta_std": 0.01854143850505352, "beta_low": 0.7969055771827698, "beta_high": 0.0, "orthogonal_to_active": 0.05858401209115982, "output_influence": 0.4871007204055786, "residual_influence": 0.16720189154148102, "state_active": 0.10268869996070862, "state_memory": 0.038201913237571716, "state_peak": 95.01341247558594, "growth_active": 0.9390020966529846, "growth_memory": 1.0787473917007446, "heads": [{"head": 0, "memory_to_active": 0.02444572187960148, "contribution_to_active": 0.5678529739379883, "mix_mean": 0.5831562280654907, "mix_low": 0.00044727325439453125, "mix_high": 0.00202178955078125, "read_cosine": 0.4088479280471802, "beta_mean": 9.349092579213902e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014934156090021133, "mix_std": 0.268875390291214, "beta_std": 0.0002233361592516303, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014922608621418476, "evidence_weights": [-0.20375867187976837, -0.26739153265953064, 0.7446939945220947], "agreement_weight": 0.11341629922389984, "orthogonal_weight": -1.1030930280685425}, {"head": 1, "memory_to_active": 0.01600608602166176, "contribution_to_active": 0.5734993815422058, "mix_mean": 0.5329678058624268, "mix_low": 0.0018329620361328125, "mix_high": 0.0024328231811523438, "read_cosine": 0.20287278294563293, "beta_mean": 5.573980160988867e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009067153558135033, "mix_std": 0.27233487367630005, "beta_std": 0.00010441368067404255, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011089997366070747, "evidence_weights": [-0.18458916246891022, -0.26776060461997986, 0.6978758573532104], "agreement_weight": 0.3431920111179352, "orthogonal_weight": -1.2739853858947754}, {"head": 2, "memory_to_active": 0.1008668839931488, "contribution_to_active": 0.45152968168258667, "mix_mean": 0.37324199080467224, "mix_low": 0.00194549560546875, "mix_high": 0.00011348724365234375, "read_cosine": 0.6519832611083984, "beta_mean": 0.015275171026587486, "nonfinite": 0.0, "weighted_memory_to_active": 0.04476756602525711, "mix_std": 0.22695806622505188, "beta_std": 0.046080511063337326, "beta_low": 0.7933349609375, "beta_high": 0.0, "orthogonal_to_active": 0.021967360749840736, "evidence_weights": [-0.19713234901428223, -0.19598262012004852, 0.4218350052833557], "agreement_weight": 0.008741823956370354, "orthogonal_weight": -0.8143952488899231}, {"head": 3, "memory_to_active": 0.1488015353679657, "contribution_to_active": 0.5695890188217163, "mix_mean": 0.6280959844589233, "mix_low": 0.02332782745361328, "mix_high": 0.027021408081054688, "read_cosine": -0.019911982119083405, "beta_mean": 0.0011908623855561018, "nonfinite": 0.0, "weighted_memory_to_active": 0.10980533808469772, "mix_std": 0.29224225878715515, "beta_std": 0.009764842689037323, "beta_low": 0.99072265625, "beta_high": 0.0, "orthogonal_to_active": 0.1020897626876831, "evidence_weights": [-0.3285027742385864, -0.2487710416316986, 1.847388744354248], "agreement_weight": 4.293066501617432, "orthogonal_weight": -1.020146131515503}, {"head": 4, "memory_to_active": 0.411056786775589, "contribution_to_active": 0.2765931785106659, "mix_mean": 0.33904582262039185, "mix_low": 0.005995750427246094, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8568068146705627, "beta_mean": 0.07611922174692154, "nonfinite": 0.0, "weighted_memory_to_active": 0.11248709261417389, "mix_std": 0.21522659063339233, "beta_std": 0.05133393406867981, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.15995830297470093, "evidence_weights": [-0.12093333154916763, -0.23120765388011932, 0.4816168546676636], "agreement_weight": -0.0023758176248520613, "orthogonal_weight": 2.1581523418426514}, {"head": 5, "memory_to_active": 0.08365780115127563, "contribution_to_active": 0.673133134841919, "mix_mean": 0.6314253211021423, "mix_low": 0.0024900436401367188, "mix_high": 0.008862495422363281, "read_cosine": 0.3220351040363312, "beta_mean": 0.0005527549074031413, "nonfinite": 0.0, "weighted_memory_to_active": 0.04976740851998329, "mix_std": 0.27438268065452576, "beta_std": 0.0037415805272758007, "beta_low": 0.99664306640625, "beta_high": 0.0, "orthogonal_to_active": 0.04147601127624512, "evidence_weights": [-0.22162777185440063, -0.30372482538223267, 0.8237136602401733], "agreement_weight": 0.1312016248703003, "orthogonal_weight": -0.9217103123664856}]}, {"layer": 14, "memory_to_active": 0.24859318137168884, "contribution_to_active": 0.4192212224006653, "mix_mean": 0.42289549112319946, "mix_low": 0.024170400574803352, "mix_high": 0.005477428436279297, "read_cosine": 0.6085168123245239, "beta_mean": 0.0659237876534462, "nonfinite": 0.0, "weighted_memory_to_active": 0.10913874208927155, "mix_std": 0.2486819624900818, "beta_std": 0.06164856255054474, "beta_low": 0.4698689877986908, "beta_high": 0.0, "orthogonal_to_active": 0.04465380683541298, "output_influence": 0.33035358786582947, "residual_influence": 0.12754930555820465, "state_active": 0.05954521894454956, "state_memory": 0.016032040119171143, "state_peak": 34.76056671142578, "growth_active": 1.0931174755096436, "growth_memory": 1.3050463199615479, "heads": [{"head": 0, "memory_to_active": 0.35839971899986267, "contribution_to_active": 0.21003535389900208, "mix_mean": 0.22942622005939484, "mix_low": 0.009756088256835938, "mix_high": 1.9073486328125e-05, "read_cosine": 0.7902438044548035, "beta_mean": 0.06964976340532303, "nonfinite": 0.0, "weighted_memory_to_active": 0.09563516080379486, "mix_std": 0.19440186023712158, "beta_std": 0.060190338641405106, "beta_low": 0.0067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.08060044050216675, "evidence_weights": [-0.13841553032398224, -0.03852105140686035, 0.49655741453170776], "agreement_weight": 0.026503248140215874, "orthogonal_weight": 1.3613477945327759}, {"head": 1, "memory_to_active": 0.48204725980758667, "contribution_to_active": 0.3303838074207306, "mix_mean": 0.23852035403251648, "mix_low": 0.11808013916015625, "mix_high": 7.82012939453125e-05, "read_cosine": 0.809009850025177, "beta_mean": 0.27035921812057495, "nonfinite": 0.0, "weighted_memory_to_active": 0.21783852577209473, "mix_std": 0.2634676396846771, "beta_std": 0.25284722447395325, "beta_low": 0.1116943359375, "beta_high": 0.0, "orthogonal_to_active": 0.014770589768886566, "evidence_weights": [-0.011568105779588223, -0.22250837087631226, -0.13812467455863953], "agreement_weight": 0.051978059113025665, "orthogonal_weight": 0.15669451653957367}, {"head": 2, "memory_to_active": 0.1528264433145523, "contribution_to_active": 0.5606461763381958, "mix_mean": 0.5733494758605957, "mix_low": 0.002635955810546875, "mix_high": 0.0032291412353515625, "read_cosine": 0.4003005027770996, "beta_mean": 0.0035249311476945877, "nonfinite": 0.0, "weighted_memory_to_active": 0.09481191635131836, "mix_std": 0.26608800888061523, "beta_std": 0.0048188865184783936, "beta_low": 0.93072509765625, "beta_high": 0.0, "orthogonal_to_active": 0.07213500887155533, "evidence_weights": [-0.1510625034570694, -0.3110930323600769, 0.9050884246826172], "agreement_weight": 0.08324828743934631, "orthogonal_weight": -0.8639031648635864}, {"head": 3, "memory_to_active": 0.06234706565737724, "contribution_to_active": 0.6068224310874939, "mix_mean": 0.6649428606033325, "mix_low": 0.005925178527832031, "mix_high": 0.0194854736328125, "read_cosine": 0.06860479712486267, "beta_mean": 0.007433826569467783, "nonfinite": 0.0, "weighted_memory_to_active": 0.03831055760383606, "mix_std": 0.2535296678543091, "beta_std": 0.008014000952243805, "beta_low": 0.76190185546875, "beta_high": 0.0, "orthogonal_to_active": 0.025575745850801468, "evidence_weights": [-0.3640766739845276, -0.31857752799987793, 0.3766210675239563], "agreement_weight": 0.07224731147289276, "orthogonal_weight": -0.76530921459198}, {"head": 4, "memory_to_active": 0.1663374900817871, "contribution_to_active": 0.48899248242378235, "mix_mean": 0.4853271543979645, "mix_low": 0.004427909851074219, "mix_high": 0.009016036987304688, "read_cosine": 0.8532216548919678, "beta_mean": 0.04204089939594269, "nonfinite": 0.0, "weighted_memory_to_active": 0.09861906617879868, "mix_std": 0.2635995149612427, "beta_std": 0.030993781983852386, "beta_low": 0.05841064453125, "beta_high": 0.0, "orthogonal_to_active": 0.028168024495244026, "evidence_weights": [-0.21978649497032166, -0.25108084082603455, 0.42539462447166443], "agreement_weight": 0.12590783834457397, "orthogonal_weight": -0.5844565033912659}, {"head": 5, "memory_to_active": 0.26960110664367676, "contribution_to_active": 0.3184469938278198, "mix_mean": 0.34580689668655396, "mix_low": 0.004197120666503906, "mix_high": 0.0010366439819335938, "read_cosine": 0.7297201156616211, "beta_mean": 0.0025340886786580086, "nonfinite": 0.0, "weighted_memory_to_active": 0.1096172109246254, "mix_std": 0.25100505352020264, "beta_std": 0.01302712969481945, "beta_low": 0.94976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.04667302593588829, "evidence_weights": [-0.17930465936660767, -0.16137681901454926, 0.24433384835720062], "agreement_weight": 0.08327295631170273, "orthogonal_weight": -0.6747738718986511}]}, {"layer": 15, "memory_to_active": 0.06528633087873459, "contribution_to_active": 0.4903222918510437, "mix_mean": 0.4080224633216858, "mix_low": 0.00923172663897276, "mix_high": 0.0026523275300860405, "read_cosine": 0.6939501166343689, "beta_mean": 0.03987666964530945, "nonfinite": 0.0, "weighted_memory_to_active": 0.03195524588227272, "mix_std": 0.28415632247924805, "beta_std": 0.047013528645038605, "beta_low": 0.7072551250457764, "beta_high": 0.0, "orthogonal_to_active": 0.0065157548524439335, "output_influence": 0.3035846948623657, "residual_influence": 0.16341592371463776, "state_active": 0.1328822672367096, "state_memory": 0.010378353297710419, "state_peak": 34.469547271728516, "growth_active": 1.2839875221252441, "growth_memory": 1.4999405145645142, "heads": [{"head": 0, "memory_to_active": 0.10710315406322479, "contribution_to_active": 0.5031924247741699, "mix_mean": 0.38148579001426697, "mix_low": 0.005778312683105469, "mix_high": 0.005549430847167969, "read_cosine": 0.7195005416870117, "beta_mean": 0.036273881793022156, "nonfinite": 0.0, "weighted_memory_to_active": 0.05399961397051811, "mix_std": 0.27950331568717957, "beta_std": 0.027907345443964005, "beta_low": 0.10748291015625, "beta_high": 0.0, "orthogonal_to_active": 0.022075915709137917, "evidence_weights": [-0.1767370104789734, -0.18254530429840088, 0.3724403381347656], "agreement_weight": 0.09097810089588165, "orthogonal_weight": -0.7099010944366455}, {"head": 1, "memory_to_active": 0.13813431560993195, "contribution_to_active": 0.4111562669277191, "mix_mean": 0.4205780327320099, "mix_low": 0.002285003662109375, "mix_high": 0.0053119659423828125, "read_cosine": 0.905785322189331, "beta_mean": 0.0011324018705636263, "nonfinite": 0.0, "weighted_memory_to_active": 0.05573491379618645, "mix_std": 0.2836271822452545, "beta_std": 0.0031470388639718294, "beta_low": 0.981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.00011247102520428598, "evidence_weights": [-0.1603979915380478, -0.16984207928180695, 0.2247561365365982], "agreement_weight": 0.11195293813943863, "orthogonal_weight": -0.0035094183403998613}, {"head": 2, "memory_to_active": 0.008968288078904152, "contribution_to_active": 0.4869019091129303, "mix_mean": 0.3410072922706604, "mix_low": 0.018842697143554688, "mix_high": 6.961822509765625e-05, "read_cosine": 0.7517349720001221, "beta_mean": 0.0001487999688833952, "nonfinite": 0.0, "weighted_memory_to_active": 0.004489365499466658, "mix_std": 0.27251777052879333, "beta_std": 0.0002554451930336654, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0008468754822388291, "evidence_weights": [-0.20072217285633087, -0.27291685342788696, 0.1946401447057724], "agreement_weight": 0.01246877945959568, "orthogonal_weight": 0.4379028379917145}, {"head": 3, "memory_to_active": 0.002723543904721737, "contribution_to_active": 0.4132723808288574, "mix_mean": 0.3813629746437073, "mix_low": 0.006260871887207031, "mix_high": 0.00028514862060546875, "read_cosine": 0.6389889717102051, "beta_mean": 1.8576192815089598e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.000972032779827714, "mix_std": 0.28410401940345764, "beta_std": 3.718320294865407e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000824496615678072, "evidence_weights": [-0.2309507131576538, -0.25460943579673767, 0.2843751609325409], "agreement_weight": 0.029808010905981064, "orthogonal_weight": -0.9694870710372925}, {"head": 4, "memory_to_active": 0.11063863337039948, "contribution_to_active": 0.6030214428901672, "mix_mean": 0.36408501863479614, "mix_low": 0.011960983276367188, "mix_high": 0.0008535385131835938, "read_cosine": 0.787731409072876, "beta_mean": 0.19885817170143127, "nonfinite": 0.0, "weighted_memory_to_active": 0.060952357947826385, "mix_std": 0.30601048469543457, "beta_std": 0.22900795936584473, "beta_low": 0.183837890625, "beta_high": 0.0, "orthogonal_to_active": 0.006562093738466501, "evidence_weights": [-0.2129131406545639, -0.254016637802124, 0.001062373397871852], "agreement_weight": 0.013475959189236164, "orthogonal_weight": 0.26739439368247986}, {"head": 5, "memory_to_active": 0.02415001392364502, "contribution_to_active": 0.5243892669677734, "mix_mean": 0.5596154928207397, "mix_low": 0.010262489318847656, "mix_high": 0.0038442611694335938, "read_cosine": 0.3599591851234436, "beta_mean": 0.002828187309205532, "nonfinite": 0.0, "weighted_memory_to_active": 0.015583205968141556, "mix_std": 0.2791752219200134, "beta_std": 0.02172619104385376, "beta_low": 0.9705810546875, "beta_high": 0.0, "orthogonal_to_active": 0.008672673255205154, "evidence_weights": [-0.30967214703559875, -0.31610307097435, 0.7405039072036743], "agreement_weight": 0.2614358067512512, "orthogonal_weight": 0.7364000678062439}]}]}, {"step": 39000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9080585000010615, "memory_to_active": 0.8530610799789429, "contribution_to_active": 0.5417516231536865, "mix_mean": 0.3853587806224823, "mix_low": 0.1451878845691681, "mix_high": 0.009027203544974327, "read_cosine": 0.372444212436676, "beta_mean": 0.08035022765398026, "nonfinite": 0.0, "weighted_memory_to_active": 0.2175418734550476, "mix_std": 0.21938860416412354, "beta_std": 0.06779015809297562, "beta_low": 0.5257276296615601, "beta_high": 2.606709676911123e-05, "orthogonal_to_active": 0.058098845183849335, "samples": 4, "scored_tokens": 899, "normal_loss": 3.128920078277588, "normal_token_accuracy": 0.4271412789821625, "state_active": 0.06104397401213646, "state_memory": 0.01913345791399479, "state_peak": 120.94068145751953, "growth_active": 1.1039412021636963, "growth_memory": 1.379650592803955, "output_influence": 0.39219820499420166, "residual_influence": 0.12848076224327087, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.269131898880005, "no_memory_loss_delta": 0.477523535490036, "scrambled_loss_delta": 0.21593496203422546, "memory_only_loss_delta": 2.7755298614501953, "no_evidence_loss_delta": 0.27396881580352783, "no_agreement_loss_delta": 0.014808203093707561, "no_orthogonal_loss_delta": 0.012713533826172352}, {"index": 94, "tokens": 256, "normal_loss": 3.1674342155456543, "no_memory_loss_delta": 0.521847128868103, "scrambled_loss_delta": 0.39921116828918457, "memory_only_loss_delta": 3.0553479194641113, "no_evidence_loss_delta": 0.21656672656536102, "no_agreement_loss_delta": 0.028281431645154953, "no_orthogonal_loss_delta": 0.055980391800403595}, {"index": 188, "tokens": 256, "normal_loss": 3.5195443630218506, "no_memory_loss_delta": 0.5063493847846985, "scrambled_loss_delta": 0.14756008982658386, "memory_only_loss_delta": 2.8712446689605713, "no_evidence_loss_delta": 0.2505183815956116, "no_agreement_loss_delta": 0.03052603453397751, "no_orthogonal_loss_delta": 0.045667700469493866}, {"index": 283, "tokens": 131, "normal_loss": 2.016295909881592, "no_memory_loss_delta": 0.33734747767448425, "scrambled_loss_delta": 0.3623790442943573, "memory_only_loss_delta": 2.5319385528564453, "no_evidence_loss_delta": 0.14444249868392944, "no_agreement_loss_delta": 0.016702909022569656, "no_orthogonal_loss_delta": 0.048424795269966125}], "agreement_weight_abs_mean": 0.17343272268772125, "agreement_weight_abs_max": 4.384817123413086, "agreement_weight_mean": 0.15235884487628937, "orthogonal_weight_abs_mean": 0.7478821277618408, "orthogonal_weight_abs_max": 7.010732650756836, "orthogonal_weight_mean": -0.1395910531282425, "no_memory_loss": 3.6068472862243652, "no_memory_loss_delta": 0.4779275059700012, "no_memory_token_accuracy_gain": 0.06229143589735031, "no_memory_token_win_fraction": 0.6751946806907654, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 37.99328534809913, "scrambled_loss": 3.398913860321045, "scrambled_loss_delta": 0.2699938416481018, "scrambled_token_accuracy_gain": 0.02892102301120758, "scrambled_token_win_fraction": 0.6373748779296875, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 23.66158044891131, "memory_only_loss": 5.97589111328125, "memory_only_loss_delta": 2.846971273422241, "memory_only_token_accuracy_gain": 0.2691879868507385, "memory_only_token_win_fraction": 0.9154615998268127, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.19802189167847, "no_evidence_loss": 3.360990524291992, "no_evidence_loss_delta": 0.2320709377527237, "no_evidence_token_accuracy_gain": 0.02002224698662758, "no_evidence_token_win_fraction": 0.6129032373428345, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 20.711012451426406, "no_agreement_loss": 3.1523165702819824, "no_agreement_loss_delta": 0.023396767675876617, "no_agreement_token_accuracy_gain": -0.006674082484096289, "no_agreement_token_win_fraction": 0.48498332500457764, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.312518547873127, "no_orthogonal_loss": 3.1685421466827393, "no_orthogonal_loss_delta": 0.03962205722928047, "no_orthogonal_token_accuracy_gain": 0.004449388012290001, "no_orthogonal_token_win_fraction": 0.5761957764625549, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 3.8847368796383632, "short_no_memory_loss_delta": 0.41240194439888, "long_context_memory_gain": 0.06552556157112122, "normal_context_loss_gain": 0.13202938437461853, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.2819039821624756, "contribution_to_active": 1.0717055797576904, "mix_mean": 0.10363627225160599, "mix_low": 0.5491976737976074, "mix_high": 0.005267143715173006, "read_cosine": 0.4480406641960144, "beta_mean": 0.24689741432666779, "nonfinite": 0.0, "weighted_memory_to_active": 1.117409586906433, "mix_std": 0.08817611634731293, "beta_std": 0.14754827320575714, "beta_low": 0.036346435546875, "beta_high": 0.0, "orthogonal_to_active": 0.05848092585802078, "output_influence": 0.19822955131530762, "residual_influence": 0.23522555828094482, "state_active": 0.06668491661548615, "state_memory": 0.04705587029457092, "state_peak": 52.66584396362305, "growth_active": 1.2216179370880127, "growth_memory": 1.256678819656372, "heads": [{"head": 0, "memory_to_active": 2.4887144565582275, "contribution_to_active": 0.43609368801116943, "mix_mean": 0.019495651125907898, "mix_low": 0.6132802963256836, "mix_high": 0.0, "read_cosine": 0.2501053810119629, "beta_mean": 0.2216135412454605, "nonfinite": 0.0, "weighted_memory_to_active": 0.520534873008728, "mix_std": 0.05641983076930046, "beta_std": 0.10459252446889877, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.044776126742362976, "evidence_weights": [-0.017007283866405487, -0.0634598508477211, -0.10424765944480896], "agreement_weight": -0.14172211289405823, "orthogonal_weight": -0.09347318112850189}, {"head": 1, "memory_to_active": 5.786065101623535, "contribution_to_active": 2.2943813800811768, "mix_mean": 0.015418373979628086, "mix_low": 0.6938810348510742, "mix_high": 0.0, "read_cosine": 0.5228593349456787, "beta_mean": 0.47856950759887695, "nonfinite": 0.0, "weighted_memory_to_active": 2.607156276702881, "mix_std": 0.05216127634048462, "beta_std": 0.211112380027771, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.08008357137441635, "evidence_weights": [0.03003704361617565, -0.098256915807724, 0.4547710418701172], "agreement_weight": 0.02436971105635166, "orthogonal_weight": -0.06016405671834946}, {"head": 2, "memory_to_active": 2.281909704208374, "contribution_to_active": 0.4743582606315613, "mix_mean": 0.013257674872875214, "mix_low": 0.682647705078125, "mix_high": 0.0, "read_cosine": 0.33059796690940857, "beta_mean": 0.2705184519290924, "nonfinite": 0.0, "weighted_memory_to_active": 0.5716034173965454, "mix_std": 0.032699957489967346, "beta_std": 0.1726870983839035, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05439910292625427, "evidence_weights": [0.06085997074842453, 0.066731296479702, 0.03159646317362785], "agreement_weight": 0.011475512757897377, "orthogonal_weight": -0.2081851363182068}, {"head": 3, "memory_to_active": 6.541768550872803, "contribution_to_active": 2.7151808738708496, "mix_mean": 0.03692513704299927, "mix_low": 0.4711179733276367, "mix_high": 0.0, "read_cosine": 0.3242977261543274, "beta_mean": 0.3200411796569824, "nonfinite": 0.0, "weighted_memory_to_active": 2.894192934036255, "mix_std": 0.08394691348075867, "beta_std": 0.24245840311050415, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1455080658197403, "evidence_weights": [-0.04804378002882004, -0.06090836226940155, -0.16014783084392548], "agreement_weight": -0.017838355153799057, "orthogonal_weight": -0.059330906718969345}, {"head": 4, "memory_to_active": 2.4856419563293457, "contribution_to_active": 0.07056355476379395, "mix_mean": 0.006710385903716087, "mix_low": 0.8143281936645508, "mix_high": 0.0, "read_cosine": 0.3372170925140381, "beta_mean": 0.16170641779899597, "nonfinite": 0.0, "weighted_memory_to_active": 0.05541224777698517, "mix_std": 0.013555781915783882, "beta_std": 0.12637731432914734, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006217882968485355, "evidence_weights": [0.12875419855117798, 0.08124325424432755, -0.33738917112350464], "agreement_weight": -0.12033531069755554, "orthogonal_weight": 0.10771442949771881}, {"head": 5, "memory_to_active": 0.1073227971792221, "contribution_to_active": 0.43965545296669006, "mix_mean": 0.5300103425979614, "mix_low": 0.01993083953857422, "mix_high": 0.03160285949707031, "read_cosine": 0.9231664538383484, "beta_mean": 0.028935328125953674, "nonfinite": 0.0, "weighted_memory_to_active": 0.055557698011398315, "mix_std": 0.29027286171913147, "beta_std": 0.028061849996447563, "beta_low": 0.21807861328125, "beta_high": 0.0, "orthogonal_to_active": 0.01990080624818802, "evidence_weights": [-0.21500615775585175, -0.23247522115707397, 0.3373141884803772], "agreement_weight": 0.2778733968734741, "orthogonal_weight": -0.684991717338562}]}, {"layer": 1, "memory_to_active": 2.257139205932617, "contribution_to_active": 0.5344107151031494, "mix_mean": 0.19530615210533142, "mix_low": 0.3533000946044922, "mix_high": 0.0012935003032907844, "read_cosine": 0.3555585443973541, "beta_mean": 0.1948244571685791, "nonfinite": 0.0, "weighted_memory_to_active": 0.3890020251274109, "mix_std": 0.11984430998563766, "beta_std": 0.13837513327598572, "beta_low": 0.2892863154411316, "beta_high": 0.00010172525799134746, "orthogonal_to_active": 0.052920423448085785, "output_influence": 0.2054855227470398, "residual_influence": 0.05114918202161789, "state_active": 0.037897393107414246, "state_memory": 0.03638763353228569, "state_peak": 35.18694305419922, "growth_active": 1.3060696125030518, "growth_memory": 1.4812262058258057, "heads": [{"head": 0, "memory_to_active": 0.03513823449611664, "contribution_to_active": 0.45191723108291626, "mix_mean": 0.453224241733551, "mix_low": 0.01025390625, "mix_high": 0.0012264251708984375, "read_cosine": 0.10514093190431595, "beta_mean": 0.000263674883171916, "nonfinite": 0.0, "weighted_memory_to_active": 0.01767081394791603, "mix_std": 0.2879564166069031, "beta_std": 0.0004209695034660399, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01628006249666214, "evidence_weights": [-0.18861369788646698, -0.24685770273208618, 0.2653355598449707], "agreement_weight": 0.18107391893863678, "orthogonal_weight": -1.0007277727127075}, {"head": 1, "memory_to_active": 2.768672466278076, "contribution_to_active": 0.24784693121910095, "mix_mean": 0.024479303508996964, "mix_low": 0.43357086181640625, "mix_high": 3.814697265625e-06, "read_cosine": 0.5152145624160767, "beta_mean": 0.28992703557014465, "nonfinite": 0.0, "weighted_memory_to_active": 0.29281026124954224, "mix_std": 0.05116844177246094, "beta_std": 0.18984854221343994, "beta_low": 0.00128173828125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.015670757740736008, "evidence_weights": [0.010575090534985065, -0.04318716377019882, 0.01539156399667263], "agreement_weight": -0.0003684871189761907, "orthogonal_weight": 0.14551889896392822}, {"head": 2, "memory_to_active": 0.18822146952152252, "contribution_to_active": 0.6091131567955017, "mix_mean": 0.6268138289451599, "mix_low": 0.0008993148803710938, "mix_high": 0.0065250396728515625, "read_cosine": 0.6750273704528809, "beta_mean": 0.0103550860658288, "nonfinite": 0.0, "weighted_memory_to_active": 0.1526355892419815, "mix_std": 0.24490204453468323, "beta_std": 0.011605154722929, "beta_low": 0.6588134765625, "beta_high": 0.0, "orthogonal_to_active": 0.04999779909849167, "evidence_weights": [-0.30635014176368713, -0.3506712317466736, 0.49117225408554077], "agreement_weight": 0.02698933705687523, "orthogonal_weight": -0.5135546326637268}, {"head": 3, "memory_to_active": 2.983260154724121, "contribution_to_active": 0.22328177094459534, "mix_mean": 0.02063790336251259, "mix_low": 0.6049766540527344, "mix_high": 1.9073486328125e-06, "read_cosine": 0.23876047134399414, "beta_mean": 0.3732787072658539, "nonfinite": 0.0, "weighted_memory_to_active": 0.3431302309036255, "mix_std": 0.051964547485113144, "beta_std": 0.2595500946044922, "beta_low": 0.02978515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08506488054990768, "evidence_weights": [0.040621135383844376, -0.012017586268484592, 0.1201118752360344], "agreement_weight": -0.00462704710662365, "orthogonal_weight": -0.37331104278564453}, {"head": 4, "memory_to_active": 3.6591787338256836, "contribution_to_active": 1.6073715686798096, "mix_mean": 0.038920026272535324, "mix_low": 0.2573709487915039, "mix_high": 3.814697265625e-06, "read_cosine": 0.33067822456359863, "beta_mean": 0.18149662017822266, "nonfinite": 0.0, "weighted_memory_to_active": 1.4723150730133057, "mix_std": 0.06483093649148941, "beta_std": 0.13092324137687683, "beta_low": 0.019775390625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.14109492301940918, "evidence_weights": [-0.04491285979747772, -0.0052625397220253944, 0.014832633547484875], "agreement_weight": -0.012120470404624939, "orthogonal_weight": 0.12791195511817932}, {"head": 5, "memory_to_active": 3.9083642959594727, "contribution_to_active": 0.066933274269104, "mix_mean": 0.007761542685329914, "mix_low": 0.8127288818359375, "mix_high": 0.0, "read_cosine": 0.2685295343399048, "beta_mean": 0.313625693321228, "nonfinite": 0.0, "weighted_memory_to_active": 0.055450160056352615, "mix_std": 0.018243400380015373, "beta_std": 0.23790279030799866, "beta_low": 0.02606201171875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.00941410381346941, "evidence_weights": [0.1621839702129364, 0.08076555281877518, 0.2640107572078705], "agreement_weight": -0.1327497363090515, "orthogonal_weight": 0.1692417562007904}]}, {"layer": 2, "memory_to_active": 1.1194405555725098, "contribution_to_active": 0.349714457988739, "mix_mean": 0.37179020047187805, "mix_low": 0.2510903775691986, "mix_high": 0.004154523368924856, "read_cosine": 0.3277831971645355, "beta_mean": 0.09616139531135559, "nonfinite": 0.0, "weighted_memory_to_active": 0.0794566199183464, "mix_std": 0.19385939836502075, "beta_std": 0.0731528103351593, "beta_low": 0.5538228750228882, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.020041650161147118, "output_influence": 0.3466815948486328, "residual_influence": 0.08656182885169983, "state_active": 0.036206915974617004, "state_memory": 0.017726412042975426, "state_peak": 32.719642639160156, "growth_active": 0.9926488995552063, "growth_memory": 1.586737036705017, "heads": [{"head": 0, "memory_to_active": 0.007336056791245937, "contribution_to_active": 0.3596794605255127, "mix_mean": 0.5738030672073364, "mix_low": 0.0015287399291992188, "mix_high": 0.007801055908203125, "read_cosine": 0.17853471636772156, "beta_mean": 5.5764554417692125e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0041712443344295025, "mix_std": 0.2887794077396393, "beta_std": 7.406621443806216e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002728251973167062, "evidence_weights": [-0.3361234962940216, -0.2054605334997177, 0.08504277467727661], "agreement_weight": 0.21976396441459656, "orthogonal_weight": -0.6832135319709778}, {"head": 1, "memory_to_active": 3.0368385314941406, "contribution_to_active": 0.05062844231724739, "mix_mean": 0.004925965331494808, "mix_low": 0.8974103927612305, "mix_high": 0.0, "read_cosine": 0.19609038531780243, "beta_mean": 0.2699187695980072, "nonfinite": 0.0, "weighted_memory_to_active": 0.036068011075258255, "mix_std": 0.01142763253301382, "beta_std": 0.156678706407547, "beta_low": 0.0015869140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.014777584932744503, "evidence_weights": [0.1207483559846878, 0.11116867512464523, 0.508949339389801], "agreement_weight": 0.025353817269206047, "orthogonal_weight": 0.41934528946876526}, {"head": 2, "memory_to_active": 0.012074703350663185, "contribution_to_active": 0.4124875068664551, "mix_mean": 0.5062427520751953, "mix_low": 0.0015783309936523438, "mix_high": 0.0023393630981445312, "read_cosine": 0.33522501587867737, "beta_mean": 2.9539718525484204e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004773169755935669, "mix_std": 0.2810860574245453, "beta_std": 5.363022501114756e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007872950518503785, "evidence_weights": [-0.24135714769363403, -0.2789212465286255, 0.47333377599716187], "agreement_weight": 0.03090008720755577, "orthogonal_weight": -0.19459228217601776}, {"head": 3, "memory_to_active": 0.19033212959766388, "contribution_to_active": 0.49793514609336853, "mix_mean": 0.514838695526123, "mix_low": 0.0004892349243164062, "mix_high": 0.007755279541015625, "read_cosine": 0.6329392194747925, "beta_mean": 0.03676462918519974, "nonfinite": 0.0, "weighted_memory_to_active": 0.11176944524049759, "mix_std": 0.26662713289260864, "beta_std": 0.04559040814638138, "beta_low": 0.2950439453125, "beta_high": 0.0, "orthogonal_to_active": 0.0742567777633667, "evidence_weights": [-0.18470056354999542, -0.28985050320625305, 0.8533574342727661], "agreement_weight": 0.05902307108044624, "orthogonal_weight": -0.8903062343597412}, {"head": 4, "memory_to_active": 3.454342842102051, "contribution_to_active": 0.26734456419944763, "mix_mean": 0.017629768699407578, "mix_low": 0.604008674621582, "mix_high": 0.0, "read_cosine": 0.20281894505023956, "beta_mean": 0.270149290561676, "nonfinite": 0.0, "weighted_memory_to_active": 0.3096579313278198, "mix_std": 0.042820002883672714, "beta_std": 0.2364410012960434, "beta_low": 0.02630615234375, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.027309469878673553, "evidence_weights": [0.01905740238726139, 0.010294062085449696, 0.0531049519777298], "agreement_weight": 0.009594921953976154, "orthogonal_weight": -0.09284921735525131}, {"head": 5, "memory_to_active": 0.015718480572104454, "contribution_to_active": 0.5102115869522095, "mix_mean": 0.6133009791374207, "mix_low": 0.0015268325805664062, "mix_high": 0.007031440734863281, "read_cosine": 0.4210909307003021, "beta_mean": 5.03647061123047e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01029992289841175, "mix_std": 0.27241599559783936, "beta_std": 7.90520862210542e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00039052224019542336, "evidence_weights": [-0.2702115774154663, -0.2861558496952057, 0.9294014573097229], "agreement_weight": 0.13881750404834747, "orthogonal_weight": 0.05560087040066719}]}, {"layer": 3, "memory_to_active": 0.5885034203529358, "contribution_to_active": 0.4260530173778534, "mix_mean": 0.4039827585220337, "mix_low": 0.08552567660808563, "mix_high": 0.00863663386553526, "read_cosine": 0.22131863236427307, "beta_mean": 0.036671288311481476, "nonfinite": 0.0, "weighted_memory_to_active": 0.15222926437854767, "mix_std": 0.25965023040771484, "beta_std": 0.030901234596967697, "beta_low": 0.6776326894760132, "beta_high": 0.0, "orthogonal_to_active": 0.06580297648906708, "output_influence": 0.34952306747436523, "residual_influence": 0.07098248600959778, "state_active": 0.022527815774083138, "state_memory": 0.008319440297782421, "state_peak": 18.99566078186035, "growth_active": 0.9493716955184937, "growth_memory": 1.780407428741455, "heads": [{"head": 0, "memory_to_active": 0.4507485628128052, "contribution_to_active": 0.5513985753059387, "mix_mean": 0.5125916004180908, "mix_low": 0.00846099853515625, "mix_high": 0.014790534973144531, "read_cosine": 0.23029392957687378, "beta_mean": 0.01228381972759962, "nonfinite": 0.0, "weighted_memory_to_active": 0.2662656605243683, "mix_std": 0.3177736699581146, "beta_std": 0.019803818315267563, "beta_low": 0.649169921875, "beta_high": 0.0, "orthogonal_to_active": 0.18013720214366913, "evidence_weights": [-0.19481658935546875, -0.2258075326681137, 0.8007009029388428], "agreement_weight": 0.12001160532236099, "orthogonal_weight": -0.7202281355857849}, {"head": 1, "memory_to_active": 0.014294525608420372, "contribution_to_active": 0.403129518032074, "mix_mean": 0.48573026061058044, "mix_low": 0.010807991027832031, "mix_high": 0.00705718994140625, "read_cosine": 0.18248048424720764, "beta_mean": 7.681461283937097e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007711050100624561, "mix_std": 0.31268954277038574, "beta_std": 0.0001346154313068837, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0055579314939677715, "evidence_weights": [-0.2302263081073761, -0.1944412887096405, 0.6012254953384399], "agreement_weight": 0.3975432217121124, "orthogonal_weight": -0.7831531763076782}, {"head": 2, "memory_to_active": 2.577986240386963, "contribution_to_active": 0.30688953399658203, "mix_mean": 0.046098195016384125, "mix_low": 0.36260032653808594, "mix_high": 0.0, "read_cosine": 0.2051529884338379, "beta_mean": 0.18743929266929626, "nonfinite": 0.0, "weighted_memory_to_active": 0.3668466806411743, "mix_std": 0.09963146597146988, "beta_std": 0.14289017021656036, "beta_low": 0.01080322265625, "beta_high": 0.0, "orthogonal_to_active": 0.0005350329447537661, "evidence_weights": [-0.0773438811302185, 0.0120372474193573, -0.01961345225572586], "agreement_weight": -0.017790503799915314, "orthogonal_weight": -0.0019135428592562675}, {"head": 3, "memory_to_active": 0.01895678974688053, "contribution_to_active": 0.4979044198989868, "mix_mean": 0.5499510765075684, "mix_low": 0.009256362915039062, "mix_high": 0.012305259704589844, "read_cosine": 0.28943654894828796, "beta_mean": 8.240468741860241e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012513821013271809, "mix_std": 0.3028196096420288, "beta_std": 0.00017861888045445085, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002918718848377466, "evidence_weights": [-0.22361309826374054, -0.22602838277816772, 2.057485342025757], "agreement_weight": 1.6216703653335571, "orthogonal_weight": -0.30467087030410767}, {"head": 4, "memory_to_active": 0.1982927918434143, "contribution_to_active": 0.17301297187805176, "mix_mean": 0.19303545355796814, "mix_low": 0.1158294677734375, "mix_high": 0.0010967254638671875, "read_cosine": 0.1863943189382553, "beta_mean": 0.0021101662423461676, "nonfinite": 0.0, "weighted_memory_to_active": 0.06678013503551483, "mix_std": 0.23772865533828735, "beta_std": 0.0029707723297178745, "beta_low": 0.97271728515625, "beta_high": 0.0, "orthogonal_to_active": 0.05821920931339264, "evidence_weights": [-0.050395816564559937, -0.09767547994852066, 0.2350592166185379], "agreement_weight": 0.0014726710505783558, "orthogonal_weight": -0.9032863974571228}, {"head": 5, "memory_to_active": 0.27074170112609863, "contribution_to_active": 0.6239829063415527, "mix_mean": 0.636489748954773, "mix_low": 0.006198883056640625, "mix_high": 0.016570091247558594, "read_cosine": 0.23415356874465942, "beta_mean": 0.01803521439433098, "nonfinite": 0.0, "weighted_memory_to_active": 0.1932581663131714, "mix_std": 0.28725844621658325, "beta_std": 0.019429398700594902, "beta_low": 0.43310546875, "beta_high": 0.0, "orthogonal_to_active": 0.14744971692562103, "evidence_weights": [-0.24305014312267303, -0.28357675671577454, 0.512410581111908], "agreement_weight": 0.13499607145786285, "orthogonal_weight": -0.819488525390625}]}, {"layer": 4, "memory_to_active": 1.15530526638031, "contribution_to_active": 0.44191235303878784, "mix_mean": 0.34977462887763977, "mix_low": 0.2846536636352539, "mix_high": 0.006273587699979544, "read_cosine": 0.34954655170440674, "beta_mean": 0.12633278965950012, "nonfinite": 0.0, "weighted_memory_to_active": 0.1386873424053192, "mix_std": 0.15400654077529907, "beta_std": 0.1149885430932045, "beta_low": 0.3709818720817566, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04315308853983879, "output_influence": 0.419512540102005, "residual_influence": 0.11508437991142273, "state_active": 0.03731696680188179, "state_memory": 0.019044747576117516, "state_peak": 40.3331298828125, "growth_active": 1.1969479322433472, "growth_memory": 1.5604450702667236, "heads": [{"head": 0, "memory_to_active": 0.22937488555908203, "contribution_to_active": 0.7032976746559143, "mix_mean": 0.6553023457527161, "mix_low": 0.013246536254882812, "mix_high": 0.014645576477050781, "read_cosine": 0.5031731128692627, "beta_mean": 0.27251869440078735, "nonfinite": 0.0, "weighted_memory_to_active": 0.16796977818012238, "mix_std": 0.34106817841529846, "beta_std": 0.27729812264442444, "beta_low": 0.212646484375, "beta_high": 0.0, "orthogonal_to_active": 0.06264863163232803, "evidence_weights": [-0.2714281380176544, -0.37146303057670593, 0.1837715208530426], "agreement_weight": 0.01833200454711914, "orthogonal_weight": 0.4532679617404938}, {"head": 1, "memory_to_active": 0.07327732443809509, "contribution_to_active": 0.6156079173088074, "mix_mean": 0.6488267779350281, "mix_low": 8.487701416015625e-05, "mix_high": 0.00860595703125, "read_cosine": 0.6542148590087891, "beta_mean": 0.0015963702462613583, "nonfinite": 0.0, "weighted_memory_to_active": 0.05126286298036575, "mix_std": 0.24654600024223328, "beta_std": 0.008424146100878716, "beta_low": 0.97589111328125, "beta_high": 0.0, "orthogonal_to_active": 0.02934887446463108, "evidence_weights": [-0.23670855164527893, -0.3331867754459381, 0.7226242423057556], "agreement_weight": 0.18752692639827728, "orthogonal_weight": -0.8167626857757568}, {"head": 2, "memory_to_active": 2.324284553527832, "contribution_to_active": 0.22684457898139954, "mix_mean": 0.017810946330428123, "mix_low": 0.5995473861694336, "mix_high": 1.52587890625e-05, "read_cosine": 0.22986289858818054, "beta_mean": 0.11698439717292786, "nonfinite": 0.0, "weighted_memory_to_active": 0.19474563002586365, "mix_std": 0.037101324647665024, "beta_std": 0.09713029116392136, "beta_low": 0.00555419921875, "beta_high": 0.0, "orthogonal_to_active": 0.019897757098078728, "evidence_weights": [0.06168872490525246, 0.031216105446219444, 0.024302193894982338], "agreement_weight": 0.1324685662984848, "orthogonal_weight": -0.1502591222524643}, {"head": 3, "memory_to_active": 1.8904584646224976, "contribution_to_active": 0.13946940004825592, "mix_mean": 0.029248088598251343, "mix_low": 0.4342832565307617, "mix_high": 0.0, "read_cosine": 0.23078793287277222, "beta_mean": 0.13231325149536133, "nonfinite": 0.0, "weighted_memory_to_active": 0.2065686136484146, "mix_std": 0.05385461449623108, "beta_std": 0.12462060898542404, "beta_low": 0.03179931640625, "beta_high": 0.0, "orthogonal_to_active": 0.007907988503575325, "evidence_weights": [-0.023081740364432335, 0.056178443133831024, -0.23255400359630585], "agreement_weight": 0.06876929104328156, "orthogonal_weight": -0.05614864081144333}, {"head": 4, "memory_to_active": 0.01812177151441574, "contribution_to_active": 0.6430583000183105, "mix_mean": 0.7344765067100525, "mix_low": 0.00011157989501953125, "mix_high": 0.014368057250976562, "read_cosine": 0.1907193511724472, "beta_mean": 3.6117755371378735e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013119383715093136, "mix_std": 0.21698246896266937, "beta_std": 0.00010370920790592209, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003307500621303916, "evidence_weights": [-0.3297044038772583, -0.33265748620033264, 0.6547255516052246], "agreement_weight": 0.3040837049484253, "orthogonal_weight": -0.27356013655662537}, {"head": 5, "memory_to_active": 2.3963141441345215, "contribution_to_active": 0.3231964111328125, "mix_mean": 0.012983016669750214, "mix_low": 0.6606483459472656, "mix_high": 6.67572021484375e-06, "read_cosine": 0.2885209023952484, "beta_mean": 0.23454783856868744, "nonfinite": 0.0, "weighted_memory_to_active": 0.1984577775001526, "mix_std": 0.028486574068665504, "beta_std": 0.18235434591770172, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.13580775260925293, "evidence_weights": [0.050255581736564636, -0.03326762840151787, -0.27328935265541077], "agreement_weight": -0.06208263337612152, "orthogonal_weight": 0.8741379976272583}]}, {"layer": 5, "memory_to_active": 0.5226221680641174, "contribution_to_active": 0.5021971464157104, "mix_mean": 0.4629274308681488, "mix_low": 0.06883367151021957, "mix_high": 0.015296141617000103, "read_cosine": 0.12467102706432343, "beta_mean": 0.061917878687381744, "nonfinite": 0.0, "weighted_memory_to_active": 0.15706127882003784, "mix_std": 0.25204524397850037, "beta_std": 0.07860556989908218, "beta_low": 0.623931884765625, "beta_high": 0.0, "orthogonal_to_active": 0.057752788066864014, "output_influence": 0.50599205493927, "residual_influence": 0.13244780898094177, "state_active": 0.04416259750723839, "state_memory": 0.012529632076621056, "state_peak": 64.96202850341797, "growth_active": 1.0781255960464478, "growth_memory": 1.3336766958236694, "heads": [{"head": 0, "memory_to_active": 0.6749417185783386, "contribution_to_active": 0.43015626072883606, "mix_mean": 0.2826077938079834, "mix_low": 0.04042816162109375, "mix_high": 0.00032520294189453125, "read_cosine": -0.14395035803318024, "beta_mean": 0.04389088600873947, "nonfinite": 0.0, "weighted_memory_to_active": 0.15787269175052643, "mix_std": 0.255556583404541, "beta_std": 0.07093682885169983, "beta_low": 0.3135986328125, "beta_high": 0.0, "orthogonal_to_active": 0.040903884917497635, "evidence_weights": [-0.10966729372739792, -0.14005272090435028, -0.1696050763130188], "agreement_weight": -0.015758488327264786, "orthogonal_weight": -0.2538296580314636}, {"head": 1, "memory_to_active": 0.017892027273774147, "contribution_to_active": 0.5422160625457764, "mix_mean": 0.661636233329773, "mix_low": 0.0020856857299804688, "mix_high": 0.02680683135986328, "read_cosine": 0.15872012078762054, "beta_mean": 4.428568354342133e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010125338099896908, "mix_std": 0.2859749495983124, "beta_std": 8.552579674869776e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009470188058912754, "evidence_weights": [-0.24259856343269348, -0.28166651725769043, 0.9047783017158508], "agreement_weight": 0.5694062113761902, "orthogonal_weight": -0.9713571667671204}, {"head": 2, "memory_to_active": 0.011481927707791328, "contribution_to_active": 0.5827997326850891, "mix_mean": 0.6405266523361206, "mix_low": 0.0032606124877929688, "mix_high": 0.01922130584716797, "read_cosine": -0.12132914364337921, "beta_mean": 0.00012474700633902103, "nonfinite": 0.0, "weighted_memory_to_active": 0.00738105783239007, "mix_std": 0.27447012066841125, "beta_std": 0.0002187691134167835, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018099030712619424, "evidence_weights": [-0.25443023443222046, -0.3464162051677704, 0.778075635433197], "agreement_weight": 0.015385663136839867, "orthogonal_weight": -0.2980417013168335}, {"head": 3, "memory_to_active": 1.848324179649353, "contribution_to_active": 0.2321234941482544, "mix_mean": 0.08924894034862518, "mix_low": 0.16205120086669922, "mix_high": 1.9073486328125e-06, "read_cosine": 0.13932043313980103, "beta_mean": 0.12766337394714355, "nonfinite": 0.0, "weighted_memory_to_active": 0.3802393674850464, "mix_std": 0.1265612542629242, "beta_std": 0.1685613989830017, "beta_low": 0.15289306640625, "beta_high": 0.0, "orthogonal_to_active": 0.1416563093662262, "evidence_weights": [-0.06396378576755524, -0.0917699858546257, 0.01917083002626896], "agreement_weight": 0.06380679458379745, "orthogonal_weight": -0.46393513679504395}, {"head": 4, "memory_to_active": 0.4412676692008972, "contribution_to_active": 0.5815579891204834, "mix_mean": 0.3160991072654724, "mix_low": 0.20218276977539062, "mix_high": 0.005974769592285156, "read_cosine": 0.5464599132537842, "beta_mean": 0.1993381530046463, "nonfinite": 0.0, "weighted_memory_to_active": 0.29034823179244995, "mix_std": 0.3377046585083008, "beta_std": 0.22924405336380005, "beta_low": 0.2835693359375, "beta_high": 0.0, "orthogonal_to_active": 0.09094350785017014, "evidence_weights": [-0.11251730471849442, -0.2794908881187439, -0.07471431791782379], "agreement_weight": 0.0398329421877861, "orthogonal_weight": 0.49727538228034973}, {"head": 5, "memory_to_active": 0.14182527363300323, "contribution_to_active": 0.6443291902542114, "mix_mean": 0.7874458432197571, "mix_low": 0.0029935836791992188, "mix_high": 0.03944683074951172, "read_cosine": 0.16880522668361664, "beta_mean": 0.00044580118265002966, "nonfinite": 0.0, "weighted_memory_to_active": 0.09640099108219147, "mix_std": 0.2320038229227066, "beta_std": 0.002586837625131011, "beta_low": 0.9935302734375, "beta_high": 0.0, "orthogonal_to_active": 0.061732932925224304, "evidence_weights": [-0.3220779597759247, -0.3263758420944214, 1.9782525300979614], "agreement_weight": 0.4168701469898224, "orthogonal_weight": -0.676400899887085}]}, {"layer": 6, "memory_to_active": 0.35216882824897766, "contribution_to_active": 0.6001965999603271, "mix_mean": 0.5614675283432007, "mix_low": 0.06387488543987274, "mix_high": 0.027621110901236534, "read_cosine": 0.28722092509269714, "beta_mean": 0.0734482854604721, "nonfinite": 0.0, "weighted_memory_to_active": 0.16879552602767944, "mix_std": 0.2375693917274475, "beta_std": 0.08140668272972107, "beta_low": 0.490631103515625, "beta_high": 0.0, "orthogonal_to_active": 0.039402276277542114, "output_influence": 0.5064959526062012, "residual_influence": 0.1669781655073166, "state_active": 0.04996541142463684, "state_memory": 0.009298540651798248, "state_peak": 93.41573333740234, "growth_active": 1.1840537786483765, "growth_memory": 1.1703325510025024, "heads": [{"head": 0, "memory_to_active": 0.15474608540534973, "contribution_to_active": 0.6413379907608032, "mix_mean": 0.7585803866386414, "mix_low": 0.00013065338134765625, "mix_high": 0.018375396728515625, "read_cosine": 0.0692235678434372, "beta_mean": 0.019872594624757767, "nonfinite": 0.0, "weighted_memory_to_active": 0.12165454030036926, "mix_std": 0.2196590006351471, "beta_std": 0.02715141326189041, "beta_low": 0.4542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.09835643321275711, "evidence_weights": [-0.3166683614253998, -0.33700644969940186, 0.857029378414154], "agreement_weight": 0.25349321961402893, "orthogonal_weight": -0.9007155895233154}, {"head": 1, "memory_to_active": 0.005526689346879721, "contribution_to_active": 0.49339139461517334, "mix_mean": 0.5378554463386536, "mix_low": 0.001689910888671875, "mix_high": 0.010240554809570312, "read_cosine": 0.5821358561515808, "beta_mean": 1.2256025001988746e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002398488577455282, "mix_std": 0.28597286343574524, "beta_std": 2.036754995060619e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013056867755949497, "evidence_weights": [-0.20090366899967194, -0.2939116060733795, 1.0981152057647705], "agreement_weight": 0.05962091311812401, "orthogonal_weight": -0.736078143119812}, {"head": 2, "memory_to_active": 1.030835509300232, "contribution_to_active": 0.09143975377082825, "mix_mean": 0.033287182450294495, "mix_low": 0.3729705810546875, "mix_high": 0.0, "read_cosine": 0.22576574981212616, "beta_mean": 0.056148599833250046, "nonfinite": 0.0, "weighted_memory_to_active": 0.07161817699670792, "mix_std": 0.055092357099056244, "beta_std": 0.05688989534974098, "beta_low": 0.0440673828125, "beta_high": 0.0, "orthogonal_to_active": 0.0003113096463494003, "evidence_weights": [0.08105097711086273, -0.022047009319067, 0.28165945410728455], "agreement_weight": 0.142785906791687, "orthogonal_weight": 0.005347012542188168}, {"head": 3, "memory_to_active": 0.14943131804466248, "contribution_to_active": 0.6705834865570068, "mix_mean": 0.6392503976821899, "mix_low": 0.0074596405029296875, "mix_high": 0.015080451965332031, "read_cosine": 0.46062541007995605, "beta_mean": 0.2288733720779419, "nonfinite": 0.0, "weighted_memory_to_active": 0.10754276067018509, "mix_std": 0.3255212903022766, "beta_std": 0.25285226106643677, "beta_low": 0.186279296875, "beta_high": 0.0, "orthogonal_to_active": 0.060106176882982254, "evidence_weights": [-0.17537617683410645, -0.3364863991737366, 0.20792458951473236], "agreement_weight": -0.016446681693196297, "orthogonal_weight": 0.6438871622085571}, {"head": 4, "memory_to_active": 0.7394634485244751, "contribution_to_active": 1.1093746423721313, "mix_mean": 0.7625672221183777, "mix_low": 0.00043392181396484375, "mix_high": 0.11884307861328125, "read_cosine": 0.11903929710388184, "beta_mean": 0.1355508267879486, "nonfinite": 0.0, "weighted_memory_to_active": 0.6869698166847229, "mix_std": 0.2826768755912781, "beta_std": 0.1509881317615509, "beta_low": 0.259765625, "beta_high": 0.0, "orthogonal_to_active": 0.05375590920448303, "evidence_weights": [-0.24698178470134735, -0.5989600419998169, -0.16990764439105988], "agreement_weight": -0.016941774636507034, "orthogonal_weight": 0.09405464679002762}, {"head": 5, "memory_to_active": 0.03300987184047699, "contribution_to_active": 0.5950521230697632, "mix_mean": 0.6372643709182739, "mix_low": 0.0005645751953125, "mix_high": 0.0031871795654296875, "read_cosine": 0.26653555035591125, "beta_mean": 0.00023203101591207087, "nonfinite": 0.0, "weighted_memory_to_active": 0.022589383646845818, "mix_std": 0.2564939856529236, "beta_std": 0.0005380252259783447, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.022578144446015358, "evidence_weights": [-0.24223607778549194, -0.3102419674396515, 0.43659108877182007], "agreement_weight": 0.14412705600261688, "orthogonal_weight": -1.0671448707580566}]}, {"layer": 7, "memory_to_active": 1.7743675708770752, "contribution_to_active": 0.3093637526035309, "mix_mean": 0.12125158309936523, "mix_low": 0.2686729431152344, "mix_high": 0.0002557436819188297, "read_cosine": 0.2572067975997925, "beta_mean": 0.13110846281051636, "nonfinite": 0.0, "weighted_memory_to_active": 0.2831340432167053, "mix_std": 0.13329339027404785, "beta_std": 0.10169579088687897, "beta_low": 0.1919657438993454, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.0653916522860527, "output_influence": 0.2579384446144104, "residual_influence": 0.05835816264152527, "state_active": 0.01344138104468584, "state_memory": 0.029279984533786774, "state_peak": 53.7861442565918, "growth_active": 1.139356017112732, "growth_memory": 1.3421740531921387, "heads": [{"head": 0, "memory_to_active": 2.808651924133301, "contribution_to_active": 0.7073156833648682, "mix_mean": 0.03719101846218109, "mix_low": 0.4188833236694336, "mix_high": 0.0, "read_cosine": 0.3801274299621582, "beta_mean": 0.15824812650680542, "nonfinite": 0.0, "weighted_memory_to_active": 0.7679062485694885, "mix_std": 0.07604534924030304, "beta_std": 0.11752022057771683, "beta_low": 0.0006103515625, "beta_high": 0.00048828125, "orthogonal_to_active": 0.015866080299019814, "evidence_weights": [0.018427137285470963, -0.08004529029130936, 0.0963030457496643], "agreement_weight": -0.038195155560970306, "orthogonal_weight": 0.03897571936249733}, {"head": 1, "memory_to_active": 1.0364043712615967, "contribution_to_active": 0.3789765536785126, "mix_mean": 0.20801123976707458, "mix_low": 0.13551998138427734, "mix_high": 0.0013818740844726562, "read_cosine": 0.3728145360946655, "beta_mean": 0.10866769403219223, "nonfinite": 0.0, "weighted_memory_to_active": 0.34158340096473694, "mix_std": 0.23183374106884003, "beta_std": 0.06824883073568344, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.021360810846090317, "evidence_weights": [-0.20356762409210205, -0.11647161096334457, -0.19619207084178925], "agreement_weight": 0.03536897152662277, "orthogonal_weight": 0.07380663603544235}, {"head": 2, "memory_to_active": 0.819710910320282, "contribution_to_active": 0.29009613394737244, "mix_mean": 0.13215239346027374, "mix_low": 0.07946109771728516, "mix_high": 0.00010395050048828125, "read_cosine": 0.14078067243099213, "beta_mean": 0.06353326886892319, "nonfinite": 0.0, "weighted_memory_to_active": 0.2000778466463089, "mix_std": 0.16502931714057922, "beta_std": 0.048351529985666275, "beta_low": 0.0360107421875, "beta_high": 0.0, "orthogonal_to_active": 0.0764913260936737, "evidence_weights": [-0.05258019641041756, -0.13784024119377136, 0.5407384634017944], "agreement_weight": 0.025005273520946503, "orthogonal_weight": -0.3908889889717102}, {"head": 3, "memory_to_active": 4.2047505378723145, "contribution_to_active": 0.0868106484413147, "mix_mean": 0.03518681973218918, "mix_low": 0.31864166259765625, "mix_high": 0.0, "read_cosine": 0.14001013338565826, "beta_mean": 0.3600584864616394, "nonfinite": 0.0, "weighted_memory_to_active": 0.30159854888916016, "mix_std": 0.05136340856552124, "beta_std": 0.27674469351768494, "beta_low": 0.0078125, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.253406286239624, "evidence_weights": [0.05842436105012894, -0.016126425936818123, 0.6795603036880493], "agreement_weight": 0.05040612444281578, "orthogonal_weight": -0.9251964688301086}, {"head": 4, "memory_to_active": 1.6983259916305542, "contribution_to_active": 0.07794671505689621, "mix_mean": 0.016004487872123718, "mix_low": 0.6502418518066406, "mix_high": 0.0, "read_cosine": 0.23929618299007416, "beta_mean": 0.09565488249063492, "nonfinite": 0.0, "weighted_memory_to_active": 0.06014489382505417, "mix_std": 0.03612741455435753, "beta_std": 0.09836884588003159, "beta_low": 0.10791015625, "beta_high": 0.0, "orthogonal_to_active": 0.008434698916971684, "evidence_weights": [0.13911958038806915, 0.04345521330833435, 0.25302138924598694], "agreement_weight": 0.04384193196892738, "orthogonal_weight": -0.16679567098617554}, {"head": 5, "memory_to_active": 0.0783618614077568, "contribution_to_active": 0.3150368928909302, "mix_mean": 0.2989635467529297, "mix_low": 0.009289741516113281, "mix_high": 4.863739013671875e-05, "read_cosine": 0.27021175622940063, "beta_mean": 0.0004882889916189015, "nonfinite": 0.0, "weighted_memory_to_active": 0.027493217960000038, "mix_std": 0.23936104774475098, "beta_std": 0.0009406032040715218, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.016790710389614105, "evidence_weights": [-0.11762838810682297, -0.1766028106212616, 0.5172806978225708], "agreement_weight": 0.11364960670471191, "orthogonal_weight": -0.6857277154922485}]}, {"layer": 8, "memory_to_active": 0.3974616527557373, "contribution_to_active": 0.527469277381897, "mix_mean": 0.42327114939689636, "mix_low": 0.08330965042114258, "mix_high": 0.001054922817274928, "read_cosine": 0.4618484377861023, "beta_mean": 0.025788798928260803, "nonfinite": 0.0, "weighted_memory_to_active": 0.09436771273612976, "mix_std": 0.22074462473392487, "beta_std": 0.033912595361471176, "beta_low": 0.6227213740348816, "beta_high": 0.0, "orthogonal_to_active": 0.06565351784229279, "output_influence": 0.457996666431427, "residual_influence": 0.1450229436159134, "state_active": 0.047634851187467575, "state_memory": 0.00892028771340847, "state_peak": 36.89799118041992, "growth_active": 1.0786442756652832, "growth_memory": 1.2966101169586182, "heads": [{"head": 0, "memory_to_active": 0.05794500559568405, "contribution_to_active": 0.5509439706802368, "mix_mean": 0.5936301946640015, "mix_low": 0.00029087066650390625, "mix_high": 0.0010900497436523438, "read_cosine": 0.4957984983921051, "beta_mean": 0.0027484861202538013, "nonfinite": 0.0, "weighted_memory_to_active": 0.04041919857263565, "mix_std": 0.24313269555568695, "beta_std": 0.008843145333230495, "beta_low": 0.938720703125, "beta_high": 0.0, "orthogonal_to_active": 0.02524341456592083, "evidence_weights": [-0.17026682198047638, -0.39180195331573486, 0.6875113248825073], "agreement_weight": 0.1501084864139557, "orthogonal_weight": -0.8001628518104553}, {"head": 1, "memory_to_active": 0.12153664231300354, "contribution_to_active": 0.5009430050849915, "mix_mean": 0.46453142166137695, "mix_low": 0.00032138824462890625, "mix_high": 0.000247955322265625, "read_cosine": 0.2977316975593567, "beta_mean": 0.002382212318480015, "nonfinite": 0.0, "weighted_memory_to_active": 0.0562608540058136, "mix_std": 0.24714645743370056, "beta_std": 0.01328823622316122, "beta_low": 0.9715576171875, "beta_high": 0.0, "orthogonal_to_active": 0.037516139447689056, "evidence_weights": [-0.18711605668067932, -0.2590804994106293, 0.6646384000778198], "agreement_weight": 0.08940578252077103, "orthogonal_weight": -0.7344453930854797}, {"head": 2, "memory_to_active": 0.01794528402388096, "contribution_to_active": 0.6529327630996704, "mix_mean": 0.6320721507072449, "mix_low": 0.0010347366333007812, "mix_high": 0.0032281875610351562, "read_cosine": 0.3984755873680115, "beta_mean": 0.0001846005761763081, "nonfinite": 0.0, "weighted_memory_to_active": 0.013611111789941788, "mix_std": 0.2336254119873047, "beta_std": 0.00257409387268126, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.008982710540294647, "evidence_weights": [-0.22202180325984955, -0.34091389179229736, 0.9643468856811523], "agreement_weight": 0.22121116518974304, "orthogonal_weight": -0.7579182386398315}, {"head": 3, "memory_to_active": 0.12644809484481812, "contribution_to_active": 0.6272951364517212, "mix_mean": 0.55732262134552, "mix_low": 0.0021963119506835938, "mix_high": 0.001636505126953125, "read_cosine": 0.7870503067970276, "beta_mean": 0.011344235390424728, "nonfinite": 0.0, "weighted_memory_to_active": 0.09473930299282074, "mix_std": 0.25772663950920105, "beta_std": 0.009111345745623112, "beta_low": 0.53509521484375, "beta_high": 0.0, "orthogonal_to_active": 0.031584884971380234, "evidence_weights": [-0.18288317322731018, -0.30424100160598755, 0.6278728246688843], "agreement_weight": 0.18356828391551971, "orthogonal_weight": -0.6426323652267456}, {"head": 4, "memory_to_active": 1.7166225910186768, "contribution_to_active": 0.17416200041770935, "mix_mean": 0.033685214817523956, "mix_low": 0.37521839141845703, "mix_high": 0.0, "read_cosine": 0.3183403015136719, "beta_mean": 0.06140812113881111, "nonfinite": 0.0, "weighted_memory_to_active": 0.16628031432628632, "mix_std": 0.05604031682014465, "beta_std": 0.07364747673273087, "beta_low": 0.135009765625, "beta_high": 0.0, "orthogonal_to_active": 0.0007253257208503783, "evidence_weights": [0.05536098778247833, -0.03565794974565506, -0.04732091352343559], "agreement_weight": 0.06926035135984421, "orthogonal_weight": 0.004657726734876633}, {"head": 5, "memory_to_active": 0.3442721366882324, "contribution_to_active": 0.6585387587547302, "mix_mean": 0.25838521122932434, "mix_low": 0.12079620361328125, "mix_high": 0.00012683868408203125, "read_cosine": 0.47369420528411865, "beta_mean": 0.07666513323783875, "nonfinite": 0.0, "weighted_memory_to_active": 0.19489550590515137, "mix_std": 0.2867961525917053, "beta_std": 0.09601128101348877, "beta_low": 0.15606689453125, "beta_high": 0.0, "orthogonal_to_active": 0.28986868262290955, "evidence_weights": [-0.07896196842193604, -0.237150639295578, -0.2739103138446808], "agreement_weight": -0.007843381725251675, "orthogonal_weight": 1.7650216817855835}]}, {"layer": 9, "memory_to_active": 1.1343377828598022, "contribution_to_active": 0.6615002155303955, "mix_mean": 0.3228515386581421, "mix_low": 0.14538638293743134, "mix_high": 0.0037212371826171875, "read_cosine": 0.3788105845451355, "beta_mean": 0.07175937294960022, "nonfinite": 0.0, "weighted_memory_to_active": 0.39178457856178284, "mix_std": 0.21133291721343994, "beta_std": 0.04740038886666298, "beta_low": 0.5015360713005066, "beta_high": 0.0, "orthogonal_to_active": 0.0388687327504158, "output_influence": 0.49815356731414795, "residual_influence": 0.13752050697803497, "state_active": 0.03965531662106514, "state_memory": 0.020116757601499557, "state_peak": 41.070068359375, "growth_active": 1.1206458806991577, "growth_memory": 1.3751670122146606, "heads": [{"head": 0, "memory_to_active": 0.0021778785157948732, "contribution_to_active": 0.5466703176498413, "mix_mean": 0.6184321641921997, "mix_low": 0.001567840576171875, "mix_high": 0.01678466796875, "read_cosine": 0.25737473368644714, "beta_mean": 4.373503998067463e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0013838965678587556, "mix_std": 0.2915654480457306, "beta_std": 1.692517616902478e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012323677074164152, "evidence_weights": [-0.2950085699558258, -0.3041279911994934, 0.8863394260406494], "agreement_weight": 0.1104608103632927, "orthogonal_weight": -0.9390969276428223}, {"head": 1, "memory_to_active": 0.019137779250741005, "contribution_to_active": 0.630853533744812, "mix_mean": 0.5061235427856445, "mix_low": 0.00865936279296875, "mix_high": 0.0017232894897460938, "read_cosine": 0.41333654522895813, "beta_mean": 5.9784899349324405e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01258391048759222, "mix_std": 0.2870761454105377, "beta_std": 0.00011379289207980037, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00880839116871357, "evidence_weights": [-0.22362443804740906, -0.2726283073425293, 1.1131117343902588], "agreement_weight": 0.36900782585144043, "orthogonal_weight": -0.8500948548316956}, {"head": 2, "memory_to_active": 0.17002896964550018, "contribution_to_active": 0.6282111406326294, "mix_mean": 0.5649930238723755, "mix_low": 0.0016765594482421875, "mix_high": 0.0033426284790039062, "read_cosine": 0.3708949387073517, "beta_mean": 0.0007273706723935902, "nonfinite": 0.0, "weighted_memory_to_active": 0.11295077204704285, "mix_std": 0.2703607976436615, "beta_std": 0.003833048976957798, "beta_low": 0.99493408203125, "beta_high": 0.0, "orthogonal_to_active": 0.06065753474831581, "evidence_weights": [-0.19604036211967468, -0.2656843662261963, 0.7086286544799805], "agreement_weight": 0.27805063128471375, "orthogonal_weight": -0.634274423122406}, {"head": 3, "memory_to_active": 1.547489881515503, "contribution_to_active": 0.1835160255432129, "mix_mean": 0.044244032353162766, "mix_low": 0.37944793701171875, "mix_high": 2.765655517578125e-05, "read_cosine": 0.4185766279697418, "beta_mean": 0.09018202126026154, "nonfinite": 0.0, "weighted_memory_to_active": 0.1524510681629181, "mix_std": 0.09660094231367111, "beta_std": 0.056631118059158325, "beta_low": 0.00189208984375, "beta_high": 0.0, "orthogonal_to_active": 0.015116678550839424, "evidence_weights": [0.07916919887065887, 0.021261872723698616, -0.013553391210734844], "agreement_weight": 0.08588599413633347, "orthogonal_weight": 0.10651115328073502}, {"head": 4, "memory_to_active": 3.4225006103515625, "contribution_to_active": 1.3981218338012695, "mix_mean": 0.12827108800411224, "mix_low": 0.15425777435302734, "mix_high": 0.00034046173095703125, "read_cosine": 0.30854639410972595, "beta_mean": 0.19044190645217896, "nonfinite": 0.0, "weighted_memory_to_active": 1.4269051551818848, "mix_std": 0.18831095099449158, "beta_std": 0.10389912873506546, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.028044458478689194, "evidence_weights": [-0.06671702861785889, -0.15098430216312408, -0.013712621293962002], "agreement_weight": -0.055678267031908035, "orthogonal_weight": -0.021089958027005196}, {"head": 5, "memory_to_active": 1.6446913480758667, "contribution_to_active": 0.5816280841827393, "mix_mean": 0.07504545152187347, "mix_low": 0.3267087936401367, "mix_high": 0.0001087188720703125, "read_cosine": 0.5041342377662659, "beta_mean": 0.14914074540138245, "nonfinite": 0.0, "weighted_memory_to_active": 0.6444326639175415, "mix_std": 0.13408319652080536, "beta_std": 0.11990830302238464, "beta_low": 0.01239013671875, "beta_high": 0.0, "orthogonal_to_active": 0.11935295164585114, "evidence_weights": [0.10695141553878784, -0.07951822131872177, 0.15145407617092133], "agreement_weight": 0.01627720519900322, "orthogonal_weight": 0.27333056926727295}]}, {"layer": 10, "memory_to_active": 0.2504224181175232, "contribution_to_active": 0.6751868724822998, "mix_mean": 0.5035200715065002, "mix_low": 0.018884819000959396, "mix_high": 0.006367683410644531, "read_cosine": 0.3704998791217804, "beta_mean": 0.04650406166911125, "nonfinite": 0.0, "weighted_memory_to_active": 0.1232042983174324, "mix_std": 0.2691223621368408, "beta_std": 0.046973273158073425, "beta_low": 0.6579691767692566, "beta_high": 0.0, "orthogonal_to_active": 0.1913384199142456, "output_influence": 0.4319531321525574, "residual_influence": 0.14685475826263428, "state_active": 0.050607845187187195, "state_memory": 0.014738870784640312, "state_peak": 120.94068145751953, "growth_active": 1.0463217496871948, "growth_memory": 0.9639047980308533, "heads": [{"head": 0, "memory_to_active": 0.9893451929092407, "contribution_to_active": 1.8272150754928589, "mix_mean": 0.5834972858428955, "mix_low": 0.07120323181152344, "mix_high": 0.020567893981933594, "read_cosine": 0.09359616786241531, "beta_mean": 0.2059340626001358, "nonfinite": 0.0, "weighted_memory_to_active": 0.6165958642959595, "mix_std": 0.39013391733169556, "beta_std": 0.21537475287914276, "beta_low": 0.12530517578125, "beta_high": 0.0, "orthogonal_to_active": 1.0641382932662964, "evidence_weights": [-0.1189681813120842, -0.38923338055610657, -1.2739735841751099], "agreement_weight": 0.0157682616263628, "orthogonal_weight": 1.7459083795547485}, {"head": 1, "memory_to_active": 0.016482532024383545, "contribution_to_active": 0.5946725606918335, "mix_mean": 0.6055285930633545, "mix_low": 0.0009441375732421875, "mix_high": 0.011471748352050781, "read_cosine": 0.27660882472991943, "beta_mean": 0.00010303972521796823, "nonfinite": 0.0, "weighted_memory_to_active": 0.009035182185471058, "mix_std": 0.2685566246509552, "beta_std": 0.00015107073704712093, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013191744685173035, "evidence_weights": [-0.26428282260894775, -0.20173344016075134, 0.5837047696113586], "agreement_weight": 0.46937552094459534, "orthogonal_weight": -1.5771056413650513}, {"head": 2, "memory_to_active": 0.021509749814867973, "contribution_to_active": 0.5997795462608337, "mix_mean": 0.6190379858016968, "mix_low": 0.0002994537353515625, "mix_high": 0.005581855773925781, "read_cosine": 0.03017299436032772, "beta_mean": 4.3443389586173e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008869207464158535, "mix_std": 0.2557927072048187, "beta_std": 0.0002629064256325364, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0041519273072481155, "evidence_weights": [-0.24116982519626617, -0.2747359573841095, 0.5690703392028809], "agreement_weight": -0.15553052723407745, "orthogonal_weight": -0.44677022099494934}, {"head": 3, "memory_to_active": 0.33495718240737915, "contribution_to_active": 0.10115494579076767, "mix_mean": 0.27707356214523315, "mix_low": 0.03757476806640625, "mix_high": 8.58306884765625e-05, "read_cosine": 0.9097211360931396, "beta_mean": 0.06627917289733887, "nonfinite": 0.0, "weighted_memory_to_active": 0.03844092786312103, "mix_std": 0.22304993867874146, "beta_std": 0.05798838287591934, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.045027125626802444, "evidence_weights": [-0.10965224355459213, -0.14487165212631226, 0.13277912139892578], "agreement_weight": 0.0024263362865895033, "orthogonal_weight": 1.470901608467102}, {"head": 4, "memory_to_active": 0.004187737591564655, "contribution_to_active": 0.5214035511016846, "mix_mean": 0.5461439490318298, "mix_low": 0.0003814697265625, "mix_high": 0.0004949569702148438, "read_cosine": 0.053717076778411865, "beta_mean": 2.4608507374068722e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022763414308428764, "mix_std": 0.24877461791038513, "beta_std": 6.0836318880319595e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019553834572434425, "evidence_weights": [-0.1812940090894699, -0.2599940001964569, 0.7920039892196655], "agreement_weight": 0.08696015179157257, "orthogonal_weight": -0.8988862633705139}, {"head": 5, "memory_to_active": 0.13605210185050964, "contribution_to_active": 0.4068954586982727, "mix_mean": 0.3898388147354126, "mix_low": 0.0029058456420898438, "mix_high": 3.814697265625e-06, "read_cosine": 0.8591830730438232, "beta_mean": 0.006640034727752209, "nonfinite": 0.0, "weighted_memory_to_active": 0.06400828808546066, "mix_std": 0.2284262478351593, "beta_std": 0.008001681417226791, "beta_low": 0.82183837890625, "beta_high": 0.0, "orthogonal_to_active": 0.019565967842936516, "evidence_weights": [-0.1477946937084198, -0.1614355891942978, 0.2858937978744507], "agreement_weight": 0.09078510850667953, "orthogonal_weight": -0.634771466255188}]}, {"layer": 11, "memory_to_active": 0.11457476019859314, "contribution_to_active": 0.590816855430603, "mix_mean": 0.48135891556739807, "mix_low": 0.10088380426168442, "mix_high": 0.04558531567454338, "read_cosine": 0.4037717878818512, "beta_mean": 0.03091762587428093, "nonfinite": 0.0, "weighted_memory_to_active": 0.0551937073469162, "mix_std": 0.3233499526977539, "beta_std": 0.0455646812915802, "beta_low": 0.6100565791130066, "beta_high": 0.0, "orthogonal_to_active": 0.050005435943603516, "output_influence": 0.5224698781967163, "residual_influence": 0.12831154465675354, "state_active": 0.19963596761226654, "state_memory": 0.0063429600559175014, "state_peak": 28.902774810791016, "growth_active": 1.0611438751220703, "growth_memory": 1.677157998085022, "heads": [{"head": 0, "memory_to_active": 0.008126501925289631, "contribution_to_active": 0.5825128555297852, "mix_mean": 0.6220603585243225, "mix_low": 0.0010166168212890625, "mix_high": 0.008376121520996094, "read_cosine": 0.34347373247146606, "beta_mean": 8.068255556281656e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00495614018291235, "mix_std": 0.2628001570701599, "beta_std": 0.0038858039770275354, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0036921524442732334, "evidence_weights": [-0.16800101101398468, -0.36764347553253174, 0.7046785950660706], "agreement_weight": 0.2832030951976776, "orthogonal_weight": -0.7887928485870361}, {"head": 1, "memory_to_active": 0.2692227363586426, "contribution_to_active": 0.32137125730514526, "mix_mean": 0.37060102820396423, "mix_low": 0.011126518249511719, "mix_high": 0.000141143798828125, "read_cosine": 0.5490024089813232, "beta_mean": 0.02567896991968155, "nonfinite": 0.0, "weighted_memory_to_active": 0.11618320643901825, "mix_std": 0.26061755418777466, "beta_std": 0.05099654197692871, "beta_low": 0.50469970703125, "beta_high": 0.0, "orthogonal_to_active": 0.04133929684758186, "evidence_weights": [-0.16507959365844727, -0.184991717338562, 0.47333601117134094], "agreement_weight": 0.07221759855747223, "orthogonal_weight": -0.44962409138679504}, {"head": 2, "memory_to_active": 0.011555084958672523, "contribution_to_active": 0.9793686866760254, "mix_mean": 0.432746946811676, "mix_low": 0.2578306198120117, "mix_high": 0.16428470611572266, "read_cosine": 0.1928783506155014, "beta_mean": 0.07705074548721313, "nonfinite": 0.0, "weighted_memory_to_active": 0.008449138142168522, "mix_std": 0.4444121718406677, "beta_std": 0.10217572748661041, "beta_low": 0.4390869140625, "beta_high": 0.0, "orthogonal_to_active": 0.05478520691394806, "evidence_weights": [0.041627559810876846, -0.4356826841831207, -1.2634598016738892], "agreement_weight": 0.005975320469588041, "orthogonal_weight": 6.687125205993652}, {"head": 3, "memory_to_active": 0.011010747402906418, "contribution_to_active": 0.8536248803138733, "mix_mean": 0.40204980969429016, "mix_low": 0.32418060302734375, "mix_high": 0.09534645080566406, "read_cosine": 0.5446203351020813, "beta_mean": 0.059228092432022095, "nonfinite": 0.0, "weighted_memory_to_active": 0.006802695337682962, "mix_std": 0.4402969181537628, "beta_std": 0.08768492937088013, "beta_low": 0.4786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.03353818506002426, "evidence_weights": [0.14879414439201355, -0.3268699645996094, -0.2711564600467682], "agreement_weight": 0.04711449146270752, "orthogonal_weight": 7.010732650756836}, {"head": 4, "memory_to_active": 0.29024645686149597, "contribution_to_active": 0.4176042675971985, "mix_mean": 0.5309228897094727, "mix_low": 0.009436607360839844, "mix_high": 0.000957489013671875, "read_cosine": 0.10547569394111633, "beta_mean": 0.01569516211748123, "nonfinite": 0.0, "weighted_memory_to_active": 0.14771917462348938, "mix_std": 0.27229833602905273, "beta_std": 0.01684909500181675, "beta_low": 0.4698486328125, "beta_high": 0.0, "orthogonal_to_active": 0.14210256934165955, "evidence_weights": [-0.23307748138904572, -0.21342384815216064, 1.8979463577270508], "agreement_weight": -0.1924286186695099, "orthogonal_weight": -0.976713240146637}, {"head": 5, "memory_to_active": 0.09728696942329407, "contribution_to_active": 0.3904188275337219, "mix_mean": 0.5297724008560181, "mix_low": 0.0017118453979492188, "mix_high": 0.004405975341796875, "read_cosine": 0.6871799826622009, "beta_mean": 0.007772095501422882, "nonfinite": 0.0, "weighted_memory_to_active": 0.047051891684532166, "mix_std": 0.2596745491027832, "beta_std": 0.011795997619628906, "beta_low": 0.76812744140625, "beta_high": 0.0, "orthogonal_to_active": 0.024575194343924522, "evidence_weights": [-0.2188195288181305, -0.298126220703125, 0.6041377782821655], "agreement_weight": 0.07986002415418625, "orthogonal_weight": -0.7210568189620972}]}, {"layer": 12, "memory_to_active": 0.25448402762413025, "contribution_to_active": 0.5528558492660522, "mix_mean": 0.5107528567314148, "mix_low": 0.009193738922476768, "mix_high": 0.004699866287410259, "read_cosine": 0.25813359022140503, "beta_mean": 0.016673346981406212, "nonfinite": 0.0, "weighted_memory_to_active": 0.1375802755355835, "mix_std": 0.26104244589805603, "beta_std": 0.01536529790610075, "beta_low": 0.8243408203125, "beta_high": 0.0, "orthogonal_to_active": 0.06918705254793167, "output_influence": 0.45970213413238525, "residual_influence": 0.12793974578380585, "state_active": 0.036348260939121246, "state_memory": 0.009710259735584259, "state_peak": 52.18260955810547, "growth_active": 1.103594183921814, "growth_memory": 1.2818865776062012, "heads": [{"head": 0, "memory_to_active": 0.049317095428705215, "contribution_to_active": 0.5221152901649475, "mix_mean": 0.5159580111503601, "mix_low": 0.001300811767578125, "mix_high": 0.00258636474609375, "read_cosine": 0.3320176899433136, "beta_mean": 0.000768361147493124, "nonfinite": 0.0, "weighted_memory_to_active": 0.023387987166643143, "mix_std": 0.2694300413131714, "beta_std": 0.0025451139081269503, "beta_low": 0.9947509765625, "beta_high": 0.0, "orthogonal_to_active": 0.01939779706299305, "evidence_weights": [-0.19734342396259308, -0.26853805780410767, 0.2335807830095291], "agreement_weight": 0.10082174092531204, "orthogonal_weight": -0.8860586881637573}, {"head": 1, "memory_to_active": 0.11258290708065033, "contribution_to_active": 0.5156548619270325, "mix_mean": 0.5373849868774414, "mix_low": 0.0007448196411132812, "mix_high": 0.0010023117065429688, "read_cosine": 0.48534905910491943, "beta_mean": 0.001348665333352983, "nonfinite": 0.0, "weighted_memory_to_active": 0.06238778680562973, "mix_std": 0.2625986933708191, "beta_std": 0.007591123692691326, "beta_low": 0.98077392578125, "beta_high": 0.0, "orthogonal_to_active": 0.0502212792634964, "evidence_weights": [-0.20881323516368866, -0.2746501863002777, 0.9715490341186523], "agreement_weight": 0.15442471206188202, "orthogonal_weight": -0.9251072406768799}, {"head": 2, "memory_to_active": 0.6093501448631287, "contribution_to_active": 0.4805006682872772, "mix_mean": 0.3248400092124939, "mix_low": 0.02523326873779297, "mix_high": 7.62939453125e-06, "read_cosine": 0.47390931844711304, "beta_mean": 0.09641264379024506, "nonfinite": 0.0, "weighted_memory_to_active": 0.2546826899051666, "mix_std": 0.240364670753479, "beta_std": 0.06581463664770126, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.24006137251853943, "evidence_weights": [-0.24055913090705872, -0.050003353506326675, -0.07350939512252808], "agreement_weight": 0.020825644955039024, "orthogonal_weight": 1.2204678058624268}, {"head": 3, "memory_to_active": 0.03550412505865097, "contribution_to_active": 0.7232310771942139, "mix_mean": 0.6914529204368591, "mix_low": 0.000270843505859375, "mix_high": 0.015522956848144531, "read_cosine": 0.0949544757604599, "beta_mean": 0.0002640873717609793, "nonfinite": 0.0, "weighted_memory_to_active": 0.025929264724254608, "mix_std": 0.23668238520622253, "beta_std": 0.0042012156918644905, "beta_low": 0.998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.024645695462822914, "evidence_weights": [-0.28913119435310364, -0.3396066427230835, 1.3502097129821777], "agreement_weight": 0.3944770395755768, "orthogonal_weight": -0.9746878743171692}, {"head": 4, "memory_to_active": 0.6632025241851807, "contribution_to_active": 0.5996952652931213, "mix_mean": 0.37516140937805176, "mix_low": 0.02047443389892578, "mix_high": 0.003223419189453125, "read_cosine": -0.08374877274036407, "beta_mean": 0.0008781867800280452, "nonfinite": 0.0, "weighted_memory_to_active": 0.4225141108036041, "mix_std": 0.2791599929332733, "beta_std": 0.009593519382178783, "beta_low": 0.97967529296875, "beta_high": 0.0, "orthogonal_to_active": 0.052395958453416824, "evidence_weights": [-0.1248357743024826, -0.21610772609710693, 0.012554571032524109], "agreement_weight": -0.0030886393506079912, "orthogonal_weight": -0.13046333193778992}, {"head": 5, "memory_to_active": 0.05694718658924103, "contribution_to_active": 0.4759380519390106, "mix_mean": 0.6197196841239929, "mix_low": 0.007138252258300781, "mix_high": 0.005856513977050781, "read_cosine": 0.24631962180137634, "beta_mean": 0.0003681327507365495, "nonfinite": 0.0, "weighted_memory_to_active": 0.036579810082912445, "mix_std": 0.2780187129974365, "beta_std": 0.0024461678694933653, "beta_low": 0.9920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.028400210663676262, "evidence_weights": [-0.24876545369625092, -0.3129849135875702, 1.262597680091858], "agreement_weight": 0.32325005531311035, "orthogonal_weight": -0.8357681632041931}]}, {"layer": 13, "memory_to_active": 0.13311807811260223, "contribution_to_active": 0.5257359743118286, "mix_mean": 0.5269787311553955, "mix_low": 0.005006313323974609, "mix_high": 0.006428241729736328, "read_cosine": 0.4130023121833801, "beta_mean": 0.01872786320745945, "nonfinite": 0.0, "weighted_memory_to_active": 0.05785603076219559, "mix_std": 0.25669318437576294, "beta_std": 0.02089015208184719, "beta_low": 0.794097900390625, "beta_high": 0.0, "orthogonal_to_active": 0.059503138065338135, "output_influence": 0.48707258701324463, "residual_influence": 0.1637614369392395, "state_active": 0.10647164285182953, "state_memory": 0.039330486208200455, "state_peak": 94.95906066894531, "growth_active": 0.91779625415802, "growth_memory": 1.0171384811401367, "heads": [{"head": 0, "memory_to_active": 0.032798632979393005, "contribution_to_active": 0.5722914934158325, "mix_mean": 0.5844680666923523, "mix_low": 0.0003833770751953125, "mix_high": 0.0020313262939453125, "read_cosine": 0.4265694320201874, "beta_mean": 0.00012806218001060188, "nonfinite": 0.0, "weighted_memory_to_active": 0.020410781726241112, "mix_std": 0.2673145532608032, "beta_std": 0.0002822046517394483, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01988258585333824, "evidence_weights": [-0.20119357109069824, -0.2655048668384552, 0.7535884976387024], "agreement_weight": 0.11910835653543472, "orthogonal_weight": -1.0991168022155762}, {"head": 1, "memory_to_active": 0.012385244481265545, "contribution_to_active": 0.5894057750701904, "mix_mean": 0.554167628288269, "mix_low": 0.0015306472778320312, "mix_high": 0.0025787353515625, "read_cosine": 0.2289000153541565, "beta_mean": 4.495550456340425e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007150651887059212, "mix_std": 0.2729973793029785, "beta_std": 8.044046990107745e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008685361593961716, "evidence_weights": [-0.18780608475208282, -0.2710888683795929, 0.6855192184448242], "agreement_weight": 0.3412809669971466, "orthogonal_weight": -1.2725000381469727}, {"head": 2, "memory_to_active": 0.10860127955675125, "contribution_to_active": 0.44631683826446533, "mix_mean": 0.3844679892063141, "mix_low": 0.0023708343505859375, "mix_high": 7.915496826171875e-05, "read_cosine": 0.6609396934509277, "beta_mean": 0.01839246228337288, "nonfinite": 0.0, "weighted_memory_to_active": 0.049181923270225525, "mix_std": 0.2282775193452835, "beta_std": 0.053311318159103394, "beta_low": 0.78375244140625, "beta_high": 0.0, "orthogonal_to_active": 0.025075051933526993, "evidence_weights": [-0.19705402851104736, -0.19809484481811523, 0.43267953395843506], "agreement_weight": 0.010057391598820686, "orthogonal_weight": -0.8118337392807007}, {"head": 3, "memory_to_active": 0.13756130635738373, "contribution_to_active": 0.5878956913948059, "mix_mean": 0.6469959020614624, "mix_low": 0.01825428009033203, "mix_high": 0.02553081512451172, "read_cosine": 0.009307930245995522, "beta_mean": 0.0012066689087077975, "nonfinite": 0.0, "weighted_memory_to_active": 0.10306736826896667, "mix_std": 0.28492262959480286, "beta_std": 0.009833656251430511, "beta_low": 0.984619140625, "beta_high": 0.0, "orthogonal_to_active": 0.09559622406959534, "evidence_weights": [-0.32760366797447205, -0.2488386332988739, 1.8224464654922485], "agreement_weight": 4.384817123413086, "orthogonal_weight": -1.0184561014175415}, {"head": 4, "memory_to_active": 0.4153172969818115, "contribution_to_active": 0.2752002477645874, "mix_mean": 0.3561542332172394, "mix_low": 0.005176544189453125, "mix_high": 1.9073486328125e-06, "read_cosine": 0.866810142993927, "beta_mean": 0.09205983579158783, "nonfinite": 0.0, "weighted_memory_to_active": 0.11448562145233154, "mix_std": 0.21396726369857788, "beta_std": 0.05794660374522209, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.16306978464126587, "evidence_weights": [-0.11914850771427155, -0.2335374653339386, 0.4809776842594147], "agreement_weight": 0.008635899052023888, "orthogonal_weight": 2.1708290576934814}, {"head": 5, "memory_to_active": 0.09204470366239548, "contribution_to_active": 0.683305561542511, "mix_mean": 0.6356185078620911, "mix_low": 0.0023221969604492188, "mix_high": 0.008347511291503906, "read_cosine": 0.28548663854599, "beta_mean": 0.000535195111297071, "nonfinite": 0.0, "weighted_memory_to_active": 0.052839815616607666, "mix_std": 0.2726796567440033, "beta_std": 0.003886684775352478, "beta_low": 0.9962158203125, "beta_high": 0.0, "orthogonal_to_active": 0.044709812849760056, "evidence_weights": [-0.22246497869491577, -0.30566221475601196, 0.8160094022750854], "agreement_weight": 0.12193597853183746, "orthogonal_weight": -0.9211351275444031}]}, {"layer": 14, "memory_to_active": 0.2521383464336395, "contribution_to_active": 0.4143589437007904, "mix_mean": 0.4191707372665405, "mix_low": 0.025366466492414474, "mix_high": 0.005546092987060547, "read_cosine": 0.6089127659797668, "beta_mean": 0.06785877048969269, "nonfinite": 0.0, "weighted_memory_to_active": 0.10658401250839233, "mix_std": 0.2457365095615387, "beta_std": 0.06200941652059555, "beta_low": 0.4563598930835724, "beta_high": 0.0, "orthogonal_to_active": 0.0455198772251606, "output_influence": 0.33428457379341125, "residual_influence": 0.12750543653964996, "state_active": 0.0600675567984581, "state_memory": 0.016831444576382637, "state_peak": 38.3798713684082, "growth_active": 1.1085560321807861, "growth_memory": 1.314538836479187, "heads": [{"head": 0, "memory_to_active": 0.36363738775253296, "contribution_to_active": 0.18916302919387817, "mix_mean": 0.21044011414051056, "mix_low": 0.014340400695800781, "mix_high": 6.866455078125e-05, "read_cosine": 0.7919236421585083, "beta_mean": 0.07470749318599701, "nonfinite": 0.0, "weighted_memory_to_active": 0.08768519759178162, "mix_std": 0.18659421801567078, "beta_std": 0.06307250261306763, "beta_low": 0.0064697265625, "beta_high": 0.0, "orthogonal_to_active": 0.07849377393722534, "evidence_weights": [-0.13614435493946075, -0.03718085214495659, 0.49089446663856506], "agreement_weight": 0.02226777747273445, "orthogonal_weight": 1.3961491584777832}, {"head": 1, "memory_to_active": 0.491506963968277, "contribution_to_active": 0.3081841468811035, "mix_mean": 0.22317755222320557, "mix_low": 0.12085533142089844, "mix_high": 6.67572021484375e-06, "read_cosine": 0.795739471912384, "beta_mean": 0.26574182510375977, "nonfinite": 0.0, "weighted_memory_to_active": 0.20447781682014465, "mix_std": 0.25224539637565613, "beta_std": 0.2486620545387268, "beta_low": 0.09564208984375, "beta_high": 0.0, "orthogonal_to_active": 0.015426421537995338, "evidence_weights": [-0.01203873474150896, -0.21921348571777344, -0.1487845629453659], "agreement_weight": 0.05333653464913368, "orthogonal_weight": 0.1614195555448532}, {"head": 2, "memory_to_active": 0.15101894736289978, "contribution_to_active": 0.5613319873809814, "mix_mean": 0.5751786231994629, "mix_low": 0.0024280548095703125, "mix_high": 0.0036706924438476562, "read_cosine": 0.4109528660774231, "beta_mean": 0.0032737157307565212, "nonfinite": 0.0, "weighted_memory_to_active": 0.09475678205490112, "mix_std": 0.26673877239227295, "beta_std": 0.004258609842509031, "beta_low": 0.94085693359375, "beta_high": 0.0, "orthogonal_to_active": 0.07207690924406052, "evidence_weights": [-0.15163512527942657, -0.312246710062027, 0.9034978151321411], "agreement_weight": 0.08197299391031265, "orthogonal_weight": -0.8666387796401978}, {"head": 3, "memory_to_active": 0.07082286477088928, "contribution_to_active": 0.6230642795562744, "mix_mean": 0.6659786105155945, "mix_low": 0.0057849884033203125, "mix_high": 0.01965045928955078, "read_cosine": 0.07907994091510773, "beta_mean": 0.008585099130868912, "nonfinite": 0.0, "weighted_memory_to_active": 0.046214476227760315, "mix_std": 0.2537747025489807, "beta_std": 0.009347516112029552, "beta_low": 0.71124267578125, "beta_high": 0.0, "orthogonal_to_active": 0.03147687762975693, "evidence_weights": [-0.3642999529838562, -0.31998908519744873, 0.36156314611434937], "agreement_weight": 0.07523410767316818, "orthogonal_weight": -0.7737146615982056}, {"head": 4, "memory_to_active": 0.16544072329998016, "contribution_to_active": 0.4803178012371063, "mix_mean": 0.4848957061767578, "mix_low": 0.004918098449707031, "mix_high": 0.008683204650878906, "read_cosine": 0.8356626033782959, "beta_mean": 0.05272495746612549, "nonfinite": 0.0, "weighted_memory_to_active": 0.0943189263343811, "mix_std": 0.2627834975719452, "beta_std": 0.03580160439014435, "beta_low": 0.02105712890625, "beta_high": 0.0, "orthogonal_to_active": 0.028360918164253235, "evidence_weights": [-0.22107045352458954, -0.2509799897670746, 0.4270157516002655], "agreement_weight": 0.12460152059793472, "orthogonal_weight": -0.5952349901199341}, {"head": 5, "memory_to_active": 0.27040308713912964, "contribution_to_active": 0.3240922689437866, "mix_mean": 0.35535359382629395, "mix_low": 0.003871917724609375, "mix_high": 0.0011968612670898438, "read_cosine": 0.7401180267333984, "beta_mean": 0.0021195164881646633, "nonfinite": 0.0, "weighted_memory_to_active": 0.112050861120224, "mix_std": 0.2522823214530945, "beta_std": 0.010914154350757599, "beta_low": 0.962890625, "beta_high": 0.0, "orthogonal_to_active": 0.047284360975027084, "evidence_weights": [-0.17962482571601868, -0.16249287128448486, 0.2493741810321808], "agreement_weight": 0.08249960094690323, "orthogonal_weight": -0.6780962944030762}]}, {"layer": 15, "memory_to_active": 0.060989897698163986, "contribution_to_active": 0.4845486283302307, "mix_mean": 0.40770018100738525, "mix_low": 0.009826183319091797, "mix_high": 0.0022335052490234375, "read_cosine": 0.6927815079689026, "beta_mean": 0.04001183807849884, "nonfinite": 0.0, "weighted_memory_to_active": 0.02832348830997944, "mix_std": 0.28375107049942017, "beta_std": 0.04585271328687668, "beta_low": 0.7099609375, "beta_high": 0.0, "orthogonal_to_active": 0.006559546571224928, "output_influence": 0.2936802804470062, "residual_influence": 0.16198799014091492, "state_active": 0.12807871401309967, "state_memory": 0.010501980781555176, "state_peak": 43.10001754760742, "growth_active": 1.1581666469573975, "growth_memory": 1.6363292932510376, "heads": [{"head": 0, "memory_to_active": 0.10928434133529663, "contribution_to_active": 0.454929381608963, "mix_mean": 0.37332454323768616, "mix_low": 0.004662513732910156, "mix_high": 0.0040569305419921875, "read_cosine": 0.6944818496704102, "beta_mean": 0.03916417807340622, "nonfinite": 0.0, "weighted_memory_to_active": 0.05226973444223404, "mix_std": 0.2764818072319031, "beta_std": 0.030928872525691986, "beta_low": 0.108642578125, "beta_high": 0.0, "orthogonal_to_active": 0.022323856130242348, "evidence_weights": [-0.17546644806861877, -0.1814858764410019, 0.3692576289176941], "agreement_weight": 0.09463690221309662, "orthogonal_weight": -0.713435709476471}, {"head": 1, "memory_to_active": 0.12501220405101776, "contribution_to_active": 0.398924320936203, "mix_mean": 0.4129193425178528, "mix_low": 0.002269744873046875, "mix_high": 0.004628181457519531, "read_cosine": 0.9120157361030579, "beta_mean": 0.0008690288523212075, "nonfinite": 0.0, "weighted_memory_to_active": 0.04928237572312355, "mix_std": 0.2826586067676544, "beta_std": 0.0023986436426639557, "beta_low": 0.98834228515625, "beta_high": 0.0, "orthogonal_to_active": 7.288051710929722e-05, "evidence_weights": [-0.16033898293972015, -0.16949665546417236, 0.22291703522205353], "agreement_weight": 0.11246940493583679, "orthogonal_weight": 0.002622604602947831}, {"head": 2, "memory_to_active": 0.008871829137206078, "contribution_to_active": 0.5015693306922913, "mix_mean": 0.3469102382659912, "mix_low": 0.018453598022460938, "mix_high": 7.724761962890625e-05, "read_cosine": 0.788804292678833, "beta_mean": 0.0001275995746254921, "nonfinite": 0.0, "weighted_memory_to_active": 0.004517906345427036, "mix_std": 0.2737944722175598, "beta_std": 0.0002160718577215448, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000834511243738234, "evidence_weights": [-0.20009253919124603, -0.2710864245891571, 0.19235211610794067], "agreement_weight": 0.011972619220614433, "orthogonal_weight": 0.4650111496448517}, {"head": 3, "memory_to_active": 0.004178293514996767, "contribution_to_active": 0.4109557569026947, "mix_mean": 0.38782158493995667, "mix_low": 0.005938529968261719, "mix_high": 0.00012874603271484375, "read_cosine": 0.6246920228004456, "beta_mean": 2.5329401978524402e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0015211732825264335, "mix_std": 0.28374311327934265, "beta_std": 5.3263414883986115e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001315380446612835, "evidence_weights": [-0.2298176884651184, -0.25428470969200134, 0.285505473613739], "agreement_weight": 0.03132958337664604, "orthogonal_weight": -1.00274658203125}, {"head": 4, "memory_to_active": 0.09253689646720886, "contribution_to_active": 0.5991294384002686, "mix_mean": 0.35765472054481506, "mix_low": 0.018167495727539062, "mix_high": 0.0006208419799804688, "read_cosine": 0.7334089279174805, "beta_mean": 0.1981969028711319, "nonfinite": 0.0, "weighted_memory_to_active": 0.044964682310819626, "mix_std": 0.3068152964115143, "beta_std": 0.2299133539199829, "beta_low": 0.18951416015625, "beta_high": 0.0, "orthogonal_to_active": 0.005557160824537277, "evidence_weights": [-0.210166797041893, -0.2516586482524872, -0.013885884545743465], "agreement_weight": 0.012716386467218399, "orthogonal_weight": 0.27147871255874634}, {"head": 5, "memory_to_active": 0.026055827736854553, "contribution_to_active": 0.541783332824707, "mix_mean": 0.5675705671310425, "mix_low": 0.009465217590332031, "mix_high": 0.0038890838623046875, "read_cosine": 0.4032861590385437, "beta_mean": 0.0016879773465916514, "nonfinite": 0.0, "weighted_memory_to_active": 0.017385058104991913, "mix_std": 0.2790129780769348, "beta_std": 0.011606094427406788, "beta_low": 0.9732666015625, "beta_high": 0.0, "orthogonal_to_active": 0.009253489784896374, "evidence_weights": [-0.30558937788009644, -0.3154233694076538, 0.7602634429931641], "agreement_weight": 0.2626894414424896, "orthogonal_weight": 0.7397205233573914}]}]}, {"step": 40000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9287149869996938, "memory_to_active": 0.8464269042015076, "contribution_to_active": 0.5356413125991821, "mix_mean": 0.38662096858024597, "mix_low": 0.14256469905376434, "mix_high": 0.009377380833029747, "read_cosine": 0.36403924226760864, "beta_mean": 0.07722706347703934, "nonfinite": 0.0, "weighted_memory_to_active": 0.2076403796672821, "mix_std": 0.21879734098911285, "beta_std": 0.0659162849187851, "beta_low": 0.5312169790267944, "beta_high": 2.2888185412739404e-05, "orthogonal_to_active": 0.05623018741607666, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1273717880249023, "normal_token_accuracy": 0.4449388086795807, "state_active": 0.060337964445352554, "state_memory": 0.018546905368566513, "state_peak": 116.88668823242188, "growth_active": 1.128605842590332, "growth_memory": 1.3751059770584106, "output_influence": 0.3923283815383911, "residual_influence": 0.12927332520484924, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2312142848968506, "no_memory_loss_delta": 0.5050123929977417, "scrambled_loss_delta": 0.18943674862384796, "memory_only_loss_delta": 3.239044427871704, "no_evidence_loss_delta": 0.3346620500087738, "no_agreement_loss_delta": 0.05213778465986252, "no_orthogonal_loss_delta": 0.017219705507159233}, {"index": 94, "tokens": 256, "normal_loss": 3.2122669219970703, "no_memory_loss_delta": 0.5831901431083679, "scrambled_loss_delta": 0.31382322311401367, "memory_only_loss_delta": 3.726907730102539, "no_evidence_loss_delta": 0.260057657957077, "no_agreement_loss_delta": 0.03801727294921875, "no_orthogonal_loss_delta": 0.047027137130498886}, {"index": 188, "tokens": 256, "normal_loss": 3.486809730529785, "no_memory_loss_delta": 0.4775286018848419, "scrambled_loss_delta": 0.13821552693843842, "memory_only_loss_delta": 3.170118570327759, "no_evidence_loss_delta": 0.2476806789636612, "no_agreement_loss_delta": 0.038642920553684235, "no_orthogonal_loss_delta": 0.0360979400575161}, {"index": 283, "tokens": 131, "normal_loss": 2.0561301708221436, "no_memory_loss_delta": 0.2469685673713684, "scrambled_loss_delta": 0.19426564872264862, "memory_only_loss_delta": 2.923935651779175, "no_evidence_loss_delta": 0.11493296176195145, "no_agreement_loss_delta": -0.010438942350447178, "no_orthogonal_loss_delta": 0.009539265185594559}], "agreement_weight_abs_mean": 0.17660623788833618, "agreement_weight_abs_max": 4.481337547302246, "agreement_weight_mean": 0.1541992425918579, "orthogonal_weight_abs_mean": 0.7564615607261658, "orthogonal_weight_abs_max": 7.180414199829102, "orthogonal_weight_mean": -0.13797897100448608, "no_memory_loss": 3.6092185974121094, "no_memory_loss_delta": 0.48184654116630554, "no_memory_token_accuracy_gain": 0.06451612710952759, "no_memory_token_win_fraction": 0.656284749507904, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 38.23581629116008, "scrambled_loss": 3.3383471965789795, "scrambled_loss_delta": 0.21097500622272491, "scrambled_token_accuracy_gain": 0.031145717948675156, "scrambled_token_win_fraction": 0.6117908954620361, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.02056935531224, "memory_only_loss": 6.439796447753906, "memory_only_loss_delta": 3.312424659729004, "memory_only_token_accuracy_gain": 0.28142380714416504, "memory_only_token_win_fraction": 0.9265850782394409, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.35722582900857, "no_evidence_loss": 3.384002447128296, "no_evidence_loss_delta": 0.25663039088249207, "no_evidence_token_accuracy_gain": 0.025583982467651367, "no_evidence_token_win_fraction": 0.6017797589302063, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 22.634588945977978, "no_agreement_loss": 3.162527561187744, "no_agreement_loss_delta": 0.03515548259019852, "no_agreement_token_accuracy_gain": 0.0011123470030725002, "no_agreement_token_win_fraction": 0.4927697479724884, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 3.4544706902591633, "no_orthogonal_loss": 3.157336473464966, "no_orthogonal_loss_delta": 0.029964307323098183, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5672969818115234, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 2.951982803444683, "short_no_memory_loss_delta": 0.43772226572036743, "long_context_memory_gain": 0.04412427544593811, "normal_context_loss_gain": 0.1440923660993576, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.3483495712280273, "contribution_to_active": 1.0726855993270874, "mix_mean": 0.10400746017694473, "mix_low": 0.5357064008712769, "mix_high": 0.004920005798339844, "read_cosine": 0.43804773688316345, "beta_mean": 0.24058151245117188, "nonfinite": 0.0, "weighted_memory_to_active": 1.1140328645706177, "mix_std": 0.08730203658342361, "beta_std": 0.15210722386837006, "beta_low": 0.0286051444709301, "beta_high": 0.0, "orthogonal_to_active": 0.05877440422773361, "output_influence": 0.19587405025959015, "residual_influence": 0.23419497907161713, "state_active": 0.06689593195915222, "state_memory": 0.04622684046626091, "state_peak": 51.110721588134766, "growth_active": 1.2017568349838257, "growth_memory": 1.2559720277786255, "heads": [{"head": 0, "memory_to_active": 2.6776602268218994, "contribution_to_active": 0.46517622470855713, "mix_mean": 0.020328547805547714, "mix_low": 0.5918941497802734, "mix_high": 0.0, "read_cosine": 0.2331405133008957, "beta_mean": 0.22961296141147614, "nonfinite": 0.0, "weighted_memory_to_active": 0.5619300603866577, "mix_std": 0.05854451656341553, "beta_std": 0.11325816065073013, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04816754162311554, "evidence_weights": [-0.019889339804649353, -0.06599581241607666, -0.10140565037727356], "agreement_weight": -0.14763270318508148, "orthogonal_weight": -0.09351427853107452}, {"head": 1, "memory_to_active": 5.488251686096191, "contribution_to_active": 2.1219563484191895, "mix_mean": 0.015606703236699104, "mix_low": 0.6852283477783203, "mix_high": 0.0, "read_cosine": 0.526295006275177, "beta_mean": 0.4175970256328583, "nonfinite": 0.0, "weighted_memory_to_active": 2.418882369995117, "mix_std": 0.050105154514312744, "beta_std": 0.2132377326488495, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06959618628025055, "evidence_weights": [0.027007553726434708, -0.1001480221748352, 0.45473870635032654], "agreement_weight": 0.027470245957374573, "orthogonal_weight": -0.05790877714753151}, {"head": 2, "memory_to_active": 2.4712135791778564, "contribution_to_active": 0.42419785261154175, "mix_mean": 0.014129474759101868, "mix_low": 0.6498146057128906, "mix_high": 0.0, "read_cosine": 0.3187669515609741, "beta_mean": 0.28805649280548096, "nonfinite": 0.0, "weighted_memory_to_active": 0.5046822428703308, "mix_std": 0.029436210170388222, "beta_std": 0.1951930969953537, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05071070417761803, "evidence_weights": [0.05822208151221275, 0.06222410127520561, 0.020290963351726532], "agreement_weight": 0.006477863993495703, "orthogonal_weight": -0.2135692983865738}, {"head": 3, "memory_to_active": 7.002523422241211, "contribution_to_active": 2.9051761627197266, "mix_mean": 0.03541610762476921, "mix_low": 0.46726226806640625, "mix_high": 0.0, "read_cosine": 0.29591596126556396, "beta_mean": 0.34767287969589233, "nonfinite": 0.0, "weighted_memory_to_active": 3.0900769233703613, "mix_std": 0.08092878013849258, "beta_std": 0.2504810094833374, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.15991957485675812, "evidence_weights": [-0.045325327664613724, -0.056326329708099365, -0.16716518998146057], "agreement_weight": -0.01614195667207241, "orthogonal_weight": -0.05973052978515625}, {"head": 4, "memory_to_active": 2.3493237495422363, "contribution_to_active": 0.07921261340379715, "mix_mean": 0.007151057943701744, "mix_low": 0.8031072616577148, "mix_high": 0.0, "read_cosine": 0.3235628306865692, "beta_mean": 0.130979523062706, "nonfinite": 0.0, "weighted_memory_to_active": 0.057935405522584915, "mix_std": 0.01612994819879532, "beta_std": 0.11636511981487274, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.006858492270112038, "evidence_weights": [0.12991949915885925, 0.08275501430034637, -0.3492758870124817], "agreement_weight": -0.12357308715581894, "orthogonal_weight": 0.1083407998085022}, {"head": 5, "memory_to_active": 0.10112372040748596, "contribution_to_active": 0.44039469957351685, "mix_mean": 0.5314128398895264, "mix_low": 0.016931533813476562, "mix_high": 0.029520034790039062, "read_cosine": 0.9306052327156067, "beta_mean": 0.029570048674941063, "nonfinite": 0.0, "weighted_memory_to_active": 0.050690099596977234, "mix_std": 0.28866758942604065, "beta_std": 0.024108227342367172, "beta_low": 0.1708984375, "beta_high": 0.0, "orthogonal_to_active": 0.01739393174648285, "evidence_weights": [-0.2123936414718628, -0.23216262459754944, 0.3416926860809326], "agreement_weight": 0.27888110280036926, "orthogonal_weight": -0.6961362957954407}]}, {"layer": 1, "memory_to_active": 2.2982828617095947, "contribution_to_active": 0.5157384872436523, "mix_mean": 0.19390186667442322, "mix_low": 0.3600039780139923, "mix_high": 0.0010008811950683594, "read_cosine": 0.3169155418872833, "beta_mean": 0.17977923154830933, "nonfinite": 0.0, "weighted_memory_to_active": 0.35769638419151306, "mix_std": 0.11839935928583145, "beta_std": 0.13153699040412903, "beta_low": 0.2992045283317566, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.051482416689395905, "output_influence": 0.20225825905799866, "residual_influence": 0.05090905725955963, "state_active": 0.03602766990661621, "state_memory": 0.03550662100315094, "state_peak": 35.39260482788086, "growth_active": 1.3151532411575317, "growth_memory": 1.5138828754425049, "heads": [{"head": 0, "memory_to_active": 0.031265951693058014, "contribution_to_active": 0.45510193705558777, "mix_mean": 0.45086470246315, "mix_low": 0.009888648986816406, "mix_high": 0.0014486312866210938, "read_cosine": 0.024262182414531708, "beta_mean": 0.00022309231280814856, "nonfinite": 0.0, "weighted_memory_to_active": 0.016923701390624046, "mix_std": 0.28548741340637207, "beta_std": 0.00035662081791087985, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01592136174440384, "evidence_weights": [-0.1891893446445465, -0.24728775024414062, 0.291939377784729], "agreement_weight": 0.1729958951473236, "orthogonal_weight": -1.0026816129684448}, {"head": 1, "memory_to_active": 2.7782201766967773, "contribution_to_active": 0.21213851869106293, "mix_mean": 0.02281428873538971, "mix_low": 0.45459461212158203, "mix_high": 2.86102294921875e-06, "read_cosine": 0.5091379880905151, "beta_mean": 0.2981298267841339, "nonfinite": 0.0, "weighted_memory_to_active": 0.23855863511562347, "mix_std": 0.044768765568733215, "beta_std": 0.18858614563941956, "beta_low": 0.00238037109375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.015245744027197361, "evidence_weights": [0.012939760461449623, -0.04251830652356148, 0.020979195833206177], "agreement_weight": -0.0052667525596916676, "orthogonal_weight": 0.14656727015972137}, {"head": 2, "memory_to_active": 0.15333637595176697, "contribution_to_active": 0.5841687917709351, "mix_mean": 0.6230394244194031, "mix_low": 0.0008993148803710938, "mix_high": 0.004551887512207031, "read_cosine": 0.6168831586837769, "beta_mean": 0.009408149868249893, "nonfinite": 0.0, "weighted_memory_to_active": 0.12003810703754425, "mix_std": 0.24253711104393005, "beta_std": 0.011823927983641624, "beta_low": 0.7032470703125, "beta_high": 0.0, "orthogonal_to_active": 0.04679486155509949, "evidence_weights": [-0.3007526695728302, -0.34635043144226074, 0.519832193851471], "agreement_weight": 0.02351314015686512, "orthogonal_weight": -0.547886848449707}, {"head": 3, "memory_to_active": 2.8811798095703125, "contribution_to_active": 0.15509779751300812, "mix_mean": 0.02113964408636093, "mix_low": 0.5928258895874023, "mix_high": 0.0, "read_cosine": 0.22419321537017822, "beta_mean": 0.32757312059402466, "nonfinite": 0.0, "weighted_memory_to_active": 0.2406103014945984, "mix_std": 0.05476013571023941, "beta_std": 0.2498539239168167, "beta_low": 0.0445556640625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.06812399625778198, "evidence_weights": [0.036956656724214554, -0.015842240303754807, 0.11290570348501205], "agreement_weight": -0.0280634555965662, "orthogonal_weight": -0.3791077136993408}, {"head": 4, "memory_to_active": 3.830826997756958, "contribution_to_active": 1.6244679689407349, "mix_mean": 0.03871920332312584, "mix_low": 0.26372241973876953, "mix_high": 1.9073486328125e-06, "read_cosine": 0.3137046694755554, "beta_mean": 0.18082424998283386, "nonfinite": 0.0, "weighted_memory_to_active": 1.4789714813232422, "mix_std": 0.06499112397432327, "beta_std": 0.12217828631401062, "beta_low": 0.01318359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.15410013496875763, "evidence_weights": [-0.04702778533101082, -0.006972318980842829, 0.05598895624279976], "agreement_weight": -0.015133604407310486, "orthogonal_weight": 0.13668206334114075}, {"head": 5, "memory_to_active": 4.114867210388184, "contribution_to_active": 0.06345590204000473, "mix_mean": 0.006833934690803289, "mix_low": 0.8380928039550781, "mix_high": 0.0, "read_cosine": 0.21331177651882172, "beta_mean": 0.2625170350074768, "nonfinite": 0.0, "weighted_memory_to_active": 0.05107598751783371, "mix_std": 0.017851553857326508, "beta_std": 0.21642303466796875, "beta_low": 0.0318603515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.008708378300070763, "evidence_weights": [0.1658080518245697, 0.0841323509812355, 0.2530808746814728], "agreement_weight": -0.14086300134658813, "orthogonal_weight": 0.16734613478183746}]}, {"layer": 2, "memory_to_active": 1.0375880002975464, "contribution_to_active": 0.34599050879478455, "mix_mean": 0.37383317947387695, "mix_low": 0.25249356031417847, "mix_high": 0.003666400909423828, "read_cosine": 0.3332539200782776, "beta_mean": 0.09921901673078537, "nonfinite": 0.0, "weighted_memory_to_active": 0.07617337256669998, "mix_std": 0.1920664757490158, "beta_std": 0.07440917193889618, "beta_low": 0.5606384873390198, "beta_high": 0.00017293295240961015, "orthogonal_to_active": 0.016301751136779785, "output_influence": 0.35996657609939575, "residual_influence": 0.09321901947259903, "state_active": 0.04030478000640869, "state_memory": 0.01766045205295086, "state_peak": 34.55183410644531, "growth_active": 1.056724190711975, "growth_memory": 1.5852116346359253, "heads": [{"head": 0, "memory_to_active": 0.006068181246519089, "contribution_to_active": 0.32345929741859436, "mix_mean": 0.5787067413330078, "mix_low": 0.0013599395751953125, "mix_high": 0.006400108337402344, "read_cosine": 0.16686028242111206, "beta_mean": 5.052400229033083e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003627236932516098, "mix_std": 0.2872074842453003, "beta_std": 8.202812023228034e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025278902612626553, "evidence_weights": [-0.33721259236335754, -0.20509487390518188, 0.09777692705392838], "agreement_weight": 0.22568202018737793, "orthogonal_weight": -0.7278048396110535}, {"head": 1, "memory_to_active": 2.836904525756836, "contribution_to_active": 0.041625410318374634, "mix_mean": 0.004657885991036892, "mix_low": 0.8979682922363281, "mix_high": 0.0, "read_cosine": 0.20334967970848083, "beta_mean": 0.28605490922927856, "nonfinite": 0.0, "weighted_memory_to_active": 0.030562305822968483, "mix_std": 0.009439405053853989, "beta_std": 0.16614976525306702, "beta_low": 0.00030517578125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.012229904532432556, "evidence_weights": [0.128046914935112, 0.11565631628036499, 0.518848180770874], "agreement_weight": 0.037588898092508316, "orthogonal_weight": 0.4190329611301422}, {"head": 2, "memory_to_active": 0.007857042364776134, "contribution_to_active": 0.4049663543701172, "mix_mean": 0.508360743522644, "mix_low": 0.0014781951904296875, "mix_high": 0.0022802352905273438, "read_cosine": 0.2941439747810364, "beta_mean": 1.732850978442002e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0028488487005233765, "mix_std": 0.2803885042667389, "beta_std": 4.375565185910091e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005357591435313225, "evidence_weights": [-0.24202123284339905, -0.27966129779815674, 0.48259085416793823], "agreement_weight": 0.029513023793697357, "orthogonal_weight": -0.205174520611763}, {"head": 3, "memory_to_active": 0.13906554877758026, "contribution_to_active": 0.5167139768600464, "mix_mean": 0.5200624465942383, "mix_low": 0.0004968643188476562, "mix_high": 0.007519721984863281, "read_cosine": 0.6725786924362183, "beta_mean": 0.031973712146282196, "nonfinite": 0.0, "weighted_memory_to_active": 0.07901514321565628, "mix_std": 0.26674893498420715, "beta_std": 0.041096512228250504, "beta_low": 0.326171875, "beta_high": 0.0, "orthogonal_to_active": 0.050991471856832504, "evidence_weights": [-0.186494380235672, -0.2919122278690338, 0.8557531833648682], "agreement_weight": 0.05924958363175392, "orthogonal_weight": -0.8887734413146973}, {"head": 4, "memory_to_active": 3.2218680381774902, "contribution_to_active": 0.2732315957546234, "mix_mean": 0.01687452755868435, "mix_low": 0.6124114990234375, "mix_high": 0.0, "read_cosine": 0.21330901980400085, "beta_mean": 0.2771765887737274, "nonfinite": 0.0, "weighted_memory_to_active": 0.3328888714313507, "mix_std": 0.039981547743082047, "beta_std": 0.23902159929275513, "beta_low": 0.037353515625, "beta_high": 0.0009765625, "orthogonal_to_active": 0.031270649284124374, "evidence_weights": [0.023131484165787697, 0.011809834279119968, 0.056318480521440506], "agreement_weight": 0.0027707002591341734, "orthogonal_weight": -0.10528980940580368}, {"head": 5, "memory_to_active": 0.013764486648142338, "contribution_to_active": 0.5159462690353394, "mix_mean": 0.6143366098403931, "mix_low": 0.0012464523315429688, "mix_high": 0.00579833984375, "read_cosine": 0.4492816627025604, "beta_mean": 4.1012455767486244e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008097827434539795, "mix_std": 0.2686328887939453, "beta_std": 6.134614523034543e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00025483284844085574, "evidence_weights": [-0.26941800117492676, -0.28653427958488464, 0.91507887840271], "agreement_weight": 0.11997465789318085, "orthogonal_weight": 0.046870049089193344}]}, {"layer": 3, "memory_to_active": 0.5869012475013733, "contribution_to_active": 0.4232240915298462, "mix_mean": 0.4087459146976471, "mix_low": 0.08085934817790985, "mix_high": 0.007634004577994347, "read_cosine": 0.21512126922607422, "beta_mean": 0.03362462297081947, "nonfinite": 0.0, "weighted_memory_to_active": 0.14449812471866608, "mix_std": 0.25899049639701843, "beta_std": 0.028316117823123932, "beta_low": 0.6820170283317566, "beta_high": 0.0, "orthogonal_to_active": 0.06710383296012878, "output_influence": 0.3603540062904358, "residual_influence": 0.07376512140035629, "state_active": 0.022515717893838882, "state_memory": 0.00810597836971283, "state_peak": 20.261917114257812, "growth_active": 1.0118978023529053, "growth_memory": 1.7750144004821777, "heads": [{"head": 0, "memory_to_active": 0.4600633978843689, "contribution_to_active": 0.5504029989242554, "mix_mean": 0.5178701877593994, "mix_low": 0.006829261779785156, "mix_high": 0.015417098999023438, "read_cosine": 0.22983242571353912, "beta_mean": 0.011956516653299332, "nonfinite": 0.0, "weighted_memory_to_active": 0.2635694146156311, "mix_std": 0.31706318259239197, "beta_std": 0.01837250404059887, "beta_low": 0.651123046875, "beta_high": 0.0, "orthogonal_to_active": 0.1805628389120102, "evidence_weights": [-0.1972002387046814, -0.22824545204639435, 0.7959280014038086], "agreement_weight": 0.1293705701828003, "orthogonal_weight": -0.723458468914032}, {"head": 1, "memory_to_active": 0.01867017149925232, "contribution_to_active": 0.39468643069267273, "mix_mean": 0.48880335688591003, "mix_low": 0.009991645812988281, "mix_high": 0.005367279052734375, "read_cosine": 0.18627771735191345, "beta_mean": 9.075468551600352e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0098002003505826, "mix_std": 0.309192955493927, "beta_std": 0.00018747894500847906, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00696583092212677, "evidence_weights": [-0.23426520824432373, -0.19868884980678558, 0.5795770883560181], "agreement_weight": 0.3996775150299072, "orthogonal_weight": -0.7743142247200012}, {"head": 2, "memory_to_active": 2.5666589736938477, "contribution_to_active": 0.2783255875110626, "mix_mean": 0.04629245400428772, "mix_low": 0.3560314178466797, "mix_high": 0.0, "read_cosine": 0.2000594586133957, "beta_mean": 0.17160281538963318, "nonfinite": 0.0, "weighted_memory_to_active": 0.315424382686615, "mix_std": 0.09788016229867935, "beta_std": 0.13104408979415894, "beta_low": 0.0106201171875, "beta_high": 0.0, "orthogonal_to_active": 0.003670461243018508, "evidence_weights": [-0.07726369053125381, 0.013948502019047737, -0.012666958384215832], "agreement_weight": -0.013914033770561218, "orthogonal_weight": -0.014507519081234932}, {"head": 3, "memory_to_active": 0.01699899509549141, "contribution_to_active": 0.5078373551368713, "mix_mean": 0.5469775199890137, "mix_low": 0.009453773498535156, "mix_high": 0.00974273681640625, "read_cosine": 0.2963683009147644, "beta_mean": 7.153773913159966e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01108052022755146, "mix_std": 0.3022698760032654, "beta_std": 0.00016430635878350586, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026424964889883995, "evidence_weights": [-0.22492536902427673, -0.22656935453414917, 2.0937249660491943], "agreement_weight": 1.64516019821167, "orthogonal_weight": -0.30885592103004456}, {"head": 4, "memory_to_active": 0.18689614534378052, "contribution_to_active": 0.176982581615448, "mix_mean": 0.20827439427375793, "mix_low": 0.09673213958740234, "mix_high": 0.00044345855712890625, "read_cosine": 0.15586231648921967, "beta_mean": 0.001532792579382658, "nonfinite": 0.0, "weighted_memory_to_active": 0.07072742283344269, "mix_std": 0.24224179983139038, "beta_std": 0.002681280951946974, "beta_low": 0.98114013671875, "beta_high": 0.0, "orthogonal_to_active": 0.062333717942237854, "evidence_weights": [-0.06340096890926361, -0.10779137164354324, 0.23770146071910858], "agreement_weight": -0.011048548854887486, "orthogonal_weight": -0.9170505404472351}, {"head": 5, "memory_to_active": 0.2721196711063385, "contribution_to_active": 0.6311094760894775, "mix_mean": 0.6442574262619019, "mix_low": 0.006117820739746094, "mix_high": 0.014833450317382812, "read_cosine": 0.22232741117477417, "beta_mean": 0.016493311151862144, "nonfinite": 0.0, "weighted_memory_to_active": 0.19638681411743164, "mix_std": 0.28529495000839233, "beta_std": 0.01744704507291317, "beta_low": 0.44927978515625, "beta_high": 0.0, "orthogonal_to_active": 0.14644765853881836, "evidence_weights": [-0.24536429345607758, -0.28670626878738403, 0.4980783760547638], "agreement_weight": 0.12947292625904083, "orthogonal_weight": -0.8101632595062256}]}, {"layer": 4, "memory_to_active": 1.167324185371399, "contribution_to_active": 0.4267704486846924, "mix_mean": 0.3499988615512848, "mix_low": 0.2821706235408783, "mix_high": 0.006913026329129934, "read_cosine": 0.3474830090999603, "beta_mean": 0.13035638630390167, "nonfinite": 0.0, "weighted_memory_to_active": 0.13806414604187012, "mix_std": 0.1550188660621643, "beta_std": 0.11425162106752396, "beta_low": 0.37548828125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03829639405012131, "output_influence": 0.40934574604034424, "residual_influence": 0.11315585672855377, "state_active": 0.038787633180618286, "state_memory": 0.019124560058116913, "state_peak": 41.01544952392578, "growth_active": 1.2472509145736694, "growth_memory": 1.5384441614151, "heads": [{"head": 0, "memory_to_active": 0.22687721252441406, "contribution_to_active": 0.7130842208862305, "mix_mean": 0.651910126209259, "mix_low": 0.014936447143554688, "mix_high": 0.01807880401611328, "read_cosine": 0.49873974919319153, "beta_mean": 0.26808255910873413, "nonfinite": 0.0, "weighted_memory_to_active": 0.1700901985168457, "mix_std": 0.343453049659729, "beta_std": 0.2739321291446686, "beta_low": 0.21917724609375, "beta_high": 0.0, "orthogonal_to_active": 0.0633576512336731, "evidence_weights": [-0.26750680804252625, -0.36815333366394043, 0.16791999340057373], "agreement_weight": 0.015065799467265606, "orthogonal_weight": 0.45705997943878174}, {"head": 1, "memory_to_active": 0.07238764315843582, "contribution_to_active": 0.6130386590957642, "mix_mean": 0.6552392840385437, "mix_low": 9.250640869140625e-05, "mix_high": 0.009939193725585938, "read_cosine": 0.6562775373458862, "beta_mean": 0.0009694627951830626, "nonfinite": 0.0, "weighted_memory_to_active": 0.05082540214061737, "mix_std": 0.24401527643203735, "beta_std": 0.004709336906671524, "beta_low": 0.98443603515625, "beta_high": 0.0, "orthogonal_to_active": 0.02911362424492836, "evidence_weights": [-0.23693126440048218, -0.33610594272613525, 0.7181622982025146], "agreement_weight": 0.18498407304286957, "orthogonal_weight": -0.8188638687133789}, {"head": 2, "memory_to_active": 2.375792980194092, "contribution_to_active": 0.22694168984889984, "mix_mean": 0.01762407273054123, "mix_low": 0.6131191253662109, "mix_high": 3.814697265625e-06, "read_cosine": 0.2645390033721924, "beta_mean": 0.11249395459890366, "nonfinite": 0.0, "weighted_memory_to_active": 0.23723997175693512, "mix_std": 0.03870707377791405, "beta_std": 0.09220612794160843, "beta_low": 0.00482177734375, "beta_high": 0.0, "orthogonal_to_active": 0.023499304428696632, "evidence_weights": [0.06421405076980591, 0.032700058072805405, 0.034083303064107895], "agreement_weight": 0.1335265189409256, "orthogonal_weight": -0.1571289300918579}, {"head": 3, "memory_to_active": 1.848860263824463, "contribution_to_active": 0.14017978310585022, "mix_mean": 0.030039552599191666, "mix_low": 0.42139434814453125, "mix_high": 0.0, "read_cosine": 0.22303539514541626, "beta_mean": 0.14965710043907166, "nonfinite": 0.0, "weighted_memory_to_active": 0.19656884670257568, "mix_std": 0.05384118854999542, "beta_std": 0.1370076835155487, "beta_low": 0.04449462890625, "beta_high": 0.0, "orthogonal_to_active": 0.00857093371450901, "evidence_weights": [-0.022460663691163063, 0.057542528957128525, -0.22843511402606964], "agreement_weight": 0.07430407404899597, "orthogonal_weight": -0.060655947774648666}, {"head": 4, "memory_to_active": 0.018219608813524246, "contribution_to_active": 0.624330997467041, "mix_mean": 0.7312021255493164, "mix_low": 0.00013446807861328125, "mix_high": 0.013446807861328125, "read_cosine": 0.16046515107154846, "beta_mean": 3.398706030566245e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013634522445499897, "mix_std": 0.2183057963848114, "beta_std": 0.00015151486149989069, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0036687811370939016, "evidence_weights": [-0.32801827788352966, -0.3296252489089966, 0.6259067058563232], "agreement_weight": 0.3135448694229126, "orthogonal_weight": -0.2918117046356201}, {"head": 5, "memory_to_active": 2.4618072509765625, "contribution_to_active": 0.24304693937301636, "mix_mean": 0.013977928087115288, "mix_low": 0.6433467864990234, "mix_high": 9.5367431640625e-06, "read_cosine": 0.28184112906455994, "beta_mean": 0.2509011924266815, "nonfinite": 0.0, "weighted_memory_to_active": 0.16002589464187622, "mix_std": 0.03179074078798294, "beta_std": 0.17750291526317596, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1015680730342865, "evidence_weights": [0.047319624572992325, -0.03494612127542496, -0.2471277117729187], "agreement_weight": -0.06618199497461319, "orthogonal_weight": 0.8813115954399109}]}, {"layer": 5, "memory_to_active": 0.5069551467895508, "contribution_to_active": 0.5018755197525024, "mix_mean": 0.4642994999885559, "mix_low": 0.0672200545668602, "mix_high": 0.013885974884033203, "read_cosine": 0.09992478787899017, "beta_mean": 0.0606483519077301, "nonfinite": 0.0, "weighted_memory_to_active": 0.15028919279575348, "mix_std": 0.253486692905426, "beta_std": 0.07610206305980682, "beta_low": 0.63427734375, "beta_high": 0.0, "orthogonal_to_active": 0.054146669805049896, "output_influence": 0.49937206506729126, "residual_influence": 0.1368274837732315, "state_active": 0.04780321195721626, "state_memory": 0.012409152463078499, "state_peak": 65.17369842529297, "growth_active": 1.1680926084518433, "growth_memory": 1.2686799764633179, "heads": [{"head": 0, "memory_to_active": 0.5278632640838623, "contribution_to_active": 0.45273369550704956, "mix_mean": 0.29098260402679443, "mix_low": 0.040328025817871094, "mix_high": 0.00036907196044921875, "read_cosine": -0.16783463954925537, "beta_mean": 0.03427845239639282, "nonfinite": 0.0, "weighted_memory_to_active": 0.13726229965686798, "mix_std": 0.2580384612083435, "beta_std": 0.05774768814444542, "beta_low": 0.37841796875, "beta_high": 0.0, "orthogonal_to_active": 0.0351494699716568, "evidence_weights": [-0.11182057857513428, -0.14218217134475708, -0.14654091000556946], "agreement_weight": -0.021863361820578575, "orthogonal_weight": -0.26090729236602783}, {"head": 1, "memory_to_active": 0.022723661735653877, "contribution_to_active": 0.5546368360519409, "mix_mean": 0.6559911966323853, "mix_low": 0.0026235580444335938, "mix_high": 0.021009445190429688, "read_cosine": 0.16957998275756836, "beta_mean": 5.7508535974193364e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012553229928016663, "mix_std": 0.28449931740760803, "beta_std": 0.00011818346683867276, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011850012466311455, "evidence_weights": [-0.23810534179210663, -0.27729737758636475, 0.8918007016181946], "agreement_weight": 0.5713244676589966, "orthogonal_weight": -0.9803614020347595}, {"head": 2, "memory_to_active": 0.012429839931428432, "contribution_to_active": 0.5444043278694153, "mix_mean": 0.6353535652160645, "mix_low": 0.003108978271484375, "mix_high": 0.019847869873046875, "read_cosine": -0.25612035393714905, "beta_mean": 0.00011975705274380744, "nonfinite": 0.0, "weighted_memory_to_active": 0.008062202483415604, "mix_std": 0.27449750900268555, "beta_std": 0.00020223463070578873, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020605039317160845, "evidence_weights": [-0.25564080476760864, -0.34654268622398376, 0.7357076406478882], "agreement_weight": 0.002432387787848711, "orthogonal_weight": -0.31126779317855835}, {"head": 3, "memory_to_active": 1.9091317653656006, "contribution_to_active": 0.21648073196411133, "mix_mean": 0.09154567122459412, "mix_low": 0.15714263916015625, "mix_high": 9.5367431640625e-07, "read_cosine": 0.13319478929042816, "beta_mean": 0.11997994780540466, "nonfinite": 0.0, "weighted_memory_to_active": 0.3612736463546753, "mix_std": 0.12800706923007965, "beta_std": 0.16092734038829803, "beta_low": 0.15997314453125, "beta_high": 0.0, "orthogonal_to_active": 0.12921525537967682, "evidence_weights": [-0.06129118427634239, -0.09088204056024551, 0.005872275680303574], "agreement_weight": 0.060277387499809265, "orthogonal_weight": -0.46099960803985596}, {"head": 4, "memory_to_active": 0.447196900844574, "contribution_to_active": 0.590949535369873, "mix_mean": 0.32819363474845886, "mix_low": 0.19720840454101562, "mix_high": 0.008253097534179688, "read_cosine": 0.5602130889892578, "beta_mean": 0.2090957760810852, "nonfinite": 0.0, "weighted_memory_to_active": 0.300245076417923, "mix_std": 0.3439275324344635, "beta_std": 0.23595575988292694, "beta_low": 0.2723388671875, "beta_high": 0.0, "orthogonal_to_active": 0.09418490529060364, "evidence_weights": [-0.10999014973640442, -0.2767133414745331, -0.07344669103622437], "agreement_weight": 0.03763004392385483, "orthogonal_weight": 0.49667033553123474}, {"head": 5, "memory_to_active": 0.12238526344299316, "contribution_to_active": 0.6520478129386902, "mix_mean": 0.783730149269104, "mix_low": 0.0029087066650390625, "mix_high": 0.033835411071777344, "read_cosine": 0.16051587462425232, "beta_mean": 0.00035866163671016693, "nonfinite": 0.0, "weighted_memory_to_active": 0.08233864605426788, "mix_std": 0.2319503277540207, "beta_std": 0.0016611868049949408, "beta_low": 0.99493408203125, "beta_high": 0.0, "orthogonal_to_active": 0.052419841289520264, "evidence_weights": [-0.3214797079563141, -0.3272457718849182, 1.9594335556030273], "agreement_weight": 0.4251636564731598, "orthogonal_weight": -0.6801183819770813}]}, {"layer": 6, "memory_to_active": 0.34669923782348633, "contribution_to_active": 0.61048424243927, "mix_mean": 0.5651917457580566, "mix_low": 0.06011565774679184, "mix_high": 0.030590057373046875, "read_cosine": 0.27871647477149963, "beta_mean": 0.06927885115146637, "nonfinite": 0.0, "weighted_memory_to_active": 0.17182642221450806, "mix_std": 0.23829235136508942, "beta_std": 0.07668539881706238, "beta_low": 0.5055745244026184, "beta_high": 0.0, "orthogonal_to_active": 0.03822600841522217, "output_influence": 0.5109158158302307, "residual_influence": 0.1675054430961609, "state_active": 0.04741766303777695, "state_memory": 0.008815734647214413, "state_peak": 93.0486068725586, "growth_active": 1.1688076257705688, "growth_memory": 1.1655980348587036, "heads": [{"head": 0, "memory_to_active": 0.11630815267562866, "contribution_to_active": 0.6703690886497498, "mix_mean": 0.7609935998916626, "mix_low": 0.0001983642578125, "mix_high": 0.0193634033203125, "read_cosine": 0.029261697083711624, "beta_mean": 0.017222438007593155, "nonfinite": 0.0, "weighted_memory_to_active": 0.09138709306716919, "mix_std": 0.22051730751991272, "beta_std": 0.023735780268907547, "beta_low": 0.5032958984375, "beta_high": 0.0, "orthogonal_to_active": 0.07833585143089294, "evidence_weights": [-0.3186080753803253, -0.3386264443397522, 0.894976019859314], "agreement_weight": 0.26883867383003235, "orthogonal_weight": -0.9003768563270569}, {"head": 1, "memory_to_active": 0.005142915993928909, "contribution_to_active": 0.49737441539764404, "mix_mean": 0.5443665981292725, "mix_low": 0.0016965866088867188, "mix_high": 0.010556221008300781, "read_cosine": 0.5290893912315369, "beta_mean": 1.1398658898542635e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0024263616651296616, "mix_std": 0.28763896226882935, "beta_std": 2.4177792511181906e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013437630841508508, "evidence_weights": [-0.20023693144321442, -0.29396817088127136, 1.0728176832199097], "agreement_weight": 0.04653661698102951, "orthogonal_weight": -0.7487893104553223}, {"head": 2, "memory_to_active": 0.9917171001434326, "contribution_to_active": 0.09268375486135483, "mix_mean": 0.035626370459795, "mix_low": 0.35173797607421875, "mix_high": 0.0, "read_cosine": 0.24597667157649994, "beta_mean": 0.043735526502132416, "nonfinite": 0.0, "weighted_memory_to_active": 0.07396280020475388, "mix_std": 0.0590062215924263, "beta_std": 0.0384749136865139, "beta_low": 0.0643310546875, "beta_high": 0.0, "orthogonal_to_active": 0.0001145591086242348, "evidence_weights": [0.07996421307325363, -0.023449741303920746, 0.26306480169296265], "agreement_weight": 0.13713550567626953, "orthogonal_weight": -0.0019346500048413873}, {"head": 3, "memory_to_active": 0.15809692442417145, "contribution_to_active": 0.6669889092445374, "mix_mean": 0.6399548053741455, "mix_low": 0.005856513977050781, "mix_high": 0.01788330078125, "read_cosine": 0.4684184193611145, "beta_mean": 0.21911361813545227, "nonfinite": 0.0, "weighted_memory_to_active": 0.11267487704753876, "mix_std": 0.32571864128112793, "beta_std": 0.24511165916919708, "beta_low": 0.19085693359375, "beta_high": 0.0, "orthogonal_to_active": 0.06122121959924698, "evidence_weights": [-0.17376244068145752, -0.33349063992500305, 0.1930953562259674], "agreement_weight": -0.007988367229700089, "orthogonal_weight": 0.6343115568161011}, {"head": 4, "memory_to_active": 0.7745321393013, "contribution_to_active": 1.1340305805206299, "mix_mean": 0.7695276141166687, "mix_low": 0.0006570816040039062, "mix_high": 0.1326313018798828, "read_cosine": 0.1368883103132248, "beta_mean": 0.13536512851715088, "nonfinite": 0.0, "weighted_memory_to_active": 0.7268693447113037, "mix_std": 0.2825666069984436, "beta_std": 0.1521623432636261, "beta_low": 0.27557373046875, "beta_high": 0.0, "orthogonal_to_active": 0.06490904837846756, "evidence_weights": [-0.24824932217597961, -0.5994787812232971, -0.15946988761425018], "agreement_weight": -0.020133640617132187, "orthogonal_weight": 0.10428086668252945}, {"head": 5, "memory_to_active": 0.03439810872077942, "contribution_to_active": 0.6014583110809326, "mix_mean": 0.6406810879707336, "mix_low": 0.0005474090576171875, "mix_high": 0.0031061172485351562, "read_cosine": 0.2626643776893616, "beta_mean": 0.0002249726967420429, "nonfinite": 0.0, "weighted_memory_to_active": 0.02363797277212143, "mix_std": 0.2543063163757324, "beta_std": 0.0006034670514054596, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.023431584239006042, "evidence_weights": [-0.24211770296096802, -0.31038910150527954, 0.4307459592819214], "agreement_weight": 0.1475377231836319, "orthogonal_weight": -1.0580195188522339}]}, {"layer": 7, "memory_to_active": 1.7564959526062012, "contribution_to_active": 0.23280474543571472, "mix_mean": 0.12356600165367126, "mix_low": 0.27059635519981384, "mix_high": 0.0003371238708496094, "read_cosine": 0.24745550751686096, "beta_mean": 0.1268492490053177, "nonfinite": 0.0, "weighted_memory_to_active": 0.20394079387187958, "mix_std": 0.13407720625400543, "beta_std": 0.09879407286643982, "beta_low": 0.1986490935087204, "beta_high": 0.00010172526526730508, "orthogonal_to_active": 0.06401197612285614, "output_influence": 0.24295902252197266, "residual_influence": 0.054626550525426865, "state_active": 0.013469676487147808, "state_memory": 0.02564014308154583, "state_peak": 26.44830894470215, "growth_active": 1.1217588186264038, "growth_memory": 1.3004283905029297, "heads": [{"head": 0, "memory_to_active": 2.460916519165039, "contribution_to_active": 0.23337164521217346, "mix_mean": 0.0358506478369236, "mix_low": 0.42334556579589844, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3619157075881958, "beta_mean": 0.16556328535079956, "nonfinite": 0.0, "weighted_memory_to_active": 0.29401105642318726, "mix_std": 0.07503554224967957, "beta_std": 0.12521451711654663, "beta_low": 0.0062255859375, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.005942186340689659, "evidence_weights": [0.02265523187816143, -0.07765813171863556, 0.07955310493707657], "agreement_weight": -0.03147316351532936, "orthogonal_weight": 0.03169143199920654}, {"head": 1, "memory_to_active": 1.0658740997314453, "contribution_to_active": 0.38690078258514404, "mix_mean": 0.20570316910743713, "mix_low": 0.13870716094970703, "mix_high": 0.0018377304077148438, "read_cosine": 0.35752952098846436, "beta_mean": 0.09980622678995132, "nonfinite": 0.0, "weighted_memory_to_active": 0.3426607549190521, "mix_std": 0.23351889848709106, "beta_std": 0.06618475914001465, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02627059817314148, "evidence_weights": [-0.20262368023395538, -0.11273892223834991, -0.19679102301597595], "agreement_weight": 0.030909128487110138, "orthogonal_weight": 0.08945413678884506}, {"head": 2, "memory_to_active": 0.776060938835144, "contribution_to_active": 0.30042925477027893, "mix_mean": 0.1356109380722046, "mix_low": 0.07493400573730469, "mix_high": 0.00012111663818359375, "read_cosine": 0.11844494193792343, "beta_mean": 0.05377654731273651, "nonfinite": 0.0, "weighted_memory_to_active": 0.2173389047384262, "mix_std": 0.16827577352523804, "beta_std": 0.045190006494522095, "beta_low": 0.0736083984375, "beta_high": 0.0, "orthogonal_to_active": 0.08357127010822296, "evidence_weights": [-0.05260884761810303, -0.13871608674526215, 0.5532421469688416], "agreement_weight": 0.027758991345763206, "orthogonal_weight": -0.3949926793575287}, {"head": 3, "memory_to_active": 4.443161487579346, "contribution_to_active": 0.08516387641429901, "mix_mean": 0.03311603143811226, "mix_low": 0.3332548141479492, "mix_high": 0.0, "read_cosine": 0.10336978733539581, "beta_mean": 0.35050827264785767, "nonfinite": 0.0, "weighted_memory_to_active": 0.29010382294654846, "mix_std": 0.048007458448410034, "beta_std": 0.2645242512226105, "beta_low": 0.00494384765625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.24824562668800354, "evidence_weights": [0.0629793331027031, -0.01480636652559042, 0.6812337636947632], "agreement_weight": 0.05633898824453354, "orthogonal_weight": -0.9387324452400208}, {"head": 4, "memory_to_active": 1.736981749534607, "contribution_to_active": 0.06828959286212921, "mix_mean": 0.016171148046851158, "mix_low": 0.6450424194335938, "mix_high": 0.0, "read_cosine": 0.2478824257850647, "beta_mean": 0.09110096096992493, "nonfinite": 0.0, "weighted_memory_to_active": 0.05959787219762802, "mix_std": 0.03646092116832733, "beta_std": 0.09100715816020966, "beta_low": 0.10748291015625, "beta_high": 0.0, "orthogonal_to_active": 0.007788488175719976, "evidence_weights": [0.13723574578762054, 0.0411677211523056, 0.24606341123580933], "agreement_weight": 0.04358059912919998, "orthogonal_weight": -0.16423889994621277}, {"head": 5, "memory_to_active": 0.05598036199808121, "contribution_to_active": 0.3226732611656189, "mix_mean": 0.31494399905204773, "mix_low": 0.008294105529785156, "mix_high": 6.29425048828125e-05, "read_cosine": 0.2955906093120575, "beta_mean": 0.0003401930443942547, "nonfinite": 0.0, "weighted_memory_to_active": 0.019932270050048828, "mix_std": 0.24316465854644775, "beta_std": 0.0006437058909796178, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.012253707274794579, "evidence_weights": [-0.1235266774892807, -0.1826765090227127, 0.5444395542144775], "agreement_weight": 0.11908670514822006, "orthogonal_weight": -0.695971667766571}]}, {"layer": 8, "memory_to_active": 0.36011916399002075, "contribution_to_active": 0.5186203122138977, "mix_mean": 0.4268013834953308, "mix_low": 0.08055448532104492, "mix_high": 0.0009385744924657047, "read_cosine": 0.45445871353149414, "beta_mean": 0.022249864414334297, "nonfinite": 0.0, "weighted_memory_to_active": 0.07942547649145126, "mix_std": 0.21714113652706146, "beta_std": 0.029200522229075432, "beta_low": 0.6420491933822632, "beta_high": 0.0, "orthogonal_to_active": 0.05696955323219299, "output_influence": 0.462380975484848, "residual_influence": 0.14766950905323029, "state_active": 0.049412697553634644, "state_memory": 0.008255206979811192, "state_peak": 40.62392044067383, "growth_active": 1.1143783330917358, "growth_memory": 1.3084746599197388, "heads": [{"head": 0, "memory_to_active": 0.05023885890841484, "contribution_to_active": 0.5486987829208374, "mix_mean": 0.5934587121009827, "mix_low": 0.0002593994140625, "mix_high": 0.000919342041015625, "read_cosine": 0.482384592294693, "beta_mean": 0.0024920781143009663, "nonfinite": 0.0, "weighted_memory_to_active": 0.03470822423696518, "mix_std": 0.24236643314361572, "beta_std": 0.008315503597259521, "beta_low": 0.947021484375, "beta_high": 0.0, "orthogonal_to_active": 0.02249704673886299, "evidence_weights": [-0.16994552314281464, -0.39188626408576965, 0.6708522439002991], "agreement_weight": 0.14985181391239166, "orthogonal_weight": -0.8070372343063354}, {"head": 1, "memory_to_active": 0.09228178858757019, "contribution_to_active": 0.5081402659416199, "mix_mean": 0.4749510884284973, "mix_low": 0.00030803680419921875, "mix_high": 0.0004930496215820312, "read_cosine": 0.24130326509475708, "beta_mean": 0.0016075316816568375, "nonfinite": 0.0, "weighted_memory_to_active": 0.04226037859916687, "mix_std": 0.24626141786575317, "beta_std": 0.008105089887976646, "beta_low": 0.9833984375, "beta_high": 0.0, "orthogonal_to_active": 0.02881719544529915, "evidence_weights": [-0.18882805109024048, -0.26220569014549255, 0.6907762289047241], "agreement_weight": 0.09472540766000748, "orthogonal_weight": -0.736589252948761}, {"head": 2, "memory_to_active": 0.016290750354528427, "contribution_to_active": 0.6561168432235718, "mix_mean": 0.6427492499351501, "mix_low": 0.0010204315185546875, "mix_high": 0.0030546188354492188, "read_cosine": 0.3927091658115387, "beta_mean": 0.0001631192135391757, "nonfinite": 0.0, "weighted_memory_to_active": 0.012177272699773312, "mix_std": 0.22928279638290405, "beta_std": 0.002639981685206294, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.008045829832553864, "evidence_weights": [-0.22227229177951813, -0.3417586386203766, 0.9872427582740784], "agreement_weight": 0.22334669530391693, "orthogonal_weight": -0.7547062039375305}, {"head": 3, "memory_to_active": 0.11289657652378082, "contribution_to_active": 0.6349053382873535, "mix_mean": 0.5677339434623718, "mix_low": 0.0015420913696289062, "mix_high": 0.0011348724365234375, "read_cosine": 0.78160560131073, "beta_mean": 0.010081150569021702, "nonfinite": 0.0, "weighted_memory_to_active": 0.08549419045448303, "mix_std": 0.2529752850532532, "beta_std": 0.008250311017036438, "beta_low": 0.596923828125, "beta_high": 0.0, "orthogonal_to_active": 0.028707634657621384, "evidence_weights": [-0.18351344764232635, -0.3056199252605438, 0.6209462285041809], "agreement_weight": 0.18287795782089233, "orthogonal_weight": -0.6442833542823792}, {"head": 4, "memory_to_active": 1.5832544565200806, "contribution_to_active": 0.1455206573009491, "mix_mean": 0.032937295734882355, "mix_low": 0.36227989196777344, "mix_high": 0.0, "read_cosine": 0.34418562054634094, "beta_mean": 0.05063237249851227, "nonfinite": 0.0, "weighted_memory_to_active": 0.13376730680465698, "mix_std": 0.05364103615283966, "beta_std": 0.05923137813806534, "beta_low": 0.14569091796875, "beta_high": 0.0, "orthogonal_to_active": 0.001502768136560917, "evidence_weights": [0.05791464075446129, -0.0354728028178215, -0.03748190402984619], "agreement_weight": 0.07498253136873245, "orthogonal_weight": -0.01199326291680336}, {"head": 5, "memory_to_active": 0.30575263500213623, "contribution_to_active": 0.6183396577835083, "mix_mean": 0.2489778995513916, "mix_low": 0.11791706085205078, "mix_high": 2.956390380859375e-05, "read_cosine": 0.4845638573169708, "beta_mean": 0.06852292269468307, "nonfinite": 0.0, "weighted_memory_to_active": 0.16814543306827545, "mix_std": 0.2783198356628418, "beta_std": 0.08866085857152939, "beta_low": 0.1793212890625, "beta_high": 0.0, "orthogonal_to_active": 0.25224682688713074, "evidence_weights": [-0.08027107268571854, -0.23075751960277557, -0.2537331283092499], "agreement_weight": -0.006753351539373398, "orthogonal_weight": 1.7787505388259888}]}, {"layer": 9, "memory_to_active": 1.0854918956756592, "contribution_to_active": 0.6746694445610046, "mix_mean": 0.3285437822341919, "mix_low": 0.1374526023864746, "mix_high": 0.004002094268798828, "read_cosine": 0.35064417123794556, "beta_mean": 0.062320370227098465, "nonfinite": 0.0, "weighted_memory_to_active": 0.38385987281799316, "mix_std": 0.21294528245925903, "beta_std": 0.04289939999580383, "beta_low": 0.5013936758041382, "beta_high": 0.0, "orthogonal_to_active": 0.03984678536653519, "output_influence": 0.5088765621185303, "residual_influence": 0.13896211981773376, "state_active": 0.04229942709207535, "state_memory": 0.01886896975338459, "state_peak": 45.491397857666016, "growth_active": 1.165701985359192, "growth_memory": 1.351348638534546, "heads": [{"head": 0, "memory_to_active": 0.0038662187289446592, "contribution_to_active": 0.5496081113815308, "mix_mean": 0.6234445571899414, "mix_low": 0.00121307373046875, "mix_high": 0.01802349090576172, "read_cosine": 0.25445544719696045, "beta_mean": 7.480436579498928e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0024488510098308325, "mix_std": 0.28899914026260376, "beta_std": 2.8736954845953733e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0022005781065672636, "evidence_weights": [-0.2966535985469818, -0.3058270812034607, 0.8965051174163818], "agreement_weight": 0.122220978140831, "orthogonal_weight": -0.9490938186645508}, {"head": 1, "memory_to_active": 0.017372550442814827, "contribution_to_active": 0.6507636308670044, "mix_mean": 0.5197033286094666, "mix_low": 0.008053779602050781, "mix_high": 0.001323699951171875, "read_cosine": 0.39284735918045044, "beta_mean": 4.950456059304997e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011949839070439339, "mix_std": 0.28445911407470703, "beta_std": 9.644282545195892e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00851498544216156, "evidence_weights": [-0.22853440046310425, -0.2761211693286896, 1.13323175907135], "agreement_weight": 0.3858691155910492, "orthogonal_weight": -0.8549618124961853}, {"head": 2, "memory_to_active": 0.169380322098732, "contribution_to_active": 0.6381916999816895, "mix_mean": 0.568728506565094, "mix_low": 0.0022077560424804688, "mix_high": 0.004244804382324219, "read_cosine": 0.3803708255290985, "beta_mean": 0.000641938706394285, "nonfinite": 0.0, "weighted_memory_to_active": 0.11188894510269165, "mix_std": 0.27207833528518677, "beta_std": 0.003445182926952839, "beta_low": 0.99603271484375, "beta_high": 0.0, "orthogonal_to_active": 0.06055884063243866, "evidence_weights": [-0.19631050527095795, -0.26677897572517395, 0.7102474570274353], "agreement_weight": 0.2781667411327362, "orthogonal_weight": -0.6395922899246216}, {"head": 3, "memory_to_active": 1.4204483032226562, "contribution_to_active": 0.19056852161884308, "mix_mean": 0.048783425241708755, "mix_low": 0.3568696975708008, "mix_high": 8.106231689453125e-05, "read_cosine": 0.3972914218902588, "beta_mean": 0.07657723128795624, "nonfinite": 0.0, "weighted_memory_to_active": 0.15451322495937347, "mix_std": 0.1034836620092392, "beta_std": 0.05219006538391113, "beta_low": 0.00299072265625, "beta_high": 0.0, "orthogonal_to_active": 0.01475534401834011, "evidence_weights": [0.0772385373711586, 0.01671747863292694, 0.0026231894735246897], "agreement_weight": 0.08772218227386475, "orthogonal_weight": 0.10056816041469574}, {"head": 4, "memory_to_active": 3.1554577350616455, "contribution_to_active": 1.4121053218841553, "mix_mean": 0.13203860819339752, "mix_low": 0.15579700469970703, "mix_high": 0.00029277801513671875, "read_cosine": 0.1970623880624771, "beta_mean": 0.1784885823726654, "nonfinite": 0.0, "weighted_memory_to_active": 1.3752388954162598, "mix_std": 0.19318872690200806, "beta_std": 0.09998384863138199, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02999986708164215, "evidence_weights": [-0.06757361441850662, -0.15150317549705505, -0.021762503311038017], "agreement_weight": -0.056685082614421844, "orthogonal_weight": -0.022502223029732704}, {"head": 5, "memory_to_active": 1.7464264631271362, "contribution_to_active": 0.6067794561386108, "mix_mean": 0.0785641148686409, "mix_low": 0.30057430267333984, "mix_high": 4.673004150390625e-05, "read_cosine": 0.4818374812602997, "beta_mean": 0.11815744638442993, "nonfinite": 0.0, "weighted_memory_to_active": 0.647119402885437, "mix_std": 0.13546264171600342, "beta_std": 0.1016521155834198, "beta_low": 0.00933837890625, "beta_high": 0.0, "orthogonal_to_active": 0.12305109947919846, "evidence_weights": [0.10960081219673157, -0.0807189866900444, 0.15706078708171844], "agreement_weight": 0.01575077883899212, "orthogonal_weight": 0.2764415740966797}]}, {"layer": 10, "memory_to_active": 0.23994669318199158, "contribution_to_active": 0.6725623607635498, "mix_mean": 0.5031400918960571, "mix_low": 0.01683155819773674, "mix_high": 0.0066960654221475124, "read_cosine": 0.37438416481018066, "beta_mean": 0.041961707174777985, "nonfinite": 0.0, "weighted_memory_to_active": 0.11916783452033997, "mix_std": 0.26670601963996887, "beta_std": 0.04456306993961334, "beta_low": 0.6671143174171448, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.19189350306987762, "output_influence": 0.4273533821105957, "residual_influence": 0.144285187125206, "state_active": 0.05282681807875633, "state_memory": 0.01584990695118904, "state_peak": 116.88668823242188, "growth_active": 1.0460773706436157, "growth_memory": 0.9304811954498291, "heads": [{"head": 0, "memory_to_active": 0.9539047479629517, "contribution_to_active": 1.8114022016525269, "mix_mean": 0.5990564823150635, "mix_low": 0.05979728698730469, "mix_high": 0.024618148803710938, "read_cosine": 0.09941365569829941, "beta_mean": 0.18427956104278564, "nonfinite": 0.0, "weighted_memory_to_active": 0.5998976230621338, "mix_std": 0.3928902745246887, "beta_std": 0.2037995457649231, "beta_low": 0.14715576171875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.07614266872406, "evidence_weights": [-0.11767793446779251, -0.39240017533302307, -1.3247206211090088], "agreement_weight": 0.014990226365625858, "orthogonal_weight": 1.8145397901535034}, {"head": 1, "memory_to_active": 0.011445470154285431, "contribution_to_active": 0.6001315712928772, "mix_mean": 0.6081579923629761, "mix_low": 0.0009164810180664062, "mix_high": 0.010783195495605469, "read_cosine": 0.278613418340683, "beta_mean": 7.42578849894926e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00649426132440567, "mix_std": 0.2664225697517395, "beta_std": 0.00010208664753008634, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009417126886546612, "evidence_weights": [-0.26466846466064453, -0.20232008397579193, 0.5789259076118469], "agreement_weight": 0.4851134419441223, "orthogonal_weight": -1.5718454122543335}, {"head": 2, "memory_to_active": 0.01320037804543972, "contribution_to_active": 0.5916417837142944, "mix_mean": 0.6209793090820312, "mix_low": 0.00028514862060546875, "mix_high": 0.004448890686035156, "read_cosine": 0.024775728583335876, "beta_mean": 3.050488157896325e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005569974891841412, "mix_std": 0.25245147943496704, "beta_std": 0.00010807461512740701, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026424271054565907, "evidence_weights": [-0.24261930584907532, -0.27567899227142334, 0.5711982250213623], "agreement_weight": -0.16268955171108246, "orthogonal_weight": -0.45703351497650146}, {"head": 3, "memory_to_active": 0.3226272463798523, "contribution_to_active": 0.09899970144033432, "mix_mean": 0.26036056876182556, "mix_low": 0.03691577911376953, "mix_high": 9.822845458984375e-05, "read_cosine": 0.9077581763267517, "beta_mean": 0.06141230836510658, "nonfinite": 0.0, "weighted_memory_to_active": 0.03698036074638367, "mix_std": 0.21510398387908936, "beta_std": 0.055944133549928665, "beta_low": 0.0030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.041536398231983185, "evidence_weights": [-0.10693913698196411, -0.14372269809246063, 0.14087028801441193], "agreement_weight": 0.005465017165988684, "orthogonal_weight": 1.4576117992401123}, {"head": 4, "memory_to_active": 0.006611641496419907, "contribution_to_active": 0.5270979404449463, "mix_mean": 0.5476275682449341, "mix_low": 0.000339508056640625, "mix_high": 0.0002193450927734375, "read_cosine": 0.07093298435211182, "beta_mean": 3.3345710107823834e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0036033678334206343, "mix_std": 0.24776385724544525, "beta_std": 9.714721818454564e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003314678091555834, "evidence_weights": [-0.1811634600162506, -0.2600415050983429, 0.8229878544807434], "agreement_weight": 0.0982581377029419, "orthogonal_weight": -0.9694334864616394}, {"head": 5, "memory_to_active": 0.13189075887203217, "contribution_to_active": 0.4061008095741272, "mix_mean": 0.382658451795578, "mix_low": 0.002735137939453125, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8648109436035156, "beta_mean": 0.005940255708992481, "nonfinite": 0.0, "weighted_memory_to_active": 0.062461405992507935, "mix_std": 0.22560375928878784, "beta_std": 0.007327424827963114, "beta_low": 0.85247802734375, "beta_high": 0.0, "orthogonal_to_active": 0.01830771565437317, "evidence_weights": [-0.14560529589653015, -0.15991255640983582, 0.2849355936050415], "agreement_weight": 0.09070286899805069, "orthogonal_weight": -0.6311967372894287}]}, {"layer": 11, "memory_to_active": 0.10830317437648773, "contribution_to_active": 0.59070885181427, "mix_mean": 0.4863346219062805, "mix_low": 0.08971135318279266, "mix_high": 0.05090188980102539, "read_cosine": 0.3780905604362488, "beta_mean": 0.03150072693824768, "nonfinite": 0.0, "weighted_memory_to_active": 0.05106852948665619, "mix_std": 0.32270073890686035, "beta_std": 0.04566652327775955, "beta_low": 0.607818603515625, "beta_high": 0.0, "orthogonal_to_active": 0.04767097532749176, "output_influence": 0.5194512009620667, "residual_influence": 0.1297505795955658, "state_active": 0.1695193350315094, "state_memory": 0.006074450444430113, "state_peak": 24.93084716796875, "growth_active": 1.0158973932266235, "growth_memory": 1.717117428779602, "heads": [{"head": 0, "memory_to_active": 0.010563197545707226, "contribution_to_active": 0.5861849188804626, "mix_mean": 0.6252530813217163, "mix_low": 0.00086212158203125, "mix_high": 0.008990287780761719, "read_cosine": 0.3018006682395935, "beta_mean": 0.00012230148422531784, "nonfinite": 0.0, "weighted_memory_to_active": 0.007255489006638527, "mix_std": 0.26160115003585815, "beta_std": 0.004007970914244652, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.00539420498535037, "evidence_weights": [-0.16737549006938934, -0.3680967390537262, 0.7084060311317444], "agreement_weight": 0.2879656255245209, "orthogonal_weight": -0.7920845746994019}, {"head": 1, "memory_to_active": 0.2625432312488556, "contribution_to_active": 0.3201371729373932, "mix_mean": 0.3644585609436035, "mix_low": 0.010198593139648438, "mix_high": 0.00014495849609375, "read_cosine": 0.521856427192688, "beta_mean": 0.025233758613467216, "nonfinite": 0.0, "weighted_memory_to_active": 0.1084766760468483, "mix_std": 0.2561167776584625, "beta_std": 0.05125630274415016, "beta_low": 0.512939453125, "beta_high": 0.0, "orthogonal_to_active": 0.04050769656896591, "evidence_weights": [-0.16145162284374237, -0.18500125408172607, 0.49561789631843567], "agreement_weight": 0.06867720186710358, "orthogonal_weight": -0.4632538855075836}, {"head": 2, "memory_to_active": 0.010418339632451534, "contribution_to_active": 0.9800238609313965, "mix_mean": 0.4464002251625061, "mix_low": 0.23318958282470703, "mix_high": 0.18171405792236328, "read_cosine": 0.20112073421478271, "beta_mean": 0.07790517807006836, "nonfinite": 0.0, "weighted_memory_to_active": 0.00765659473836422, "mix_std": 0.4443749189376831, "beta_std": 0.10107099264860153, "beta_low": 0.428466796875, "beta_high": 0.0, "orthogonal_to_active": 0.05032563582062721, "evidence_weights": [0.039753507822752, -0.4329277276992798, -1.265591025352478], "agreement_weight": 0.003091801656410098, "orthogonal_weight": 6.843817710876465}, {"head": 3, "memory_to_active": 0.011103824712336063, "contribution_to_active": 0.8304073214530945, "mix_mean": 0.4190571904182434, "mix_low": 0.2827425003051758, "mix_high": 0.11025524139404297, "read_cosine": 0.547630786895752, "beta_mean": 0.06124520301818848, "nonfinite": 0.0, "weighted_memory_to_active": 0.006693700794130564, "mix_std": 0.4413057565689087, "beta_std": 0.08845849335193634, "beta_low": 0.46148681640625, "beta_high": 0.0, "orthogonal_to_active": 0.03639694303274155, "evidence_weights": [0.14984992146492004, -0.32897794246673584, -0.25508391857147217], "agreement_weight": 0.047254517674446106, "orthogonal_weight": 7.180414199829102}, {"head": 4, "memory_to_active": 0.27078986167907715, "contribution_to_active": 0.43327242136001587, "mix_mean": 0.5272667407989502, "mix_low": 0.00984954833984375, "mix_high": 0.0005540847778320312, "read_cosine": 0.03358161449432373, "beta_mean": 0.017205774784088135, "nonfinite": 0.0, "weighted_memory_to_active": 0.1355886608362198, "mix_std": 0.271758109331131, "beta_std": 0.0188590195029974, "beta_low": 0.4637451171875, "beta_high": 0.0, "orthogonal_to_active": 0.13138821721076965, "evidence_weights": [-0.23801816999912262, -0.21965202689170837, 1.905995488166809], "agreement_weight": -0.19268465042114258, "orthogonal_weight": -0.9818899631500244}, {"head": 5, "memory_to_active": 0.08440052717924118, "contribution_to_active": 0.3942272663116455, "mix_mean": 0.5355719327926636, "mix_low": 0.0014257431030273438, "mix_high": 0.0037527084350585938, "read_cosine": 0.6625531911849976, "beta_mean": 0.007292141206562519, "nonfinite": 0.0, "weighted_memory_to_active": 0.04074005037546158, "mix_std": 0.2610475718975067, "beta_std": 0.010346327908337116, "beta_low": 0.78082275390625, "beta_high": 0.0, "orthogonal_to_active": 0.022013148292899132, "evidence_weights": [-0.21798108518123627, -0.30004817247390747, 0.6085204482078552], "agreement_weight": 0.08318968117237091, "orthogonal_weight": -0.7260065078735352}]}, {"layer": 12, "memory_to_active": 0.2625775933265686, "contribution_to_active": 0.5590612888336182, "mix_mean": 0.5107839703559875, "mix_low": 0.009472529403865337, "mix_high": 0.004360993858426809, "read_cosine": 0.2369265854358673, "beta_mean": 0.015034331940114498, "nonfinite": 0.0, "weighted_memory_to_active": 0.14234663546085358, "mix_std": 0.26049622893333435, "beta_std": 0.015146546065807343, "beta_low": 0.824676513671875, "beta_high": 0.0, "orthogonal_to_active": 0.06971374154090881, "output_influence": 0.457820862531662, "residual_influence": 0.12633779644966125, "state_active": 0.03645855933427811, "state_memory": 0.009083354845643044, "state_peak": 50.853511810302734, "growth_active": 1.1053894758224487, "growth_memory": 1.22257399559021, "heads": [{"head": 0, "memory_to_active": 0.05729858577251434, "contribution_to_active": 0.5289229154586792, "mix_mean": 0.5235657691955566, "mix_low": 0.0009136199951171875, "mix_high": 0.00260162353515625, "read_cosine": 0.30457445979118347, "beta_mean": 0.0009440793073736131, "nonfinite": 0.0, "weighted_memory_to_active": 0.027843907475471497, "mix_std": 0.2691091299057007, "beta_std": 0.002894842065870762, "beta_low": 0.9920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.022896289825439453, "evidence_weights": [-0.20047682523727417, -0.27123019099235535, 0.22447139024734497], "agreement_weight": 0.10320301353931427, "orthogonal_weight": -0.8847130537033081}, {"head": 1, "memory_to_active": 0.09576183557510376, "contribution_to_active": 0.5122994184494019, "mix_mean": 0.5308558940887451, "mix_low": 0.0007419586181640625, "mix_high": 0.00099945068359375, "read_cosine": 0.45524901151657104, "beta_mean": 0.0008519841358065605, "nonfinite": 0.0, "weighted_memory_to_active": 0.05066797882318497, "mix_std": 0.26204848289489746, "beta_std": 0.0074630784802138805, "beta_low": 0.99371337890625, "beta_high": 0.0, "orthogonal_to_active": 0.041757047176361084, "evidence_weights": [-0.20865079760551453, -0.27425000071525574, 0.9834968447685242], "agreement_weight": 0.15241272747516632, "orthogonal_weight": -0.9306673407554626}, {"head": 2, "memory_to_active": 0.5755555629730225, "contribution_to_active": 0.47336193919181824, "mix_mean": 0.3249583840370178, "mix_low": 0.026228904724121094, "mix_high": 0.0004138946533203125, "read_cosine": 0.4566979706287384, "beta_mean": 0.08650697022676468, "nonfinite": 0.0, "weighted_memory_to_active": 0.23772311210632324, "mix_std": 0.23916961252689362, "beta_std": 0.061689380556344986, "beta_low": 0.0020751953125, "beta_high": 0.0, "orthogonal_to_active": 0.23782140016555786, "evidence_weights": [-0.2394227683544159, -0.04999043047428131, -0.07115618139505386], "agreement_weight": 0.026380490511655807, "orthogonal_weight": 1.2253707647323608}, {"head": 3, "memory_to_active": 0.03128860145807266, "contribution_to_active": 0.7314548492431641, "mix_mean": 0.6966575384140015, "mix_low": 0.00025653839111328125, "mix_high": 0.014509201049804688, "read_cosine": 0.0887129083275795, "beta_mean": 0.00024602969642728567, "nonfinite": 0.0, "weighted_memory_to_active": 0.02265850454568863, "mix_std": 0.23432734608650208, "beta_std": 0.0034618102945387363, "beta_low": 0.9979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.021351441740989685, "evidence_weights": [-0.2898949682712555, -0.34091904759407043, 1.3928285837173462], "agreement_weight": 0.4044382870197296, "orthogonal_weight": -0.9634374380111694}, {"head": 4, "memory_to_active": 0.7295844554901123, "contribution_to_active": 0.6289460062980652, "mix_mean": 0.37237322330474854, "mix_low": 0.021341323852539062, "mix_high": 0.003208160400390625, "read_cosine": -0.10572053492069244, "beta_mean": 0.0011697798036038876, "nonfinite": 0.0, "weighted_memory_to_active": 0.4585860073566437, "mix_std": 0.28000980615615845, "beta_std": 0.010071451775729656, "beta_low": 0.97161865234375, "beta_high": 0.0, "orthogonal_to_active": 0.05040483549237251, "evidence_weights": [-0.1253059357404709, -0.2142086923122406, 0.006259606685489416], "agreement_weight": -0.006265164352953434, "orthogonal_weight": -0.11579719185829163}, {"head": 5, "memory_to_active": 0.08597646653652191, "contribution_to_active": 0.4793824553489685, "mix_mean": 0.6162927150726318, "mix_low": 0.0073528289794921875, "mix_high": 0.004433631896972656, "read_cosine": 0.2220456749200821, "beta_mean": 0.0004871496057603508, "nonfinite": 0.0, "weighted_memory_to_active": 0.05660024285316467, "mix_std": 0.2783128619194031, "beta_std": 0.005298711359500885, "beta_low": 0.99066162109375, "beta_high": 0.0, "orthogonal_to_active": 0.044051408767700195, "evidence_weights": [-0.24721074104309082, -0.31374144554138184, 1.2731596231460571], "agreement_weight": 0.3331131637096405, "orthogonal_weight": -0.833487868309021}]}, {"layer": 13, "memory_to_active": 0.12951529026031494, "contribution_to_active": 0.5184059739112854, "mix_mean": 0.5171465277671814, "mix_low": 0.005141735076904297, "mix_high": 0.006617069710046053, "read_cosine": 0.4289931058883667, "beta_mean": 0.015210816636681557, "nonfinite": 0.0, "weighted_memory_to_active": 0.05453142151236534, "mix_std": 0.25618648529052734, "beta_std": 0.017423653975129128, "beta_low": 0.7982280254364014, "beta_high": 0.0, "orthogonal_to_active": 0.05548352748155594, "output_influence": 0.49065959453582764, "residual_influence": 0.16371718049049377, "state_active": 0.105842724442482, "state_memory": 0.039200156927108765, "state_peak": 94.17632293701172, "growth_active": 0.9365653395652771, "growth_memory": 1.0380834341049194, "heads": [{"head": 0, "memory_to_active": 0.019136805087327957, "contribution_to_active": 0.5695098638534546, "mix_mean": 0.5850319862365723, "mix_low": 0.000385284423828125, "mix_high": 0.0022554397583007812, "read_cosine": 0.4588779807090759, "beta_mean": 7.117482891771942e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011666620150208473, "mix_std": 0.26745232939720154, "beta_std": 0.00013989693252369761, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01143031194806099, "evidence_weights": [-0.20189771056175232, -0.26680833101272583, 0.756739616394043], "agreement_weight": 0.11773864924907684, "orthogonal_weight": -1.1058204174041748}, {"head": 1, "memory_to_active": 0.014579322189092636, "contribution_to_active": 0.5836414694786072, "mix_mean": 0.5470483303070068, "mix_low": 0.0018396377563476562, "mix_high": 0.0024814605712890625, "read_cosine": 0.20555704832077026, "beta_mean": 5.381726077757776e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008277649991214275, "mix_std": 0.27173712849617004, "beta_std": 9.657843474997208e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010166497901082039, "evidence_weights": [-0.18592415750026703, -0.2683964669704437, 0.6910122036933899], "agreement_weight": 0.3533591330051422, "orthogonal_weight": -1.2794382572174072}, {"head": 2, "memory_to_active": 0.09571033716201782, "contribution_to_active": 0.44260895252227783, "mix_mean": 0.3744986057281494, "mix_low": 0.00193023681640625, "mix_high": 0.000110626220703125, "read_cosine": 0.7092681527137756, "beta_mean": 0.012706659734249115, "nonfinite": 0.0, "weighted_memory_to_active": 0.04341547563672066, "mix_std": 0.22543007135391235, "beta_std": 0.038457125425338745, "beta_low": 0.80255126953125, "beta_high": 0.0, "orthogonal_to_active": 0.019853997975587845, "evidence_weights": [-0.1946178823709488, -0.1979834884405136, 0.42749547958374023], "agreement_weight": 0.009378299117088318, "orthogonal_weight": -0.8123899698257446}, {"head": 3, "memory_to_active": 0.15409858524799347, "contribution_to_active": 0.5893441438674927, "mix_mean": 0.639674186706543, "mix_low": 0.019317626953125, "mix_high": 0.026604652404785156, "read_cosine": 0.010136055760085583, "beta_mean": 0.0011196606792509556, "nonfinite": 0.0, "weighted_memory_to_active": 0.1130468025803566, "mix_std": 0.2905174195766449, "beta_std": 0.008911702781915665, "beta_low": 0.988525390625, "beta_high": 0.0, "orthogonal_to_active": 0.10495555400848389, "evidence_weights": [-0.32483407855033875, -0.24643968045711517, 1.8176043033599854], "agreement_weight": 4.481337547302246, "orthogonal_weight": -1.0054256916046143}, {"head": 4, "memory_to_active": 0.40956220030784607, "contribution_to_active": 0.25460168719291687, "mix_mean": 0.3262934386730194, "mix_low": 0.0051116943359375, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8736007809638977, "beta_mean": 0.0768769234418869, "nonfinite": 0.0, "weighted_memory_to_active": 0.10285338759422302, "mix_std": 0.20685046911239624, "beta_std": 0.05303337424993515, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.14630097150802612, "evidence_weights": [-0.11487017571926117, -0.23233917355537415, 0.4767400026321411], "agreement_weight": -0.0011809554416686296, "orthogonal_weight": 2.181615114212036}, {"head": 5, "memory_to_active": 0.08400452136993408, "contribution_to_active": 0.6707296967506409, "mix_mean": 0.6303322315216064, "mix_low": 0.00226593017578125, "mix_high": 0.008245468139648438, "read_cosine": 0.31651851534843445, "beta_mean": 0.00043665673001669347, "nonfinite": 0.0, "weighted_memory_to_active": 0.047928594052791595, "mix_std": 0.2751314043998718, "beta_std": 0.0039032455533742905, "beta_low": 0.9979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.04019378870725632, "evidence_weights": [-0.2184266746044159, -0.3022410273551941, 0.8346503973007202], "agreement_weight": 0.12997967004776, "orthogonal_weight": -0.9194270968437195}]}, {"layer": 14, "memory_to_active": 0.248111292719841, "contribution_to_active": 0.41955065727233887, "mix_mean": 0.42128893733024597, "mix_low": 0.02393341064453125, "mix_high": 0.005377610679715872, "read_cosine": 0.6295132040977478, "beta_mean": 0.06480563431978226, "nonfinite": 0.0, "weighted_memory_to_active": 0.10658625513315201, "mix_std": 0.24405407905578613, "beta_std": 0.06118842214345932, "beta_low": 0.4659932553768158, "beta_high": 0.0, "orthogonal_to_active": 0.04309920594096184, "output_influence": 0.3279261589050293, "residual_influence": 0.12664194405078888, "state_active": 0.059246987104415894, "state_memory": 0.01582960970699787, "state_peak": 38.08798599243164, "growth_active": 1.110802412033081, "growth_memory": 1.2735978364944458, "heads": [{"head": 0, "memory_to_active": 0.3722643256187439, "contribution_to_active": 0.19767513871192932, "mix_mean": 0.21095454692840576, "mix_low": 0.013333320617675781, "mix_high": 3.814697265625e-06, "read_cosine": 0.8024324178695679, "beta_mean": 0.06567545980215073, "nonfinite": 0.0, "weighted_memory_to_active": 0.09008149802684784, "mix_std": 0.1855686902999878, "beta_std": 0.057984985411167145, "beta_low": 0.0087890625, "beta_high": 0.0, "orthogonal_to_active": 0.07724826037883759, "evidence_weights": [-0.13458184897899628, -0.035582561045885086, 0.47783714532852173], "agreement_weight": 0.02410603128373623, "orthogonal_weight": 1.4217255115509033}, {"head": 1, "memory_to_active": 0.5021313428878784, "contribution_to_active": 0.2998223602771759, "mix_mean": 0.22361502051353455, "mix_low": 0.11521339416503906, "mix_high": 3.4332275390625e-05, "read_cosine": 0.8214672803878784, "beta_mean": 0.2541709542274475, "nonfinite": 0.0, "weighted_memory_to_active": 0.2208411991596222, "mix_std": 0.24995547533035278, "beta_std": 0.2468244731426239, "beta_low": 0.106689453125, "beta_high": 0.0, "orthogonal_to_active": 0.014101753942668438, "evidence_weights": [-0.013198199681937695, -0.2182067334651947, -0.14849595725536346], "agreement_weight": 0.05311128869652748, "orthogonal_weight": 0.15375982224941254}, {"head": 2, "memory_to_active": 0.15400728583335876, "contribution_to_active": 0.5670430064201355, "mix_mean": 0.5739471912384033, "mix_low": 0.0023622512817382812, "mix_high": 0.0027666091918945312, "read_cosine": 0.42034459114074707, "beta_mean": 0.0031303726136684418, "nonfinite": 0.0, "weighted_memory_to_active": 0.09821511059999466, "mix_std": 0.2656062841415405, "beta_std": 0.004402509890496731, "beta_low": 0.94061279296875, "beta_high": 0.0, "orthogonal_to_active": 0.07294164597988129, "evidence_weights": [-0.1498813033103943, -0.31200212240219116, 0.9011257886886597], "agreement_weight": 0.086105577647686, "orthogonal_weight": -0.8527455925941467}, {"head": 3, "memory_to_active": 0.05247946083545685, "contribution_to_active": 0.6346740126609802, "mix_mean": 0.674025297164917, "mix_low": 0.005234718322753906, "mix_high": 0.019926071166992188, "read_cosine": 0.1466987133026123, "beta_mean": 0.007761616725474596, "nonfinite": 0.0, "weighted_memory_to_active": 0.03351016342639923, "mix_std": 0.2519438862800598, "beta_std": 0.008034584112465382, "beta_low": 0.74542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.022838246077299118, "evidence_weights": [-0.3641788959503174, -0.3197609782218933, 0.3571400046348572], "agreement_weight": 0.07746800780296326, "orthogonal_weight": -0.777194082736969}, {"head": 4, "memory_to_active": 0.16170312464237213, "contribution_to_active": 0.49132633209228516, "mix_mean": 0.49157077074050903, "mix_low": 0.0037279129028320312, "mix_high": 0.008256912231445312, "read_cosine": 0.8464900255203247, "beta_mean": 0.05621380731463432, "nonfinite": 0.0, "weighted_memory_to_active": 0.09517717361450195, "mix_std": 0.2611278295516968, "beta_std": 0.039623506367206573, "beta_low": 0.02490234375, "beta_high": 0.0, "orthogonal_to_active": 0.02846798300743103, "evidence_weights": [-0.22042444348335266, -0.25038591027259827, 0.42675694823265076], "agreement_weight": 0.12633094191551208, "orthogonal_weight": -0.6028608679771423}, {"head": 5, "memory_to_active": 0.24608208239078522, "contribution_to_active": 0.32676318287849426, "mix_mean": 0.3536206781864166, "mix_low": 0.0037288665771484375, "mix_high": 0.001277923583984375, "read_cosine": 0.7396460771560669, "beta_mean": 0.0018815582152456045, "nonfinite": 0.0, "weighted_memory_to_active": 0.10169236361980438, "mix_std": 0.2501223087310791, "beta_std": 0.010260450653731823, "beta_low": 0.96954345703125, "beta_high": 0.0, "orthogonal_to_active": 0.0429973378777504, "evidence_weights": [-0.17959414422512054, -0.16419334709644318, 0.25133216381073], "agreement_weight": 0.08252795040607452, "orthogonal_weight": -0.6850267052650452}]}, {"layer": 15, "memory_to_active": 0.06016991659998894, "contribution_to_active": 0.48710882663726807, "mix_mean": 0.4083516597747803, "mix_low": 0.008771419525146484, "mix_high": 0.0021963119506835938, "read_cosine": 0.6946987509727478, "beta_mean": 0.04221238195896149, "nonfinite": 0.0, "weighted_memory_to_active": 0.028738748282194138, "mix_std": 0.2828938066959381, "beta_std": 0.04636972397565842, "beta_low": 0.7077434062957764, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.006662257015705109, "output_influence": 0.3017394244670868, "residual_influence": 0.16680552065372467, "state_active": 0.13657860457897186, "state_memory": 0.010099349543452263, "state_peak": 43.44447326660156, "growth_active": 1.2714394330978394, "growth_memory": 1.7567862272262573, "heads": [{"head": 0, "memory_to_active": 0.10790165513753891, "contribution_to_active": 0.4740782082080841, "mix_mean": 0.3818512558937073, "mix_low": 0.005343437194824219, "mix_high": 0.004721641540527344, "read_cosine": 0.6906871199607849, "beta_mean": 0.042848750948905945, "nonfinite": 0.0, "weighted_memory_to_active": 0.05125004053115845, "mix_std": 0.2771458029747009, "beta_std": 0.036677248775959015, "beta_low": 0.10858154296875, "beta_high": 0.0, "orthogonal_to_active": 0.023615267127752304, "evidence_weights": [-0.18060849606990814, -0.18451033532619476, 0.3785995543003082], "agreement_weight": 0.09621325135231018, "orthogonal_weight": -0.7177543044090271}, {"head": 1, "memory_to_active": 0.13044646382331848, "contribution_to_active": 0.41455891728401184, "mix_mean": 0.4250459671020508, "mix_low": 0.0020608901977539062, "mix_high": 0.0044002532958984375, "read_cosine": 0.9194657802581787, "beta_mean": 0.0009005344472825527, "nonfinite": 0.0, "weighted_memory_to_active": 0.0542820580303669, "mix_std": 0.28340592980384827, "beta_std": 0.002503511030226946, "beta_low": 0.98675537109375, "beta_high": 0.0, "orthogonal_to_active": 0.00042089622002094984, "evidence_weights": [-0.1587609499692917, -0.16838590800762177, 0.22093260288238525], "agreement_weight": 0.11654481291770935, "orthogonal_weight": 0.0147286681458354}, {"head": 2, "memory_to_active": 0.006858195643872023, "contribution_to_active": 0.4893682897090912, "mix_mean": 0.34372687339782715, "mix_low": 0.015307426452636719, "mix_high": 7.82012939453125e-05, "read_cosine": 0.7904199361801147, "beta_mean": 0.0001069265854312107, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034900924656540155, "mix_std": 0.27149680256843567, "beta_std": 0.00025559833738952875, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.000675236398819834, "evidence_weights": [-0.20093710720539093, -0.2711640000343323, 0.20484890043735504], "agreement_weight": 0.009631057269871235, "orthogonal_weight": 0.4894578456878662}, {"head": 3, "memory_to_active": 0.0037688210140913725, "contribution_to_active": 0.3982524871826172, "mix_mean": 0.37796780467033386, "mix_low": 0.006557464599609375, "mix_high": 0.000225067138671875, "read_cosine": 0.6116518378257751, "beta_mean": 2.4666569515829906e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.001353031606413424, "mix_std": 0.28434979915618896, "beta_std": 6.140959885669872e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011976899113506079, "evidence_weights": [-0.23091353476047516, -0.2509952187538147, 0.27803125977516174], "agreement_weight": 0.03358937427401543, "orthogonal_weight": -0.9823556542396545}, {"head": 4, "memory_to_active": 0.09031909704208374, "contribution_to_active": 0.6161500811576843, "mix_mean": 0.3616965413093567, "mix_low": 0.01334381103515625, "mix_high": 0.00057220458984375, "read_cosine": 0.7629993557929993, "beta_mean": 0.2082786113023758, "nonfinite": 0.0, "weighted_memory_to_active": 0.047881707549095154, "mix_std": 0.30378326773643494, "beta_std": 0.23136328160762787, "beta_low": 0.17254638671875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0062228040769696236, "evidence_weights": [-0.21332989633083344, -0.2542979419231415, -0.027510054409503937], "agreement_weight": 0.013066835701465607, "orthogonal_weight": 0.281423956155777}, {"head": 5, "memory_to_active": 0.021725233644247055, "contribution_to_active": 0.5302448272705078, "mix_mean": 0.559821367263794, "mix_low": 0.010015487670898438, "mix_high": 0.0031805038452148438, "read_cosine": 0.3929685354232788, "beta_mean": 0.0011147897457703948, "nonfinite": 0.0, "weighted_memory_to_active": 0.014175552874803543, "mix_std": 0.27718114852905273, "beta_std": 0.007357290014624596, "beta_low": 0.9786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.007841648533940315, "evidence_weights": [-0.3034133315086365, -0.31531229615211487, 0.7675540447235107], "agreement_weight": 0.2646290957927704, "orthogonal_weight": 0.7473172545433044}]}]}, {"step": 41000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 35.92186724600015, "memory_to_active": 0.827393651008606, "contribution_to_active": 0.53495854139328, "mix_mean": 0.38832736015319824, "mix_low": 0.1447458565235138, "mix_high": 0.00975263211876154, "read_cosine": 0.36471039056777954, "beta_mean": 0.07542911171913147, "nonfinite": 0.0, "weighted_memory_to_active": 0.2032942920923233, "mix_std": 0.21901249885559082, "beta_std": 0.06579631567001343, "beta_low": 0.5330905914306641, "beta_high": 2.288818359375e-05, "orthogonal_to_active": 0.05630403012037277, "samples": 4, "scored_tokens": 899, "normal_loss": 3.138275623321533, "normal_token_accuracy": 0.44382646679878235, "state_active": 0.06216529756784439, "state_memory": 0.018476638942956924, "state_peak": 121.4257583618164, "growth_active": 1.1060731410980225, "growth_memory": 1.352000117301941, "output_influence": 0.3965483009815216, "residual_influence": 0.13074178993701935, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.237610340118408, "no_memory_loss_delta": 0.5346198081970215, "scrambled_loss_delta": 0.1521749049425125, "memory_only_loss_delta": 2.9422409534454346, "no_evidence_loss_delta": 0.3030169606208801, "no_agreement_loss_delta": 0.022476328536868095, "no_orthogonal_loss_delta": 0.005114639177918434}, {"index": 94, "tokens": 256, "normal_loss": 3.280654191970825, "no_memory_loss_delta": 0.5585425496101379, "scrambled_loss_delta": 0.31153321266174316, "memory_only_loss_delta": 3.2671234607696533, "no_evidence_loss_delta": 0.25706976652145386, "no_agreement_loss_delta": 0.042085401713848114, "no_orthogonal_loss_delta": 0.04367068409919739}, {"index": 188, "tokens": 256, "normal_loss": 3.432748317718506, "no_memory_loss_delta": 0.6172062158584595, "scrambled_loss_delta": 0.13640397787094116, "memory_only_loss_delta": 2.8319971561431885, "no_evidence_loss_delta": 0.32109135389328003, "no_agreement_loss_delta": 0.04209539666771889, "no_orthogonal_loss_delta": 0.038082800805568695}, {"index": 283, "tokens": 131, "normal_loss": 2.0904626846313477, "no_memory_loss_delta": 0.1948060542345047, "scrambled_loss_delta": 0.17675216495990753, "memory_only_loss_delta": 2.645685911178589, "no_evidence_loss_delta": -0.005010322667658329, "no_agreement_loss_delta": 0.0030929488129913807, "no_orthogonal_loss_delta": -0.010013299994170666}], "agreement_weight_abs_mean": 0.17952394485473633, "agreement_weight_abs_max": 4.583496570587158, "agreement_weight_mean": 0.1573231965303421, "orthogonal_weight_abs_mean": 0.7655003666877747, "orthogonal_weight_abs_max": 7.332150459289551, "orthogonal_weight_mean": -0.13767561316490173, "no_memory_loss": 3.6537084579467773, "no_memory_loss_delta": 0.5154326558113098, "no_memory_token_accuracy_gain": 0.08342602849006653, "no_memory_token_win_fraction": 0.6785317063331604, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 40.27578615709673, "scrambled_loss": 3.3349196910858154, "scrambled_loss_delta": 0.19664430618286133, "scrambled_token_accuracy_gain": 0.027808675542473793, "scrambled_token_win_fraction": 0.5973303914070129, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 17.8517222302485, "memory_only_loss": 6.098423480987549, "memory_only_loss_delta": 2.960148334503174, "memory_only_token_accuracy_gain": 0.27586206793785095, "memory_only_token_win_fraction": 0.906562864780426, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.81887687905471, "no_evidence_loss": 3.3884706497192383, "no_evidence_loss_delta": 0.2501949071884155, "no_evidence_token_accuracy_gain": 0.03893214836716652, "no_evidence_token_win_fraction": 0.5895439386367798, "no_evidence_probe_win_fraction": 0.75, "no_evidence_ppl_reduction_pct": 22.13509960076621, "no_agreement_loss": 3.169097900390625, "no_agreement_loss_delta": 0.030822468921542168, "no_agreement_token_accuracy_gain": 0.0055617354810237885, "no_agreement_token_win_fraction": 0.4838709533214569, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.0352299601424817, "no_orthogonal_loss": 3.161553144454956, "no_orthogonal_loss_delta": 0.023277530446648598, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5528364777565002, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.3008698687902047, "short_no_memory_loss_delta": 0.44187161326408386, "long_context_memory_gain": 0.07356104254722595, "normal_context_loss_gain": 0.1329013556241989, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.3654212951660156, "contribution_to_active": 1.095288634300232, "mix_mean": 0.10233810544013977, "mix_low": 0.549018919467926, "mix_high": 0.0049915313720703125, "read_cosine": 0.42808422446250916, "beta_mean": 0.2489503026008606, "nonfinite": 0.0, "weighted_memory_to_active": 1.1390576362609863, "mix_std": 0.08885842561721802, "beta_std": 0.16105903685092926, "beta_low": 0.03466796875, "beta_high": 0.0, "orthogonal_to_active": 0.05507120490074158, "output_influence": 0.1964491307735443, "residual_influence": 0.23512409627437592, "state_active": 0.06642965972423553, "state_memory": 0.04767270386219025, "state_peak": 45.314815521240234, "growth_active": 1.2081279754638672, "growth_memory": 1.2595194578170776, "heads": [{"head": 0, "memory_to_active": 2.5572071075439453, "contribution_to_active": 0.4653535485267639, "mix_mean": 0.01943906769156456, "mix_low": 0.6251134872436523, "mix_high": 0.0, "read_cosine": 0.23140718042850494, "beta_mean": 0.21907450258731842, "nonfinite": 0.0, "weighted_memory_to_active": 0.5600209832191467, "mix_std": 0.05919816344976425, "beta_std": 0.11921659857034683, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.047399360686540604, "evidence_weights": [-0.017887774854898453, -0.06329840421676636, -0.11032383888959885], "agreement_weight": -0.15213435888290405, "orthogonal_weight": -0.0929446741938591}, {"head": 1, "memory_to_active": 5.88383150100708, "contribution_to_active": 2.3873512744903564, "mix_mean": 0.016240060329437256, "mix_low": 0.6850967407226562, "mix_high": 0.0, "read_cosine": 0.5004411935806274, "beta_mean": 0.4627983272075653, "nonfinite": 0.0, "weighted_memory_to_active": 2.7098989486694336, "mix_std": 0.05255153775215149, "beta_std": 0.2141684740781784, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0679682195186615, "evidence_weights": [0.024106420576572418, -0.10526055842638016, 0.4512503445148468], "agreement_weight": 0.026982074603438377, "orthogonal_weight": -0.0516468770802021}, {"head": 2, "memory_to_active": 2.53120493888855, "contribution_to_active": 0.4351889491081238, "mix_mean": 0.012955831363797188, "mix_low": 0.6822881698608398, "mix_high": 0.0, "read_cosine": 0.2856873869895935, "beta_mean": 0.31468695402145386, "nonfinite": 0.0, "weighted_memory_to_active": 0.5156627297401428, "mix_std": 0.028198251500725746, "beta_std": 0.2312505692243576, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05206575244665146, "evidence_weights": [0.06235736235976219, 0.0680183693766594, 0.03098217584192753], "agreement_weight": 0.010064244270324707, "orthogonal_weight": -0.2127571702003479}, {"head": 3, "memory_to_active": 6.735774993896484, "contribution_to_active": 2.7514805793762207, "mix_mean": 0.03560277074575424, "mix_low": 0.49115562438964844, "mix_high": 0.0, "read_cosine": 0.30431047081947327, "beta_mean": 0.338286817073822, "nonfinite": 0.0, "weighted_memory_to_active": 2.9234373569488525, "mix_std": 0.08244350552558899, "beta_std": 0.259189635515213, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.13203755021095276, "evidence_weights": [-0.04496314376592636, -0.054795924574136734, -0.15388084948062897], "agreement_weight": -0.006857989355921745, "orthogonal_weight": -0.05323419347405434}, {"head": 4, "memory_to_active": 2.377885103225708, "contribution_to_active": 0.09814997762441635, "mix_mean": 0.007835952565073967, "mix_low": 0.7928733825683594, "mix_high": 0.0, "read_cosine": 0.3358170688152313, "beta_mean": 0.12704285979270935, "nonfinite": 0.0, "weighted_memory_to_active": 0.07064241915941238, "mix_std": 0.01834823004901409, "beta_std": 0.11207395792007446, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.00905739888548851, "evidence_weights": [0.12835468351840973, 0.08146021515130997, -0.33399689197540283], "agreement_weight": -0.11936461180448532, "orthogonal_weight": 0.11741259694099426}, {"head": 5, "memory_to_active": 0.10662344098091125, "contribution_to_active": 0.43420711159706116, "mix_mean": 0.5219549536705017, "mix_low": 0.01758575439453125, "mix_high": 0.029949188232421875, "read_cosine": 0.9108420014381409, "beta_mean": 0.03181235119700432, "nonfinite": 0.0, "weighted_memory_to_active": 0.054683636873960495, "mix_std": 0.29241085052490234, "beta_std": 0.0304549653083086, "beta_low": 0.2078857421875, "beta_high": 0.0, "orthogonal_to_active": 0.021898943930864334, "evidence_weights": [-0.20810289680957794, -0.22975720465183258, 0.3435841202735901], "agreement_weight": 0.2800063192844391, "orthogonal_weight": -0.7110478281974792}]}, {"layer": 1, "memory_to_active": 2.160449504852295, "contribution_to_active": 0.47916826605796814, "mix_mean": 0.19609808921813965, "mix_low": 0.36065277457237244, "mix_high": 0.0010431609116494656, "read_cosine": 0.31341809034347534, "beta_mean": 0.17080265283584595, "nonfinite": 0.0, "weighted_memory_to_active": 0.321616530418396, "mix_std": 0.11937379837036133, "beta_std": 0.12564006447792053, "beta_low": 0.308868408203125, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.04954829439520836, "output_influence": 0.20715665817260742, "residual_influence": 0.053374119102954865, "state_active": 0.03566181659698486, "state_memory": 0.035158444195985794, "state_peak": 35.241607666015625, "growth_active": 1.2747697830200195, "growth_memory": 1.4956926107406616, "heads": [{"head": 0, "memory_to_active": 0.041819263249635696, "contribution_to_active": 0.4650058150291443, "mix_mean": 0.45828312635421753, "mix_low": 0.009107589721679688, "mix_high": 0.0019369125366210938, "read_cosine": -0.002366070169955492, "beta_mean": 0.00029696442652493715, "nonfinite": 0.0, "weighted_memory_to_active": 0.02327302284538746, "mix_std": 0.28649333119392395, "beta_std": 0.000362182327080518, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.022157741710543633, "evidence_weights": [-0.19385555386543274, -0.2529614269733429, 0.29245996475219727], "agreement_weight": 0.188783198595047, "orthogonal_weight": -1.0274714231491089}, {"head": 1, "memory_to_active": 2.797943592071533, "contribution_to_active": 0.2477310597896576, "mix_mean": 0.023525673896074295, "mix_low": 0.4590139389038086, "mix_high": 2.86102294921875e-06, "read_cosine": 0.48405781388282776, "beta_mean": 0.2715960144996643, "nonfinite": 0.0, "weighted_memory_to_active": 0.271286278963089, "mix_std": 0.04915926977992058, "beta_std": 0.18466097116470337, "beta_low": 0.00091552734375, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.017149262130260468, "evidence_weights": [0.012739907018840313, -0.04531339555978775, 0.02118452824652195], "agreement_weight": -0.00812153983861208, "orthogonal_weight": 0.14913037419319153}, {"head": 2, "memory_to_active": 0.14899702370166779, "contribution_to_active": 0.5973696112632751, "mix_mean": 0.6271578669548035, "mix_low": 0.000881195068359375, "mix_high": 0.004317283630371094, "read_cosine": 0.6372752785682678, "beta_mean": 0.007842056453227997, "nonfinite": 0.0, "weighted_memory_to_active": 0.11831314861774445, "mix_std": 0.2412136197090149, "beta_std": 0.009260943159461021, "beta_low": 0.762451171875, "beta_high": 0.0, "orthogonal_to_active": 0.04765360802412033, "evidence_weights": [-0.3006241023540497, -0.3466442823410034, 0.5393075942993164], "agreement_weight": 0.024739239364862442, "orthogonal_weight": -0.5786311030387878}, {"head": 3, "memory_to_active": 2.8163816928863525, "contribution_to_active": 0.185638889670372, "mix_mean": 0.020891552790999413, "mix_low": 0.6111526489257812, "mix_high": 9.5367431640625e-07, "read_cosine": 0.23563626408576965, "beta_mean": 0.32347720861434937, "nonfinite": 0.0, "weighted_memory_to_active": 0.2812785506248474, "mix_std": 0.05784674733877182, "beta_std": 0.24334822595119476, "beta_low": 0.0440673828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.07613050937652588, "evidence_weights": [0.0424543060362339, -0.008524227887392044, 0.10999154299497604], "agreement_weight": -0.02189362235367298, "orthogonal_weight": -0.3825167417526245}, {"head": 4, "memory_to_active": 3.480071783065796, "contribution_to_active": 1.3052998781204224, "mix_mean": 0.039344049990177155, "mix_low": 0.26703834533691406, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3096344769001007, "beta_mean": 0.15977485477924347, "nonfinite": 0.0, "weighted_memory_to_active": 1.182328224182129, "mix_std": 0.06456003338098526, "beta_std": 0.11652377247810364, "beta_low": 0.0146484375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12351922690868378, "evidence_weights": [-0.049600668251514435, -0.007474479265511036, 0.04788384214043617], "agreement_weight": -0.02041298896074295, "orthogonal_weight": 0.1330355554819107}, {"head": 5, "memory_to_active": 3.6774840354919434, "contribution_to_active": 0.07396412640810013, "mix_mean": 0.0073861838318407536, "mix_low": 0.8167228698730469, "mix_high": 0.0, "read_cosine": 0.21627077460289001, "beta_mean": 0.2618287205696106, "nonfinite": 0.0, "weighted_memory_to_active": 0.05322011932730675, "mix_std": 0.016969770193099976, "beta_std": 0.199684277176857, "beta_low": 0.0311279296875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.010679397732019424, "evidence_weights": [0.16012147068977356, 0.07809841632843018, 0.22967448830604553], "agreement_weight": -0.14141006767749786, "orthogonal_weight": 0.1857488751411438}]}, {"layer": 2, "memory_to_active": 1.0855109691619873, "contribution_to_active": 0.3442390561103821, "mix_mean": 0.37565380334854126, "mix_low": 0.25512266159057617, "mix_high": 0.003768603317439556, "read_cosine": 0.3221350908279419, "beta_mean": 0.09731155633926392, "nonfinite": 0.0, "weighted_memory_to_active": 0.07629287987947464, "mix_std": 0.19249564409255981, "beta_std": 0.07345623522996902, "beta_low": 0.561004638671875, "beta_high": 0.00014241535973269492, "orthogonal_to_active": 0.019147705286741257, "output_influence": 0.3584398329257965, "residual_influence": 0.0884900763630867, "state_active": 0.03254441171884537, "state_memory": 0.017474735155701637, "state_peak": 30.771440505981445, "growth_active": 0.9109294414520264, "growth_memory": 1.5743135213851929, "heads": [{"head": 0, "memory_to_active": 0.00612298771739006, "contribution_to_active": 0.3173002004623413, "mix_mean": 0.5818529725074768, "mix_low": 0.001483917236328125, "mix_high": 0.0072021484375, "read_cosine": 0.17343375086784363, "beta_mean": 5.857802170794457e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034788434859365225, "mix_std": 0.2866434156894684, "beta_std": 0.00010387005750089884, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025667031295597553, "evidence_weights": [-0.33889976143836975, -0.20556120574474335, 0.1444607973098755], "agreement_weight": 0.25745636224746704, "orthogonal_weight": -0.7734895944595337}, {"head": 1, "memory_to_active": 3.0619938373565674, "contribution_to_active": 0.04375447332859039, "mix_mean": 0.00477387011051178, "mix_low": 0.8953704833984375, "mix_high": 0.0, "read_cosine": 0.17704366147518158, "beta_mean": 0.30033451318740845, "nonfinite": 0.0, "weighted_memory_to_active": 0.033714886754751205, "mix_std": 0.010036952793598175, "beta_std": 0.16836996376514435, "beta_low": 0.00067138671875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.012605874799191952, "evidence_weights": [0.12567541003227234, 0.11262991279363632, 0.5091363787651062], "agreement_weight": 0.03433385491371155, "orthogonal_weight": 0.4152849316596985}, {"head": 2, "memory_to_active": 0.015284933149814606, "contribution_to_active": 0.4139341115951538, "mix_mean": 0.5114206671714783, "mix_low": 0.001445770263671875, "mix_high": 0.0024585723876953125, "read_cosine": 0.2891290783882141, "beta_mean": 2.9401180654531345e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005894465837627649, "mix_std": 0.28093546628952026, "beta_std": 5.180472362553701e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009833575459197164, "evidence_weights": [-0.2414989024400711, -0.2804941236972809, 0.4769798517227173], "agreement_weight": 0.024785803630948067, "orthogonal_weight": -0.1957491934299469}, {"head": 3, "memory_to_active": 0.16797654330730438, "contribution_to_active": 0.5024619698524475, "mix_mean": 0.5170460939407349, "mix_low": 0.0004711151123046875, "mix_high": 0.006007194519042969, "read_cosine": 0.6215610504150391, "beta_mean": 0.032163433730602264, "nonfinite": 0.0, "weighted_memory_to_active": 0.09665454924106598, "mix_std": 0.2662203907966614, "beta_std": 0.04135102406144142, "beta_low": 0.32537841796875, "beta_high": 0.0, "orthogonal_to_active": 0.0652962327003479, "evidence_weights": [-0.18619970977306366, -0.2918887138366699, 0.855479896068573], "agreement_weight": 0.05404185131192207, "orthogonal_weight": -0.8897661566734314}, {"head": 4, "memory_to_active": 3.251636505126953, "contribution_to_active": 0.2573854327201843, "mix_mean": 0.016488276422023773, "mix_low": 0.6306591033935547, "mix_high": 0.0, "read_cosine": 0.20807941257953644, "beta_mean": 0.2512509226799011, "nonfinite": 0.0, "weighted_memory_to_active": 0.3122687339782715, "mix_std": 0.04141823574900627, "beta_std": 0.23081167042255402, "beta_low": 0.03997802734375, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.03330063819885254, "evidence_weights": [0.025284405797719955, 0.015352671034634113, 0.04393167048692703], "agreement_weight": 0.01350037008523941, "orthogonal_weight": -0.11347657442092896}, {"head": 5, "memory_to_active": 0.010050158947706223, "contribution_to_active": 0.5305981636047363, "mix_mean": 0.6223407983779907, "mix_low": 0.0013055801391601562, "mix_high": 0.006943702697753906, "read_cosine": 0.46356362104415894, "beta_mean": 3.2405681849922985e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0057457853108644485, "mix_std": 0.26971936225891113, "beta_std": 4.904842717223801e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00013341459271032363, "evidence_weights": [-0.27276235818862915, -0.2871379554271698, 0.9054538607597351], "agreement_weight": 0.09398725628852844, "orthogonal_weight": 0.03464363515377045}]}, {"layer": 3, "memory_to_active": 0.5380823612213135, "contribution_to_active": 0.4250674247741699, "mix_mean": 0.42081350088119507, "mix_low": 0.07402960956096649, "mix_high": 0.008183320984244347, "read_cosine": 0.22935496270656586, "beta_mean": 0.02936650440096855, "nonfinite": 0.0, "weighted_memory_to_active": 0.1392921656370163, "mix_std": 0.2617335319519043, "beta_std": 0.02695263922214508, "beta_low": 0.6944580674171448, "beta_high": 0.0, "orthogonal_to_active": 0.0615832582116127, "output_influence": 0.3467133045196533, "residual_influence": 0.0736638531088829, "state_active": 0.02148633822798729, "state_memory": 0.00783568061888218, "state_peak": 20.6239013671875, "growth_active": 0.9588121771812439, "growth_memory": 1.8075839281082153, "heads": [{"head": 0, "memory_to_active": 0.46553951501846313, "contribution_to_active": 0.5360069274902344, "mix_mean": 0.5257174372673035, "mix_low": 0.0054683685302734375, "mix_high": 0.015497207641601562, "read_cosine": 0.2553713321685791, "beta_mean": 0.011258463375270367, "nonfinite": 0.0, "weighted_memory_to_active": 0.2700747549533844, "mix_std": 0.31471312046051025, "beta_std": 0.014852301217615604, "beta_low": 0.65679931640625, "beta_high": 0.0, "orthogonal_to_active": 0.1843988299369812, "evidence_weights": [-0.19873513281345367, -0.22940528392791748, 0.8059980273246765], "agreement_weight": 0.1262325942516327, "orthogonal_weight": -0.7270456552505493}, {"head": 1, "memory_to_active": 0.017097381874918938, "contribution_to_active": 0.4042111337184906, "mix_mean": 0.49547791481018066, "mix_low": 0.00933837890625, "mix_high": 0.006783485412597656, "read_cosine": 0.21238084137439728, "beta_mean": 9.299537487095222e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00890576932579279, "mix_std": 0.31276676058769226, "beta_std": 0.00017101057164836675, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00637274282053113, "evidence_weights": [-0.23346228897571564, -0.19653047621250153, 0.6147995591163635], "agreement_weight": 0.402469664812088, "orthogonal_weight": -0.7813963890075684}, {"head": 2, "memory_to_active": 2.371438980102539, "contribution_to_active": 0.29415997862815857, "mix_mean": 0.04464760050177574, "mix_low": 0.3432579040527344, "mix_high": 0.0, "read_cosine": 0.20488187670707703, "beta_mean": 0.14941483736038208, "nonfinite": 0.0, "weighted_memory_to_active": 0.32194802165031433, "mix_std": 0.0942205935716629, "beta_std": 0.12938563525676727, "beta_low": 0.01947021484375, "beta_high": 0.0, "orthogonal_to_active": 0.001150656957179308, "evidence_weights": [-0.08433586359024048, 0.007543701212853193, 0.004463886842131615], "agreement_weight": -0.006885727867484093, "orthogonal_weight": -0.004429515916854143}, {"head": 3, "memory_to_active": 0.01827765442430973, "contribution_to_active": 0.49667537212371826, "mix_mean": 0.5657804608345032, "mix_low": 0.00899505615234375, "mix_high": 0.011895179748535156, "read_cosine": 0.29340237379074097, "beta_mean": 8.01238784333691e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012285895645618439, "mix_std": 0.302626371383667, "beta_std": 0.00019064289517700672, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003076370805501938, "evidence_weights": [-0.22948335111141205, -0.22937941551208496, 2.1186208724975586], "agreement_weight": 1.6845811605453491, "orthogonal_weight": -0.32279151678085327}, {"head": 4, "memory_to_active": 0.08718892931938171, "contribution_to_active": 0.19926905632019043, "mix_mean": 0.24385342001914978, "mix_low": 0.07187843322753906, "mix_high": 0.0004291534423828125, "read_cosine": 0.19103717803955078, "beta_mean": 0.0008631307864561677, "nonfinite": 0.0, "weighted_memory_to_active": 0.031295593827962875, "mix_std": 0.2614089548587799, "beta_std": 0.0012596603482961655, "beta_low": 0.9976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.027981627732515335, "evidence_weights": [-0.08198563009500504, -0.12263834476470947, 0.23933996260166168], "agreement_weight": -0.002355043776333332, "orthogonal_weight": -0.9296499490737915}, {"head": 5, "memory_to_active": 0.2689516246318817, "contribution_to_active": 0.6200817823410034, "mix_mean": 0.6494041681289673, "mix_low": 0.0052394866943359375, "mix_high": 0.014494895935058594, "read_cosine": 0.2190561145544052, "beta_mean": 0.014489484019577503, "nonfinite": 0.0, "weighted_memory_to_active": 0.19124296307563782, "mix_std": 0.2846653461456299, "beta_std": 0.01585656963288784, "beta_low": 0.4927978515625, "beta_high": 0.0, "orthogonal_to_active": 0.14651930332183838, "evidence_weights": [-0.24545985460281372, -0.28955546021461487, 0.5188551545143127], "agreement_weight": 0.12749844789505005, "orthogonal_weight": -0.820712685585022}]}, {"layer": 4, "memory_to_active": 1.1158188581466675, "contribution_to_active": 0.4324195384979248, "mix_mean": 0.3503974676132202, "mix_low": 0.28981876373291016, "mix_high": 0.006661256309598684, "read_cosine": 0.34702858328819275, "beta_mean": 0.11721494048833847, "nonfinite": 0.0, "weighted_memory_to_active": 0.12353505194187164, "mix_std": 0.152958944439888, "beta_std": 0.10911378264427185, "beta_low": 0.3771871030330658, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03980930894613266, "output_influence": 0.428514301776886, "residual_influence": 0.11763052642345428, "state_active": 0.0376645028591156, "state_memory": 0.018096771091222763, "state_peak": 36.44029235839844, "growth_active": 1.1979562044143677, "growth_memory": 1.5447856187820435, "heads": [{"head": 0, "memory_to_active": 0.21053382754325867, "contribution_to_active": 0.6889688372612, "mix_mean": 0.635282576084137, "mix_low": 0.014863014221191406, "mix_high": 0.0119476318359375, "read_cosine": 0.5067463517189026, "beta_mean": 0.2693525552749634, "nonfinite": 0.0, "weighted_memory_to_active": 0.15272104740142822, "mix_std": 0.34422174096107483, "beta_std": 0.2739308476448059, "beta_low": 0.21612548828125, "beta_high": 0.0, "orthogonal_to_active": 0.05822901055216789, "evidence_weights": [-0.26544496417045593, -0.3665317893028259, 0.1570092737674713], "agreement_weight": 0.015538480132818222, "orthogonal_weight": 0.46369922161102295}, {"head": 1, "memory_to_active": 0.06279686838388443, "contribution_to_active": 0.630549430847168, "mix_mean": 0.6661264896392822, "mix_low": 7.534027099609375e-05, "mix_high": 0.010396957397460938, "read_cosine": 0.6776989698410034, "beta_mean": 0.0009753112681210041, "nonfinite": 0.0, "weighted_memory_to_active": 0.04495161399245262, "mix_std": 0.24181526899337769, "beta_std": 0.004509380552917719, "beta_low": 0.98443603515625, "beta_high": 0.0, "orthogonal_to_active": 0.024606548249721527, "evidence_weights": [-0.23948785662651062, -0.3409387469291687, 0.7131223678588867], "agreement_weight": 0.19768251478672028, "orthogonal_weight": -0.8138208389282227}, {"head": 2, "memory_to_active": 2.2231597900390625, "contribution_to_active": 0.22275757789611816, "mix_mean": 0.017894120886921883, "mix_low": 0.6086330413818359, "mix_high": 8.58306884765625e-06, "read_cosine": 0.26659247279167175, "beta_mean": 0.10894954204559326, "nonfinite": 0.0, "weighted_memory_to_active": 0.1935492604970932, "mix_std": 0.039303023368120193, "beta_std": 0.0985579639673233, "beta_low": 0.0181884765625, "beta_high": 0.0, "orthogonal_to_active": 0.021142860874533653, "evidence_weights": [0.06211172416806221, 0.028880959376692772, 0.0386803112924099], "agreement_weight": 0.13196952641010284, "orthogonal_weight": -0.15759631991386414}, {"head": 3, "memory_to_active": 1.7065441608428955, "contribution_to_active": 0.10254821181297302, "mix_mean": 0.02588754892349243, "mix_low": 0.4727745056152344, "mix_high": 0.0, "read_cosine": 0.2011018693447113, "beta_mean": 0.11485660076141357, "nonfinite": 0.0, "weighted_memory_to_active": 0.1592569351196289, "mix_std": 0.04953273385763168, "beta_std": 0.11297676712274551, "beta_low": 0.04437255859375, "beta_high": 0.0, "orthogonal_to_active": 0.006186143960803747, "evidence_weights": [-0.017998946830630302, 0.06251901388168335, -0.23733268678188324], "agreement_weight": 0.07013886421918869, "orthogonal_weight": -0.05744706839323044}, {"head": 4, "memory_to_active": 0.015371693298220634, "contribution_to_active": 0.6479368209838867, "mix_mean": 0.7434820532798767, "mix_low": 4.9591064453125e-05, "mix_high": 0.01760387420654297, "read_cosine": 0.16806867718696594, "beta_mean": 3.291814209660515e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011276892386376858, "mix_std": 0.21391266584396362, "beta_std": 0.00010912952711805701, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003332295920699835, "evidence_weights": [-0.3310539126396179, -0.33159300684928894, 0.6102427244186401], "agreement_weight": 0.33344343304634094, "orthogonal_weight": -0.3218120038509369}, {"head": 5, "memory_to_active": 2.4765071868896484, "contribution_to_active": 0.3017560839653015, "mix_mean": 0.013712050393223763, "mix_low": 0.64251708984375, "mix_high": 1.049041748046875e-05, "read_cosine": 0.26196303963661194, "beta_mean": 0.2091226875782013, "nonfinite": 0.0, "weighted_memory_to_active": 0.17945453524589539, "mix_std": 0.028968211263418198, "beta_std": 0.1645985245704651, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1253589689731598, "evidence_weights": [0.04876825958490372, -0.03360208868980408, -0.25525665283203125], "agreement_weight": -0.07640188932418823, "orthogonal_weight": 0.8848901987075806}]}, {"layer": 5, "memory_to_active": 0.4547497034072876, "contribution_to_active": 0.49769532680511475, "mix_mean": 0.46211081743240356, "mix_low": 0.06852738559246063, "mix_high": 0.012922605499625206, "read_cosine": 0.09520872682332993, "beta_mean": 0.053116150200366974, "nonfinite": 0.0, "weighted_memory_to_active": 0.13848023116588593, "mix_std": 0.2525182366371155, "beta_std": 0.07039926201105118, "beta_low": 0.6486104726791382, "beta_high": 0.0, "orthogonal_to_active": 0.050257910043001175, "output_influence": 0.4941815733909607, "residual_influence": 0.13254226744174957, "state_active": 0.0450589582324028, "state_memory": 0.01152834389358759, "state_peak": 63.80834197998047, "growth_active": 1.0933398008346558, "growth_memory": 1.2708457708358765, "heads": [{"head": 0, "memory_to_active": 0.43839049339294434, "contribution_to_active": 0.4649692475795746, "mix_mean": 0.2934011220932007, "mix_low": 0.03186321258544922, "mix_high": 0.000457763671875, "read_cosine": -0.16081687808036804, "beta_mean": 0.02857782319188118, "nonfinite": 0.0, "weighted_memory_to_active": 0.1123286709189415, "mix_std": 0.25458621978759766, "beta_std": 0.055766817182302475, "beta_low": 0.4263916015625, "beta_high": 0.0, "orthogonal_to_active": 0.030190380290150642, "evidence_weights": [-0.11466927081346512, -0.14550143480300903, -0.13819269835948944], "agreement_weight": -0.019003281369805336, "orthogonal_weight": -0.27039575576782227}, {"head": 1, "memory_to_active": 0.016190601512789726, "contribution_to_active": 0.5369112491607666, "mix_mean": 0.6512085795402527, "mix_low": 0.0021944046020507812, "mix_high": 0.021346092224121094, "read_cosine": 0.1860434114933014, "beta_mean": 3.8010042771929875e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008947813883423805, "mix_std": 0.2862056791782379, "beta_std": 8.138059638440609e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008429252542555332, "evidence_weights": [-0.2399832308292389, -0.27927419543266296, 0.8793911933898926], "agreement_weight": 0.582897961139679, "orthogonal_weight": -0.9815998673439026}, {"head": 2, "memory_to_active": 0.008678149431943893, "contribution_to_active": 0.535747230052948, "mix_mean": 0.6386417150497437, "mix_low": 0.0021791458129882812, "mix_high": 0.019390106201171875, "read_cosine": -0.3343179523944855, "beta_mean": 6.822969589848071e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005441483575850725, "mix_std": 0.2750515937805176, "beta_std": 0.00011582956358324736, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001510729780420661, "evidence_weights": [-0.25552502274513245, -0.3464816212654114, 0.6970500946044922], "agreement_weight": -0.0122292535379529, "orthogonal_weight": -0.33300691843032837}, {"head": 3, "memory_to_active": 1.7210417985916138, "contribution_to_active": 0.2147122621536255, "mix_mean": 0.08852969110012054, "mix_low": 0.17740726470947266, "mix_high": 0.0, "read_cosine": 0.15254241228103638, "beta_mean": 0.0986635759472847, "nonfinite": 0.0, "weighted_memory_to_active": 0.3440421223640442, "mix_std": 0.12726053595542908, "beta_std": 0.1461363285779953, "beta_low": 0.20050048828125, "beta_high": 0.0, "orthogonal_to_active": 0.1303197294473648, "evidence_weights": [-0.057852763682603836, -0.08717139810323715, 0.016127049922943115], "agreement_weight": 0.05243172124028206, "orthogonal_weight": -0.457324743270874}, {"head": 4, "memory_to_active": 0.451166033744812, "contribution_to_active": 0.5750645399093628, "mix_mean": 0.32016992568969727, "mix_low": 0.19469833374023438, "mix_high": 0.0059452056884765625, "read_cosine": 0.5501481890678406, "beta_mean": 0.19105151295661926, "nonfinite": 0.0, "weighted_memory_to_active": 0.297183632850647, "mix_std": 0.3374098837375641, "beta_std": 0.21898943185806274, "beta_low": 0.26788330078125, "beta_high": 0.0, "orthogonal_to_active": 0.09084147214889526, "evidence_weights": [-0.11071307957172394, -0.27641698718070984, -0.0730554535984993], "agreement_weight": 0.0386296845972538, "orthogonal_weight": 0.49187302589416504}, {"head": 5, "memory_to_active": 0.09303118288516998, "contribution_to_active": 0.6587674617767334, "mix_mean": 0.7807139158248901, "mix_low": 0.0028219223022460938, "mix_high": 0.030396461486816406, "read_cosine": 0.17765317857265472, "beta_mean": 0.0002977472322527319, "nonfinite": 0.0, "weighted_memory_to_active": 0.06293756514787674, "mix_std": 0.2345954179763794, "beta_std": 0.0013057352043688297, "beta_low": 0.99688720703125, "beta_high": 0.0, "orthogonal_to_active": 0.04025588929653168, "evidence_weights": [-0.31809040904045105, -0.32535481452941895, 1.9251585006713867], "agreement_weight": 0.4227488040924072, "orthogonal_weight": -0.6781997680664062}]}, {"layer": 6, "memory_to_active": 0.34414106607437134, "contribution_to_active": 0.5987673401832581, "mix_mean": 0.5686686038970947, "mix_low": 0.06654977798461914, "mix_high": 0.03245067596435547, "read_cosine": 0.288312166929245, "beta_mean": 0.06466469913721085, "nonfinite": 0.0, "weighted_memory_to_active": 0.15821045637130737, "mix_std": 0.23579049110412598, "beta_std": 0.07340206950902939, "beta_low": 0.4993489682674408, "beta_high": 0.0, "orthogonal_to_active": 0.03992180526256561, "output_influence": 0.5233765244483948, "residual_influence": 0.1686851978302002, "state_active": 0.0466139055788517, "state_memory": 0.008561627008020878, "state_peak": 85.53338623046875, "growth_active": 1.1471753120422363, "growth_memory": 1.1808574199676514, "heads": [{"head": 0, "memory_to_active": 0.15224267542362213, "contribution_to_active": 0.6851363182067871, "mix_mean": 0.7737404108047485, "mix_low": 0.00017070770263671875, "mix_high": 0.019356727600097656, "read_cosine": 0.07320788502693176, "beta_mean": 0.020229047164320946, "nonfinite": 0.0, "weighted_memory_to_active": 0.1215122640132904, "mix_std": 0.20912966132164001, "beta_std": 0.025094227865338326, "beta_low": 0.42156982421875, "beta_high": 0.0, "orthogonal_to_active": 0.10095050930976868, "evidence_weights": [-0.3184424936771393, -0.33855220675468445, 0.9280070662498474], "agreement_weight": 0.2782849371433258, "orthogonal_weight": -0.9015433192253113}, {"head": 1, "memory_to_active": 0.006697343662381172, "contribution_to_active": 0.4895555377006531, "mix_mean": 0.546642005443573, "mix_low": 0.0015993118286132812, "mix_high": 0.010163307189941406, "read_cosine": 0.5572018623352051, "beta_mean": 1.333980071649421e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003029505256563425, "mix_std": 0.28568780422210693, "beta_std": 1.968027208931744e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016229924513027072, "evidence_weights": [-0.19947251677513123, -0.2939498722553253, 1.0932025909423828], "agreement_weight": 0.04657553508877754, "orthogonal_weight": -0.7491947412490845}, {"head": 2, "memory_to_active": 1.047593116760254, "contribution_to_active": 0.08320999145507812, "mix_mean": 0.030791975557804108, "mix_low": 0.38904666900634766, "mix_high": 0.0, "read_cosine": 0.22955358028411865, "beta_mean": 0.04457160830497742, "nonfinite": 0.0, "weighted_memory_to_active": 0.06382948160171509, "mix_std": 0.05004793405532837, "beta_std": 0.03962543234229088, "beta_low": 0.06201171875, "beta_high": 0.0, "orthogonal_to_active": 0.0008974575903266668, "evidence_weights": [0.08728543668985367, -0.01838415116071701, 0.2561032176017761], "agreement_weight": 0.13872946798801422, "orthogonal_weight": -0.01730523630976677}, {"head": 3, "memory_to_active": 0.1400526463985443, "contribution_to_active": 0.6779559254646301, "mix_mean": 0.6455415487289429, "mix_low": 0.007407188415527344, "mix_high": 0.018006324768066406, "read_cosine": 0.4458833932876587, "beta_mean": 0.20046228170394897, "nonfinite": 0.0, "weighted_memory_to_active": 0.09996041655540466, "mix_std": 0.32599306106567383, "beta_std": 0.2353346198797226, "beta_low": 0.22723388671875, "beta_high": 0.0, "orthogonal_to_active": 0.055418867617845535, "evidence_weights": [-0.17835399508476257, -0.3366401791572571, 0.20546241104602814], "agreement_weight": -0.0029401914216578007, "orthogonal_weight": 0.6357195973396301}, {"head": 4, "memory_to_active": 0.6853935718536377, "contribution_to_active": 1.0438010692596436, "mix_mean": 0.7633605003356934, "mix_low": 0.0008106231689453125, "mix_high": 0.14326190948486328, "read_cosine": 0.16246557235717773, "beta_mean": 0.12251028418540955, "nonfinite": 0.0, "weighted_memory_to_active": 0.6374199986457825, "mix_std": 0.2909160256385803, "beta_std": 0.13978329300880432, "beta_low": 0.28564453125, "beta_high": 0.0, "orthogonal_to_active": 0.05735605210065842, "evidence_weights": [-0.2479134052991867, -0.5982322692871094, -0.17959053814411163], "agreement_weight": -0.017715375870466232, "orthogonal_weight": 0.10450102388858795}, {"head": 5, "memory_to_active": 0.032867006957530975, "contribution_to_active": 0.612945020198822, "mix_mean": 0.651934802532196, "mix_low": 0.00026416778564453125, "mix_high": 0.0039157867431640625, "read_cosine": 0.2615606486797333, "beta_mean": 0.00020161642169114202, "nonfinite": 0.0, "weighted_memory_to_active": 0.023511050269007683, "mix_std": 0.25296854972839355, "beta_std": 0.0005551563808694482, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.023284945636987686, "evidence_weights": [-0.2427421510219574, -0.31164881587028503, 0.43452388048171997], "agreement_weight": 0.14878599345684052, "orthogonal_weight": -1.0570296049118042}]}, {"layer": 7, "memory_to_active": 1.6919752359390259, "contribution_to_active": 0.2321321964263916, "mix_mean": 0.12811915576457977, "mix_low": 0.2705693244934082, "mix_high": 0.0004083315725438297, "read_cosine": 0.24569788575172424, "beta_mean": 0.1244278997182846, "nonfinite": 0.0, "weighted_memory_to_active": 0.19519482553005219, "mix_std": 0.13711732625961304, "beta_std": 0.10078265517950058, "beta_low": 0.1939493864774704, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.06445536017417908, "output_influence": 0.2420385181903839, "residual_influence": 0.055290356278419495, "state_active": 0.013370124623179436, "state_memory": 0.02527511492371559, "state_peak": 31.44462013244629, "growth_active": 1.1296770572662354, "growth_memory": 1.3171342611312866, "heads": [{"head": 0, "memory_to_active": 2.3561720848083496, "contribution_to_active": 0.20665328204631805, "mix_mean": 0.03652241826057434, "mix_low": 0.4264059066772461, "mix_high": 9.5367431640625e-07, "read_cosine": 0.35714805126190186, "beta_mean": 0.16083312034606934, "nonfinite": 0.0, "weighted_memory_to_active": 0.2505321502685547, "mix_std": 0.07742597907781601, "beta_std": 0.12746718525886536, "beta_low": 0.00653076171875, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.005188356153666973, "evidence_weights": [0.021786201745271683, -0.07843070477247238, 0.07078755646944046], "agreement_weight": -0.03852763772010803, "orthogonal_weight": 0.028151050209999084}, {"head": 1, "memory_to_active": 1.0231988430023193, "contribution_to_active": 0.3571855127811432, "mix_mean": 0.1975807100534439, "mix_low": 0.13948631286621094, "mix_high": 0.0021514892578125, "read_cosine": 0.38525548577308655, "beta_mean": 0.0932900458574295, "nonfinite": 0.0, "weighted_memory_to_active": 0.31863346695899963, "mix_std": 0.2286761850118637, "beta_std": 0.0637616515159607, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.025590896606445312, "evidence_weights": [-0.20108121633529663, -0.10812144726514816, -0.2027140110731125], "agreement_weight": 0.0341431125998497, "orthogonal_weight": 0.09562399983406067}, {"head": 2, "memory_to_active": 0.7424701452255249, "contribution_to_active": 0.30663901567459106, "mix_mean": 0.14489160478115082, "mix_low": 0.06673622131347656, "mix_high": 0.0002079010009765625, "read_cosine": 0.12665420770645142, "beta_mean": 0.05428801476955414, "nonfinite": 0.0, "weighted_memory_to_active": 0.21122030913829803, "mix_std": 0.17549629509449005, "beta_std": 0.044041767716407776, "beta_low": 0.04949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.08316200226545334, "evidence_weights": [-0.05918304994702339, -0.14353662729263306, 0.5662868022918701], "agreement_weight": 0.020034022629261017, "orthogonal_weight": -0.40382495522499084}, {"head": 3, "memory_to_active": 4.233590126037598, "contribution_to_active": 0.09554919600486755, "mix_mean": 0.03511185944080353, "mix_low": 0.32962894439697266, "mix_high": 0.0, "read_cosine": 0.09518569707870483, "beta_mean": 0.33596447110176086, "nonfinite": 0.0, "weighted_memory_to_active": 0.29891517758369446, "mix_std": 0.05111723765730858, "beta_std": 0.26388031244277954, "beta_low": 0.00811767578125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.24764296412467957, "evidence_weights": [0.06075603514909744, -0.016329491510987282, 0.6866494417190552], "agreement_weight": 0.05266556516289711, "orthogonal_weight": -0.9237223863601685}, {"head": 4, "memory_to_active": 1.723062515258789, "contribution_to_active": 0.07784755527973175, "mix_mean": 0.016545230522751808, "mix_low": 0.6541061401367188, "mix_high": 0.0, "read_cosine": 0.21509981155395508, "beta_mean": 0.10173970460891724, "nonfinite": 0.0, "weighted_memory_to_active": 0.06291525065898895, "mix_std": 0.039763957262039185, "beta_std": 0.10456157475709915, "beta_low": 0.1004638671875, "beta_high": 0.0, "orthogonal_to_active": 0.0074022687040269375, "evidence_weights": [0.14207302033901215, 0.03788761794567108, 0.24553881585597992], "agreement_weight": 0.052871786057949066, "orthogonal_weight": -0.15372437238693237}, {"head": 5, "memory_to_active": 0.0733579471707344, "contribution_to_active": 0.3489185869693756, "mix_mean": 0.3380630314350128, "mix_low": 0.007052421569824219, "mix_high": 8.96453857421875e-05, "read_cosine": 0.29484397172927856, "beta_mean": 0.0004519750946201384, "nonfinite": 0.0, "weighted_memory_to_active": 0.028952602297067642, "mix_std": 0.2502243220806122, "beta_std": 0.000983433099463582, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.01774568110704422, "evidence_weights": [-0.1299281269311905, -0.18902811408042908, 0.5620480179786682], "agreement_weight": 0.13135124742984772, "orthogonal_weight": -0.7090092897415161}]}, {"layer": 8, "memory_to_active": 0.3997112214565277, "contribution_to_active": 0.5278958082199097, "mix_mean": 0.42746466398239136, "mix_low": 0.07793887704610825, "mix_high": 0.0010644595604389906, "read_cosine": 0.4719376564025879, "beta_mean": 0.02532730996608734, "nonfinite": 0.0, "weighted_memory_to_active": 0.09272803366184235, "mix_std": 0.21782717108726501, "beta_std": 0.03365737944841385, "beta_low": 0.6343587636947632, "beta_high": 0.0, "orthogonal_to_active": 0.06199916824698448, "output_influence": 0.47224053740501404, "residual_influence": 0.1522866189479828, "state_active": 0.047133490443229675, "state_memory": 0.008494558744132519, "state_peak": 32.56974411010742, "growth_active": 1.074640154838562, "growth_memory": 1.2956645488739014, "heads": [{"head": 0, "memory_to_active": 0.06135430559515953, "contribution_to_active": 0.5589404106140137, "mix_mean": 0.5984556674957275, "mix_low": 0.0003299713134765625, "mix_high": 0.0009708404541015625, "read_cosine": 0.4967634677886963, "beta_mean": 0.002972058020532131, "nonfinite": 0.0, "weighted_memory_to_active": 0.04285659268498421, "mix_std": 0.24414348602294922, "beta_std": 0.01059788465499878, "beta_low": 0.93707275390625, "beta_high": 0.0, "orthogonal_to_active": 0.026821963489055634, "evidence_weights": [-0.17041154205799103, -0.3917146623134613, 0.6806488037109375], "agreement_weight": 0.16241107881069183, "orthogonal_weight": -0.8031379580497742}, {"head": 1, "memory_to_active": 0.12637418508529663, "contribution_to_active": 0.5061858296394348, "mix_mean": 0.4777001440525055, "mix_low": 0.0002231597900390625, "mix_high": 0.000408172607421875, "read_cosine": 0.2458709180355072, "beta_mean": 0.0027883038856089115, "nonfinite": 0.0, "weighted_memory_to_active": 0.05874572694301605, "mix_std": 0.24582378566265106, "beta_std": 0.013084092177450657, "beta_low": 0.9583740234375, "beta_high": 0.0, "orthogonal_to_active": 0.04026097431778908, "evidence_weights": [-0.18813233077526093, -0.2631753981113434, 0.6842407584190369], "agreement_weight": 0.0893804207444191, "orthogonal_weight": -0.7420473694801331}, {"head": 2, "memory_to_active": 0.016257289797067642, "contribution_to_active": 0.6556076407432556, "mix_mean": 0.6436865329742432, "mix_low": 0.0013275146484375, "mix_high": 0.0033540725708007812, "read_cosine": 0.4181057810783386, "beta_mean": 0.0001552844187244773, "nonfinite": 0.0, "weighted_memory_to_active": 0.012164459563791752, "mix_std": 0.23038172721862793, "beta_std": 0.002915541874244809, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00796682108193636, "evidence_weights": [-0.22291190922260284, -0.34239956736564636, 0.9857814908027649], "agreement_weight": 0.22764618694782257, "orthogonal_weight": -0.7574945092201233}, {"head": 3, "memory_to_active": 0.1097281351685524, "contribution_to_active": 0.6446052193641663, "mix_mean": 0.5677498579025269, "mix_low": 0.001766204833984375, "mix_high": 0.0015764236450195312, "read_cosine": 0.8270227909088135, "beta_mean": 0.008920274674892426, "nonfinite": 0.0, "weighted_memory_to_active": 0.08412684500217438, "mix_std": 0.2554529011249542, "beta_std": 0.007079032249748707, "beta_low": 0.65997314453125, "beta_high": 0.0, "orthogonal_to_active": 0.02528158575296402, "evidence_weights": [-0.18409636616706848, -0.3057534098625183, 0.6198870539665222], "agreement_weight": 0.1916024535894394, "orthogonal_weight": -0.6429483890533447}, {"head": 4, "memory_to_active": 1.7526499032974243, "contribution_to_active": 0.18263059854507446, "mix_mean": 0.036331675946712494, "mix_low": 0.3395538330078125, "mix_high": 0.0, "read_cosine": 0.34378111362457275, "beta_mean": 0.05758947134017944, "nonfinite": 0.0, "weighted_memory_to_active": 0.17897860705852509, "mix_std": 0.05849049612879753, "beta_std": 0.06747487932443619, "beta_low": 0.12628173828125, "beta_high": 0.0, "orthogonal_to_active": 0.0037940028123557568, "evidence_weights": [0.05170389264822006, -0.041967470198869705, -0.056250739842653275], "agreement_weight": 0.08186382800340652, "orthogonal_weight": -0.022782161831855774}, {"head": 5, "memory_to_active": 0.3319033980369568, "contribution_to_active": 0.6194048523902893, "mix_mean": 0.24086415767669678, "mix_low": 0.12443256378173828, "mix_high": 7.724761962890625e-05, "read_cosine": 0.5000818371772766, "beta_mean": 0.07953847199678421, "nonfinite": 0.0, "weighted_memory_to_active": 0.1794959455728531, "mix_std": 0.27267053723335266, "beta_std": 0.10079286247491837, "beta_low": 0.12451171875, "beta_high": 0.0, "orthogonal_to_active": 0.26786965131759644, "evidence_weights": [-0.07672552019357681, -0.21986117959022522, -0.2351125329732895], "agreement_weight": 0.0006504242774099112, "orthogonal_weight": 1.7854896783828735}]}, {"layer": 9, "memory_to_active": 1.0385063886642456, "contribution_to_active": 0.6481236219406128, "mix_mean": 0.3297852873802185, "mix_low": 0.1387070119380951, "mix_high": 0.004039923660457134, "read_cosine": 0.3572043776512146, "beta_mean": 0.06213073432445526, "nonfinite": 0.0, "weighted_memory_to_active": 0.3649253249168396, "mix_std": 0.21170979738235474, "beta_std": 0.04356089234352112, "beta_low": 0.5014750957489014, "beta_high": 0.0, "orthogonal_to_active": 0.038681067526340485, "output_influence": 0.5065990686416626, "residual_influence": 0.14155393838882446, "state_active": 0.04216386005282402, "state_memory": 0.018438613042235374, "state_peak": 43.76577377319336, "growth_active": 1.1551847457885742, "growth_memory": 1.3684719800949097, "heads": [{"head": 0, "memory_to_active": 0.0031563611701130867, "contribution_to_active": 0.5575555562973022, "mix_mean": 0.6229212880134583, "mix_low": 0.0014886856079101562, "mix_high": 0.018218994140625, "read_cosine": 0.2663176953792572, "beta_mean": 6.118945748312399e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002025463618338108, "mix_std": 0.2910948395729065, "beta_std": 2.3968177629285492e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018471097573637962, "evidence_weights": [-0.29820016026496887, -0.30749112367630005, 0.8912730813026428], "agreement_weight": 0.11447180062532425, "orthogonal_weight": -0.9651768207550049}, {"head": 1, "memory_to_active": 0.019939959049224854, "contribution_to_active": 0.6612296104431152, "mix_mean": 0.5254998207092285, "mix_low": 0.008462905883789062, "mix_high": 0.0013170242309570312, "read_cosine": 0.409384548664093, "beta_mean": 5.518995021702722e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013917380012571812, "mix_std": 0.2838421165943146, "beta_std": 0.00011932708002859727, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009795531630516052, "evidence_weights": [-0.23068314790725708, -0.2783443033695221, 1.1417354345321655], "agreement_weight": 0.39832884073257446, "orthogonal_weight": -0.8621142506599426}, {"head": 2, "memory_to_active": 0.2049247920513153, "contribution_to_active": 0.6328607797622681, "mix_mean": 0.5728256702423096, "mix_low": 0.0020771026611328125, "mix_high": 0.004057884216308594, "read_cosine": 0.3437178134918213, "beta_mean": 0.0007704764138907194, "nonfinite": 0.0, "weighted_memory_to_active": 0.1368672251701355, "mix_std": 0.272579550743103, "beta_std": 0.0047331941314041615, "beta_low": 0.994140625, "beta_high": 0.0, "orthogonal_to_active": 0.07585266977548599, "evidence_weights": [-0.19930292665958405, -0.2698751389980316, 0.7048301696777344], "agreement_weight": 0.28234556317329407, "orthogonal_weight": -0.6450304985046387}, {"head": 3, "memory_to_active": 1.3991284370422363, "contribution_to_active": 0.17672696709632874, "mix_mean": 0.044669851660728455, "mix_low": 0.3710746765136719, "mix_high": 3.814697265625e-05, "read_cosine": 0.3944953978061676, "beta_mean": 0.07701171934604645, "nonfinite": 0.0, "weighted_memory_to_active": 0.13750624656677246, "mix_std": 0.09499004483222961, "beta_std": 0.05234921723604202, "beta_low": 0.002685546875, "beta_high": 0.0, "orthogonal_to_active": 0.011322524398565292, "evidence_weights": [0.080089271068573, 0.017320401966571808, 0.01868957094848156], "agreement_weight": 0.08382842689752579, "orthogonal_weight": 0.08454246073961258}, {"head": 4, "memory_to_active": 3.0085904598236084, "contribution_to_active": 1.3376482725143433, "mix_mean": 0.13093553483486176, "mix_low": 0.1502847671508789, "mix_high": 0.000530242919921875, "read_cosine": 0.18622872233390808, "beta_mean": 0.1720617711544037, "nonfinite": 0.0, "weighted_memory_to_active": 1.2970430850982666, "mix_std": 0.18982288241386414, "beta_std": 0.10046940296888351, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.029369311407208443, "evidence_weights": [-0.068549744784832, -0.1517505645751953, -0.015251794829964638], "agreement_weight": -0.05747013911604881, "orthogonal_weight": -0.02334417961537838}, {"head": 5, "memory_to_active": 1.595298171043396, "contribution_to_active": 0.5227207541465759, "mix_mean": 0.08185964822769165, "mix_low": 0.29885387420654297, "mix_high": 7.724761962890625e-05, "read_cosine": 0.5430819392204285, "beta_mean": 0.12287913262844086, "nonfinite": 0.0, "weighted_memory_to_active": 0.602192223072052, "mix_std": 0.13792932033538818, "beta_std": 0.10367023944854736, "beta_low": 0.01202392578125, "beta_high": 0.0, "orthogonal_to_active": 0.10389924794435501, "evidence_weights": [0.10958380997180939, -0.08332761377096176, 0.15667539834976196], "agreement_weight": 0.01629154570400715, "orthogonal_weight": 0.27341675758361816}]}, {"layer": 10, "memory_to_active": 0.23870399594306946, "contribution_to_active": 0.6811662912368774, "mix_mean": 0.5019229054450989, "mix_low": 0.017874719575047493, "mix_high": 0.007565022446215153, "read_cosine": 0.37882277369499207, "beta_mean": 0.046383462846279144, "nonfinite": 0.0, "weighted_memory_to_active": 0.12098611891269684, "mix_std": 0.26680511236190796, "beta_std": 0.04796389117836952, "beta_low": 0.6584879755973816, "beta_high": 0.0, "orthogonal_to_active": 0.20069217681884766, "output_influence": 0.42783409357070923, "residual_influence": 0.14550986886024475, "state_active": 0.05485743284225464, "state_memory": 0.01559645589441061, "state_peak": 121.4257583618164, "growth_active": 1.0608583688735962, "growth_memory": 0.9434334635734558, "heads": [{"head": 0, "memory_to_active": 0.9527243375778198, "contribution_to_active": 1.8833012580871582, "mix_mean": 0.6060880422592163, "mix_low": 0.05711841583251953, "mix_high": 0.02819061279296875, "read_cosine": 0.0951811745762825, "beta_mean": 0.2060631811618805, "nonfinite": 0.0, "weighted_memory_to_active": 0.6119368076324463, "mix_std": 0.39220577478408813, "beta_std": 0.21879149973392487, "beta_low": 0.1199951171875, "beta_high": 0.0, "orthogonal_to_active": 1.1351646184921265, "evidence_weights": [-0.12300155311822891, -0.39778265357017517, -1.353579044342041], "agreement_weight": 0.012950113043189049, "orthogonal_weight": 1.8757174015045166}, {"head": 1, "memory_to_active": 0.00901821255683899, "contribution_to_active": 0.5961565971374512, "mix_mean": 0.6124822497367859, "mix_low": 0.0007648468017578125, "mix_high": 0.012101173400878906, "read_cosine": 0.2983483076095581, "beta_mean": 6.158708856673911e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005249744281172752, "mix_std": 0.2685896158218384, "beta_std": 7.741797890048474e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007440641988068819, "evidence_weights": [-0.2633018493652344, -0.20190419256687164, 0.5824868083000183], "agreement_weight": 0.4869869649410248, "orthogonal_weight": -1.5657936334609985}, {"head": 2, "memory_to_active": 0.009876251220703125, "contribution_to_active": 0.584335207939148, "mix_mean": 0.6163924932479858, "mix_low": 0.0003414154052734375, "mix_high": 0.004506111145019531, "read_cosine": 0.025977889075875282, "beta_mean": 2.500524715287611e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004178708419203758, "mix_std": 0.25432172417640686, "beta_std": 8.641387103125453e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002089323243126273, "evidence_weights": [-0.24235056340694427, -0.2754174470901489, 0.5894791483879089], "agreement_weight": -0.16442053020000458, "orthogonal_weight": -0.4817991256713867}, {"head": 3, "memory_to_active": 0.31211143732070923, "contribution_to_active": 0.0853453129529953, "mix_mean": 0.23579607903957367, "mix_low": 0.046082496643066406, "mix_high": 2.47955322265625e-05, "read_cosine": 0.9135169386863708, "beta_mean": 0.06566575169563293, "nonfinite": 0.0, "weighted_memory_to_active": 0.03286036476492882, "mix_std": 0.21285812556743622, "beta_std": 0.06095211207866669, "beta_low": 0.002197265625, "beta_high": 0.0, "orthogonal_to_active": 0.03427839279174805, "evidence_weights": [-0.10744085907936096, -0.1441267877817154, 0.14063993096351624], "agreement_weight": 0.007115539629012346, "orthogonal_weight": 1.450673222541809}, {"head": 4, "memory_to_active": 0.009416273795068264, "contribution_to_active": 0.5295277237892151, "mix_mean": 0.556216835975647, "mix_low": 0.0004558563232421875, "mix_high": 0.0005626678466796875, "read_cosine": 0.07858964055776596, "beta_mean": 4.48678110842593e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005220208317041397, "mix_std": 0.24821919202804565, "beta_std": 0.00017880434461403638, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.005091092549264431, "evidence_weights": [-0.18239612877368927, -0.261375367641449, 0.8419786095619202], "agreement_weight": 0.119659923017025, "orthogonal_weight": -1.023777961730957}, {"head": 5, "memory_to_active": 0.1390773504972458, "contribution_to_active": 0.40833163261413574, "mix_mean": 0.3845616579055786, "mix_low": 0.0024852752685546875, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8613226413726807, "beta_mean": 0.006440385244786739, "nonfinite": 0.0, "weighted_memory_to_active": 0.0664709061384201, "mix_std": 0.2246362864971161, "beta_std": 0.007697084452956915, "beta_low": 0.828857421875, "beta_high": 0.0, "orthogonal_to_active": 0.020088989287614822, "evidence_weights": [-0.14561782777309418, -0.16029468178749084, 0.2834392488002777], "agreement_weight": 0.09062042087316513, "orthogonal_weight": -0.6397913098335266}]}, {"layer": 11, "memory_to_active": 0.10426246374845505, "contribution_to_active": 0.5947526693344116, "mix_mean": 0.48838555812835693, "mix_low": 0.09346755594015121, "mix_high": 0.05175670236349106, "read_cosine": 0.3834744691848755, "beta_mean": 0.02966209501028061, "nonfinite": 0.0, "weighted_memory_to_active": 0.04994077607989311, "mix_std": 0.3226468563079834, "beta_std": 0.04320750758051872, "beta_low": 0.6147563457489014, "beta_high": 0.0, "orthogonal_to_active": 0.04610765352845192, "output_influence": 0.5354047417640686, "residual_influence": 0.13717453181743622, "state_active": 0.20685625076293945, "state_memory": 0.005915752146393061, "state_peak": 28.12908935546875, "growth_active": 1.1453591585159302, "growth_memory": 1.6168673038482666, "heads": [{"head": 0, "memory_to_active": 0.007490966003388166, "contribution_to_active": 0.5886692404747009, "mix_mean": 0.6221439838409424, "mix_low": 0.001186370849609375, "mix_high": 0.007175445556640625, "read_cosine": 0.31368520855903625, "beta_mean": 9.493825928075239e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005021628923714161, "mix_std": 0.260860413312912, "beta_std": 0.003855298738926649, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.003723887959495187, "evidence_weights": [-0.16663621366024017, -0.3673267364501953, 0.7218778729438782], "agreement_weight": 0.29253673553466797, "orthogonal_weight": -0.7956080436706543}, {"head": 1, "memory_to_active": 0.2600354254245758, "contribution_to_active": 0.34349703788757324, "mix_mean": 0.38344913721084595, "mix_low": 0.007762908935546875, "mix_high": 0.0002002716064453125, "read_cosine": 0.554552435874939, "beta_mean": 0.02391532063484192, "nonfinite": 0.0, "weighted_memory_to_active": 0.10912621766328812, "mix_std": 0.25762176513671875, "beta_std": 0.048615358769893646, "beta_low": 0.5257568359375, "beta_high": 0.0, "orthogonal_to_active": 0.042656127363443375, "evidence_weights": [-0.16704685986042023, -0.19392025470733643, 0.5078451037406921], "agreement_weight": 0.07668488472700119, "orthogonal_weight": -0.48889046907424927}, {"head": 2, "memory_to_active": 0.010241932235658169, "contribution_to_active": 0.9790292978286743, "mix_mean": 0.44392460584640503, "mix_low": 0.24553489685058594, "mix_high": 0.17671489715576172, "read_cosine": 0.20735052227973938, "beta_mean": 0.07295647263526917, "nonfinite": 0.0, "weighted_memory_to_active": 0.0072022597305476665, "mix_std": 0.4432919919490814, "beta_std": 0.09401547163724899, "beta_low": 0.43316650390625, "beta_high": 0.0, "orthogonal_to_active": 0.048546407371759415, "evidence_weights": [0.045959558337926865, -0.4264640510082245, -1.262148380279541], "agreement_weight": 0.0018609045073390007, "orthogonal_weight": 6.991311550140381}, {"head": 3, "memory_to_active": 0.010519005358219147, "contribution_to_active": 0.8532804250717163, "mix_mean": 0.41940075159072876, "mix_low": 0.29515933990478516, "mix_high": 0.12135124206542969, "read_cosine": 0.5404274463653564, "beta_mean": 0.0576428659260273, "nonfinite": 0.0, "weighted_memory_to_active": 0.006621204316616058, "mix_std": 0.4411065876483917, "beta_std": 0.08337883651256561, "beta_low": 0.46820068359375, "beta_high": 0.0, "orthogonal_to_active": 0.034639161080121994, "evidence_weights": [0.15080617368221283, -0.32977524399757385, -0.23646654188632965], "agreement_weight": 0.04827554151415825, "orthogonal_weight": 7.332150459289551}, {"head": 4, "memory_to_active": 0.24732063710689545, "contribution_to_active": 0.4263322353363037, "mix_mean": 0.5292847752571106, "mix_low": 0.008462905883789062, "mix_high": 0.000682830810546875, "read_cosine": 0.09126482903957367, "beta_mean": 0.014576395973563194, "nonfinite": 0.0, "weighted_memory_to_active": 0.1263808012008667, "mix_std": 0.27138474583625793, "beta_std": 0.016249237582087517, "beta_low": 0.52032470703125, "beta_high": 0.0, "orthogonal_to_active": 0.12179012596607208, "evidence_weights": [-0.2378365844488144, -0.2196422517299652, 1.9144268035888672], "agreement_weight": -0.19749188423156738, "orthogonal_weight": -0.985575795173645}, {"head": 5, "memory_to_active": 0.089966781437397, "contribution_to_active": 0.3777073919773102, "mix_mean": 0.5321099162101746, "mix_low": 0.0026988983154296875, "mix_high": 0.0044155120849609375, "read_cosine": 0.5935662984848022, "beta_mean": 0.008786585181951523, "nonfinite": 0.0, "weighted_memory_to_active": 0.04529254138469696, "mix_std": 0.261615514755249, "beta_std": 0.013130813837051392, "beta_low": 0.741455078125, "beta_high": 0.0, "orthogonal_to_active": 0.02529020421206951, "evidence_weights": [-0.21663647890090942, -0.30087175965309143, 0.6092953085899353], "agreement_weight": 0.08014834672212601, "orthogonal_weight": -0.7258409261703491}]}, {"layer": 12, "memory_to_active": 0.26407259702682495, "contribution_to_active": 0.5686043500900269, "mix_mean": 0.513396143913269, "mix_low": 0.010669867508113384, "mix_high": 0.004427274223417044, "read_cosine": 0.24170881509780884, "beta_mean": 0.014222264289855957, "nonfinite": 0.0, "weighted_memory_to_active": 0.14079472422599792, "mix_std": 0.2596171498298645, "beta_std": 0.015269327908754349, "beta_low": 0.8267212510108948, "beta_high": 0.0, "orthogonal_to_active": 0.06584836542606354, "output_influence": 0.46676796674728394, "residual_influence": 0.12767142057418823, "state_active": 0.03412619233131409, "state_memory": 0.00893245916813612, "state_peak": 47.34836959838867, "growth_active": 1.0985864400863647, "growth_memory": 1.1970899105072021, "heads": [{"head": 0, "memory_to_active": 0.047844111919403076, "contribution_to_active": 0.5538748502731323, "mix_mean": 0.5368252992630005, "mix_low": 0.0008859634399414062, "mix_high": 0.00373077392578125, "read_cosine": 0.28627467155456543, "beta_mean": 0.000758569804020226, "nonfinite": 0.0, "weighted_memory_to_active": 0.024189051240682602, "mix_std": 0.26934880018234253, "beta_std": 0.002977315802127123, "beta_low": 0.99700927734375, "beta_high": 0.0, "orthogonal_to_active": 0.020502222701907158, "evidence_weights": [-0.2015390843153, -0.27201247215270996, 0.20379523932933807], "agreement_weight": 0.09856268763542175, "orthogonal_weight": -0.9029361009597778}, {"head": 1, "memory_to_active": 0.08749891817569733, "contribution_to_active": 0.527069628238678, "mix_mean": 0.5439466238021851, "mix_low": 0.0005903244018554688, "mix_high": 0.001373291015625, "read_cosine": 0.5149840712547302, "beta_mean": 0.0008970431517809629, "nonfinite": 0.0, "weighted_memory_to_active": 0.047896608710289, "mix_std": 0.2639213800430298, "beta_std": 0.0061622802168130875, "beta_low": 0.990966796875, "beta_high": 0.0, "orthogonal_to_active": 0.037525542080402374, "evidence_weights": [-0.20758967101573944, -0.27279749512672424, 0.9805530905723572], "agreement_weight": 0.15953604876995087, "orthogonal_weight": -0.9231091737747192}, {"head": 2, "memory_to_active": 0.6028067469596863, "contribution_to_active": 0.4578479528427124, "mix_mean": 0.29190289974212646, "mix_low": 0.03482818603515625, "mix_high": 3.0517578125e-05, "read_cosine": 0.4599849581718445, "beta_mean": 0.08179520815610886, "nonfinite": 0.0, "weighted_memory_to_active": 0.224137082695961, "mix_std": 0.23313570022583008, "beta_std": 0.06153513863682747, "beta_low": 0.00286865234375, "beta_high": 0.0, "orthogonal_to_active": 0.23116301000118256, "evidence_weights": [-0.23654544353485107, -0.047673147171735764, -0.06896103173494339], "agreement_weight": 0.01680567115545273, "orthogonal_weight": 1.2390934228897095}, {"head": 3, "memory_to_active": 0.03018810600042343, "contribution_to_active": 0.7386866211891174, "mix_mean": 0.700342059135437, "mix_low": 0.000255584716796875, "mix_high": 0.013406753540039062, "read_cosine": 0.05939936637878418, "beta_mean": 0.000269134936388582, "nonfinite": 0.0, "weighted_memory_to_active": 0.023295335471630096, "mix_std": 0.23391187191009521, "beta_std": 0.0055889249779284, "beta_low": 0.99822998046875, "beta_high": 0.0, "orthogonal_to_active": 0.02224196493625641, "evidence_weights": [-0.2906775176525116, -0.3415478765964508, 1.4215056896209717], "agreement_weight": 0.4090671241283417, "orthogonal_weight": -0.9741005897521973}, {"head": 4, "memory_to_active": 0.7575522661209106, "contribution_to_active": 0.6479419469833374, "mix_mean": 0.3823557496070862, "mix_low": 0.020689964294433594, "mix_high": 0.00307464599609375, "read_cosine": -0.11644051969051361, "beta_mean": 0.0013134912587702274, "nonfinite": 0.0, "weighted_memory_to_active": 0.4872478246688843, "mix_std": 0.28005263209342957, "beta_std": 0.012182940728962421, "beta_low": 0.97454833984375, "beta_high": 0.0, "orthogonal_to_active": 0.0541800893843174, "evidence_weights": [-0.12763109803199768, -0.2144482284784317, 0.0009542222833260894], "agreement_weight": 0.001188610214740038, "orthogonal_weight": -0.11614804714918137}, {"head": 5, "memory_to_active": 0.058545202016830444, "contribution_to_active": 0.48620525002479553, "mix_mean": 0.6250040531158447, "mix_low": 0.0067691802978515625, "mix_high": 0.004947662353515625, "read_cosine": 0.24605029821395874, "beta_mean": 0.00030013234936632216, "nonfinite": 0.0, "weighted_memory_to_active": 0.0380023792386055, "mix_std": 0.2773324251174927, "beta_std": 0.0031693638302385807, "beta_low": 0.9967041015625, "beta_high": 0.0, "orthogonal_to_active": 0.029477300122380257, "evidence_weights": [-0.2457447648048401, -0.31295615434646606, 1.2876815795898438], "agreement_weight": 0.3438762426376343, "orthogonal_weight": -0.8323951959609985}]}, {"layer": 13, "memory_to_active": 0.13408344984054565, "contribution_to_active": 0.5302925705909729, "mix_mean": 0.5236729979515076, "mix_low": 0.0053952536545693874, "mix_high": 0.008643945679068565, "read_cosine": 0.42807477712631226, "beta_mean": 0.016886651515960693, "nonfinite": 0.0, "weighted_memory_to_active": 0.06064107269048691, "mix_std": 0.2571951746940613, "beta_std": 0.019580403342843056, "beta_low": 0.80010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.05976863205432892, "output_influence": 0.5061159133911133, "residual_influence": 0.17009228467941284, "state_active": 0.11408128589391708, "state_memory": 0.04048916697502136, "state_peak": 100.91389465332031, "growth_active": 0.9273900985717773, "growth_memory": 0.9844602942466736, "heads": [{"head": 0, "memory_to_active": 0.028983427211642265, "contribution_to_active": 0.5620813369750977, "mix_mean": 0.5874395370483398, "mix_low": 0.000530242919921875, "mix_high": 0.0021572113037109375, "read_cosine": 0.4180099666118622, "beta_mean": 0.00011498833919176832, "nonfinite": 0.0, "weighted_memory_to_active": 0.017765700817108154, "mix_std": 0.2687104046344757, "beta_std": 0.00025053019635379314, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.017578689381480217, "evidence_weights": [-0.20171256363391876, -0.2666908800601959, 0.7615651488304138], "agreement_weight": 0.11786674708127975, "orthogonal_weight": -1.1004000902175903}, {"head": 1, "memory_to_active": 0.01609456166625023, "contribution_to_active": 0.5859370231628418, "mix_mean": 0.5537393689155579, "mix_low": 0.00165557861328125, "mix_high": 0.0028409957885742188, "read_cosine": 0.20887815952301025, "beta_mean": 5.789250644738786e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00927259773015976, "mix_std": 0.273393839597702, "beta_std": 0.00011303859355393797, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011446673423051834, "evidence_weights": [-0.18698658049106598, -0.2687748670578003, 0.6816478967666626], "agreement_weight": 0.3493236005306244, "orthogonal_weight": -1.2907274961471558}, {"head": 2, "memory_to_active": 0.09135127812623978, "contribution_to_active": 0.45705050230026245, "mix_mean": 0.3725445866584778, "mix_low": 0.0021123886108398438, "mix_high": 0.00017452239990234375, "read_cosine": 0.6563079953193665, "beta_mean": 0.013528899289667606, "nonfinite": 0.0, "weighted_memory_to_active": 0.04004378616809845, "mix_std": 0.2277441918849945, "beta_std": 0.042580656707286835, "beta_low": 0.81280517578125, "beta_high": 0.0, "orthogonal_to_active": 0.019359972327947617, "evidence_weights": [-0.1942378580570221, -0.20004026591777802, 0.4283815920352936], "agreement_weight": 0.009998507797718048, "orthogonal_weight": -0.8155045509338379}, {"head": 3, "memory_to_active": 0.19102603197097778, "contribution_to_active": 0.6510973572731018, "mix_mean": 0.6651546955108643, "mix_low": 0.01805400848388672, "mix_high": 0.03713703155517578, "read_cosine": 0.10827470570802689, "beta_mean": 0.0013122099917382002, "nonfinite": 0.0, "weighted_memory_to_active": 0.15294872224330902, "mix_std": 0.2861959636211395, "beta_std": 0.010746411979198456, "beta_low": 0.98992919921875, "beta_high": 0.0, "orthogonal_to_active": 0.1316443383693695, "evidence_weights": [-0.32619211077690125, -0.24842393398284912, 1.8142814636230469], "agreement_weight": 4.583496570587158, "orthogonal_weight": -1.0054714679718018}, {"head": 4, "memory_to_active": 0.39675867557525635, "contribution_to_active": 0.24185273051261902, "mix_mean": 0.32311224937438965, "mix_low": 0.007917404174804688, "mix_high": 0.0, "read_cosine": 0.8804414868354797, "beta_mean": 0.08587701618671417, "nonfinite": 0.0, "weighted_memory_to_active": 0.0972122997045517, "mix_std": 0.21489271521568298, "beta_std": 0.05978412553668022, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.13910932838916779, "evidence_weights": [-0.11418670415878296, -0.23318730294704437, 0.4755052328109741], "agreement_weight": 0.005690073128789663, "orthogonal_weight": 2.18795108795166}, {"head": 5, "memory_to_active": 0.08028668165206909, "contribution_to_active": 0.6837363839149475, "mix_mean": 0.6400474309921265, "mix_low": 0.002101898193359375, "mix_high": 0.009553909301757812, "read_cosine": 0.29653632640838623, "beta_mean": 0.0004289011121727526, "nonfinite": 0.0, "weighted_memory_to_active": 0.046603307127952576, "mix_std": 0.27223390340805054, "beta_std": 0.004007658921182156, "beta_low": 0.9976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.03947274386882782, "evidence_weights": [-0.2196410745382309, -0.30376461148262024, 0.8309095501899719], "agreement_weight": 0.13291659951210022, "orthogonal_weight": -0.9214373826980591}]}, {"layer": 14, "memory_to_active": 0.24898283183574677, "contribution_to_active": 0.42041367292404175, "mix_mean": 0.4184552729129791, "mix_low": 0.02865855023264885, "mix_high": 0.005719820968806744, "read_cosine": 0.6212911009788513, "beta_mean": 0.06693153083324432, "nonfinite": 0.0, "weighted_memory_to_active": 0.1051199734210968, "mix_std": 0.24377302825450897, "beta_std": 0.06281633675098419, "beta_low": 0.4639282524585724, "beta_high": 0.0, "orthogonal_to_active": 0.041807956993579865, "output_influence": 0.33366820216178894, "residual_influence": 0.12702128291130066, "state_active": 0.05865687504410744, "state_memory": 0.016070976853370667, "state_peak": 41.27272415161133, "growth_active": 1.0689775943756104, "growth_memory": 1.259303092956543, "heads": [{"head": 0, "memory_to_active": 0.3815794587135315, "contribution_to_active": 0.19210362434387207, "mix_mean": 0.17761535942554474, "mix_low": 0.02613544464111328, "mix_high": 0.0, "read_cosine": 0.8052424788475037, "beta_mean": 0.07261838018894196, "nonfinite": 0.0, "weighted_memory_to_active": 0.08531704545021057, "mix_std": 0.173808291554451, "beta_std": 0.06363819539546967, "beta_low": 0.0111083984375, "beta_high": 0.0, "orthogonal_to_active": 0.07010074704885483, "evidence_weights": [-0.13172438740730286, -0.03220989555120468, 0.47261226177215576], "agreement_weight": 0.024322111159563065, "orthogonal_weight": 1.4501348733901978}, {"head": 1, "memory_to_active": 0.4859313368797302, "contribution_to_active": 0.29497969150543213, "mix_mean": 0.22076056897640228, "mix_low": 0.12960243225097656, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8060919046401978, "beta_mean": 0.26950544118881226, "nonfinite": 0.0, "weighted_memory_to_active": 0.20380336046218872, "mix_std": 0.2509068250656128, "beta_std": 0.255828320980072, "beta_low": 0.08843994140625, "beta_high": 0.0, "orthogonal_to_active": 0.013458498753607273, "evidence_weights": [-0.016337014734745026, -0.21862325072288513, -0.13340073823928833], "agreement_weight": 0.05192195251584053, "orthogonal_weight": 0.14855137467384338}, {"head": 2, "memory_to_active": 0.14290590584278107, "contribution_to_active": 0.5678086280822754, "mix_mean": 0.5772855877876282, "mix_low": 0.0024480819702148438, "mix_high": 0.0033626556396484375, "read_cosine": 0.4197237491607666, "beta_mean": 0.003005381440743804, "nonfinite": 0.0, "weighted_memory_to_active": 0.0908377394080162, "mix_std": 0.26683950424194336, "beta_std": 0.004088046029210091, "beta_low": 0.94659423828125, "beta_high": 0.0, "orthogonal_to_active": 0.06938433647155762, "evidence_weights": [-0.1493765115737915, -0.31272584199905396, 0.8922239542007446], "agreement_weight": 0.08163122832775116, "orthogonal_weight": -0.8761481046676636}, {"head": 3, "memory_to_active": 0.04990041255950928, "contribution_to_active": 0.631962776184082, "mix_mean": 0.6746312379837036, "mix_low": 0.005280494689941406, "mix_high": 0.020285606384277344, "read_cosine": 0.07555586099624634, "beta_mean": 0.007172516081482172, "nonfinite": 0.0, "weighted_memory_to_active": 0.03218654543161392, "mix_std": 0.25294509530067444, "beta_std": 0.007482978980988264, "beta_low": 0.771240234375, "beta_high": 0.0, "orthogonal_to_active": 0.02204246073961258, "evidence_weights": [-0.3620220422744751, -0.31895771622657776, 0.33846351504325867], "agreement_weight": 0.07824061810970306, "orthogonal_weight": -0.7774767279624939}, {"head": 4, "memory_to_active": 0.17305238544940948, "contribution_to_active": 0.4970729351043701, "mix_mean": 0.4981924295425415, "mix_low": 0.004189491271972656, "mix_high": 0.009065628051757812, "read_cosine": 0.8667595386505127, "beta_mean": 0.046261757612228394, "nonfinite": 0.0, "weighted_memory_to_active": 0.10213519632816315, "mix_std": 0.26330697536468506, "beta_std": 0.03216880187392235, "beta_low": 0.031494140625, "beta_high": 0.0, "orthogonal_to_active": 0.028898760676383972, "evidence_weights": [-0.22081434726715088, -0.2508292496204376, 0.42772403359413147], "agreement_weight": 0.12697114050388336, "orthogonal_weight": -0.5991334915161133}, {"head": 5, "memory_to_active": 0.26052749156951904, "contribution_to_active": 0.3385540843009949, "mix_mean": 0.36224645376205444, "mix_low": 0.00429534912109375, "mix_high": 0.0016031265258789062, "read_cosine": 0.754372775554657, "beta_mean": 0.003025741782039404, "nonfinite": 0.0, "weighted_memory_to_active": 0.11643994599580765, "mix_std": 0.2548315227031708, "beta_std": 0.013691647909581661, "beta_low": 0.9346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.0469629280269146, "evidence_weights": [-0.18198725581169128, -0.1682896614074707, 0.2548684775829315], "agreement_weight": 0.08429712802171707, "orthogonal_weight": -0.6888934373855591}]}, {"layer": 15, "memory_to_active": 0.05382709577679634, "contribution_to_active": 0.4833095073699951, "mix_mean": 0.40595510601997375, "mix_low": 0.00893259048461914, "mix_high": 0.002395471092313528, "read_cosine": 0.683612585067749, "beta_mean": 0.039467066526412964, "nonfinite": 0.0, "weighted_memory_to_active": 0.0258927159011364, "mix_std": 0.2837793529033661, "beta_std": 0.045879583805799484, "beta_low": 0.711517333984375, "beta_high": 0.0, "orthogonal_to_active": 0.0061646332032978535, "output_influence": 0.29927241802215576, "residual_influence": 0.16575825214385986, "state_active": 0.13793964684009552, "state_memory": 0.010084825567901134, "state_peak": 45.966190338134766, "growth_active": 1.2453867197036743, "growth_memory": 1.5159780979156494, "heads": [{"head": 0, "memory_to_active": 0.08669700473546982, "contribution_to_active": 0.4763031601905823, "mix_mean": 0.3709977865219116, "mix_low": 0.007519721984863281, "mix_high": 0.004932403564453125, "read_cosine": 0.7020687460899353, "beta_mean": 0.03335030749440193, "nonfinite": 0.0, "weighted_memory_to_active": 0.04163821414113045, "mix_std": 0.27840927243232727, "beta_std": 0.0270365122705698, "beta_low": 0.12884521484375, "beta_high": 0.0, "orthogonal_to_active": 0.019208667799830437, "evidence_weights": [-0.1795920729637146, -0.18314489722251892, 0.3771819472312927], "agreement_weight": 0.09615221619606018, "orthogonal_weight": -0.721526026725769}, {"head": 1, "memory_to_active": 0.12462060898542404, "contribution_to_active": 0.4119972884654999, "mix_mean": 0.41532862186431885, "mix_low": 0.0022430419921875, "mix_high": 0.0045948028564453125, "read_cosine": 0.9126982092857361, "beta_mean": 0.000743656069971621, "nonfinite": 0.0, "weighted_memory_to_active": 0.05184481292963028, "mix_std": 0.2830878794193268, "beta_std": 0.0023153251968324184, "beta_low": 0.99102783203125, "beta_high": 0.0, "orthogonal_to_active": 0.0003993151185568422, "evidence_weights": [-0.15609262883663177, -0.16590411961078644, 0.2144094705581665], "agreement_weight": 0.11632204800844193, "orthogonal_weight": 0.014794163405895233}, {"head": 2, "memory_to_active": 0.0056166513822972775, "contribution_to_active": 0.4635240137577057, "mix_mean": 0.3394542336463928, "mix_low": 0.013974189758300781, "mix_high": 6.771087646484375e-05, "read_cosine": 0.8092613220214844, "beta_mean": 9.271847375202924e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0027080709114670753, "mix_std": 0.2695084810256958, "beta_std": 0.00015552995319012552, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005707292584702373, "evidence_weights": [-0.20525845885276794, -0.27181434631347656, 0.18713633716106415], "agreement_weight": 0.007793205324560404, "orthogonal_weight": 0.5220746994018555}, {"head": 3, "memory_to_active": 0.0020627821795642376, "contribution_to_active": 0.3969506025314331, "mix_mean": 0.37784719467163086, "mix_low": 0.007599830627441406, "mix_high": 0.0002498626708984375, "read_cosine": 0.6100515127182007, "beta_mean": 1.4852932508802041e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0006887878407724202, "mix_std": 0.2870802581310272, "beta_std": 3.5503060644259676e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006515197455883026, "evidence_weights": [-0.23516127467155457, -0.25131556391716003, 0.2551954984664917], "agreement_weight": 0.02885097824037075, "orthogonal_weight": -0.9805715680122375}, {"head": 4, "memory_to_active": 0.07771489024162292, "contribution_to_active": 0.6192150115966797, "mix_mean": 0.3661390244960785, "mix_low": 0.0130615234375, "mix_high": 0.0009603500366210938, "read_cosine": 0.7059636116027832, "beta_mean": 0.20080585777759552, "nonfinite": 0.0, "weighted_memory_to_active": 0.041356198489665985, "mix_std": 0.30795586109161377, "beta_std": 0.2317768782377243, "beta_low": 0.176025390625, "beta_high": 0.0, "orthogonal_to_active": 0.0063740527257323265, "evidence_weights": [-0.21452733874320984, -0.25418907403945923, -0.027363112196326256], "agreement_weight": 0.012300029397010803, "orthogonal_weight": 0.2987126410007477}, {"head": 5, "memory_to_active": 0.026250634342432022, "contribution_to_active": 0.531866729259491, "mix_mean": 0.5659635663032532, "mix_low": 0.009197235107421875, "mix_high": 0.0035676956176757812, "read_cosine": 0.3616315722465515, "beta_mean": 0.0017949913162738085, "nonfinite": 0.0, "weighted_memory_to_active": 0.01712018996477127, "mix_std": 0.2766343951225281, "beta_std": 0.01395774818956852, "beta_low": 0.97320556640625, "beta_high": 0.0, "orthogonal_to_active": 0.009783511981368065, "evidence_weights": [-0.3039229214191437, -0.31569746136665344, 0.7449893951416016], "agreement_weight": 0.2674873173236847, "orthogonal_weight": 0.741259753704071}]}]}, {"step": 42000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8944571960000758, "memory_to_active": 0.8339742422103882, "contribution_to_active": 0.5378016233444214, "mix_mean": 0.3915073871612549, "mix_low": 0.14206908643245697, "mix_high": 0.009078025817871094, "read_cosine": 0.36015772819519043, "beta_mean": 0.0760970190167427, "nonfinite": 0.0, "weighted_memory_to_active": 0.20676535367965698, "mix_std": 0.21996350586414337, "beta_std": 0.06569495797157288, "beta_low": 0.5322609543800354, "beta_high": 1.843770223786123e-05, "orthogonal_to_active": 0.054968513548374176, "samples": 4, "scored_tokens": 899, "normal_loss": 3.111457347869873, "normal_token_accuracy": 0.4449388086795807, "state_active": 0.06293541193008423, "state_memory": 0.018561722710728645, "state_peak": 121.8446044921875, "growth_active": 1.1310911178588867, "growth_memory": 1.360012173652649, "output_influence": 0.3964840769767761, "residual_influence": 0.13064788281917572, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2629809379577637, "no_memory_loss_delta": 0.5025583505630493, "scrambled_loss_delta": 0.138154998421669, "memory_only_loss_delta": 3.3600780963897705, "no_evidence_loss_delta": 0.2711646556854248, "no_agreement_loss_delta": 0.034279964864254, "no_orthogonal_loss_delta": 0.009169496595859528}, {"index": 94, "tokens": 256, "normal_loss": 3.196925640106201, "no_memory_loss_delta": 0.6132941842079163, "scrambled_loss_delta": 0.34198763966560364, "memory_only_loss_delta": 3.144935131072998, "no_evidence_loss_delta": 0.2896984815597534, "no_agreement_loss_delta": 0.03736630082130432, "no_orthogonal_loss_delta": 0.030901797115802765}, {"index": 188, "tokens": 256, "normal_loss": 3.449307918548584, "no_memory_loss_delta": 0.6209864616394043, "scrambled_loss_delta": 0.16203519701957703, "memory_only_loss_delta": 3.0654356479644775, "no_evidence_loss_delta": 0.3188198208808899, "no_agreement_loss_delta": 0.05946040526032448, "no_orthogonal_loss_delta": 0.041142698377370834}, {"index": 283, "tokens": 131, "normal_loss": 1.9881037473678589, "no_memory_loss_delta": 0.49432677030563354, "scrambled_loss_delta": 0.215146005153656, "memory_only_loss_delta": 2.864389181137085, "no_evidence_loss_delta": 0.13837577402591705, "no_agreement_loss_delta": 0.028768448159098625, "no_orthogonal_loss_delta": 0.003229317720979452}], "agreement_weight_abs_mean": 0.1824762374162674, "agreement_weight_abs_max": 4.6641035079956055, "agreement_weight_mean": 0.16026397049427032, "orthogonal_weight_abs_mean": 0.7724345326423645, "orthogonal_weight_abs_max": 7.48113489151001, "orthogonal_weight_mean": -0.1350981742143631, "no_memory_loss": 3.6780734062194824, "no_memory_loss_delta": 0.5666157603263855, "no_memory_token_accuracy_gain": 0.09010010957717896, "no_memory_token_win_fraction": 0.6518353819847107, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 43.25574455144661, "scrambled_loss": 3.3256752490997314, "scrambled_loss_delta": 0.2142176330089569, "scrambled_token_accuracy_gain": 0.023359287530183792, "scrambled_token_win_fraction": 0.628476083278656, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.28273015173389, "memory_only_loss": 6.254137992858887, "memory_only_loss_delta": 3.1426804065704346, "memory_only_token_accuracy_gain": 0.2825361490249634, "memory_only_token_win_fraction": 0.8987764120101929, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.6833062248396, "no_evidence_loss": 3.3821206092834473, "no_evidence_loss_delta": 0.2706630229949951, "no_evidence_token_accuracy_gain": 0.04338153451681137, "no_evidence_token_win_fraction": 0.6129032373428345, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 23.712647606825783, "no_agreement_loss": 3.1529839038848877, "no_agreement_loss_delta": 0.04152611643075943, "no_agreement_token_accuracy_gain": 0.0011123470030725002, "no_agreement_token_win_fraction": 0.510567307472229, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 4.067571911185036, "no_orthogonal_loss": 3.135054588317871, "no_orthogonal_loss_delta": 0.02359713427722454, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5695216655731201, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 2.332089895738223, "short_no_memory_loss_delta": 0.4793422222137451, "long_context_memory_gain": 0.08727353811264038, "normal_context_loss_gain": 0.13495224714279175, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.413827896118164, "contribution_to_active": 1.1124904155731201, "mix_mean": 0.10429300367832184, "mix_low": 0.5447689890861511, "mix_high": 0.004970550537109375, "read_cosine": 0.43226146697998047, "beta_mean": 0.23520877957344055, "nonfinite": 0.0, "weighted_memory_to_active": 1.1569548845291138, "mix_std": 0.08749772608280182, "beta_std": 0.14918318390846252, "beta_low": 0.0320027694106102, "beta_high": 0.0, "orthogonal_to_active": 0.05259503424167633, "output_influence": 0.1935214400291443, "residual_influence": 0.23296087980270386, "state_active": 0.06809787452220917, "state_memory": 0.046827532351017, "state_peak": 48.211299896240234, "growth_active": 1.2418811321258545, "growth_memory": 1.2637841701507568, "heads": [{"head": 0, "memory_to_active": 2.592806100845337, "contribution_to_active": 0.4625534415245056, "mix_mean": 0.018904302269220352, "mix_low": 0.616551399230957, "mix_high": 0.0, "read_cosine": 0.24426332116127014, "beta_mean": 0.20032736659049988, "nonfinite": 0.0, "weighted_memory_to_active": 0.5581839084625244, "mix_std": 0.05777396634221077, "beta_std": 0.10360172390937805, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05094556137919426, "evidence_weights": [-0.01457060407847166, -0.06035847216844559, -0.10969686508178711], "agreement_weight": -0.15379025042057037, "orthogonal_weight": -0.09882055222988129}, {"head": 1, "memory_to_active": 6.17796516418457, "contribution_to_active": 2.604729413986206, "mix_mean": 0.015838701277971268, "mix_low": 0.6838665008544922, "mix_high": 0.0, "read_cosine": 0.4810670018196106, "beta_mean": 0.45886215567588806, "nonfinite": 0.0, "weighted_memory_to_active": 2.932129383087158, "mix_std": 0.05311332643032074, "beta_std": 0.21358567476272583, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06616981327533722, "evidence_weights": [0.02606324665248394, -0.10349797457456589, 0.46515682339668274], "agreement_weight": 0.030464433133602142, "orthogonal_weight": -0.04503783956170082}, {"head": 2, "memory_to_active": 2.4411044120788574, "contribution_to_active": 0.4419098496437073, "mix_mean": 0.014136770740151405, "mix_low": 0.6534957885742188, "mix_high": 0.0, "read_cosine": 0.3055621385574341, "beta_mean": 0.2888856530189514, "nonfinite": 0.0, "weighted_memory_to_active": 0.5257381200790405, "mix_std": 0.0308232381939888, "beta_std": 0.20257392525672913, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.048982731997966766, "evidence_weights": [0.053044866770505905, 0.06013637036085129, 0.054442424327135086], "agreement_weight": 0.007908179424703121, "orthogonal_weight": -0.20311325788497925}, {"head": 3, "memory_to_active": 6.937570095062256, "contribution_to_active": 2.6616592407226562, "mix_mean": 0.03470391780138016, "mix_low": 0.4948406219482422, "mix_high": 0.0, "read_cosine": 0.29785874485969543, "beta_mean": 0.3229012191295624, "nonfinite": 0.0, "weighted_memory_to_active": 2.8218696117401123, "mix_std": 0.07891151309013367, "beta_std": 0.25078845024108887, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.1257888376712799, "evidence_weights": [-0.04266408458352089, -0.0518680065870285, -0.15503783524036407], "agreement_weight": -0.01506276335567236, "orthogonal_weight": -0.051994435489177704}, {"head": 4, "memory_to_active": 2.2348461151123047, "contribution_to_active": 0.06880425661802292, "mix_mean": 0.007357852533459663, "mix_low": 0.8010416030883789, "mix_high": 0.0, "read_cosine": 0.33407652378082275, "beta_mean": 0.11403908580541611, "nonfinite": 0.0, "weighted_memory_to_active": 0.053932975977659225, "mix_std": 0.014587219804525375, "beta_std": 0.1059970036149025, "beta_low": 0.004150390625, "beta_high": 0.0, "orthogonal_to_active": 0.00629381462931633, "evidence_weights": [0.1332724392414093, 0.08694934099912643, -0.31706857681274414], "agreement_weight": -0.09796932339668274, "orthogonal_weight": 0.11249040812253952}, {"head": 5, "memory_to_active": 0.09867434948682785, "contribution_to_active": 0.43528664112091064, "mix_mean": 0.5348163843154907, "mix_low": 0.018817901611328125, "mix_high": 0.02982330322265625, "read_cosine": 0.9307410717010498, "beta_mean": 0.02623709663748741, "nonfinite": 0.0, "weighted_memory_to_active": 0.04987525939941406, "mix_std": 0.28977710008621216, "beta_std": 0.018552282825112343, "beta_low": 0.18780517578125, "beta_high": 0.0, "orthogonal_to_active": 0.017389439046382904, "evidence_weights": [-0.2036823034286499, -0.2293097972869873, 0.35186946392059326], "agreement_weight": 0.28071075677871704, "orthogonal_weight": -0.701438844203949}]}, {"layer": 1, "memory_to_active": 2.263418197631836, "contribution_to_active": 0.503156840801239, "mix_mean": 0.19716790318489075, "mix_low": 0.36272287368774414, "mix_high": 0.0011048316955566406, "read_cosine": 0.3205299973487854, "beta_mean": 0.17179524898529053, "nonfinite": 0.0, "weighted_memory_to_active": 0.34929174184799194, "mix_std": 0.11879457533359528, "beta_std": 0.12645961344242096, "beta_low": 0.31414794921875, "beta_high": 5.086263263365254e-05, "orthogonal_to_active": 0.052953727543354034, "output_influence": 0.207651287317276, "residual_influence": 0.05247853696346283, "state_active": 0.03528899699449539, "state_memory": 0.035932548344135284, "state_peak": 38.3681640625, "growth_active": 1.3043270111083984, "growth_memory": 1.5175961256027222, "heads": [{"head": 0, "memory_to_active": 0.041813336312770844, "contribution_to_active": 0.46954530477523804, "mix_mean": 0.4659770727157593, "mix_low": 0.009038925170898438, "mix_high": 0.0016546249389648438, "read_cosine": 0.06698699295520782, "beta_mean": 0.0003046611091122031, "nonfinite": 0.0, "weighted_memory_to_active": 0.02174258977174759, "mix_std": 0.28619834780693054, "beta_std": 0.00046655096230097115, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.02080768719315529, "evidence_weights": [-0.19428442418575287, -0.2538222670555115, 0.30625200271606445], "agreement_weight": 0.18270243704319, "orthogonal_weight": -1.0193095207214355}, {"head": 1, "memory_to_active": 2.8708791732788086, "contribution_to_active": 0.2184569537639618, "mix_mean": 0.02391374297440052, "mix_low": 0.44900989532470703, "mix_high": 3.814697265625e-06, "read_cosine": 0.5026590824127197, "beta_mean": 0.27448421716690063, "nonfinite": 0.0, "weighted_memory_to_active": 0.2443830668926239, "mix_std": 0.05014372617006302, "beta_std": 0.18627850711345673, "beta_low": 0.001953125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.015451292507350445, "evidence_weights": [0.015335429459810257, -0.043965425342321396, 0.02039100043475628], "agreement_weight": -0.005796034820377827, "orthogonal_weight": 0.14427632093429565}, {"head": 2, "memory_to_active": 0.1594630777835846, "contribution_to_active": 0.5718199014663696, "mix_mean": 0.6278085112571716, "mix_low": 0.0008602142333984375, "mix_high": 0.0049686431884765625, "read_cosine": 0.5872249603271484, "beta_mean": 0.00724133662879467, "nonfinite": 0.0, "weighted_memory_to_active": 0.1267169713973999, "mix_std": 0.24059432744979858, "beta_std": 0.01013796217739582, "beta_low": 0.78973388671875, "beta_high": 0.0, "orthogonal_to_active": 0.05662382021546364, "evidence_weights": [-0.2970261573791504, -0.3446546196937561, 0.5680186152458191], "agreement_weight": 0.026137173175811768, "orthogonal_weight": -0.6003041863441467}, {"head": 3, "memory_to_active": 3.0080556869506836, "contribution_to_active": 0.17015784978866577, "mix_mean": 0.02020467258989811, "mix_low": 0.6125164031982422, "mix_high": 0.0, "read_cosine": 0.20242074131965637, "beta_mean": 0.33603185415267944, "nonfinite": 0.0, "weighted_memory_to_active": 0.2505607604980469, "mix_std": 0.05456945672631264, "beta_std": 0.2543531656265259, "beta_low": 0.03875732421875, "beta_high": 0.0, "orthogonal_to_active": 0.07527250796556473, "evidence_weights": [0.04239801689982414, -0.009215468540787697, 0.10732050240039825], "agreement_weight": -0.02708061970770359, "orthogonal_weight": -0.38650357723236084}, {"head": 4, "memory_to_active": 3.7046282291412354, "contribution_to_active": 1.5247838497161865, "mix_mean": 0.03816860914230347, "mix_low": 0.2652320861816406, "mix_high": 1.9073486328125e-06, "read_cosine": 0.31668856739997864, "beta_mean": 0.14475823938846588, "nonfinite": 0.0, "weighted_memory_to_active": 1.4008231163024902, "mix_std": 0.06302862614393234, "beta_std": 0.10265276581048965, "beta_low": 0.01751708984375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.13990484178066254, "evidence_weights": [-0.0440390408039093, -0.004778318572789431, 0.029512977227568626], "agreement_weight": -0.012438695877790451, "orthogonal_weight": 0.13199880719184875}, {"head": 5, "memory_to_active": 3.7956695556640625, "contribution_to_active": 0.06417697668075562, "mix_mean": 0.006934851873666048, "mix_low": 0.8396797180175781, "mix_high": 0.0, "read_cosine": 0.24719969928264618, "beta_mean": 0.26795125007629395, "nonfinite": 0.0, "weighted_memory_to_active": 0.05152392387390137, "mix_std": 0.018232928588986397, "beta_std": 0.2048686444759369, "beta_low": 0.03692626953125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.009662171825766563, "evidence_weights": [0.16416150331497192, 0.0819450244307518, 0.24048787355422974], "agreement_weight": -0.14598478376865387, "orthogonal_weight": 0.1933155357837677}]}, {"layer": 2, "memory_to_active": 1.0475027561187744, "contribution_to_active": 0.3445560932159424, "mix_mean": 0.3771343231201172, "mix_low": 0.25543326139450073, "mix_high": 0.004474481102079153, "read_cosine": 0.3253158926963806, "beta_mean": 0.09711972624063492, "nonfinite": 0.0, "weighted_memory_to_active": 0.06551431119441986, "mix_std": 0.19280491769313812, "beta_std": 0.07567087560892105, "beta_low": 0.5540364980697632, "beta_high": 0.00010172525799134746, "orthogonal_to_active": 0.016807518899440765, "output_influence": 0.35433757305145264, "residual_influence": 0.09050457924604416, "state_active": 0.038498543202877045, "state_memory": 0.01683943346142769, "state_peak": 32.87289810180664, "growth_active": 1.0194000005722046, "growth_memory": 1.5754865407943726, "heads": [{"head": 0, "memory_to_active": 0.010172927752137184, "contribution_to_active": 0.3462385833263397, "mix_mean": 0.5785633325576782, "mix_low": 0.0012912750244140625, "mix_high": 0.007740020751953125, "read_cosine": 0.16377389430999756, "beta_mean": 8.438360237050802e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005931027699261904, "mix_std": 0.2878052294254303, "beta_std": 0.00015227597032207996, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004775278735905886, "evidence_weights": [-0.34163618087768555, -0.2064134180545807, 0.12684056162834167], "agreement_weight": 0.2589142620563507, "orthogonal_weight": -0.8446325063705444}, {"head": 1, "memory_to_active": 2.9709548950195312, "contribution_to_active": 0.03535905107855797, "mix_mean": 0.004567210096865892, "mix_low": 0.9055719375610352, "mix_high": 0.0, "read_cosine": 0.18526019155979156, "beta_mean": 0.2998628616333008, "nonfinite": 0.0, "weighted_memory_to_active": 0.027471061795949936, "mix_std": 0.009875726886093616, "beta_std": 0.18018272519111633, "beta_low": 0.0008544921875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.010528068989515305, "evidence_weights": [0.1310616284608841, 0.11826973408460617, 0.5363615155220032], "agreement_weight": 0.028099045157432556, "orthogonal_weight": 0.41757932305336}, {"head": 2, "memory_to_active": 0.009885513223707676, "contribution_to_active": 0.4323098957538605, "mix_mean": 0.5064343214035034, "mix_low": 0.0015172958374023438, "mix_high": 0.0030841827392578125, "read_cosine": 0.2685467600822449, "beta_mean": 2.0796669559786096e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003862992161884904, "mix_std": 0.2817375063896179, "beta_std": 3.928353180526756e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000708724488504231, "evidence_weights": [-0.24292443692684174, -0.28151094913482666, 0.49083250761032104], "agreement_weight": 0.016629314050078392, "orthogonal_weight": -0.20779883861541748}, {"head": 3, "memory_to_active": 0.1632285714149475, "contribution_to_active": 0.5166800618171692, "mix_mean": 0.5304095149040222, "mix_low": 0.0004520416259765625, "mix_high": 0.00730133056640625, "read_cosine": 0.6837090253829956, "beta_mean": 0.03243742138147354, "nonfinite": 0.0, "weighted_memory_to_active": 0.09575077891349792, "mix_std": 0.26580893993377686, "beta_std": 0.042610105127096176, "beta_low": 0.29559326171875, "beta_high": 0.0, "orthogonal_to_active": 0.05881277844309807, "evidence_weights": [-0.19001947343349457, -0.2963964641094208, 0.8529617786407471], "agreement_weight": 0.05890115350484848, "orthogonal_weight": -0.8844403624534607}, {"head": 4, "memory_to_active": 3.121819019317627, "contribution_to_active": 0.20129752159118652, "mix_mean": 0.016850560903549194, "mix_low": 0.6224374771118164, "mix_high": 0.0, "read_cosine": 0.19558407366275787, "beta_mean": 0.2502824366092682, "nonfinite": 0.0, "weighted_memory_to_active": 0.25467202067375183, "mix_std": 0.04252041131258011, "beta_std": 0.23099470138549805, "beta_low": 0.02777099609375, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.025965319946408272, "evidence_weights": [0.023485643789172173, 0.013525434769690037, 0.030833780765533447], "agreement_weight": 0.024809179827570915, "orthogonal_weight": -0.11367793381214142}, {"head": 5, "memory_to_active": 0.008955459110438824, "contribution_to_active": 0.5354514122009277, "mix_mean": 0.6259811520576477, "mix_low": 0.0013294219970703125, "mix_high": 0.008721351623535156, "read_cosine": 0.4550211429595947, "beta_mean": 3.0449218684225343e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005397963337600231, "mix_std": 0.2690816819667816, "beta_std": 4.613208511727862e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 5.4930977057665586e-05, "evidence_weights": [-0.2764836251735687, -0.29068872332572937, 0.9455639719963074], "agreement_weight": 0.11347252130508423, "orthogonal_weight": 0.015115279704332352}]}, {"layer": 3, "memory_to_active": 0.5333757996559143, "contribution_to_active": 0.4341135323047638, "mix_mean": 0.4266741871833801, "mix_low": 0.06896289438009262, "mix_high": 0.009092330932617188, "read_cosine": 0.22878655791282654, "beta_mean": 0.029113229364156723, "nonfinite": 0.0, "weighted_memory_to_active": 0.14105211198329926, "mix_std": 0.26288023591041565, "beta_std": 0.026093807071447372, "beta_low": 0.7024027705192566, "beta_high": 0.0, "orthogonal_to_active": 0.06019279360771179, "output_influence": 0.3578066825866699, "residual_influence": 0.07838962972164154, "state_active": 0.02450869418680668, "state_memory": 0.007788518909364939, "state_peak": 19.82993507385254, "growth_active": 1.053788661956787, "growth_memory": 1.7738951444625854, "heads": [{"head": 0, "memory_to_active": 0.4673920273780823, "contribution_to_active": 0.5553904175758362, "mix_mean": 0.5250084400177002, "mix_low": 0.0049495697021484375, "mix_high": 0.01739025115966797, "read_cosine": 0.22928598523139954, "beta_mean": 0.011648910120129585, "nonfinite": 0.0, "weighted_memory_to_active": 0.2697773873806, "mix_std": 0.31784912943840027, "beta_std": 0.0188327357172966, "beta_low": 0.6519775390625, "beta_high": 0.0, "orthogonal_to_active": 0.18567171692848206, "evidence_weights": [-0.20072788000106812, -0.23312941193580627, 0.812578558921814], "agreement_weight": 0.1295480579137802, "orthogonal_weight": -0.7282097935676575}, {"head": 1, "memory_to_active": 0.013252045959234238, "contribution_to_active": 0.4080621004104614, "mix_mean": 0.5030810832977295, "mix_low": 0.009882926940917969, "mix_high": 0.008035659790039062, "read_cosine": 0.20144321024417877, "beta_mean": 7.510781870223582e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006993270479142666, "mix_std": 0.3129143714904785, "beta_std": 0.00014957775420043617, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005054799374192953, "evidence_weights": [-0.23849771916866302, -0.19941897690296173, 0.578582227230072], "agreement_weight": 0.38826197385787964, "orthogonal_weight": -0.7880246639251709}, {"head": 2, "memory_to_active": 2.3548190593719482, "contribution_to_active": 0.30710145831108093, "mix_mean": 0.04836948961019516, "mix_low": 0.33030223846435547, "mix_high": 0.0, "read_cosine": 0.21332895755767822, "beta_mean": 0.14912912249565125, "nonfinite": 0.0, "weighted_memory_to_active": 0.34026145935058594, "mix_std": 0.10068380832672119, "beta_std": 0.12154686450958252, "beta_low": 0.0147705078125, "beta_high": 0.0, "orthogonal_to_active": 0.0019390481757000089, "evidence_weights": [-0.08469561487436295, 0.008184883743524551, 0.005828098859637976], "agreement_weight": -0.015435587614774704, "orthogonal_weight": 0.007147067692130804}, {"head": 3, "memory_to_active": 0.026948492974042892, "contribution_to_active": 0.4847714304924011, "mix_mean": 0.5650143623352051, "mix_low": 0.008207321166992188, "mix_high": 0.011493682861328125, "read_cosine": 0.30043724179267883, "beta_mean": 0.00011189388897037134, "nonfinite": 0.0, "weighted_memory_to_active": 0.018143458291888237, "mix_std": 0.3006784915924072, "beta_std": 0.0003117923915851861, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0044720410369336605, "evidence_weights": [-0.22854158282279968, -0.2272803634405136, 2.132504940032959], "agreement_weight": 1.7130874395370483, "orthogonal_weight": -0.322612464427948}, {"head": 4, "memory_to_active": 0.09245134890079498, "contribution_to_active": 0.19902527332305908, "mix_mean": 0.2555142045021057, "mix_low": 0.05598163604736328, "mix_high": 0.000514984130859375, "read_cosine": 0.19745272397994995, "beta_mean": 0.0008315872983075678, "nonfinite": 0.0, "weighted_memory_to_active": 0.03407354652881622, "mix_std": 0.26389095187187195, "beta_std": 0.001241748221218586, "beta_low": 0.9979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.030790839344263077, "evidence_weights": [-0.09317565709352493, -0.13104303181171417, 0.19810286164283752], "agreement_weight": -0.01208517700433731, "orthogonal_weight": -0.9445304870605469}, {"head": 5, "memory_to_active": 0.2453918308019638, "contribution_to_active": 0.6503301858901978, "mix_mean": 0.6630574464797974, "mix_low": 0.0044536590576171875, "mix_high": 0.017119407653808594, "read_cosine": 0.2307712733745575, "beta_mean": 0.012882744893431664, "nonfinite": 0.0, "weighted_memory_to_active": 0.1770634949207306, "mix_std": 0.2812645435333252, "beta_std": 0.014480114914476871, "beta_low": 0.5498046875, "beta_high": 0.0, "orthogonal_to_active": 0.13322831690311432, "evidence_weights": [-0.25218653678894043, -0.2967454195022583, 0.48792606592178345], "agreement_weight": 0.1326572448015213, "orthogonal_weight": -0.8129554986953735}]}, {"layer": 4, "memory_to_active": 1.0845900774002075, "contribution_to_active": 0.44565969705581665, "mix_mean": 0.3534705638885498, "mix_low": 0.27699074149131775, "mix_high": 0.007561843376606703, "read_cosine": 0.3424822986125946, "beta_mean": 0.1163116991519928, "nonfinite": 0.0, "weighted_memory_to_active": 0.1315881907939911, "mix_std": 0.15480098128318787, "beta_std": 0.1057666540145874, "beta_low": 0.3758138120174408, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04491116851568222, "output_influence": 0.4270102381706238, "residual_influence": 0.11762745678424835, "state_active": 0.035528186708688736, "state_memory": 0.018476812168955803, "state_peak": 40.18435287475586, "growth_active": 1.1579654216766357, "growth_memory": 1.5610549449920654, "heads": [{"head": 0, "memory_to_active": 0.21626269817352295, "contribution_to_active": 0.7295318841934204, "mix_mean": 0.6626867651939392, "mix_low": 0.010865211486816406, "mix_high": 0.02059173583984375, "read_cosine": 0.4882979989051819, "beta_mean": 0.2656525671482086, "nonfinite": 0.0, "weighted_memory_to_active": 0.1692846417427063, "mix_std": 0.3399520218372345, "beta_std": 0.27186906337738037, "beta_low": 0.224853515625, "beta_high": 0.0, "orthogonal_to_active": 0.06512760370969772, "evidence_weights": [-0.26840418577194214, -0.3687979280948639, 0.1504463255405426], "agreement_weight": 0.015704834833741188, "orthogonal_weight": 0.47323358058929443}, {"head": 1, "memory_to_active": 0.06819646805524826, "contribution_to_active": 0.6208053231239319, "mix_mean": 0.6612711548805237, "mix_low": 0.00012874603271484375, "mix_high": 0.009279251098632812, "read_cosine": 0.6425994634628296, "beta_mean": 0.0011910238536074758, "nonfinite": 0.0, "weighted_memory_to_active": 0.04794778674840927, "mix_std": 0.2424708902835846, "beta_std": 0.006267846561968327, "beta_low": 0.9798583984375, "beta_high": 0.0, "orthogonal_to_active": 0.027856729924678802, "evidence_weights": [-0.23978960514068604, -0.3439446985721588, 0.7155488729476929], "agreement_weight": 0.2013927847146988, "orthogonal_weight": -0.8177017569541931}, {"head": 2, "memory_to_active": 2.0069103240966797, "contribution_to_active": 0.2367335855960846, "mix_mean": 0.02042975462973118, "mix_low": 0.5856990814208984, "mix_high": 2.193450927734375e-05, "read_cosine": 0.252605140209198, "beta_mean": 0.08397091925144196, "nonfinite": 0.0, "weighted_memory_to_active": 0.224067822098732, "mix_std": 0.04574490338563919, "beta_std": 0.07538209855556488, "beta_low": 0.0164794921875, "beta_high": 0.0, "orthogonal_to_active": 0.024694152176380157, "evidence_weights": [0.06135112792253494, 0.02679632045328617, 0.03420822322368622], "agreement_weight": 0.12480805069208145, "orthogonal_weight": -0.1614820957183838}, {"head": 3, "memory_to_active": 1.836686134338379, "contribution_to_active": 0.121397465467453, "mix_mean": 0.029952552169561386, "mix_low": 0.42386341094970703, "mix_high": 0.0, "read_cosine": 0.24444329738616943, "beta_mean": 0.12469250708818436, "nonfinite": 0.0, "weighted_memory_to_active": 0.161971315741539, "mix_std": 0.05300658196210861, "beta_std": 0.11491362005472183, "beta_low": 0.03369140625, "beta_high": 0.0, "orthogonal_to_active": 0.006812273990362883, "evidence_weights": [-0.019849244505167007, 0.060086820274591446, -0.23601897060871124], "agreement_weight": 0.07665527611970901, "orthogonal_weight": -0.05494202300906181}, {"head": 4, "memory_to_active": 0.01216890849173069, "contribution_to_active": 0.6462324857711792, "mix_mean": 0.7324403524398804, "mix_low": 0.0001049041748046875, "mix_high": 0.015474319458007812, "read_cosine": 0.16746407747268677, "beta_mean": 2.402061909378972e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008821539580821991, "mix_std": 0.21810851991176605, "beta_std": 6.944400956854224e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002664984902366996, "evidence_weights": [-0.3316253125667572, -0.33165448904037476, 0.618240475654602], "agreement_weight": 0.31694287061691284, "orthogonal_weight": -0.3265504539012909}, {"head": 5, "memory_to_active": 2.3673160076141357, "contribution_to_active": 0.31925731897354126, "mix_mean": 0.014042784459888935, "mix_low": 0.6412830352783203, "mix_high": 3.814697265625e-06, "read_cosine": 0.2594839334487915, "beta_mean": 0.22233915328979492, "nonfinite": 0.0, "weighted_memory_to_active": 0.1774359941482544, "mix_std": 0.02952287532389164, "beta_std": 0.16609779000282288, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1423112452030182, "evidence_weights": [0.050334036350250244, -0.031630806624889374, -0.2744514048099518], "agreement_weight": -0.06308984011411667, "orthogonal_weight": 0.8860123157501221}]}, {"layer": 5, "memory_to_active": 0.47255566716194153, "contribution_to_active": 0.5057425498962402, "mix_mean": 0.4685913622379303, "mix_low": 0.062937892973423, "mix_high": 0.013356050476431847, "read_cosine": 0.09703698754310608, "beta_mean": 0.05656209960579872, "nonfinite": 0.0, "weighted_memory_to_active": 0.14806953072547913, "mix_std": 0.253225177526474, "beta_std": 0.0723741352558136, "beta_low": 0.631072998046875, "beta_high": 0.0, "orthogonal_to_active": 0.055439867079257965, "output_influence": 0.5083063840866089, "residual_influence": 0.1353425681591034, "state_active": 0.04221533611416817, "state_memory": 0.011277120560407639, "state_peak": 64.50360870361328, "growth_active": 1.0870321989059448, "growth_memory": 1.2728075981140137, "heads": [{"head": 0, "memory_to_active": 0.575949490070343, "contribution_to_active": 0.4689863622188568, "mix_mean": 0.3100307881832123, "mix_low": 0.0288238525390625, "mix_high": 0.00042247772216796875, "read_cosine": -0.09113401174545288, "beta_mean": 0.03665725886821747, "nonfinite": 0.0, "weighted_memory_to_active": 0.15181684494018555, "mix_std": 0.2591139078140259, "beta_std": 0.060784757137298584, "beta_low": 0.34259033203125, "beta_high": 0.0, "orthogonal_to_active": 0.042951758950948715, "evidence_weights": [-0.1177893579006195, -0.1494498997926712, -0.13349509239196777], "agreement_weight": -0.02095642127096653, "orthogonal_weight": -0.28223374485969543}, {"head": 1, "memory_to_active": 0.01730157807469368, "contribution_to_active": 0.5508655905723572, "mix_mean": 0.6660959124565125, "mix_low": 0.00182342529296875, "mix_high": 0.02573871612548828, "read_cosine": 0.21923184394836426, "beta_mean": 4.4218264520168304e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009911995381116867, "mix_std": 0.2842596471309662, "beta_std": 9.676822810433805e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00942504033446312, "evidence_weights": [-0.24369771778583527, -0.28198277950286865, 0.879906952381134], "agreement_weight": 0.5945334434509277, "orthogonal_weight": -0.9955747723579407}, {"head": 2, "memory_to_active": 0.014249160885810852, "contribution_to_active": 0.5511486530303955, "mix_mean": 0.6327513456344604, "mix_low": 0.0031032562255859375, "mix_high": 0.016819000244140625, "read_cosine": -0.37519532442092896, "beta_mean": 0.0001109565346268937, "nonfinite": 0.0, "weighted_memory_to_active": 0.009469838812947273, "mix_std": 0.2750873863697052, "beta_std": 0.00016437951126135886, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026436829939484596, "evidence_weights": [-0.25293418765068054, -0.34246206283569336, 0.6702994108200073], "agreement_weight": -0.003038229653611779, "orthogonal_weight": -0.3343117833137512}, {"head": 3, "memory_to_active": 1.6714410781860352, "contribution_to_active": 0.22739319503307343, "mix_mean": 0.093885637819767, "mix_low": 0.15519237518310547, "mix_high": 3.814697265625e-06, "read_cosine": 0.13817371428012848, "beta_mean": 0.09973502159118652, "nonfinite": 0.0, "weighted_memory_to_active": 0.34780773520469666, "mix_std": 0.12842024862766266, "beta_std": 0.1433221995830536, "beta_low": 0.18524169921875, "beta_high": 0.0, "orthogonal_to_active": 0.13540337979793549, "evidence_weights": [-0.06030619889497757, -0.09057268500328064, 0.007469269912689924], "agreement_weight": 0.049161240458488464, "orthogonal_weight": -0.47026678919792175}, {"head": 4, "memory_to_active": 0.45118167996406555, "contribution_to_active": 0.5861028432846069, "mix_mean": 0.3278168737888336, "mix_low": 0.18625831604003906, "mix_high": 0.0069141387939453125, "read_cosine": 0.5443481802940369, "beta_mean": 0.20247355103492737, "nonfinite": 0.0, "weighted_memory_to_active": 0.2967703640460968, "mix_std": 0.33965688943862915, "beta_std": 0.2283947914838791, "beta_low": 0.26318359375, "beta_high": 0.0, "orthogonal_to_active": 0.09529971331357956, "evidence_weights": [-0.11317393183708191, -0.2798044979572296, -0.06397503614425659], "agreement_weight": 0.040787726640701294, "orthogonal_weight": 0.4930727183818817}, {"head": 5, "memory_to_active": 0.10521106421947479, "contribution_to_active": 0.6499587893486023, "mix_mean": 0.7809674739837646, "mix_low": 0.0024261474609375, "mix_high": 0.03023815155029297, "read_cosine": 0.14679747819900513, "beta_mean": 0.0003515693824738264, "nonfinite": 0.0, "weighted_memory_to_active": 0.07264039665460587, "mix_std": 0.232812762260437, "beta_std": 0.0014819297939538956, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.04691564291715622, "evidence_weights": [-0.31824687123298645, -0.32648587226867676, 1.929890751838684], "agreement_weight": 0.43130719661712646, "orthogonal_weight": -0.6832578182220459}]}, {"layer": 6, "memory_to_active": 0.3448731303215027, "contribution_to_active": 0.60341876745224, "mix_mean": 0.5694317817687988, "mix_low": 0.05982478708028793, "mix_high": 0.032054901123046875, "read_cosine": 0.290731817483902, "beta_mean": 0.07191456854343414, "nonfinite": 0.0, "weighted_memory_to_active": 0.1619124710559845, "mix_std": 0.23650725185871124, "beta_std": 0.08086370676755905, "beta_low": 0.4975789487361908, "beta_high": 0.0, "orthogonal_to_active": 0.03712589293718338, "output_influence": 0.5145947933197021, "residual_influence": 0.17060473561286926, "state_active": 0.04666886106133461, "state_memory": 0.008633690886199474, "state_peak": 88.93450927734375, "growth_active": 1.1733932495117188, "growth_memory": 1.1441236734390259, "heads": [{"head": 0, "memory_to_active": 0.11019869893789291, "contribution_to_active": 0.6750739812850952, "mix_mean": 0.7730022668838501, "mix_low": 0.00016021728515625, "mix_high": 0.023258209228515625, "read_cosine": 0.11494141072034836, "beta_mean": 0.019609995186328888, "nonfinite": 0.0, "weighted_memory_to_active": 0.08798443526029587, "mix_std": 0.21210691332817078, "beta_std": 0.026451818645000458, "beta_low": 0.45806884765625, "beta_high": 0.0, "orthogonal_to_active": 0.07356786727905273, "evidence_weights": [-0.31814298033714294, -0.33861255645751953, 0.9731822609901428], "agreement_weight": 0.29925012588500977, "orthogonal_weight": -0.897580087184906}, {"head": 1, "memory_to_active": 0.005456635262817144, "contribution_to_active": 0.484228253364563, "mix_mean": 0.5527637004852295, "mix_low": 0.0015363693237304688, "mix_high": 0.01131439208984375, "read_cosine": 0.514835774898529, "beta_mean": 1.0498084520804696e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0024125855416059494, "mix_std": 0.286355197429657, "beta_std": 1.9084831365034916e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013288960326462984, "evidence_weights": [-0.2005237340927124, -0.29544398188591003, 1.1041020154953003], "agreement_weight": 0.04541532322764397, "orthogonal_weight": -0.7496093511581421}, {"head": 2, "memory_to_active": 1.0363717079162598, "contribution_to_active": 0.09055712074041367, "mix_mean": 0.03525959700345993, "mix_low": 0.34490966796875, "mix_high": 0.0, "read_cosine": 0.21872054040431976, "beta_mean": 0.04584804177284241, "nonfinite": 0.0, "weighted_memory_to_active": 0.06729568541049957, "mix_std": 0.05602019280195236, "beta_std": 0.04065156728029251, "beta_low": 0.04833984375, "beta_high": 0.0, "orthogonal_to_active": 0.0012745076091960073, "evidence_weights": [0.07904459536075592, -0.028654057532548904, 0.2694057524204254], "agreement_weight": 0.1453280746936798, "orthogonal_weight": -0.022358577698469162}, {"head": 3, "memory_to_active": 0.15055306255817413, "contribution_to_active": 0.6624287366867065, "mix_mean": 0.6358141899108887, "mix_low": 0.010922431945800781, "mix_high": 0.017442703247070312, "read_cosine": 0.4603440761566162, "beta_mean": 0.22389481961727142, "nonfinite": 0.0, "weighted_memory_to_active": 0.10504136979579926, "mix_std": 0.32818007469177246, "beta_std": 0.25673872232437134, "beta_low": 0.211669921875, "beta_high": 0.0, "orthogonal_to_active": 0.058270685374736786, "evidence_weights": [-0.17571836709976196, -0.3325665593147278, 0.19891974329948425], "agreement_weight": -0.006833590567111969, "orthogonal_weight": 0.6353791356086731}, {"head": 4, "memory_to_active": 0.7357670068740845, "contribution_to_active": 1.091193437576294, "mix_mean": 0.766494631767273, "mix_low": 0.0009126663208007812, "mix_high": 0.13570880889892578, "read_cosine": 0.154072105884552, "beta_mean": 0.14193807542324066, "nonfinite": 0.0, "weighted_memory_to_active": 0.6872297525405884, "mix_std": 0.28294485807418823, "beta_std": 0.1609465330839157, "beta_low": 0.2674560546875, "beta_high": 0.0, "orthogonal_to_active": 0.06726297736167908, "evidence_weights": [-0.24487216770648956, -0.5950607657432556, -0.19242733716964722], "agreement_weight": -0.01654733717441559, "orthogonal_weight": 0.11404496431350708}, {"head": 5, "memory_to_active": 0.03089151903986931, "contribution_to_active": 0.6170308589935303, "mix_mean": 0.6532561779022217, "mix_low": 0.000507354736328125, "mix_high": 0.004605293273925781, "read_cosine": 0.28147685527801514, "beta_mean": 0.000185956057976, "nonfinite": 0.0, "weighted_memory_to_active": 0.021510910242795944, "mix_std": 0.25343620777130127, "beta_std": 0.00037448122748173773, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.021050438284873962, "evidence_weights": [-0.2431892603635788, -0.3119589686393738, 0.4323979318141937], "agreement_weight": 0.15385441482067108, "orthogonal_weight": -1.0501196384429932}]}, {"layer": 7, "memory_to_active": 1.7064905166625977, "contribution_to_active": 0.24241548776626587, "mix_mean": 0.1265030950307846, "mix_low": 0.2702614665031433, "mix_high": 0.0003703435359057039, "read_cosine": 0.23208209872245789, "beta_mean": 0.130372554063797, "nonfinite": 0.0, "weighted_memory_to_active": 0.20423024892807007, "mix_std": 0.13489235937595367, "beta_std": 0.10099511593580246, "beta_low": 0.1976521909236908, "beta_high": 0.00013224282884038985, "orthogonal_to_active": 0.06438647955656052, "output_influence": 0.23989593982696533, "residual_influence": 0.055198971182107925, "state_active": 0.013491283170878887, "state_memory": 0.025695133954286575, "state_peak": 30.266090393066406, "growth_active": 1.1566181182861328, "growth_memory": 1.318852186203003, "heads": [{"head": 0, "memory_to_active": 2.4783406257629395, "contribution_to_active": 0.25824758410453796, "mix_mean": 0.036397382616996765, "mix_low": 0.42255306243896484, "mix_high": 1.9073486328125e-06, "read_cosine": 0.3417477607727051, "beta_mean": 0.1667986959218979, "nonfinite": 0.0, "weighted_memory_to_active": 0.304768443107605, "mix_std": 0.07675925642251968, "beta_std": 0.12566229701042175, "beta_low": 0.0054931640625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.006513654254376888, "evidence_weights": [0.021380050107836723, -0.07915881276130676, 0.08973372727632523], "agreement_weight": -0.03393889591097832, "orthogonal_weight": 0.03159932792186737}, {"head": 1, "memory_to_active": 1.07100248336792, "contribution_to_active": 0.3737534284591675, "mix_mean": 0.1926083266735077, "mix_low": 0.1291360855102539, "mix_high": 0.0018491744995117188, "read_cosine": 0.35752609372138977, "beta_mean": 0.1064685806632042, "nonfinite": 0.0, "weighted_memory_to_active": 0.3259674310684204, "mix_std": 0.222483828663826, "beta_std": 0.068697988986969, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02986343950033188, "evidence_weights": [-0.20243912935256958, -0.10737599432468414, -0.23917336761951447], "agreement_weight": 0.03281767666339874, "orthogonal_weight": 0.10620016604661942}, {"head": 2, "memory_to_active": 0.776802659034729, "contribution_to_active": 0.2991000711917877, "mix_mean": 0.1423262357711792, "mix_low": 0.0707406997680664, "mix_high": 0.0001697540283203125, "read_cosine": 0.08825170993804932, "beta_mean": 0.056077249348163605, "nonfinite": 0.0, "weighted_memory_to_active": 0.21223175525665283, "mix_std": 0.1716488152742386, "beta_std": 0.04813957214355469, "beta_low": 0.05389404296875, "beta_high": 0.0, "orthogonal_to_active": 0.08614926040172577, "evidence_weights": [-0.059156086295843124, -0.14188455045223236, 0.5834606289863586], "agreement_weight": 0.02356231026351452, "orthogonal_weight": -0.4153631329536438}, {"head": 3, "memory_to_active": 4.261168956756592, "contribution_to_active": 0.09273305535316467, "mix_mean": 0.032482828944921494, "mix_low": 0.3537425994873047, "mix_high": 0.0, "read_cosine": 0.10330662876367569, "beta_mean": 0.37065696716308594, "nonfinite": 0.0, "weighted_memory_to_active": 0.2923628091812134, "mix_std": 0.048039957880973816, "beta_std": 0.2726949453353882, "beta_low": 0.00775146484375, "beta_high": 0.00048828125, "orthogonal_to_active": 0.240962952375412, "evidence_weights": [0.06648006290197372, -0.010037819854915142, 0.6950876712799072], "agreement_weight": 0.05042450875043869, "orthogonal_weight": -0.928197979927063}, {"head": 4, "memory_to_active": 1.5832786560058594, "contribution_to_active": 0.08018132299184799, "mix_mean": 0.017507508397102356, "mix_low": 0.6380186080932617, "mix_high": 0.0, "read_cosine": 0.22872915863990784, "beta_mean": 0.08179286122322083, "nonfinite": 0.0, "weighted_memory_to_active": 0.06476700305938721, "mix_std": 0.03988347575068474, "beta_std": 0.08938811719417572, "beta_low": 0.12060546875, "beta_high": 0.0, "orthogonal_to_active": 0.00717002060264349, "evidence_weights": [0.1443728506565094, 0.03243492171168327, 0.2466636449098587], "agreement_weight": 0.04637520760297775, "orthogonal_weight": -0.1535363644361496}, {"head": 5, "memory_to_active": 0.0683484673500061, "contribution_to_active": 0.350477397441864, "mix_mean": 0.33769625425338745, "mix_low": 0.00737762451171875, "mix_high": 0.00020122528076171875, "read_cosine": 0.27293121814727783, "beta_mean": 0.0004409748362377286, "nonfinite": 0.0, "weighted_memory_to_active": 0.025283966213464737, "mix_std": 0.25053876638412476, "beta_std": 0.0013877059100195765, "beta_low": 0.9981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.015659546479582787, "evidence_weights": [-0.13298659026622772, -0.1922157108783722, 0.575303852558136], "agreement_weight": 0.14829525351524353, "orthogonal_weight": -0.7031299471855164}]}, {"layer": 8, "memory_to_active": 0.40398305654525757, "contribution_to_active": 0.5246816277503967, "mix_mean": 0.4308985769748688, "mix_low": 0.07260148227214813, "mix_high": 0.0011142095318064094, "read_cosine": 0.45520591735839844, "beta_mean": 0.02671195939183235, "nonfinite": 0.0, "weighted_memory_to_active": 0.09285072982311249, "mix_std": 0.21997378766536713, "beta_std": 0.035163603723049164, "beta_low": 0.6281942129135132, "beta_high": 0.0, "orthogonal_to_active": 0.061612263321876526, "output_influence": 0.46774399280548096, "residual_influence": 0.14923681318759918, "state_active": 0.048094894737005234, "state_memory": 0.008554903790354729, "state_peak": 32.97462463378906, "growth_active": 1.0872986316680908, "growth_memory": 1.3099119663238525, "heads": [{"head": 0, "memory_to_active": 0.05848468467593193, "contribution_to_active": 0.5579090118408203, "mix_mean": 0.6000711917877197, "mix_low": 0.0003757476806640625, "mix_high": 0.0012989044189453125, "read_cosine": 0.4596696197986603, "beta_mean": 0.003525202628225088, "nonfinite": 0.0, "weighted_memory_to_active": 0.040552619844675064, "mix_std": 0.24518968164920807, "beta_std": 0.0124766631051898, "beta_low": 0.92694091796875, "beta_high": 0.0, "orthogonal_to_active": 0.025953970849514008, "evidence_weights": [-0.1712862104177475, -0.3922003209590912, 0.6692422032356262], "agreement_weight": 0.16592179238796234, "orthogonal_weight": -0.7960106134414673}, {"head": 1, "memory_to_active": 0.12960495054721832, "contribution_to_active": 0.5113659501075745, "mix_mean": 0.48093926906585693, "mix_low": 0.000232696533203125, "mix_high": 0.00043010711669921875, "read_cosine": 0.25111496448516846, "beta_mean": 0.002942127175629139, "nonfinite": 0.0, "weighted_memory_to_active": 0.06288418918848038, "mix_std": 0.24721726775169373, "beta_std": 0.013289244845509529, "beta_low": 0.9583740234375, "beta_high": 0.0, "orthogonal_to_active": 0.0439816489815712, "evidence_weights": [-0.18748196959495544, -0.2625914514064789, 0.6905646324157715], "agreement_weight": 0.10303337872028351, "orthogonal_weight": -0.7495497465133667}, {"head": 2, "memory_to_active": 0.014912789687514305, "contribution_to_active": 0.6667426228523254, "mix_mean": 0.6501996517181396, "mix_low": 0.0010890960693359375, "mix_high": 0.00345611572265625, "read_cosine": 0.3813033401966095, "beta_mean": 0.00014643663598690182, "nonfinite": 0.0, "weighted_memory_to_active": 0.011208781972527504, "mix_std": 0.23151327669620514, "beta_std": 0.003246292006224394, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.007429292891174555, "evidence_weights": [-0.22426830232143402, -0.34316790103912354, 0.9812900424003601], "agreement_weight": 0.2315046489238739, "orthogonal_weight": -0.7536882758140564}, {"head": 3, "memory_to_active": 0.12259121239185333, "contribution_to_active": 0.6136071681976318, "mix_mean": 0.566369891166687, "mix_low": 0.00141143798828125, "mix_high": 0.0014276504516601562, "read_cosine": 0.8027628064155579, "beta_mean": 0.009767662733793259, "nonfinite": 0.0, "weighted_memory_to_active": 0.09218955039978027, "mix_std": 0.2562932074069977, "beta_std": 0.00766235264018178, "beta_low": 0.61328125, "beta_high": 0.0, "orthogonal_to_active": 0.030143316835165024, "evidence_weights": [-0.1824049949645996, -0.3044676184654236, 0.6149290204048157], "agreement_weight": 0.1901494562625885, "orthogonal_weight": -0.634238064289093}, {"head": 4, "memory_to_active": 1.7813643217086792, "contribution_to_active": 0.18686187267303467, "mix_mean": 0.039919011294841766, "mix_low": 0.3167715072631836, "mix_high": 0.0, "read_cosine": 0.3453754484653473, "beta_mean": 0.06987549364566803, "nonfinite": 0.0, "weighted_memory_to_active": 0.1800292283296585, "mix_std": 0.06412369012832642, "beta_std": 0.08164291083812714, "beta_low": 0.12158203125, "beta_high": 0.0, "orthogonal_to_active": 0.005073933396488428, "evidence_weights": [0.050060711801052094, -0.04618029296398163, -0.04411744326353073], "agreement_weight": 0.08369326591491699, "orthogonal_weight": -0.03052579052746296}, {"head": 5, "memory_to_active": 0.31694018840789795, "contribution_to_active": 0.6116029620170593, "mix_mean": 0.24789217114448547, "mix_low": 0.11572837829589844, "mix_high": 7.2479248046875e-05, "read_cosine": 0.4910092353820801, "beta_mean": 0.0740148276090622, "nonfinite": 0.0, "weighted_memory_to_active": 0.17023999989032745, "mix_std": 0.2755056321620941, "beta_std": 0.09266413003206253, "beta_low": 0.1490478515625, "beta_high": 0.0, "orthogonal_to_active": 0.2570914030075073, "evidence_weights": [-0.08350930362939835, -0.21643906831741333, -0.23418697714805603], "agreement_weight": -0.010364691726863384, "orthogonal_weight": 1.7961862087249756}]}, {"layer": 9, "memory_to_active": 1.0453368425369263, "contribution_to_active": 0.6520931720733643, "mix_mean": 0.33039990067481995, "mix_low": 0.14070352911949158, "mix_high": 0.004596551414579153, "read_cosine": 0.34352344274520874, "beta_mean": 0.06669969111680984, "nonfinite": 0.0, "weighted_memory_to_active": 0.3679203391075134, "mix_std": 0.2140483558177948, "beta_std": 0.045890260487794876, "beta_low": 0.5013021230697632, "beta_high": 0.0, "orthogonal_to_active": 0.039931342005729675, "output_influence": 0.5074353218078613, "residual_influence": 0.13881655037403107, "state_active": 0.04113691300153732, "state_memory": 0.01900426112115383, "state_peak": 43.219093322753906, "growth_active": 1.1750664710998535, "growth_memory": 1.3755683898925781, "heads": [{"head": 0, "memory_to_active": 0.002285997848957777, "contribution_to_active": 0.5530326962471008, "mix_mean": 0.6242794990539551, "mix_low": 0.0017709732055664062, "mix_high": 0.020936012268066406, "read_cosine": 0.2769809365272522, "beta_mean": 5.2077652981097344e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014649808872491121, "mix_std": 0.2913845479488373, "beta_std": 1.9702354620676488e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013609351590275764, "evidence_weights": [-0.2979690730571747, -0.3073841333389282, 0.863305926322937], "agreement_weight": 0.10802090913057327, "orthogonal_weight": -0.9776626229286194}, {"head": 1, "memory_to_active": 0.020720100030303, "contribution_to_active": 0.6580318808555603, "mix_mean": 0.524504542350769, "mix_low": 0.007856369018554688, "mix_high": 0.0016412734985351562, "read_cosine": 0.40060940384864807, "beta_mean": 6.0962662246311083e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014564374461770058, "mix_std": 0.2840173542499542, "beta_std": 0.00013395427959039807, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010272943414747715, "evidence_weights": [-0.23130232095718384, -0.279677152633667, 1.153074026107788], "agreement_weight": 0.3989641070365906, "orthogonal_weight": -0.872068464756012}, {"head": 2, "memory_to_active": 0.19561180472373962, "contribution_to_active": 0.6292213201522827, "mix_mean": 0.5697400569915771, "mix_low": 0.0024662017822265625, "mix_high": 0.004306793212890625, "read_cosine": 0.34650611877441406, "beta_mean": 0.0008326547686010599, "nonfinite": 0.0, "weighted_memory_to_active": 0.13011550903320312, "mix_std": 0.27345865964889526, "beta_std": 0.005019476171582937, "beta_low": 0.99365234375, "beta_high": 0.0, "orthogonal_to_active": 0.07217860966920853, "evidence_weights": [-0.19717499613761902, -0.2680901288986206, 0.7122172117233276], "agreement_weight": 0.28419873118400574, "orthogonal_weight": -0.6458038687705994}, {"head": 3, "memory_to_active": 1.411572813987732, "contribution_to_active": 0.16784197092056274, "mix_mean": 0.04770796373486519, "mix_low": 0.3691253662109375, "mix_high": 1.9073486328125e-06, "read_cosine": 0.4088083505630493, "beta_mean": 0.0836770087480545, "nonfinite": 0.0, "weighted_memory_to_active": 0.14485770463943481, "mix_std": 0.10116764903068542, "beta_std": 0.05592759698629379, "beta_low": 0.00213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.00854634027928114, "evidence_weights": [0.07819462567567825, 0.015417748130857944, 0.03283398970961571], "agreement_weight": 0.0874861627817154, "orthogonal_weight": 0.06448434293270111}, {"head": 4, "memory_to_active": 2.9504024982452393, "contribution_to_active": 1.355629324913025, "mix_mean": 0.13475367426872253, "mix_low": 0.15050315856933594, "mix_high": 0.0005197525024414062, "read_cosine": 0.1528157889842987, "beta_mean": 0.1777149736881256, "nonfinite": 0.0, "weighted_memory_to_active": 1.3400661945343018, "mix_std": 0.19361770153045654, "beta_std": 0.09994333237409592, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03338496387004852, "evidence_weights": [-0.06964386254549026, -0.1530795693397522, -0.01666460558772087], "agreement_weight": -0.05692760646343231, "orthogonal_weight": -0.0261435117572546}, {"head": 5, "memory_to_active": 1.6914277076721191, "contribution_to_active": 0.5488015413284302, "mix_mean": 0.08141347765922546, "mix_low": 0.3124990463256836, "mix_high": 0.0001735687255859375, "read_cosine": 0.475420206785202, "beta_mean": 0.13790732622146606, "nonfinite": 0.0, "weighted_memory_to_active": 0.5764532089233398, "mix_std": 0.1406441330909729, "beta_std": 0.11429749429225922, "beta_low": 0.01202392578125, "beta_high": 0.0, "orthogonal_to_active": 0.11384424567222595, "evidence_weights": [0.10881172120571136, -0.08816778659820557, 0.15632852911949158], "agreement_weight": 0.014909913763403893, "orthogonal_weight": 0.2766118347644806}]}, {"layer": 10, "memory_to_active": 0.21488574147224426, "contribution_to_active": 0.6396687030792236, "mix_mean": 0.5111919045448303, "mix_low": 0.017308076843619347, "mix_high": 0.00625928258523345, "read_cosine": 0.36592718958854675, "beta_mean": 0.04783044382929802, "nonfinite": 0.0, "weighted_memory_to_active": 0.1006539836525917, "mix_std": 0.2708457112312317, "beta_std": 0.04782713204622269, "beta_low": 0.6623942255973816, "beta_high": 0.0, "orthogonal_to_active": 0.16781210899353027, "output_influence": 0.4347323477268219, "residual_influence": 0.14629527926445007, "state_active": 0.053283605724573135, "state_memory": 0.014275931753218174, "state_peak": 121.8446044921875, "growth_active": 1.0764402151107788, "growth_memory": 0.9510513544082642, "heads": [{"head": 0, "memory_to_active": 0.7947099208831787, "contribution_to_active": 1.5679036378860474, "mix_mean": 0.5882629156112671, "mix_low": 0.06707954406738281, "mix_high": 0.017187118530273438, "read_cosine": 0.10122501850128174, "beta_mean": 0.21054352819919586, "nonfinite": 0.0, "weighted_memory_to_active": 0.48375752568244934, "mix_std": 0.38923379778862, "beta_std": 0.21957768499851227, "beta_low": 0.112548828125, "beta_high": 0.0, "orthogonal_to_active": 0.9261132478713989, "evidence_weights": [-0.12159350514411926, -0.3973304033279419, -1.3832054138183594], "agreement_weight": 0.017330219969153404, "orthogonal_weight": 1.9414113759994507}, {"head": 1, "memory_to_active": 0.008099241182208061, "contribution_to_active": 0.6015720367431641, "mix_mean": 0.611946702003479, "mix_low": 0.0007839202880859375, "mix_high": 0.0128173828125, "read_cosine": 0.2790997624397278, "beta_mean": 5.245546708465554e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004666227847337723, "mix_std": 0.26879599690437317, "beta_std": 7.012348214630038e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006719935219734907, "evidence_weights": [-0.26143547892570496, -0.2005474716424942, 0.5801051259040833], "agreement_weight": 0.4882740080356598, "orthogonal_weight": -1.5631135702133179}, {"head": 2, "memory_to_active": 0.010343887843191624, "contribution_to_active": 0.6117918491363525, "mix_mean": 0.6300884485244751, "mix_low": 0.00047206878662109375, "mix_high": 0.006463050842285156, "read_cosine": 0.022677548229694366, "beta_mean": 2.638616570038721e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004721756558865309, "mix_std": 0.2560478448867798, "beta_std": 8.799170609563589e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0022805689368396997, "evidence_weights": [-0.24675582349300385, -0.2787308692932129, 0.5916898846626282], "agreement_weight": -0.175611674785614, "orthogonal_weight": -0.47123363614082336}, {"head": 3, "memory_to_active": 0.33277428150177, "contribution_to_active": 0.10438918322324753, "mix_mean": 0.28470975160598755, "mix_low": 0.032413482666015625, "mix_high": 0.0002994537353515625, "read_cosine": 0.9118814468383789, "beta_mean": 0.07051803171634674, "nonfinite": 0.0, "weighted_memory_to_active": 0.04090223088860512, "mix_std": 0.2305631786584854, "beta_std": 0.060424625873565674, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.04434249922633171, "evidence_weights": [-0.11066817492246628, -0.1488952487707138, 0.15702460706233978], "agreement_weight": 0.009792475029826164, "orthogonal_weight": 1.448356032371521}, {"head": 4, "memory_to_active": 0.014952151104807854, "contribution_to_active": 0.5365902185440063, "mix_mean": 0.5587395429611206, "mix_low": 0.0003185272216796875, "mix_high": 0.0007801055908203125, "read_cosine": 0.019069720059633255, "beta_mean": 4.540299778454937e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008414335548877716, "mix_std": 0.25137728452682495, "beta_std": 0.00012921848974656314, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00861247070133686, "evidence_weights": [-0.18352709710597992, -0.2624666690826416, 0.8665493726730347], "agreement_weight": 0.12794770300388336, "orthogonal_weight": -1.0770806074142456}, {"head": 5, "memory_to_active": 0.12843486666679382, "contribution_to_active": 0.4157653748989105, "mix_mean": 0.3934038281440735, "mix_low": 0.002780914306640625, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8616095781326294, "beta_mean": 0.005796853918582201, "nonfinite": 0.0, "weighted_memory_to_active": 0.06146182119846344, "mix_std": 0.22905614972114563, "beta_std": 0.006673166528344154, "beta_low": 0.86138916015625, "beta_high": 0.0, "orthogonal_to_active": 0.0188040379434824, "evidence_weights": [-0.14480476081371307, -0.1596926599740982, 0.2825201451778412], "agreement_weight": 0.08966904133558273, "orthogonal_weight": -0.6415355205535889}]}, {"layer": 11, "memory_to_active": 0.10843487083911896, "contribution_to_active": 0.5967295169830322, "mix_mean": 0.49141743779182434, "mix_low": 0.0938115194439888, "mix_high": 0.04037269204854965, "read_cosine": 0.40231209993362427, "beta_mean": 0.02903749980032444, "nonfinite": 0.0, "weighted_memory_to_active": 0.05277745798230171, "mix_std": 0.32179200649261475, "beta_std": 0.04215681552886963, "beta_low": 0.6210530996322632, "beta_high": 0.0, "orthogonal_to_active": 0.04730842262506485, "output_influence": 0.5263370275497437, "residual_influence": 0.1305137723684311, "state_active": 0.21622446179389954, "state_memory": 0.005744935944676399, "state_peak": 32.936275482177734, "growth_active": 1.1328725814819336, "growth_memory": 1.6635613441467285, "heads": [{"head": 0, "memory_to_active": 0.007185136899352074, "contribution_to_active": 0.59186190366745, "mix_mean": 0.6308315992355347, "mix_low": 0.0009317398071289062, "mix_high": 0.00885772705078125, "read_cosine": 0.3121412694454193, "beta_mean": 9.188991680275649e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00499927531927824, "mix_std": 0.2600979208946228, "beta_std": 0.003889329032972455, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.0038411524146795273, "evidence_weights": [-0.16813521087169647, -0.3702482581138611, 0.7234034538269043], "agreement_weight": 0.3057537376880646, "orthogonal_weight": -0.8088011741638184}, {"head": 1, "memory_to_active": 0.24834688007831573, "contribution_to_active": 0.34848931431770325, "mix_mean": 0.39315715432167053, "mix_low": 0.008128166198730469, "mix_high": 0.00033664703369140625, "read_cosine": 0.5354076623916626, "beta_mean": 0.022919218987226486, "nonfinite": 0.0, "weighted_memory_to_active": 0.106875479221344, "mix_std": 0.2611501216888428, "beta_std": 0.0479142852127552, "beta_low": 0.55047607421875, "beta_high": 0.0, "orthogonal_to_active": 0.04153056442737579, "evidence_weights": [-0.16681623458862305, -0.19579148292541504, 0.5169779658317566], "agreement_weight": 0.07353495061397552, "orthogonal_weight": -0.48703649640083313}, {"head": 2, "memory_to_active": 0.00790422037243843, "contribution_to_active": 0.9809805750846863, "mix_mean": 0.4411858916282654, "mix_low": 0.24243927001953125, "mix_high": 0.1525106430053711, "read_cosine": 0.2067115753889084, "beta_mean": 0.07174147665500641, "nonfinite": 0.0, "weighted_memory_to_active": 0.005684077274054289, "mix_std": 0.4420497417449951, "beta_std": 0.09074614942073822, "beta_low": 0.435791015625, "beta_high": 0.0, "orthogonal_to_active": 0.038773372769355774, "evidence_weights": [0.040050335228443146, -0.4210890531539917, -1.2630295753479004], "agreement_weight": 0.010755523107945919, "orthogonal_weight": 7.134232997894287}, {"head": 3, "memory_to_active": 0.010563166812062263, "contribution_to_active": 0.843715488910675, "mix_mean": 0.4049396514892578, "mix_low": 0.3020362854003906, "mix_high": 0.07572746276855469, "read_cosine": 0.5951542854309082, "beta_mean": 0.05528973042964935, "nonfinite": 0.0, "weighted_memory_to_active": 0.006456943694502115, "mix_std": 0.434963583946228, "beta_std": 0.08024857938289642, "beta_low": 0.481689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0340244360268116, "evidence_weights": [0.16011571884155273, -0.32103633880615234, -0.23275327682495117], "agreement_weight": 0.049295615404844284, "orthogonal_weight": 7.48113489151001}, {"head": 4, "memory_to_active": 0.28252676129341125, "contribution_to_active": 0.4300137758255005, "mix_mean": 0.5419230461120605, "mix_low": 0.007891654968261719, "mix_high": 0.001068115234375, "read_cosine": 0.0731474831700325, "beta_mean": 0.015887172892689705, "nonfinite": 0.0, "weighted_memory_to_active": 0.14730557799339294, "mix_std": 0.27062928676605225, "beta_std": 0.017964588478207588, "beta_low": 0.50067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.14140033721923828, "evidence_weights": [-0.2373073548078537, -0.2205284982919693, 1.9222344160079956], "agreement_weight": -0.19282551109790802, "orthogonal_weight": -0.980612576007843}, {"head": 5, "memory_to_active": 0.0940830260515213, "contribution_to_active": 0.38531601428985596, "mix_mean": 0.5364671945571899, "mix_low": 0.00144195556640625, "mix_high": 0.0037355422973632812, "read_cosine": 0.6913101673126221, "beta_mean": 0.008295496925711632, "nonfinite": 0.0, "weighted_memory_to_active": 0.04534338414669037, "mix_std": 0.2618613839149475, "beta_std": 0.012177940458059311, "beta_low": 0.758056640625, "beta_high": 0.0, "orthogonal_to_active": 0.024280665442347527, "evidence_weights": [-0.21689552068710327, -0.30313795804977417, 0.6175124645233154], "agreement_weight": 0.08198407292366028, "orthogonal_weight": -0.7345516681671143}]}, {"layer": 12, "memory_to_active": 0.27605682611465454, "contribution_to_active": 0.5716546773910522, "mix_mean": 0.5138974189758301, "mix_low": 0.009571234695613384, "mix_high": 0.004269282333552837, "read_cosine": 0.23627668619155884, "beta_mean": 0.016801105812191963, "nonfinite": 0.0, "weighted_memory_to_active": 0.15024329721927643, "mix_std": 0.26059210300445557, "beta_std": 0.01656278967857361, "beta_low": 0.8218892812728882, "beta_high": 0.0, "orthogonal_to_active": 0.075496606528759, "output_influence": 0.46855059266090393, "residual_influence": 0.13103260099887848, "state_active": 0.036777012050151825, "state_memory": 0.009618516080081463, "state_peak": 49.51757049560547, "growth_active": 1.1479514837265015, "growth_memory": 1.2372629642486572, "heads": [{"head": 0, "memory_to_active": 0.05692986398935318, "contribution_to_active": 0.557713508605957, "mix_mean": 0.5342130064964294, "mix_low": 0.0008783340454101562, "mix_high": 0.0030183792114257812, "read_cosine": 0.28303056955337524, "beta_mean": 0.0010191076435148716, "nonfinite": 0.0, "weighted_memory_to_active": 0.028822127729654312, "mix_std": 0.2714938521385193, "beta_std": 0.002557306783273816, "beta_low": 0.9921875, "beta_high": 0.0, "orthogonal_to_active": 0.02435293421149254, "evidence_weights": [-0.20255766808986664, -0.27267950773239136, 0.1979704648256302], "agreement_weight": 0.10279768705368042, "orthogonal_weight": -0.9054968953132629}, {"head": 1, "memory_to_active": 0.10941626876592636, "contribution_to_active": 0.526107907295227, "mix_mean": 0.5405479669570923, "mix_low": 0.0006561279296875, "mix_high": 0.0010290145874023438, "read_cosine": 0.4730641841888428, "beta_mean": 0.0011201166780665517, "nonfinite": 0.0, "weighted_memory_to_active": 0.06243191659450531, "mix_std": 0.2634378671646118, "beta_std": 0.008080619387328625, "beta_low": 0.98785400390625, "beta_high": 0.0, "orthogonal_to_active": 0.04990740865468979, "evidence_weights": [-0.20755566656589508, -0.2733965516090393, 0.9898303151130676], "agreement_weight": 0.1633239984512329, "orthogonal_weight": -0.9246909022331238}, {"head": 2, "memory_to_active": 0.6065884828567505, "contribution_to_active": 0.47984880208969116, "mix_mean": 0.31317463517189026, "mix_low": 0.0267791748046875, "mix_high": 9.059906005859375e-05, "read_cosine": 0.4680714011192322, "beta_mean": 0.09627542644739151, "nonfinite": 0.0, "weighted_memory_to_active": 0.24307957291603088, "mix_std": 0.2339625060558319, "beta_std": 0.06386119872331619, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.2469875067472458, "evidence_weights": [-0.2374388426542282, -0.04971802234649658, -0.06840475648641586], "agreement_weight": 0.03263819217681885, "orthogonal_weight": 1.2483608722686768}, {"head": 3, "memory_to_active": 0.03713330626487732, "contribution_to_active": 0.7350136637687683, "mix_mean": 0.6955316662788391, "mix_low": 0.00032711029052734375, "mix_high": 0.012729644775390625, "read_cosine": 0.11305026710033417, "beta_mean": 0.00035661834408529103, "nonfinite": 0.0, "weighted_memory_to_active": 0.02703440748155117, "mix_std": 0.2353324592113495, "beta_std": 0.005658832844346762, "beta_low": 0.99481201171875, "beta_high": 0.0, "orthogonal_to_active": 0.02560902200639248, "evidence_weights": [-0.28955787420272827, -0.33995145559310913, 1.4447047710418701], "agreement_weight": 0.4208574593067169, "orthogonal_weight": -0.9691442251205444}, {"head": 4, "memory_to_active": 0.7464283108711243, "contribution_to_active": 0.6446822881698608, "mix_mean": 0.3785942792892456, "mix_low": 0.021360397338867188, "mix_high": 0.0030317306518554688, "read_cosine": -0.12232381105422974, "beta_mean": 0.0012194759910926223, "nonfinite": 0.0, "weighted_memory_to_active": 0.4729700982570648, "mix_std": 0.2794336676597595, "beta_std": 0.012852796353399754, "beta_low": 0.9757080078125, "beta_high": 0.0, "orthogonal_to_active": 0.05406692996621132, "evidence_weights": [-0.12935125827789307, -0.21551784873008728, -0.0014798396732658148], "agreement_weight": 0.0010301887523382902, "orthogonal_weight": -0.12018703669309616}, {"head": 5, "memory_to_active": 0.09984458237886429, "contribution_to_active": 0.48656177520751953, "mix_mean": 0.621322751045227, "mix_low": 0.007426261901855469, "mix_high": 0.0057163238525390625, "read_cosine": 0.202767476439476, "beta_mean": 0.0008158866548910737, "nonfinite": 0.0, "weighted_memory_to_active": 0.06712154299020767, "mix_std": 0.27989229559898376, "beta_std": 0.00636597303673625, "beta_low": 0.98046875, "beta_high": 0.0, "orthogonal_to_active": 0.05205583944916725, "evidence_weights": [-0.24509529769420624, -0.3139572739601135, 1.2932602167129517], "agreement_weight": 0.34426406025886536, "orthogonal_weight": -0.8367453217506409}]}, {"layer": 13, "memory_to_active": 0.1255502849817276, "contribution_to_active": 0.5232647657394409, "mix_mean": 0.5214036703109741, "mix_low": 0.005425135605037212, "mix_high": 0.007073243614286184, "read_cosine": 0.42312484979629517, "beta_mean": 0.017599143087863922, "nonfinite": 0.0, "weighted_memory_to_active": 0.05248298496007919, "mix_std": 0.2587328255176544, "beta_std": 0.02037634328007698, "beta_low": 0.7996012568473816, "beta_high": 0.0, "orthogonal_to_active": 0.05211593955755234, "output_influence": 0.4968263804912567, "residual_influence": 0.1663450449705124, "state_active": 0.11500013619661331, "state_memory": 0.04189446568489075, "state_peak": 103.65249633789062, "growth_active": 0.9588015675544739, "growth_memory": 1.0419020652770996, "heads": [{"head": 0, "memory_to_active": 0.02032199501991272, "contribution_to_active": 0.561295747756958, "mix_mean": 0.5837098360061646, "mix_low": 0.0004367828369140625, "mix_high": 0.002437591552734375, "read_cosine": 0.41830652952194214, "beta_mean": 7.831874245312065e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012602523900568485, "mix_std": 0.2694457173347473, "beta_std": 0.00018816038209479302, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012674509547650814, "evidence_weights": [-0.20057007670402527, -0.2662191092967987, 0.7519601583480835], "agreement_weight": 0.11335528641939163, "orthogonal_weight": -1.1159446239471436}, {"head": 1, "memory_to_active": 0.01852061226963997, "contribution_to_active": 0.5822334289550781, "mix_mean": 0.5429030656814575, "mix_low": 0.0020704269409179688, "mix_high": 0.0027151107788085938, "read_cosine": 0.200552299618721, "beta_mean": 6.757816299796104e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010349474847316742, "mix_std": 0.2741627097129822, "beta_std": 0.0001295704278163612, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0128908297047019, "evidence_weights": [-0.18507671356201172, -0.2666095793247223, 0.6794201135635376], "agreement_weight": 0.3580065667629242, "orthogonal_weight": -1.299017310142517}, {"head": 2, "memory_to_active": 0.10298498719930649, "contribution_to_active": 0.4508407711982727, "mix_mean": 0.3754221200942993, "mix_low": 0.0018444061279296875, "mix_high": 8.678436279296875e-05, "read_cosine": 0.7119358777999878, "beta_mean": 0.01673857867717743, "nonfinite": 0.0, "weighted_memory_to_active": 0.04381173476576805, "mix_std": 0.22552266716957092, "beta_std": 0.054580315947532654, "beta_low": 0.80523681640625, "beta_high": 0.0, "orthogonal_to_active": 0.01919926516711712, "evidence_weights": [-0.19276565313339233, -0.2008260190486908, 0.42518773674964905], "agreement_weight": 0.00777787808328867, "orthogonal_weight": -0.8115988969802856}, {"head": 3, "memory_to_active": 0.138349249958992, "contribution_to_active": 0.6119552254676819, "mix_mean": 0.6462829113006592, "mix_low": 0.019112586975097656, "mix_high": 0.0276947021484375, "read_cosine": 0.03842981904745102, "beta_mean": 0.001001560827717185, "nonfinite": 0.0, "weighted_memory_to_active": 0.104301817715168, "mix_std": 0.28858333826065063, "beta_std": 0.007888654246926308, "beta_low": 0.9940185546875, "beta_high": 0.0, "orthogonal_to_active": 0.09181900322437286, "evidence_weights": [-0.3239540755748749, -0.24821189045906067, 1.7870514392852783], "agreement_weight": 4.6641035079956055, "orthogonal_weight": -0.99906325340271}, {"head": 4, "memory_to_active": 0.40891367197036743, "contribution_to_active": 0.2488439530134201, "mix_mean": 0.3480505347251892, "mix_low": 0.0064239501953125, "mix_high": 0.0, "read_cosine": 0.8911789655685425, "beta_mean": 0.0873287096619606, "nonfinite": 0.0, "weighted_memory_to_active": 0.10302723199129105, "mix_std": 0.218007892370224, "beta_std": 0.05566076934337616, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14132219552993774, "evidence_weights": [-0.11311807483434677, -0.23471489548683167, 0.47264063358306885], "agreement_weight": -0.0004131287569180131, "orthogonal_weight": 2.1975836753845215}, {"head": 5, "memory_to_active": 0.0642111748456955, "contribution_to_active": 0.6844193339347839, "mix_mean": 0.6320531368255615, "mix_low": 0.00266265869140625, "mix_high": 0.009505271911621094, "read_cosine": 0.2783457040786743, "beta_mean": 0.00038010248681530356, "nonfinite": 0.0, "weighted_memory_to_active": 0.0408051423728466, "mix_std": 0.27667468786239624, "beta_std": 0.0038105908315628767, "beta_low": 0.99835205078125, "beta_high": 0.0, "orthogonal_to_active": 0.0347897931933403, "evidence_weights": [-0.21913065016269684, -0.304198682308197, 0.8253287672996521], "agreement_weight": 0.13182459771633148, "orthogonal_weight": -0.9236825704574585}]}, {"layer": 14, "memory_to_active": 0.2430979609489441, "contribution_to_active": 0.42148828506469727, "mix_mean": 0.4264861047267914, "mix_low": 0.023348968476057053, "mix_high": 0.0059420270845294, "read_cosine": 0.618444561958313, "beta_mean": 0.0656023621559143, "nonfinite": 0.0, "weighted_memory_to_active": 0.10474791377782822, "mix_std": 0.2466323971748352, "beta_std": 0.06109090521931648, "beta_low": 0.4693095088005066, "beta_high": 0.0, "orthogonal_to_active": 0.04367653653025627, "output_influence": 0.33951449394226074, "residual_influence": 0.1294880360364914, "state_active": 0.05853544920682907, "state_memory": 0.01607794687151909, "state_peak": 38.42176818847656, "growth_active": 1.084466576576233, "growth_memory": 1.2625430822372437, "heads": [{"head": 0, "memory_to_active": 0.36691346764564514, "contribution_to_active": 0.20581869781017303, "mix_mean": 0.20572471618652344, "mix_low": 0.013976097106933594, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8017716407775879, "beta_mean": 0.07564626634120941, "nonfinite": 0.0, "weighted_memory_to_active": 0.08994078636169434, "mix_std": 0.18299178779125214, "beta_std": 0.06427586078643799, "beta_low": 0.00384521484375, "beta_high": 0.0, "orthogonal_to_active": 0.07819874584674835, "evidence_weights": [-0.13155733048915863, -0.03297761455178261, 0.4756183326244354], "agreement_weight": 0.02707710489630699, "orthogonal_weight": 1.4718929529190063}, {"head": 1, "memory_to_active": 0.5022903084754944, "contribution_to_active": 0.29698219895362854, "mix_mean": 0.23666062951087952, "mix_low": 0.10934066772460938, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8291075229644775, "beta_mean": 0.25893634557724, "nonfinite": 0.0, "weighted_memory_to_active": 0.2163538634777069, "mix_std": 0.25570452213287354, "beta_std": 0.24593545496463776, "beta_low": 0.09515380859375, "beta_high": 0.0, "orthogonal_to_active": 0.014196320436894894, "evidence_weights": [-0.016306614503264427, -0.21726448833942413, -0.12297501415014267], "agreement_weight": 0.05233005806803703, "orthogonal_weight": 0.1544734090566635}, {"head": 2, "memory_to_active": 0.15798383951187134, "contribution_to_active": 0.5637130737304688, "mix_mean": 0.5810058116912842, "mix_low": 0.0027551651000976562, "mix_high": 0.0037240982055664062, "read_cosine": 0.40174606442451477, "beta_mean": 0.0032849013805389404, "nonfinite": 0.0, "weighted_memory_to_active": 0.10074413567781448, "mix_std": 0.2673863470554352, "beta_std": 0.0043610744178295135, "beta_low": 0.94317626953125, "beta_high": 0.0, "orthogonal_to_active": 0.07720907777547836, "evidence_weights": [-0.14935889840126038, -0.3135598599910736, 0.8914740085601807], "agreement_weight": 0.086265929043293, "orthogonal_weight": -0.8706281781196594}, {"head": 3, "memory_to_active": 0.05832073092460632, "contribution_to_active": 0.6270906329154968, "mix_mean": 0.6744241118431091, "mix_low": 0.0052394866943359375, "mix_high": 0.021764755249023438, "read_cosine": 0.06493456661701202, "beta_mean": 0.00756542943418026, "nonfinite": 0.0, "weighted_memory_to_active": 0.03856491297483444, "mix_std": 0.25367075204849243, "beta_std": 0.007975142449140549, "beta_low": 0.761474609375, "beta_high": 0.0, "orthogonal_to_active": 0.027013573795557022, "evidence_weights": [-0.36124566197395325, -0.318709135055542, 0.3322109282016754], "agreement_weight": 0.08111108839511871, "orthogonal_weight": -0.7911749482154846}, {"head": 4, "memory_to_active": 0.14371666312217712, "contribution_to_active": 0.4934206008911133, "mix_mean": 0.49609601497650146, "mix_low": 0.0049915313720703125, "mix_high": 0.008231163024902344, "read_cosine": 0.867768406867981, "beta_mean": 0.046166371554136276, "nonfinite": 0.0, "weighted_memory_to_active": 0.08323251456022263, "mix_std": 0.2625892162322998, "beta_std": 0.03292873501777649, "beta_low": 0.04718017578125, "beta_high": 0.0, "orthogonal_to_active": 0.02312745526432991, "evidence_weights": [-0.21921199560165405, -0.24873226881027222, 0.4271358549594879], "agreement_weight": 0.1280076801776886, "orthogonal_weight": -0.5956043004989624}, {"head": 5, "memory_to_active": 0.22936275601387024, "contribution_to_active": 0.3419044613838196, "mix_mean": 0.365005224943161, "mix_low": 0.0037908554077148438, "mix_high": 0.0019245147705078125, "read_cosine": 0.7453387379646301, "beta_mean": 0.0020148474723100662, "nonfinite": 0.0, "weighted_memory_to_active": 0.09965124726295471, "mix_std": 0.25745171308517456, "beta_std": 0.01106917206197977, "beta_low": 0.96502685546875, "beta_high": 0.0, "orthogonal_to_active": 0.04231401905417442, "evidence_weights": [-0.18002520501613617, -0.16793403029441833, 0.25733140110969543], "agreement_weight": 0.08419820666313171, "orthogonal_weight": -0.6963458061218262}]}, {"layer": 15, "memory_to_active": 0.05960782244801521, "contribution_to_active": 0.48369163274765015, "mix_mean": 0.41515663266181946, "mix_low": 0.008432547561824322, "mix_high": 0.002635797020047903, "read_cosine": 0.6484820246696472, "beta_mean": 0.038872141391038895, "nonfinite": 0.0, "weighted_memory_to_active": 0.02795552648603916, "mix_std": 0.2853957414627075, "beta_std": 0.04464434087276459, "beta_low": 0.7077230215072632, "beta_high": 0.0, "orthogonal_to_active": 0.0071304719895124435, "output_influence": 0.29948025941848755, "residual_influence": 0.16553077101707458, "state_active": 0.1336163729429245, "state_memory": 0.010345791466534138, "state_peak": 36.977195739746094, "growth_active": 1.2401537895202637, "growth_memory": 1.4907934665679932, "heads": [{"head": 0, "memory_to_active": 0.10914023965597153, "contribution_to_active": 0.4591185450553894, "mix_mean": 0.37141481041908264, "mix_low": 0.006903648376464844, "mix_high": 0.005217552185058594, "read_cosine": 0.6641915440559387, "beta_mean": 0.04017839580774307, "nonfinite": 0.0, "weighted_memory_to_active": 0.04939501732587814, "mix_std": 0.2815870940685272, "beta_std": 0.030381154268980026, "beta_low": 0.09405517578125, "beta_high": 0.0, "orthogonal_to_active": 0.023637300357222557, "evidence_weights": [-0.1788971871137619, -0.18229417502880096, 0.3842715620994568], "agreement_weight": 0.09269274026155472, "orthogonal_weight": -0.7278578281402588}, {"head": 1, "memory_to_active": 0.14315903186798096, "contribution_to_active": 0.4121899902820587, "mix_mean": 0.4261332154273987, "mix_low": 0.001926422119140625, "mix_high": 0.005791664123535156, "read_cosine": 0.9119021892547607, "beta_mean": 0.0010346760973334312, "nonfinite": 0.0, "weighted_memory_to_active": 0.05938003584742546, "mix_std": 0.28591281175613403, "beta_std": 0.0030389560852199793, "beta_low": 0.98553466796875, "beta_high": 0.0, "orthogonal_to_active": 0.0005300320917740464, "evidence_weights": [-0.15713927149772644, -0.16716839373111725, 0.21141889691352844], "agreement_weight": 0.11826957017183304, "orthogonal_weight": 0.017009204253554344}, {"head": 2, "memory_to_active": 0.008223317563533783, "contribution_to_active": 0.4733262062072754, "mix_mean": 0.3451184630393982, "mix_low": 0.01616954803466797, "mix_high": 9.441375732421875e-05, "read_cosine": 0.7542366981506348, "beta_mean": 0.00012183011131128296, "nonfinite": 0.0, "weighted_memory_to_active": 0.003999360837042332, "mix_std": 0.2726055979728699, "beta_std": 0.00027503445744514465, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0009598804172128439, "evidence_weights": [-0.2043977677822113, -0.2731996476650238, 0.18422837555408478], "agreement_weight": 0.008832637220621109, "orthogonal_weight": 0.5514994263648987}, {"head": 3, "memory_to_active": 0.003378374734893441, "contribution_to_active": 0.4045800268650055, "mix_mean": 0.38803669810295105, "mix_low": 0.006041526794433594, "mix_high": 0.00039386749267578125, "read_cosine": 0.6422606110572815, "beta_mean": 2.4260643840534613e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.001224451931193471, "mix_std": 0.28506284952163696, "beta_std": 7.117180939530954e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010774253169074655, "evidence_weights": [-0.23899297416210175, -0.25365781784057617, 0.23417654633522034], "agreement_weight": 0.03783278539776802, "orthogonal_weight": -0.925114095211029}, {"head": 4, "memory_to_active": 0.0689484104514122, "contribution_to_active": 0.6197373867034912, "mix_mean": 0.3922405242919922, "mix_low": 0.010110855102539062, "mix_high": 0.0009031295776367188, "read_cosine": 0.5621639490127563, "beta_mean": 0.19002734124660492, "nonfinite": 0.0, "weighted_memory_to_active": 0.037151142954826355, "mix_std": 0.31101399660110474, "beta_std": 0.22021670639514923, "beta_low": 0.1934814453125, "beta_high": 0.0, "orthogonal_to_active": 0.006849520839750767, "evidence_weights": [-0.21420320868492126, -0.25463953614234924, -0.006701063830405474], "agreement_weight": 0.013689275830984116, "orthogonal_weight": 0.2963927090167999}, {"head": 5, "memory_to_active": 0.024797555059194565, "contribution_to_active": 0.5331975221633911, "mix_mean": 0.5679959654808044, "mix_low": 0.009443283081054688, "mix_high": 0.003414154052734375, "read_cosine": 0.35613709688186646, "beta_mean": 0.001846345141530037, "nonfinite": 0.0, "weighted_memory_to_active": 0.016583142802119255, "mix_std": 0.2761921286582947, "beta_std": 0.01388300396502018, "beta_low": 0.97332763671875, "beta_high": 0.0, "orthogonal_to_active": 0.009728672914206982, "evidence_weights": [-0.3002428710460663, -0.3137874901294708, 0.7448481917381287], "agreement_weight": 0.2701045274734497, "orthogonal_weight": 0.7510538697242737}]}]}, {"step": 43000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8728377890001866, "memory_to_active": 0.8445115685462952, "contribution_to_active": 0.5476627945899963, "mix_mean": 0.3932531476020813, "mix_low": 0.14010252058506012, "mix_high": 0.009009758941829205, "read_cosine": 0.3595094680786133, "beta_mean": 0.07555476576089859, "nonfinite": 0.0, "weighted_memory_to_active": 0.21575339138507843, "mix_std": 0.2204534113407135, "beta_std": 0.0652364119887352, "beta_low": 0.5354284048080444, "beta_high": 1.9073486328125e-05, "orthogonal_to_active": 0.05829090252518654, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1348366737365723, "normal_token_accuracy": 0.442714124917984, "state_active": 0.06132014840841293, "state_memory": 0.01866055466234684, "state_peak": 106.07532501220703, "growth_active": 1.1295270919799805, "growth_memory": 1.3664002418518066, "output_influence": 0.40051180124282837, "residual_influence": 0.13207221031188965, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.235111951828003, "no_memory_loss_delta": 0.5213523507118225, "scrambled_loss_delta": 0.19525697827339172, "memory_only_loss_delta": 3.0668275356292725, "no_evidence_loss_delta": 0.3080368638038635, "no_agreement_loss_delta": 0.02833499014377594, "no_orthogonal_loss_delta": 0.012646124698221684}, {"index": 94, "tokens": 256, "normal_loss": 3.2455837726593018, "no_memory_loss_delta": 0.5740649104118347, "scrambled_loss_delta": 0.36920708417892456, "memory_only_loss_delta": 3.128716230392456, "no_evidence_loss_delta": 0.2694898545742035, "no_agreement_loss_delta": 0.03372187912464142, "no_orthogonal_loss_delta": 0.04950662702322006}, {"index": 188, "tokens": 256, "normal_loss": 3.4937214851379395, "no_memory_loss_delta": 0.6180020570755005, "scrambled_loss_delta": 0.15237173438072205, "memory_only_loss_delta": 2.8849244117736816, "no_evidence_loss_delta": 0.30928826332092285, "no_agreement_loss_delta": 0.04245582967996597, "no_orthogonal_loss_delta": 0.03008488565683365}, {"index": 283, "tokens": 131, "normal_loss": 2.021127223968506, "no_memory_loss_delta": 0.4470009207725525, "scrambled_loss_delta": 0.27735137939453125, "memory_only_loss_delta": 2.78334903717041, "no_evidence_loss_delta": 0.185397669672966, "no_agreement_loss_delta": 0.0033793249167501926, "no_orthogonal_loss_delta": 0.040358759462833405}], "agreement_weight_abs_mean": 0.18606780469417572, "agreement_weight_abs_max": 4.757392406463623, "agreement_weight_mean": 0.16284170746803284, "orthogonal_weight_abs_mean": 0.7797583937644958, "orthogonal_weight_abs_max": 7.631902694702148, "orthogonal_weight_mean": -0.13358831405639648, "no_memory_loss": 3.687887191772461, "no_memory_loss_delta": 0.5530506372451782, "no_memory_token_accuracy_gain": 0.0856507197022438, "no_memory_token_win_fraction": 0.6618465185165405, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 42.48075722682706, "scrambled_loss": 3.3793787956237793, "scrambled_loss_delta": 0.244541734457016, "scrambled_token_accuracy_gain": 0.02892102301120758, "scrambled_token_win_fraction": 0.6307007670402527, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 21.6936693038526, "memory_only_loss": 6.126180648803711, "memory_only_loss_delta": 2.9913442134857178, "memory_only_token_accuracy_gain": 0.27252504229545593, "memory_only_token_win_fraction": 0.9032257795333862, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.97801149149308, "no_evidence_loss": 3.4143826961517334, "no_evidence_loss_delta": 0.2795458734035492, "no_evidence_token_accuracy_gain": 0.03559510409832001, "no_evidence_token_win_fraction": 0.6151279211044312, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 24.38729591013105, "no_agreement_loss": 3.165090560913086, "no_agreement_loss_delta": 0.03025355003774166, "no_agreement_token_accuracy_gain": 0.006674082484096289, "no_agreement_token_win_fraction": 0.486095666885376, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.980049176196786, "no_orthogonal_loss": 3.1669838428497314, "no_orthogonal_loss_delta": 0.0321466438472271, "no_orthogonal_token_accuracy_gain": 0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5661846399307251, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 3.163543303898713, "short_no_memory_loss_delta": 0.4745493233203888, "long_context_memory_gain": 0.07850131392478943, "normal_context_loss_gain": 0.15113650262355804, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.450438976287842, "contribution_to_active": 1.1355781555175781, "mix_mean": 0.1051013246178627, "mix_low": 0.5412225723266602, "mix_high": 0.005086421966552734, "read_cosine": 0.43535885214805603, "beta_mean": 0.23394499719142914, "nonfinite": 0.0, "weighted_memory_to_active": 1.1841555833816528, "mix_std": 0.0898524820804596, "beta_std": 0.14655713737010956, "beta_low": 0.0415547713637352, "beta_high": 0.0, "orthogonal_to_active": 0.05398936569690704, "output_influence": 0.19347450137138367, "residual_influence": 0.23373855650424957, "state_active": 0.06464441120624542, "state_memory": 0.04569326713681221, "state_peak": 43.95194625854492, "growth_active": 1.2034937143325806, "growth_memory": 1.263848066329956, "heads": [{"head": 0, "memory_to_active": 2.6183834075927734, "contribution_to_active": 0.4726974070072174, "mix_mean": 0.019662700593471527, "mix_low": 0.6098403930664062, "mix_high": 0.0, "read_cosine": 0.22274696826934814, "beta_mean": 0.20861975848674774, "nonfinite": 0.0, "weighted_memory_to_active": 0.5725648999214172, "mix_std": 0.05974292755126953, "beta_std": 0.10639669001102448, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.049507901072502136, "evidence_weights": [-0.016138674691319466, -0.06156984716653824, -0.10886429995298386], "agreement_weight": -0.15859197080135345, "orthogonal_weight": -0.09427172690629959}, {"head": 1, "memory_to_active": 6.278892517089844, "contribution_to_active": 2.517164945602417, "mix_mean": 0.01634906604886055, "mix_low": 0.6747426986694336, "mix_high": 0.0, "read_cosine": 0.48106083273887634, "beta_mean": 0.4367108643054962, "nonfinite": 0.0, "weighted_memory_to_active": 2.821279525756836, "mix_std": 0.05293631926178932, "beta_std": 0.20676428079605103, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03962159901857376, "evidence_weights": [0.02376697212457657, -0.10490557551383972, 0.47056180238723755], "agreement_weight": 0.029244938865303993, "orthogonal_weight": -0.029848525300621986}, {"head": 2, "memory_to_active": 2.4739465713500977, "contribution_to_active": 0.4697161614894867, "mix_mean": 0.01528118271380663, "mix_low": 0.6451053619384766, "mix_high": 0.0, "read_cosine": 0.28785011172294617, "beta_mean": 0.26642197370529175, "nonfinite": 0.0, "weighted_memory_to_active": 0.5546160340309143, "mix_std": 0.03464619815349579, "beta_std": 0.16640102863311768, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.053343452513217926, "evidence_weights": [0.055249545723199844, 0.05998552218079567, 0.0685960054397583], "agreement_weight": 0.005574952345341444, "orthogonal_weight": -0.1981564164161682}, {"head": 3, "memory_to_active": 6.938331127166748, "contribution_to_active": 2.8342669010162354, "mix_mean": 0.034258875995874405, "mix_low": 0.5132131576538086, "mix_high": 0.0, "read_cosine": 0.34039920568466187, "beta_mean": 0.3547738492488861, "nonfinite": 0.0, "weighted_memory_to_active": 3.0406033992767334, "mix_std": 0.08231647312641144, "beta_std": 0.2578798532485962, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.15595586597919464, "evidence_weights": [-0.04211146757006645, -0.04966132342815399, -0.1566305160522461], "agreement_weight": -0.008774873800575733, "orthogonal_weight": -0.06121279299259186}, {"head": 4, "memory_to_active": 2.2816083431243896, "contribution_to_active": 0.08351733535528183, "mix_mean": 0.007844346575438976, "mix_low": 0.7881708145141602, "mix_high": 0.0, "read_cosine": 0.3397676348686218, "beta_mean": 0.11219590157270432, "nonfinite": 0.0, "weighted_memory_to_active": 0.060598500072956085, "mix_std": 0.017765387892723083, "beta_std": 0.12316875159740448, "beta_low": 0.01190185546875, "beta_high": 0.0, "orthogonal_to_active": 0.007445854600518942, "evidence_weights": [0.131156325340271, 0.08549480140209198, -0.2884120047092438], "agreement_weight": -0.11361262947320938, "orthogonal_weight": 0.10990765690803528}, {"head": 5, "memory_to_active": 0.11147040128707886, "contribution_to_active": 0.4361061453819275, "mix_mean": 0.5372116565704346, "mix_low": 0.01626300811767578, "mix_high": 0.030518531799316406, "read_cosine": 0.9403283596038818, "beta_mean": 0.024947527796030045, "nonfinite": 0.0, "weighted_memory_to_active": 0.05527110770344734, "mix_std": 0.29170751571655273, "beta_std": 0.018732214346528053, "beta_low": 0.2374267578125, "beta_high": 0.0, "orthogonal_to_active": 0.01806151680648327, "evidence_weights": [-0.20268253982067108, -0.22930952906608582, 0.3540743291378021], "agreement_weight": 0.2850372791290283, "orthogonal_weight": -0.696774423122406}]}, {"layer": 1, "memory_to_active": 2.2710046768188477, "contribution_to_active": 0.5189982652664185, "mix_mean": 0.20106245577335358, "mix_low": 0.3532085418701172, "mix_high": 0.0010315576801076531, "read_cosine": 0.3232092261314392, "beta_mean": 0.17877113819122314, "nonfinite": 0.0, "weighted_memory_to_active": 0.36554384231567383, "mix_std": 0.11820770055055618, "beta_std": 0.13201522827148438, "beta_low": 0.3100993037223816, "beta_high": 0.00010172526526730508, "orthogonal_to_active": 0.05473041906952858, "output_influence": 0.22582325339317322, "residual_influence": 0.05825494974851608, "state_active": 0.03817950189113617, "state_memory": 0.035622186958789825, "state_peak": 35.958152770996094, "growth_active": 1.4260262250900269, "growth_memory": 1.4870731830596924, "heads": [{"head": 0, "memory_to_active": 0.03935631364583969, "contribution_to_active": 0.4757423400878906, "mix_mean": 0.4785532057285309, "mix_low": 0.008004188537597656, "mix_high": 0.0018482208251953125, "read_cosine": 0.04116356745362282, "beta_mean": 0.0002818881766870618, "nonfinite": 0.0, "weighted_memory_to_active": 0.022539960220456123, "mix_std": 0.2850289046764374, "beta_std": 0.00038564138230867684, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.022194107994437218, "evidence_weights": [-0.19745977222919464, -0.2581869065761566, 0.32872846722602844], "agreement_weight": 0.19685199856758118, "orthogonal_weight": -1.0481970310211182}, {"head": 1, "memory_to_active": 2.940444231033325, "contribution_to_active": 0.2725968360900879, "mix_mean": 0.02432241290807724, "mix_low": 0.4397878646850586, "mix_high": 5.7220458984375e-06, "read_cosine": 0.5084961652755737, "beta_mean": 0.2748108506202698, "nonfinite": 0.0, "weighted_memory_to_active": 0.2972952723503113, "mix_std": 0.047199785709381104, "beta_std": 0.1884319931268692, "beta_low": 0.00042724609375, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.01893453672528267, "evidence_weights": [0.014216107316315174, -0.04584649205207825, 0.021897248923778534], "agreement_weight": -0.008075280115008354, "orthogonal_weight": 0.1498005986213684}, {"head": 2, "memory_to_active": 0.13120469450950623, "contribution_to_active": 0.5783727765083313, "mix_mean": 0.6369687914848328, "mix_low": 0.0009508132934570312, "mix_high": 0.004329681396484375, "read_cosine": 0.6241014003753662, "beta_mean": 0.007911084219813347, "nonfinite": 0.0, "weighted_memory_to_active": 0.10334348678588867, "mix_std": 0.23812419176101685, "beta_std": 0.010886119678616524, "beta_low": 0.74945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.04631217196583748, "evidence_weights": [-0.29637378454208374, -0.3467065691947937, 0.5828176140785217], "agreement_weight": 0.025674548000097275, "orthogonal_weight": -0.6184508800506592}, {"head": 3, "memory_to_active": 2.9994332790374756, "contribution_to_active": 0.27675777673721313, "mix_mean": 0.02078329771757126, "mix_low": 0.5975513458251953, "mix_high": 0.0, "read_cosine": 0.21662184596061707, "beta_mean": 0.3201945126056671, "nonfinite": 0.0, "weighted_memory_to_active": 0.4019199013710022, "mix_std": 0.057341791689395905, "beta_std": 0.25023919343948364, "beta_low": 0.05206298828125, "beta_high": 0.0, "orthogonal_to_active": 0.09678001701831818, "evidence_weights": [0.04290751740336418, -0.008692240342497826, 0.10964594036340714], "agreement_weight": -0.030034661293029785, "orthogonal_weight": -0.3939785957336426}, {"head": 4, "memory_to_active": 3.782069683074951, "contribution_to_active": 1.4364707469940186, "mix_mean": 0.03817838430404663, "mix_low": 0.26570796966552734, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3100148141384125, "beta_mean": 0.17768150568008423, "nonfinite": 0.0, "weighted_memory_to_active": 1.3124626874923706, "mix_std": 0.06234709173440933, "beta_std": 0.12021466344594955, "beta_low": 0.0284423828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.13218027353286743, "evidence_weights": [-0.042820803821086884, -0.005212375428527594, 0.06258651614189148], "agreement_weight": -0.022032996639609337, "orthogonal_weight": 0.13391844928264618}, {"head": 5, "memory_to_active": 3.7335193157196045, "contribution_to_active": 0.07404915988445282, "mix_mean": 0.007568575907498598, "mix_low": 0.8072490692138672, "mix_high": 4.76837158203125e-06, "read_cosine": 0.23885752260684967, "beta_mean": 0.29174691438674927, "nonfinite": 0.0, "weighted_memory_to_active": 0.055701836943626404, "mix_std": 0.01920442283153534, "beta_std": 0.22193367779254913, "beta_low": 0.03021240234375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.011981393210589886, "evidence_weights": [0.1587551087141037, 0.07588845491409302, 0.2753804624080658], "agreement_weight": -0.12941136956214905, "orthogonal_weight": 0.20329737663269043}]}, {"layer": 2, "memory_to_active": 1.0578835010528564, "contribution_to_active": 0.35154038667678833, "mix_mean": 0.38394343852996826, "mix_low": 0.2460087239742279, "mix_high": 0.004541873931884766, "read_cosine": 0.33000802993774414, "beta_mean": 0.09255363047122955, "nonfinite": 0.0, "weighted_memory_to_active": 0.07033897936344147, "mix_std": 0.19291406869888306, "beta_std": 0.07289361208677292, "beta_low": 0.5560404658317566, "beta_high": 7.120767986634746e-05, "orthogonal_to_active": 0.018743330612778664, "output_influence": 0.3576158583164215, "residual_influence": 0.09228719770908356, "state_active": 0.037371061742305756, "state_memory": 0.016358181834220886, "state_peak": 30.715709686279297, "growth_active": 1.024887204170227, "growth_memory": 1.5418486595153809, "heads": [{"head": 0, "memory_to_active": 0.00988011434674263, "contribution_to_active": 0.3658295273780823, "mix_mean": 0.6002600193023682, "mix_low": 0.0011043548583984375, "mix_high": 0.008679389953613281, "read_cosine": 0.15455111861228943, "beta_mean": 7.158080552471802e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005991414189338684, "mix_std": 0.28380286693573, "beta_std": 0.00013901828788220882, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004995736293494701, "evidence_weights": [-0.3469557762145996, -0.2100234031677246, 0.11747469753026962], "agreement_weight": 0.24601896107196808, "orthogonal_weight": -0.8681612014770508}, {"head": 1, "memory_to_active": 3.0081348419189453, "contribution_to_active": 0.050274357199668884, "mix_mean": 0.005639123730361462, "mix_low": 0.8790826797485352, "mix_high": 0.0, "read_cosine": 0.18799294531345367, "beta_mean": 0.2680307626724243, "nonfinite": 0.0, "weighted_memory_to_active": 0.03637244179844856, "mix_std": 0.014947004616260529, "beta_std": 0.16485896706581116, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.01574970781803131, "evidence_weights": [0.12419585883617401, 0.10931525379419327, 0.5538724660873413], "agreement_weight": 0.03783976659178734, "orthogonal_weight": 0.41793638467788696}, {"head": 2, "memory_to_active": 0.010604266077280045, "contribution_to_active": 0.42064741253852844, "mix_mean": 0.5132899284362793, "mix_low": 0.001495361328125, "mix_high": 0.0024557113647460938, "read_cosine": 0.3316652476787567, "beta_mean": 2.350994327571243e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0043870857916772366, "mix_std": 0.27863845229148865, "beta_std": 4.173045090283267e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000715067028068006, "evidence_weights": [-0.24250417947769165, -0.28133630752563477, 0.47989392280578613], "agreement_weight": 0.021467287093400955, "orthogonal_weight": -0.2071501463651657}, {"head": 3, "memory_to_active": 0.16584962606430054, "contribution_to_active": 0.5207963585853577, "mix_mean": 0.5346412658691406, "mix_low": 0.00029659271240234375, "mix_high": 0.0072536468505859375, "read_cosine": 0.655580997467041, "beta_mean": 0.03303144499659538, "nonfinite": 0.0, "weighted_memory_to_active": 0.09663787484169006, "mix_std": 0.2639712691307068, "beta_std": 0.04395352676510811, "beta_low": 0.30938720703125, "beta_high": 0.0, "orthogonal_to_active": 0.062267497181892395, "evidence_weights": [-0.1905483603477478, -0.2986603379249573, 0.8497638702392578], "agreement_weight": 0.0537789985537529, "orthogonal_weight": -0.8916550278663635}, {"head": 4, "memory_to_active": 3.137578010559082, "contribution_to_active": 0.2139110118150711, "mix_mean": 0.01881387084722519, "mix_low": 0.5929670333862305, "mix_high": 0.0, "read_cosine": 0.19583427906036377, "beta_mean": 0.25412115454673767, "nonfinite": 0.0, "weighted_memory_to_active": 0.26759564876556396, "mix_std": 0.04751981794834137, "beta_std": 0.22828897833824158, "beta_low": 0.026611328125, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.028636163100600243, "evidence_weights": [0.018922945484519005, 0.006958947516977787, 0.03314613178372383], "agreement_weight": 0.01671668328344822, "orthogonal_weight": -0.1161220446228981}, {"head": 5, "memory_to_active": 0.015253654681146145, "contribution_to_active": 0.5377835035324097, "mix_mean": 0.6310161352157593, "mix_low": 0.00110626220703125, "mix_high": 0.008862495422363281, "read_cosine": 0.45442354679107666, "beta_mean": 4.330721276346594e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011049375869333744, "mix_std": 0.26860499382019043, "beta_std": 7.94399602455087e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 9.580658661434427e-05, "evidence_weights": [-0.27551522850990295, -0.2891286015510559, 0.9621917009353638], "agreement_weight": 0.13008122146129608, "orthogonal_weight": 0.014861363917589188}]}, {"layer": 3, "memory_to_active": 0.5394302606582642, "contribution_to_active": 0.4417083263397217, "mix_mean": 0.43319302797317505, "mix_low": 0.06468900293111801, "mix_high": 0.008965810760855675, "read_cosine": 0.21132782101631165, "beta_mean": 0.030097410082817078, "nonfinite": 0.0, "weighted_memory_to_active": 0.1430920958518982, "mix_std": 0.2615835666656494, "beta_std": 0.02865273877978325, "beta_low": 0.7022705078125, "beta_high": 0.0, "orthogonal_to_active": 0.06104974448680878, "output_influence": 0.3786764144897461, "residual_influence": 0.0819351077079773, "state_active": 0.023861223831772804, "state_memory": 0.007929747924208641, "state_peak": 20.09530258178711, "growth_active": 0.9974650740623474, "growth_memory": 1.793946623802185, "heads": [{"head": 0, "memory_to_active": 0.44013315439224243, "contribution_to_active": 0.5555051565170288, "mix_mean": 0.5283191204071045, "mix_low": 0.0051059722900390625, "mix_high": 0.016902923583984375, "read_cosine": 0.18327027559280396, "beta_mean": 0.013712333515286446, "nonfinite": 0.0, "weighted_memory_to_active": 0.25386232137680054, "mix_std": 0.31569504737854004, "beta_std": 0.02723674476146698, "beta_low": 0.60675048828125, "beta_high": 0.0, "orthogonal_to_active": 0.1778685450553894, "evidence_weights": [-0.20119789242744446, -0.23356807231903076, 0.8373453617095947], "agreement_weight": 0.11331384629011154, "orthogonal_weight": -0.7294538021087646}, {"head": 1, "memory_to_active": 0.018203752115368843, "contribution_to_active": 0.4043160080909729, "mix_mean": 0.5035048723220825, "mix_low": 0.008635520935058594, "mix_high": 0.0072879791259765625, "read_cosine": 0.19108016788959503, "beta_mean": 0.00010105258115800098, "nonfinite": 0.0, "weighted_memory_to_active": 0.009339148178696632, "mix_std": 0.3082646429538727, "beta_std": 0.0002392198657616973, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.006800998002290726, "evidence_weights": [-0.23878256976604462, -0.19872787594795227, 0.6030660271644592], "agreement_weight": 0.4155700206756592, "orthogonal_weight": -0.8027094006538391}, {"head": 2, "memory_to_active": 2.420182943344116, "contribution_to_active": 0.31428030133247375, "mix_mean": 0.04914335161447525, "mix_low": 0.3154106140136719, "mix_high": 0.0, "read_cosine": 0.2178138792514801, "beta_mean": 0.15464769303798676, "nonfinite": 0.0, "weighted_memory_to_active": 0.34995153546333313, "mix_std": 0.10005167126655579, "beta_std": 0.130133718252182, "beta_low": 0.01678466796875, "beta_high": 0.0, "orthogonal_to_active": 0.0008033741032704711, "evidence_weights": [-0.08572431653738022, 0.008239616639912128, -0.002333021489903331], "agreement_weight": -0.017201170325279236, "orthogonal_weight": -0.0028379657305777073}, {"head": 3, "memory_to_active": 0.03232870250940323, "contribution_to_active": 0.5115307569503784, "mix_mean": 0.5788072347640991, "mix_low": 0.007967948913574219, "mix_high": 0.013939857482910156, "read_cosine": 0.30661293864250183, "beta_mean": 0.00012497206625994295, "nonfinite": 0.0, "weighted_memory_to_active": 0.021452315151691437, "mix_std": 0.297981858253479, "beta_std": 0.00033471244387328625, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0052787428721785545, "evidence_weights": [-0.2334461659193039, -0.22992850840091705, 2.1215057373046875], "agreement_weight": 1.7506053447723389, "orthogonal_weight": -0.3306351900100708}, {"head": 4, "memory_to_active": 0.05826320871710777, "contribution_to_active": 0.22776511311531067, "mix_mean": 0.272171288728714, "mix_low": 0.047163963317871094, "mix_high": 0.00054168701171875, "read_cosine": 0.1917761266231537, "beta_mean": 0.0004027383401989937, "nonfinite": 0.0, "weighted_memory_to_active": 0.02535175532102585, "mix_std": 0.26876312494277954, "beta_std": 0.0005773932207375765, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.023693837225437164, "evidence_weights": [-0.1036958172917366, -0.13852059841156006, 0.174025297164917], "agreement_weight": -0.01937473937869072, "orthogonal_weight": -0.9846564531326294}, {"head": 5, "memory_to_active": 0.26746970415115356, "contribution_to_active": 0.636852502822876, "mix_mean": 0.6672121286392212, "mix_low": 0.0038499832153320312, "mix_high": 0.015122413635253906, "read_cosine": 0.1774134337902069, "beta_mean": 0.011595668271183968, "nonfinite": 0.0, "weighted_memory_to_active": 0.19859540462493896, "mix_std": 0.27874505519866943, "beta_std": 0.013394635170698166, "beta_low": 0.59033203125, "beta_high": 0.0, "orthogonal_to_active": 0.15185296535491943, "evidence_weights": [-0.25091663002967834, -0.29802441596984863, 0.5027328133583069], "agreement_weight": 0.12990403175354004, "orthogonal_weight": -0.8040235638618469}]}, {"layer": 4, "memory_to_active": 1.1625189781188965, "contribution_to_active": 0.47628724575042725, "mix_mean": 0.3551512360572815, "mix_low": 0.274697482585907, "mix_high": 0.007086118217557669, "read_cosine": 0.3350670337677002, "beta_mean": 0.11801251769065857, "nonfinite": 0.0, "weighted_memory_to_active": 0.16172003746032715, "mix_std": 0.15555773675441742, "beta_std": 0.10832740366458893, "beta_low": 0.3711954951286316, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05807812139391899, "output_influence": 0.4200226664543152, "residual_influence": 0.1137007623910904, "state_active": 0.03823542222380638, "state_memory": 0.019470252096652985, "state_peak": 38.68305587768555, "growth_active": 1.2415187358856201, "growth_memory": 1.5470777750015259, "heads": [{"head": 0, "memory_to_active": 0.2166370153427124, "contribution_to_active": 0.7110767960548401, "mix_mean": 0.6619422435760498, "mix_low": 0.011762619018554688, "mix_high": 0.016096115112304688, "read_cosine": 0.5237897038459778, "beta_mean": 0.25972867012023926, "nonfinite": 0.0, "weighted_memory_to_active": 0.16171783208847046, "mix_std": 0.3413504660129547, "beta_std": 0.26328903436660767, "beta_low": 0.20513916015625, "beta_high": 0.0, "orthogonal_to_active": 0.06119060888886452, "evidence_weights": [-0.26776766777038574, -0.36710959672927856, 0.1325090378522873], "agreement_weight": 0.006293855607509613, "orthogonal_weight": 0.4835657477378845}, {"head": 1, "memory_to_active": 0.08006825298070908, "contribution_to_active": 0.6229328513145447, "mix_mean": 0.6700361967086792, "mix_low": 0.00010395050048828125, "mix_high": 0.011107444763183594, "read_cosine": 0.6268303394317627, "beta_mean": 0.0013854217249900103, "nonfinite": 0.0, "weighted_memory_to_active": 0.057605866342782974, "mix_std": 0.24197068810462952, "beta_std": 0.0070986030623316765, "beta_low": 0.97479248046875, "beta_high": 0.0, "orthogonal_to_active": 0.033678799867630005, "evidence_weights": [-0.23999284207820892, -0.34637942910194397, 0.7149388194084167], "agreement_weight": 0.21063163876533508, "orthogonal_weight": -0.8103567361831665}, {"head": 2, "memory_to_active": 2.091304302215576, "contribution_to_active": 0.2414160668849945, "mix_mean": 0.02174878492951393, "mix_low": 0.5665721893310547, "mix_high": 1.811981201171875e-05, "read_cosine": 0.24197401106357574, "beta_mean": 0.09520092606544495, "nonfinite": 0.0, "weighted_memory_to_active": 0.2280714511871338, "mix_std": 0.04703843966126442, "beta_std": 0.08411329239606857, "beta_low": 0.0181884765625, "beta_high": 0.0, "orthogonal_to_active": 0.026098527014255524, "evidence_weights": [0.056811779737472534, 0.021386059001088142, 0.03710819408297539], "agreement_weight": 0.14008720219135284, "orthogonal_weight": -0.16644611954689026}, {"head": 3, "memory_to_active": 2.1053171157836914, "contribution_to_active": 0.14476388692855835, "mix_mean": 0.030169161036610603, "mix_low": 0.4338703155517578, "mix_high": 0.0, "read_cosine": 0.2121886909008026, "beta_mean": 0.1364975869655609, "nonfinite": 0.0, "weighted_memory_to_active": 0.24474194645881653, "mix_std": 0.054249461740255356, "beta_std": 0.12231763452291489, "beta_low": 0.02886962890625, "beta_high": 0.0, "orthogonal_to_active": 0.007620543707162142, "evidence_weights": [-0.023010864853858948, 0.06017220765352249, -0.23356691002845764], "agreement_weight": 0.07449633628129959, "orthogonal_weight": -0.05169151350855827}, {"head": 4, "memory_to_active": 0.012245487421751022, "contribution_to_active": 0.6398998498916626, "mix_mean": 0.7328042984008789, "mix_low": 0.0002002716064453125, "mix_high": 0.01528167724609375, "read_cosine": 0.1527237594127655, "beta_mean": 2.5181852834066376e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008951367810368538, "mix_std": 0.21907787024974823, "beta_std": 7.775084668537602e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028185592964291573, "evidence_weights": [-0.3331882059574127, -0.3306456208229065, 0.6171988248825073], "agreement_weight": 0.31330013275146484, "orthogonal_weight": -0.3412567675113678}, {"head": 5, "memory_to_active": 2.469541072845459, "contribution_to_active": 0.4976338744163513, "mix_mean": 0.01420679036527872, "mix_low": 0.6356754302978516, "mix_high": 1.33514404296875e-05, "read_cosine": 0.252895712852478, "beta_mean": 0.21523728966712952, "nonfinite": 0.0, "weighted_memory_to_active": 0.26923173666000366, "mix_std": 0.029659409075975418, "beta_std": 0.17306813597679138, "beta_low": 0.00018310546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.21706166863441467, "evidence_weights": [0.04639432579278946, -0.03437769412994385, -0.2895881235599518], "agreement_weight": -0.07437702268362045, "orthogonal_weight": 0.897939920425415}]}, {"layer": 5, "memory_to_active": 0.4614752531051636, "contribution_to_active": 0.503575325012207, "mix_mean": 0.47603124380111694, "mix_low": 0.06354920566082001, "mix_high": 0.01491673942655325, "read_cosine": 0.08036467432975769, "beta_mean": 0.054868027567863464, "nonfinite": 0.0, "weighted_memory_to_active": 0.14745041728019714, "mix_std": 0.2542869448661804, "beta_std": 0.07130416482686996, "beta_low": 0.6462911367416382, "beta_high": 0.0, "orthogonal_to_active": 0.05521037057042122, "output_influence": 0.5160098671913147, "residual_influence": 0.13980968296527863, "state_active": 0.0426318421959877, "state_memory": 0.011007965542376041, "state_peak": 61.65814971923828, "growth_active": 1.0823585987091064, "growth_memory": 1.2286101579666138, "heads": [{"head": 0, "memory_to_active": 0.4913873076438904, "contribution_to_active": 0.4581086337566376, "mix_mean": 0.32491910457611084, "mix_low": 0.02814960479736328, "mix_high": 0.0006837844848632812, "read_cosine": -0.12365816533565521, "beta_mean": 0.026468368247151375, "nonfinite": 0.0, "weighted_memory_to_active": 0.13406112790107727, "mix_std": 0.2620052099227905, "beta_std": 0.04648138955235481, "beta_low": 0.4302978515625, "beta_high": 0.0, "orthogonal_to_active": 0.03929588198661804, "evidence_weights": [-0.1190037801861763, -0.14973251521587372, -0.11613758653402328], "agreement_weight": -0.022308286279439926, "orthogonal_weight": -0.29342004656791687}, {"head": 1, "memory_to_active": 0.01727363094687462, "contribution_to_active": 0.5471775531768799, "mix_mean": 0.6782422065734863, "mix_low": 0.0020008087158203125, "mix_high": 0.03061389923095703, "read_cosine": 0.19990286231040955, "beta_mean": 4.1460152715444565e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00987678486853838, "mix_std": 0.2819339632987976, "beta_std": 7.995367923285812e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00930686853826046, "evidence_weights": [-0.24554632604122162, -0.28299251198768616, 0.8609561324119568], "agreement_weight": 0.600981593132019, "orthogonal_weight": -0.9805156588554382}, {"head": 2, "memory_to_active": 0.0101310508325696, "contribution_to_active": 0.5341951251029968, "mix_mean": 0.6445934176445007, "mix_low": 0.0037994384765625, "mix_high": 0.017377853393554688, "read_cosine": -0.45084404945373535, "beta_mean": 7.704336167080328e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006485980935394764, "mix_std": 0.27364498376846313, "beta_std": 0.00013110223517287523, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018618659814819694, "evidence_weights": [-0.2580137550830841, -0.3452077805995941, 0.6186972856521606], "agreement_weight": 0.00017111559282056987, "orthogonal_weight": -0.3570455312728882}, {"head": 3, "memory_to_active": 1.704115390777588, "contribution_to_active": 0.22320064902305603, "mix_mean": 0.09624887257814407, "mix_low": 0.15239620208740234, "mix_high": 2.86102294921875e-06, "read_cosine": 0.1383826732635498, "beta_mean": 0.1105397567152977, "nonfinite": 0.0, "weighted_memory_to_active": 0.36278781294822693, "mix_std": 0.13129377365112305, "beta_std": 0.1561489850282669, "beta_low": 0.18310546875, "beta_high": 0.0, "orthogonal_to_active": 0.13853353261947632, "evidence_weights": [-0.0630430281162262, -0.09154605865478516, 0.014034254476428032], "agreement_weight": 0.06443072110414505, "orthogonal_weight": -0.471439391374588}, {"head": 4, "memory_to_active": 0.4354068338871002, "contribution_to_active": 0.5997369289398193, "mix_mean": 0.32802605628967285, "mix_low": 0.19219684600830078, "mix_high": 0.007166862487792969, "read_cosine": 0.5503578186035156, "beta_mean": 0.19171375036239624, "nonfinite": 0.0, "weighted_memory_to_active": 0.2954047918319702, "mix_std": 0.34361112117767334, "beta_std": 0.22331343591213226, "beta_low": 0.270263671875, "beta_high": 0.0, "orthogonal_to_active": 0.0932474434375763, "evidence_weights": [-0.11153280735015869, -0.2766639292240143, -0.07035823166370392], "agreement_weight": 0.047271110117435455, "orthogonal_weight": 0.4872349202632904}, {"head": 5, "memory_to_active": 0.11053730547428131, "contribution_to_active": 0.6590330004692078, "mix_mean": 0.7841576933860779, "mix_low": 0.0027523040771484375, "mix_high": 0.03365516662597656, "read_cosine": 0.16804687678813934, "beta_mean": 0.0003677660715766251, "nonfinite": 0.0, "weighted_memory_to_active": 0.07608599215745926, "mix_std": 0.23323258757591248, "beta_std": 0.001670122961513698, "beta_low": 0.99407958984375, "beta_high": 0.0, "orthogonal_to_active": 0.04901660233736038, "evidence_weights": [-0.31810224056243896, -0.3291700482368469, 1.9699798822402954], "agreement_weight": 0.435660183429718, "orthogonal_weight": -0.6759659647941589}]}, {"layer": 6, "memory_to_active": 0.3533781170845032, "contribution_to_active": 0.6190484762191772, "mix_mean": 0.5737960934638977, "mix_low": 0.05972910299897194, "mix_high": 0.03220780938863754, "read_cosine": 0.2682603895664215, "beta_mean": 0.0684656947851181, "nonfinite": 0.0, "weighted_memory_to_active": 0.17011339962482452, "mix_std": 0.2359563410282135, "beta_std": 0.07624323666095734, "beta_low": 0.4950154721736908, "beta_high": 0.0, "orthogonal_to_active": 0.04019560664892197, "output_influence": 0.5346755981445312, "residual_influence": 0.17171378433704376, "state_active": 0.04892846196889877, "state_memory": 0.009203501977026463, "state_peak": 98.24494934082031, "growth_active": 1.206594705581665, "growth_memory": 1.2061712741851807, "heads": [{"head": 0, "memory_to_active": 0.12618210911750793, "contribution_to_active": 0.6989412307739258, "mix_mean": 0.7811951041221619, "mix_low": 0.00020694732666015625, "mix_high": 0.02523517608642578, "read_cosine": -0.0011668670922517776, "beta_mean": 0.0188822653144598, "nonfinite": 0.0, "weighted_memory_to_active": 0.1021454781293869, "mix_std": 0.21051213145256042, "beta_std": 0.02334897592663765, "beta_low": 0.4549560546875, "beta_high": 0.0, "orthogonal_to_active": 0.08706638216972351, "evidence_weights": [-0.32055726647377014, -0.3425261378288269, 1.00845468044281], "agreement_weight": 0.30863407254219055, "orthogonal_weight": -0.8848234415054321}, {"head": 1, "memory_to_active": 0.006026884075254202, "contribution_to_active": 0.5043392777442932, "mix_mean": 0.5554909110069275, "mix_low": 0.0013523101806640625, "mix_high": 0.011481285095214844, "read_cosine": 0.5251463651657104, "beta_mean": 1.1896858268301003e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0026766424998641014, "mix_std": 0.2847522795200348, "beta_std": 2.1148518499103375e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001471569063141942, "evidence_weights": [-0.20019251108169556, -0.2961389124393463, 1.0960723161697388], "agreement_weight": 0.050758909434080124, "orthogonal_weight": -0.7575363516807556}, {"head": 2, "memory_to_active": 1.043636679649353, "contribution_to_active": 0.09000667929649353, "mix_mean": 0.0350385420024395, "mix_low": 0.34879302978515625, "mix_high": 0.0, "read_cosine": 0.23138728737831116, "beta_mean": 0.04516417533159256, "nonfinite": 0.0, "weighted_memory_to_active": 0.06893151998519897, "mix_std": 0.056659385561943054, "beta_std": 0.04124407842755318, "beta_low": 0.04730224609375, "beta_high": 0.0, "orthogonal_to_active": 0.00134675158187747, "evidence_weights": [0.08022676408290863, -0.028415320441126823, 0.2697755694389343], "agreement_weight": 0.13556143641471863, "orthogonal_weight": -0.022943414747714996}, {"head": 3, "memory_to_active": 0.1432361602783203, "contribution_to_active": 0.6711785793304443, "mix_mean": 0.654381513595581, "mix_low": 0.006659507751464844, "mix_high": 0.01974010467529297, "read_cosine": 0.44994765520095825, "beta_mean": 0.21585184335708618, "nonfinite": 0.0, "weighted_memory_to_active": 0.10413897037506104, "mix_std": 0.3229944705963135, "beta_std": 0.2450067102909088, "beta_low": 0.20355224609375, "beta_high": 0.0, "orthogonal_to_active": 0.05751659721136093, "evidence_weights": [-0.17518799006938934, -0.33101192116737366, 0.19370917975902557], "agreement_weight": -0.005830632522702217, "orthogonal_weight": 0.6332122087478638}, {"head": 4, "memory_to_active": 0.7660074830055237, "contribution_to_active": 1.1349223852157593, "mix_mean": 0.7642343044281006, "mix_low": 0.0010194778442382812, "mix_high": 0.13211345672607422, "read_cosine": 0.12578555941581726, "beta_mean": 0.13065999746322632, "nonfinite": 0.0, "weighted_memory_to_active": 0.7175582647323608, "mix_std": 0.28493642807006836, "beta_std": 0.1473381072282791, "beta_low": 0.2645263671875, "beta_high": 0.0, "orthogonal_to_active": 0.06893816590309143, "evidence_weights": [-0.24387826025485992, -0.5966075658798218, -0.2376241534948349], "agreement_weight": -0.02042090892791748, "orthogonal_weight": 0.11357458680868149}, {"head": 5, "memory_to_active": 0.03517916053533554, "contribution_to_active": 0.6149023771286011, "mix_mean": 0.6524361371994019, "mix_low": 0.00034332275390625, "mix_high": 0.00467681884765625, "read_cosine": 0.2784622609615326, "beta_mean": 0.00022400770103558898, "nonfinite": 0.0, "weighted_memory_to_active": 0.02522948756814003, "mix_std": 0.2558833360671997, "beta_std": 0.0005003910628147423, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.024834180250763893, "evidence_weights": [-0.24316027760505676, -0.3124946057796478, 0.4344199299812317], "agreement_weight": 0.1611059159040451, "orthogonal_weight": -1.0572166442871094}]}, {"layer": 7, "memory_to_active": 1.7107880115509033, "contribution_to_active": 0.25203144550323486, "mix_mean": 0.13090330362319946, "mix_low": 0.26591938734054565, "mix_high": 0.0005679130554199219, "read_cosine": 0.22504815459251404, "beta_mean": 0.13226473331451416, "nonfinite": 0.0, "weighted_memory_to_active": 0.215162456035614, "mix_std": 0.13494086265563965, "beta_std": 0.10485093295574188, "beta_low": 0.1974080502986908, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.06630686670541763, "output_influence": 0.2552221119403839, "residual_influence": 0.057000190019607544, "state_active": 0.013450615108013153, "state_memory": 0.027007196098566055, "state_peak": 37.754295349121094, "growth_active": 1.1246566772460938, "growth_memory": 1.3463460206985474, "heads": [{"head": 0, "memory_to_active": 2.4760191440582275, "contribution_to_active": 0.33781957626342773, "mix_mean": 0.034889642149209976, "mix_low": 0.42417240142822266, "mix_high": 2.86102294921875e-06, "read_cosine": 0.29713329672813416, "beta_mean": 0.17689494788646698, "nonfinite": 0.0, "weighted_memory_to_active": 0.38424575328826904, "mix_std": 0.07355247437953949, "beta_std": 0.14293330907821655, "beta_low": 0.005126953125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.006658675614744425, "evidence_weights": [0.02551608718931675, -0.07401745766401291, 0.056475818157196045], "agreement_weight": -0.03266734257340431, "orthogonal_weight": 0.02639494091272354}, {"head": 1, "memory_to_active": 1.0077085494995117, "contribution_to_active": 0.38153180480003357, "mix_mean": 0.2076580971479416, "mix_low": 0.11211681365966797, "mix_high": 0.0030727386474609375, "read_cosine": 0.36916518211364746, "beta_mean": 0.10342074930667877, "nonfinite": 0.0, "weighted_memory_to_active": 0.33148032426834106, "mix_std": 0.2309156358242035, "beta_std": 0.06504710018634796, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03207249566912651, "evidence_weights": [-0.20629099011421204, -0.10832667350769043, -0.2330084890127182], "agreement_weight": 0.028103820979595184, "orthogonal_weight": 0.11432047188282013}, {"head": 2, "memory_to_active": 0.8186584711074829, "contribution_to_active": 0.2890506684780121, "mix_mean": 0.13701945543289185, "mix_low": 0.07536697387695312, "mix_high": 0.00018405914306640625, "read_cosine": 0.07521025836467743, "beta_mean": 0.06011040508747101, "nonfinite": 0.0, "weighted_memory_to_active": 0.2098398655653, "mix_std": 0.16885077953338623, "beta_std": 0.05245593935251236, "beta_low": 0.05560302734375, "beta_high": 0.0, "orthogonal_to_active": 0.08676803112030029, "evidence_weights": [-0.05595202371478081, -0.1388487070798874, 0.6031482219696045], "agreement_weight": 0.027028346434235573, "orthogonal_weight": -0.4244335889816284}, {"head": 3, "memory_to_active": 4.296422481536865, "contribution_to_active": 0.08492952585220337, "mix_mean": 0.034915246069431305, "mix_low": 0.3262805938720703, "mix_high": 0.0, "read_cosine": 0.09520967304706573, "beta_mean": 0.36343255639076233, "nonfinite": 0.0, "weighted_memory_to_active": 0.29124486446380615, "mix_std": 0.05082542449235916, "beta_std": 0.27325674891471863, "beta_low": 0.010986328125, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.24993330240249634, "evidence_weights": [0.0640764907002449, -0.013245993293821812, 0.6917153000831604], "agreement_weight": 0.05216703563928604, "orthogonal_weight": -0.9296500086784363}, {"head": 4, "memory_to_active": 1.6073143482208252, "contribution_to_active": 0.0636468231678009, "mix_mean": 0.015197796747088432, "mix_low": 0.6510047912597656, "mix_high": 0.0, "read_cosine": 0.22312712669372559, "beta_mean": 0.08937402069568634, "nonfinite": 0.0, "weighted_memory_to_active": 0.05067875608801842, "mix_std": 0.03147970885038376, "beta_std": 0.09469635039567947, "beta_low": 0.11358642578125, "beta_high": 0.0, "orthogonal_to_active": 0.007128995377570391, "evidence_weights": [0.14819256961345673, 0.03448472544550896, 0.2140038013458252], "agreement_weight": 0.04320785775780678, "orthogonal_weight": -0.170429065823555}, {"head": 5, "memory_to_active": 0.05860449746251106, "contribution_to_active": 0.35521018505096436, "mix_mean": 0.3557395935058594, "mix_low": 0.0065746307373046875, "mix_high": 0.00014781951904296875, "read_cosine": 0.2904433608055115, "beta_mean": 0.00035569589817896485, "nonfinite": 0.0, "weighted_memory_to_active": 0.023485148325562477, "mix_std": 0.25402113795280457, "beta_std": 0.0007161170360632241, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.015279697254300117, "evidence_weights": [-0.13687212765216827, -0.19637444615364075, 0.5817785263061523], "agreement_weight": 0.1559249609708786, "orthogonal_weight": -0.7330015897750854}]}, {"layer": 8, "memory_to_active": 0.4071429967880249, "contribution_to_active": 0.5326576232910156, "mix_mean": 0.4353369474411011, "mix_low": 0.06549756228923798, "mix_high": 0.0010466575622558594, "read_cosine": 0.46757882833480835, "beta_mean": 0.026168987154960632, "nonfinite": 0.0, "weighted_memory_to_active": 0.09399605542421341, "mix_std": 0.22041350603103638, "beta_std": 0.03338218852877617, "beta_low": 0.63037109375, "beta_high": 0.0, "orthogonal_to_active": 0.06594027578830719, "output_influence": 0.4677089750766754, "residual_influence": 0.15166378021240234, "state_active": 0.043841708451509476, "state_memory": 0.008755684830248356, "state_peak": 34.962249755859375, "growth_active": 1.0367077589035034, "growth_memory": 1.318543553352356, "heads": [{"head": 0, "memory_to_active": 0.06606103479862213, "contribution_to_active": 0.5595468878746033, "mix_mean": 0.606441855430603, "mix_low": 0.0003223419189453125, "mix_high": 0.0010814666748046875, "read_cosine": 0.48442187905311584, "beta_mean": 0.003614675020799041, "nonfinite": 0.0, "weighted_memory_to_active": 0.04617543891072273, "mix_std": 0.24179647862911224, "beta_std": 0.011692765168845654, "beta_low": 0.9173583984375, "beta_high": 0.0, "orthogonal_to_active": 0.029554281383752823, "evidence_weights": [-0.17254920303821564, -0.39268410205841064, 0.6695559024810791], "agreement_weight": 0.17056548595428467, "orthogonal_weight": -0.8037408590316772}, {"head": 1, "memory_to_active": 0.10793936252593994, "contribution_to_active": 0.5276581048965454, "mix_mean": 0.4892449378967285, "mix_low": 0.000152587890625, "mix_high": 0.00046443939208984375, "read_cosine": 0.24985623359680176, "beta_mean": 0.002229757374152541, "nonfinite": 0.0, "weighted_memory_to_active": 0.052758995443582535, "mix_std": 0.24506725370883942, "beta_std": 0.008282158523797989, "beta_low": 0.968505859375, "beta_high": 0.0, "orthogonal_to_active": 0.036290086805820465, "evidence_weights": [-0.19015014171600342, -0.26630350947380066, 0.6912016272544861], "agreement_weight": 0.09018891304731369, "orthogonal_weight": -0.7399415373802185}, {"head": 2, "memory_to_active": 0.01612476073205471, "contribution_to_active": 0.6640910506248474, "mix_mean": 0.6495387554168701, "mix_low": 0.0009002685546875, "mix_high": 0.00318145751953125, "read_cosine": 0.4149585962295532, "beta_mean": 0.00013574579497799277, "nonfinite": 0.0, "weighted_memory_to_active": 0.012410733848810196, "mix_std": 0.22885118424892426, "beta_std": 0.000429646170232445, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.008133059367537498, "evidence_weights": [-0.22548894584178925, -0.34452393651008606, 0.9756543636322021], "agreement_weight": 0.24339531362056732, "orthogonal_weight": -0.7571817636489868}, {"head": 3, "memory_to_active": 0.11647379398345947, "contribution_to_active": 0.6214199662208557, "mix_mean": 0.5652756690979004, "mix_low": 0.0013570785522460938, "mix_high": 0.0014181137084960938, "read_cosine": 0.8037987947463989, "beta_mean": 0.009479325264692307, "nonfinite": 0.0, "weighted_memory_to_active": 0.0883481577038765, "mix_std": 0.25571104884147644, "beta_std": 0.007772411219775677, "beta_low": 0.63323974609375, "beta_high": 0.0, "orthogonal_to_active": 0.02786754071712494, "evidence_weights": [-0.18227963149547577, -0.3038161098957062, 0.6090931296348572], "agreement_weight": 0.1964542716741562, "orthogonal_weight": -0.6338096261024475}, {"head": 4, "memory_to_active": 1.79306161403656, "contribution_to_active": 0.17890754342079163, "mix_mean": 0.04343631863594055, "mix_low": 0.29315662384033203, "mix_high": 0.0, "read_cosine": 0.35631558299064636, "beta_mean": 0.06403661519289017, "nonfinite": 0.0, "weighted_memory_to_active": 0.17701807618141174, "mix_std": 0.06956550478935242, "beta_std": 0.08016250282526016, "beta_low": 0.1580810546875, "beta_high": 0.0, "orthogonal_to_active": 0.0096054682508111, "evidence_weights": [0.046441514045000076, -0.0522453673183918, -0.026289915665984154], "agreement_weight": 0.07918909192085266, "orthogonal_weight": -0.05811816081404686}, {"head": 5, "memory_to_active": 0.3431972861289978, "contribution_to_active": 0.644322395324707, "mix_mean": 0.25808393955230713, "mix_low": 0.09709644317626953, "mix_high": 0.00013446807861328125, "read_cosine": 0.49612191319465637, "beta_mean": 0.0775177925825119, "nonfinite": 0.0, "weighted_memory_to_active": 0.18726493418216705, "mix_std": 0.2814895212650299, "beta_std": 0.09195365011692047, "beta_low": 0.10516357421875, "beta_high": 0.0, "orthogonal_to_active": 0.28419119119644165, "evidence_weights": [-0.08888982236385345, -0.21183164417743683, -0.22218310832977295], "agreement_weight": -0.014937528409063816, "orthogonal_weight": 1.8123209476470947}]}, {"layer": 9, "memory_to_active": 1.087180495262146, "contribution_to_active": 0.6923462748527527, "mix_mean": 0.3336910009384155, "mix_low": 0.14055411517620087, "mix_high": 0.004677772521972656, "read_cosine": 0.3518691658973694, "beta_mean": 0.06393104791641235, "nonfinite": 0.0, "weighted_memory_to_active": 0.41694319248199463, "mix_std": 0.21488147974014282, "beta_std": 0.04471549764275551, "beta_low": 0.5026041865348816, "beta_high": 0.0, "orthogonal_to_active": 0.04197823256254196, "output_influence": 0.5087847113609314, "residual_influence": 0.14052046835422516, "state_active": 0.040746696293354034, "state_memory": 0.019197871908545494, "state_peak": 43.043758392333984, "growth_active": 1.1737314462661743, "growth_memory": 1.412906527519226, "heads": [{"head": 0, "memory_to_active": 0.0022395236883312464, "contribution_to_active": 0.5527488589286804, "mix_mean": 0.6233435869216919, "mix_low": 0.0015277862548828125, "mix_high": 0.02049541473388672, "read_cosine": 0.2640036940574646, "beta_mean": 4.4204834921401925e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014181723818182945, "mix_std": 0.29319894313812256, "beta_std": 1.5781051843077876e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001298452727496624, "evidence_weights": [-0.2982393205165863, -0.3067563772201538, 0.8252459764480591], "agreement_weight": 0.11113502830266953, "orthogonal_weight": -0.9673466682434082}, {"head": 1, "memory_to_active": 0.03528178110718727, "contribution_to_active": 0.6584665179252625, "mix_mean": 0.5343272686004639, "mix_low": 0.008496284484863281, "mix_high": 0.0019121170043945312, "read_cosine": 0.397908478975296, "beta_mean": 0.00010394438140792772, "nonfinite": 0.0, "weighted_memory_to_active": 0.0252365879714489, "mix_std": 0.2850106358528137, "beta_std": 0.00021692464360967278, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.017481591552495956, "evidence_weights": [-0.2311730682849884, -0.28033730387687683, 1.1662671566009521], "agreement_weight": 0.40330803394317627, "orthogonal_weight": -0.8545131683349609}, {"head": 2, "memory_to_active": 0.21280476450920105, "contribution_to_active": 0.6340111494064331, "mix_mean": 0.5778604745864868, "mix_low": 0.0023975372314453125, "mix_high": 0.005034446716308594, "read_cosine": 0.3698046803474426, "beta_mean": 0.0008784696110524237, "nonfinite": 0.0, "weighted_memory_to_active": 0.14714065194129944, "mix_std": 0.2728336453437805, "beta_std": 0.00319162686355412, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.08092661201953888, "evidence_weights": [-0.1974223405122757, -0.2688191831111908, 0.7237933874130249], "agreement_weight": 0.2931627929210663, "orthogonal_weight": -0.6502313017845154}, {"head": 3, "memory_to_active": 1.435448169708252, "contribution_to_active": 0.15206922590732574, "mix_mean": 0.044710226356983185, "mix_low": 0.3786592483520508, "mix_high": 8.58306884765625e-06, "read_cosine": 0.40601396560668945, "beta_mean": 0.08739876747131348, "nonfinite": 0.0, "weighted_memory_to_active": 0.13304400444030762, "mix_std": 0.09833788126707077, "beta_std": 0.05755317211151123, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.0056665632873773575, "evidence_weights": [0.08229217678308487, 0.017919300124049187, 0.03174386918544769], "agreement_weight": 0.08812486380338669, "orthogonal_weight": 0.046238914132118225}, {"head": 4, "memory_to_active": 3.1676886081695557, "contribution_to_active": 1.6207581758499146, "mix_mean": 0.14102703332901, "mix_low": 0.14607715606689453, "mix_high": 0.0005321502685546875, "read_cosine": 0.17623934149742126, "beta_mean": 0.15720617771148682, "nonfinite": 0.0, "weighted_memory_to_active": 1.6094108819961548, "mix_std": 0.20023876428604126, "beta_std": 0.08895275741815567, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.036693401634693146, "evidence_weights": [-0.07349590212106705, -0.15604142844676971, -0.017301084473729134], "agreement_weight": -0.05854851007461548, "orthogonal_weight": -0.023661235347390175}, {"head": 5, "memory_to_active": 1.6696197986602783, "contribution_to_active": 0.5360237956047058, "mix_mean": 0.08087734133005142, "mix_low": 0.3061666488647461, "mix_high": 8.392333984375e-05, "read_cosine": 0.497244656085968, "beta_mean": 0.13799448311328888, "nonfinite": 0.0, "weighted_memory_to_active": 0.5854089260101318, "mix_std": 0.13966894149780273, "beta_std": 0.11836268752813339, "beta_low": 0.0216064453125, "beta_high": 0.0, "orthogonal_to_active": 0.10980277508497238, "evidence_weights": [0.10970801115036011, -0.0882670134305954, 0.15023116767406464], "agreement_weight": 0.015563945285975933, "orthogonal_weight": 0.2772189676761627}]}, {"layer": 10, "memory_to_active": 0.23213107883930206, "contribution_to_active": 0.6603498458862305, "mix_mean": 0.5054100155830383, "mix_low": 0.01815176010131836, "mix_high": 0.006939093582332134, "read_cosine": 0.3721810281276703, "beta_mean": 0.04624103009700775, "nonfinite": 0.0, "weighted_memory_to_active": 0.11309637874364853, "mix_std": 0.27164262533187866, "beta_std": 0.04715155065059662, "beta_low": 0.6458536982536316, "beta_high": 0.0, "orthogonal_to_active": 0.1955346316099167, "output_influence": 0.4295817017555237, "residual_influence": 0.1465800702571869, "state_active": 0.051010485738515854, "state_memory": 0.01587197557091713, "state_peak": 106.07532501220703, "growth_active": 1.0435587167739868, "growth_memory": 0.9506235122680664, "heads": [{"head": 0, "memory_to_active": 0.8524816036224365, "contribution_to_active": 1.752413034439087, "mix_mean": 0.5953414440155029, "mix_low": 0.0704498291015625, "mix_high": 0.02319812774658203, "read_cosine": 0.11047181487083435, "beta_mean": 0.20311713218688965, "nonfinite": 0.0, "weighted_memory_to_active": 0.5427178144454956, "mix_std": 0.3943544030189514, "beta_std": 0.21525919437408447, "beta_low": 0.12774658203125, "beta_high": 0.0, "orthogonal_to_active": 1.0774894952774048, "evidence_weights": [-0.1172783374786377, -0.39358824491500854, -1.4102786779403687], "agreement_weight": 0.02366570010781288, "orthogonal_weight": 2.009305238723755}, {"head": 1, "memory_to_active": 0.01784617453813553, "contribution_to_active": 0.5964908003807068, "mix_mean": 0.6065534353256226, "mix_low": 0.0008544921875, "mix_high": 0.012614250183105469, "read_cosine": 0.27128687500953674, "beta_mean": 0.00010698841651901603, "nonfinite": 0.0, "weighted_memory_to_active": 0.009833408519625664, "mix_std": 0.2708682715892792, "beta_std": 0.00015449798956979066, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014365077950060368, "evidence_weights": [-0.260996550321579, -0.19975724816322327, 0.567233145236969], "agreement_weight": 0.5044126510620117, "orthogonal_weight": -1.5772470235824585}, {"head": 2, "memory_to_active": 0.014445004053413868, "contribution_to_active": 0.590798556804657, "mix_mean": 0.619443953037262, "mix_low": 0.0005159378051757812, "mix_high": 0.004833221435546875, "read_cosine": 0.027793865650892258, "beta_mean": 3.191491850884631e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00638989731669426, "mix_std": 0.25670915842056274, "beta_std": 0.00010617072985041887, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003199474886059761, "evidence_weights": [-0.2461785078048706, -0.27768754959106445, 0.5808312892913818], "agreement_weight": -0.18028850853443146, "orthogonal_weight": -0.48343512415885925}, {"head": 3, "memory_to_active": 0.3409329056739807, "contribution_to_active": 0.10088904947042465, "mix_mean": 0.27458545565605164, "mix_low": 0.034379005432128906, "mix_high": 0.0001773834228515625, "read_cosine": 0.9102792739868164, "beta_mean": 0.06594444811344147, "nonfinite": 0.0, "weighted_memory_to_active": 0.04122345894575119, "mix_std": 0.22842642664909363, "beta_std": 0.05827996879816055, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.04541067034006119, "evidence_weights": [-0.10704120248556137, -0.1457049548625946, 0.1479865461587906], "agreement_weight": 0.007964645512402058, "orthogonal_weight": 1.4418282508850098}, {"head": 4, "memory_to_active": 0.018667154014110565, "contribution_to_active": 0.5319875478744507, "mix_mean": 0.551444411277771, "mix_low": 0.0003604888916015625, "mix_high": 0.0008087158203125, "read_cosine": 0.058601781725883484, "beta_mean": 6.212711741682142e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01049003005027771, "mix_std": 0.2524028420448303, "beta_std": 0.000232237609452568, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.011229329742491245, "evidence_weights": [-0.183505579829216, -0.26187950372695923, 0.8694918751716614], "agreement_weight": 0.13013440370559692, "orthogonal_weight": -1.1299985647201538}, {"head": 5, "memory_to_active": 0.14841361343860626, "contribution_to_active": 0.38952013850212097, "mix_mean": 0.38509124517440796, "mix_low": 0.0023508071899414062, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8546525239944458, "beta_mean": 0.00818357989192009, "nonfinite": 0.0, "weighted_memory_to_active": 0.06792367994785309, "mix_std": 0.2270946204662323, "beta_std": 0.008877204731106758, "beta_low": 0.7464599609375, "beta_high": 0.0, "orthogonal_to_active": 0.021513691172003746, "evidence_weights": [-0.1448153257369995, -0.16020548343658447, 0.2839077115058899], "agreement_weight": 0.08931782096624374, "orthogonal_weight": -0.6384034156799316}]}, {"layer": 11, "memory_to_active": 0.10038568079471588, "contribution_to_active": 0.5939050316810608, "mix_mean": 0.4835512340068817, "mix_low": 0.09972159564495087, "mix_high": 0.03741741180419922, "read_cosine": 0.38966506719589233, "beta_mean": 0.02620518207550049, "nonfinite": 0.0, "weighted_memory_to_active": 0.04860248416662216, "mix_std": 0.32193267345428467, "beta_std": 0.03971875086426735, "beta_low": 0.65313720703125, "beta_high": 0.0, "orthogonal_to_active": 0.044781628996133804, "output_influence": 0.5200716853141785, "residual_influence": 0.1326097846031189, "state_active": 0.19623073935508728, "state_memory": 0.005702758673578501, "state_peak": 29.168506622314453, "growth_active": 1.1272295713424683, "growth_memory": 1.681561827659607, "heads": [{"head": 0, "memory_to_active": 0.008892780169844627, "contribution_to_active": 0.5833855867385864, "mix_mean": 0.6235024929046631, "mix_low": 0.0012273788452148438, "mix_high": 0.00787353515625, "read_cosine": 0.33047938346862793, "beta_mean": 6.0305272199912e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0060658506117761135, "mix_std": 0.2599920332431793, "beta_std": 0.0018907487392425537, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.004588231910020113, "evidence_weights": [-0.16622969508171082, -0.36908841133117676, 0.7224587202072144], "agreement_weight": 0.2966046929359436, "orthogonal_weight": -0.8025239109992981}, {"head": 1, "memory_to_active": 0.25121214985847473, "contribution_to_active": 0.34221377968788147, "mix_mean": 0.39129501581192017, "mix_low": 0.008488655090332031, "mix_high": 0.00029659271240234375, "read_cosine": 0.5370569229125977, "beta_mean": 0.0217280350625515, "nonfinite": 0.0, "weighted_memory_to_active": 0.10883525013923645, "mix_std": 0.2601073682308197, "beta_std": 0.04800376668572426, "beta_low": 0.57769775390625, "beta_high": 0.0, "orthogonal_to_active": 0.042179033160209656, "evidence_weights": [-0.1686258167028427, -0.20175229012966156, 0.5351806879043579], "agreement_weight": 0.07256515324115753, "orthogonal_weight": -0.4999608099460602}, {"head": 2, "memory_to_active": 0.008816957473754883, "contribution_to_active": 0.9803996086120605, "mix_mean": 0.4260188043117523, "mix_low": 0.25963878631591797, "mix_high": 0.13775253295898438, "read_cosine": 0.20239689946174622, "beta_mean": 0.06509000062942505, "nonfinite": 0.0, "weighted_memory_to_active": 0.006194541696459055, "mix_std": 0.4416939616203308, "beta_std": 0.08710838109254837, "beta_low": 0.46807861328125, "beta_high": 0.0, "orthogonal_to_active": 0.04359067976474762, "evidence_weights": [0.03888605907559395, -0.41805964708328247, -1.2986736297607422], "agreement_weight": -0.0013640981633216143, "orthogonal_weight": 7.279565811157227}, {"head": 3, "memory_to_active": 0.011348540894687176, "contribution_to_active": 0.8446956276893616, "mix_mean": 0.3959078788757324, "mix_low": 0.31763267517089844, "mix_high": 0.07296943664550781, "read_cosine": 0.5505175590515137, "beta_mean": 0.04987287521362305, "nonfinite": 0.0, "weighted_memory_to_active": 0.006888033822178841, "mix_std": 0.43482619524002075, "beta_std": 0.07643391191959381, "beta_low": 0.51220703125, "beta_high": 0.0, "orthogonal_to_active": 0.03673720359802246, "evidence_weights": [0.16223551332950592, -0.3210209608078003, -0.23126336932182312], "agreement_weight": 0.04743373021483421, "orthogonal_weight": 7.631902694702148}, {"head": 4, "memory_to_active": 0.24544022977352142, "contribution_to_active": 0.4290909171104431, "mix_mean": 0.5354669094085693, "mix_low": 0.009455680847167969, "mix_high": 0.0007600784301757812, "read_cosine": 0.06272768974304199, "beta_mean": 0.013483542017638683, "nonfinite": 0.0, "weighted_memory_to_active": 0.12647069990634918, "mix_std": 0.2728511691093445, "beta_std": 0.015790434554219246, "beta_low": 0.56817626953125, "beta_high": 0.0, "orthogonal_to_active": 0.12124896049499512, "evidence_weights": [-0.239130437374115, -0.22187715768814087, 1.9515174627304077], "agreement_weight": -0.19700069725513458, "orthogonal_weight": -0.9794241189956665}, {"head": 5, "memory_to_active": 0.07660342752933502, "contribution_to_active": 0.38364458084106445, "mix_mean": 0.5291163325309753, "mix_low": 0.0018863677978515625, "mix_high": 0.004852294921875, "read_cosine": 0.6548118591308594, "beta_mean": 0.006996334530413151, "nonfinite": 0.0, "weighted_memory_to_active": 0.037160519510507584, "mix_std": 0.26212525367736816, "beta_std": 0.009085258468985558, "beta_low": 0.79296875, "beta_high": 0.0, "orthogonal_to_active": 0.02034563198685646, "evidence_weights": [-0.21589940786361694, -0.30390626192092896, 0.6163493394851685], "agreement_weight": 0.08345893025398254, "orthogonal_weight": -0.7475716471672058}]}, {"layer": 12, "memory_to_active": 0.24746638536453247, "contribution_to_active": 0.5614933967590332, "mix_mean": 0.5141558051109314, "mix_low": 0.01006460189819336, "mix_high": 0.004784584045410156, "read_cosine": 0.2477392554283142, "beta_mean": 0.017282256856560707, "nonfinite": 0.0, "weighted_memory_to_active": 0.13251608610153198, "mix_std": 0.2626574635505676, "beta_std": 0.015019158832728863, "beta_low": 0.8260701894760132, "beta_high": 0.0, "orthogonal_to_active": 0.06796436011791229, "output_influence": 0.46630731225013733, "residual_influence": 0.13037073612213135, "state_active": 0.03632662445306778, "state_memory": 0.009315622970461845, "state_peak": 51.49055862426758, "growth_active": 1.1060757637023926, "growth_memory": 1.222955346107483, "heads": [{"head": 0, "memory_to_active": 0.04768846556544304, "contribution_to_active": 0.5391458868980408, "mix_mean": 0.5327408313751221, "mix_low": 0.0010423660278320312, "mix_high": 0.0033636093139648438, "read_cosine": 0.31859147548675537, "beta_mean": 0.0007264544838108122, "nonfinite": 0.0, "weighted_memory_to_active": 0.023552460595965385, "mix_std": 0.270203173160553, "beta_std": 0.0015018890844658017, "beta_low": 0.99560546875, "beta_high": 0.0, "orthogonal_to_active": 0.01987193338572979, "evidence_weights": [-0.20299243927001953, -0.2730056643486023, 0.1896478682756424], "agreement_weight": 0.10201113671064377, "orthogonal_weight": -0.9080918431282043}, {"head": 1, "memory_to_active": 0.10042206943035126, "contribution_to_active": 0.5145633220672607, "mix_mean": 0.5291470289230347, "mix_low": 0.0008897781372070312, "mix_high": 0.0011949539184570312, "read_cosine": 0.45465514063835144, "beta_mean": 0.0008265624055638909, "nonfinite": 0.0, "weighted_memory_to_active": 0.05608157813549042, "mix_std": 0.2657216489315033, "beta_std": 0.005779411643743515, "beta_low": 0.99395751953125, "beta_high": 0.0, "orthogonal_to_active": 0.04570254310965538, "evidence_weights": [-0.20614202320575714, -0.2720271646976471, 0.9879105687141418], "agreement_weight": 0.17377425730228424, "orthogonal_weight": -0.9236525297164917}, {"head": 2, "memory_to_active": 0.6114524602890015, "contribution_to_active": 0.49222737550735474, "mix_mean": 0.3159591257572174, "mix_low": 0.03038311004638672, "mix_high": 8.0108642578125e-05, "read_cosine": 0.4988848865032196, "beta_mean": 0.10061655938625336, "nonfinite": 0.0, "weighted_memory_to_active": 0.2507244348526001, "mix_std": 0.2375498265028, "beta_std": 0.06763363629579544, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.2516612410545349, "evidence_weights": [-0.23472945392131805, -0.05009201914072037, -0.06359459459781647], "agreement_weight": 0.01911041885614395, "orthogonal_weight": 1.2633000612258911}, {"head": 3, "memory_to_active": 0.019208058714866638, "contribution_to_active": 0.7237111330032349, "mix_mean": 0.6884541511535645, "mix_low": 0.0002956390380859375, "mix_high": 0.014407157897949219, "read_cosine": 0.08331894129514694, "beta_mean": 0.00014585073222406209, "nonfinite": 0.0, "weighted_memory_to_active": 0.014267200604081154, "mix_std": 0.24054652452468872, "beta_std": 0.001346738077700138, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.013627380132675171, "evidence_weights": [-0.289485365152359, -0.34005752205848694, 1.4310684204101562], "agreement_weight": 0.4359656274318695, "orthogonal_weight": -0.9793189764022827}, {"head": 4, "memory_to_active": 0.6408798694610596, "contribution_to_active": 0.604299783706665, "mix_mean": 0.3913366198539734, "mix_low": 0.021439552307128906, "mix_high": 0.003875732421875, "read_cosine": -0.11170066893100739, "beta_mean": 0.0010600287932902575, "nonfinite": 0.0, "weighted_memory_to_active": 0.40685510635375977, "mix_std": 0.28432852029800415, "beta_std": 0.010532436892390251, "beta_low": 0.9715576171875, "beta_high": 0.0, "orthogonal_to_active": 0.043241534382104874, "evidence_weights": [-0.1324199140071869, -0.2167806178331375, -0.006463026627898216], "agreement_weight": 0.0009031571680679917, "orthogonal_weight": -0.11136123538017273}, {"head": 5, "memory_to_active": 0.06514732539653778, "contribution_to_active": 0.49501290917396545, "mix_mean": 0.6272968053817749, "mix_low": 0.006337165832519531, "mix_high": 0.005785942077636719, "read_cosine": 0.24268561601638794, "beta_mean": 0.0003180847270414233, "nonfinite": 0.0, "weighted_memory_to_active": 0.043615713715553284, "mix_std": 0.2775951623916626, "beta_std": 0.0033208439126610756, "beta_low": 0.99566650390625, "beta_high": 0.0, "orthogonal_to_active": 0.033681537955999374, "evidence_weights": [-0.2473895251750946, -0.316498726606369, 1.3230171203613281], "agreement_weight": 0.34972769021987915, "orthogonal_weight": -0.834987998008728}]}, {"layer": 13, "memory_to_active": 0.12661123275756836, "contribution_to_active": 0.5208872556686401, "mix_mean": 0.5285161733627319, "mix_low": 0.005676746368408203, "mix_high": 0.006556670181453228, "read_cosine": 0.41842496395111084, "beta_mean": 0.016146032139658928, "nonfinite": 0.0, "weighted_memory_to_active": 0.05488653481006622, "mix_std": 0.2605038285255432, "beta_std": 0.017437869682908058, "beta_low": 0.8001912832260132, "beta_high": 0.0, "orthogonal_to_active": 0.056227341294288635, "output_influence": 0.49460169672966003, "residual_influence": 0.1677733063697815, "state_active": 0.10673358291387558, "state_memory": 0.04029114171862602, "state_peak": 96.66444396972656, "growth_active": 0.9368452429771423, "growth_memory": 1.0556710958480835, "heads": [{"head": 0, "memory_to_active": 0.026045575737953186, "contribution_to_active": 0.5721977353096008, "mix_mean": 0.5881409645080566, "mix_low": 0.0005064010620117188, "mix_high": 0.002521514892578125, "read_cosine": 0.4424630105495453, "beta_mean": 0.0001022235446725972, "nonfinite": 0.0, "weighted_memory_to_active": 0.01621227152645588, "mix_std": 0.2702851891517639, "beta_std": 0.00021707138512283564, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015982013195753098, "evidence_weights": [-0.2002004086971283, -0.2660970687866211, 0.755428671836853], "agreement_weight": 0.1274479627609253, "orthogonal_weight": -1.105589747428894}, {"head": 1, "memory_to_active": 0.017383966594934464, "contribution_to_active": 0.589049756526947, "mix_mean": 0.5550546646118164, "mix_low": 0.0017633438110351562, "mix_high": 0.003444671630859375, "read_cosine": 0.2073180228471756, "beta_mean": 6.642489461228251e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009939951822161674, "mix_std": 0.27564537525177, "beta_std": 0.00012610417616087943, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012462988495826721, "evidence_weights": [-0.18514645099639893, -0.26712915301322937, 0.6738940477371216], "agreement_weight": 0.36431244015693665, "orthogonal_weight": -1.3067611455917358}, {"head": 2, "memory_to_active": 0.09191691130399704, "contribution_to_active": 0.44581493735313416, "mix_mean": 0.3873523473739624, "mix_low": 0.0022754669189453125, "mix_high": 0.0001316070556640625, "read_cosine": 0.7034206390380859, "beta_mean": 0.013044234365224838, "nonfinite": 0.0, "weighted_memory_to_active": 0.041742220520973206, "mix_std": 0.23349252343177795, "beta_std": 0.041384272277355194, "beta_low": 0.81207275390625, "beta_high": 0.0, "orthogonal_to_active": 0.020358538255095482, "evidence_weights": [-0.19205786287784576, -0.20330195128917694, 0.42345067858695984], "agreement_weight": 0.007086056284606457, "orthogonal_weight": -0.821333110332489}, {"head": 3, "memory_to_active": 0.14313572645187378, "contribution_to_active": 0.5552793741226196, "mix_mean": 0.6307600140571594, "mix_low": 0.021831512451171875, "mix_high": 0.022796630859375, "read_cosine": -0.04567263275384903, "beta_mean": 0.001011586282402277, "nonfinite": 0.0, "weighted_memory_to_active": 0.10712355375289917, "mix_std": 0.29088228940963745, "beta_std": 0.007921449840068817, "beta_low": 0.99078369140625, "beta_high": 0.0, "orthogonal_to_active": 0.0970485731959343, "evidence_weights": [-0.3222607374191284, -0.24673223495483398, 1.8015960454940796], "agreement_weight": 4.757392406463623, "orthogonal_weight": -1.003139615058899}, {"head": 4, "memory_to_active": 0.41119515895843506, "contribution_to_active": 0.2705104947090149, "mix_mean": 0.3692028522491455, "mix_low": 0.00551605224609375, "mix_high": 7.62939453125e-06, "read_cosine": 0.878581166267395, "beta_mean": 0.08228322863578796, "nonfinite": 0.0, "weighted_memory_to_active": 0.11118384450674057, "mix_std": 0.21947425603866577, "beta_std": 0.05321362987160683, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.1554400771856308, "evidence_weights": [-0.1115332618355751, -0.2365131974220276, 0.4727446734905243], "agreement_weight": 0.003282153746113181, "orthogonal_weight": 2.2036490440368652}, {"head": 5, "memory_to_active": 0.06999005377292633, "contribution_to_active": 0.6924709677696228, "mix_mean": 0.640586256980896, "mix_low": 0.0021677017211914062, "mix_high": 0.010437965393066406, "read_cosine": 0.3244395852088928, "beta_mean": 0.0003684959374368191, "nonfinite": 0.0, "weighted_memory_to_active": 0.043117351830005646, "mix_std": 0.2732433080673218, "beta_std": 0.0017646923661231995, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.03607183322310448, "evidence_weights": [-0.22089730203151703, -0.3057018518447876, 0.8292677402496338], "agreement_weight": 0.1409304440021515, "orthogonal_weight": -0.9194009900093079}]}, {"layer": 14, "memory_to_active": 0.24811199307441711, "contribution_to_active": 0.41994789242744446, "mix_mean": 0.4235042929649353, "mix_low": 0.024153076112270355, "mix_high": 0.005860011093318462, "read_cosine": 0.6102828979492188, "beta_mean": 0.06823036819696426, "nonfinite": 0.0, "weighted_memory_to_active": 0.10721227526664734, "mix_std": 0.24760442972183228, "beta_std": 0.06332146376371384, "beta_low": 0.4798685908317566, "beta_high": 0.0, "orthogonal_to_active": 0.04548945650458336, "output_influence": 0.3383995294570923, "residual_influence": 0.13035129010677338, "state_active": 0.05811566114425659, "state_memory": 0.0168467964977026, "state_peak": 38.08564376831055, "growth_active": 1.0809234380722046, "growth_memory": 1.2931557893753052, "heads": [{"head": 0, "memory_to_active": 0.37335389852523804, "contribution_to_active": 0.2029535174369812, "mix_mean": 0.22505876421928406, "mix_low": 0.010418891906738281, "mix_high": 8.58306884765625e-06, "read_cosine": 0.7897665500640869, "beta_mean": 0.08161391317844391, "nonfinite": 0.0, "weighted_memory_to_active": 0.09151308238506317, "mix_std": 0.18887224793434143, "beta_std": 0.06696023792028427, "beta_low": 0.0018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.08668369054794312, "evidence_weights": [-0.13197752833366394, -0.033909548074007034, 0.4717007577419281], "agreement_weight": 0.024589749053120613, "orthogonal_weight": 1.4933805465698242}, {"head": 1, "memory_to_active": 0.5054389834403992, "contribution_to_active": 0.3052928149700165, "mix_mean": 0.22664086520671844, "mix_low": 0.11592864990234375, "mix_high": 1.33514404296875e-05, "read_cosine": 0.8213920593261719, "beta_mean": 0.2655485272407532, "nonfinite": 0.0, "weighted_memory_to_active": 0.22376284003257751, "mix_std": 0.2545856237411499, "beta_std": 0.2551914155483246, "beta_low": 0.09820556640625, "beta_high": 0.0, "orthogonal_to_active": 0.015316968783736229, "evidence_weights": [-0.013349837623536587, -0.21346426010131836, -0.12582193315029144], "agreement_weight": 0.05535625293850899, "orthogonal_weight": 0.15737193822860718}, {"head": 2, "memory_to_active": 0.15472102165222168, "contribution_to_active": 0.5711511373519897, "mix_mean": 0.5783171653747559, "mix_low": 0.0032491683959960938, "mix_high": 0.003387451171875, "read_cosine": 0.4186067283153534, "beta_mean": 0.002754614455625415, "nonfinite": 0.0, "weighted_memory_to_active": 0.0997130498290062, "mix_std": 0.2685353755950928, "beta_std": 0.0034269848838448524, "beta_low": 0.9586181640625, "beta_high": 0.0, "orthogonal_to_active": 0.07522868365049362, "evidence_weights": [-0.14658790826797485, -0.3133566677570343, 0.8949903249740601], "agreement_weight": 0.08340347558259964, "orthogonal_weight": -0.8649364709854126}, {"head": 3, "memory_to_active": 0.05098814517259598, "contribution_to_active": 0.6184236407279968, "mix_mean": 0.6653177738189697, "mix_low": 0.006023406982421875, "mix_high": 0.021440505981445312, "read_cosine": 0.04745468869805336, "beta_mean": 0.005919479764997959, "nonfinite": 0.0, "weighted_memory_to_active": 0.033195752650499344, "mix_std": 0.2572474181652069, "beta_std": 0.006319898180663586, "beta_low": 0.83172607421875, "beta_high": 0.0, "orthogonal_to_active": 0.02294817939400673, "evidence_weights": [-0.3568858206272125, -0.31445667147636414, 0.32382065057754517], "agreement_weight": 0.07813409715890884, "orthogonal_weight": -0.7917078733444214}, {"head": 4, "memory_to_active": 0.1595023274421692, "contribution_to_active": 0.49060386419296265, "mix_mean": 0.4885137677192688, "mix_low": 0.0050373077392578125, "mix_high": 0.00891876220703125, "read_cosine": 0.8503740429878235, "beta_mean": 0.0512351430952549, "nonfinite": 0.0, "weighted_memory_to_active": 0.09351924061775208, "mix_std": 0.26323187351226807, "beta_std": 0.035901084542274475, "beta_low": 0.03521728515625, "beta_high": 0.0, "orthogonal_to_active": 0.02796287089586258, "evidence_weights": [-0.2182222455739975, -0.24755403399467468, 0.42795172333717346], "agreement_weight": 0.12740054726600647, "orthogonal_weight": -0.5997529029846191}, {"head": 5, "memory_to_active": 0.24466760456562042, "contribution_to_active": 0.33126240968704224, "mix_mean": 0.3571772277355194, "mix_low": 0.004261016845703125, "mix_high": 0.0013914108276367188, "read_cosine": 0.734103262424469, "beta_mean": 0.002310500480234623, "nonfinite": 0.0, "weighted_memory_to_active": 0.10156968235969543, "mix_std": 0.25315406918525696, "beta_std": 0.012129154056310654, "beta_low": 0.95361328125, "beta_high": 0.0, "orthogonal_to_active": 0.044796332716941833, "evidence_weights": [-0.17771178483963013, -0.1669481098651886, 0.259540319442749], "agreement_weight": 0.08299871534109116, "orthogonal_weight": -0.6982837319374084}]}, {"layer": 15, "memory_to_active": 0.05623737722635269, "contribution_to_active": 0.4822498559951782, "mix_mean": 0.4087029695510864, "mix_low": 0.00879669189453125, "mix_high": 0.0024696986656636, "read_cosine": 0.6857661604881287, "beta_mean": 0.03569314628839493, "nonfinite": 0.0, "weighted_memory_to_active": 0.02722449041903019, "mix_std": 0.2843188941478729, "beta_std": 0.04219178110361099, "beta_low": 0.7088826894760132, "beta_high": 0.0, "orthogonal_to_active": 0.0064347232691943645, "output_influence": 0.30121299624443054, "residual_influence": 0.16484574973583221, "state_active": 0.14081434905529022, "state_memory": 0.010294726118445396, "state_peak": 42.82447814941406, "growth_active": 1.260361909866333, "growth_memory": 1.5120646953582764, "heads": [{"head": 0, "memory_to_active": 0.09304608404636383, "contribution_to_active": 0.4825979173183441, "mix_mean": 0.375105082988739, "mix_low": 0.006275177001953125, "mix_high": 0.0053882598876953125, "read_cosine": 0.6598764657974243, "beta_mean": 0.04236597567796707, "nonfinite": 0.0, "weighted_memory_to_active": 0.045269645750522614, "mix_std": 0.28053200244903564, "beta_std": 0.032401230186223984, "beta_low": 0.083251953125, "beta_high": 0.0, "orthogonal_to_active": 0.022213242948055267, "evidence_weights": [-0.18369092047214508, -0.186982199549675, 0.39012160897254944], "agreement_weight": 0.0941663607954979, "orthogonal_weight": -0.7251179814338684}, {"head": 1, "memory_to_active": 0.12596918642520905, "contribution_to_active": 0.4180286228656769, "mix_mean": 0.4225831627845764, "mix_low": 0.0020313262939453125, "mix_high": 0.0049610137939453125, "read_cosine": 0.9152464866638184, "beta_mean": 0.0008491023327223957, "nonfinite": 0.0, "weighted_memory_to_active": 0.052579302340745926, "mix_std": 0.28344014286994934, "beta_std": 0.002486756071448326, "beta_low": 0.98822021484375, "beta_high": 0.0, "orthogonal_to_active": 0.00041288070497103035, "evidence_weights": [-0.15618199110031128, -0.16632696986198425, 0.20784913003444672], "agreement_weight": 0.1199951171875, "orthogonal_weight": 0.014857565984129906}, {"head": 2, "memory_to_active": 0.0033577787689864635, "contribution_to_active": 0.461969792842865, "mix_mean": 0.3385806679725647, "mix_low": 0.015813827514648438, "mix_high": 5.054473876953125e-05, "read_cosine": 0.7757464647293091, "beta_mean": 5.360605427995324e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.001577779301442206, "mix_std": 0.2703288495540619, "beta_std": 8.909973985282704e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00034237871295772493, "evidence_weights": [-0.20719170570373535, -0.2728855013847351, 0.16998933255672455], "agreement_weight": 0.011096988804638386, "orthogonal_weight": 0.5653303861618042}, {"head": 3, "memory_to_active": 0.0014520343393087387, "contribution_to_active": 0.40126752853393555, "mix_mean": 0.37703564763069153, "mix_low": 0.006592750549316406, "mix_high": 0.00031280517578125, "read_cosine": 0.6425191760063171, "beta_mean": 8.92881871550344e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005310555570758879, "mix_std": 0.2850378751754761, "beta_std": 2.0551018678816035e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004199348040856421, "evidence_weights": [-0.2387610226869583, -0.25128060579299927, 0.22089166939258575], "agreement_weight": 0.04053199663758278, "orthogonal_weight": -0.8936319947242737}, {"head": 4, "memory_to_active": 0.08918765932321548, "contribution_to_active": 0.5985844135284424, "mix_mean": 0.3725883960723877, "mix_low": 0.012002944946289062, "mix_high": 0.0007257461547851562, "read_cosine": 0.7418864965438843, "beta_mean": 0.1693524420261383, "nonfinite": 0.0, "weighted_memory_to_active": 0.0471615307033062, "mix_std": 0.307319700717926, "beta_std": 0.20727704465389252, "beta_low": 0.20703125, "beta_high": 0.0, "orthogonal_to_active": 0.006264375988394022, "evidence_weights": [-0.21363715827465057, -0.25453275442123413, -0.0060508968308568], "agreement_weight": 0.011998471803963184, "orthogonal_weight": 0.29020360112190247}, {"head": 5, "memory_to_active": 0.024411512538790703, "contribution_to_active": 0.5310507416725159, "mix_mean": 0.5663248300552368, "mix_low": 0.010064125061035156, "mix_high": 0.00337982177734375, "read_cosine": 0.37932154536247253, "beta_mean": 0.0015287961577996612, "nonfinite": 0.0, "weighted_memory_to_active": 0.016227617859840393, "mix_std": 0.2792547047138214, "beta_std": 0.010876008309423923, "beta_low": 0.97479248046875, "beta_high": 0.0, "orthogonal_to_active": 0.008955524303019047, "evidence_weights": [-0.29841145873069763, -0.31224870681762695, 0.7370853424072266], "agreement_weight": 0.2758982181549072, "orthogonal_weight": 0.7437260746955872}]}]}, {"step": 44000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.89736606400038, "memory_to_active": 0.8304214477539062, "contribution_to_active": 0.5467806458473206, "mix_mean": 0.39473703503608704, "mix_low": 0.14106670022010803, "mix_high": 0.009496688842773438, "read_cosine": 0.35873204469680786, "beta_mean": 0.07308539748191833, "nonfinite": 0.0, "weighted_memory_to_active": 0.2147289663553238, "mix_std": 0.22071674466133118, "beta_std": 0.06424465030431747, "beta_low": 0.5419260859489441, "beta_high": 3.24249267578125e-05, "orthogonal_to_active": 0.056793082505464554, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1368680000305176, "normal_token_accuracy": 0.44716352224349976, "state_active": 0.05996856838464737, "state_memory": 0.018365591764450073, "state_peak": 117.90721130371094, "growth_active": 1.1123135089874268, "growth_memory": 1.35191011428833, "output_influence": 0.4019263684749603, "residual_influence": 0.13303455710411072, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2371973991394043, "no_memory_loss_delta": 0.5294026136398315, "scrambled_loss_delta": 0.19443883001804352, "memory_only_loss_delta": 3.1676244735717773, "no_evidence_loss_delta": 0.30854278802871704, "no_agreement_loss_delta": 0.02804182842373848, "no_orthogonal_loss_delta": 0.0032268096692860126}, {"index": 94, "tokens": 256, "normal_loss": 3.3218579292297363, "no_memory_loss_delta": 0.5270076394081116, "scrambled_loss_delta": 0.2787016034126282, "memory_only_loss_delta": 3.195700168609619, "no_evidence_loss_delta": 0.22562259435653687, "no_agreement_loss_delta": 0.02172023057937622, "no_orthogonal_loss_delta": 0.04782537743449211}, {"index": 188, "tokens": 256, "normal_loss": 3.456888198852539, "no_memory_loss_delta": 0.6195827126502991, "scrambled_loss_delta": 0.14225734770298004, "memory_only_loss_delta": 3.066175937652588, "no_evidence_loss_delta": 0.336473286151886, "no_agreement_loss_delta": 0.04048643261194229, "no_orthogonal_loss_delta": 0.02566945180296898}, {"index": 283, "tokens": 131, "normal_loss": 1.9539169073104858, "no_memory_loss_delta": 0.24065148830413818, "scrambled_loss_delta": 0.24044707417488098, "memory_only_loss_delta": 2.5990867614746094, "no_evidence_loss_delta": 0.13125120103359222, "no_agreement_loss_delta": 0.0020258964505046606, "no_orthogonal_loss_delta": 0.013761294074356556}], "agreement_weight_abs_mean": 0.1885846108198166, "agreement_weight_abs_max": 4.854716777801514, "agreement_weight_mean": 0.16495078802108765, "orthogonal_weight_abs_mean": 0.7869251370429993, "orthogonal_weight_abs_max": 7.780999183654785, "orthogonal_weight_mean": -0.13174468278884888, "no_memory_loss": 3.6491928100585938, "no_memory_loss_delta": 0.512324333190918, "no_memory_token_accuracy_gain": 0.06562847644090652, "no_memory_token_win_fraction": 0.6473859548568726, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 40.08985521524778, "scrambled_loss": 3.347146987915039, "scrambled_loss_delta": 0.21027854084968567, "scrambled_token_accuracy_gain": 0.03559510409832001, "scrambled_token_win_fraction": 0.6295884251594543, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.96415034127552, "memory_only_loss": 6.200753211975098, "memory_only_loss_delta": 3.063884973526001, "memory_only_token_accuracy_gain": 0.2803114652633667, "memory_only_token_win_fraction": 0.8976640701293945, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.3294108853965, "no_evidence_loss": 3.4039177894592285, "no_evidence_loss_delta": 0.2670494019985199, "no_evidence_token_accuracy_gain": 0.036707453429698944, "no_evidence_token_win_fraction": 0.5995550751686096, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 23.436475338025016, "no_agreement_loss": 3.162862777709961, "no_agreement_loss_delta": 0.025994446128606796, "no_agreement_token_accuracy_gain": -0.0022246940061450005, "no_agreement_token_win_fraction": 0.4949944317340851, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.565949904444215, "no_orthogonal_loss": 3.1607210636138916, "no_orthogonal_loss_delta": 0.023852579295635223, "no_orthogonal_token_accuracy_gain": 0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5572858452796936, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 2.357035490583326, "short_no_memory_loss_delta": 0.45826560258865356, "long_context_memory_gain": 0.054058730602264404, "normal_context_loss_gain": 0.1296595335006714, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.498444080352783, "contribution_to_active": 1.1823638677597046, "mix_mean": 0.10408774018287659, "mix_low": 0.5405668020248413, "mix_high": 0.005351861473172903, "read_cosine": 0.4417286813259125, "beta_mean": 0.22431010007858276, "nonfinite": 0.0, "weighted_memory_to_active": 1.23246169090271, "mix_std": 0.08926695585250854, "beta_std": 0.14955252408981323, "beta_low": 0.0383707694709301, "beta_high": 0.0, "orthogonal_to_active": 0.053189799189567566, "output_influence": 0.2011347860097885, "residual_influence": 0.24016042053699493, "state_active": 0.06586594879627228, "state_memory": 0.044117193669080734, "state_peak": 43.14808654785156, "growth_active": 1.232932209968567, "growth_memory": 1.2663840055465698, "heads": [{"head": 0, "memory_to_active": 2.639096975326538, "contribution_to_active": 0.49970847368240356, "mix_mean": 0.02020736038684845, "mix_low": 0.5920314788818359, "mix_high": 0.0, "read_cosine": 0.248055100440979, "beta_mean": 0.1900574117898941, "nonfinite": 0.0, "weighted_memory_to_active": 0.6002709269523621, "mix_std": 0.05968666821718216, "beta_std": 0.10647135972976685, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05134253948926926, "evidence_weights": [-0.01640632003545761, -0.06081245467066765, -0.11378981173038483], "agreement_weight": -0.163999542593956, "orthogonal_weight": -0.09419703483581543}, {"head": 1, "memory_to_active": 5.865094184875488, "contribution_to_active": 2.387700080871582, "mix_mean": 0.017239918932318687, "mix_low": 0.6550006866455078, "mix_high": 0.0, "read_cosine": 0.48847341537475586, "beta_mean": 0.41501402854919434, "nonfinite": 0.0, "weighted_memory_to_active": 2.687100887298584, "mix_std": 0.05263847857713699, "beta_std": 0.23091168701648712, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03298487886786461, "evidence_weights": [0.0229447390884161, -0.10318505764007568, 0.4786728620529175], "agreement_weight": 0.026547010987997055, "orthogonal_weight": -0.02632446028292179}, {"head": 2, "memory_to_active": 2.771291732788086, "contribution_to_active": 0.5356907844543457, "mix_mean": 0.014913585036993027, "mix_low": 0.6601524353027344, "mix_high": 0.0, "read_cosine": 0.31052133440971375, "beta_mean": 0.2766598165035248, "nonfinite": 0.0, "weighted_memory_to_active": 0.627669095993042, "mix_std": 0.03370215371251106, "beta_std": 0.20648953318595886, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05294038727879524, "evidence_weights": [0.05551944673061371, 0.0597693994641304, 0.06974298506975174], "agreement_weight": 0.007812057621777058, "orthogonal_weight": -0.1884104311466217}, {"head": 3, "memory_to_active": 7.537055015563965, "contribution_to_active": 3.1646175384521484, "mix_mean": 0.034094445407390594, "mix_low": 0.505284309387207, "mix_high": 0.0, "read_cosine": 0.3365006148815155, "beta_mean": 0.34313106536865234, "nonfinite": 0.0, "weighted_memory_to_active": 3.3741588592529297, "mix_std": 0.08187941461801529, "beta_std": 0.2543129324913025, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.15814220905303955, "evidence_weights": [-0.04253615438938141, -0.049203865230083466, -0.14958779513835907], "agreement_weight": -0.010474038310348988, "orthogonal_weight": -0.055818695574998856}, {"head": 4, "memory_to_active": 2.070194721221924, "contribution_to_active": 0.07029561698436737, "mix_mean": 0.007108848076313734, "mix_low": 0.8153753280639648, "mix_high": 0.0, "read_cosine": 0.33218473196029663, "beta_mean": 0.09302331507205963, "nonfinite": 0.0, "weighted_memory_to_active": 0.050911977887153625, "mix_std": 0.016395926475524902, "beta_std": 0.0727822408080101, "beta_low": 0.00701904296875, "beta_high": 0.0, "orthogonal_to_active": 0.005234569311141968, "evidence_weights": [0.14041152596473694, 0.09565289318561554, -0.3097487986087799], "agreement_weight": -0.13753212988376617, "orthogonal_weight": 0.09246920049190521}, {"head": 5, "memory_to_active": 0.10793166607618332, "contribution_to_active": 0.43617111444473267, "mix_mean": 0.5309622883796692, "mix_low": 0.01555633544921875, "mix_high": 0.032111167907714844, "read_cosine": 0.9346369504928589, "beta_mean": 0.027974940836429596, "nonfinite": 0.0, "weighted_memory_to_active": 0.05465780198574066, "mix_std": 0.29129907488822937, "beta_std": 0.026347417384386063, "beta_low": 0.22320556640625, "beta_high": 0.0, "orthogonal_to_active": 0.01849418878555298, "evidence_weights": [-0.1991092711687088, -0.22591784596443176, 0.3541402816772461], "agreement_weight": 0.28460127115249634, "orthogonal_weight": -0.7039740681648254}]}, {"layer": 1, "memory_to_active": 2.1777231693267822, "contribution_to_active": 0.46942389011383057, "mix_mean": 0.20225954055786133, "mix_low": 0.3658428192138672, "mix_high": 0.0011108716716989875, "read_cosine": 0.3325837254524231, "beta_mean": 0.16219744086265564, "nonfinite": 0.0, "weighted_memory_to_active": 0.3123811185359955, "mix_std": 0.11683294922113419, "beta_std": 0.12141038477420807, "beta_low": 0.3287251889705658, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.043580830097198486, "output_influence": 0.21599945425987244, "residual_influence": 0.05540389567613602, "state_active": 0.03218936547636986, "state_memory": 0.03464553877711296, "state_peak": 38.20493698120117, "growth_active": 1.234689474105835, "growth_memory": 1.502724528312683, "heads": [{"head": 0, "memory_to_active": 0.04658214747905731, "contribution_to_active": 0.46655187010765076, "mix_mean": 0.4805375635623932, "mix_low": 0.008803367614746094, "mix_high": 0.001880645751953125, "read_cosine": 0.06696200370788574, "beta_mean": 0.000360443809768185, "nonfinite": 0.0, "weighted_memory_to_active": 0.025218579918146133, "mix_std": 0.2826806902885437, "beta_std": 0.000539685133844614, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.024146057665348053, "evidence_weights": [-0.19838887453079224, -0.2587667405605316, 0.356853187084198], "agreement_weight": 0.19470956921577454, "orthogonal_weight": -1.0292295217514038}, {"head": 1, "memory_to_active": 2.61635684967041, "contribution_to_active": 0.20218592882156372, "mix_mean": 0.02499374933540821, "mix_low": 0.4270210266113281, "mix_high": 2.86102294921875e-06, "read_cosine": 0.5326094031333923, "beta_mean": 0.2315244823694229, "nonfinite": 0.0, "weighted_memory_to_active": 0.2377442717552185, "mix_std": 0.04889211803674698, "beta_std": 0.16579994559288025, "beta_low": 0.00067138671875, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.01497689075767994, "evidence_weights": [0.012671076692640781, -0.05150814726948738, 0.026992695406079292], "agreement_weight": -0.0049723852425813675, "orthogonal_weight": 0.15243810415267944}, {"head": 2, "memory_to_active": 0.0955989807844162, "contribution_to_active": 0.5650720000267029, "mix_mean": 0.6474725008010864, "mix_low": 0.0007028579711914062, "mix_high": 0.004778861999511719, "read_cosine": 0.5581858158111572, "beta_mean": 0.0049883523024618626, "nonfinite": 0.0, "weighted_memory_to_active": 0.07231520116329193, "mix_std": 0.23764267563819885, "beta_std": 0.006582111120223999, "beta_low": 0.8695068359375, "beta_high": 0.0, "orthogonal_to_active": 0.03753884136676788, "evidence_weights": [-0.29875466227531433, -0.3522326350212097, 0.625146746635437], "agreement_weight": 0.02775641344487667, "orthogonal_weight": -0.6584072113037109}, {"head": 3, "memory_to_active": 2.793226718902588, "contribution_to_active": 0.13985486328601837, "mix_mean": 0.018711687996983528, "mix_low": 0.6335172653198242, "mix_high": 0.0, "read_cosine": 0.22515586018562317, "beta_mean": 0.3219911456108093, "nonfinite": 0.0, "weighted_memory_to_active": 0.2193019688129425, "mix_std": 0.052071936428546906, "beta_std": 0.2553189992904663, "beta_low": 0.0467529296875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.05771467834711075, "evidence_weights": [0.04839549586176872, -0.001985861686989665, 0.09086274355649948], "agreement_weight": -0.019244777038693428, "orthogonal_weight": -0.41704443097114563}, {"head": 4, "memory_to_active": 3.642885446548462, "contribution_to_active": 1.3724907636642456, "mix_mean": 0.03494778275489807, "mix_low": 0.2891254425048828, "mix_high": 2.86102294921875e-06, "read_cosine": 0.34584394097328186, "beta_mean": 0.14927750825881958, "nonfinite": 0.0, "weighted_memory_to_active": 1.265159249305725, "mix_std": 0.058620572090148926, "beta_std": 0.10267659276723862, "beta_low": 0.02313232421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.11522485315799713, "evidence_weights": [-0.03897838294506073, -0.0009886627085506916, 0.05363016575574875], "agreement_weight": -0.014815631322562695, "orthogonal_weight": 0.12482825666666031}, {"head": 5, "memory_to_active": 3.8716883659362793, "contribution_to_active": 0.07038787007331848, "mix_mean": 0.006893886253237724, "mix_low": 0.8358869552612305, "mix_high": 0.0, "read_cosine": 0.2667454481124878, "beta_mean": 0.2650427222251892, "nonfinite": 0.0, "weighted_memory_to_active": 0.05454732105135918, "mix_std": 0.02108968421816826, "beta_std": 0.19754494726657867, "beta_low": 0.03228759765625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.011883658356964588, "evidence_weights": [0.16654038429260254, 0.08101334422826767, 0.23929308354854584], "agreement_weight": -0.13377134501934052, "orthogonal_weight": 0.21415992081165314}]}, {"layer": 2, "memory_to_active": 0.9869104027748108, "contribution_to_active": 0.3482058346271515, "mix_mean": 0.3813632130622864, "mix_low": 0.24713151156902313, "mix_high": 0.003976345062255859, "read_cosine": 0.3220008313655853, "beta_mean": 0.09267526119947433, "nonfinite": 0.0, "weighted_memory_to_active": 0.07485786825418472, "mix_std": 0.19230765104293823, "beta_std": 0.07293461263179779, "beta_low": 0.5595703125, "beta_high": 0.0001932780141942203, "orthogonal_to_active": 0.017170535400509834, "output_influence": 0.3640254735946655, "residual_influence": 0.09238573908805847, "state_active": 0.037002094089984894, "state_memory": 0.016626432538032532, "state_peak": 33.083900451660156, "growth_active": 1.0223017930984497, "growth_memory": 1.5652434825897217, "heads": [{"head": 0, "memory_to_active": 0.013202771544456482, "contribution_to_active": 0.37057679891586304, "mix_mean": 0.5917675495147705, "mix_low": 0.0012693405151367188, "mix_high": 0.007891654968261719, "read_cosine": 0.15432533621788025, "beta_mean": 9.028022759594023e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007793716620653868, "mix_std": 0.28240031003952026, "beta_std": 0.0001520475634606555, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006448534317314625, "evidence_weights": [-0.34838026762008667, -0.2097712904214859, 0.10924696177244186], "agreement_weight": 0.2272050380706787, "orthogonal_weight": -0.858932614326477}, {"head": 1, "memory_to_active": 2.7701497077941895, "contribution_to_active": 0.04321420192718506, "mix_mean": 0.0051158517599105835, "mix_low": 0.8916759490966797, "mix_high": 0.0, "read_cosine": 0.1800377070903778, "beta_mean": 0.25649887323379517, "nonfinite": 0.0, "weighted_memory_to_active": 0.03500189259648323, "mix_std": 0.011878452263772488, "beta_std": 0.16989631950855255, "beta_low": 0.00250244140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.012344003655016422, "evidence_weights": [0.12507392466068268, 0.11281494796276093, 0.5366937518119812], "agreement_weight": 0.032473377883434296, "orthogonal_weight": 0.39056816697120667}, {"head": 2, "memory_to_active": 0.008766509592533112, "contribution_to_active": 0.410114586353302, "mix_mean": 0.5231775045394897, "mix_low": 0.001392364501953125, "mix_high": 0.0024347305297851562, "read_cosine": 0.33543887734413147, "beta_mean": 1.9442497432464734e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034491100814193487, "mix_std": 0.2787034511566162, "beta_std": 4.668383553507738e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000599128135945648, "evidence_weights": [-0.2461037039756775, -0.28440380096435547, 0.4951108992099762], "agreement_weight": 0.0016352829989045858, "orthogonal_weight": -0.20431236922740936}, {"head": 3, "memory_to_active": 0.1366250067949295, "contribution_to_active": 0.5223335027694702, "mix_mean": 0.5262943506240845, "mix_low": 0.0003185272216796875, "mix_high": 0.0062694549560546875, "read_cosine": 0.6444321870803833, "beta_mean": 0.02719363570213318, "nonfinite": 0.0, "weighted_memory_to_active": 0.07985446602106094, "mix_std": 0.26306551694869995, "beta_std": 0.03422502055764198, "beta_low": 0.3363037109375, "beta_high": 0.0, "orthogonal_to_active": 0.05263921990990639, "evidence_weights": [-0.18996112048625946, -0.29875391721725464, 0.8454402089118958], "agreement_weight": 0.05927429720759392, "orthogonal_weight": -0.8934085965156555}, {"head": 4, "memory_to_active": 2.9795656204223633, "contribution_to_active": 0.2339581549167633, "mix_mean": 0.01956489495933056, "mix_low": 0.5867233276367188, "mix_high": 0.0, "read_cosine": 0.20036685466766357, "beta_mean": 0.2722042500972748, "nonfinite": 0.0, "weighted_memory_to_active": 0.3155156075954437, "mix_std": 0.049816615879535675, "beta_std": 0.23322781920433044, "beta_low": 0.01861572265625, "beta_high": 0.0010986328125, "orthogonal_to_active": 0.030987754464149475, "evidence_weights": [0.01943313144147396, 0.0066401055082678795, 0.024770183488726616], "agreement_weight": 0.022771183401346207, "orthogonal_weight": -0.1170707419514656}, {"head": 5, "memory_to_active": 0.013152636587619781, "contribution_to_active": 0.5090375542640686, "mix_mean": 0.6222588419914246, "mix_low": 0.0014095306396484375, "mix_high": 0.007262229919433594, "read_cosine": 0.41740402579307556, "beta_mean": 4.503979653236456e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0075324042700231075, "mix_std": 0.26798155903816223, "beta_std": 5.9751175285782665e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 4.576903847919311e-06, "evidence_weights": [-0.27400967478752136, -0.28793996572494507, 0.9662138819694519], "agreement_weight": 0.13929708302021027, "orthogonal_weight": -0.0008375736651942134}]}, {"layer": 3, "memory_to_active": 0.5200421214103699, "contribution_to_active": 0.43480217456817627, "mix_mean": 0.44248586893081665, "mix_low": 0.06781339645385742, "mix_high": 0.008921623229980469, "read_cosine": 0.21910901367664337, "beta_mean": 0.029621975496411324, "nonfinite": 0.0, "weighted_memory_to_active": 0.14186076819896698, "mix_std": 0.2627326250076294, "beta_std": 0.026416432112455368, "beta_low": 0.7266134023666382, "beta_high": 0.0, "orthogonal_to_active": 0.06115759164094925, "output_influence": 0.3710471987724304, "residual_influence": 0.07886424660682678, "state_active": 0.022106017917394638, "state_memory": 0.007582188583910465, "state_peak": 22.52851676940918, "growth_active": 1.0048927068710327, "growth_memory": 1.7271268367767334, "heads": [{"head": 0, "memory_to_active": 0.4274003803730011, "contribution_to_active": 0.5566032528877258, "mix_mean": 0.5489636659622192, "mix_low": 0.0039196014404296875, "mix_high": 0.02180767059326172, "read_cosine": 0.2858846187591553, "beta_mean": 0.012084497138857841, "nonfinite": 0.0, "weighted_memory_to_active": 0.2631644010543823, "mix_std": 0.31637832522392273, "beta_std": 0.01652011275291443, "beta_low": 0.6231689453125, "beta_high": 0.0, "orthogonal_to_active": 0.17930302023887634, "evidence_weights": [-0.20439861714839935, -0.23651820421218872, 0.8441937565803528], "agreement_weight": 0.12077756971120834, "orthogonal_weight": -0.7369420528411865}, {"head": 1, "memory_to_active": 0.016215451061725616, "contribution_to_active": 0.3894616961479187, "mix_mean": 0.517073392868042, "mix_low": 0.008106231689453125, "mix_high": 0.007374763488769531, "read_cosine": 0.1571432650089264, "beta_mean": 8.439793600700796e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008559506386518478, "mix_std": 0.3092477023601532, "beta_std": 0.0001772555406205356, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006408365443348885, "evidence_weights": [-0.24374079704284668, -0.20366109907627106, 0.6043747663497925], "agreement_weight": 0.4114391803741455, "orthogonal_weight": -0.8107662200927734}, {"head": 2, "memory_to_active": 2.3188633918762207, "contribution_to_active": 0.29283568263053894, "mix_mean": 0.04577898234128952, "mix_low": 0.3503456115722656, "mix_high": 0.0, "read_cosine": 0.22949998080730438, "beta_mean": 0.15647616982460022, "nonfinite": 0.0, "weighted_memory_to_active": 0.3380638062953949, "mix_std": 0.09802268445491791, "beta_std": 0.1297314465045929, "beta_low": 0.01544189453125, "beta_high": 0.0, "orthogonal_to_active": 0.0023250833619385958, "evidence_weights": [-0.08201278746128082, 0.013027170673012733, -0.00843522883951664], "agreement_weight": -0.008641839027404785, "orthogonal_weight": -0.008839716203510761}, {"head": 3, "memory_to_active": 0.030939551070332527, "contribution_to_active": 0.5134327411651611, "mix_mean": 0.5698493719100952, "mix_low": 0.008077621459960938, "mix_high": 0.008928298950195312, "read_cosine": 0.28941845893859863, "beta_mean": 0.00011336884927004576, "nonfinite": 0.0, "weighted_memory_to_active": 0.020591722801327705, "mix_std": 0.2962861657142639, "beta_std": 0.00032197576365433633, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005355366505682468, "evidence_weights": [-0.23249170184135437, -0.2283651977777481, 2.1324665546417236], "agreement_weight": 1.775986909866333, "orthogonal_weight": -0.34524163603782654}, {"head": 4, "memory_to_active": 0.061955526471138, "contribution_to_active": 0.24345247447490692, "mix_mean": 0.30300602316856384, "mix_low": 0.031864166259765625, "mix_high": 0.0007419586181640625, "read_cosine": 0.19746097922325134, "beta_mean": 0.0004100670921616256, "nonfinite": 0.0, "weighted_memory_to_active": 0.028843849897384644, "mix_std": 0.27395159006118774, "beta_std": 0.0005582526791840792, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.026430625468492508, "evidence_weights": [-0.12206489592790604, -0.1524345427751541, 0.1637813001871109], "agreement_weight": -0.021785715594887733, "orthogonal_weight": -0.9666365385055542}, {"head": 5, "memory_to_active": 0.2648783326148987, "contribution_to_active": 0.6130273342132568, "mix_mean": 0.6702437400817871, "mix_low": 0.004567146301269531, "mix_high": 0.014677047729492188, "read_cosine": 0.15524671971797943, "beta_mean": 0.008563335984945297, "nonfinite": 0.0, "weighted_memory_to_active": 0.19194121658802032, "mix_std": 0.2825092673301697, "beta_std": 0.011189552024006844, "beta_low": 0.72119140625, "beta_high": 0.0, "orthogonal_to_active": 0.1471230685710907, "evidence_weights": [-0.2502513527870178, -0.2995317578315735, 0.5043656229972839], "agreement_weight": 0.13342101871967316, "orthogonal_weight": -0.8086647987365723}]}, {"layer": 4, "memory_to_active": 1.1142133474349976, "contribution_to_active": 0.4508030116558075, "mix_mean": 0.3580721616744995, "mix_low": 0.2784341275691986, "mix_high": 0.008088271133601665, "read_cosine": 0.3302236795425415, "beta_mean": 0.11871498078107834, "nonfinite": 0.0, "weighted_memory_to_active": 0.13881705701351166, "mix_std": 0.1561635136604309, "beta_std": 0.10855347663164139, "beta_low": 0.3775329887866974, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04293616861104965, "output_influence": 0.4360272288322449, "residual_influence": 0.12151428312063217, "state_active": 0.038274653255939484, "state_memory": 0.018380654975771904, "state_peak": 37.25277328491211, "growth_active": 1.2248624563217163, "growth_memory": 1.5358308553695679, "heads": [{"head": 0, "memory_to_active": 0.21656738221645355, "contribution_to_active": 0.719450831413269, "mix_mean": 0.6670680046081543, "mix_low": 0.011380195617675781, "mix_high": 0.02105712890625, "read_cosine": 0.5067300796508789, "beta_mean": 0.26001113653182983, "nonfinite": 0.0, "weighted_memory_to_active": 0.16830205917358398, "mix_std": 0.341153085231781, "beta_std": 0.27038857340812683, "beta_low": 0.22943115234375, "beta_high": 0.0, "orthogonal_to_active": 0.06473612040281296, "evidence_weights": [-0.26819559931755066, -0.3690866529941559, 0.13626518845558167], "agreement_weight": 0.014907727017998695, "orthogonal_weight": 0.4922798275947571}, {"head": 1, "memory_to_active": 0.05895909294486046, "contribution_to_active": 0.6286680102348328, "mix_mean": 0.6715519428253174, "mix_low": 6.198883056640625e-05, "mix_high": 0.011002540588378906, "read_cosine": 0.655066967010498, "beta_mean": 0.0008493763161823153, "nonfinite": 0.0, "weighted_memory_to_active": 0.04258010536432266, "mix_std": 0.24307341873645782, "beta_std": 0.0045730131678283215, "beta_low": 0.98583984375, "beta_high": 0.0, "orthogonal_to_active": 0.023926159366965294, "evidence_weights": [-0.2384355515241623, -0.346879780292511, 0.7044986486434937], "agreement_weight": 0.22409425675868988, "orthogonal_weight": -0.8132272362709045}, {"head": 2, "memory_to_active": 2.0287492275238037, "contribution_to_active": 0.24008002877235413, "mix_mean": 0.02218587137758732, "mix_low": 0.5666561126708984, "mix_high": 7.62939453125e-06, "read_cosine": 0.19665879011154175, "beta_mean": 0.09015975892543793, "nonfinite": 0.0, "weighted_memory_to_active": 0.20920643210411072, "mix_std": 0.04884936660528183, "beta_std": 0.08628448843955994, "beta_low": 0.02496337890625, "beta_high": 0.0, "orthogonal_to_active": 0.027134479954838753, "evidence_weights": [0.057548023760318756, 0.022361235693097115, 0.05132674053311348], "agreement_weight": 0.13153794407844543, "orthogonal_weight": -0.17895068228244781}, {"head": 3, "memory_to_active": 2.097062587738037, "contribution_to_active": 0.15255165100097656, "mix_mean": 0.031066888943314552, "mix_low": 0.43628597259521484, "mix_high": 0.0, "read_cosine": 0.21824368834495544, "beta_mean": 0.1330312341451645, "nonfinite": 0.0, "weighted_memory_to_active": 0.2059730589389801, "mix_std": 0.05718237906694412, "beta_std": 0.12218627333641052, "beta_low": 0.02484130859375, "beta_high": 0.0, "orthogonal_to_active": 0.006474202498793602, "evidence_weights": [-0.02223879285156727, 0.06273377686738968, -0.23988930881023407], "agreement_weight": 0.06594929099082947, "orthogonal_weight": -0.04318564012646675}, {"head": 4, "memory_to_active": 0.021710649132728577, "contribution_to_active": 0.6549967527389526, "mix_mean": 0.7430040240287781, "mix_low": 0.0001163482666015625, "mix_high": 0.016454696655273438, "read_cosine": 0.14127066731452942, "beta_mean": 3.514252239256166e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015523970127105713, "mix_std": 0.21655301749706268, "beta_std": 0.00012216497270856053, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00522010400891304, "evidence_weights": [-0.33676373958587646, -0.332571417093277, 0.5964679718017578], "agreement_weight": 0.3307986855506897, "orthogonal_weight": -0.35920989513397217}, {"head": 5, "memory_to_active": 2.26223087310791, "contribution_to_active": 0.3090706467628479, "mix_mean": 0.013556144200265408, "mix_low": 0.6561040878295898, "mix_high": 7.62939453125e-06, "read_cosine": 0.26337191462516785, "beta_mean": 0.2282031625509262, "nonfinite": 0.0, "weighted_memory_to_active": 0.19131672382354736, "mix_std": 0.03016972541809082, "beta_std": 0.16776631772518158, "beta_low": 0.0001220703125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.13012592494487762, "evidence_weights": [0.051211658865213394, -0.027847053483128548, -0.3029232919216156], "agreement_weight": -0.06827707588672638, "orthogonal_weight": 0.8991646766662598}]}, {"layer": 5, "memory_to_active": 0.46672523021698, "contribution_to_active": 0.5155342817306519, "mix_mean": 0.4816475808620453, "mix_low": 0.06375821679830551, "mix_high": 0.016797225922346115, "read_cosine": 0.08845958113670349, "beta_mean": 0.05406564474105835, "nonfinite": 0.0, "weighted_memory_to_active": 0.1499333381652832, "mix_std": 0.25627636909484863, "beta_std": 0.0714445561170578, "beta_low": 0.6492106318473816, "beta_high": 0.0, "orthogonal_to_active": 0.05618217587471008, "output_influence": 0.5187765955924988, "residual_influence": 0.14208471775054932, "state_active": 0.043616313487291336, "state_memory": 0.011181565001606941, "state_peak": 64.16127014160156, "growth_active": 1.1102603673934937, "growth_memory": 1.266550064086914, "heads": [{"head": 0, "memory_to_active": 0.6176521182060242, "contribution_to_active": 0.5098228454589844, "mix_mean": 0.3385254740715027, "mix_low": 0.027459144592285156, "mix_high": 0.0010242462158203125, "read_cosine": -0.09907360374927521, "beta_mean": 0.030435804277658463, "nonfinite": 0.0, "weighted_memory_to_active": 0.15399441123008728, "mix_std": 0.27048036456108093, "beta_std": 0.05735688656568527, "beta_low": 0.418212890625, "beta_high": 0.0, "orthogonal_to_active": 0.04922078177332878, "evidence_weights": [-0.12177343666553497, -0.15191417932510376, -0.09874420613050461], "agreement_weight": -0.023094555363059044, "orthogonal_weight": -0.3166506290435791}, {"head": 1, "memory_to_active": 0.01839800924062729, "contribution_to_active": 0.5558301210403442, "mix_mean": 0.6772168278694153, "mix_low": 0.001667022705078125, "mix_high": 0.031658172607421875, "read_cosine": 0.16261112689971924, "beta_mean": 4.318158244132064e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010410519316792488, "mix_std": 0.28071796894073486, "beta_std": 8.99401493370533e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009798233397305012, "evidence_weights": [-0.24535216391086578, -0.2814503312110901, 0.875091552734375], "agreement_weight": 0.6099367737770081, "orthogonal_weight": -0.9836573004722595}, {"head": 2, "memory_to_active": 0.015967776998877525, "contribution_to_active": 0.5432512760162354, "mix_mean": 0.6432031393051147, "mix_low": 0.0037994384765625, "mix_high": 0.02075481414794922, "read_cosine": -0.36391669511795044, "beta_mean": 0.00016154631157405674, "nonfinite": 0.0, "weighted_memory_to_active": 0.009860423393547535, "mix_std": 0.27534031867980957, "beta_std": 0.00022891766275279224, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003048686543479562, "evidence_weights": [-0.25832030177116394, -0.3446817398071289, 0.5675895810127258], "agreement_weight": -0.0014313817955553532, "orthogonal_weight": -0.39112988114356995}, {"head": 3, "memory_to_active": 1.580013632774353, "contribution_to_active": 0.21222512423992157, "mix_mean": 0.09810344129800797, "mix_low": 0.15637493133544922, "mix_high": 4.76837158203125e-06, "read_cosine": 0.14993947744369507, "beta_mean": 0.11630142480134964, "nonfinite": 0.0, "weighted_memory_to_active": 0.34067609906196594, "mix_std": 0.135370135307312, "beta_std": 0.16149885952472687, "beta_low": 0.18194580078125, "beta_high": 0.0, "orthogonal_to_active": 0.12655261158943176, "evidence_weights": [-0.06355249881744385, -0.09402323514223099, 0.018621662631630898], "agreement_weight": 0.05947539582848549, "orthogonal_weight": -0.4762847125530243}, {"head": 4, "memory_to_active": 0.4636170566082001, "contribution_to_active": 0.595902681350708, "mix_mean": 0.335828959941864, "mix_low": 0.1908407211303711, "mix_high": 0.007950782775878906, "read_cosine": 0.5224782228469849, "beta_mean": 0.17714570462703705, "nonfinite": 0.0, "weighted_memory_to_active": 0.31145453453063965, "mix_std": 0.34786659479141235, "beta_std": 0.20806261897087097, "beta_low": 0.29949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.10291652381420135, "evidence_weights": [-0.11341749876737595, -0.2778843939304352, -0.07792513072490692], "agreement_weight": 0.03812660276889801, "orthogonal_weight": 0.48714959621429443}, {"head": 5, "memory_to_active": 0.10470256209373474, "contribution_to_active": 0.6761736869812012, "mix_mean": 0.7970076203346252, "mix_low": 0.0024080276489257812, "mix_high": 0.03939056396484375, "read_cosine": 0.15871897339820862, "beta_mean": 0.00030617142328992486, "nonfinite": 0.0, "weighted_memory_to_active": 0.07320401072502136, "mix_std": 0.2278827726840973, "beta_std": 0.0014301041373983026, "beta_low": 0.99560546875, "beta_high": 0.0, "orthogonal_to_active": 0.04555618017911911, "evidence_weights": [-0.320291131734848, -0.3322615623474121, 1.980884075164795], "agreement_weight": 0.42639344930648804, "orthogonal_weight": -0.6628342270851135}]}, {"layer": 6, "memory_to_active": 0.3355065584182739, "contribution_to_active": 0.6066706776618958, "mix_mean": 0.5697734355926514, "mix_low": 0.057686805725097656, "mix_high": 0.029404325410723686, "read_cosine": 0.3067971169948578, "beta_mean": 0.06746678799390793, "nonfinite": 0.0, "weighted_memory_to_active": 0.16455990076065063, "mix_std": 0.23847582936286926, "beta_std": 0.07598169893026352, "beta_low": 0.5088399648666382, "beta_high": 0.0, "orthogonal_to_active": 0.03899596631526947, "output_influence": 0.5205787420272827, "residual_influence": 0.16768518090248108, "state_active": 0.04857373982667923, "state_memory": 0.008727256208658218, "state_peak": 96.46726989746094, "growth_active": 1.187667965888977, "growth_memory": 1.1193586587905884, "heads": [{"head": 0, "memory_to_active": 0.11357884109020233, "contribution_to_active": 0.680475652217865, "mix_mean": 0.7768131494522095, "mix_low": 0.000225067138671875, "mix_high": 0.0250701904296875, "read_cosine": 0.1820584088563919, "beta_mean": 0.016747815534472466, "nonfinite": 0.0, "weighted_memory_to_active": 0.0911603718996048, "mix_std": 0.2151806503534317, "beta_std": 0.02032366581261158, "beta_low": 0.48748779296875, "beta_high": 0.0, "orthogonal_to_active": 0.07468033581972122, "evidence_weights": [-0.3187222480773926, -0.341596394777298, 1.0493378639221191], "agreement_weight": 0.32259371876716614, "orthogonal_weight": -0.8866716623306274}, {"head": 1, "memory_to_active": 0.005007867701351643, "contribution_to_active": 0.4888783097267151, "mix_mean": 0.5535846948623657, "mix_low": 0.0010967254638671875, "mix_high": 0.012269020080566406, "read_cosine": 0.556808590888977, "beta_mean": 1.0806160389620345e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002121392637491226, "mix_std": 0.28724098205566406, "beta_std": 1.7973581634578295e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011384047102183104, "evidence_weights": [-0.19917377829551697, -0.2961631715297699, 1.0687519311904907], "agreement_weight": 0.0674055740237236, "orthogonal_weight": -0.7351492047309875}, {"head": 2, "memory_to_active": 0.9519070386886597, "contribution_to_active": 0.08874786645174026, "mix_mean": 0.03673567250370979, "mix_low": 0.33614063262939453, "mix_high": 0.0, "read_cosine": 0.23620954155921936, "beta_mean": 0.03691357374191284, "nonfinite": 0.0, "weighted_memory_to_active": 0.06531068682670593, "mix_std": 0.05955156683921814, "beta_std": 0.035639628767967224, "beta_low": 0.08062744140625, "beta_high": 0.0, "orthogonal_to_active": 0.0025286669842898846, "evidence_weights": [0.08042062073945999, -0.03031696006655693, 0.2874813675880432], "agreement_weight": 0.1378830522298813, "orthogonal_weight": -0.045160118490457535}, {"head": 3, "memory_to_active": 0.1510780155658722, "contribution_to_active": 0.6556442975997925, "mix_mean": 0.6395200490951538, "mix_low": 0.0074863433837890625, "mix_high": 0.016420364379882812, "read_cosine": 0.4540301561355591, "beta_mean": 0.2195107638835907, "nonfinite": 0.0, "weighted_memory_to_active": 0.10506860911846161, "mix_std": 0.32574719190597534, "beta_std": 0.25081878900527954, "beta_low": 0.2081298828125, "beta_high": 0.0, "orthogonal_to_active": 0.05745397135615349, "evidence_weights": [-0.17495891451835632, -0.3306646943092346, 0.18983015418052673], "agreement_weight": -0.006759734358638525, "orthogonal_weight": 0.6277987957000732}, {"head": 4, "memory_to_active": 0.7574671506881714, "contribution_to_active": 1.1086162328720093, "mix_mean": 0.752727210521698, "mix_low": 0.0007658004760742188, "mix_high": 0.11751174926757812, "read_cosine": 0.13632804155349731, "beta_mean": 0.1313944309949875, "nonfinite": 0.0, "weighted_memory_to_active": 0.6995177865028381, "mix_std": 0.29066550731658936, "beta_std": 0.14862282574176788, "beta_low": 0.277099609375, "beta_high": 0.0, "orthogonal_to_active": 0.07419826835393906, "evidence_weights": [-0.24047471582889557, -0.5939876437187195, -0.23947006464004517], "agreement_weight": -0.01970856264233589, "orthogonal_weight": 0.12267542630434036}, {"head": 5, "memory_to_active": 0.034000225365161896, "contribution_to_active": 0.6176614761352539, "mix_mean": 0.659259557723999, "mix_low": 0.0004062652587890625, "mix_high": 0.005154609680175781, "read_cosine": 0.27534791827201843, "beta_mean": 0.00022330487263388932, "nonfinite": 0.0, "weighted_memory_to_active": 0.024180524051189423, "mix_std": 0.252469003200531, "beta_std": 0.0004673149378504604, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.023976147174835205, "evidence_weights": [-0.24365994334220886, -0.3130326569080353, 0.41817328333854675], "agreement_weight": 0.1634962260723114, "orthogonal_weight": -1.0622413158416748}]}, {"layer": 7, "memory_to_active": 1.7727634906768799, "contribution_to_active": 0.26781392097473145, "mix_mean": 0.13737106323242188, "mix_low": 0.2666899561882019, "mix_high": 0.0008123716106638312, "read_cosine": 0.22341033816337585, "beta_mean": 0.13430353999137878, "nonfinite": 0.0, "weighted_memory_to_active": 0.23132310807704926, "mix_std": 0.14029039442539215, "beta_std": 0.10082947462797165, "beta_low": 0.1979573667049408, "beta_high": 0.00020345053053461015, "orthogonal_to_active": 0.07130042463541031, "output_influence": 0.2666934132575989, "residual_influence": 0.059290602803230286, "state_active": 0.013028981164097786, "state_memory": 0.026691481471061707, "state_peak": 30.905588150024414, "growth_active": 1.1276347637176514, "growth_memory": 1.310394287109375, "heads": [{"head": 0, "memory_to_active": 2.4721806049346924, "contribution_to_active": 0.35261112451553345, "mix_mean": 0.037036310881376266, "mix_low": 0.4300870895385742, "mix_high": 9.5367431640625e-07, "read_cosine": 0.33914270997047424, "beta_mean": 0.16715484857559204, "nonfinite": 0.0, "weighted_memory_to_active": 0.4231380820274353, "mix_std": 0.07854161411523819, "beta_std": 0.12083537876605988, "beta_low": 0.00048828125, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.0043604401871562, "evidence_weights": [0.026226267218589783, -0.0751764103770256, 0.07270223647356033], "agreement_weight": -0.0351874865591526, "orthogonal_weight": 0.018268857151269913}, {"head": 1, "memory_to_active": 1.0610857009887695, "contribution_to_active": 0.40858662128448486, "mix_mean": 0.21199586987495422, "mix_low": 0.12551021575927734, "mix_high": 0.004385948181152344, "read_cosine": 0.3333592712879181, "beta_mean": 0.09973552078008652, "nonfinite": 0.0, "weighted_memory_to_active": 0.3485701084136963, "mix_std": 0.2400180846452713, "beta_std": 0.0705743208527565, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.03753811866044998, "evidence_weights": [-0.20651619136333466, -0.10621233284473419, -0.22708654403686523], "agreement_weight": 0.04283321648836136, "orthogonal_weight": 0.12634357810020447}, {"head": 2, "memory_to_active": 0.7579089403152466, "contribution_to_active": 0.3113161027431488, "mix_mean": 0.15424728393554688, "mix_low": 0.064453125, "mix_high": 0.00031757354736328125, "read_cosine": 0.08822618424892426, "beta_mean": 0.04844389855861664, "nonfinite": 0.0, "weighted_memory_to_active": 0.22053588926792145, "mix_std": 0.18070152401924133, "beta_std": 0.04097878187894821, "beta_low": 0.06689453125, "beta_high": 0.0, "orthogonal_to_active": 0.09297719597816467, "evidence_weights": [-0.06213068217039108, -0.14400489628314972, 0.6229895949363708], "agreement_weight": 0.024299541488289833, "orthogonal_weight": -0.43255606293678284}, {"head": 3, "memory_to_active": 4.584263801574707, "contribution_to_active": 0.09820696711540222, "mix_mean": 0.03283381462097168, "mix_low": 0.3462200164794922, "mix_high": 0.0, "read_cosine": 0.08220022916793823, "beta_mean": 0.3977653980255127, "nonfinite": 0.0, "weighted_memory_to_active": 0.31223076581954956, "mix_std": 0.04805488884449005, "beta_std": 0.2724386751651764, "beta_low": 0.00408935546875, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.26552051305770874, "evidence_weights": [0.07126407325267792, -0.008999956771731377, 0.6906283497810364], "agreement_weight": 0.051345039159059525, "orthogonal_weight": -0.9378674626350403}, {"head": 4, "memory_to_active": 1.6939908266067505, "contribution_to_active": 0.0677366703748703, "mix_mean": 0.016919076442718506, "mix_low": 0.6285343170166016, "mix_high": 0.0, "read_cosine": 0.2093663513660431, "beta_mean": 0.09235401451587677, "nonfinite": 0.0, "weighted_memory_to_active": 0.05664487183094025, "mix_std": 0.035910360515117645, "beta_std": 0.09934064745903015, "beta_low": 0.1171875, "beta_high": 0.0, "orthogonal_to_active": 0.008500387892127037, "evidence_weights": [0.14380784332752228, 0.02859380841255188, 0.21571224927902222], "agreement_weight": 0.045193884521722794, "orthogonal_weight": -0.18615791201591492}, {"head": 5, "memory_to_active": 0.06715063750743866, "contribution_to_active": 0.36842602491378784, "mix_mean": 0.3711940348148346, "mix_low": 0.0053348541259765625, "mix_high": 0.0001697540283203125, "read_cosine": 0.2881672978401184, "beta_mean": 0.0003675668267533183, "nonfinite": 0.0, "weighted_memory_to_active": 0.026818880811333656, "mix_std": 0.2585158944129944, "beta_std": 0.0008089903858490288, "beta_low": 0.9990234375, "beta_high": 0.0, "orthogonal_to_active": 0.018905874341726303, "evidence_weights": [-0.1424430012702942, -0.202209934592247, 0.5870292782783508], "agreement_weight": 0.16312062740325928, "orthogonal_weight": -0.777038037776947}]}, {"layer": 8, "memory_to_active": 0.3395867347717285, "contribution_to_active": 0.529966413974762, "mix_mean": 0.4330873191356659, "mix_low": 0.06471633911132812, "mix_high": 0.0011358261108398438, "read_cosine": 0.4491484463214874, "beta_mean": 0.02046358212828636, "nonfinite": 0.0, "weighted_memory_to_active": 0.09053659439086914, "mix_std": 0.2187337726354599, "beta_std": 0.02964327298104763, "beta_low": 0.6756591796875, "beta_high": 0.0, "orthogonal_to_active": 0.0677720233798027, "output_influence": 0.4749177098274231, "residual_influence": 0.15254774689674377, "state_active": 0.0449376255273819, "state_memory": 0.007893125526607037, "state_peak": 32.073123931884766, "growth_active": 1.0519322156906128, "growth_memory": 1.2713367938995361, "heads": [{"head": 0, "memory_to_active": 0.06190704181790352, "contribution_to_active": 0.5536817908287048, "mix_mean": 0.5989168286323547, "mix_low": 0.0003795623779296875, "mix_high": 0.0010223388671875, "read_cosine": 0.447765052318573, "beta_mean": 0.003162251552566886, "nonfinite": 0.0, "weighted_memory_to_active": 0.043401800096035004, "mix_std": 0.24374359846115112, "beta_std": 0.01080256886780262, "beta_low": 0.92950439453125, "beta_high": 0.0, "orthogonal_to_active": 0.028153229504823685, "evidence_weights": [-0.16961942613124847, -0.3888099193572998, 0.6707491278648376], "agreement_weight": 0.1757642924785614, "orthogonal_weight": -0.8037837147712708}, {"head": 1, "memory_to_active": 0.12790557742118835, "contribution_to_active": 0.5206958055496216, "mix_mean": 0.4917911887168884, "mix_low": 0.00018978118896484375, "mix_high": 0.000728607177734375, "read_cosine": 0.24557790160179138, "beta_mean": 0.0029607200995087624, "nonfinite": 0.0, "weighted_memory_to_active": 0.06367605179548264, "mix_std": 0.24384748935699463, "beta_std": 0.013112717308104038, "beta_low": 0.95849609375, "beta_high": 0.0, "orthogonal_to_active": 0.04321172460913658, "evidence_weights": [-0.1912764608860016, -0.26896563172340393, 0.693832278251648], "agreement_weight": 0.08582156151533127, "orthogonal_weight": -0.733285665512085}, {"head": 2, "memory_to_active": 0.01323908381164074, "contribution_to_active": 0.6694332957267761, "mix_mean": 0.654335081577301, "mix_low": 0.0011425018310546875, "mix_high": 0.0030984878540039062, "read_cosine": 0.39001137018203735, "beta_mean": 0.00014672367251478136, "nonfinite": 0.0, "weighted_memory_to_active": 0.010166811756789684, "mix_std": 0.22584083676338196, "beta_std": 0.0036605186760425568, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006545231211930513, "evidence_weights": [-0.2249375730752945, -0.34339189529418945, 0.9905063509941101], "agreement_weight": 0.23664745688438416, "orthogonal_weight": -0.7310810089111328}, {"head": 3, "memory_to_active": 0.11326659470796585, "contribution_to_active": 0.6293258666992188, "mix_mean": 0.5667177438735962, "mix_low": 0.001277923583984375, "mix_high": 0.0019016265869140625, "read_cosine": 0.8034087419509888, "beta_mean": 0.008241585455834866, "nonfinite": 0.0, "weighted_memory_to_active": 0.08576828241348267, "mix_std": 0.2561156153678894, "beta_std": 0.007316869683563709, "beta_low": 0.7098388671875, "beta_high": 0.0, "orthogonal_to_active": 0.027292273938655853, "evidence_weights": [-0.1827685683965683, -0.30324774980545044, 0.6020689606666565], "agreement_weight": 0.20184868574142456, "orthogonal_weight": -0.6321589350700378}, {"head": 4, "memory_to_active": 1.3708564043045044, "contribution_to_active": 0.16735076904296875, "mix_mean": 0.04686553776264191, "mix_low": 0.2702608108520508, "mix_high": 0.0, "read_cosine": 0.34653228521347046, "beta_mean": 0.03988410532474518, "nonfinite": 0.0, "weighted_memory_to_active": 0.15529479086399078, "mix_std": 0.07231573760509491, "beta_std": 0.055122293531894684, "beta_low": 0.2869873046875, "beta_high": 0.0, "orthogonal_to_active": 0.010480286553502083, "evidence_weights": [0.04331010580062866, -0.05881278216838837, -0.01995672471821308], "agreement_weight": 0.0810951292514801, "orthogonal_weight": -0.07277359813451767}, {"head": 5, "memory_to_active": 0.35034555196762085, "contribution_to_active": 0.6393108367919922, "mix_mean": 0.23989759385585785, "mix_low": 0.11504745483398438, "mix_high": 6.389617919921875e-05, "read_cosine": 0.4615952968597412, "beta_mean": 0.06838610023260117, "nonfinite": 0.0, "weighted_memory_to_active": 0.1849118024110794, "mix_std": 0.27053940296173096, "beta_std": 0.08784466981887817, "beta_low": 0.169189453125, "beta_high": 0.0, "orthogonal_to_active": 0.29094937443733215, "evidence_weights": [-0.08759599924087524, -0.20351487398147583, -0.20637182891368866], "agreement_weight": -0.019036926329135895, "orthogonal_weight": 1.8261886835098267}]}, {"layer": 9, "memory_to_active": 1.0534024238586426, "contribution_to_active": 0.6881994605064392, "mix_mean": 0.3337233066558838, "mix_low": 0.13901424407958984, "mix_high": 0.004795710556209087, "read_cosine": 0.346691370010376, "beta_mean": 0.06024985760450363, "nonfinite": 0.0, "weighted_memory_to_active": 0.40430518984794617, "mix_std": 0.21520096063613892, "beta_std": 0.04488327354192734, "beta_low": 0.5040690302848816, "beta_high": 0.0, "orthogonal_to_active": 0.037668563425540924, "output_influence": 0.5146845579147339, "residual_influence": 0.1413077414035797, "state_active": 0.04143820330500603, "state_memory": 0.018644962459802628, "state_peak": 48.96394729614258, "growth_active": 1.1650999784469604, "growth_memory": 1.375207543373108, "heads": [{"head": 0, "memory_to_active": 0.002314738230779767, "contribution_to_active": 0.5616817474365234, "mix_mean": 0.6279853582382202, "mix_low": 0.0012540817260742188, "mix_high": 0.021248817443847656, "read_cosine": 0.2564968466758728, "beta_mean": 4.491482286539394e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014934162609279156, "mix_std": 0.29089510440826416, "beta_std": 1.559425436425954e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013730587670579553, "evidence_weights": [-0.298767626285553, -0.30838996171951294, 0.806816577911377], "agreement_weight": 0.11036650836467743, "orthogonal_weight": -0.9671959280967712}, {"head": 1, "memory_to_active": 0.02225356735289097, "contribution_to_active": 0.6580172777175903, "mix_mean": 0.5349282622337341, "mix_low": 0.006758689880371094, "mix_high": 0.0021753311157226562, "read_cosine": 0.4002958834171295, "beta_mean": 6.125414074631408e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015276803635060787, "mix_std": 0.28624844551086426, "beta_std": 0.00016714217781554908, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01090651098638773, "evidence_weights": [-0.23397834599018097, -0.28405314683914185, 1.1691045761108398], "agreement_weight": 0.4019118547439575, "orthogonal_weight": -0.867877185344696}, {"head": 2, "memory_to_active": 0.18205547332763672, "contribution_to_active": 0.6278927326202393, "mix_mean": 0.5730787515640259, "mix_low": 0.0027265548706054688, "mix_high": 0.00466156005859375, "read_cosine": 0.35805967450141907, "beta_mean": 0.0006715197814628482, "nonfinite": 0.0, "weighted_memory_to_active": 0.12075842916965485, "mix_std": 0.2725018858909607, "beta_std": 0.004569019190967083, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.066832534968853, "evidence_weights": [-0.19710543751716614, -0.2685215175151825, 0.7263295650482178], "agreement_weight": 0.2945839762687683, "orthogonal_weight": -0.6502425074577332}, {"head": 3, "memory_to_active": 1.4021143913269043, "contribution_to_active": 0.1768427938222885, "mix_mean": 0.04811659827828407, "mix_low": 0.36236000061035156, "mix_high": 4.76837158203125e-06, "read_cosine": 0.40071696043014526, "beta_mean": 0.0794389545917511, "nonfinite": 0.0, "weighted_memory_to_active": 0.1530894935131073, "mix_std": 0.10364042967557907, "beta_std": 0.058553870767354965, "beta_low": 0.00274658203125, "beta_high": 0.0, "orthogonal_to_active": 0.005438339430838823, "evidence_weights": [0.0784795954823494, 0.013016724959015846, 0.05483546480536461], "agreement_weight": 0.09220678359270096, "orthogonal_weight": 0.03764011710882187}, {"head": 4, "memory_to_active": 3.1015162467956543, "contribution_to_active": 1.5466748476028442, "mix_mean": 0.136800155043602, "mix_low": 0.14616775512695312, "mix_high": 0.0006017684936523438, "read_cosine": 0.19271337985992432, "beta_mean": 0.16579598188400269, "nonfinite": 0.0, "weighted_memory_to_active": 1.5312190055847168, "mix_std": 0.19602787494659424, "beta_std": 0.0998939648270607, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0266620684415102, "evidence_weights": [-0.073237843811512, -0.15620571374893188, -0.026964811608195305], "agreement_weight": -0.0574079304933548, "orthogonal_weight": -0.018231650814414024}, {"head": 5, "memory_to_active": 1.6101598739624023, "contribution_to_active": 0.5580873489379883, "mix_mean": 0.0814308226108551, "mix_low": 0.3148183822631836, "mix_high": 8.20159912109375e-05, "read_cosine": 0.4718652367591858, "beta_mean": 0.11552692949771881, "nonfinite": 0.0, "weighted_memory_to_active": 0.6039939522743225, "mix_std": 0.1418919414281845, "beta_std": 0.10610002279281616, "beta_low": 0.0262451171875, "beta_high": 0.0, "orthogonal_to_active": 0.11479885876178741, "evidence_weights": [0.10985991358757019, -0.08933261781930923, 0.14159396290779114], "agreement_weight": 0.016997504979372025, "orthogonal_weight": 0.2745240032672882}]}, {"layer": 10, "memory_to_active": 0.2217046618461609, "contribution_to_active": 0.6585671305656433, "mix_mean": 0.5087389945983887, "mix_low": 0.018236637115478516, "mix_high": 0.006165027618408203, "read_cosine": 0.3789435029029846, "beta_mean": 0.04399503394961357, "nonfinite": 0.0, "weighted_memory_to_active": 0.10711981356143951, "mix_std": 0.26950111985206604, "beta_std": 0.04509858787059784, "beta_low": 0.6590780019760132, "beta_high": 0.0, "orthogonal_to_active": 0.19074061512947083, "output_influence": 0.4313817322254181, "residual_influence": 0.1475343108177185, "state_active": 0.05206332728266716, "state_memory": 0.015156079083681107, "state_peak": 117.90721130371094, "growth_active": 1.043880581855774, "growth_memory": 0.9833718538284302, "heads": [{"head": 0, "memory_to_active": 0.8149421811103821, "contribution_to_active": 1.7105358839035034, "mix_mean": 0.5887938737869263, "mix_low": 0.06819343566894531, "mix_high": 0.02005481719970703, "read_cosine": 0.10386881232261658, "beta_mean": 0.19118858873844147, "nonfinite": 0.0, "weighted_memory_to_active": 0.513431191444397, "mix_std": 0.3914805054664612, "beta_std": 0.20551474392414093, "beta_low": 0.1632080078125, "beta_high": 0.0, "orthogonal_to_active": 1.0527026653289795, "evidence_weights": [-0.11678765714168549, -0.3938186466693878, -1.432304859161377], "agreement_weight": 0.02184944972395897, "orthogonal_weight": 2.0736639499664307}, {"head": 1, "memory_to_active": 0.017743950709700584, "contribution_to_active": 0.5991318821907043, "mix_mean": 0.6107889413833618, "mix_low": 0.0009965896606445312, "mix_high": 0.011786460876464844, "read_cosine": 0.27018141746520996, "beta_mean": 0.0001115111299441196, "nonfinite": 0.0, "weighted_memory_to_active": 0.00975036434829235, "mix_std": 0.2681178152561188, "beta_std": 0.00017376970208715647, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014498094096779823, "evidence_weights": [-0.26065558195114136, -0.20032836496829987, 0.5745439529418945], "agreement_weight": 0.502849817276001, "orthogonal_weight": -1.5936170816421509}, {"head": 2, "memory_to_active": 0.011622302234172821, "contribution_to_active": 0.6030896902084351, "mix_mean": 0.6310783624649048, "mix_low": 0.0004177093505859375, "mix_high": 0.0039577484130859375, "read_cosine": 0.0544305220246315, "beta_mean": 2.8730717531288974e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0052697123028337955, "mix_std": 0.25159692764282227, "beta_std": 0.0001257865224033594, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0026330030523240566, "evidence_weights": [-0.2491137683391571, -0.2804056406021118, 0.6065600514411926], "agreement_weight": -0.19259518384933472, "orthogonal_weight": -0.48156479001045227}, {"head": 3, "memory_to_active": 0.33092227578163147, "contribution_to_active": 0.10383981466293335, "mix_mean": 0.2747419476509094, "mix_low": 0.03733253479003906, "mix_high": 0.000446319580078125, "read_cosine": 0.914393961429596, "beta_mean": 0.06516169011592865, "nonfinite": 0.0, "weighted_memory_to_active": 0.04030793905258179, "mix_std": 0.22785921394824982, "beta_std": 0.05603554844856262, "beta_low": 0.00189208984375, "beta_high": 0.0, "orthogonal_to_active": 0.04297611117362976, "evidence_weights": [-0.10747388750314713, -0.14674684405326843, 0.1508876383304596], "agreement_weight": 0.011780308559536934, "orthogonal_weight": 1.447773814201355}, {"head": 4, "memory_to_active": 0.017879080027341843, "contribution_to_active": 0.5343769192695618, "mix_mean": 0.5603317618370056, "mix_low": 0.00037288665771484375, "mix_high": 0.0007381439208984375, "read_cosine": 0.07880353927612305, "beta_mean": 6.948934606043622e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010110931470990181, "mix_std": 0.25202447175979614, "beta_std": 0.0002516162348911166, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.011297493241727352, "evidence_weights": [-0.1842568814754486, -0.26251837611198425, 0.8828462362289429], "agreement_weight": 0.12902617454528809, "orthogonal_weight": -1.1746644973754883}, {"head": 5, "memory_to_active": 0.1371181756258011, "contribution_to_active": 0.4004283547401428, "mix_mean": 0.3866990804672241, "mix_low": 0.0021066665649414062, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8519826531410217, "beta_mean": 0.007410182151943445, "nonfinite": 0.0, "weighted_memory_to_active": 0.06384870409965515, "mix_std": 0.22592765092849731, "beta_std": 0.008490055799484253, "beta_low": 0.78955078125, "beta_high": 0.0, "orthogonal_to_active": 0.020336391404271126, "evidence_weights": [-0.14358776807785034, -0.15928055346012115, 0.2808704376220703], "agreement_weight": 0.08973166346549988, "orthogonal_weight": -0.6412546038627625}]}, {"layer": 11, "memory_to_active": 0.10593800246715546, "contribution_to_active": 0.5924698710441589, "mix_mean": 0.4959421455860138, "mix_low": 0.09677983075380325, "mix_high": 0.044981639832258224, "read_cosine": 0.3840984106063843, "beta_mean": 0.027908693999052048, "nonfinite": 0.0, "weighted_memory_to_active": 0.053450051695108414, "mix_std": 0.3233336806297302, "beta_std": 0.041833098977804184, "beta_low": 0.63641357421875, "beta_high": 0.0, "orthogonal_to_active": 0.05090203881263733, "output_influence": 0.5174602270126343, "residual_influence": 0.13112673163414001, "state_active": 0.1794421374797821, "state_memory": 0.0054533579386770725, "state_peak": 30.919782638549805, "growth_active": 1.05580735206604, "growth_memory": 1.6441588401794434, "heads": [{"head": 0, "memory_to_active": 0.008098705671727657, "contribution_to_active": 0.583645224571228, "mix_mean": 0.6307262778282166, "mix_low": 0.001201629638671875, "mix_high": 0.00881195068359375, "read_cosine": 0.2885094881057739, "beta_mean": 9.109315578825772e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005548669956624508, "mix_std": 0.26140734553337097, "beta_std": 0.003951610531657934, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.004268305376172066, "evidence_weights": [-0.16688820719718933, -0.36904001235961914, 0.7278836965560913], "agreement_weight": 0.3023113012313843, "orthogonal_weight": -0.8003620505332947}, {"head": 1, "memory_to_active": 0.23840804398059845, "contribution_to_active": 0.35236871242523193, "mix_mean": 0.40604960918426514, "mix_low": 0.007632255554199219, "mix_high": 0.0004892349243164062, "read_cosine": 0.5361677408218384, "beta_mean": 0.019189827144145966, "nonfinite": 0.0, "weighted_memory_to_active": 0.1091383621096611, "mix_std": 0.2638583183288574, "beta_std": 0.042940717190504074, "beta_low": 0.634033203125, "beta_high": 0.0, "orthogonal_to_active": 0.04431680589914322, "evidence_weights": [-0.1710691601037979, -0.20899344980716705, 0.5487014055252075], "agreement_weight": 0.07678145915269852, "orthogonal_weight": -0.509087085723877}, {"head": 2, "memory_to_active": 0.009199580177664757, "contribution_to_active": 0.9806927442550659, "mix_mean": 0.43903982639312744, "mix_low": 0.2503032684326172, "mix_high": 0.16648387908935547, "read_cosine": 0.2042892426252365, "beta_mean": 0.06737548857927322, "nonfinite": 0.0, "weighted_memory_to_active": 0.0065618352964520454, "mix_std": 0.44362983107566833, "beta_std": 0.0905999168753624, "beta_low": 0.48114013671875, "beta_high": 0.0, "orthogonal_to_active": 0.046925969421863556, "evidence_weights": [0.03365572541952133, -0.41827401518821716, -1.2899903059005737], "agreement_weight": 0.005112200044095516, "orthogonal_weight": 7.434454917907715}, {"head": 3, "memory_to_active": 0.011990871280431747, "contribution_to_active": 0.8178045749664307, "mix_mean": 0.4076942503452301, "mix_low": 0.3105888366699219, "mix_high": 0.0880270004272461, "read_cosine": 0.5791831612586975, "beta_mean": 0.05427894741296768, "nonfinite": 0.0, "weighted_memory_to_active": 0.006792167201638222, "mix_std": 0.43813782930374146, "beta_std": 0.08098998665809631, "beta_low": 0.513916015625, "beta_high": 0.0, "orthogonal_to_active": 0.041699185967445374, "evidence_weights": [0.16521018743515015, -0.3216603100299835, -0.21907198429107666], "agreement_weight": 0.0459732860326767, "orthogonal_weight": 7.780999183654785}, {"head": 4, "memory_to_active": 0.28229019045829773, "contribution_to_active": 0.43077442049980164, "mix_mean": 0.5582104921340942, "mix_low": 0.009006500244140625, "mix_high": 0.001026153564453125, "read_cosine": 0.07287583500146866, "beta_mean": 0.01858702301979065, "nonfinite": 0.0, "weighted_memory_to_active": 0.1494077444076538, "mix_std": 0.27166497707366943, "beta_std": 0.02100449427962303, "beta_low": 0.43280029296875, "beta_high": 0.0, "orthogonal_to_active": 0.14359964430332184, "evidence_weights": [-0.24080908298492432, -0.2253054827451706, 1.9614439010620117], "agreement_weight": -0.1956881284713745, "orthogonal_weight": -0.9784877896308899}, {"head": 5, "memory_to_active": 0.08564060926437378, "contribution_to_active": 0.38953328132629395, "mix_mean": 0.5339323878288269, "mix_low": 0.0019464492797851562, "mix_high": 0.005051612854003906, "read_cosine": 0.6235648393630981, "beta_mean": 0.007929773069918156, "nonfinite": 0.0, "weighted_memory_to_active": 0.04325152933597565, "mix_std": 0.26130378246307373, "beta_std": 0.011511852964758873, "beta_low": 0.75677490234375, "beta_high": 0.0, "orthogonal_to_active": 0.024602320045232773, "evidence_weights": [-0.2144828587770462, -0.3039584457874298, 0.6220118999481201], "agreement_weight": 0.07826776057481766, "orthogonal_weight": -0.7465251088142395}]}, {"layer": 12, "memory_to_active": 0.2720980942249298, "contribution_to_active": 0.5763906836509705, "mix_mean": 0.5213121175765991, "mix_low": 0.009669622406363487, "mix_high": 0.004776637069880962, "read_cosine": 0.23646655678749084, "beta_mean": 0.01575157232582569, "nonfinite": 0.0, "weighted_memory_to_active": 0.15021727979183197, "mix_std": 0.26122474670410156, "beta_std": 0.0159757100045681, "beta_low": 0.8235880732536316, "beta_high": 0.0, "orthogonal_to_active": 0.0723438709974289, "output_influence": 0.4725126624107361, "residual_influence": 0.1309703290462494, "state_active": 0.037987664341926575, "state_memory": 0.009300528094172478, "state_peak": 56.42913055419922, "growth_active": 1.14036226272583, "growth_memory": 1.2208225727081299, "heads": [{"head": 0, "memory_to_active": 0.04863545298576355, "contribution_to_active": 0.5614787340164185, "mix_mean": 0.5415964126586914, "mix_low": 0.0005922317504882812, "mix_high": 0.002796173095703125, "read_cosine": 0.2652033567428589, "beta_mean": 0.0008064366993494332, "nonfinite": 0.0, "weighted_memory_to_active": 0.025605324655771255, "mix_std": 0.2685869634151459, "beta_std": 0.0018835609080269933, "beta_low": 0.9962158203125, "beta_high": 0.0, "orthogonal_to_active": 0.02176530659198761, "evidence_weights": [-0.2045297920703888, -0.27440667152404785, 0.1841658055782318], "agreement_weight": 0.09591186791658401, "orthogonal_weight": -0.9145938754081726}, {"head": 1, "memory_to_active": 0.1139598861336708, "contribution_to_active": 0.5356617569923401, "mix_mean": 0.5476781129837036, "mix_low": 0.00044918060302734375, "mix_high": 0.0019683837890625, "read_cosine": 0.46029287576675415, "beta_mean": 0.0011730510741472244, "nonfinite": 0.0, "weighted_memory_to_active": 0.06541602313518524, "mix_std": 0.26380518078804016, "beta_std": 0.009014615789055824, "beta_low": 0.98809814453125, "beta_high": 0.0, "orthogonal_to_active": 0.052814118564128876, "evidence_weights": [-0.2084847241640091, -0.2742159962654114, 0.991739809513092], "agreement_weight": 0.17163801193237305, "orthogonal_weight": -0.923572301864624}, {"head": 2, "memory_to_active": 0.5893301367759705, "contribution_to_active": 0.47591516375541687, "mix_mean": 0.3212413191795349, "mix_low": 0.02913951873779297, "mix_high": 0.00020599365234375, "read_cosine": 0.4952675700187683, "beta_mean": 0.09027144312858582, "nonfinite": 0.0, "weighted_memory_to_active": 0.24070611596107483, "mix_std": 0.24247626960277557, "beta_std": 0.060401029884815216, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.24370785057544708, "evidence_weights": [-0.2337249219417572, -0.05080460384488106, -0.052136022597551346], "agreement_weight": 0.02044185996055603, "orthogonal_weight": 1.2673736810684204}, {"head": 3, "memory_to_active": 0.03942599147558212, "contribution_to_active": 0.7366322875022888, "mix_mean": 0.7025583982467651, "mix_low": 0.00021457672119140625, "mix_high": 0.014455795288085938, "read_cosine": 0.1054956316947937, "beta_mean": 0.00039657781599089503, "nonfinite": 0.0, "weighted_memory_to_active": 0.029569976031780243, "mix_std": 0.2332078218460083, "beta_std": 0.005968123208731413, "beta_low": 0.99420166015625, "beta_high": 0.0, "orthogonal_to_active": 0.028328144922852516, "evidence_weights": [-0.289742648601532, -0.3407352566719055, 1.4726057052612305], "agreement_weight": 0.4408886730670929, "orthogonal_weight": -0.9806176424026489}, {"head": 4, "memory_to_active": 0.7793971300125122, "contribution_to_active": 0.6647371649742126, "mix_mean": 0.3914351761341095, "mix_low": 0.020827293395996094, "mix_high": 0.0034656524658203125, "read_cosine": -0.12117749452590942, "beta_mean": 0.0015331882750615478, "nonfinite": 0.0, "weighted_memory_to_active": 0.4987821578979492, "mix_std": 0.2822726368904114, "beta_std": 0.015065066516399384, "beta_low": 0.96832275390625, "beta_high": 0.0, "orthogonal_to_active": 0.05534553900361061, "evidence_weights": [-0.13263382017612457, -0.2154787927865982, -0.01497254241257906], "agreement_weight": 0.0011320732301101089, "orthogonal_weight": -0.11656511574983597}, {"head": 5, "memory_to_active": 0.0618399977684021, "contribution_to_active": 0.48391905426979065, "mix_mean": 0.6233630776405334, "mix_low": 0.006794929504394531, "mix_high": 0.005767822265625, "read_cosine": 0.21371731162071228, "beta_mean": 0.00032873303280211985, "nonfinite": 0.0, "weighted_memory_to_active": 0.04122406244277954, "mix_std": 0.27699950337409973, "beta_std": 0.003521861508488655, "beta_low": 0.9942626953125, "beta_high": 0.0, "orthogonal_to_active": 0.03210226818919182, "evidence_weights": [-0.2446586638689041, -0.31606343388557434, 1.3166128396987915], "agreement_weight": 0.36382395029067993, "orthogonal_weight": -0.8333524465560913}]}, {"layer": 13, "memory_to_active": 0.1254950910806656, "contribution_to_active": 0.522685170173645, "mix_mean": 0.5249453783035278, "mix_low": 0.00573333166539669, "mix_high": 0.007342815399169922, "read_cosine": 0.4141249656677246, "beta_mean": 0.016400841996073723, "nonfinite": 0.0, "weighted_memory_to_active": 0.05463286489248276, "mix_std": 0.2600513696670532, "beta_std": 0.01829950138926506, "beta_low": 0.8001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.055096130818128586, "output_influence": 0.4946550130844116, "residual_influence": 0.16778269410133362, "state_active": 0.11373431980609894, "state_memory": 0.04263463616371155, "state_peak": 100.02778625488281, "growth_active": 0.9307504296302795, "growth_memory": 1.0688059329986572, "heads": [{"head": 0, "memory_to_active": 0.021372323855757713, "contribution_to_active": 0.5686683058738708, "mix_mean": 0.5870964527130127, "mix_low": 0.00047016143798828125, "mix_high": 0.0028409957885742188, "read_cosine": 0.4296388626098633, "beta_mean": 7.387679943349212e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013042788952589035, "mix_std": 0.26982370018959045, "beta_std": 0.00015684487880207598, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012972015887498856, "evidence_weights": [-0.19824182987213135, -0.2648729085922241, 0.7531786561012268], "agreement_weight": 0.13428175449371338, "orthogonal_weight": -1.108871579170227}, {"head": 1, "memory_to_active": 0.01519069541245699, "contribution_to_active": 0.5861913561820984, "mix_mean": 0.5535927414894104, "mix_low": 0.0015859603881835938, "mix_high": 0.0029993057250976562, "read_cosine": 0.22566674649715424, "beta_mean": 5.789937131339684e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008618395775556564, "mix_std": 0.27484673261642456, "beta_std": 0.00011079135583713651, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010835733264684677, "evidence_weights": [-0.18505476415157318, -0.26737716794013977, 0.6674643158912659], "agreement_weight": 0.37721359729766846, "orthogonal_weight": -1.314105749130249}, {"head": 2, "memory_to_active": 0.08822125196456909, "contribution_to_active": 0.4505994915962219, "mix_mean": 0.37876954674720764, "mix_low": 0.0018033981323242188, "mix_high": 0.00012683868408203125, "read_cosine": 0.6354494094848633, "beta_mean": 0.014188271015882492, "nonfinite": 0.0, "weighted_memory_to_active": 0.038752809166908264, "mix_std": 0.22970788180828094, "beta_std": 0.04444213584065437, "beta_low": 0.8138427734375, "beta_high": 0.0, "orthogonal_to_active": 0.01999005861580372, "evidence_weights": [-0.19116660952568054, -0.20370042324066162, 0.42886000871658325], "agreement_weight": 0.009823944419622421, "orthogonal_weight": -0.8183965682983398}, {"head": 3, "memory_to_active": 0.16633062064647675, "contribution_to_active": 0.581692099571228, "mix_mean": 0.6408692598342896, "mix_low": 0.02326202392578125, "mix_high": 0.028345108032226562, "read_cosine": -0.02042289264500141, "beta_mean": 0.0011123648146167397, "nonfinite": 0.0, "weighted_memory_to_active": 0.12314383685588837, "mix_std": 0.2909027934074402, "beta_std": 0.011187569238245487, "beta_low": 0.98809814453125, "beta_high": 0.0, "orthogonal_to_active": 0.11213159561157227, "evidence_weights": [-0.3210941553115845, -0.24608752131462097, 1.7762054204940796], "agreement_weight": 4.854716777801514, "orthogonal_weight": -0.9962983727455139}, {"head": 4, "memory_to_active": 0.402952641248703, "contribution_to_active": 0.2549254894256592, "mix_mean": 0.34850287437438965, "mix_low": 0.004994392395019531, "mix_high": 1.71661376953125e-05, "read_cosine": 0.8856066465377808, "beta_mean": 0.08264322578907013, "nonfinite": 0.0, "weighted_memory_to_active": 0.10619818419218063, "mix_std": 0.22199538350105286, "beta_std": 0.050717707723379135, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14305303990840912, "evidence_weights": [-0.10728037357330322, -0.23612025380134583, 0.47565391659736633], "agreement_weight": 0.003009140258654952, "orthogonal_weight": 2.2098894119262695}, {"head": 5, "memory_to_active": 0.058903004974126816, "contribution_to_active": 0.694034218788147, "mix_mean": 0.6408412456512451, "mix_low": 0.0022840499877929688, "mix_high": 0.00972747802734375, "read_cosine": 0.3288108706474304, "beta_mean": 0.00032941531389951706, "nonfinite": 0.0, "weighted_memory_to_active": 0.038041166961193085, "mix_std": 0.2730315923690796, "beta_std": 0.003181952051818371, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.03159433230757713, "evidence_weights": [-0.2210458368062973, -0.3050472140312195, 0.8438833951950073], "agreement_weight": 0.14466767013072968, "orthogonal_weight": -0.9165289998054504}]}, {"layer": 14, "memory_to_active": 0.23887667059898376, "contribution_to_active": 0.4166996479034424, "mix_mean": 0.42170408368110657, "mix_low": 0.024307092651724815, "mix_high": 0.005609194748103619, "read_cosine": 0.6159992218017578, "beta_mean": 0.06379307061433792, "nonfinite": 0.0, "weighted_memory_to_active": 0.1017501950263977, "mix_std": 0.24636131525039673, "beta_std": 0.06071730703115463, "beta_low": 0.473114013671875, "beta_high": 0.0, "orthogonal_to_active": 0.0428038127720356, "output_influence": 0.3305782973766327, "residual_influence": 0.12831749022006989, "state_active": 0.05727483332157135, "state_memory": 0.016149811446666718, "state_peak": 37.01285171508789, "growth_active": 1.0784363746643066, "growth_memory": 1.2824177742004395, "heads": [{"head": 0, "memory_to_active": 0.362614244222641, "contribution_to_active": 0.189902663230896, "mix_mean": 0.19788837432861328, "mix_low": 0.016536712646484375, "mix_high": 4.1961669921875e-05, "read_cosine": 0.8086406588554382, "beta_mean": 0.07312364131212234, "nonfinite": 0.0, "weighted_memory_to_active": 0.08443432301282883, "mix_std": 0.18349239230155945, "beta_std": 0.06019711494445801, "beta_low": 0.0030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.07539742439985275, "evidence_weights": [-0.12850549817085266, -0.03304239362478256, 0.4646035432815552], "agreement_weight": 0.026619846001267433, "orthogonal_weight": 1.5172758102416992}, {"head": 1, "memory_to_active": 0.47377830743789673, "contribution_to_active": 0.3088892698287964, "mix_mean": 0.23234841227531433, "mix_low": 0.11122703552246094, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8272210359573364, "beta_mean": 0.2540452480316162, "nonfinite": 0.0, "weighted_memory_to_active": 0.20504581928253174, "mix_std": 0.25554609298706055, "beta_std": 0.24809764325618744, "beta_low": 0.10003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.013817817904055119, "evidence_weights": [-0.012907557189464569, -0.21301037073135376, -0.11550825089216232], "agreement_weight": 0.05299421399831772, "orthogonal_weight": 0.15585900843143463}, {"head": 2, "memory_to_active": 0.1579911708831787, "contribution_to_active": 0.5621734261512756, "mix_mean": 0.5799263715744019, "mix_low": 0.0024471282958984375, "mix_high": 0.0027313232421875, "read_cosine": 0.4222239851951599, "beta_mean": 0.002612283919006586, "nonfinite": 0.0, "weighted_memory_to_active": 0.10103729367256165, "mix_std": 0.26643529534339905, "beta_std": 0.0035113831982016563, "beta_low": 0.966796875, "beta_high": 0.0, "orthogonal_to_active": 0.07612745463848114, "evidence_weights": [-0.14674261212348938, -0.315132200717926, 0.8887653350830078], "agreement_weight": 0.07781488448381424, "orthogonal_weight": -0.8658093810081482}, {"head": 3, "memory_to_active": 0.05118691176176071, "contribution_to_active": 0.6054520010948181, "mix_mean": 0.6607768535614014, "mix_low": 0.006279945373535156, "mix_high": 0.020204544067382812, "read_cosine": 0.03389016166329384, "beta_mean": 0.007408072240650654, "nonfinite": 0.0, "weighted_memory_to_active": 0.03309319168329239, "mix_std": 0.2553369700908661, "beta_std": 0.008421601727604866, "beta_low": 0.75927734375, "beta_high": 0.0, "orthogonal_to_active": 0.023531006649136543, "evidence_weights": [-0.3560950458049774, -0.315677672624588, 0.35186564922332764], "agreement_weight": 0.07634128630161285, "orthogonal_weight": -0.8067517280578613}, {"head": 4, "memory_to_active": 0.1607409417629242, "contribution_to_active": 0.5017377138137817, "mix_mean": 0.49820107221603394, "mix_low": 0.005291938781738281, "mix_high": 0.009280204772949219, "read_cosine": 0.8598284125328064, "beta_mean": 0.04342193901538849, "nonfinite": 0.0, "weighted_memory_to_active": 0.0948665514588356, "mix_std": 0.2641032338142395, "beta_std": 0.0308518148958683, "beta_low": 0.04644775390625, "beta_high": 0.0, "orthogonal_to_active": 0.027278825640678406, "evidence_weights": [-0.22050747275352478, -0.24943116307258606, 0.4295204281806946], "agreement_weight": 0.13003824651241302, "orthogonal_weight": -0.6001713275909424}, {"head": 5, "memory_to_active": 0.22694852948188782, "contribution_to_active": 0.3320426046848297, "mix_mean": 0.36108338832855225, "mix_low": 0.004059791564941406, "mix_high": 0.0013942718505859375, "read_cosine": 0.7441911697387695, "beta_mean": 0.0021471718791872263, "nonfinite": 0.0, "weighted_memory_to_active": 0.09202396869659424, "mix_std": 0.25325390696525574, "beta_std": 0.013224282301962376, "beta_low": 0.96307373046875, "beta_high": 0.0, "orthogonal_to_active": 0.04067035764455795, "evidence_weights": [-0.1779661625623703, -0.16712652146816254, 0.26107046008110046], "agreement_weight": 0.08059380948543549, "orthogonal_weight": -0.7026728987693787}]}, {"layer": 15, "memory_to_active": 0.05731257051229477, "contribution_to_active": 0.4878944754600525, "mix_mean": 0.3992791175842285, "mix_low": 0.010686556808650494, "mix_high": 0.002677281852811575, "read_cosine": 0.6499274969100952, "beta_mean": 0.03744795173406601, "nonfinite": 0.0, "weighted_memory_to_active": 0.02745674178004265, "mix_std": 0.28471463918685913, "beta_std": 0.04434049502015114, "beta_low": 0.7119039297103882, "beta_high": 0.0, "orthogonal_to_active": 0.006848818622529507, "output_influence": 0.3003489077091217, "residual_influence": 0.17157690227031708, "state_active": 0.131961852312088, "state_memory": 0.010664643719792366, "state_peak": 37.49168014526367, "growth_active": 1.185505747795105, "growth_memory": 1.4908279180526733, "heads": [{"head": 0, "memory_to_active": 0.0956680178642273, "contribution_to_active": 0.4880155622959137, "mix_mean": 0.3640180230140686, "mix_low": 0.008303642272949219, "mix_high": 0.005573272705078125, "read_cosine": 0.6113210916519165, "beta_mean": 0.0386931374669075, "nonfinite": 0.0, "weighted_memory_to_active": 0.04527423530817032, "mix_std": 0.27861493825912476, "beta_std": 0.03117506392300129, "beta_low": 0.12188720703125, "beta_high": 0.0, "orthogonal_to_active": 0.02387283742427826, "evidence_weights": [-0.18573983013629913, -0.18982620537281036, 0.3961615264415741], "agreement_weight": 0.0943959504365921, "orthogonal_weight": -0.7311468124389648}, {"head": 1, "memory_to_active": 0.13043375313282013, "contribution_to_active": 0.4153640866279602, "mix_mean": 0.4302721917629242, "mix_low": 0.0019254684448242188, "mix_high": 0.006186485290527344, "read_cosine": 0.9170300960540771, "beta_mean": 0.000955307565163821, "nonfinite": 0.0, "weighted_memory_to_active": 0.0541953407227993, "mix_std": 0.28502023220062256, "beta_std": 0.002905889181420207, "beta_low": 0.988525390625, "beta_high": 0.0, "orthogonal_to_active": 0.0005553992814384401, "evidence_weights": [-0.15398477017879486, -0.16422681510448456, 0.20404231548309326], "agreement_weight": 0.12098607420921326, "orthogonal_weight": 0.019405953586101532}, {"head": 2, "memory_to_active": 0.00566972466185689, "contribution_to_active": 0.48580533266067505, "mix_mean": 0.32833611965179443, "mix_low": 0.019878387451171875, "mix_high": 5.054473876953125e-05, "read_cosine": 0.7809137105941772, "beta_mean": 8.805142715573311e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002866549650207162, "mix_std": 0.2719014883041382, "beta_std": 0.0002244797651655972, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0006482221069745719, "evidence_weights": [-0.20529013872146606, -0.27134788036346436, 0.1529512256383896], "agreement_weight": 0.009331753477454185, "orthogonal_weight": 0.5884210467338562}, {"head": 3, "memory_to_active": 0.002874231431633234, "contribution_to_active": 0.39220672845840454, "mix_mean": 0.3758038282394409, "mix_low": 0.00696563720703125, "mix_high": 0.0003528594970703125, "read_cosine": 0.5561555027961731, "beta_mean": 1.8218048353446648e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0010335553670302033, "mix_std": 0.28780698776245117, "beta_std": 4.502572846831754e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008059547981247306, "evidence_weights": [-0.24012476205825806, -0.24997317790985107, 0.2039230912923813], "agreement_weight": 0.049983445554971695, "orthogonal_weight": -0.8514730930328369}, {"head": 4, "memory_to_active": 0.08354341983795166, "contribution_to_active": 0.62884122133255, "mix_mean": 0.3532392978668213, "mix_low": 0.017597198486328125, "mix_high": 0.0003948211669921875, "read_cosine": 0.7026159763336182, "beta_mean": 0.18288291990756989, "nonfinite": 0.0, "weighted_memory_to_active": 0.044733427464962006, "mix_std": 0.30506110191345215, "beta_std": 0.21612654626369476, "beta_low": 0.188720703125, "beta_high": 0.0, "orthogonal_to_active": 0.005507388152182102, "evidence_weights": [-0.21216218173503876, -0.25397738814353943, -0.007396323140710592], "agreement_weight": 0.012684609740972519, "orthogonal_weight": 0.30216649174690247}, {"head": 5, "memory_to_active": 0.02568628266453743, "contribution_to_active": 0.5171340107917786, "mix_mean": 0.5440050363540649, "mix_low": 0.009449005126953125, "mix_high": 0.003505706787109375, "read_cosine": 0.33152884244918823, "beta_mean": 0.002050074515864253, "nonfinite": 0.0, "weighted_memory_to_active": 0.016637329012155533, "mix_std": 0.27988293766975403, "beta_std": 0.015565937384963036, "beta_low": 0.972412109375, "beta_high": 0.0, "orthogonal_to_active": 0.009703107178211212, "evidence_weights": [-0.29565638303756714, -0.309760183095932, 0.7379841804504395], "agreement_weight": 0.274339884519577, "orthogonal_weight": 0.7648827433586121}]}]}, {"step": 45000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9064128329991945, "memory_to_active": 0.8328052759170532, "contribution_to_active": 0.5482307076454163, "mix_mean": 0.3968532085418701, "mix_low": 0.1414438784122467, "mix_high": 0.009647050872445107, "read_cosine": 0.35645467042922974, "beta_mean": 0.07432900369167328, "nonfinite": 0.0, "weighted_memory_to_active": 0.21345199644565582, "mix_std": 0.22109857201576233, "beta_std": 0.06448078900575638, "beta_low": 0.5376027822494507, "beta_high": 2.415974813629873e-05, "orthogonal_to_active": 0.0601712130010128, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0837481021881104, "normal_token_accuracy": 0.442714124917984, "state_active": 0.060816552489995956, "state_memory": 0.01843463070690632, "state_peak": 115.61193084716797, "growth_active": 1.1389875411987305, "growth_memory": 1.356810450553894, "output_influence": 0.4049398899078369, "residual_influence": 0.1339212954044342, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.15364933013916, "no_memory_loss_delta": 0.5450456142425537, "scrambled_loss_delta": 0.22182995080947876, "memory_only_loss_delta": 3.747291088104248, "no_evidence_loss_delta": 0.31635311245918274, "no_agreement_loss_delta": 0.03945007175207138, "no_orthogonal_loss_delta": 0.012168342247605324}, {"index": 94, "tokens": 256, "normal_loss": 3.2376925945281982, "no_memory_loss_delta": 0.5730865001678467, "scrambled_loss_delta": 0.3100103735923767, "memory_only_loss_delta": 3.390799045562744, "no_evidence_loss_delta": 0.2712339758872986, "no_agreement_loss_delta": 0.04812061786651611, "no_orthogonal_loss_delta": 0.03624687343835831}, {"index": 188, "tokens": 256, "normal_loss": 3.4528653621673584, "no_memory_loss_delta": 0.6297075152397156, "scrambled_loss_delta": 0.1499449610710144, "memory_only_loss_delta": 2.875246047973633, "no_evidence_loss_delta": 0.33511191606521606, "no_agreement_loss_delta": 0.061203472316265106, "no_orthogonal_loss_delta": 0.03701721876859665}, {"index": 283, "tokens": 131, "normal_loss": 1.9249814748764038, "no_memory_loss_delta": 0.5139492154121399, "scrambled_loss_delta": 0.3291102945804596, "memory_only_loss_delta": 2.8100569248199463, "no_evidence_loss_delta": 0.26260995864868164, "no_agreement_loss_delta": 0.03699631616473198, "no_orthogonal_loss_delta": -0.0012311007594689727}], "agreement_weight_abs_mean": 0.19150781631469727, "agreement_weight_abs_max": 4.945950031280518, "agreement_weight_mean": 0.16773660480976105, "orthogonal_weight_abs_mean": 0.794719934463501, "orthogonal_weight_abs_max": 7.930002212524414, "orthogonal_weight_mean": -0.1295250803232193, "no_memory_loss": 3.656355857849121, "no_memory_loss_delta": 0.5726077556610107, "no_memory_token_accuracy_gain": 0.08008898794651031, "no_memory_token_win_fraction": 0.6607341766357422, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 43.59473922384172, "scrambled_loss": 3.3258509635925293, "scrambled_loss_delta": 0.24210287630558014, "scrambled_token_accuracy_gain": 0.03337041288614273, "scrambled_token_win_fraction": 0.6407119035720825, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 21.502458197319786, "memory_only_loss": 6.34462833404541, "memory_only_loss_delta": 3.260880470275879, "memory_only_token_accuracy_gain": 0.27919912338256836, "memory_only_token_win_fraction": 0.9043381810188293, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.16453869503285, "no_evidence_loss": 3.384763479232788, "no_evidence_loss_delta": 0.3010154068470001, "no_evidence_token_accuracy_gain": 0.031145717948675156, "no_evidence_token_win_fraction": 0.6117908954620361, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 25.99336294304692, "no_agreement_loss": 3.1315040588378906, "no_agreement_loss_delta": 0.04775606840848923, "no_agreement_token_accuracy_gain": 0.0011123470030725002, "no_agreement_token_win_fraction": 0.5183537006378174, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 4.663368512160548, "no_orthogonal_loss": 3.1078968048095703, "no_orthogonal_loss_delta": 0.02414841763675213, "no_orthogonal_token_accuracy_gain": 0.0033370412420481443, "no_orthogonal_token_win_fraction": 0.5928809642791748, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.3859177507749085, "short_no_memory_loss_delta": 0.4898415505886078, "long_context_memory_gain": 0.08276620507240295, "normal_context_loss_gain": 0.15475282073020935, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.43936824798584, "contribution_to_active": 1.1114962100982666, "mix_mean": 0.1035437136888504, "mix_low": 0.5511102676391602, "mix_high": 0.005277157295495272, "read_cosine": 0.4378168284893036, "beta_mean": 0.23026281595230103, "nonfinite": 0.0, "weighted_memory_to_active": 1.1585713624954224, "mix_std": 0.0884397029876709, "beta_std": 0.1528562605381012, "beta_low": 0.0330607146024704, "beta_high": 0.0, "orthogonal_to_active": 0.04935759678483009, "output_influence": 0.20538660883903503, "residual_influence": 0.24536658823490143, "state_active": 0.0620153471827507, "state_memory": 0.045392971485853195, "state_peak": 44.48945236206055, "growth_active": 1.1993550062179565, "growth_memory": 1.274510383605957, "heads": [{"head": 0, "memory_to_active": 2.556483745574951, "contribution_to_active": 0.4709753096103668, "mix_mean": 0.019897058606147766, "mix_low": 0.6238937377929688, "mix_high": 0.0, "read_cosine": 0.23890943825244904, "beta_mean": 0.23078317940235138, "nonfinite": 0.0, "weighted_memory_to_active": 0.5758507251739502, "mix_std": 0.06202448904514313, "beta_std": 0.10780096054077148, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04898089915513992, "evidence_weights": [-0.01720886491239071, -0.061350442469120026, -0.10502960532903671], "agreement_weight": -0.16486215591430664, "orthogonal_weight": -0.09538240730762482}, {"head": 1, "memory_to_active": 6.2530412673950195, "contribution_to_active": 2.5313994884490967, "mix_mean": 0.016319187358021736, "mix_low": 0.6750469207763672, "mix_high": 0.0, "read_cosine": 0.4579375982284546, "beta_mean": 0.38163262605667114, "nonfinite": 0.0, "weighted_memory_to_active": 2.8309240341186523, "mix_std": 0.05109550058841705, "beta_std": 0.2349696010351181, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02702297270298004, "evidence_weights": [0.02458755113184452, -0.10159104317426682, 0.4844221770763397], "agreement_weight": 0.0259019136428833, "orthogonal_weight": -0.02132652886211872}, {"head": 2, "memory_to_active": 2.903316020965576, "contribution_to_active": 0.5217536091804504, "mix_mean": 0.014398615807294846, "mix_low": 0.663792610168457, "mix_high": 0.0, "read_cosine": 0.3212010860443115, "beta_mean": 0.3182964324951172, "nonfinite": 0.0, "weighted_memory_to_active": 0.6116743683815002, "mix_std": 0.03245954588055611, "beta_std": 0.21951302886009216, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.05720528960227966, "evidence_weights": [0.05604255571961403, 0.05838516354560852, 0.08699742704629898], "agreement_weight": 0.006681113038212061, "orthogonal_weight": -0.2026108056306839}, {"head": 3, "memory_to_active": 6.744464874267578, "contribution_to_active": 2.6538214683532715, "mix_mean": 0.0333113893866539, "mix_low": 0.5160303115844727, "mix_high": 0.0, "read_cosine": 0.3239479064941406, "beta_mean": 0.3276156485080719, "nonfinite": 0.0, "weighted_memory_to_active": 2.830994129180908, "mix_std": 0.08050191402435303, "beta_std": 0.24867099523544312, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.14116188883781433, "evidence_weights": [-0.0423286147415638, -0.047538213431835175, -0.1604200154542923], "agreement_weight": -0.012644597329199314, "orthogonal_weight": -0.05908391997218132}, {"head": 4, "memory_to_active": 2.0593669414520264, "contribution_to_active": 0.04886354133486748, "mix_mean": 0.007013694383203983, "mix_low": 0.8123722076416016, "mix_high": 0.0, "read_cosine": 0.342021644115448, "beta_mean": 0.0954667404294014, "nonfinite": 0.0, "weighted_memory_to_active": 0.04108942672610283, "mix_std": 0.01459323987364769, "beta_std": 0.08385182172060013, "beta_low": 0.0224609375, "beta_high": 0.0, "orthogonal_to_active": 0.003683308372274041, "evidence_weights": [0.14138242602348328, 0.09953474998474121, -0.2884194254875183], "agreement_weight": -0.111402228474617, "orthogonal_weight": 0.08947441726922989}, {"head": 5, "memory_to_active": 0.11953619867563248, "contribution_to_active": 0.44216400384902954, "mix_mean": 0.5303223133087158, "mix_low": 0.01552581787109375, "mix_high": 0.031662940979003906, "read_cosine": 0.9428833723068237, "beta_mean": 0.027782224118709564, "nonfinite": 0.0, "weighted_memory_to_active": 0.06089533865451813, "mix_std": 0.2899635136127472, "beta_std": 0.022331124171614647, "beta_low": 0.17578125, "beta_high": 0.0, "orthogonal_to_active": 0.018091194331645966, "evidence_weights": [-0.19378434121608734, -0.2227543145418167, 0.35755687952041626], "agreement_weight": 0.2855112850666046, "orthogonal_weight": -0.6870014667510986}]}, {"layer": 1, "memory_to_active": 2.1848764419555664, "contribution_to_active": 0.4806470274925232, "mix_mean": 0.201328843832016, "mix_low": 0.3714369237422943, "mix_high": 0.0010223389836028218, "read_cosine": 0.31701552867889404, "beta_mean": 0.17483559250831604, "nonfinite": 0.0, "weighted_memory_to_active": 0.33018842339515686, "mix_std": 0.11622209846973419, "beta_std": 0.1275559365749359, "beta_low": 0.3319600522518158, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.050649452954530716, "output_influence": 0.22207361459732056, "residual_influence": 0.05781552940607071, "state_active": 0.03127316012978554, "state_memory": 0.03503558039665222, "state_peak": 37.706851959228516, "growth_active": 1.212841272354126, "growth_memory": 1.4968657493591309, "heads": [{"head": 0, "memory_to_active": 0.042439814656972885, "contribution_to_active": 0.46095678210258484, "mix_mean": 0.47914549708366394, "mix_low": 0.008001327514648438, "mix_high": 0.0012798309326171875, "read_cosine": 0.035926640033721924, "beta_mean": 0.00028524675872176886, "nonfinite": 0.0, "weighted_memory_to_active": 0.025563795119524002, "mix_std": 0.2817707657814026, "beta_std": 0.00042543699964880943, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.024976491928100586, "evidence_weights": [-0.19720889627933502, -0.25844499468803406, 0.35187652707099915], "agreement_weight": 0.18989168107509613, "orthogonal_weight": -1.0342202186584473}, {"head": 1, "memory_to_active": 2.6502814292907715, "contribution_to_active": 0.19422227144241333, "mix_mean": 0.022695783525705338, "mix_low": 0.46094799041748047, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5304586887359619, "beta_mean": 0.28917521238327026, "nonfinite": 0.0, "weighted_memory_to_active": 0.22849828004837036, "mix_std": 0.0443180650472641, "beta_std": 0.18795613944530487, "beta_low": 0.00054931640625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.014591513201594353, "evidence_weights": [0.016272656619548798, -0.04865426570177078, 0.020802631974220276], "agreement_weight": -0.0035365691874176264, "orthogonal_weight": 0.15313602983951569}, {"head": 2, "memory_to_active": 0.09497515112161636, "contribution_to_active": 0.5622032880783081, "mix_mean": 0.644140899181366, "mix_low": 0.0005788803100585938, "mix_high": 0.00484466552734375, "read_cosine": 0.5396309494972229, "beta_mean": 0.00432711374014616, "nonfinite": 0.0, "weighted_memory_to_active": 0.07206689566373825, "mix_std": 0.23697969317436218, "beta_std": 0.006147865206003189, "beta_low": 0.89599609375, "beta_high": 0.0, "orthogonal_to_active": 0.0392252653837204, "evidence_weights": [-0.29384279251098633, -0.3507671654224396, 0.6454827189445496], "agreement_weight": 0.02723058871924877, "orthogonal_weight": -0.6825658679008484}, {"head": 3, "memory_to_active": 2.7865030765533447, "contribution_to_active": 0.21123795211315155, "mix_mean": 0.020922761410474777, "mix_low": 0.6054353713989258, "mix_high": 0.0, "read_cosine": 0.21972671151161194, "beta_mean": 0.31844663619995117, "nonfinite": 0.0, "weighted_memory_to_active": 0.32782816886901855, "mix_std": 0.05728648975491524, "beta_std": 0.25202685594558716, "beta_low": 0.05059814453125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.09115172922611237, "evidence_weights": [0.03904617205262184, -0.010424064472317696, 0.07987681776285172], "agreement_weight": -0.020844750106334686, "orthogonal_weight": -0.4257875680923462}, {"head": 4, "memory_to_active": 3.630760669708252, "contribution_to_active": 1.3916234970092773, "mix_mean": 0.035034410655498505, "mix_low": 0.2901620864868164, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3295750021934509, "beta_mean": 0.14711205661296844, "nonfinite": 0.0, "weighted_memory_to_active": 1.2799748182296753, "mix_std": 0.057734109461307526, "beta_std": 0.09861445426940918, "beta_low": 0.020263671875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12202402949333191, "evidence_weights": [-0.04239407926797867, -0.004507044330239296, 0.05813869461417198], "agreement_weight": -0.01358837354928255, "orthogonal_weight": 0.12650860846042633}, {"head": 5, "memory_to_active": 3.9042983055114746, "contribution_to_active": 0.0636381059885025, "mix_mean": 0.006033698562532663, "mix_low": 0.8634958267211914, "mix_high": 7.62939453125e-06, "read_cosine": 0.2467752993106842, "beta_mean": 0.28966718912124634, "nonfinite": 0.0, "weighted_memory_to_active": 0.04719848185777664, "mix_std": 0.01924343779683113, "beta_std": 0.22016476094722748, "beta_low": 0.02435302734375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.011927671730518341, "evidence_weights": [0.17334330081939697, 0.08879604935646057, 0.23102550208568573], "agreement_weight": -0.13004642724990845, "orthogonal_weight": 0.22398591041564941}]}, {"layer": 2, "memory_to_active": 1.076465368270874, "contribution_to_active": 0.34362661838531494, "mix_mean": 0.379602313041687, "mix_low": 0.2534979283809662, "mix_high": 0.0032971701584756374, "read_cosine": 0.32668671011924744, "beta_mean": 0.09224316477775574, "nonfinite": 0.0, "weighted_memory_to_active": 0.08430731296539307, "mix_std": 0.18989872932434082, "beta_std": 0.07367895543575287, "beta_low": 0.5536702871322632, "beta_high": 0.00016276040696538985, "orthogonal_to_active": 0.019801320508122444, "output_influence": 0.3670566976070404, "residual_influence": 0.09325595200061798, "state_active": 0.03750636801123619, "state_memory": 0.016880784183740616, "state_peak": 29.6326847076416, "growth_active": 1.051694393157959, "growth_memory": 1.581830382347107, "heads": [{"head": 0, "memory_to_active": 0.00858731847256422, "contribution_to_active": 0.32455015182495117, "mix_mean": 0.5884413719177246, "mix_low": 0.0012178421020507812, "mix_high": 0.0056133270263671875, "read_cosine": 0.1799078732728958, "beta_mean": 7.412007835227996e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004957085940986872, "mix_std": 0.2805226445198059, "beta_std": 0.00013942734221927822, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004371284041553736, "evidence_weights": [-0.3480384051799774, -0.20843037962913513, 0.10637225210666656], "agreement_weight": 0.19733195006847382, "orthogonal_weight": -0.9205958247184753}, {"head": 1, "memory_to_active": 2.9952187538146973, "contribution_to_active": 0.03766971826553345, "mix_mean": 0.004951730370521545, "mix_low": 0.8932580947875977, "mix_high": 0.0, "read_cosine": 0.17156195640563965, "beta_mean": 0.25292712450027466, "nonfinite": 0.0, "weighted_memory_to_active": 0.02745664305984974, "mix_std": 0.01065184734761715, "beta_std": 0.1642989069223404, "beta_low": 0.00146484375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.010476039722561836, "evidence_weights": [0.1281673014163971, 0.11690513044595718, 0.5076947212219238], "agreement_weight": 0.0299778301268816, "orthogonal_weight": 0.39123356342315674}, {"head": 2, "memory_to_active": 0.01117037981748581, "contribution_to_active": 0.40289556980133057, "mix_mean": 0.5133581161499023, "mix_low": 0.00173187255859375, "mix_high": 0.0017528533935546875, "read_cosine": 0.4108344316482544, "beta_mean": 3.0295415854197927e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004167784936726093, "mix_std": 0.2774021625518799, "beta_std": 7.160100358305499e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005799487698823214, "evidence_weights": [-0.24393758177757263, -0.28276368975639343, 0.5078961253166199], "agreement_weight": -0.004817408975213766, "orthogonal_weight": -0.1651640683412552}, {"head": 3, "memory_to_active": 0.16126129031181335, "contribution_to_active": 0.5082021355628967, "mix_mean": 0.5297166705131531, "mix_low": 0.000370025634765625, "mix_high": 0.0063648223876953125, "read_cosine": 0.5806940197944641, "beta_mean": 0.03255976364016533, "nonfinite": 0.0, "weighted_memory_to_active": 0.09524306654930115, "mix_std": 0.2651398777961731, "beta_std": 0.0387047715485096, "beta_low": 0.2952880859375, "beta_high": 0.0, "orthogonal_to_active": 0.06539120525121689, "evidence_weights": [-0.19411969184875488, -0.30110490322113037, 0.839467465877533], "agreement_weight": 0.055740971118211746, "orthogonal_weight": -0.8862037062644958}, {"head": 4, "memory_to_active": 3.272209882736206, "contribution_to_active": 0.25762176513671875, "mix_mean": 0.017038334161043167, "mix_low": 0.6232395172119141, "mix_high": 0.0, "read_cosine": 0.20088841021060944, "beta_mean": 0.26783087849617004, "nonfinite": 0.0, "weighted_memory_to_active": 0.36781057715415955, "mix_std": 0.04133724421262741, "beta_std": 0.23880994319915771, "beta_low": 0.0252685546875, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.037861745804548264, "evidence_weights": [0.024625929072499275, 0.01059923879802227, 0.03369682654738426], "agreement_weight": 0.020054174587130547, "orthogonal_weight": -0.1290511041879654}, {"head": 5, "memory_to_active": 0.010344628244638443, "contribution_to_active": 0.530820369720459, "mix_mean": 0.6241075396537781, "mix_low": 0.0011701583862304688, "mix_high": 0.0060520172119140625, "read_cosine": 0.4162334203720093, "beta_mean": 3.6802561226068065e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006208733655512333, "mix_std": 0.2643386125564575, "beta_std": 4.9084468628279865e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00012770401372108608, "evidence_weights": [-0.27682673931121826, -0.28927528858184814, 0.9507383704185486], "agreement_weight": 0.16457690298557281, "orthogonal_weight": -0.028352933004498482}]}, {"layer": 3, "memory_to_active": 0.5523905754089355, "contribution_to_active": 0.4418174922466278, "mix_mean": 0.44614148139953613, "mix_low": 0.06488101184368134, "mix_high": 0.008867263793945312, "read_cosine": 0.2232874035835266, "beta_mean": 0.03277445584535599, "nonfinite": 0.0, "weighted_memory_to_active": 0.1377972960472107, "mix_std": 0.2600831389427185, "beta_std": 0.029026197269558907, "beta_low": 0.7266845703125, "beta_high": 0.0, "orthogonal_to_active": 0.06150813400745392, "output_influence": 0.37208330631256104, "residual_influence": 0.0807272270321846, "state_active": 0.024333767592906952, "state_memory": 0.007442450150847435, "state_peak": 22.18526840209961, "growth_active": 1.0796905755996704, "growth_memory": 1.711402177810669, "heads": [{"head": 0, "memory_to_active": 0.4275815486907959, "contribution_to_active": 0.5665545463562012, "mix_mean": 0.5546296834945679, "mix_low": 0.0033979415893554688, "mix_high": 0.021615028381347656, "read_cosine": 0.25592878460884094, "beta_mean": 0.012440362945199013, "nonfinite": 0.0, "weighted_memory_to_active": 0.2644771337509155, "mix_std": 0.312682569026947, "beta_std": 0.021101273596286774, "beta_low": 0.649658203125, "beta_high": 0.0, "orthogonal_to_active": 0.18142779171466827, "evidence_weights": [-0.20982016623020172, -0.2427619993686676, 0.8510770797729492], "agreement_weight": 0.12170680612325668, "orthogonal_weight": -0.7338911890983582}, {"head": 1, "memory_to_active": 0.013125751167535782, "contribution_to_active": 0.39814841747283936, "mix_mean": 0.5195286273956299, "mix_low": 0.00722503662109375, "mix_high": 0.006955146789550781, "read_cosine": 0.17999474704265594, "beta_mean": 6.677317287540063e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007143836468458176, "mix_std": 0.3061777651309967, "beta_std": 0.00012392736971378326, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005339008755981922, "evidence_weights": [-0.24226920306682587, -0.2017308473587036, 0.6103009581565857], "agreement_weight": 0.42395737767219543, "orthogonal_weight": -0.8090764284133911}, {"head": 2, "memory_to_active": 2.5328946113586426, "contribution_to_active": 0.2758054733276367, "mix_mean": 0.044621750712394714, "mix_low": 0.34075927734375, "mix_high": 0.0, "read_cosine": 0.20942425727844238, "beta_mean": 0.1746181845664978, "nonfinite": 0.0, "weighted_memory_to_active": 0.30850353837013245, "mix_std": 0.0916539654135704, "beta_std": 0.13999344408512115, "beta_low": 0.012939453125, "beta_high": 0.0, "orthogonal_to_active": 0.001664995332248509, "evidence_weights": [-0.08018817007541656, 0.014640743844211102, 0.0018841223791241646], "agreement_weight": -0.017306478694081306, "orthogonal_weight": -0.006695343181490898}, {"head": 3, "memory_to_active": 0.02439914084970951, "contribution_to_active": 0.5235100984573364, "mix_mean": 0.5740227699279785, "mix_low": 0.00772857666015625, "mix_high": 0.008704185485839844, "read_cosine": 0.3173796832561493, "beta_mean": 9.962805052055046e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0168471559882164, "mix_std": 0.2961941361427307, "beta_std": 0.00026502623222768307, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004229906015098095, "evidence_weights": [-0.23316681385040283, -0.22788408398628235, 2.1360256671905518], "agreement_weight": 1.798445701599121, "orthogonal_weight": -0.34442415833473206}, {"head": 4, "memory_to_active": 0.04634104669094086, "contribution_to_active": 0.24008703231811523, "mix_mean": 0.3121315836906433, "mix_low": 0.026468276977539062, "mix_high": 0.00091552734375, "read_cosine": 0.180378258228302, "beta_mean": 0.00030359410447999835, "nonfinite": 0.0, "weighted_memory_to_active": 0.021760286763310432, "mix_std": 0.27415281534194946, "beta_std": 0.00041336618596687913, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0198625810444355, "evidence_weights": [-0.13060715794563293, -0.1578349471092224, 0.1472083032131195], "agreement_weight": -0.020960092544555664, "orthogonal_weight": -0.963133692741394}, {"head": 5, "memory_to_active": 0.27000129222869873, "contribution_to_active": 0.6467992663383484, "mix_mean": 0.6719145178794861, "mix_low": 0.0037069320678710938, "mix_high": 0.015013694763183594, "read_cosine": 0.19661858677864075, "beta_mean": 0.009118172340095043, "nonfinite": 0.0, "weighted_memory_to_active": 0.20805181562900543, "mix_std": 0.2796373665332794, "beta_std": 0.012260130606591702, "beta_low": 0.69757080078125, "beta_high": 0.0, "orthogonal_to_active": 0.15652449429035187, "evidence_weights": [-0.25393885374069214, -0.30514174699783325, 0.4761318862438202], "agreement_weight": 0.13118736445903778, "orthogonal_weight": -0.8003063797950745}]}, {"layer": 4, "memory_to_active": 1.0714490413665771, "contribution_to_active": 0.4515451192855835, "mix_mean": 0.3572021424770355, "mix_low": 0.2816642224788666, "mix_high": 0.008174896240234375, "read_cosine": 0.33676981925964355, "beta_mean": 0.12043558806180954, "nonfinite": 0.0, "weighted_memory_to_active": 0.14266647398471832, "mix_std": 0.15694859623908997, "beta_std": 0.10651889443397522, "beta_low": 0.3761190176010132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.0448295883834362, "output_influence": 0.4339439570903778, "residual_influence": 0.12087297439575195, "state_active": 0.038480184972286224, "state_memory": 0.018646948039531708, "state_peak": 36.78693389892578, "growth_active": 1.2453727722167969, "growth_memory": 1.5558853149414062, "heads": [{"head": 0, "memory_to_active": 0.2099873125553131, "contribution_to_active": 0.7353993654251099, "mix_mean": 0.666985273361206, "mix_low": 0.012014389038085938, "mix_high": 0.02041149139404297, "read_cosine": 0.49015533924102783, "beta_mean": 0.26220017671585083, "nonfinite": 0.0, "weighted_memory_to_active": 0.16554513573646545, "mix_std": 0.3426654040813446, "beta_std": 0.26811954379081726, "beta_low": 0.22418212890625, "beta_high": 0.0, "orthogonal_to_active": 0.06664103269577026, "evidence_weights": [-0.26519379019737244, -0.3649318814277649, 0.12893562018871307], "agreement_weight": 0.013004610314965248, "orthogonal_weight": 0.497466504573822}, {"head": 1, "memory_to_active": 0.06427737325429916, "contribution_to_active": 0.6259972453117371, "mix_mean": 0.6664153337478638, "mix_low": 0.00012969970703125, "mix_high": 0.011449813842773438, "read_cosine": 0.6383312344551086, "beta_mean": 0.0008566974429413676, "nonfinite": 0.0, "weighted_memory_to_active": 0.045594990253448486, "mix_std": 0.24462461471557617, "beta_std": 0.004678766708821058, "beta_low": 0.9854736328125, "beta_high": 0.0, "orthogonal_to_active": 0.025811679661273956, "evidence_weights": [-0.2378184199333191, -0.3479999005794525, 0.699311375617981], "agreement_weight": 0.22525033354759216, "orthogonal_weight": -0.802943766117096}, {"head": 2, "memory_to_active": 2.0344583988189697, "contribution_to_active": 0.254520446062088, "mix_mean": 0.022073758766055107, "mix_low": 0.5884227752685547, "mix_high": 2.86102294921875e-06, "read_cosine": 0.25630801916122437, "beta_mean": 0.08452901989221573, "nonfinite": 0.0, "weighted_memory_to_active": 0.25088152289390564, "mix_std": 0.05057770758867264, "beta_std": 0.08010871708393097, "beta_low": 0.02655029296875, "beta_high": 0.0, "orthogonal_to_active": 0.031031783670186996, "evidence_weights": [0.06377525627613068, 0.026690904051065445, 0.04457227885723114], "agreement_weight": 0.1323021948337555, "orthogonal_weight": -0.1882801055908203}, {"head": 3, "memory_to_active": 1.8051345348358154, "contribution_to_active": 0.1383848488330841, "mix_mean": 0.029134493321180344, "mix_low": 0.4466705322265625, "mix_high": 0.0, "read_cosine": 0.23015466332435608, "beta_mean": 0.1417052149772644, "nonfinite": 0.0, "weighted_memory_to_active": 0.2065843641757965, "mix_std": 0.053929202258586884, "beta_std": 0.12467952072620392, "beta_low": 0.0205078125, "beta_high": 0.0, "orthogonal_to_active": 0.004792716354131699, "evidence_weights": [-0.01989763416349888, 0.06453008204698563, -0.23454423248767853], "agreement_weight": 0.07210821658372879, "orthogonal_weight": -0.03444328159093857}, {"head": 4, "memory_to_active": 0.010853651911020279, "contribution_to_active": 0.6316171288490295, "mix_mean": 0.7441065311431885, "mix_low": 0.00012683868408203125, "mix_high": 0.017180442810058594, "read_cosine": 0.15121129155158997, "beta_mean": 2.5775472749955952e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007804925553500652, "mix_std": 0.21727298200130463, "beta_std": 7.206388545455411e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029276118148118258, "evidence_weights": [-0.3377171456813812, -0.3317480683326721, 0.5816907286643982], "agreement_weight": 0.33962392807006836, "orthogonal_weight": -0.39874905347824097}, {"head": 5, "memory_to_active": 2.3039824962615967, "contribution_to_active": 0.32335132360458374, "mix_mean": 0.014497310854494572, "mix_low": 0.6426210403442383, "mix_high": 4.76837158203125e-06, "read_cosine": 0.25445830821990967, "beta_mean": 0.23329657316207886, "nonfinite": 0.0, "weighted_memory_to_active": 0.17958784103393555, "mix_std": 0.03262167423963547, "beta_std": 0.16145478188991547, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.13777269423007965, "evidence_weights": [0.04683978110551834, -0.03219730406999588, -0.3119327425956726], "agreement_weight": -0.0717051699757576, "orthogonal_weight": 0.9085069298744202}]}, {"layer": 5, "memory_to_active": 0.4420832097530365, "contribution_to_active": 0.5170615911483765, "mix_mean": 0.481345534324646, "mix_low": 0.06244516745209694, "mix_high": 0.01586151123046875, "read_cosine": 0.07654370367527008, "beta_mean": 0.054574303328990936, "nonfinite": 0.0, "weighted_memory_to_active": 0.14855128526687622, "mix_std": 0.25598621368408203, "beta_std": 0.07005943357944489, "beta_low": 0.6391805410385132, "beta_high": 0.0, "orthogonal_to_active": 0.05399885028600693, "output_influence": 0.5179896354675293, "residual_influence": 0.14064662158489227, "state_active": 0.04175668954849243, "state_memory": 0.01126495935022831, "state_peak": 64.67597198486328, "growth_active": 1.096015214920044, "growth_memory": 1.2610479593276978, "heads": [{"head": 0, "memory_to_active": 0.45457154512405396, "contribution_to_active": 0.5196543335914612, "mix_mean": 0.35078126192092896, "mix_low": 0.02104663848876953, "mix_high": 0.0013303756713867188, "read_cosine": -0.15203849971294403, "beta_mean": 0.030981946736574173, "nonfinite": 0.0, "weighted_memory_to_active": 0.14334005117416382, "mix_std": 0.26951515674591064, "beta_std": 0.048187967389822006, "beta_low": 0.3685302734375, "beta_high": 0.0, "orthogonal_to_active": 0.04423122853040695, "evidence_weights": [-0.12580575048923492, -0.15593871474266052, -0.07009574770927429], "agreement_weight": -0.0320248156785965, "orthogonal_weight": -0.31432855129241943}, {"head": 1, "memory_to_active": 0.014493884518742561, "contribution_to_active": 0.5436487197875977, "mix_mean": 0.6665784120559692, "mix_low": 0.001979827880859375, "mix_high": 0.02813720703125, "read_cosine": 0.2115362584590912, "beta_mean": 3.39713878929615e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007990842685103416, "mix_std": 0.2811189889907837, "beta_std": 7.807854854036123e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0074964105151593685, "evidence_weights": [-0.2409377545118332, -0.27764153480529785, 0.8777397871017456], "agreement_weight": 0.6077238917350769, "orthogonal_weight": -0.984998345375061}, {"head": 2, "memory_to_active": 0.01313930656760931, "contribution_to_active": 0.5682359933853149, "mix_mean": 0.6512356400489807, "mix_low": 0.0030879974365234375, "mix_high": 0.021646499633789062, "read_cosine": -0.4694957435131073, "beta_mean": 0.0001048431295203045, "nonfinite": 0.0, "weighted_memory_to_active": 0.008128763176500797, "mix_std": 0.2715919017791748, "beta_std": 0.00013782919268123806, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026299296878278255, "evidence_weights": [-0.26065802574157715, -0.3463000953197479, 0.5288416743278503], "agreement_weight": 0.007714451290667057, "orthogonal_weight": -0.4159502685070038}, {"head": 3, "memory_to_active": 1.641174077987671, "contribution_to_active": 0.21563254296779633, "mix_mean": 0.10154835879802704, "mix_low": 0.15763282775878906, "mix_high": 3.814697265625e-06, "read_cosine": 0.1528535783290863, "beta_mean": 0.10706315189599991, "nonfinite": 0.0, "weighted_memory_to_active": 0.3775934875011444, "mix_std": 0.14366178214550018, "beta_std": 0.15443168580532074, "beta_low": 0.1884765625, "beta_high": 0.0, "orthogonal_to_active": 0.14026960730552673, "evidence_weights": [-0.0613836906850338, -0.09031370282173157, 0.008518092334270477], "agreement_weight": 0.05536654219031334, "orthogonal_weight": -0.47045621275901794}, {"head": 4, "memory_to_active": 0.4532006084918976, "contribution_to_active": 0.5820705890655518, "mix_mean": 0.3270642161369324, "mix_low": 0.18874073028564453, "mix_high": 0.006488800048828125, "read_cosine": 0.5315836071968079, "beta_mean": 0.18901942670345306, "nonfinite": 0.0, "weighted_memory_to_active": 0.30233174562454224, "mix_std": 0.3398968577384949, "beta_std": 0.2165750414133072, "beta_low": 0.27978515625, "beta_high": 0.0, "orthogonal_to_active": 0.09623792767524719, "evidence_weights": [-0.11262357234954834, -0.27520233392715454, -0.05902421474456787], "agreement_weight": 0.04413836821913719, "orthogonal_weight": 0.4956226944923401}, {"head": 5, "memory_to_active": 0.07591971755027771, "contribution_to_active": 0.6731274724006653, "mix_mean": 0.7908652424812317, "mix_low": 0.0021829605102539062, "mix_high": 0.03756237030029297, "read_cosine": 0.18482297658920288, "beta_mean": 0.000242462832829915, "nonfinite": 0.0, "weighted_memory_to_active": 0.05192284658551216, "mix_std": 0.23013250529766083, "beta_std": 0.0009459591237828135, "beta_low": 0.998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.033127982169389725, "evidence_weights": [-0.3173984885215759, -0.330917626619339, 1.9613531827926636], "agreement_weight": 0.4314386248588562, "orthogonal_weight": -0.6747046113014221}]}, {"layer": 6, "memory_to_active": 0.3344719409942627, "contribution_to_active": 0.6034146547317505, "mix_mean": 0.5718017816543579, "mix_low": 0.05736494064331055, "mix_high": 0.03173398971557617, "read_cosine": 0.28870293498039246, "beta_mean": 0.06344962120056152, "nonfinite": 0.0, "weighted_memory_to_active": 0.15817171335220337, "mix_std": 0.2380295693874359, "beta_std": 0.07237577438354492, "beta_low": 0.51080322265625, "beta_high": 0.0, "orthogonal_to_active": 0.03790608420968056, "output_influence": 0.5172334313392639, "residual_influence": 0.16584765911102295, "state_active": 0.04722961038351059, "state_memory": 0.008572950959205627, "state_peak": 95.13921356201172, "growth_active": 1.2086119651794434, "growth_memory": 1.2015199661254883, "heads": [{"head": 0, "memory_to_active": 0.11546167731285095, "contribution_to_active": 0.6870104074478149, "mix_mean": 0.7986347079277039, "mix_low": 0.00026226043701171875, "mix_high": 0.03187751770019531, "read_cosine": 0.11738280206918716, "beta_mean": 0.01824772171676159, "nonfinite": 0.0, "weighted_memory_to_active": 0.0936344563961029, "mix_std": 0.20132476091384888, "beta_std": 0.024963026866316795, "beta_low": 0.4886474609375, "beta_high": 0.0, "orthogonal_to_active": 0.07774470001459122, "evidence_weights": [-0.3209758698940277, -0.34440144896507263, 1.0642684698104858], "agreement_weight": 0.3452771008014679, "orthogonal_weight": -0.8777053356170654}, {"head": 1, "memory_to_active": 0.005679843947291374, "contribution_to_active": 0.5011389255523682, "mix_mean": 0.5467686057090759, "mix_low": 0.0014057159423828125, "mix_high": 0.012720108032226562, "read_cosine": 0.5254115462303162, "beta_mean": 1.1316744348732755e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002457607537508011, "mix_std": 0.2907761335372925, "beta_std": 1.855806476669386e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014318842440843582, "evidence_weights": [-0.1941818743944168, -0.2920096814632416, 1.089489459991455], "agreement_weight": 0.07909919321537018, "orthogonal_weight": -0.7769807577133179}, {"head": 2, "memory_to_active": 1.0034935474395752, "contribution_to_active": 0.09208738058805466, "mix_mean": 0.03687499463558197, "mix_low": 0.33232593536376953, "mix_high": 0.0, "read_cosine": 0.23270748555660248, "beta_mean": 0.04328429698944092, "nonfinite": 0.0, "weighted_memory_to_active": 0.07392936199903488, "mix_std": 0.06139969825744629, "beta_std": 0.040805622935295105, "beta_low": 0.047119140625, "beta_high": 0.0, "orthogonal_to_active": 0.0022834581322968006, "evidence_weights": [0.07889440655708313, -0.03224042430520058, 0.2840682566165924], "agreement_weight": 0.1401214897632599, "orthogonal_weight": -0.03662727400660515}, {"head": 3, "memory_to_active": 0.1365496814250946, "contribution_to_active": 0.6518222689628601, "mix_mean": 0.6373603343963623, "mix_low": 0.008927345275878906, "mix_high": 0.01728057861328125, "read_cosine": 0.44762855768203735, "beta_mean": 0.19709523022174835, "nonfinite": 0.0, "weighted_memory_to_active": 0.09604629874229431, "mix_std": 0.3291710913181305, "beta_std": 0.23003064095973969, "beta_low": 0.2265625, "beta_high": 0.0, "orthogonal_to_active": 0.05202443152666092, "evidence_weights": [-0.17696738243103027, -0.33162352442741394, 0.18393737077713013], "agreement_weight": -0.008851557970046997, "orthogonal_weight": 0.6200840473175049}, {"head": 4, "memory_to_active": 0.7131838798522949, "contribution_to_active": 1.0777419805526733, "mix_mean": 0.757509708404541, "mix_low": 0.0011339187622070312, "mix_high": 0.12319469451904297, "read_cosine": 0.134059339761734, "beta_mean": 0.1218547448515892, "nonfinite": 0.0, "weighted_memory_to_active": 0.6601918935775757, "mix_std": 0.2893209457397461, "beta_std": 0.13801923394203186, "beta_low": 0.30255126953125, "beta_high": 0.0, "orthogonal_to_active": 0.07138228416442871, "evidence_weights": [-0.23795028030872345, -0.5925084352493286, -0.2607133686542511], "agreement_weight": -0.023065030574798584, "orthogonal_weight": 0.12182067334651947}, {"head": 5, "memory_to_active": 0.0324631929397583, "contribution_to_active": 0.6106868982315063, "mix_mean": 0.6536623239517212, "mix_low": 0.00013446807861328125, "mix_high": 0.0053310394287109375, "read_cosine": 0.27502769231796265, "beta_mean": 0.00020437961211428046, "nonfinite": 0.0, "weighted_memory_to_active": 0.02277066931128502, "mix_std": 0.25618481636047363, "beta_std": 0.00041754558333195746, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.022569723427295685, "evidence_weights": [-0.24161911010742188, -0.3111776113510132, 0.4186497628688812], "agreement_weight": 0.16059499979019165, "orthogonal_weight": -1.0647820234298706}]}, {"layer": 7, "memory_to_active": 1.7425944805145264, "contribution_to_active": 0.26367685198783875, "mix_mean": 0.14030331373214722, "mix_low": 0.26380157470703125, "mix_high": 0.0008859634981490672, "read_cosine": 0.23163266479969025, "beta_mean": 0.13413502275943756, "nonfinite": 0.0, "weighted_memory_to_active": 0.22673815488815308, "mix_std": 0.14167538285255432, "beta_std": 0.10038246959447861, "beta_low": 0.1952006071805954, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.07135748863220215, "output_influence": 0.2644335627555847, "residual_influence": 0.059517644345760345, "state_active": 0.014602788724005222, "state_memory": 0.025810183957219124, "state_peak": 33.836158752441406, "growth_active": 1.1980550289154053, "growth_memory": 1.3208577632904053, "heads": [{"head": 0, "memory_to_active": 2.462749719619751, "contribution_to_active": 0.3263375461101532, "mix_mean": 0.03727271780371666, "mix_low": 0.42134857177734375, "mix_high": 1.9073486328125e-06, "read_cosine": 0.33841970562934875, "beta_mean": 0.18344572186470032, "nonfinite": 0.0, "weighted_memory_to_active": 0.40086549520492554, "mix_std": 0.08073738217353821, "beta_std": 0.13938260078430176, "beta_low": 0.00079345703125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0048157391138374805, "evidence_weights": [0.02849689871072769, -0.07724873721599579, 0.07873861491680145], "agreement_weight": -0.035734791308641434, "orthogonal_weight": 0.020629137754440308}, {"head": 1, "memory_to_active": 1.0257197618484497, "contribution_to_active": 0.39815083146095276, "mix_mean": 0.21882769465446472, "mix_low": 0.11196136474609375, "mix_high": 0.004933357238769531, "read_cosine": 0.3750549852848053, "beta_mean": 0.10865981876850128, "nonfinite": 0.0, "weighted_memory_to_active": 0.3415549695491791, "mix_std": 0.23949602246284485, "beta_std": 0.06862887740135193, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03631550446152687, "evidence_weights": [-0.20989076793193817, -0.10801468789577484, -0.22360244393348694], "agreement_weight": 0.039940640330314636, "orthogonal_weight": 0.1255483776330948}, {"head": 2, "memory_to_active": 0.7714862823486328, "contribution_to_active": 0.320592999458313, "mix_mean": 0.16142283380031586, "mix_low": 0.056052207946777344, "mix_high": 0.00026988983154296875, "read_cosine": 0.12847909331321716, "beta_mean": 0.055211834609508514, "nonfinite": 0.0, "weighted_memory_to_active": 0.21887299418449402, "mix_std": 0.1852473020553589, "beta_std": 0.045638952404260635, "beta_low": 0.0533447265625, "beta_high": 0.0, "orthogonal_to_active": 0.09482568502426147, "evidence_weights": [-0.06373544037342072, -0.1469564586877823, 0.6616999506950378], "agreement_weight": 0.03337132930755615, "orthogonal_weight": -0.44568774104118347}, {"head": 3, "memory_to_active": 4.40771484375, "contribution_to_active": 0.08960964530706406, "mix_mean": 0.031518228352069855, "mix_low": 0.34876346588134766, "mix_high": 0.0, "read_cosine": 0.10268400609493256, "beta_mean": 0.37887272238731384, "nonfinite": 0.0, "weighted_memory_to_active": 0.3058578372001648, "mix_std": 0.046741142868995667, "beta_std": 0.2642919719219208, "beta_low": 0.00262451171875, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.26399752497673035, "evidence_weights": [0.07157547771930695, -0.010924209840595722, 0.6976402401924133], "agreement_weight": 0.052618615329265594, "orthogonal_weight": -0.9403802752494812}, {"head": 4, "memory_to_active": 1.7250248193740845, "contribution_to_active": 0.07984607666730881, "mix_mean": 0.017346341162919998, "mix_low": 0.6397275924682617, "mix_high": 0.0, "read_cosine": 0.19315867125988007, "beta_mean": 0.07826574146747589, "nonfinite": 0.0, "weighted_memory_to_active": 0.06802193820476532, "mix_std": 0.041449081152677536, "beta_std": 0.08356928080320358, "beta_low": 0.11529541015625, "beta_high": 0.0, "orthogonal_to_active": 0.009242307394742966, "evidence_weights": [0.14365391433238983, 0.025762444362044334, 0.24760158360004425], "agreement_weight": 0.04960399493575096, "orthogonal_weight": -0.1902294009923935}, {"head": 5, "memory_to_active": 0.06287133693695068, "contribution_to_active": 0.3675239086151123, "mix_mean": 0.3754320442676544, "mix_low": 0.004956245422363281, "mix_high": 0.000110626220703125, "read_cosine": 0.25199949741363525, "beta_mean": 0.0003542792983353138, "nonfinite": 0.0, "weighted_memory_to_active": 0.025255637243390083, "mix_std": 0.2563813626766205, "beta_std": 0.0007830789545550942, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.01894814893603325, "evidence_weights": [-0.14510582387447357, -0.20403876900672913, 0.5752589106559753], "agreement_weight": 0.16699986159801483, "orthogonal_weight": -0.8094528317451477}]}, {"layer": 8, "memory_to_active": 0.3631795048713684, "contribution_to_active": 0.5411659479141235, "mix_mean": 0.4385889172554016, "mix_low": 0.05704721063375473, "mix_high": 0.0012712478637695312, "read_cosine": 0.4418717622756958, "beta_mean": 0.022416502237319946, "nonfinite": 0.0, "weighted_memory_to_active": 0.09553316980600357, "mix_std": 0.22347132861614227, "beta_std": 0.03126026690006256, "beta_low": 0.6487630605697632, "beta_high": 0.0, "orthogonal_to_active": 0.0721299946308136, "output_influence": 0.4825953245162964, "residual_influence": 0.15754956007003784, "state_active": 0.04744720086455345, "state_memory": 0.008442336693406105, "state_peak": 32.75162887573242, "growth_active": 1.1015071868896484, "growth_memory": 1.3040857315063477, "heads": [{"head": 0, "memory_to_active": 0.056348174810409546, "contribution_to_active": 0.5621518492698669, "mix_mean": 0.6014506816864014, "mix_low": 0.00032901763916015625, "mix_high": 0.0011930465698242188, "read_cosine": 0.4634054899215698, "beta_mean": 0.0024734812323004007, "nonfinite": 0.0, "weighted_memory_to_active": 0.040052108466625214, "mix_std": 0.2469099909067154, "beta_std": 0.00866059958934784, "beta_low": 0.94232177734375, "beta_high": 0.0, "orthogonal_to_active": 0.025411896407604218, "evidence_weights": [-0.16976292431354523, -0.38952407240867615, 0.6666558980941772], "agreement_weight": 0.18842174112796783, "orthogonal_weight": -0.8018236756324768}, {"head": 1, "memory_to_active": 0.09609369933605194, "contribution_to_active": 0.5267710089683533, "mix_mean": 0.491430401802063, "mix_low": 0.00011539459228515625, "mix_high": 0.0006074905395507812, "read_cosine": 0.21519075334072113, "beta_mean": 0.001709822565317154, "nonfinite": 0.0, "weighted_memory_to_active": 0.04291233420372009, "mix_std": 0.24666829407215118, "beta_std": 0.008309785276651382, "beta_low": 0.97882080078125, "beta_high": 0.0, "orthogonal_to_active": 0.029977573081851006, "evidence_weights": [-0.1896788626909256, -0.26929789781570435, 0.7024343609809875], "agreement_weight": 0.07675514370203018, "orthogonal_weight": -0.7448135614395142}, {"head": 2, "memory_to_active": 0.01645205169916153, "contribution_to_active": 0.6673613786697388, "mix_mean": 0.6567769050598145, "mix_low": 0.0015888214111328125, "mix_high": 0.003757476806640625, "read_cosine": 0.3844689428806305, "beta_mean": 0.00015914255345705897, "nonfinite": 0.0, "weighted_memory_to_active": 0.012432938441634178, "mix_std": 0.22832462191581726, "beta_std": 0.003761168336495757, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.008024664595723152, "evidence_weights": [-0.2243545949459076, -0.34194526076316833, 0.9954361319541931], "agreement_weight": 0.23460491001605988, "orthogonal_weight": -0.7325454950332642}, {"head": 3, "memory_to_active": 0.11888620257377625, "contribution_to_active": 0.6333966851234436, "mix_mean": 0.5694348812103271, "mix_low": 0.0010595321655273438, "mix_high": 0.00191497802734375, "read_cosine": 0.7991058230400085, "beta_mean": 0.010479530319571495, "nonfinite": 0.0, "weighted_memory_to_active": 0.0904328003525734, "mix_std": 0.2579699754714966, "beta_std": 0.008934988640248775, "beta_low": 0.5848388671875, "beta_high": 0.0, "orthogonal_to_active": 0.02857125923037529, "evidence_weights": [-0.18268927931785583, -0.3028973937034607, 0.5966913104057312], "agreement_weight": 0.2068263441324234, "orthogonal_weight": -0.6280012726783752}, {"head": 4, "memory_to_active": 1.5093228816986084, "contribution_to_active": 0.1868436634540558, "mix_mean": 0.05388198420405388, "mix_low": 0.23566722869873047, "mix_high": 0.0, "read_cosine": 0.32628244161605835, "beta_mean": 0.047966908663511276, "nonfinite": 0.0, "weighted_memory_to_active": 0.18278738856315613, "mix_std": 0.08069992810487747, "beta_std": 0.06881752610206604, "beta_low": 0.25946044921875, "beta_high": 0.0, "orthogonal_to_active": 0.01413671113550663, "evidence_weights": [0.039021532982587814, -0.06580290198326111, -0.004349919967353344], "agreement_weight": 0.07693172246217728, "orthogonal_weight": -0.08431464433670044}, {"head": 5, "memory_to_active": 0.3819739520549774, "contribution_to_active": 0.6704710721969604, "mix_mean": 0.25855863094329834, "mix_low": 0.10352325439453125, "mix_high": 0.0001544952392578125, "read_cosine": 0.46277719736099243, "beta_mean": 0.07171011716127396, "nonfinite": 0.0, "weighted_memory_to_active": 0.2045813798904419, "mix_std": 0.28025510907173157, "beta_std": 0.08907755464315414, "beta_low": 0.127197265625, "beta_high": 0.0, "orthogonal_to_active": 0.32665783166885376, "evidence_weights": [-0.09083030372858047, -0.1974152773618698, -0.20164501667022705], "agreement_weight": -0.018593154847621918, "orthogonal_weight": 1.8439432382583618}]}, {"layer": 9, "memory_to_active": 1.0634093284606934, "contribution_to_active": 0.7085599899291992, "mix_mean": 0.34032130241394043, "mix_low": 0.1324310302734375, "mix_high": 0.005172252655029297, "read_cosine": 0.36367642879486084, "beta_mean": 0.050773609429597855, "nonfinite": 0.0, "weighted_memory_to_active": 0.418891578912735, "mix_std": 0.21877653896808624, "beta_std": 0.03794137015938759, "beta_low": 0.5058492422103882, "beta_high": 0.0, "orthogonal_to_active": 0.03897751122713089, "output_influence": 0.5257386565208435, "residual_influence": 0.14464637637138367, "state_active": 0.0403580516576767, "state_memory": 0.018152834847569466, "state_peak": 51.51095199584961, "growth_active": 1.173567533493042, "growth_memory": 1.3325740098953247, "heads": [{"head": 0, "memory_to_active": 0.001705903559923172, "contribution_to_active": 0.5607708692550659, "mix_mean": 0.6362286806106567, "mix_low": 0.001338958740234375, "mix_high": 0.021760940551757812, "read_cosine": 0.2603577673435211, "beta_mean": 3.5792954804492183e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0011065149446949363, "mix_std": 0.29064539074897766, "beta_std": 1.2137137673562393e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010505429236218333, "evidence_weights": [-0.3012688159942627, -0.30978795886039734, 0.798338770866394], "agreement_weight": 0.11719400435686111, "orthogonal_weight": -0.9930641055107117}, {"head": 1, "memory_to_active": 0.025497999042272568, "contribution_to_active": 0.682930588722229, "mix_mean": 0.5375254154205322, "mix_low": 0.007477760314941406, "mix_high": 0.0026950836181640625, "read_cosine": 0.4208373725414276, "beta_mean": 6.652504089288414e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.018596310168504715, "mix_std": 0.28745290637016296, "beta_std": 0.00016973815218079835, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012678713537752628, "evidence_weights": [-0.2335585057735443, -0.28447413444519043, 1.1707653999328613], "agreement_weight": 0.4166163504123688, "orthogonal_weight": -0.8652583360671997}, {"head": 2, "memory_to_active": 0.18176501989364624, "contribution_to_active": 0.6473596096038818, "mix_mean": 0.5862728953361511, "mix_low": 0.0014696121215820312, "mix_high": 0.0054454803466796875, "read_cosine": 0.3504883646965027, "beta_mean": 0.000667797401547432, "nonfinite": 0.0, "weighted_memory_to_active": 0.12098278105258942, "mix_std": 0.2707979083061218, "beta_std": 0.005115640349686146, "beta_low": 0.99591064453125, "beta_high": 0.0, "orthogonal_to_active": 0.06781160831451416, "evidence_weights": [-0.199996218085289, -0.2720390856266022, 0.7237406373023987], "agreement_weight": 0.2973422110080719, "orthogonal_weight": -0.654323935508728}, {"head": 3, "memory_to_active": 1.275148868560791, "contribution_to_active": 0.16568246483802795, "mix_mean": 0.053022030740976334, "mix_low": 0.34023189544677734, "mix_high": 0.000125885009765625, "read_cosine": 0.4062657356262207, "beta_mean": 0.05565428361296654, "nonfinite": 0.0, "weighted_memory_to_active": 0.13500475883483887, "mix_std": 0.11123526096343994, "beta_std": 0.04143399000167847, "beta_low": 0.0020751953125, "beta_high": 0.0, "orthogonal_to_active": 0.0019266240997239947, "evidence_weights": [0.07280862331390381, 0.005666912533342838, 0.06165888532996178], "agreement_weight": 0.0753108486533165, "orthogonal_weight": 0.01534796692430973}, {"head": 4, "memory_to_active": 3.275919198989868, "contribution_to_active": 1.5844508409500122, "mix_mean": 0.13983839750289917, "mix_low": 0.1537332534790039, "mix_high": 0.0007429122924804688, "read_cosine": 0.2086942493915558, "beta_mean": 0.148373544216156, "nonfinite": 0.0, "weighted_memory_to_active": 1.5681939125061035, "mix_std": 0.20206204056739807, "beta_std": 0.088438980281353, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.026783548295497894, "evidence_weights": [-0.07297468930482864, -0.156003937125206, -0.02805999293923378], "agreement_weight": -0.06431249529123306, "orthogonal_weight": -0.017738686874508858}, {"head": 5, "memory_to_active": 1.6204187870025635, "contribution_to_active": 0.6101657152175903, "mix_mean": 0.08904033899307251, "mix_low": 0.29033470153808594, "mix_high": 0.000263214111328125, "read_cosine": 0.5354150533676147, "beta_mean": 0.09987591207027435, "nonfinite": 0.0, "weighted_memory_to_active": 0.6694651246070862, "mix_std": 0.15046563744544983, "beta_std": 0.0924777090549469, "beta_low": 0.037109375, "beta_high": 0.0, "orthogonal_to_active": 0.12361401319503784, "evidence_weights": [0.10545819997787476, -0.09578675031661987, 0.14426761865615845], "agreement_weight": 0.018055768683552742, "orthogonal_weight": 0.2703202962875366}]}, {"layer": 10, "memory_to_active": 0.2402540147304535, "contribution_to_active": 0.6944301724433899, "mix_mean": 0.5076502561569214, "mix_low": 0.018988609313964844, "mix_high": 0.005691846366971731, "read_cosine": 0.3864268362522125, "beta_mean": 0.0488152801990509, "nonfinite": 0.0, "weighted_memory_to_active": 0.11732497066259384, "mix_std": 0.27117884159088135, "beta_std": 0.04875973239541054, "beta_low": 0.6563212275505066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.21866115927696228, "output_influence": 0.43713289499282837, "residual_influence": 0.1493559330701828, "state_active": 0.05226796865463257, "state_memory": 0.015843985602259636, "state_peak": 115.61193084716797, "growth_active": 1.0671498775482178, "growth_memory": 0.9350913166999817, "heads": [{"head": 0, "memory_to_active": 0.9358150959014893, "contribution_to_active": 1.9153082370758057, "mix_mean": 0.5785000920295715, "mix_low": 0.07546520233154297, "mix_high": 0.014940261840820312, "read_cosine": 0.10839290916919708, "beta_mean": 0.22080475091934204, "nonfinite": 0.0, "weighted_memory_to_active": 0.5783987641334534, "mix_std": 0.3891410231590271, "beta_std": 0.22641882300376892, "beta_low": 0.099853515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.2234699726104736, "evidence_weights": [-0.11085492372512817, -0.38705047965049744, -1.4875260591506958], "agreement_weight": 0.01167439203709364, "orthogonal_weight": 2.1382699012756348}, {"head": 1, "memory_to_active": 0.012797738425433636, "contribution_to_active": 0.6007224321365356, "mix_mean": 0.6118104457855225, "mix_low": 0.000881195068359375, "mix_high": 0.013646125793457031, "read_cosine": 0.28770047426223755, "beta_mean": 7.843830098863691e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007049777079373598, "mix_std": 0.26857197284698486, "beta_std": 0.00011042472760891542, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010260498151183128, "evidence_weights": [-0.26017671823501587, -0.20078198611736298, 0.588607132434845], "agreement_weight": 0.517199695110321, "orthogonal_weight": -1.5729396343231201}, {"head": 2, "memory_to_active": 0.009155476465821266, "contribution_to_active": 0.5901445746421814, "mix_mean": 0.6232250332832336, "mix_low": 0.00046253204345703125, "mix_high": 0.00447845458984375, "read_cosine": 0.05953118950128555, "beta_mean": 2.5891949917422608e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004083806648850441, "mix_std": 0.25698959827423096, "beta_std": 9.176204912364483e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019470620900392532, "evidence_weights": [-0.24745771288871765, -0.27862438559532166, 0.628693699836731], "agreement_weight": -0.18639108538627625, "orthogonal_weight": -0.4630303978919983}, {"head": 3, "memory_to_active": 0.33921751379966736, "contribution_to_active": 0.10844570398330688, "mix_mean": 0.29022032022476196, "mix_low": 0.034142494201660156, "mix_high": 0.00034809112548828125, "read_cosine": 0.9071372747421265, "beta_mean": 0.06560124456882477, "nonfinite": 0.0, "weighted_memory_to_active": 0.0423431396484375, "mix_std": 0.2311239242553711, "beta_std": 0.058521732687950134, "beta_low": 0.001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.04705836623907089, "evidence_weights": [-0.1075780913233757, -0.14691254496574402, 0.15491671860218048], "agreement_weight": 0.008845830336213112, "orthogonal_weight": 1.4439841508865356}, {"head": 4, "memory_to_active": 0.016294308006763458, "contribution_to_active": 0.534968376159668, "mix_mean": 0.5602931976318359, "mix_low": 0.00042819976806640625, "mix_high": 0.0007314682006835938, "read_cosine": 0.09099571406841278, "beta_mean": 6.937585567357019e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00925061572343111, "mix_std": 0.2519872486591339, "beta_std": 0.0002126756007783115, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.01053097378462553, "evidence_weights": [-0.18265584111213684, -0.26148486137390137, 0.8977649211883545], "agreement_weight": 0.1461055874824524, "orthogonal_weight": -1.2008774280548096}, {"head": 5, "memory_to_active": 0.12824395298957825, "contribution_to_active": 0.41699156165122986, "mix_mean": 0.38185232877731323, "mix_low": 0.002552032470703125, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8648034334182739, "beta_mean": 0.006311967968940735, "nonfinite": 0.0, "weighted_memory_to_active": 0.06282368302345276, "mix_std": 0.22925913333892822, "beta_std": 0.007202947977930307, "beta_low": 0.83648681640625, "beta_high": 0.0, "orthogonal_to_active": 0.01869991049170494, "evidence_weights": [-0.14127817749977112, -0.1579030305147171, 0.2771625816822052], "agreement_weight": 0.09144891798496246, "orthogonal_weight": -0.6471648812294006}]}, {"layer": 11, "memory_to_active": 0.10711875557899475, "contribution_to_active": 0.5963660478591919, "mix_mean": 0.49759775400161743, "mix_low": 0.09581375122070312, "mix_high": 0.046209655702114105, "read_cosine": 0.3694533705711365, "beta_mean": 0.028755873441696167, "nonfinite": 0.0, "weighted_memory_to_active": 0.05258888006210327, "mix_std": 0.32195621728897095, "beta_std": 0.04127468913793564, "beta_low": 0.6135661005973816, "beta_high": 0.0, "orthogonal_to_active": 0.05132412165403366, "output_influence": 0.5139039754867554, "residual_influence": 0.12918727099895477, "state_active": 0.18238040804862976, "state_memory": 0.005511441268026829, "state_peak": 31.48910903930664, "growth_active": 1.0998708009719849, "growth_memory": 1.6476718187332153, "heads": [{"head": 0, "memory_to_active": 0.007812567055225372, "contribution_to_active": 0.6041281223297119, "mix_mean": 0.631869375705719, "mix_low": 0.0014934539794921875, "mix_high": 0.0076923370361328125, "read_cosine": 0.2636984586715698, "beta_mean": 9.219085768563673e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005325310863554478, "mix_std": 0.25939232110977173, "beta_std": 0.0040884679183363914, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.004107566550374031, "evidence_weights": [-0.16730433702468872, -0.3694989085197449, 0.7347172498703003], "agreement_weight": 0.3084738850593567, "orthogonal_weight": -0.7993466258049011}, {"head": 1, "memory_to_active": 0.23022720217704773, "contribution_to_active": 0.35596612095832825, "mix_mean": 0.40955889225006104, "mix_low": 0.0067043304443359375, "mix_high": 0.000255584716796875, "read_cosine": 0.5101540088653564, "beta_mean": 0.019388800486922264, "nonfinite": 0.0, "weighted_memory_to_active": 0.10266512632369995, "mix_std": 0.26096993684768677, "beta_std": 0.04044022038578987, "beta_low": 0.6004638671875, "beta_high": 0.0, "orthogonal_to_active": 0.044074155390262604, "evidence_weights": [-0.17048904299736023, -0.21285338699817657, 0.5590857863426208], "agreement_weight": 0.07064440846443176, "orthogonal_weight": -0.5174306035041809}, {"head": 2, "memory_to_active": 0.009652967564761639, "contribution_to_active": 0.9823147654533386, "mix_mean": 0.4469267725944519, "mix_low": 0.25064563751220703, "mix_high": 0.17138385772705078, "read_cosine": 0.20890465378761292, "beta_mean": 0.06977219134569168, "nonfinite": 0.0, "weighted_memory_to_active": 0.0073524885810911655, "mix_std": 0.44328761100769043, "beta_std": 0.08956891298294067, "beta_low": 0.4432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.0538773350417614, "evidence_weights": [0.0372636541724205, -0.4178772270679474, -1.3111293315887451], "agreement_weight": 0.005648487247526646, "orthogonal_weight": 7.586947917938232}, {"head": 3, "memory_to_active": 0.011283712461590767, "contribution_to_active": 0.8404736518859863, "mix_mean": 0.4148593544960022, "mix_low": 0.30556297302246094, "mix_high": 0.09281635284423828, "read_cosine": 0.6115037798881531, "beta_mean": 0.05491572618484497, "nonfinite": 0.0, "weighted_memory_to_active": 0.006902122404426336, "mix_std": 0.43733516335487366, "beta_std": 0.07840897887945175, "beta_low": 0.47821044921875, "beta_high": 0.0, "orthogonal_to_active": 0.03781729191541672, "evidence_weights": [0.16543641686439514, -0.3227471113204956, -0.20504723489284515], "agreement_weight": 0.05173075944185257, "orthogonal_weight": 7.930002212524414}, {"head": 4, "memory_to_active": 0.29418647289276123, "contribution_to_active": 0.4238508641719818, "mix_mean": 0.5498461723327637, "mix_low": 0.008451461791992188, "mix_high": 0.0009450912475585938, "read_cosine": 0.04936331510543823, "beta_mean": 0.019465703517198563, "nonfinite": 0.0, "weighted_memory_to_active": 0.1486557126045227, "mix_std": 0.2702485918998718, "beta_std": 0.021829966455698013, "beta_low": 0.41656494140625, "beta_high": 0.0, "orthogonal_to_active": 0.14187686145305634, "evidence_weights": [-0.2413029670715332, -0.22504602372646332, 1.9500643014907837], "agreement_weight": -0.2003297358751297, "orthogonal_weight": -0.9707338809967041}, {"head": 5, "memory_to_active": 0.08954964578151703, "contribution_to_active": 0.37146270275115967, "mix_mean": 0.5325257182121277, "mix_low": 0.0020246505737304688, "mix_high": 0.004164695739746094, "read_cosine": 0.5730956792831421, "beta_mean": 0.00890062190592289, "nonfinite": 0.0, "weighted_memory_to_active": 0.04463252052664757, "mix_std": 0.26050353050231934, "beta_std": 0.013311569578945637, "beta_low": 0.74322509765625, "beta_high": 0.0, "orthogonal_to_active": 0.026191499084234238, "evidence_weights": [-0.21412119269371033, -0.30491140484809875, 0.6338775157928467], "agreement_weight": 0.08032167702913284, "orthogonal_weight": -0.7465656995773315}]}, {"layer": 12, "memory_to_active": 0.2708337604999542, "contribution_to_active": 0.5902566313743591, "mix_mean": 0.5316641330718994, "mix_low": 0.008680185303092003, "mix_high": 0.0054972972720861435, "read_cosine": 0.22358188033103943, "beta_mean": 0.01536436378955841, "nonfinite": 0.0, "weighted_memory_to_active": 0.15295886993408203, "mix_std": 0.2634720802307129, "beta_std": 0.016735047101974487, "beta_low": 0.8231608867645264, "beta_high": 0.0, "orthogonal_to_active": 0.08067354559898376, "output_influence": 0.48310327529907227, "residual_influence": 0.1331155002117157, "state_active": 0.035847317427396774, "state_memory": 0.009400326758623123, "state_peak": 51.646385192871094, "growth_active": 1.1318691968917847, "growth_memory": 1.2204350233078003, "heads": [{"head": 0, "memory_to_active": 0.04275522381067276, "contribution_to_active": 0.5795186758041382, "mix_mean": 0.5518534183502197, "mix_low": 0.0004024505615234375, "mix_high": 0.003833770751953125, "read_cosine": 0.2714002728462219, "beta_mean": 0.0007329238578677177, "nonfinite": 0.0, "weighted_memory_to_active": 0.02284894697368145, "mix_std": 0.2699698209762573, "beta_std": 0.001641839393414557, "beta_low": 0.9952392578125, "beta_high": 0.0, "orthogonal_to_active": 0.020048268139362335, "evidence_weights": [-0.20710548758506775, -0.27705734968185425, 0.16886621713638306], "agreement_weight": 0.09898711740970612, "orthogonal_weight": -0.9342357516288757}, {"head": 1, "memory_to_active": 0.12020977586507797, "contribution_to_active": 0.5375257134437561, "mix_mean": 0.5518406629562378, "mix_low": 0.0003414154052734375, "mix_high": 0.0017137527465820312, "read_cosine": 0.4222017824649811, "beta_mean": 0.0012375962687656283, "nonfinite": 0.0, "weighted_memory_to_active": 0.07019065320491791, "mix_std": 0.26559174060821533, "beta_std": 0.012510562315583229, "beta_low": 0.988037109375, "beta_high": 0.0, "orthogonal_to_active": 0.058413222432136536, "evidence_weights": [-0.20907142758369446, -0.27518555521965027, 1.0043025016784668], "agreement_weight": 0.17158709466457367, "orthogonal_weight": -0.9265605807304382}, {"head": 2, "memory_to_active": 0.6202005743980408, "contribution_to_active": 0.5396981239318848, "mix_mean": 0.35247766971588135, "mix_low": 0.023286819458007812, "mix_high": 0.00013828277587890625, "read_cosine": 0.49054914712905884, "beta_mean": 0.08796190470457077, "nonfinite": 0.0, "weighted_memory_to_active": 0.27633780241012573, "mix_std": 0.24569174647331238, "beta_std": 0.061692818999290466, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.2788187563419342, "evidence_weights": [-0.2346186488866806, -0.05359719321131706, -0.032355908304452896], "agreement_weight": 0.02751634456217289, "orthogonal_weight": 1.2773592472076416}, {"head": 3, "memory_to_active": 0.03100760467350483, "contribution_to_active": 0.7560454607009888, "mix_mean": 0.7023723125457764, "mix_low": 0.0002155303955078125, "mix_high": 0.017004966735839844, "read_cosine": 0.09492579102516174, "beta_mean": 0.0002636265126056969, "nonfinite": 0.0, "weighted_memory_to_active": 0.023727506399154663, "mix_std": 0.2348044514656067, "beta_std": 0.004764088895171881, "beta_low": 0.99615478515625, "beta_high": 0.0, "orthogonal_to_active": 0.02266533300280571, "evidence_weights": [-0.2911599278450012, -0.3410470485687256, 1.4999202489852905], "agreement_weight": 0.45798730850219727, "orthogonal_weight": -0.9753912687301636}, {"head": 4, "memory_to_active": 0.7148776054382324, "contribution_to_active": 0.6478563547134399, "mix_mean": 0.3991277813911438, "mix_low": 0.020691871643066406, "mix_high": 0.003930091857910156, "read_cosine": -0.12771961092948914, "beta_mean": 0.00141068734228611, "nonfinite": 0.0, "weighted_memory_to_active": 0.46047142148017883, "mix_std": 0.2849210500717163, "beta_std": 0.01389478798955679, "beta_low": 0.97479248046875, "beta_high": 0.0, "orthogonal_to_active": 0.05394016206264496, "evidence_weights": [-0.13554595410823822, -0.21615299582481384, -0.032187264412641525], "agreement_weight": 0.003248529741540551, "orthogonal_weight": -0.1232159435749054}, {"head": 5, "memory_to_active": 0.09595170617103577, "contribution_to_active": 0.4808952808380127, "mix_mean": 0.6323128938674927, "mix_low": 0.0071430206298828125, "mix_high": 0.0063629150390625, "read_cosine": 0.19013383984565735, "beta_mean": 0.0005794459138996899, "nonfinite": 0.0, "weighted_memory_to_active": 0.06417685747146606, "mix_std": 0.2798536717891693, "beta_std": 0.0059061842039227486, "beta_low": 0.98370361328125, "beta_high": 0.0, "orthogonal_to_active": 0.05015551298856735, "evidence_weights": [-0.24424472451210022, -0.31669074296951294, 1.3206872940063477], "agreement_weight": 0.36851462721824646, "orthogonal_weight": -0.8333556652069092}]}, {"layer": 13, "memory_to_active": 0.13866513967514038, "contribution_to_active": 0.5349169373512268, "mix_mean": 0.5309382081031799, "mix_low": 0.005972862243652344, "mix_high": 0.007993539795279503, "read_cosine": 0.4066632390022278, "beta_mean": 0.01695976033806801, "nonfinite": 0.0, "weighted_memory_to_active": 0.06144562363624573, "mix_std": 0.26133906841278076, "beta_std": 0.01971764862537384, "beta_low": 0.8025615215301514, "beta_high": 0.0, "orthogonal_to_active": 0.06320153921842575, "output_influence": 0.5076149702072144, "residual_influence": 0.16940894722938538, "state_active": 0.11003421247005463, "state_memory": 0.04230904579162598, "state_peak": 95.87413024902344, "growth_active": 0.9325698614120483, "growth_memory": 1.0171403884887695, "heads": [{"head": 0, "memory_to_active": 0.025322824716567993, "contribution_to_active": 0.5711958408355713, "mix_mean": 0.5957432985305786, "mix_low": 0.0005273818969726562, "mix_high": 0.0031042098999023438, "read_cosine": 0.42014139890670776, "beta_mean": 9.868515189737082e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015733880922198296, "mix_std": 0.26849278807640076, "beta_std": 0.00018991481920238584, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015549608506262302, "evidence_weights": [-0.19993674755096436, -0.2671090364456177, 0.7749556303024292], "agreement_weight": 0.1400758922100067, "orthogonal_weight": -1.0999903678894043}, {"head": 1, "memory_to_active": 0.010769614018499851, "contribution_to_active": 0.59283447265625, "mix_mean": 0.5550830364227295, "mix_low": 0.0018253326416015625, "mix_high": 0.0033588409423828125, "read_cosine": 0.20592179894447327, "beta_mean": 4.395415089675225e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005994477774947882, "mix_std": 0.2748732566833496, "beta_std": 8.152174996212125e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007668480277061462, "evidence_weights": [-0.1841464638710022, -0.2666439712047577, 0.6643660068511963], "agreement_weight": 0.3801236152648926, "orthogonal_weight": -1.3281691074371338}, {"head": 2, "memory_to_active": 0.09654536843299866, "contribution_to_active": 0.4588859975337982, "mix_mean": 0.38143104314804077, "mix_low": 0.0016775131225585938, "mix_high": 0.00017642974853515625, "read_cosine": 0.6793765425682068, "beta_mean": 0.01344798319041729, "nonfinite": 0.0, "weighted_memory_to_active": 0.04319915175437927, "mix_std": 0.2323610931634903, "beta_std": 0.04457788169384003, "beta_low": 0.826416015625, "beta_high": 0.0, "orthogonal_to_active": 0.020799929276108742, "evidence_weights": [-0.19206632673740387, -0.2052476704120636, 0.4277171194553375], "agreement_weight": 0.0078054629266262054, "orthogonal_weight": -0.825711727142334}, {"head": 3, "memory_to_active": 0.19541746377944946, "contribution_to_active": 0.5901790857315063, "mix_mean": 0.63981032371521, "mix_low": 0.02500438690185547, "mix_high": 0.03068065643310547, "read_cosine": -0.0402836799621582, "beta_mean": 0.0011094613000750542, "nonfinite": 0.0, "weighted_memory_to_active": 0.13860204815864563, "mix_std": 0.29719170928001404, "beta_std": 0.01134822703897953, "beta_low": 0.99017333984375, "beta_high": 0.0, "orthogonal_to_active": 0.1254737675189972, "evidence_weights": [-0.32175061106681824, -0.24775947630405426, 1.7542592287063599], "agreement_weight": 4.945950031280518, "orthogonal_weight": -0.9865731596946716}, {"head": 4, "memory_to_active": 0.43737363815307617, "contribution_to_active": 0.29246532917022705, "mix_mean": 0.3646933436393738, "mix_low": 0.0048122406005859375, "mix_high": 3.814697265625e-06, "read_cosine": 0.87473464012146, "beta_mean": 0.08670264482498169, "nonfinite": 0.0, "weighted_memory_to_active": 0.12101759016513824, "mix_std": 0.22239772975444794, "beta_std": 0.05855416879057884, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.17307600378990173, "evidence_weights": [-0.10345473885536194, -0.23552477359771729, 0.4720117449760437], "agreement_weight": 0.00010425230721011758, "orthogonal_weight": 2.2116384506225586}, {"head": 5, "memory_to_active": 0.06656186282634735, "contribution_to_active": 0.7039408683776855, "mix_mean": 0.6488683223724365, "mix_low": 0.0019903182983398438, "mix_high": 0.010637283325195312, "read_cosine": 0.3000885844230652, "beta_mean": 0.00035583574208430946, "nonfinite": 0.0, "weighted_memory_to_active": 0.04412660002708435, "mix_std": 0.27271783351898193, "beta_std": 0.003554184455424547, "beta_low": 0.99871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.036641430109739304, "evidence_weights": [-0.2208624929189682, -0.3054097890853882, 0.8503523468971252], "agreement_weight": 0.16214364767074585, "orthogonal_weight": -0.9185325503349304}]}, {"layer": 14, "memory_to_active": 0.24059997498989105, "contribution_to_active": 0.4113183319568634, "mix_mean": 0.4172170162200928, "mix_low": 0.02910693548619747, "mix_high": 0.005139350891113281, "read_cosine": 0.6123743057250977, "beta_mean": 0.06505507230758667, "nonfinite": 0.0, "weighted_memory_to_active": 0.10212185978889465, "mix_std": 0.24552181363105774, "beta_std": 0.06081799790263176, "beta_low": 0.4825134575366974, "beta_high": 0.0, "orthogonal_to_active": 0.041476525366306305, "output_influence": 0.3309401869773865, "residual_influence": 0.12852931022644043, "state_active": 0.060825809836387634, "state_memory": 0.015861090272665024, "state_peak": 36.81573486328125, "growth_active": 1.1243443489074707, "growth_memory": 1.2882390022277832, "heads": [{"head": 0, "memory_to_active": 0.35184961557388306, "contribution_to_active": 0.1872280091047287, "mix_mean": 0.1833430528640747, "mix_low": 0.019765853881835938, "mix_high": 9.5367431640625e-06, "read_cosine": 0.8026783466339111, "beta_mean": 0.06459931284189224, "nonfinite": 0.0, "weighted_memory_to_active": 0.07761193066835403, "mix_std": 0.177750825881958, "beta_std": 0.05592899024486542, "beta_low": 0.00787353515625, "beta_high": 0.0, "orthogonal_to_active": 0.0716266930103302, "evidence_weights": [-0.12876412272453308, -0.03590298071503639, 0.46048909425735474], "agreement_weight": 0.02719087153673172, "orthogonal_weight": 1.5369631052017212}, {"head": 1, "memory_to_active": 0.5002590417861938, "contribution_to_active": 0.3010943531990051, "mix_mean": 0.22169296443462372, "mix_low": 0.13764572143554688, "mix_high": 1.33514404296875e-05, "read_cosine": 0.8513846397399902, "beta_mean": 0.272098183631897, "nonfinite": 0.0, "weighted_memory_to_active": 0.2203574776649475, "mix_std": 0.2547245919704437, "beta_std": 0.25778937339782715, "beta_low": 0.1015625, "beta_high": 0.0, "orthogonal_to_active": 0.014413814060389996, "evidence_weights": [-0.012979158200323582, -0.2100333422422409, -0.11011248826980591], "agreement_weight": 0.0563003309071064, "orthogonal_weight": 0.16143137216567993}, {"head": 2, "memory_to_active": 0.1469677984714508, "contribution_to_active": 0.5589122772216797, "mix_mean": 0.5755562782287598, "mix_low": 0.0026731491088867188, "mix_high": 0.0024919509887695312, "read_cosine": 0.41176486015319824, "beta_mean": 0.0024298173375427723, "nonfinite": 0.0, "weighted_memory_to_active": 0.09329931437969208, "mix_std": 0.2651192247867584, "beta_std": 0.0035115499049425125, "beta_low": 0.968505859375, "beta_high": 0.0, "orthogonal_to_active": 0.06987269967794418, "evidence_weights": [-0.14609499275684357, -0.3161689341068268, 0.8883113265037537], "agreement_weight": 0.08284664154052734, "orthogonal_weight": -0.8582738041877747}, {"head": 3, "memory_to_active": 0.058268941938877106, "contribution_to_active": 0.6057497262954712, "mix_mean": 0.6626293659210205, "mix_low": 0.005557060241699219, "mix_high": 0.018590927124023438, "read_cosine": -0.0038181915879249573, "beta_mean": 0.006750733591616154, "nonfinite": 0.0, "weighted_memory_to_active": 0.037575699388980865, "mix_std": 0.25609076023101807, "beta_std": 0.007559956982731819, "beta_low": 0.7916259765625, "beta_high": 0.0, "orthogonal_to_active": 0.026384923607110977, "evidence_weights": [-0.3560309112071991, -0.3151688277721405, 0.3327256441116333], "agreement_weight": 0.07343156635761261, "orthogonal_weight": -0.8108828067779541}, {"head": 4, "memory_to_active": 0.15488632023334503, "contribution_to_active": 0.48478296399116516, "mix_mean": 0.4932444989681244, "mix_low": 0.005078315734863281, "mix_high": 0.008507728576660156, "read_cosine": 0.8677877187728882, "beta_mean": 0.0425025075674057, "nonfinite": 0.0, "weighted_memory_to_active": 0.08858354389667511, "mix_std": 0.2642975449562073, "beta_std": 0.028966456651687622, "beta_low": 0.05889892578125, "beta_high": 0.0, "orthogonal_to_active": 0.024631066247820854, "evidence_weights": [-0.22047211229801178, -0.24887093901634216, 0.43007349967956543], "agreement_weight": 0.13016889989376068, "orthogonal_weight": -0.5992098450660706}, {"head": 5, "memory_to_active": 0.23136813938617706, "contribution_to_active": 0.3301427364349365, "mix_mean": 0.3668361008167267, "mix_low": 0.0039215087890625, "mix_high": 0.0012226104736328125, "read_cosine": 0.7444486021995544, "beta_mean": 0.0019498454639688134, "nonfinite": 0.0, "weighted_memory_to_active": 0.09530319273471832, "mix_std": 0.2551478445529938, "beta_std": 0.01115165464580059, "beta_low": 0.96661376953125, "beta_high": 0.0, "orthogonal_to_active": 0.041929941624403, "evidence_weights": [-0.17832210659980774, -0.16861207783222198, 0.2679041624069214], "agreement_weight": 0.07851767539978027, "orthogonal_weight": -0.7046148777008057}]}, {"layer": 15, "memory_to_active": 0.05712371692061424, "contribution_to_active": 0.481391966342926, "mix_mean": 0.4044046401977539, "mix_low": 0.008859475143253803, "mix_high": 0.0022573473397642374, "read_cosine": 0.6607711315155029, "beta_mean": 0.03841298073530197, "nonfinite": 0.0, "weighted_memory_to_active": 0.027374887838959694, "mix_std": 0.2845779061317444, "beta_std": 0.042731959372758865, "beta_low": 0.7022298574447632, "beta_high": 0.0, "orthogonal_to_active": 0.006886494345963001, "output_influence": 0.2978076636791229, "residual_influence": 0.16689781844615936, "state_active": 0.14670595526695251, "state_memory": 0.010386189445853233, "state_peak": 38.20766067504883, "growth_active": 1.3012866973876953, "growth_memory": 1.5598106384277344, "heads": [{"head": 0, "memory_to_active": 0.1043754294514656, "contribution_to_active": 0.4760884642601013, "mix_mean": 0.3660185933113098, "mix_low": 0.005961418151855469, "mix_high": 0.0036840438842773438, "read_cosine": 0.6352825164794922, "beta_mean": 0.041594814509153366, "nonfinite": 0.0, "weighted_memory_to_active": 0.04984423145651817, "mix_std": 0.2777673602104187, "beta_std": 0.03179104998707771, "beta_low": 0.0693359375, "beta_high": 0.0, "orthogonal_to_active": 0.023654216900467873, "evidence_weights": [-0.1847849041223526, -0.18895277380943298, 0.4043578505516052], "agreement_weight": 0.09530401974916458, "orthogonal_weight": -0.7339283227920532}, {"head": 1, "memory_to_active": 0.12197217345237732, "contribution_to_active": 0.4169747233390808, "mix_mean": 0.4205017685890198, "mix_low": 0.0019445419311523438, "mix_high": 0.0054073333740234375, "read_cosine": 0.9159719347953796, "beta_mean": 0.0008438096847385168, "nonfinite": 0.0, "weighted_memory_to_active": 0.05031263083219528, "mix_std": 0.2848901152610779, "beta_std": 0.0025831833481788635, "beta_low": 0.98822021484375, "beta_high": 0.0, "orthogonal_to_active": 0.0004180092364549637, "evidence_weights": [-0.15250705182552338, -0.16273710131645203, 0.20024792850017548], "agreement_weight": 0.12183558940887451, "orthogonal_weight": 0.015706965699791908}, {"head": 2, "memory_to_active": 0.006244816351681948, "contribution_to_active": 0.46042895317077637, "mix_mean": 0.334536075592041, "mix_low": 0.018115997314453125, "mix_high": 4.482269287109375e-05, "read_cosine": 0.7881808280944824, "beta_mean": 8.651318057673052e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029880753718316555, "mix_std": 0.27143925428390503, "beta_std": 0.00021118970471434295, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0007668281905353069, "evidence_weights": [-0.20482976734638214, -0.26926663517951965, 0.16562119126319885], "agreement_weight": 0.008427041582763195, "orthogonal_weight": 0.6125811338424683}, {"head": 3, "memory_to_active": 0.002007417380809784, "contribution_to_active": 0.39321279525756836, "mix_mean": 0.38001227378845215, "mix_low": 0.0058441162109375, "mix_high": 0.0001583099365234375, "read_cosine": 0.5517870187759399, "beta_mean": 1.2178328688605689e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0006824688171036541, "mix_std": 0.2861783504486084, "beta_std": 3.088202720391564e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005803804378956556, "evidence_weights": [-0.24277392029762268, -0.2480403482913971, 0.20474383234977722], "agreement_weight": 0.04029947519302368, "orthogonal_weight": -0.843570351600647}, {"head": 4, "memory_to_active": 0.08428041636943817, "contribution_to_active": 0.6167171597480774, "mix_mean": 0.37092408537864685, "mix_low": 0.012038230895996094, "mix_high": 0.0010356903076171875, "read_cosine": 0.7032449245452881, "beta_mean": 0.18644532561302185, "nonfinite": 0.0, "weighted_memory_to_active": 0.04476409777998924, "mix_std": 0.30804508924484253, "beta_std": 0.21073144674301147, "beta_low": 0.182373046875, "beta_high": 0.0, "orthogonal_to_active": 0.006994706578552723, "evidence_weights": [-0.21531996130943298, -0.2561998665332794, -0.01168938260525465], "agreement_weight": 0.013891184702515602, "orthogonal_weight": 0.31655946373939514}, {"head": 5, "memory_to_active": 0.023862022906541824, "contribution_to_active": 0.5249297618865967, "mix_mean": 0.5544348359107971, "mix_low": 0.009252548217773438, "mix_high": 0.0032138824462890625, "read_cosine": 0.3701595067977905, "beta_mean": 0.0014952152268961072, "nonfinite": 0.0, "weighted_memory_to_active": 0.01565781608223915, "mix_std": 0.279147207736969, "beta_std": 0.01104398351162672, "beta_low": 0.9735107421875, "beta_high": 0.0, "orthogonal_to_active": 0.008904825896024704, "evidence_weights": [-0.29211321473121643, -0.3084869682788849, 0.750569760799408], "agreement_weight": 0.2768288850784302, "orthogonal_weight": 0.7942814826965332}]}]}, {"step": 46000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8837988589984889, "memory_to_active": 0.8191452622413635, "contribution_to_active": 0.5434309244155884, "mix_mean": 0.3994847536087036, "mix_low": 0.13931523263454437, "mix_high": 0.010347595438361168, "read_cosine": 0.35484907031059265, "beta_mean": 0.0732770562171936, "nonfinite": 0.0, "weighted_memory_to_active": 0.2039376050233841, "mix_std": 0.22079278528690338, "beta_std": 0.06395824998617172, "beta_low": 0.5438823699951172, "beta_high": 2.924601358245127e-05, "orthogonal_to_active": 0.058658163994550705, "samples": 4, "scored_tokens": 899, "normal_loss": 3.084700584411621, "normal_token_accuracy": 0.44048944115638733, "state_active": 0.06166989356279373, "state_memory": 0.018162816762924194, "state_peak": 124.86367797851562, "growth_active": 1.1337834596633911, "growth_memory": 1.3401552438735962, "output_influence": 0.40713411569595337, "residual_influence": 0.13505183160305023, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2277109622955322, "no_memory_loss_delta": 0.5128701329231262, "scrambled_loss_delta": 0.11135663837194443, "memory_only_loss_delta": 3.228147029876709, "no_evidence_loss_delta": 0.30864137411117554, "no_agreement_loss_delta": 0.021831005811691284, "no_orthogonal_loss_delta": -0.011484283022582531}, {"index": 94, "tokens": 256, "normal_loss": 3.262517213821411, "no_memory_loss_delta": 0.5525938868522644, "scrambled_loss_delta": 0.34375542402267456, "memory_only_loss_delta": 2.961331844329834, "no_evidence_loss_delta": 0.25096526741981506, "no_agreement_loss_delta": 0.030778011307120323, "no_orthogonal_loss_delta": 0.05071702226996422}, {"index": 188, "tokens": 256, "normal_loss": 3.3517274856567383, "no_memory_loss_delta": 0.7624428272247314, "scrambled_loss_delta": 0.1973283737897873, "memory_only_loss_delta": 3.200796604156494, "no_evidence_loss_delta": 0.4371904134750366, "no_agreement_loss_delta": 0.055338792502880096, "no_orthogonal_loss_delta": 0.05637805536389351}, {"index": 283, "tokens": 131, "normal_loss": 1.9359157085418701, "no_memory_loss_delta": 0.3557327687740326, "scrambled_loss_delta": 0.26371854543685913, "memory_only_loss_delta": 2.859360456466675, "no_evidence_loss_delta": 0.10604260116815567, "no_agreement_loss_delta": -0.018688766285777092, "no_orthogonal_loss_delta": 0.02036750316619873}], "agreement_weight_abs_mean": 0.19390320777893066, "agreement_weight_abs_max": 5.028424263000488, "agreement_weight_mean": 0.16910214722156525, "orthogonal_weight_abs_mean": 0.8018819689750671, "orthogonal_weight_abs_max": 8.074974060058594, "orthogonal_weight_mean": -0.12733422219753265, "no_memory_loss": 3.657053232192993, "no_memory_loss_delta": 0.5723528265953064, "no_memory_token_accuracy_gain": 0.07119020819664001, "no_memory_token_win_fraction": 0.6674082279205322, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 43.580358050399575, "scrambled_loss": 3.3089182376861572, "scrambled_loss_delta": 0.22421787679195404, "scrambled_token_accuracy_gain": 0.02447163499891758, "scrambled_token_win_fraction": 0.621802031993866, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.085899887738616, "memory_only_loss": 6.175342082977295, "memory_only_loss_delta": 3.090641736984253, "memory_only_token_accuracy_gain": 0.2736373841762543, "memory_only_token_win_fraction": 0.9010010957717896, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.45272365149276, "no_evidence_loss": 3.3840014934539795, "no_evidence_loss_delta": 0.29930099844932556, "no_evidence_token_accuracy_gain": 0.026696329936385155, "no_evidence_token_win_fraction": 0.621802031993866, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 25.866376520801648, "no_agreement_loss": 3.1127164363861084, "no_agreement_loss_delta": 0.02801603265106678, "no_agreement_token_accuracy_gain": 0.0022246940061450005, "no_agreement_token_win_fraction": 0.4872080087661743, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.7627223037165876, "no_orthogonal_loss": 3.114894390106201, "no_orthogonal_loss_delta": 0.0301941130310297, "no_orthogonal_token_accuracy_gain": 0.011123470962047577, "no_orthogonal_token_win_fraction": 0.5717464089393616, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.97428242935109, "short_no_memory_loss_delta": 0.4785943329334259, "long_context_memory_gain": 0.0937584936618805, "normal_context_loss_gain": 0.13301700353622437, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.387211322784424, "contribution_to_active": 1.083634614944458, "mix_mean": 0.1040651798248291, "mix_low": 0.5424908399581909, "mix_high": 0.005386829376220703, "read_cosine": 0.4364205598831177, "beta_mean": 0.22581997513771057, "nonfinite": 0.0, "weighted_memory_to_active": 1.1234617233276367, "mix_std": 0.08789646625518799, "beta_std": 0.15082550048828125, "beta_low": 0.0367533415555954, "beta_high": 0.0, "orthogonal_to_active": 0.05076735466718674, "output_influence": 0.19922012090682983, "residual_influence": 0.24098436534404755, "state_active": 0.06686501204967499, "state_memory": 0.044763579964637756, "state_peak": 41.17827606201172, "growth_active": 1.237887978553772, "growth_memory": 1.2829087972640991, "heads": [{"head": 0, "memory_to_active": 2.4950690269470215, "contribution_to_active": 0.4703926146030426, "mix_mean": 0.020757706835865974, "mix_low": 0.5733890533447266, "mix_high": 0.0, "read_cosine": 0.2245122790336609, "beta_mean": 0.20421966910362244, "nonfinite": 0.0, "weighted_memory_to_active": 0.5649558305740356, "mix_std": 0.060198523104190826, "beta_std": 0.10191522538661957, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04790967330336571, "evidence_weights": [-0.013691212050616741, -0.05761004239320755, -0.11231062561273575], "agreement_weight": -0.17020805180072784, "orthogonal_weight": -0.0930793285369873}, {"head": 1, "memory_to_active": 5.991005897521973, "contribution_to_active": 2.3865981101989746, "mix_mean": 0.015695886686444283, "mix_low": 0.7014122009277344, "mix_high": 0.0, "read_cosine": 0.4798769950866699, "beta_mean": 0.38318777084350586, "nonfinite": 0.0, "weighted_memory_to_active": 2.672872543334961, "mix_std": 0.05001828819513321, "beta_std": 0.24444270133972168, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.024140462279319763, "evidence_weights": [0.03048388659954071, -0.09485943615436554, 0.49694764614105225], "agreement_weight": 0.02724621631205082, "orthogonal_weight": -0.01959291286766529}, {"head": 2, "memory_to_active": 2.6809799671173096, "contribution_to_active": 0.49226289987564087, "mix_mean": 0.013688229024410248, "mix_low": 0.6793184280395508, "mix_high": 0.0, "read_cosine": 0.3246227204799652, "beta_mean": 0.27192434668540955, "nonfinite": 0.0, "weighted_memory_to_active": 0.5753922462463379, "mix_std": 0.031424179673194885, "beta_std": 0.19011248648166656, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.049280956387519836, "evidence_weights": [0.06134878844022751, 0.062229741364717484, 0.09255176782608032], "agreement_weight": 0.008818551898002625, "orthogonal_weight": -0.1933801919221878}, {"head": 3, "memory_to_active": 6.869638919830322, "contribution_to_active": 2.620237350463867, "mix_mean": 0.03299177810549736, "mix_low": 0.49559783935546875, "mix_high": 0.0, "read_cosine": 0.3050243854522705, "beta_mean": 0.3638690114021301, "nonfinite": 0.0, "weighted_memory_to_active": 2.8120553493499756, "mix_std": 0.0783352181315422, "beta_std": 0.257252037525177, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.15904518961906433, "evidence_weights": [-0.04109296202659607, -0.045225031673908234, -0.1599612534046173], "agreement_weight": -0.00942117627710104, "orthogonal_weight": -0.06646604090929031}, {"head": 4, "memory_to_active": 2.1779470443725586, "contribution_to_active": 0.09113946557044983, "mix_mean": 0.0077005792409181595, "mix_low": 0.7902297973632812, "mix_high": 0.0, "read_cosine": 0.3454517722129822, "beta_mean": 0.10716044902801514, "nonfinite": 0.0, "weighted_memory_to_active": 0.060679711401462555, "mix_std": 0.017561940476298332, "beta_std": 0.09217216819524765, "beta_low": 0.0096435546875, "beta_high": 0.0, "orthogonal_to_active": 0.006690053269267082, "evidence_weights": [0.13924995064735413, 0.09863854944705963, -0.2828913629055023], "agreement_weight": -0.13589000701904297, "orthogonal_weight": 0.0970863550901413}, {"head": 5, "memory_to_active": 0.10862672328948975, "contribution_to_active": 0.44117671251296997, "mix_mean": 0.5335568785667419, "mix_low": 0.014997482299804688, "mix_high": 0.03232097625732422, "read_cosine": 0.939035177230835, "beta_mean": 0.024558618664741516, "nonfinite": 0.0, "weighted_memory_to_active": 0.054814308881759644, "mix_std": 0.28984060883522034, "beta_std": 0.019058365374803543, "beta_low": 0.21087646484375, "beta_high": 0.0, "orthogonal_to_active": 0.01753777265548706, "evidence_weights": [-0.18967345356941223, -0.22207359969615936, 0.36154165863990784], "agreement_weight": 0.2916783392429352, "orthogonal_weight": -0.6966376900672913}]}, {"layer": 1, "memory_to_active": 2.2593345642089844, "contribution_to_active": 0.4827318489551544, "mix_mean": 0.20158420503139496, "mix_low": 0.37254512310028076, "mix_high": 0.0008959770784713328, "read_cosine": 0.3154953122138977, "beta_mean": 0.17673209309577942, "nonfinite": 0.0, "weighted_memory_to_active": 0.3235316872596741, "mix_std": 0.11556531488895416, "beta_std": 0.12618252635002136, "beta_low": 0.330963134765625, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.043893441557884216, "output_influence": 0.22217470407485962, "residual_influence": 0.05816055089235306, "state_active": 0.029236378148198128, "state_memory": 0.03505866974592209, "state_peak": 40.35462951660156, "growth_active": 1.14876389503479, "growth_memory": 1.4874552488327026, "heads": [{"head": 0, "memory_to_active": 0.02967309206724167, "contribution_to_active": 0.4750998020172119, "mix_mean": 0.48063021898269653, "mix_low": 0.008176803588867188, "mix_high": 0.0014524459838867188, "read_cosine": 0.04576627165079117, "beta_mean": 0.00021327912691049278, "nonfinite": 0.0, "weighted_memory_to_active": 0.017658650875091553, "mix_std": 0.28250759840011597, "beta_std": 0.00034592015435919166, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01685578189790249, "evidence_weights": [-0.1988743096590042, -0.26049527525901794, 0.36032506823539734], "agreement_weight": 0.17897388339042664, "orthogonal_weight": -1.0153077840805054}, {"head": 1, "memory_to_active": 2.906738758087158, "contribution_to_active": 0.26048219203948975, "mix_mean": 0.02340238355100155, "mix_low": 0.44801998138427734, "mix_high": 2.86102294921875e-06, "read_cosine": 0.5027846097946167, "beta_mean": 0.26343342661857605, "nonfinite": 0.0, "weighted_memory_to_active": 0.2908889055252075, "mix_std": 0.044230759143829346, "beta_std": 0.1704740822315216, "beta_low": 0.0003662109375, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.01772753894329071, "evidence_weights": [0.01617312803864479, -0.05321187153458595, 0.018706215545535088], "agreement_weight": -0.005794111173599958, "orthogonal_weight": 0.1512223780155182}, {"head": 2, "memory_to_active": 0.09737887233495712, "contribution_to_active": 0.5382182002067566, "mix_mean": 0.6452406048774719, "mix_low": 0.0008649826049804688, "mix_high": 0.003920555114746094, "read_cosine": 0.5421226024627686, "beta_mean": 0.004285994917154312, "nonfinite": 0.0, "weighted_memory_to_active": 0.07327747344970703, "mix_std": 0.23472890257835388, "beta_std": 0.005827534943819046, "beta_low": 0.9078369140625, "beta_high": 0.0, "orthogonal_to_active": 0.04150809720158577, "evidence_weights": [-0.29007428884506226, -0.35201311111450195, 0.6701672673225403], "agreement_weight": 0.029479913413524628, "orthogonal_weight": -0.6952804923057556}, {"head": 3, "memory_to_active": 2.798583984375, "contribution_to_active": 0.150517076253891, "mix_mean": 0.019352445378899574, "mix_low": 0.6258964538574219, "mix_high": 0.0, "read_cosine": 0.22693628072738647, "beta_mean": 0.29502636194229126, "nonfinite": 0.0, "weighted_memory_to_active": 0.21001774072647095, "mix_std": 0.05505269765853882, "beta_std": 0.24258816242218018, "beta_low": 0.04888916015625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.06379517912864685, "evidence_weights": [0.04295559599995613, -0.008280801586806774, 0.07977546751499176], "agreement_weight": -0.018125418573617935, "orthogonal_weight": -0.4378310739994049}, {"head": 4, "memory_to_active": 3.6709532737731934, "contribution_to_active": 1.4082467555999756, "mix_mean": 0.034639373421669006, "mix_low": 0.29813671112060547, "mix_high": 0.0, "read_cosine": 0.3182375431060791, "beta_mean": 0.16543099284172058, "nonfinite": 0.0, "weighted_memory_to_active": 1.299559235572815, "mix_std": 0.059165600687265396, "beta_std": 0.10687224566936493, "beta_low": 0.01263427734375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.11247602850198746, "evidence_weights": [-0.04027874022722244, -0.003989710938185453, 0.06316154450178146], "agreement_weight": -0.014018881134688854, "orthogonal_weight": 0.11617743223905563}, {"head": 5, "memory_to_active": 4.052678108215332, "contribution_to_active": 0.06382694840431213, "mix_mean": 0.006240084767341614, "mix_low": 0.8541755676269531, "mix_high": 0.0, "read_cosine": 0.25712451338768005, "beta_mean": 0.3320024907588959, "nonfinite": 0.0, "weighted_memory_to_active": 0.0497879795730114, "mix_std": 0.017706269398331642, "beta_std": 0.23098711669445038, "beta_low": 0.01605224609375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.010997974313795567, "evidence_weights": [0.17141053080558777, 0.08994153887033463, 0.16728045046329498], "agreement_weight": -0.1419537216424942, "orthogonal_weight": 0.22436733543872833}]}, {"layer": 2, "memory_to_active": 1.0402131080627441, "contribution_to_active": 0.3435571789741516, "mix_mean": 0.38169652223587036, "mix_low": 0.2576026916503906, "mix_high": 0.003772576805204153, "read_cosine": 0.31998494267463684, "beta_mean": 0.0958937406539917, "nonfinite": 0.0, "weighted_memory_to_active": 0.07554597407579422, "mix_std": 0.19108009338378906, "beta_std": 0.07279347628355026, "beta_low": 0.5600179433822632, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.018952075392007828, "output_influence": 0.3641546368598938, "residual_influence": 0.09356874227523804, "state_active": 0.03756081685423851, "state_memory": 0.016383768990635872, "state_peak": 28.865446090698242, "growth_active": 1.0477215051651, "growth_memory": 1.56947922706604, "heads": [{"head": 0, "memory_to_active": 0.010560249909758568, "contribution_to_active": 0.3403671979904175, "mix_mean": 0.5949878692626953, "mix_low": 0.001129150390625, "mix_high": 0.007904052734375, "read_cosine": 0.17410078644752502, "beta_mean": 8.366681868210435e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006203032098710537, "mix_std": 0.280908465385437, "beta_std": 0.00013960053911432624, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005723894108086824, "evidence_weights": [-0.351093590259552, -0.2097027748823166, 0.11600876599550247], "agreement_weight": 0.16999021172523499, "orthogonal_weight": -0.9643909931182861}, {"head": 1, "memory_to_active": 2.9671401977539062, "contribution_to_active": 0.035496704280376434, "mix_mean": 0.004206816665828228, "mix_low": 0.9192285537719727, "mix_high": 0.0, "read_cosine": 0.15415622293949127, "beta_mean": 0.2748270034790039, "nonfinite": 0.0, "weighted_memory_to_active": 0.026305336505174637, "mix_std": 0.009977864101529121, "beta_std": 0.16844619810581207, "beta_low": 0.00054931640625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.009843866340816021, "evidence_weights": [0.14135931432247162, 0.12839987874031067, 0.48775070905685425], "agreement_weight": 0.01834641955792904, "orthogonal_weight": 0.3968089818954468}, {"head": 2, "memory_to_active": 0.012931747362017632, "contribution_to_active": 0.43771666288375854, "mix_mean": 0.5235034227371216, "mix_low": 0.0013494491577148438, "mix_high": 0.0024728775024414062, "read_cosine": 0.4069138169288635, "beta_mean": 4.26348669861909e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005286990199238062, "mix_std": 0.2787056565284729, "beta_std": 0.00010333443060517311, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000621496990788728, "evidence_weights": [-0.24617928266525269, -0.2853524386882782, 0.5140042901039124], "agreement_weight": 0.01876777410507202, "orthogonal_weight": -0.14014463126659393}, {"head": 3, "memory_to_active": 0.15761767327785492, "contribution_to_active": 0.4993570148944855, "mix_mean": 0.5219436883926392, "mix_low": 0.0003643035888671875, "mix_high": 0.005767822265625, "read_cosine": 0.556334912776947, "beta_mean": 0.0274765957146883, "nonfinite": 0.0, "weighted_memory_to_active": 0.09163141250610352, "mix_std": 0.2648492157459259, "beta_std": 0.032707568258047104, "beta_low": 0.33245849609375, "beta_high": 0.0, "orthogonal_to_active": 0.06494293361902237, "evidence_weights": [-0.1920451521873474, -0.29951420426368713, 0.8311701416969299], "agreement_weight": 0.057674016803503036, "orthogonal_weight": -0.887651264667511}, {"head": 4, "memory_to_active": 3.0839080810546875, "contribution_to_active": 0.21977826952934265, "mix_mean": 0.017803343012928963, "mix_low": 0.6222820281982422, "mix_high": 0.0, "read_cosine": 0.21090427041053772, "beta_mean": 0.2729002833366394, "nonfinite": 0.0, "weighted_memory_to_active": 0.31843674182891846, "mix_std": 0.047539062798023224, "beta_std": 0.2353169023990631, "beta_low": 0.027099609375, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.03233964741230011, "evidence_weights": [0.024205217137932777, 0.011310716159641743, 0.020605292171239853], "agreement_weight": 0.0275331549346447, "orthogonal_weight": -0.13377422094345093}, {"head": 5, "memory_to_active": 0.009120642207562923, "contribution_to_active": 0.5286270380020142, "mix_mean": 0.6277340650558472, "mix_low": 0.001262664794921875, "mix_high": 0.0064907073974609375, "read_cosine": 0.4174995422363281, "beta_mean": 3.222459417884238e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005412324797362089, "mix_std": 0.26450029015541077, "beta_std": 4.724061000160873e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00024060513533186167, "evidence_weights": [-0.27782532572746277, -0.29053354263305664, 0.95215904712677], "agreement_weight": 0.15612797439098358, "orthogonal_weight": -0.0634327232837677}]}, {"layer": 3, "memory_to_active": 0.5209094285964966, "contribution_to_active": 0.45059871673583984, "mix_mean": 0.45779067277908325, "mix_low": 0.059828922152519226, "mix_high": 0.009186427108943462, "read_cosine": 0.22015267610549927, "beta_mean": 0.031288303434848785, "nonfinite": 0.0, "weighted_memory_to_active": 0.1364128291606903, "mix_std": 0.26117944717407227, "beta_std": 0.027181413024663925, "beta_low": 0.7326253652572632, "beta_high": 0.0, "orthogonal_to_active": 0.05657036602497101, "output_influence": 0.3872043490409851, "residual_influence": 0.08262208849191666, "state_active": 0.02450050227344036, "state_memory": 0.007563417311757803, "state_peak": 23.216604232788086, "growth_active": 1.0921683311462402, "growth_memory": 1.7058262825012207, "heads": [{"head": 0, "memory_to_active": 0.42638155817985535, "contribution_to_active": 0.5753751993179321, "mix_mean": 0.5576460361480713, "mix_low": 0.0026006698608398438, "mix_high": 0.02147674560546875, "read_cosine": 0.24490736424922943, "beta_mean": 0.011512422934174538, "nonfinite": 0.0, "weighted_memory_to_active": 0.26045000553131104, "mix_std": 0.312427282333374, "beta_std": 0.01287456601858139, "beta_low": 0.613525390625, "beta_high": 0.0, "orthogonal_to_active": 0.17853470146656036, "evidence_weights": [-0.20792138576507568, -0.24103350937366486, 0.8545030951499939], "agreement_weight": 0.1229131817817688, "orthogonal_weight": -0.7335568070411682}, {"head": 1, "memory_to_active": 0.01601373963057995, "contribution_to_active": 0.41528210043907166, "mix_mean": 0.5371105074882507, "mix_low": 0.006810188293457031, "mix_high": 0.007822990417480469, "read_cosine": 0.17284435033798218, "beta_mean": 8.805569086689502e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009169282391667366, "mix_std": 0.30545446276664734, "beta_std": 0.0001873662113212049, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00683208741247654, "evidence_weights": [-0.24974627792835236, -0.20632977783679962, 0.6289211511611938], "agreement_weight": 0.4299570918083191, "orthogonal_weight": -0.810907244682312}, {"head": 2, "memory_to_active": 2.390692710876465, "contribution_to_active": 0.28941965103149414, "mix_mean": 0.04689234495162964, "mix_low": 0.32323360443115234, "mix_high": 0.0, "read_cosine": 0.2042110562324524, "beta_mean": 0.168349951505661, "nonfinite": 0.0, "weighted_memory_to_active": 0.3437277674674988, "mix_std": 0.09420524537563324, "beta_std": 0.1396382451057434, "beta_low": 0.01788330078125, "beta_high": 0.0, "orthogonal_to_active": 0.006521761417388916, "evidence_weights": [-0.08573801815509796, 0.009715107269585133, -0.00856550969183445], "agreement_weight": -0.02451208233833313, "orthogonal_weight": -0.024052869528532028}, {"head": 3, "memory_to_active": 0.028379227966070175, "contribution_to_active": 0.5163666009902954, "mix_mean": 0.5813956260681152, "mix_low": 0.00684356689453125, "mix_high": 0.010443687438964844, "read_cosine": 0.3130413889884949, "beta_mean": 0.0001126897259382531, "nonfinite": 0.0, "weighted_memory_to_active": 0.019377488642930984, "mix_std": 0.2955113649368286, "beta_std": 0.000305553141515702, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004940218757838011, "evidence_weights": [-0.23485253751277924, -0.22754421830177307, 2.1347861289978027], "agreement_weight": 1.8130502700805664, "orthogonal_weight": -0.3396177291870117}, {"head": 4, "memory_to_active": 0.033859316259622574, "contribution_to_active": 0.26908454298973083, "mix_mean": 0.34825122356414795, "mix_low": 0.01648426055908203, "mix_high": 0.000865936279296875, "read_cosine": 0.20303356647491455, "beta_mean": 0.00023064337437972426, "nonfinite": 0.0, "weighted_memory_to_active": 0.016487378627061844, "mix_std": 0.2824828624725342, "beta_std": 0.0003571581037249416, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015521114692091942, "evidence_weights": [-0.14745794236660004, -0.1710711270570755, 0.11887216567993164], "agreement_weight": -0.03091222047805786, "orthogonal_weight": -0.99415522813797}, {"head": 5, "memory_to_active": 0.23012974858283997, "contribution_to_active": 0.6380642056465149, "mix_mean": 0.6754480600357056, "mix_low": 0.0030012130737304688, "mix_high": 0.014509201049804688, "read_cosine": 0.1828782558441162, "beta_mean": 0.0074360365979373455, "nonfinite": 0.0, "weighted_memory_to_active": 0.16926497220993042, "mix_std": 0.27699536085128784, "beta_std": 0.00972558744251728, "beta_low": 0.76434326171875, "beta_high": 0.0, "orthogonal_to_active": 0.1270723044872284, "evidence_weights": [-0.24980582296848297, -0.3028866946697235, 0.47965124249458313], "agreement_weight": 0.1331198513507843, "orthogonal_weight": -0.794901967048645}]}, {"layer": 4, "memory_to_active": 1.053450345993042, "contribution_to_active": 0.4737701117992401, "mix_mean": 0.3584008514881134, "mix_low": 0.27788639068603516, "mix_high": 0.008091768249869347, "read_cosine": 0.3324718177318573, "beta_mean": 0.11262861639261246, "nonfinite": 0.0, "weighted_memory_to_active": 0.14815753698349, "mix_std": 0.15502454340457916, "beta_std": 0.10205304622650146, "beta_low": 0.382659912109375, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05224180221557617, "output_influence": 0.43618687987327576, "residual_influence": 0.12425181269645691, "state_active": 0.039356812834739685, "state_memory": 0.018173301592469215, "state_peak": 36.172000885009766, "growth_active": 1.2727998495101929, "growth_memory": 1.571694254875183, "heads": [{"head": 0, "memory_to_active": 0.20396976172924042, "contribution_to_active": 0.7210952043533325, "mix_mean": 0.6686881184577942, "mix_low": 0.012807846069335938, "mix_high": 0.019876480102539062, "read_cosine": 0.5030795931816101, "beta_mean": 0.2323848158121109, "nonfinite": 0.0, "weighted_memory_to_active": 0.15601855516433716, "mix_std": 0.34391289949417114, "beta_std": 0.24526995420455933, "beta_low": 0.24517822265625, "beta_high": 0.0, "orthogonal_to_active": 0.06099851056933403, "evidence_weights": [-0.26479512453079224, -0.3647768497467041, 0.11674085259437561], "agreement_weight": 0.016318898648023605, "orthogonal_weight": 0.5020111203193665}, {"head": 1, "memory_to_active": 0.06738758832216263, "contribution_to_active": 0.6397683024406433, "mix_mean": 0.6760054230690002, "mix_low": 9.918212890625e-05, "mix_high": 0.012339591979980469, "read_cosine": 0.649816632270813, "beta_mean": 0.0010347174247726798, "nonfinite": 0.0, "weighted_memory_to_active": 0.048992790281772614, "mix_std": 0.2398846447467804, "beta_std": 0.006284675560891628, "beta_low": 0.98333740234375, "beta_high": 0.0, "orthogonal_to_active": 0.027089454233646393, "evidence_weights": [-0.23830607533454895, -0.3498706519603729, 0.6901847720146179], "agreement_weight": 0.2383677363395691, "orthogonal_weight": -0.7980601191520691}, {"head": 2, "memory_to_active": 1.9813051223754883, "contribution_to_active": 0.2562122344970703, "mix_mean": 0.021946389228105545, "mix_low": 0.5525236129760742, "mix_high": 1.52587890625e-05, "read_cosine": 0.2316228300333023, "beta_mean": 0.08325575292110443, "nonfinite": 0.0, "weighted_memory_to_active": 0.23085595667362213, "mix_std": 0.0455988310277462, "beta_std": 0.07406815141439438, "beta_low": 0.0391845703125, "beta_high": 0.0, "orthogonal_to_active": 0.030670899897813797, "evidence_weights": [0.06088035926222801, 0.020357292145490646, 0.07076907902956009], "agreement_weight": 0.13975092768669128, "orthogonal_weight": -0.20156730711460114}, {"head": 3, "memory_to_active": 1.8127434253692627, "contribution_to_active": 0.12794150412082672, "mix_mean": 0.029116230085492134, "mix_low": 0.44173717498779297, "mix_high": 0.0, "read_cosine": 0.20604979991912842, "beta_mean": 0.13109800219535828, "nonfinite": 0.0, "weighted_memory_to_active": 0.19619306921958923, "mix_std": 0.052402544766664505, "beta_std": 0.11927290260791779, "beta_low": 0.02825927734375, "beta_high": 0.0, "orthogonal_to_active": 0.003980649635195732, "evidence_weights": [-0.02454143576323986, 0.05961991474032402, -0.23277132213115692], "agreement_weight": 0.07303924858570099, "orthogonal_weight": -0.031037423759698868}, {"head": 4, "memory_to_active": 0.01043058093637228, "contribution_to_active": 0.6581459045410156, "mix_mean": 0.740958034992218, "mix_low": 0.0001163482666015625, "mix_high": 0.01631450653076172, "read_cosine": 0.14669376611709595, "beta_mean": 2.0214420146658085e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007595540955662727, "mix_std": 0.2164396196603775, "beta_std": 6.732475594617426e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029445448890328407, "evidence_weights": [-0.33879804611206055, -0.332281231880188, 0.6010600328445435], "agreement_weight": 0.34294071793556213, "orthogonal_weight": -0.414499968290329}, {"head": 5, "memory_to_active": 2.2448651790618896, "contribution_to_active": 0.4394572675228119, "mix_mean": 0.013690710067749023, "mix_low": 0.6600341796875, "mix_high": 4.76837158203125e-06, "read_cosine": 0.25756824016571045, "beta_mean": 0.22797814011573792, "nonfinite": 0.0, "weighted_memory_to_active": 0.24928921461105347, "mix_std": 0.031908635050058365, "beta_std": 0.1673552542924881, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.18776677548885345, "evidence_weights": [0.04861973971128464, -0.02955218032002449, -0.3027247488498688], "agreement_weight": -0.07597610354423523, "orthogonal_weight": 0.910652756690979}]}, {"layer": 5, "memory_to_active": 0.45525336265563965, "contribution_to_active": 0.5156731605529785, "mix_mean": 0.48413974046707153, "mix_low": 0.05933030694723129, "mix_high": 0.01687081716954708, "read_cosine": 0.0927172601222992, "beta_mean": 0.05397482216358185, "nonfinite": 0.0, "weighted_memory_to_active": 0.15348070859909058, "mix_std": 0.25609779357910156, "beta_std": 0.07146111875772476, "beta_low": 0.646148681640625, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05902157723903656, "output_influence": 0.5185620784759521, "residual_influence": 0.14328739047050476, "state_active": 0.044496942311525345, "state_memory": 0.0110155139118433, "state_peak": 62.65202331542969, "growth_active": 1.1569536924362183, "growth_memory": 1.2678120136260986, "heads": [{"head": 0, "memory_to_active": 0.4931601285934448, "contribution_to_active": 0.48707032203674316, "mix_mean": 0.35247623920440674, "mix_low": 0.019072532653808594, "mix_high": 0.0014162063598632812, "read_cosine": -0.08119864761829376, "beta_mean": 0.029236935079097748, "nonfinite": 0.0, "weighted_memory_to_active": 0.14700940251350403, "mix_std": 0.2699742317199707, "beta_std": 0.05453372746706009, "beta_low": 0.4122314453125, "beta_high": 0.0, "orthogonal_to_active": 0.049778446555137634, "evidence_weights": [-0.12678280472755432, -0.15655970573425293, -0.06113602593541145], "agreement_weight": -0.02940785512328148, "orthogonal_weight": -0.33929353952407837}, {"head": 1, "memory_to_active": 0.0136790806427598, "contribution_to_active": 0.550714910030365, "mix_mean": 0.6720422506332397, "mix_low": 0.0015783309936523438, "mix_high": 0.03043651580810547, "read_cosine": 0.21632499992847443, "beta_mean": 3.3484931918792427e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007608235813677311, "mix_std": 0.2800155282020569, "beta_std": 8.331951539730653e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007022574078291655, "evidence_weights": [-0.24722842872142792, -0.2841416001319885, 0.8877813220024109], "agreement_weight": 0.6252459287643433, "orthogonal_weight": -0.9709226489067078}, {"head": 2, "memory_to_active": 0.013972898945212364, "contribution_to_active": 0.5422175526618958, "mix_mean": 0.6417676210403442, "mix_low": 0.002819061279296875, "mix_high": 0.020048141479492188, "read_cosine": -0.4103127121925354, "beta_mean": 0.00011500569235067815, "nonfinite": 0.0, "weighted_memory_to_active": 0.00918561965227127, "mix_std": 0.27176615595817566, "beta_std": 0.00017677889263723046, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002989630214869976, "evidence_weights": [-0.25864699482917786, -0.3439127504825592, 0.5116669535636902], "agreement_weight": 0.02237125113606453, "orthogonal_weight": -0.4062242805957794}, {"head": 3, "memory_to_active": 1.6513116359710693, "contribution_to_active": 0.24403822422027588, "mix_mean": 0.10628947615623474, "mix_low": 0.1455221176147461, "mix_high": 1.9073486328125e-06, "read_cosine": 0.13307315111160278, "beta_mean": 0.10593083500862122, "nonfinite": 0.0, "weighted_memory_to_active": 0.38448530435562134, "mix_std": 0.14590127766132355, "beta_std": 0.15383264422416687, "beta_low": 0.19134521484375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.15398645401000977, "evidence_weights": [-0.06457393616437912, -0.09160219877958298, -0.011064931750297546], "agreement_weight": 0.06011977791786194, "orthogonal_weight": -0.4731583297252655}, {"head": 4, "memory_to_active": 0.46833112835884094, "contribution_to_active": 0.586161196231842, "mix_mean": 0.33647048473358154, "mix_low": 0.18496036529541016, "mix_high": 0.006946563720703125, "read_cosine": 0.5233793258666992, "beta_mean": 0.1882723569869995, "nonfinite": 0.0, "weighted_memory_to_active": 0.31110018491744995, "mix_std": 0.342279851436615, "beta_std": 0.21886616945266724, "beta_low": 0.275390625, "beta_high": 0.0, "orthogonal_to_active": 0.10210166871547699, "evidence_weights": [-0.1181756779551506, -0.2781701982021332, -0.04942922294139862], "agreement_weight": 0.044136568903923035, "orthogonal_weight": 0.49586591124534607}, {"head": 5, "memory_to_active": 0.09106525033712387, "contribution_to_active": 0.6838365793228149, "mix_mean": 0.7957922220230103, "mix_low": 0.0020294189453125, "mix_high": 0.04237556457519531, "read_cosine": 0.1750374138355255, "beta_mean": 0.00026029229047708213, "nonfinite": 0.0, "weighted_memory_to_active": 0.061495453119277954, "mix_std": 0.22664958238601685, "beta_std": 0.001274089445360005, "beta_low": 0.9979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.03825066611170769, "evidence_weights": [-0.31918641924858093, -0.33357346057891846, 1.9786970615386963], "agreement_weight": 0.4298693537712097, "orthogonal_weight": -0.6626125574111938}]}, {"layer": 6, "memory_to_active": 0.325655996799469, "contribution_to_active": 0.5992007851600647, "mix_mean": 0.5759146213531494, "mix_low": 0.05767282098531723, "mix_high": 0.03595400229096413, "read_cosine": 0.2795802652835846, "beta_mean": 0.06454680860042572, "nonfinite": 0.0, "weighted_memory_to_active": 0.1454593390226364, "mix_std": 0.236754909157753, "beta_std": 0.07237471640110016, "beta_low": 0.520355224609375, "beta_high": 0.0, "orthogonal_to_active": 0.037260428071022034, "output_influence": 0.5240844488143921, "residual_influence": 0.17051202058792114, "state_active": 0.04566866531968117, "state_memory": 0.008060355670750141, "state_peak": 81.65298461914062, "growth_active": 1.1773087978363037, "growth_memory": 1.2118189334869385, "heads": [{"head": 0, "memory_to_active": 0.10655839741230011, "contribution_to_active": 0.6988334655761719, "mix_mean": 0.7811406850814819, "mix_low": 0.00030994415283203125, "mix_high": 0.027341842651367188, "read_cosine": 0.015934612601995468, "beta_mean": 0.015330493450164795, "nonfinite": 0.0, "weighted_memory_to_active": 0.08427061140537262, "mix_std": 0.20958717167377472, "beta_std": 0.020960785448551178, "beta_low": 0.55694580078125, "beta_high": 0.0, "orthogonal_to_active": 0.07091271132230759, "evidence_weights": [-0.31587812304496765, -0.3392924964427948, 1.1025978326797485], "agreement_weight": 0.3550814390182495, "orthogonal_weight": -0.8708879947662354}, {"head": 1, "memory_to_active": 0.00452530337497592, "contribution_to_active": 0.4938544034957886, "mix_mean": 0.5565796494483948, "mix_low": 0.0010862350463867188, "mix_high": 0.013405799865722656, "read_cosine": 0.5366663336753845, "beta_mean": 1.0055015991383698e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0019880342297255993, "mix_std": 0.28825685381889343, "beta_std": 1.7522068446851335e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011296311859041452, "evidence_weights": [-0.19715188443660736, -0.2950051724910736, 1.1147255897521973], "agreement_weight": 0.05706106498837471, "orthogonal_weight": -0.7539886236190796}, {"head": 2, "memory_to_active": 1.0341697931289673, "contribution_to_active": 0.08985646069049835, "mix_mean": 0.03592139482498169, "mix_low": 0.3359518051147461, "mix_high": 0.0, "read_cosine": 0.23059609532356262, "beta_mean": 0.040809132158756256, "nonfinite": 0.0, "weighted_memory_to_active": 0.07122800499200821, "mix_std": 0.05936050042510033, "beta_std": 0.033039577305316925, "beta_low": 0.05706787109375, "beta_high": 0.0, "orthogonal_to_active": 0.002785541582852602, "evidence_weights": [0.08121130615472794, -0.02929842658340931, 0.28685370087623596], "agreement_weight": 0.13941167294979095, "orthogonal_weight": -0.046693455427885056}, {"head": 3, "memory_to_active": 0.15179277956485748, "contribution_to_active": 0.6821480393409729, "mix_mean": 0.6542180180549622, "mix_low": 0.007491111755371094, "mix_high": 0.021951675415039062, "read_cosine": 0.4622493386268616, "beta_mean": 0.21115905046463013, "nonfinite": 0.0, "weighted_memory_to_active": 0.10815151035785675, "mix_std": 0.32661011815071106, "beta_std": 0.24220415949821472, "beta_low": 0.20245361328125, "beta_high": 0.0, "orthogonal_to_active": 0.05752220004796982, "evidence_weights": [-0.17999519407749176, -0.3356301486492157, 0.18911293148994446], "agreement_weight": -0.011319713667035103, "orthogonal_weight": 0.6195299029350281}, {"head": 4, "memory_to_active": 0.6279197931289673, "contribution_to_active": 1.0179741382598877, "mix_mean": 0.7685871124267578, "mix_low": 0.0008153915405273438, "mix_high": 0.14721202850341797, "read_cosine": 0.16405758261680603, "beta_mean": 0.11976800113916397, "nonfinite": 0.0, "weighted_memory_to_active": 0.5865433812141418, "mix_std": 0.28299474716186523, "beta_std": 0.13755479454994202, "beta_low": 0.30584716796875, "beta_high": 0.0, "orthogonal_to_active": 0.07075627893209457, "evidence_weights": [-0.2349195033311844, -0.5911558866500854, -0.2853289842605591], "agreement_weight": -0.0158613920211792, "orthogonal_weight": 0.13533936440944672}, {"head": 5, "memory_to_active": 0.02896988019347191, "contribution_to_active": 0.61253821849823, "mix_mean": 0.6590405702590942, "mix_low": 0.00038242340087890625, "mix_high": 0.005812644958496094, "read_cosine": 0.26797759532928467, "beta_mean": 0.00020415012841112912, "nonfinite": 0.0, "weighted_memory_to_active": 0.020574526861310005, "mix_std": 0.25372010469436646, "beta_std": 0.0004714478272944689, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.020456213504076004, "evidence_weights": [-0.2414848506450653, -0.3119129538536072, 0.42443498969078064], "agreement_weight": 0.16272397339344025, "orthogonal_weight": -1.0602526664733887}]}, {"layer": 7, "memory_to_active": 1.7165846824645996, "contribution_to_active": 0.2551961839199066, "mix_mean": 0.1426270306110382, "mix_low": 0.2634762227535248, "mix_high": 0.0007859866600483656, "read_cosine": 0.22709859907627106, "beta_mean": 0.128762349486351, "nonfinite": 0.0, "weighted_memory_to_active": 0.20828959345817566, "mix_std": 0.14061737060546875, "beta_std": 0.10019737482070923, "beta_low": 0.1927693784236908, "beta_high": 0.000213623046875, "orthogonal_to_active": 0.07155623286962509, "output_influence": 0.26666074991226196, "residual_influence": 0.06153206527233124, "state_active": 0.013396156951785088, "state_memory": 0.02557182125747204, "state_peak": 27.90113639831543, "growth_active": 1.155795931816101, "growth_memory": 1.267920732498169, "heads": [{"head": 0, "memory_to_active": 2.3300669193267822, "contribution_to_active": 0.25059786438941956, "mix_mean": 0.03499240800738335, "mix_low": 0.4428892135620117, "mix_high": 9.5367431640625e-07, "read_cosine": 0.34778672456741333, "beta_mean": 0.15379253029823303, "nonfinite": 0.0, "weighted_memory_to_active": 0.30544161796569824, "mix_std": 0.07592624425888062, "beta_std": 0.1167510449886322, "beta_low": 0.004150390625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.0026179521810263395, "evidence_weights": [0.03310937061905861, -0.07464750856161118, 0.06491440534591675], "agreement_weight": -0.034496188163757324, "orthogonal_weight": 0.013243233785033226}, {"head": 1, "memory_to_active": 1.0083956718444824, "contribution_to_active": 0.3939072787761688, "mix_mean": 0.21417032182216644, "mix_low": 0.1195688247680664, "mix_high": 0.004216194152832031, "read_cosine": 0.35668694972991943, "beta_mean": 0.09547639638185501, "nonfinite": 0.0, "weighted_memory_to_active": 0.33136412501335144, "mix_std": 0.23938612639904022, "beta_std": 0.0661502555012703, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.038493964821100235, "evidence_weights": [-0.20892450213432312, -0.10548185557126999, -0.21790021657943726], "agreement_weight": 0.03829726204276085, "orthogonal_weight": 0.13541539013385773}, {"head": 2, "memory_to_active": 0.7288601994514465, "contribution_to_active": 0.3272082507610321, "mix_mean": 0.1627771556377411, "mix_low": 0.055843353271484375, "mix_high": 0.00027370452880859375, "read_cosine": 0.08382029831409454, "beta_mean": 0.049985721707344055, "nonfinite": 0.0, "weighted_memory_to_active": 0.22056445479393005, "mix_std": 0.18697790801525116, "beta_std": 0.04476571083068848, "beta_low": 0.0797119140625, "beta_high": 0.0, "orthogonal_to_active": 0.0966523140668869, "evidence_weights": [-0.06478646397590637, -0.14736983180046082, 0.6678593754768372], "agreement_weight": 0.021941473707556725, "orthogonal_weight": -0.44985106587409973}, {"head": 3, "memory_to_active": 4.531942367553711, "contribution_to_active": 0.09167477488517761, "mix_mean": 0.032566141337156296, "mix_low": 0.3373851776123047, "mix_high": 0.0, "read_cosine": 0.0898318737745285, "beta_mean": 0.3785521984100342, "nonfinite": 0.0, "weighted_memory_to_active": 0.30829301476478577, "mix_std": 0.04673486575484276, "beta_std": 0.27513426542282104, "beta_low": 0.00384521484375, "beta_high": 0.0009765625, "orthogonal_to_active": 0.26389768719673157, "evidence_weights": [0.06964017450809479, -0.01453379262238741, 0.6860634684562683], "agreement_weight": 0.04849465563893318, "orthogonal_weight": -0.9386098980903625}, {"head": 4, "memory_to_active": 1.6427268981933594, "contribution_to_active": 0.08079355210065842, "mix_mean": 0.01757879927754402, "mix_low": 0.6209449768066406, "mix_high": 0.0, "read_cosine": 0.21058669686317444, "beta_mean": 0.09441322833299637, "nonfinite": 0.0, "weighted_memory_to_active": 0.059035319834947586, "mix_std": 0.038331687450408936, "beta_std": 0.09765194356441498, "beta_low": 0.06939697265625, "beta_high": 0.0, "orthogonal_to_active": 0.008670404553413391, "evidence_weights": [0.14067870378494263, 0.01870950311422348, 0.2452310025691986], "agreement_weight": 0.04584624990820885, "orthogonal_weight": -0.17822431027889252}, {"head": 5, "memory_to_active": 0.05751626938581467, "contribution_to_active": 0.38699525594711304, "mix_mean": 0.3936772346496582, "mix_low": 0.004225730895996094, "mix_high": 0.000225067138671875, "read_cosine": 0.2738790512084961, "beta_mean": 0.0003539978351909667, "nonfinite": 0.0, "weighted_memory_to_active": 0.02503889426589012, "mix_std": 0.2563474178314209, "beta_std": 0.0007310486980713904, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.019005045294761658, "evidence_weights": [-0.15037916600704193, -0.20853418111801147, 0.5717210173606873], "agreement_weight": 0.17578525841236115, "orthogonal_weight": -0.8433769345283508}]}, {"layer": 8, "memory_to_active": 0.3283816874027252, "contribution_to_active": 0.5348182320594788, "mix_mean": 0.4393790364265442, "mix_low": 0.04883512109518051, "mix_high": 0.0011685689678415656, "read_cosine": 0.45523980259895325, "beta_mean": 0.02107897773385048, "nonfinite": 0.0, "weighted_memory_to_active": 0.09405234456062317, "mix_std": 0.22262714803218842, "beta_std": 0.0292857363820076, "beta_low": 0.6627706289291382, "beta_high": 0.0, "orthogonal_to_active": 0.06976348161697388, "output_influence": 0.47490233182907104, "residual_influence": 0.15286961197853088, "state_active": 0.04634685069322586, "state_memory": 0.007895390503108501, "state_peak": 34.00718307495117, "growth_active": 1.0643795728683472, "growth_memory": 1.3135625123977661, "heads": [{"head": 0, "memory_to_active": 0.06387165933847427, "contribution_to_active": 0.5584668517112732, "mix_mean": 0.6036093831062317, "mix_low": 0.00037097930908203125, "mix_high": 0.0012454986572265625, "read_cosine": 0.46511420607566833, "beta_mean": 0.0036505907773971558, "nonfinite": 0.0, "weighted_memory_to_active": 0.045411616563797, "mix_std": 0.24460314214229584, "beta_std": 0.012698713690042496, "beta_low": 0.92144775390625, "beta_high": 0.0, "orthogonal_to_active": 0.0285726860165596, "evidence_weights": [-0.17135679721832275, -0.3898490071296692, 0.6739156246185303], "agreement_weight": 0.18866440653800964, "orthogonal_weight": -0.7942127585411072}, {"head": 1, "memory_to_active": 0.10355497896671295, "contribution_to_active": 0.5317763686180115, "mix_mean": 0.4925154447555542, "mix_low": 7.343292236328125e-05, "mix_high": 0.0005702972412109375, "read_cosine": 0.24535106122493744, "beta_mean": 0.001874297740869224, "nonfinite": 0.0, "weighted_memory_to_active": 0.048582687973976135, "mix_std": 0.24479863047599792, "beta_std": 0.008010132238268852, "beta_low": 0.97503662109375, "beta_high": 0.0, "orthogonal_to_active": 0.03379376605153084, "evidence_weights": [-0.18942919373512268, -0.269362211227417, 0.7076184749603271], "agreement_weight": 0.08466927707195282, "orthogonal_weight": -0.7467945218086243}, {"head": 2, "memory_to_active": 0.017351247370243073, "contribution_to_active": 0.6642905473709106, "mix_mean": 0.6528278589248657, "mix_low": 0.0009288787841796875, "mix_high": 0.0034360885620117188, "read_cosine": 0.3912765383720398, "beta_mean": 0.00018597528105601668, "nonfinite": 0.0, "weighted_memory_to_active": 0.01322226133197546, "mix_std": 0.22671788930892944, "beta_std": 0.0038806037046015263, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.008533243089914322, "evidence_weights": [-0.22376438975334167, -0.34124723076820374, 1.0032906532287598], "agreement_weight": 0.23369739949703217, "orthogonal_weight": -0.7362194657325745}, {"head": 3, "memory_to_active": 0.10827166587114334, "contribution_to_active": 0.6166174411773682, "mix_mean": 0.5696648359298706, "mix_low": 0.0009031295776367188, "mix_high": 0.0016717910766601562, "read_cosine": 0.8075702786445618, "beta_mean": 0.008939128369092941, "nonfinite": 0.0, "weighted_memory_to_active": 0.07914890348911285, "mix_std": 0.25498297810554504, "beta_std": 0.007803014013916254, "beta_low": 0.66143798828125, "beta_high": 0.0, "orthogonal_to_active": 0.025381501764059067, "evidence_weights": [-0.18239443004131317, -0.3027097284793854, 0.5932770371437073], "agreement_weight": 0.20438559353351593, "orthogonal_weight": -0.6352528929710388}, {"head": 4, "memory_to_active": 1.315307855606079, "contribution_to_active": 0.18980614840984344, "mix_mean": 0.06166829168796539, "mix_low": 0.1959085464477539, "mix_high": 0.0, "read_cosine": 0.34678611159324646, "beta_mean": 0.03986680135130882, "nonfinite": 0.0, "weighted_memory_to_active": 0.18538598716259003, "mix_std": 0.08986324071884155, "beta_std": 0.05617426335811615, "beta_low": 0.30120849609375, "beta_high": 0.0, "orthogonal_to_active": 0.017307927832007408, "evidence_weights": [0.030392814427614212, -0.07493585348129272, 0.04097665473818779], "agreement_weight": 0.08801352232694626, "orthogonal_weight": -0.10452961921691895}, {"head": 5, "memory_to_active": 0.36193257570266724, "contribution_to_active": 0.6479520201683044, "mix_mean": 0.2559881806373596, "mix_low": 0.09482574462890625, "mix_high": 8.7738037109375e-05, "read_cosine": 0.47534066438674927, "beta_mean": 0.071957066655159, "nonfinite": 0.0, "weighted_memory_to_active": 0.19256260991096497, "mix_std": 0.2747969627380371, "beta_std": 0.08714768290519714, "beta_low": 0.11761474609375, "beta_high": 0.0, "orthogonal_to_active": 0.3049917221069336, "evidence_weights": [-0.09477339684963226, -0.19273048639297485, -0.19146977365016937], "agreement_weight": -0.015660995617508888, "orthogonal_weight": 1.8460209369659424}]}, {"layer": 9, "memory_to_active": 1.0123957395553589, "contribution_to_active": 0.6627791523933411, "mix_mean": 0.3429616391658783, "mix_low": 0.1258022040128708, "mix_high": 0.004898866172879934, "read_cosine": 0.3521515727043152, "beta_mean": 0.05070104822516441, "nonfinite": 0.0, "weighted_memory_to_active": 0.3720666766166687, "mix_std": 0.21819105744361877, "beta_std": 0.037781231105327606, "beta_low": 0.5118815302848816, "beta_high": 0.0, "orthogonal_to_active": 0.037082456052303314, "output_influence": 0.5205351114273071, "residual_influence": 0.14317156374454498, "state_active": 0.04282267019152641, "state_memory": 0.01788436435163021, "state_peak": 50.230751037597656, "growth_active": 1.1891621351242065, "growth_memory": 1.3536163568496704, "heads": [{"head": 0, "memory_to_active": 0.0013814473059028387, "contribution_to_active": 0.5584293007850647, "mix_mean": 0.6343883275985718, "mix_low": 0.0014781951904296875, "mix_high": 0.02066516876220703, "read_cosine": 0.28210994601249695, "beta_mean": 3.480562099866802e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0008958830148912966, "mix_std": 0.2898457646369934, "beta_std": 1.3126880730851553e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008662004256621003, "evidence_weights": [-0.30068621039390564, -0.3089665174484253, 0.8352257013320923], "agreement_weight": 0.12039678543806076, "orthogonal_weight": -1.0150015354156494}, {"head": 1, "memory_to_active": 0.02667665295302868, "contribution_to_active": 0.6729679107666016, "mix_mean": 0.5377689599990845, "mix_low": 0.0074920654296875, "mix_high": 0.0016889572143554688, "read_cosine": 0.3828892707824707, "beta_mean": 7.677678513573483e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01941702514886856, "mix_std": 0.28635847568511963, "beta_std": 0.0002190882369177416, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.013790464960038662, "evidence_weights": [-0.23223501443862915, -0.2842411398887634, 1.15909743309021], "agreement_weight": 0.41638943552970886, "orthogonal_weight": -0.8582437634468079}, {"head": 2, "memory_to_active": 0.17851346731185913, "contribution_to_active": 0.644547700881958, "mix_mean": 0.5953249931335449, "mix_low": 0.0014400482177734375, "mix_high": 0.005837440490722656, "read_cosine": 0.35289889574050903, "beta_mean": 0.000764245807658881, "nonfinite": 0.0, "weighted_memory_to_active": 0.11824936419725418, "mix_std": 0.2683662474155426, "beta_std": 0.004934936761856079, "beta_low": 0.9947509765625, "beta_high": 0.0, "orthogonal_to_active": 0.06655964255332947, "evidence_weights": [-0.2018573135137558, -0.2748887240886688, 0.7455982565879822], "agreement_weight": 0.2998199164867401, "orthogonal_weight": -0.6557376980781555}, {"head": 3, "memory_to_active": 1.4125652313232422, "contribution_to_active": 0.19402173161506653, "mix_mean": 0.057203762233257294, "mix_low": 0.3265094757080078, "mix_high": 0.00016498565673828125, "read_cosine": 0.38805752992630005, "beta_mean": 0.07454798370599747, "nonfinite": 0.0, "weighted_memory_to_active": 0.1752183437347412, "mix_std": 0.11449383199214935, "beta_std": 0.05593952164053917, "beta_low": 0.00531005859375, "beta_high": 0.0, "orthogonal_to_active": 0.0009505205671302974, "evidence_weights": [0.07149288803339005, 0.003943471238017082, 0.05381999909877777], "agreement_weight": 0.08993526548147202, "orthogonal_weight": -0.0058028819039464}, {"head": 4, "memory_to_active": 2.91918683052063, "contribution_to_active": 1.3558330535888672, "mix_mean": 0.13996663689613342, "mix_low": 0.14145374298095703, "mix_high": 0.0008096694946289062, "read_cosine": 0.21073143184185028, "beta_mean": 0.1540393978357315, "nonfinite": 0.0, "weighted_memory_to_active": 1.3267292976379395, "mix_std": 0.19836865365505219, "beta_std": 0.08902014791965485, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02717442996799946, "evidence_weights": [-0.0728382095694542, -0.15531055629253387, -0.024627575650811195], "agreement_weight": -0.059386592358350754, "orthogonal_weight": -0.021105334162712097}, {"head": 5, "memory_to_active": 1.5360510349273682, "contribution_to_active": 0.5508751273155212, "mix_mean": 0.09311693161725998, "mix_low": 0.2764396667480469, "mix_high": 0.0002269744873046875, "read_cosine": 0.4962224066257477, "beta_mean": 0.0747743770480156, "nonfinite": 0.0, "weighted_memory_to_active": 0.5918898582458496, "mix_std": 0.15171322226524353, "beta_std": 0.07656056433916092, "beta_low": 0.0712890625, "beta_high": 0.0, "orthogonal_to_active": 0.11315345764160156, "evidence_weights": [0.10485653579235077, -0.099240742623806, 0.13902898132801056], "agreement_weight": 0.01253180205821991, "orthogonal_weight": 0.2745862901210785}]}, {"layer": 10, "memory_to_active": 0.23523592948913574, "contribution_to_active": 0.6896496415138245, "mix_mean": 0.513462245464325, "mix_low": 0.018631458282470703, "mix_high": 0.006237030029296875, "read_cosine": 0.3745884597301483, "beta_mean": 0.04610529541969299, "nonfinite": 0.0, "weighted_memory_to_active": 0.11342040449380875, "mix_std": 0.2696443200111389, "beta_std": 0.047615569084882736, "beta_low": 0.6681417226791382, "beta_high": 0.0, "orthogonal_to_active": 0.21483831107616425, "output_influence": 0.43941837549209595, "residual_influence": 0.14990699291229248, "state_active": 0.052917808294296265, "state_memory": 0.015646733343601227, "state_peak": 124.86367797851562, "growth_active": 1.0354503393173218, "growth_memory": 0.9589027762413025, "heads": [{"head": 0, "memory_to_active": 0.8821473717689514, "contribution_to_active": 1.8774545192718506, "mix_mean": 0.5866742730140686, "mix_low": 0.06801986694335938, "mix_high": 0.01636791229248047, "read_cosine": 0.10933205485343933, "beta_mean": 0.20295017957687378, "nonfinite": 0.0, "weighted_memory_to_active": 0.5479879975318909, "mix_std": 0.3908606171607971, "beta_std": 0.21735715866088867, "beta_low": 0.14666748046875, "beta_high": 0.0, "orthogonal_to_active": 1.1969133615493774, "evidence_weights": [-0.11352851241827011, -0.39259564876556396, -1.498800277709961], "agreement_weight": 0.0186124499887228, "orthogonal_weight": 2.2113893032073975}, {"head": 1, "memory_to_active": 0.01296964567154646, "contribution_to_active": 0.601081371307373, "mix_mean": 0.615105390548706, "mix_low": 0.0013704299926757812, "mix_high": 0.014325141906738281, "read_cosine": 0.28369539976119995, "beta_mean": 8.123022416839376e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007434142753481865, "mix_std": 0.2690867781639099, "beta_std": 0.00011268431262578815, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010988250374794006, "evidence_weights": [-0.2588180899620056, -0.19972431659698486, 0.5929485559463501], "agreement_weight": 0.5227881073951721, "orthogonal_weight": -1.5998114347457886}, {"head": 2, "memory_to_active": 0.022147679701447487, "contribution_to_active": 0.6061320900917053, "mix_mean": 0.6406709551811218, "mix_low": 0.00024509429931640625, "mix_high": 0.0055255889892578125, "read_cosine": 0.017937418073415756, "beta_mean": 4.318261198932305e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009344170801341534, "mix_std": 0.2528306245803833, "beta_std": 0.00020453495380934328, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.004436993971467018, "evidence_weights": [-0.25039300322532654, -0.28136444091796875, 0.6639947891235352], "agreement_weight": -0.19011759757995605, "orthogonal_weight": -0.4504321813583374}, {"head": 3, "memory_to_active": 0.33788928389549255, "contribution_to_active": 0.10180728882551193, "mix_mean": 0.2735154628753662, "mix_low": 0.039933204650878906, "mix_high": 0.0003452301025390625, "read_cosine": 0.9085411429405212, "beta_mean": 0.06783846020698547, "nonfinite": 0.0, "weighted_memory_to_active": 0.04028970003128052, "mix_std": 0.23144882917404175, "beta_std": 0.06141873449087143, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.043165989220142365, "evidence_weights": [-0.10472655296325684, -0.14399950206279755, 0.15141530334949493], "agreement_weight": 0.009224927052855492, "orthogonal_weight": 1.437252163887024}, {"head": 4, "memory_to_active": 0.020048875361680984, "contribution_to_active": 0.5446054339408875, "mix_mean": 0.5715540647506714, "mix_low": 0.00024700164794921875, "mix_high": 0.0008535385131835938, "read_cosine": 0.06859168410301208, "beta_mean": 7.256979733938351e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011488012038171291, "mix_std": 0.2467634081840515, "beta_std": 0.00018507131608203053, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.013579150661826134, "evidence_weights": [-0.1834801733493805, -0.262043833732605, 0.9044039249420166], "agreement_weight": 0.14225398004055023, "orthogonal_weight": -1.2402794361114502}, {"head": 5, "memory_to_active": 0.13621260225772858, "contribution_to_active": 0.4068170189857483, "mix_mean": 0.3932531476020813, "mix_low": 0.0019731521606445312, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8594328761100769, "beta_mean": 0.005646150559186935, "nonfinite": 0.0, "weighted_memory_to_active": 0.06397837400436401, "mix_std": 0.2268756926059723, "beta_std": 0.006415225565433502, "beta_low": 0.8614501953125, "beta_high": 0.0, "orthogonal_to_active": 0.019946034997701645, "evidence_weights": [-0.14162799715995789, -0.15844696760177612, 0.27568742632865906], "agreement_weight": 0.09112806618213654, "orthogonal_weight": -0.6442593336105347}]}, {"layer": 11, "memory_to_active": 0.10150791704654694, "contribution_to_active": 0.5994651317596436, "mix_mean": 0.5000051259994507, "mix_low": 0.09481097012758255, "mix_high": 0.050156120210886, "read_cosine": 0.36733514070510864, "beta_mean": 0.03013714775443077, "nonfinite": 0.0, "weighted_memory_to_active": 0.050304777920246124, "mix_std": 0.3237382769584656, "beta_std": 0.04364042356610298, "beta_low": 0.635498046875, "beta_high": 0.0, "orthogonal_to_active": 0.04749599099159241, "output_influence": 0.5316599607467651, "residual_influence": 0.13424673676490784, "state_active": 0.19107982516288757, "state_memory": 0.005239959340542555, "state_peak": 29.736948013305664, "growth_active": 1.1410870552062988, "growth_memory": 1.6267832517623901, "heads": [{"head": 0, "memory_to_active": 0.009091472253203392, "contribution_to_active": 0.5964113473892212, "mix_mean": 0.6282382011413574, "mix_low": 0.0010547637939453125, "mix_high": 0.008893013000488281, "read_cosine": 0.2467225193977356, "beta_mean": 9.346862498205155e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006373193114995956, "mix_std": 0.261064738035202, "beta_std": 0.004237509798258543, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.0049333879724144936, "evidence_weights": [-0.1678936779499054, -0.37106072902679443, 0.7382334470748901], "agreement_weight": 0.3067713975906372, "orthogonal_weight": -0.802406370639801}, {"head": 1, "memory_to_active": 0.2293967604637146, "contribution_to_active": 0.35949960350990295, "mix_mean": 0.41071903705596924, "mix_low": 0.006741523742675781, "mix_high": 0.00028324127197265625, "read_cosine": 0.4886545240879059, "beta_mean": 0.01779995486140251, "nonfinite": 0.0, "weighted_memory_to_active": 0.10419012606143951, "mix_std": 0.2613655924797058, "beta_std": 0.039305951446294785, "beta_low": 0.63140869140625, "beta_high": 0.0, "orthogonal_to_active": 0.04520466551184654, "evidence_weights": [-0.16889558732509613, -0.21516922116279602, 0.561616063117981], "agreement_weight": 0.06985846161842346, "orthogonal_weight": -0.5234545469284058}, {"head": 2, "memory_to_active": 0.008815942332148552, "contribution_to_active": 0.9812048673629761, "mix_mean": 0.45077717304229736, "mix_low": 0.24531078338623047, "mix_high": 0.18602466583251953, "read_cosine": 0.17453546822071075, "beta_mean": 0.07512331008911133, "nonfinite": 0.0, "weighted_memory_to_active": 0.006264620926231146, "mix_std": 0.44755324721336365, "beta_std": 0.09676438570022583, "beta_low": 0.45391845703125, "beta_high": 0.0, "orthogonal_to_active": 0.04723617434501648, "evidence_weights": [0.029510539025068283, -0.41917482018470764, -1.301474928855896], "agreement_weight": -0.004244319628924131, "orthogonal_weight": 7.742043972015381}, {"head": 3, "memory_to_active": 0.009965960867702961, "contribution_to_active": 0.8476845026016235, "mix_mean": 0.4201502799987793, "mix_low": 0.3054695129394531, "mix_high": 0.10056495666503906, "read_cosine": 0.6082628965377808, "beta_mean": 0.06342996656894684, "nonfinite": 0.0, "weighted_memory_to_active": 0.006199071183800697, "mix_std": 0.4415598511695862, "beta_std": 0.09040375053882599, "beta_low": 0.48040771484375, "beta_high": 0.0, "orthogonal_to_active": 0.03479871153831482, "evidence_weights": [0.17094174027442932, -0.3177139461040497, -0.21571461856365204], "agreement_weight": 0.0516793467104435, "orthogonal_weight": 8.074974060058594}, {"head": 4, "memory_to_active": 0.25533396005630493, "contribution_to_active": 0.42082253098487854, "mix_mean": 0.5478044748306274, "mix_low": 0.008614540100097656, "mix_high": 0.0008296966552734375, "read_cosine": 0.04365077242255211, "beta_mean": 0.015866361558437347, "nonfinite": 0.0, "weighted_memory_to_active": 0.1304536610841751, "mix_std": 0.2723321318626404, "beta_std": 0.017929527908563614, "beta_low": 0.5013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.1260034739971161, "evidence_weights": [-0.24022045731544495, -0.2251579463481903, 1.958541750907898], "agreement_weight": -0.1986381560564041, "orthogonal_weight": -0.9777323603630066}, {"head": 5, "memory_to_active": 0.09644342213869095, "contribution_to_active": 0.39116790890693665, "mix_mean": 0.5423414707183838, "mix_low": 0.001674652099609375, "mix_high": 0.00434112548828125, "read_cosine": 0.6421843767166138, "beta_mean": 0.008509818464517593, "nonfinite": 0.0, "weighted_memory_to_active": 0.04834798723459244, "mix_std": 0.25855404138565063, "beta_std": 0.013201408088207245, "beta_low": 0.746337890625, "beta_high": 0.0, "orthogonal_to_active": 0.026799537241458893, "evidence_weights": [-0.21455234289169312, -0.30698391795158386, 0.6504808068275452], "agreement_weight": 0.08220846205949783, "orthogonal_weight": -0.7570116519927979}]}, {"layer": 12, "memory_to_active": 0.2508962154388428, "contribution_to_active": 0.5733238458633423, "mix_mean": 0.5231837034225464, "mix_low": 0.009567579254508018, "mix_high": 0.005398591514676809, "read_cosine": 0.21476438641548157, "beta_mean": 0.015428462997078896, "nonfinite": 0.0, "weighted_memory_to_active": 0.13577742874622345, "mix_std": 0.2612523138523102, "beta_std": 0.016188636422157288, "beta_low": 0.8274638056755066, "beta_high": 0.0, "orthogonal_to_active": 0.0762593075633049, "output_influence": 0.4798402190208435, "residual_influence": 0.13251960277557373, "state_active": 0.03453855589032173, "state_memory": 0.009532742202281952, "state_peak": 52.17652130126953, "growth_active": 1.1245195865631104, "growth_memory": 1.2103806734085083, "heads": [{"head": 0, "memory_to_active": 0.04704957827925682, "contribution_to_active": 0.5688087940216064, "mix_mean": 0.5480426549911499, "mix_low": 0.0004749298095703125, "mix_high": 0.004622459411621094, "read_cosine": 0.2906433641910553, "beta_mean": 0.0008201237069442868, "nonfinite": 0.0, "weighted_memory_to_active": 0.0242740698158741, "mix_std": 0.2707844078540802, "beta_std": 0.002100215991958976, "beta_low": 0.99468994140625, "beta_high": 0.0, "orthogonal_to_active": 0.021354082971811295, "evidence_weights": [-0.20655231177806854, -0.27625492215156555, 0.16787202656269073], "agreement_weight": 0.09897952526807785, "orthogonal_weight": -0.9437002539634705}, {"head": 1, "memory_to_active": 0.12242750078439713, "contribution_to_active": 0.5379193425178528, "mix_mean": 0.5496866703033447, "mix_low": 0.0004405975341796875, "mix_high": 0.0016107559204101562, "read_cosine": 0.40027886629104614, "beta_mean": 0.0008861184469424188, "nonfinite": 0.0, "weighted_memory_to_active": 0.07413872331380844, "mix_std": 0.26547810435295105, "beta_std": 0.010860087350010872, "beta_low": 0.99444580078125, "beta_high": 0.0, "orthogonal_to_active": 0.062073953449726105, "evidence_weights": [-0.20757240056991577, -0.27405890822410583, 1.013048529624939], "agreement_weight": 0.17867162823677063, "orthogonal_weight": -0.9252686500549316}, {"head": 2, "memory_to_active": 0.6406384110450745, "contribution_to_active": 0.5077868103981018, "mix_mean": 0.3007848262786865, "mix_low": 0.029799461364746094, "mix_high": 0.0002346038818359375, "read_cosine": 0.4509679675102234, "beta_mean": 0.08945894241333008, "nonfinite": 0.0, "weighted_memory_to_active": 0.2776315212249756, "mix_std": 0.23572051525115967, "beta_std": 0.06417934596538544, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.2650724947452545, "evidence_weights": [-0.2298332303762436, -0.0503702275454998, -0.0311574786901474], "agreement_weight": 0.025487534701824188, "orthogonal_weight": 1.2853318452835083}, {"head": 3, "memory_to_active": 0.03625743091106415, "contribution_to_active": 0.7486881613731384, "mix_mean": 0.7031508684158325, "mix_low": 0.00023555755615234375, "mix_high": 0.014847755432128906, "read_cosine": 0.07818234711885452, "beta_mean": 0.00028545933309942484, "nonfinite": 0.0, "weighted_memory_to_active": 0.02596864476799965, "mix_std": 0.23287925124168396, "beta_std": 0.005488258320838213, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.025101695209741592, "evidence_weights": [-0.29113906621932983, -0.3410801291465759, 1.507619857788086], "agreement_weight": 0.4612843692302704, "orthogonal_weight": -0.9870030283927917}, {"head": 4, "memory_to_active": 0.5796776413917542, "contribution_to_active": 0.5822433829307556, "mix_mean": 0.4055159389972687, "mix_low": 0.019346237182617188, "mix_high": 0.003997802734375, "read_cosine": -0.13539695739746094, "beta_mean": 0.000705597223713994, "nonfinite": 0.0, "weighted_memory_to_active": 0.3604051470756531, "mix_std": 0.2843778133392334, "beta_std": 0.009665826335549355, "beta_low": 0.98419189453125, "beta_high": 0.0, "orthogonal_to_active": 0.04348231479525566, "evidence_weights": [-0.13847212493419647, -0.21666909754276276, -0.033681027591228485], "agreement_weight": -0.004505991004407406, "orthogonal_weight": -0.1268458515405655}, {"head": 5, "memory_to_active": 0.0793265774846077, "contribution_to_active": 0.49449610710144043, "mix_mean": 0.6319212913513184, "mix_low": 0.0071086883544921875, "mix_high": 0.0070781707763671875, "read_cosine": 0.20391064882278442, "beta_mean": 0.0004145356942899525, "nonfinite": 0.0, "weighted_memory_to_active": 0.05224645137786865, "mix_std": 0.27827373147010803, "beta_std": 0.004838090389966965, "beta_low": 0.9930419921875, "beta_high": 0.0, "orthogonal_to_active": 0.040471285581588745, "evidence_weights": [-0.24329964816570282, -0.3174477219581604, 1.3165439367294312], "agreement_weight": 0.3683786690235138, "orthogonal_weight": -0.8246382474899292}]}, {"layer": 13, "memory_to_active": 0.12455859780311584, "contribution_to_active": 0.5240090489387512, "mix_mean": 0.5252110958099365, "mix_low": 0.00565083883702755, "mix_high": 0.008118152618408203, "read_cosine": 0.4059836268424988, "beta_mean": 0.015621483325958252, "nonfinite": 0.0, "weighted_memory_to_active": 0.05415375530719757, "mix_std": 0.26126551628112793, "beta_std": 0.01874426007270813, "beta_low": 0.8029683828353882, "beta_high": 0.0, "orthogonal_to_active": 0.05514591932296753, "output_influence": 0.5025251507759094, "residual_influence": 0.17001007497310638, "state_active": 0.11631014198064804, "state_memory": 0.041849639266729355, "state_peak": 101.51251983642578, "growth_active": 0.9242486357688904, "growth_memory": 0.9799626469612122, "heads": [{"head": 0, "memory_to_active": 0.019318975508213043, "contribution_to_active": 0.5700252652168274, "mix_mean": 0.5908481478691101, "mix_low": 0.000492095947265625, "mix_high": 0.0030364990234375, "read_cosine": 0.42993080615997314, "beta_mean": 7.907501276349649e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011933810077607632, "mix_std": 0.26870301365852356, "beta_std": 0.0001813727430999279, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011685049161314964, "evidence_weights": [-0.19959190487861633, -0.26742157340049744, 0.7908985018730164], "agreement_weight": 0.14306706190109253, "orthogonal_weight": -1.0890583992004395}, {"head": 1, "memory_to_active": 0.015458273701369762, "contribution_to_active": 0.5880388021469116, "mix_mean": 0.5555468797683716, "mix_low": 0.0019083023071289062, "mix_high": 0.0029382705688476562, "read_cosine": 0.18118959665298462, "beta_mean": 5.2258666983107105e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008623137138783932, "mix_std": 0.275662899017334, "beta_std": 0.00010350385127821937, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01107160747051239, "evidence_weights": [-0.1847362071275711, -0.2667035460472107, 0.6687495112419128], "agreement_weight": 0.39253881573677063, "orthogonal_weight": -1.3353413343429565}, {"head": 2, "memory_to_active": 0.08551596105098724, "contribution_to_active": 0.4672667384147644, "mix_mean": 0.37348103523254395, "mix_low": 0.0019330978393554688, "mix_high": 0.00018596649169921875, "read_cosine": 0.6391310691833496, "beta_mean": 0.012658574618399143, "nonfinite": 0.0, "weighted_memory_to_active": 0.04017047584056854, "mix_std": 0.23049113154411316, "beta_std": 0.040334612131118774, "beta_low": 0.8250732421875, "beta_high": 0.0, "orthogonal_to_active": 0.022111035883426666, "evidence_weights": [-0.19001176953315735, -0.20423708856105804, 0.4246176779270172], "agreement_weight": 0.00854775682091713, "orthogonal_weight": -0.8283908367156982}, {"head": 3, "memory_to_active": 0.15800327062606812, "contribution_to_active": 0.5699632167816162, "mix_mean": 0.644401490688324, "mix_low": 0.022329330444335938, "mix_high": 0.032192230224609375, "read_cosine": 0.005276068113744259, "beta_mean": 0.0010155938798561692, "nonfinite": 0.0, "weighted_memory_to_active": 0.11789287626743317, "mix_std": 0.2957416772842407, "beta_std": 0.01212150976061821, "beta_low": 0.99310302734375, "beta_high": 0.0, "orthogonal_to_active": 0.10514596104621887, "evidence_weights": [-0.3217519223690033, -0.24905870854854584, 1.754620909690857], "agreement_weight": 5.028424263000488, "orthogonal_weight": -0.9832006692886353}, {"head": 4, "memory_to_active": 0.41540300846099854, "contribution_to_active": 0.26310983300209045, "mix_mean": 0.348885178565979, "mix_low": 0.004769325256347656, "mix_high": 4.57763671875e-05, "read_cosine": 0.8864104747772217, "beta_mean": 0.07964803278446198, "nonfinite": 0.0, "weighted_memory_to_active": 0.1098054051399231, "mix_std": 0.2221815288066864, "beta_std": 0.05659009888768196, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.1505061686038971, "evidence_weights": [-0.10019269585609436, -0.23383747041225433, 0.4730822443962097], "agreement_weight": 0.002235093619674444, "orthogonal_weight": 2.2173855304718018}, {"head": 5, "memory_to_active": 0.05365206301212311, "contribution_to_active": 0.6856504678726196, "mix_mean": 0.6381036043167114, "mix_low": 0.0024728775024414062, "mix_high": 0.010310173034667969, "read_cosine": 0.2939634919166565, "beta_mean": 0.0002753616136033088, "nonfinite": 0.0, "weighted_memory_to_active": 0.03649681806564331, "mix_std": 0.274812787771225, "beta_std": 0.003134472295641899, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.030355684459209442, "evidence_weights": [-0.2188800871372223, -0.3030702471733093, 0.8605279326438904], "agreement_weight": 0.16439883410930634, "orthogonal_weight": -0.9151933193206787}]}, {"layer": 14, "memory_to_active": 0.23489296436309814, "contribution_to_active": 0.4195905327796936, "mix_mean": 0.4259205162525177, "mix_low": 0.02644634246826172, "mix_high": 0.00606536865234375, "read_cosine": 0.6156916618347168, "beta_mean": 0.0637388676404953, "nonfinite": 0.0, "weighted_memory_to_active": 0.09991023689508438, "mix_std": 0.24681910872459412, "beta_std": 0.06220655143260956, "beta_low": 0.4874369502067566, "beta_high": 0.0, "orthogonal_to_active": 0.040508560836315155, "output_influence": 0.34028324484825134, "residual_influence": 0.13196823000907898, "state_active": 0.06004931777715683, "state_memory": 0.015709128230810165, "state_peak": 39.85256576538086, "growth_active": 1.112457036972046, "growth_memory": 1.2455319166183472, "heads": [{"head": 0, "memory_to_active": 0.3602939546108246, "contribution_to_active": 0.19082321226596832, "mix_mean": 0.19205275177955627, "mix_low": 0.021770477294921875, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8159714937210083, "beta_mean": 0.06801317632198334, "nonfinite": 0.0, "weighted_memory_to_active": 0.08308247476816177, "mix_std": 0.18371102213859558, "beta_std": 0.061828963458538055, "beta_low": 0.011474609375, "beta_high": 0.0, "orthogonal_to_active": 0.07386073470115662, "evidence_weights": [-0.12693138420581818, -0.03555220738053322, 0.45729851722717285], "agreement_weight": 0.02689022570848465, "orthogonal_weight": 1.5555315017700195}, {"head": 1, "memory_to_active": 0.48271864652633667, "contribution_to_active": 0.31036850810050964, "mix_mean": 0.22964845597743988, "mix_low": 0.1215658187866211, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8480493426322937, "beta_mean": 0.2594236731529236, "nonfinite": 0.0, "weighted_memory_to_active": 0.215395987033844, "mix_std": 0.25473159551620483, "beta_std": 0.2559890151023865, "beta_low": 0.09600830078125, "beta_high": 0.0, "orthogonal_to_active": 0.013750292360782623, "evidence_weights": [-0.01030649896711111, -0.20623023808002472, -0.1096200942993164], "agreement_weight": 0.054666198790073395, "orthogonal_weight": 0.1595304161310196}, {"head": 2, "memory_to_active": 0.12743937969207764, "contribution_to_active": 0.5789574384689331, "mix_mean": 0.585964560508728, "mix_low": 0.0022535324096679688, "mix_high": 0.0037488937377929688, "read_cosine": 0.38373494148254395, "beta_mean": 0.002487740246579051, "nonfinite": 0.0, "weighted_memory_to_active": 0.08188358694314957, "mix_std": 0.2653951644897461, "beta_std": 0.003153617028146982, "beta_low": 0.96612548828125, "beta_high": 0.0, "orthogonal_to_active": 0.06297845393419266, "evidence_weights": [-0.1465633064508438, -0.31788668036460876, 0.8914375901222229], "agreement_weight": 0.08255957067012787, "orthogonal_weight": -0.8628669381141663}, {"head": 3, "memory_to_active": 0.0538933202624321, "contribution_to_active": 0.6088684797286987, "mix_mean": 0.6732087135314941, "mix_low": 0.005042076110839844, "mix_high": 0.021440505981445312, "read_cosine": 0.03268211707472801, "beta_mean": 0.0064832535572350025, "nonfinite": 0.0, "weighted_memory_to_active": 0.03522711992263794, "mix_std": 0.2548825740814209, "beta_std": 0.00757976109161973, "beta_low": 0.8076171875, "beta_high": 0.0, "orthogonal_to_active": 0.024453546851873398, "evidence_weights": [-0.3558276891708374, -0.3155291676521301, 0.326905757188797], "agreement_weight": 0.07052302360534668, "orthogonal_weight": -0.8059463500976562}, {"head": 4, "memory_to_active": 0.15348760783672333, "contribution_to_active": 0.49842795729637146, "mix_mean": 0.502752423286438, "mix_low": 0.0042724609375, "mix_high": 0.009007453918457031, "read_cosine": 0.8607084155082703, "beta_mean": 0.04415268823504448, "nonfinite": 0.0, "weighted_memory_to_active": 0.0905669778585434, "mix_std": 0.2643217146396637, "beta_std": 0.033441733568906784, "beta_low": 0.07421875, "beta_high": 0.0, "orthogonal_to_active": 0.026752281934022903, "evidence_weights": [-0.21920432150363922, -0.24735027551651, 0.4263157844543457], "agreement_weight": 0.1278921216726303, "orthogonal_weight": -0.6034632921218872}, {"head": 5, "memory_to_active": 0.2315247654914856, "contribution_to_active": 0.33009758591651917, "mix_mean": 0.3718961179256439, "mix_low": 0.0037736892700195312, "mix_high": 0.0021848678588867188, "read_cosine": 0.7530038356781006, "beta_mean": 0.0018726944690570235, "nonfinite": 0.0, "weighted_memory_to_active": 0.09330524504184723, "mix_std": 0.2578725516796112, "beta_std": 0.011246195994317532, "beta_low": 0.96917724609375, "beta_high": 0.0, "orthogonal_to_active": 0.04125605523586273, "evidence_weights": [-0.17813754081726074, -0.1698714792728424, 0.27437731623649597], "agreement_weight": 0.07918986678123474, "orthogonal_weight": -0.7120223045349121}]}, {"layer": 15, "memory_to_active": 0.05984264984726906, "contribution_to_active": 0.4868963658809662, "mix_mean": 0.41541415452957153, "mix_low": 0.00846592616289854, "mix_high": 0.002574443817138672, "read_cosine": 0.6679094433784485, "beta_mean": 0.0399748720228672, "nonfinite": 0.0, "weighted_memory_to_active": 0.028976671397686005, "mix_std": 0.2849310040473938, "beta_std": 0.04480038583278656, "beta_low": 0.7036641836166382, "beta_high": 0.0, "orthogonal_to_active": 0.007173310033977032, "output_influence": 0.3067333400249481, "residual_influence": 0.17121738195419312, "state_active": 0.14157181978225708, "state_memory": 0.01025669276714325, "state_peak": 37.69382858276367, "growth_active": 1.2598310708999634, "growth_memory": 1.3888283967971802, "heads": [{"head": 0, "memory_to_active": 0.1123678982257843, "contribution_to_active": 0.4793274998664856, "mix_mean": 0.38478705286979675, "mix_low": 0.005877494812011719, "mix_high": 0.0054111480712890625, "read_cosine": 0.6591310501098633, "beta_mean": 0.04074728488922119, "nonfinite": 0.0, "weighted_memory_to_active": 0.05493165925145149, "mix_std": 0.2817230820655823, "beta_std": 0.03261672332882881, "beta_low": 0.0782470703125, "beta_high": 0.0, "orthogonal_to_active": 0.024886025115847588, "evidence_weights": [-0.1873241513967514, -0.1913456916809082, 0.4093188941478729], "agreement_weight": 0.0928824171423912, "orthogonal_weight": -0.7393132448196411}, {"head": 1, "memory_to_active": 0.13413944840431213, "contribution_to_active": 0.41575199365615845, "mix_mean": 0.42818599939346313, "mix_low": 0.0017461776733398438, "mix_high": 0.005166053771972656, "read_cosine": 0.9136813282966614, "beta_mean": 0.0007997402572073042, "nonfinite": 0.0, "weighted_memory_to_active": 0.055067382752895355, "mix_std": 0.2856177091598511, "beta_std": 0.00271388771943748, "beta_low": 0.9892578125, "beta_high": 0.0, "orthogonal_to_active": 0.0003602564102038741, "evidence_weights": [-0.1520245373249054, -0.16195429861545563, 0.19815737009048462], "agreement_weight": 0.12233006209135056, "orthogonal_weight": 0.012094017118215561}, {"head": 2, "memory_to_active": 0.004318643361330032, "contribution_to_active": 0.47754284739494324, "mix_mean": 0.3398168385028839, "mix_low": 0.016876220703125, "mix_high": 7.152557373046875e-05, "read_cosine": 0.8031095266342163, "beta_mean": 6.45848413114436e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021350537426769733, "mix_std": 0.2716006338596344, "beta_std": 0.00016923608200158924, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0005472538177855313, "evidence_weights": [-0.2054820954799652, -0.2689868211746216, 0.15478277206420898], "agreement_weight": 0.012570302002131939, "orthogonal_weight": 0.6382359862327576}, {"head": 3, "memory_to_active": 0.0016868669772520661, "contribution_to_active": 0.39717409014701843, "mix_mean": 0.39654043316841125, "mix_low": 0.005223274230957031, "mix_high": 0.0004253387451171875, "read_cosine": 0.5666604042053223, "beta_mean": 1.0331974408472888e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005982960574328899, "mix_std": 0.28539374470710754, "beta_std": 2.5631688913563266e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004653785435948521, "evidence_weights": [-0.25002413988113403, -0.2518802285194397, 0.20919816195964813], "agreement_weight": 0.03484804928302765, "orthogonal_weight": -0.8084176182746887}, {"head": 4, "memory_to_active": 0.08287067711353302, "contribution_to_active": 0.626876711845398, "mix_mean": 0.38184043765068054, "mix_low": 0.010239601135253906, "mix_high": 0.0009164810180664062, "read_cosine": 0.731043815612793, "beta_mean": 0.19649183750152588, "nonfinite": 0.0, "weighted_memory_to_active": 0.045742932707071304, "mix_std": 0.3071514070034027, "beta_std": 0.22090323269367218, "beta_low": 0.18109130859375, "beta_high": 0.0, "orthogonal_to_active": 0.007328154053539038, "evidence_weights": [-0.21447859704494476, -0.2550305426120758, -0.027771713212132454], "agreement_weight": 0.014262779615819454, "orthogonal_weight": 0.32374054193496704}, {"head": 5, "memory_to_active": 0.02367234230041504, "contribution_to_active": 0.5247049927711487, "mix_mean": 0.5613139867782593, "mix_low": 0.010832786560058594, "mix_high": 0.00345611572265625, "read_cosine": 0.3338305950164795, "beta_mean": 0.001735438359901309, "nonfinite": 0.0, "weighted_memory_to_active": 0.015384702943265438, "mix_std": 0.2780994176864624, "beta_std": 0.012373631820082664, "beta_low": 0.97344970703125, "beta_high": 0.0, "orthogonal_to_active": 0.00945279374718666, "evidence_weights": [-0.29163894057273865, -0.30795252323150635, 0.7341346144676208], "agreement_weight": 0.2801204323768616, "orthogonal_weight": 0.8024590015411377}]}]}, {"step": 47000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8931606710029882, "memory_to_active": 0.8071131706237793, "contribution_to_active": 0.5405220985412598, "mix_mean": 0.39927321672439575, "mix_low": 0.13976536691188812, "mix_high": 0.00976270530372858, "read_cosine": 0.35587218403816223, "beta_mean": 0.07397832721471786, "nonfinite": 0.0, "weighted_memory_to_active": 0.2000458836555481, "mix_std": 0.22041843831539154, "beta_std": 0.06504912674427032, "beta_low": 0.5434430837631226, "beta_high": 4.00543212890625e-05, "orthogonal_to_active": 0.056981295347213745, "samples": 4, "scored_tokens": 899, "normal_loss": 3.085998296737671, "normal_token_accuracy": 0.45606228709220886, "state_active": 0.06227407604455948, "state_memory": 0.01818646676838398, "state_peak": 116.64588928222656, "growth_active": 1.126602292060852, "growth_memory": 1.3413450717926025, "output_influence": 0.4098914563655853, "residual_influence": 0.13643379509449005, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1741528511047363, "no_memory_loss_delta": 0.6220309138298035, "scrambled_loss_delta": 0.1662936508655548, "memory_only_loss_delta": 3.3972830772399902, "no_evidence_loss_delta": 0.3681415915489197, "no_agreement_loss_delta": 0.03677626699209213, "no_orthogonal_loss_delta": 0.01651003584265709}, {"index": 94, "tokens": 256, "normal_loss": 3.2517318725585938, "no_memory_loss_delta": 0.5850961208343506, "scrambled_loss_delta": 0.34209612011909485, "memory_only_loss_delta": 3.144474506378174, "no_evidence_loss_delta": 0.2718256115913391, "no_agreement_loss_delta": 0.039624396711587906, "no_orthogonal_loss_delta": 0.027993718162178993}, {"index": 188, "tokens": 256, "normal_loss": 3.399515151977539, "no_memory_loss_delta": 0.7085492014884949, "scrambled_loss_delta": 0.15299756824970245, "memory_only_loss_delta": 3.402688503265381, "no_evidence_loss_delta": 0.3916487395763397, "no_agreement_loss_delta": 0.0585860051214695, "no_orthogonal_loss_delta": 0.037960708141326904}, {"index": 283, "tokens": 131, "normal_loss": 1.9771742820739746, "no_memory_loss_delta": 0.4502728283405304, "scrambled_loss_delta": 0.2285521924495697, "memory_only_loss_delta": 2.865368604660034, "no_evidence_loss_delta": 0.23023319244384766, "no_agreement_loss_delta": 0.02090703509747982, "no_orthogonal_loss_delta": 0.00857196282595396}], "agreement_weight_abs_mean": 0.19519175589084625, "agreement_weight_abs_max": 5.118095397949219, "agreement_weight_mean": 0.17138196527957916, "orthogonal_weight_abs_mean": 0.8096404671669006, "orthogonal_weight_abs_max": 8.207582473754883, "orthogonal_weight_mean": -0.1258961260318756, "no_memory_loss": 3.697120428085327, "no_memory_loss_delta": 0.611122190952301, "no_memory_token_accuracy_gain": 0.09899888932704926, "no_memory_token_win_fraction": 0.6863181591033936, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 45.72585323583611, "scrambled_loss": 3.3076393604278564, "scrambled_loss_delta": 0.22164125740528107, "scrambled_token_accuracy_gain": 0.03893214836716652, "scrambled_token_win_fraction": 0.6262513995170593, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.8797261665824, "memory_only_loss": 6.335320472717285, "memory_only_loss_delta": 3.2493233680725098, "memory_only_token_accuracy_gain": 0.2947719693183899, "memory_only_token_win_fraction": 0.9054505228996277, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.1199547423281, "no_evidence_loss": 3.413311004638672, "no_evidence_loss_delta": 0.3273128271102905, "no_evidence_token_accuracy_gain": 0.060066740959882736, "no_evidence_token_win_fraction": 0.6262513995170593, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 27.914179623776402, "no_agreement_loss": 3.127483606338501, "no_agreement_loss_delta": 0.041485439985990524, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.517241358757019, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 4.063669641707649, "no_orthogonal_loss": 3.1107301712036133, "no_orthogonal_loss_delta": 0.02473173476755619, "no_orthogonal_token_accuracy_gain": 0.015572858974337578, "no_orthogonal_token_win_fraction": 0.5839822292327881, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 2.442841113358273, "short_no_memory_loss_delta": 0.5143460035324097, "long_context_memory_gain": 0.09677618741989136, "normal_context_loss_gain": 0.13951484858989716, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.447389602661133, "contribution_to_active": 1.1224199533462524, "mix_mean": 0.10438378900289536, "mix_low": 0.5451995730400085, "mix_high": 0.005167166702449322, "read_cosine": 0.4325917363166809, "beta_mean": 0.23093120753765106, "nonfinite": 0.0, "weighted_memory_to_active": 1.167473316192627, "mix_std": 0.08840546011924744, "beta_std": 0.15246739983558655, "beta_low": 0.0387674979865551, "beta_high": 0.0, "orthogonal_to_active": 0.053483784198760986, "output_influence": 0.21107277274131775, "residual_influence": 0.25252506136894226, "state_active": 0.06854191422462463, "state_memory": 0.04473011568188667, "state_peak": 40.08028030395508, "growth_active": 1.2820425033569336, "growth_memory": 1.2680842876434326, "heads": [{"head": 0, "memory_to_active": 2.7328600883483887, "contribution_to_active": 0.5262340307235718, "mix_mean": 0.01974937692284584, "mix_low": 0.6191482543945312, "mix_high": 0.0, "read_cosine": 0.2473650872707367, "beta_mean": 0.20966258645057678, "nonfinite": 0.0, "weighted_memory_to_active": 0.641269326210022, "mix_std": 0.060655996203422546, "beta_std": 0.10381730645895004, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.057902924716472626, "evidence_weights": [-0.012341600842773914, -0.05600585415959358, -0.08993065357208252], "agreement_weight": -0.1687917858362198, "orthogonal_weight": -0.10071834921836853}, {"head": 1, "memory_to_active": 6.180454254150391, "contribution_to_active": 2.574146032333374, "mix_mean": 0.016897406429052353, "mix_low": 0.6757631301879883, "mix_high": 0.0, "read_cosine": 0.4667738974094391, "beta_mean": 0.42270204424858093, "nonfinite": 0.0, "weighted_memory_to_active": 2.8761539459228516, "mix_std": 0.0535355806350708, "beta_std": 0.2452150285243988, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.027878692373633385, "evidence_weights": [0.02504616416990757, -0.09831154346466064, 0.4991113543510437], "agreement_weight": 0.029090438038110733, "orthogonal_weight": -0.020304322242736816}, {"head": 2, "memory_to_active": 2.6590020656585693, "contribution_to_active": 0.4377036392688751, "mix_mean": 0.012831042520701885, "mix_low": 0.6995763778686523, "mix_high": 0.0, "read_cosine": 0.3027978539466858, "beta_mean": 0.30160290002822876, "nonfinite": 0.0, "weighted_memory_to_active": 0.5144175291061401, "mix_std": 0.02945764549076557, "beta_std": 0.20422160625457764, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04266680032014847, "evidence_weights": [0.06491851061582565, 0.06228084862232208, 0.08721234649419785], "agreement_weight": 0.009201405569911003, "orthogonal_weight": -0.19431427121162415}, {"head": 3, "memory_to_active": 6.94368314743042, "contribution_to_active": 2.6628992557525635, "mix_mean": 0.033881623297929764, "mix_low": 0.47637271881103516, "mix_high": 0.0, "read_cosine": 0.29777657985687256, "beta_mean": 0.3339194059371948, "nonfinite": 0.0, "weighted_memory_to_active": 2.864612579345703, "mix_std": 0.07862810790538788, "beta_std": 0.25919535756111145, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.17037776112556458, "evidence_weights": [-0.04258901998400688, -0.04644261300563812, -0.1512763947248459], "agreement_weight": -0.004005913157016039, "orthogonal_weight": -0.0693657249212265}, {"head": 4, "memory_to_active": 2.0618350505828857, "contribution_to_active": 0.07706879079341888, "mix_mean": 0.008099343627691269, "mix_low": 0.7862768173217773, "mix_high": 0.0, "read_cosine": 0.33688703179359436, "beta_mean": 0.09218569844961166, "nonfinite": 0.0, "weighted_memory_to_active": 0.054105594754219055, "mix_std": 0.018529240041971207, "beta_std": 0.08219528198242188, "beta_low": 0.013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.005843789782375097, "evidence_weights": [0.14076678454875946, 0.09987952560186386, -0.29176992177963257], "agreement_weight": -0.13443003594875336, "orthogonal_weight": 0.09582101553678513}, {"head": 5, "memory_to_active": 0.10650119930505753, "contribution_to_active": 0.45646753907203674, "mix_mean": 0.5348439812660217, "mix_low": 0.014060020446777344, "mix_high": 0.03100299835205078, "read_cosine": 0.943949818611145, "beta_mean": 0.025514550507068634, "nonfinite": 0.0, "weighted_memory_to_active": 0.054281026124954224, "mix_std": 0.2896261513233185, "beta_std": 0.020159877836704254, "beta_low": 0.21917724609375, "beta_high": 0.0, "orthogonal_to_active": 0.016232725232839584, "evidence_weights": [-0.1901048868894577, -0.22311747074127197, 0.3679898679256439], "agreement_weight": 0.29322952032089233, "orthogonal_weight": -0.6915068626403809}]}, {"layer": 1, "memory_to_active": 2.280086040496826, "contribution_to_active": 0.4513612389564514, "mix_mean": 0.20413115620613098, "mix_low": 0.3596075475215912, "mix_high": 0.0011377334594726562, "read_cosine": 0.2950473129749298, "beta_mean": 0.17397239804267883, "nonfinite": 0.0, "weighted_memory_to_active": 0.2952461242675781, "mix_std": 0.11849617213010788, "beta_std": 0.12806907296180725, "beta_low": 0.3369242548942566, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.04446978121995926, "output_influence": 0.22658973932266235, "residual_influence": 0.058538325130939484, "state_active": 0.031305134296417236, "state_memory": 0.03524699807167053, "state_peak": 36.192771911621094, "growth_active": 1.257065773010254, "growth_memory": 1.5036875009536743, "heads": [{"head": 0, "memory_to_active": 0.049507319927215576, "contribution_to_active": 0.4698507785797119, "mix_mean": 0.49149107933044434, "mix_low": 0.008059501647949219, "mix_high": 0.0018329620361328125, "read_cosine": -0.003592479508370161, "beta_mean": 0.00035550270695239305, "nonfinite": 0.0, "weighted_memory_to_active": 0.027286622673273087, "mix_std": 0.2853352725505829, "beta_std": 0.00047427351819351315, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.02600821852684021, "evidence_weights": [-0.19956481456756592, -0.26141488552093506, 0.3475082516670227], "agreement_weight": 0.19873493909835815, "orthogonal_weight": -1.0119942426681519}, {"head": 1, "memory_to_active": 2.8503715991973877, "contribution_to_active": 0.2422386109828949, "mix_mean": 0.025028493255376816, "mix_low": 0.4369363784790039, "mix_high": 2.86102294921875e-06, "read_cosine": 0.5133504271507263, "beta_mean": 0.2720663249492645, "nonfinite": 0.0, "weighted_memory_to_active": 0.27066174149513245, "mix_std": 0.0480593703687191, "beta_std": 0.17048434913158417, "beta_low": 0.0008544921875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.018405338749289513, "evidence_weights": [0.014406038448214531, -0.05603562295436859, 0.02109643816947937], "agreement_weight": -0.005978833884000778, "orthogonal_weight": 0.16251525282859802}, {"head": 2, "memory_to_active": 0.08155079931020737, "contribution_to_active": 0.5365118980407715, "mix_mean": 0.6439789533615112, "mix_low": 0.0009365081787109375, "mix_high": 0.004985809326171875, "read_cosine": 0.5196470618247986, "beta_mean": 0.003878359915688634, "nonfinite": 0.0, "weighted_memory_to_active": 0.059650737792253494, "mix_std": 0.23897014558315277, "beta_std": 0.006104320287704468, "beta_low": 0.91796875, "beta_high": 0.0, "orthogonal_to_active": 0.034442149102687836, "evidence_weights": [-0.2868150472640991, -0.35121291875839233, 0.667585551738739], "agreement_weight": 0.030351391062140465, "orthogonal_weight": -0.7066861987113953}, {"head": 3, "memory_to_active": 3.0007729530334473, "contribution_to_active": 0.15975484251976013, "mix_mean": 0.022133301943540573, "mix_low": 0.5916881561279297, "mix_high": 0.0, "read_cosine": 0.20305722951889038, "beta_mean": 0.31393349170684814, "nonfinite": 0.0, "weighted_memory_to_active": 0.22609534859657288, "mix_std": 0.06112182140350342, "beta_std": 0.25392234325408936, "beta_low": 0.053466796875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.07695244252681732, "evidence_weights": [0.03756913170218468, -0.014357750304043293, 0.07259984314441681], "agreement_weight": -0.02468397095799446, "orthogonal_weight": -0.43777889013290405}, {"head": 4, "memory_to_active": 3.67159104347229, "contribution_to_active": 1.2279950380325317, "mix_mean": 0.03512275218963623, "mix_low": 0.29114627838134766, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3159666955471039, "beta_mean": 0.17535915970802307, "nonfinite": 0.0, "weighted_memory_to_active": 1.1335034370422363, "mix_std": 0.05729370564222336, "beta_std": 0.12394643574953079, "beta_low": 0.021240234375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.09782885015010834, "evidence_weights": [-0.03830287232995033, -0.0035185408778488636, 0.04392918571829796], "agreement_weight": -0.01532655954360962, "orthogonal_weight": 0.1116114929318428}, {"head": 5, "memory_to_active": 4.026721954345703, "contribution_to_active": 0.071816086769104, "mix_mean": 0.007032328750938177, "mix_low": 0.8288784027099609, "mix_high": 3.814697265625e-06, "read_cosine": 0.22185499966144562, "beta_mean": 0.27824148535728455, "nonfinite": 0.0, "weighted_memory_to_active": 0.05427871644496918, "mix_std": 0.02019667997956276, "beta_std": 0.21348266303539276, "beta_low": 0.02801513671875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.01318166684359312, "evidence_weights": [0.16224360466003418, 0.08228293806314468, 0.16091927886009216], "agreement_weight": -0.12931491434574127, "orthogonal_weight": 0.23370476067066193}]}, {"layer": 2, "memory_to_active": 1.023638129234314, "contribution_to_active": 0.3512516915798187, "mix_mean": 0.38573092222213745, "mix_low": 0.2552735209465027, "mix_high": 0.0036187171936035156, "read_cosine": 0.3373059034347534, "beta_mean": 0.0978781208395958, "nonfinite": 0.0, "weighted_memory_to_active": 0.08042864501476288, "mix_std": 0.18988868594169617, "beta_std": 0.07378031313419342, "beta_low": 0.5592550039291382, "beta_high": 0.0003458659048192203, "orthogonal_to_active": 0.01833461970090866, "output_influence": 0.37149709463119507, "residual_influence": 0.0941513255238533, "state_active": 0.03681366145610809, "state_memory": 0.01615379936993122, "state_peak": 29.192697525024414, "growth_active": 1.0104883909225464, "growth_memory": 1.5164824724197388, "heads": [{"head": 0, "memory_to_active": 0.01106453686952591, "contribution_to_active": 0.33215391635894775, "mix_mean": 0.6022936105728149, "mix_low": 0.0008716583251953125, "mix_high": 0.006827354431152344, "read_cosine": 0.16558176279067993, "beta_mean": 9.749787568580359e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0068204365670681, "mix_std": 0.2769829332828522, "beta_std": 0.0001363546762149781, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006217312533408403, "evidence_weights": [-0.3541327118873596, -0.21075420081615448, 0.09473998099565506], "agreement_weight": 0.14409339427947998, "orthogonal_weight": -0.9584436416625977}, {"head": 1, "memory_to_active": 2.8558292388916016, "contribution_to_active": 0.037286534905433655, "mix_mean": 0.0040727355517446995, "mix_low": 0.9216756820678711, "mix_high": 0.0, "read_cosine": 0.16697224974632263, "beta_mean": 0.2807287871837616, "nonfinite": 0.0, "weighted_memory_to_active": 0.02649051323533058, "mix_std": 0.009943685494363308, "beta_std": 0.17038384079933167, "beta_low": 0.0006103515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.010073201730847359, "evidence_weights": [0.1431238055229187, 0.13095900416374207, 0.49058231711387634], "agreement_weight": 0.03855212777853012, "orthogonal_weight": 0.39585718512535095}, {"head": 2, "memory_to_active": 0.00968471821397543, "contribution_to_active": 0.4188883304595947, "mix_mean": 0.5167545080184937, "mix_low": 0.0013818740844726562, "mix_high": 0.0022916793823242188, "read_cosine": 0.3522456884384155, "beta_mean": 2.551706347730942e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0036822448018938303, "mix_std": 0.2787141501903534, "beta_std": 4.32724627899006e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000335794611601159, "evidence_weights": [-0.24449972808361053, -0.28366345167160034, 0.5318101644515991], "agreement_weight": 0.009714098647236824, "orthogonal_weight": -0.10564108192920685}, {"head": 3, "memory_to_active": 0.15034256875514984, "contribution_to_active": 0.5308884382247925, "mix_mean": 0.5366083383560181, "mix_low": 0.00033855438232421875, "mix_high": 0.0058803558349609375, "read_cosine": 0.706587016582489, "beta_mean": 0.028757844120264053, "nonfinite": 0.0, "weighted_memory_to_active": 0.08936379104852676, "mix_std": 0.261682391166687, "beta_std": 0.03506719321012497, "beta_low": 0.3304443359375, "beta_high": 0.0, "orthogonal_to_active": 0.05357059836387634, "evidence_weights": [-0.1922149360179901, -0.3014164865016937, 0.8356534242630005], "agreement_weight": 0.057207316160202026, "orthogonal_weight": -0.883880078792572}, {"head": 4, "memory_to_active": 3.105029582977295, "contribution_to_active": 0.2506100535392761, "mix_mean": 0.01856386475265026, "mix_low": 0.6065464019775391, "mix_high": 0.0, "read_cosine": 0.21235248446464539, "beta_mean": 0.2776240110397339, "nonfinite": 0.0, "weighted_memory_to_active": 0.3499041199684143, "mix_std": 0.04814501851797104, "beta_std": 0.23699650168418884, "beta_low": 0.02447509765625, "beta_high": 0.00201416015625, "orthogonal_to_active": 0.03951634466648102, "evidence_weights": [0.02328636310994625, 0.008964655920863152, 0.009904557839035988], "agreement_weight": 0.017198937013745308, "orthogonal_weight": -0.13731639087200165}, {"head": 5, "memory_to_active": 0.009877788834273815, "contribution_to_active": 0.5376826524734497, "mix_mean": 0.6360923051834106, "mix_low": 0.0008268356323242188, "mix_high": 0.006712913513183594, "read_cosine": 0.420096218585968, "beta_mean": 3.506232678773813e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006310753524303436, "mix_std": 0.2638639211654663, "beta_std": 5.4694719437975436e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00029446452390402555, "evidence_weights": [-0.279651015996933, -0.2925170660018921, 0.9758177399635315], "agreement_weight": 0.14331337809562683, "orthogonal_weight": -0.0684603825211525}]}, {"layer": 3, "memory_to_active": 0.5022833943367004, "contribution_to_active": 0.4424322843551636, "mix_mean": 0.45498573780059814, "mix_low": 0.06684081256389618, "mix_high": 0.00845845602452755, "read_cosine": 0.21666719019412994, "beta_mean": 0.027400504797697067, "nonfinite": 0.0, "weighted_memory_to_active": 0.13575302064418793, "mix_std": 0.2598704695701599, "beta_std": 0.024955609813332558, "beta_low": 0.7302449941635132, "beta_high": 0.0, "orthogonal_to_active": 0.05839882045984268, "output_influence": 0.39084193110466003, "residual_influence": 0.08287553489208221, "state_active": 0.02493342012166977, "state_memory": 0.0074388314969837666, "state_peak": 23.704191207885742, "growth_active": 1.0657492876052856, "growth_memory": 1.8047429323196411, "heads": [{"head": 0, "memory_to_active": 0.45319947600364685, "contribution_to_active": 0.5550287961959839, "mix_mean": 0.5532757639884949, "mix_low": 0.0031795501708984375, "mix_high": 0.019514083862304688, "read_cosine": 0.2763756811618805, "beta_mean": 0.013170372694730759, "nonfinite": 0.0, "weighted_memory_to_active": 0.2716478705406189, "mix_std": 0.3127967119216919, "beta_std": 0.01692768558859825, "beta_low": 0.59075927734375, "beta_high": 0.0, "orthogonal_to_active": 0.18618172407150269, "evidence_weights": [-0.20911015570163727, -0.2425050586462021, 0.8624957203865051], "agreement_weight": 0.10831733047962189, "orthogonal_weight": -0.7365139722824097}, {"head": 1, "memory_to_active": 0.015425973571836948, "contribution_to_active": 0.40703943371772766, "mix_mean": 0.5252498984336853, "mix_low": 0.007811546325683594, "mix_high": 0.007824897766113281, "read_cosine": 0.18250055611133575, "beta_mean": 8.96426645340398e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00830994825810194, "mix_std": 0.30534061789512634, "beta_std": 0.00017647385539021343, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006359035149216652, "evidence_weights": [-0.24892254173755646, -0.20445068180561066, 0.6143837571144104], "agreement_weight": 0.4288010895252228, "orthogonal_weight": -0.8407252430915833}, {"head": 2, "memory_to_active": 2.2497847080230713, "contribution_to_active": 0.2835446298122406, "mix_mean": 0.042451161891222, "mix_low": 0.3640909194946289, "mix_high": 0.0, "read_cosine": 0.20693084597587585, "beta_mean": 0.1435263454914093, "nonfinite": 0.0, "weighted_memory_to_active": 0.32734164595603943, "mix_std": 0.09146969020366669, "beta_std": 0.1217861920595169, "beta_low": 0.0164794921875, "beta_high": 0.0, "orthogonal_to_active": 0.006463364232331514, "evidence_weights": [-0.08213728666305542, 0.011916302144527435, -0.003973442129790783], "agreement_weight": -0.016042741015553474, "orthogonal_weight": -0.024884195998311043}, {"head": 3, "memory_to_active": 0.023443302139639854, "contribution_to_active": 0.5058704018592834, "mix_mean": 0.5751661658287048, "mix_low": 0.007107734680175781, "mix_high": 0.008625030517578125, "read_cosine": 0.2808493971824646, "beta_mean": 9.856140241026878e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015767384320497513, "mix_std": 0.295742928981781, "beta_std": 0.00025434468989260495, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004232760984450579, "evidence_weights": [-0.23472587764263153, -0.22651790082454681, 2.132434368133545], "agreement_weight": 1.832018494606018, "orthogonal_weight": -0.3443591296672821}, {"head": 4, "memory_to_active": 0.016559548676013947, "contribution_to_active": 0.2733578681945801, "mix_mean": 0.3494049906730652, "mix_low": 0.01579570770263672, "mix_high": 0.000553131103515625, "read_cosine": 0.18647530674934387, "beta_mean": 0.00011496413208078593, "nonfinite": 0.0, "weighted_memory_to_active": 0.008459691889584064, "mix_std": 0.27939897775650024, "beta_std": 0.00016583460092078894, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008077885955572128, "evidence_weights": [-0.14939068257808685, -0.17076601088047028, 0.1281164586544037], "agreement_weight": -0.017490876838564873, "orthogonal_weight": -1.0135709047317505}, {"head": 5, "memory_to_active": 0.25528720021247864, "contribution_to_active": 0.6297524571418762, "mix_mean": 0.6843662858009338, "mix_low": 0.00305938720703125, "mix_high": 0.014233589172363281, "read_cosine": 0.16687142848968506, "beta_mean": 0.0074031469412148, "nonfinite": 0.0, "weighted_memory_to_active": 0.18299153447151184, "mix_std": 0.27447375655174255, "beta_std": 0.010423118248581886, "beta_low": 0.7742919921875, "beta_high": 0.0, "orthogonal_to_active": 0.13907814025878906, "evidence_weights": [-0.2556166350841522, -0.30896851420402527, 0.4680389165878296], "agreement_weight": 0.13851693272590637, "orthogonal_weight": -0.7982467412948608}]}, {"layer": 4, "memory_to_active": 1.040055513381958, "contribution_to_active": 0.4586436450481415, "mix_mean": 0.35751786828041077, "mix_low": 0.27639883756637573, "mix_high": 0.009068489074707031, "read_cosine": 0.3308945894241333, "beta_mean": 0.11766479164361954, "nonfinite": 0.0, "weighted_memory_to_active": 0.13061925768852234, "mix_std": 0.15715092420578003, "beta_std": 0.10665829479694366, "beta_low": 0.3777974545955658, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.048273541033267975, "output_influence": 0.43719589710235596, "residual_influence": 0.12425658851861954, "state_active": 0.040778301656246185, "state_memory": 0.01838114857673645, "state_peak": 39.057064056396484, "growth_active": 1.277435064315796, "growth_memory": 1.5333291292190552, "heads": [{"head": 0, "memory_to_active": 0.21007362008094788, "contribution_to_active": 0.7359527945518494, "mix_mean": 0.6702120900154114, "mix_low": 0.014112472534179688, "mix_high": 0.024575233459472656, "read_cosine": 0.49664515256881714, "beta_mean": 0.27088987827301025, "nonfinite": 0.0, "weighted_memory_to_active": 0.16567310690879822, "mix_std": 0.34487709403038025, "beta_std": 0.2764100432395935, "beta_low": 0.22674560546875, "beta_high": 0.0, "orthogonal_to_active": 0.06550955772399902, "evidence_weights": [-0.26433953642845154, -0.36459726095199585, 0.11020289361476898], "agreement_weight": 0.0071927634999156, "orthogonal_weight": 0.5175566673278809}, {"head": 1, "memory_to_active": 0.0557929128408432, "contribution_to_active": 0.6270087361335754, "mix_mean": 0.6640387177467346, "mix_low": 0.0001125335693359375, "mix_high": 0.012176513671875, "read_cosine": 0.6442883014678955, "beta_mean": 0.0008463147096335888, "nonfinite": 0.0, "weighted_memory_to_active": 0.03960179537534714, "mix_std": 0.24636702239513397, "beta_std": 0.005220350809395313, "beta_low": 0.9852294921875, "beta_high": 0.0, "orthogonal_to_active": 0.02210742048919201, "evidence_weights": [-0.23691140115261078, -0.3496857285499573, 0.6773620843887329], "agreement_weight": 0.23929718136787415, "orthogonal_weight": -0.8000356554985046}, {"head": 2, "memory_to_active": 1.8644604682922363, "contribution_to_active": 0.25423988699913025, "mix_mean": 0.02348152920603752, "mix_low": 0.5427350997924805, "mix_high": 2.09808349609375e-05, "read_cosine": 0.2376200258731842, "beta_mean": 0.07759194076061249, "nonfinite": 0.0, "weighted_memory_to_active": 0.2071351706981659, "mix_std": 0.04892761632800102, "beta_std": 0.07182534039020538, "beta_low": 0.0352783203125, "beta_high": 0.0, "orthogonal_to_active": 0.031792838126420975, "evidence_weights": [0.06010987237095833, 0.01916252076625824, 0.07062529027462006], "agreement_weight": 0.13372492790222168, "orthogonal_weight": -0.22728517651557922}, {"head": 3, "memory_to_active": 1.719048023223877, "contribution_to_active": 0.11257342249155045, "mix_mean": 0.029769983142614365, "mix_low": 0.44168758392333984, "mix_high": 0.0, "read_cosine": 0.2351057380437851, "beta_mean": 0.1344480663537979, "nonfinite": 0.0, "weighted_memory_to_active": 0.1582319438457489, "mix_std": 0.05455648899078369, "beta_std": 0.12115921825170517, "beta_low": 0.01953125, "beta_high": 0.0, "orthogonal_to_active": 0.0027026007883250713, "evidence_weights": [-0.026978211477398872, 0.057764481753110886, -0.22883819043636322], "agreement_weight": 0.07913145422935486, "orthogonal_weight": -0.025314047932624817}, {"head": 4, "memory_to_active": 0.014170380309224129, "contribution_to_active": 0.6404753923416138, "mix_mean": 0.7440257668495178, "mix_low": 9.5367431640625e-05, "mix_high": 0.017629623413085938, "read_cosine": 0.14675751328468323, "beta_mean": 2.7390316972741857e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010658529587090015, "mix_std": 0.21687589585781097, "beta_std": 7.117479981388897e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00430607795715332, "evidence_weights": [-0.3392755091190338, -0.3311624526977539, 0.5937033891677856], "agreement_weight": 0.3457018733024597, "orthogonal_weight": -0.42908626794815063}, {"head": 5, "memory_to_active": 2.3767871856689453, "contribution_to_active": 0.3816114366054535, "mix_mean": 0.013579205609858036, "mix_low": 0.6596498489379883, "mix_high": 8.58306884765625e-06, "read_cosine": 0.22495082020759583, "beta_mean": 0.2221851348876953, "nonfinite": 0.0, "weighted_memory_to_active": 0.20241490006446838, "mix_std": 0.03130137175321579, "beta_std": 0.16526362299919128, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1632227599620819, "evidence_weights": [0.048317138105630875, -0.02926304191350937, -0.31524816155433655], "agreement_weight": -0.0722184032201767, "orthogonal_weight": 0.9221584796905518}]}, {"layer": 5, "memory_to_active": 0.4295724630355835, "contribution_to_active": 0.5167877674102783, "mix_mean": 0.48400047421455383, "mix_low": 0.05894756317138672, "mix_high": 0.015416941605508327, "read_cosine": 0.08129043877124786, "beta_mean": 0.058889247477054596, "nonfinite": 0.0, "weighted_memory_to_active": 0.1430026888847351, "mix_std": 0.25541049242019653, "beta_std": 0.07508037984371185, "beta_low": 0.6358236074447632, "beta_high": 0.0, "orthogonal_to_active": 0.055424608290195465, "output_influence": 0.5141621828079224, "residual_influence": 0.14439286291599274, "state_active": 0.04553261399269104, "state_memory": 0.011468233540654182, "state_peak": 63.51133346557617, "growth_active": 1.141228437423706, "growth_memory": 1.2426717281341553, "heads": [{"head": 0, "memory_to_active": 0.4010942578315735, "contribution_to_active": 0.5036382079124451, "mix_mean": 0.36057794094085693, "mix_low": 0.021829605102539062, "mix_high": 0.001338958740234375, "read_cosine": -0.08621929585933685, "beta_mean": 0.029532426968216896, "nonfinite": 0.0, "weighted_memory_to_active": 0.1222238540649414, "mix_std": 0.27196747064590454, "beta_std": 0.04831881821155548, "beta_low": 0.38885498046875, "beta_high": 0.0, "orthogonal_to_active": 0.04328799247741699, "evidence_weights": [-0.12932436168193817, -0.1585567593574524, -0.04913892596960068], "agreement_weight": -0.025387180969119072, "orthogonal_weight": -0.35852742195129395}, {"head": 1, "memory_to_active": 0.014515608549118042, "contribution_to_active": 0.5433741807937622, "mix_mean": 0.6665728092193604, "mix_low": 0.0015134811401367188, "mix_high": 0.026477813720703125, "read_cosine": 0.2191070318222046, "beta_mean": 3.582346107577905e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008228223770856857, "mix_std": 0.28165555000305176, "beta_std": 8.647848881082609e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007690759375691414, "evidence_weights": [-0.24624018371105194, -0.2826557755470276, 0.8909419775009155], "agreement_weight": 0.6253947615623474, "orthogonal_weight": -0.9734771847724915}, {"head": 2, "memory_to_active": 0.009481292217969894, "contribution_to_active": 0.5654028654098511, "mix_mean": 0.6507364511489868, "mix_low": 0.002277374267578125, "mix_high": 0.019794464111328125, "read_cosine": -0.5008269548416138, "beta_mean": 6.443892925744876e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006191426422446966, "mix_std": 0.2699224054813385, "beta_std": 0.00011035595525754616, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0022124010138213634, "evidence_weights": [-0.25945207476615906, -0.34340545535087585, 0.47563350200653076], "agreement_weight": 0.006464722566306591, "orthogonal_weight": -0.4380912780761719}, {"head": 3, "memory_to_active": 1.6040866374969482, "contribution_to_active": 0.22284427285194397, "mix_mean": 0.10384128987789154, "mix_low": 0.13454914093017578, "mix_high": 4.76837158203125e-06, "read_cosine": 0.15461879968643188, "beta_mean": 0.12504565715789795, "nonfinite": 0.0, "weighted_memory_to_active": 0.35401514172554016, "mix_std": 0.13655222952365875, "beta_std": 0.17512892186641693, "beta_low": 0.1551513671875, "beta_high": 0.0, "orthogonal_to_active": 0.13754922151565552, "evidence_weights": [-0.06874880194664001, -0.09429313987493515, 0.01048565749078989], "agreement_weight": 0.047182708978652954, "orthogonal_weight": -0.47614821791648865}, {"head": 4, "memory_to_active": 0.4367387890815735, "contribution_to_active": 0.5908568501472473, "mix_mean": 0.33195701241493225, "mix_low": 0.19106388092041016, "mix_high": 0.006108283996582031, "read_cosine": 0.550836443901062, "beta_mean": 0.19830402731895447, "nonfinite": 0.0, "weighted_memory_to_active": 0.2923964858055115, "mix_std": 0.3424283266067505, "beta_std": 0.2253476083278656, "beta_low": 0.27642822265625, "beta_high": 0.0, "orthogonal_to_active": 0.09450291097164154, "evidence_weights": [-0.1184469535946846, -0.27765828371047974, -0.056349847465753555], "agreement_weight": 0.043738264590501785, "orthogonal_weight": 0.5010350942611694}, {"head": 5, "memory_to_active": 0.11151808500289917, "contribution_to_active": 0.6746097803115845, "mix_mean": 0.790317177772522, "mix_low": 0.0024518966674804688, "mix_high": 0.03877735137939453, "read_cosine": 0.1502266228199005, "beta_mean": 0.0003530890098772943, "nonfinite": 0.0, "weighted_memory_to_active": 0.07496094703674316, "mix_std": 0.22993698716163635, "beta_std": 0.0014901063404977322, "beta_low": 0.9945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.047304365783929825, "evidence_weights": [-0.31465455889701843, -0.33019116520881653, 1.9711432456970215], "agreement_weight": 0.4459685981273651, "orthogonal_weight": -0.6646889448165894}]}, {"layer": 6, "memory_to_active": 0.32565823197364807, "contribution_to_active": 0.6148043870925903, "mix_mean": 0.5789555907249451, "mix_low": 0.06474272906780243, "mix_high": 0.03285280987620354, "read_cosine": 0.28327691555023193, "beta_mean": 0.06797710806131363, "nonfinite": 0.0, "weighted_memory_to_active": 0.1541672945022583, "mix_std": 0.2350149154663086, "beta_std": 0.07662401348352432, "beta_low": 0.5170084834098816, "beta_high": 0.0, "orthogonal_to_active": 0.041715674102306366, "output_influence": 0.5408422946929932, "residual_influence": 0.17381663620471954, "state_active": 0.0465351901948452, "state_memory": 0.00832161121070385, "state_peak": 86.43504333496094, "growth_active": 1.1489921808242798, "growth_memory": 1.1991826295852661, "heads": [{"head": 0, "memory_to_active": 0.13509923219680786, "contribution_to_active": 0.7076587080955505, "mix_mean": 0.7950383424758911, "mix_low": 0.0001468658447265625, "mix_high": 0.033110618591308594, "read_cosine": 0.09261371195316315, "beta_mean": 0.018336830660700798, "nonfinite": 0.0, "weighted_memory_to_active": 0.10917361825704575, "mix_std": 0.2035793513059616, "beta_std": 0.02461472526192665, "beta_low": 0.49658203125, "beta_high": 0.0, "orthogonal_to_active": 0.08829335868358612, "evidence_weights": [-0.3169138431549072, -0.341063916683197, 1.1185736656188965], "agreement_weight": 0.37114065885543823, "orthogonal_weight": -0.8577967286109924}, {"head": 1, "memory_to_active": 0.004605292342603207, "contribution_to_active": 0.5263594388961792, "mix_mean": 0.5670270919799805, "mix_low": 0.0011949539184570312, "mix_high": 0.012425422668457031, "read_cosine": 0.5085554122924805, "beta_mean": 9.752719051903114e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021392006892710924, "mix_std": 0.2856943607330322, "beta_std": 1.6996607882902026e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011441786773502827, "evidence_weights": [-0.19954058527946472, -0.2988896369934082, 1.1088316440582275], "agreement_weight": 0.06540147960186005, "orthogonal_weight": -0.7307806611061096}, {"head": 2, "memory_to_active": 0.9665018916130066, "contribution_to_active": 0.08729100972414017, "mix_mean": 0.032356567680835724, "mix_low": 0.3799915313720703, "mix_high": 0.0, "read_cosine": 0.23846521973609924, "beta_mean": 0.03810136020183563, "nonfinite": 0.0, "weighted_memory_to_active": 0.06465789675712585, "mix_std": 0.055402956902980804, "beta_std": 0.03385552763938904, "beta_low": 0.0889892578125, "beta_high": 0.0, "orthogonal_to_active": 0.0030471039935946465, "evidence_weights": [0.08736053854227066, -0.02518949657678604, 0.2918663024902344], "agreement_weight": 0.14568352699279785, "orthogonal_weight": -0.05981793254613876}, {"head": 3, "memory_to_active": 0.14716458320617676, "contribution_to_active": 0.7003095149993896, "mix_mean": 0.6489620208740234, "mix_low": 0.005854606628417969, "mix_high": 0.019811630249023438, "read_cosine": 0.42855119705200195, "beta_mean": 0.21035140752792358, "nonfinite": 0.0, "weighted_memory_to_active": 0.10684368014335632, "mix_std": 0.32721802592277527, "beta_std": 0.24274945259094238, "beta_low": 0.23773193359375, "beta_high": 0.0, "orthogonal_to_active": 0.058311015367507935, "evidence_weights": [-0.1798025220632553, -0.33440646529197693, 0.18534648418426514], "agreement_weight": -0.006890615448355675, "orthogonal_weight": 0.6185097098350525}, {"head": 4, "memory_to_active": 0.6663972735404968, "contribution_to_active": 1.051435112953186, "mix_mean": 0.7656710147857666, "mix_low": 0.0006933212280273438, "mix_high": 0.12695884704589844, "read_cosine": 0.1480586677789688, "beta_mean": 0.14086291193962097, "nonfinite": 0.0, "weighted_memory_to_active": 0.6182448863983154, "mix_std": 0.2875092625617981, "beta_std": 0.15807487070560455, "beta_low": 0.2789306640625, "beta_high": 0.0, "orthogonal_to_active": 0.07583889365196228, "evidence_weights": [-0.23402053117752075, -0.5899882316589355, -0.2940201759338379], "agreement_weight": -0.01806759089231491, "orthogonal_weight": 0.13847891986370087}, {"head": 5, "memory_to_active": 0.03418096527457237, "contribution_to_active": 0.6157724261283875, "mix_mean": 0.6646783947944641, "mix_low": 0.0005750656127929688, "mix_high": 0.004810333251953125, "read_cosine": 0.28341734409332275, "beta_mean": 0.00020037303329445422, "nonfinite": 0.0, "weighted_memory_to_active": 0.023944297805428505, "mix_std": 0.2506853938102722, "beta_std": 0.0004324847541283816, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.02365949936211109, "evidence_weights": [-0.24182246625423431, -0.3119407594203949, 0.4333672523498535], "agreement_weight": 0.16906657814979553, "orthogonal_weight": -1.0585750341415405}]}, {"layer": 7, "memory_to_active": 1.6414830684661865, "contribution_to_active": 0.24363422393798828, "mix_mean": 0.1441354602575302, "mix_low": 0.2625386118888855, "mix_high": 0.0007882118225097656, "read_cosine": 0.22470198571681976, "beta_mean": 0.11779419332742691, "nonfinite": 0.0, "weighted_memory_to_active": 0.19631555676460266, "mix_std": 0.14187031984329224, "beta_std": 0.09619028866291046, "beta_low": 0.2064208984375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.07315234839916229, "output_influence": 0.2618541419506073, "residual_influence": 0.06007011979818344, "state_active": 0.013338835909962654, "state_memory": 0.024885740131139755, "state_peak": 32.17770767211914, "growth_active": 1.074047565460205, "growth_memory": 1.3216935396194458, "heads": [{"head": 0, "memory_to_active": 2.2577152252197266, "contribution_to_active": 0.18149779736995697, "mix_mean": 0.03609894961118698, "mix_low": 0.4384956359863281, "mix_high": 0.0, "read_cosine": 0.32969796657562256, "beta_mean": 0.13989193737506866, "nonfinite": 0.0, "weighted_memory_to_active": 0.2189074009656906, "mix_std": 0.07694776356220245, "beta_std": 0.1137344017624855, "beta_low": 0.0067138671875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.001757844933308661, "evidence_weights": [0.030699554830789566, -0.07668188214302063, 0.05473620444536209], "agreement_weight": -0.036225151270627975, "orthogonal_weight": 0.01121803279966116}, {"head": 1, "memory_to_active": 1.0156910419464111, "contribution_to_active": 0.3963758647441864, "mix_mean": 0.21344543993473053, "mix_low": 0.1199960708618164, "mix_high": 0.004324913024902344, "read_cosine": 0.35666629672050476, "beta_mean": 0.09278944134712219, "nonfinite": 0.0, "weighted_memory_to_active": 0.33550503849983215, "mix_std": 0.2406134009361267, "beta_std": 0.06763620674610138, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03922862187027931, "evidence_weights": [-0.20929382741451263, -0.10414610803127289, -0.2133466601371765], "agreement_weight": 0.04032108187675476, "orthogonal_weight": 0.13745853304862976}, {"head": 2, "memory_to_active": 0.7544102072715759, "contribution_to_active": 0.3210401237010956, "mix_mean": 0.16694651544094086, "mix_low": 0.05156230926513672, "mix_high": 0.00022029876708984375, "read_cosine": 0.08489987999200821, "beta_mean": 0.05163004249334335, "nonfinite": 0.0, "weighted_memory_to_active": 0.2315910905599594, "mix_std": 0.18876434862613678, "beta_std": 0.04691270738840103, "beta_low": 0.09112548828125, "beta_high": 0.0, "orthogonal_to_active": 0.10415409505367279, "evidence_weights": [-0.06559792906045914, -0.14896687865257263, 0.6669378876686096], "agreement_weight": 0.023937039077281952, "orthogonal_weight": -0.4622514545917511}, {"head": 3, "memory_to_active": 4.244483947753906, "contribution_to_active": 0.09361011534929276, "mix_mean": 0.03215814754366875, "mix_low": 0.3484983444213867, "mix_high": 0.0, "read_cosine": 0.09173639863729477, "beta_mean": 0.35628044605255127, "nonfinite": 0.0, "weighted_memory_to_active": 0.3059215843677521, "mix_std": 0.04950753599405289, "beta_std": 0.2735714912414551, "beta_low": 0.00469970703125, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.26304295659065247, "evidence_weights": [0.07048489153385162, -0.01490812748670578, 0.6871328949928284], "agreement_weight": 0.04671049863100052, "orthogonal_weight": -0.934211015701294}, {"head": 4, "memory_to_active": 1.5136313438415527, "contribution_to_active": 0.08307702094316483, "mix_mean": 0.01760496385395527, "mix_low": 0.6127166748046875, "mix_high": 0.0, "read_cosine": 0.23378658294677734, "beta_mean": 0.06585459411144257, "nonfinite": 0.0, "weighted_memory_to_active": 0.05910000205039978, "mix_std": 0.037217698991298676, "beta_std": 0.07449422776699066, "beta_low": 0.13690185546875, "beta_high": 0.0, "orthogonal_to_active": 0.009116857312619686, "evidence_weights": [0.1402990072965622, 0.01776031404733658, 0.25336283445358276], "agreement_weight": 0.04384675994515419, "orthogonal_weight": -0.19354981184005737}, {"head": 5, "memory_to_active": 0.06296593695878983, "contribution_to_active": 0.38620442152023315, "mix_mean": 0.39855873584747314, "mix_low": 0.003962516784667969, "mix_high": 0.00018405914306640625, "read_cosine": 0.25142478942871094, "beta_mean": 0.00031868997029960155, "nonfinite": 0.0, "weighted_memory_to_active": 0.026868196204304695, "mix_std": 0.2581712007522583, "beta_std": 0.0007926799007691443, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.021613717079162598, "evidence_weights": [-0.15080247819423676, -0.20841045677661896, 0.5591439008712769], "agreement_weight": 0.18015803396701813, "orthogonal_weight": -0.8606098294258118}]}, {"layer": 8, "memory_to_active": 0.2931591868400574, "contribution_to_active": 0.5374815464019775, "mix_mean": 0.4443163275718689, "mix_low": 0.044251129031181335, "mix_high": 0.0013499259948730469, "read_cosine": 0.4608270525932312, "beta_mean": 0.019484998658299446, "nonfinite": 0.0, "weighted_memory_to_active": 0.0881769135594368, "mix_std": 0.2250845581293106, "beta_std": 0.027519654482603073, "beta_low": 0.6934815049171448, "beta_high": 0.0, "orthogonal_to_active": 0.06892414391040802, "output_influence": 0.48492786288261414, "residual_influence": 0.16017329692840576, "state_active": 0.047940388321876526, "state_memory": 0.007239798083901405, "state_peak": 34.90163803100586, "growth_active": 1.112128734588623, "growth_memory": 1.217783808708191, "heads": [{"head": 0, "memory_to_active": 0.06097487732768059, "contribution_to_active": 0.5537810921669006, "mix_mean": 0.5987033247947693, "mix_low": 0.00042819976806640625, "mix_high": 0.0010776519775390625, "read_cosine": 0.4553283452987671, "beta_mean": 0.003336359281092882, "nonfinite": 0.0, "weighted_memory_to_active": 0.04251832515001297, "mix_std": 0.24800989031791687, "beta_std": 0.01119612343609333, "beta_low": 0.9228515625, "beta_high": 0.0, "orthogonal_to_active": 0.02737807109951973, "evidence_weights": [-0.1706320345401764, -0.38928261399269104, 0.6643924713134766], "agreement_weight": 0.1911478042602539, "orthogonal_weight": -0.8033900856971741}, {"head": 1, "memory_to_active": 0.10189743340015411, "contribution_to_active": 0.5345946550369263, "mix_mean": 0.4972916543483734, "mix_low": 8.678436279296875e-05, "mix_high": 0.000644683837890625, "read_cosine": 0.24077193439006805, "beta_mean": 0.0018860625568777323, "nonfinite": 0.0, "weighted_memory_to_active": 0.049910757690668106, "mix_std": 0.2440245896577835, "beta_std": 0.00910834688693285, "beta_low": 0.97509765625, "beta_high": 0.0, "orthogonal_to_active": 0.03567386418581009, "evidence_weights": [-0.18977686762809753, -0.27027544379234314, 0.7114846110343933], "agreement_weight": 0.07291257381439209, "orthogonal_weight": -0.7576885223388672}, {"head": 2, "memory_to_active": 0.01822604425251484, "contribution_to_active": 0.6793951988220215, "mix_mean": 0.6654819250106812, "mix_low": 0.0009098052978515625, "mix_high": 0.0035610198974609375, "read_cosine": 0.422063410282135, "beta_mean": 0.00016019909526221454, "nonfinite": 0.0, "weighted_memory_to_active": 0.014190156012773514, "mix_std": 0.22632774710655212, "beta_std": 0.002732523251324892, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.008956044912338257, "evidence_weights": [-0.22626443207263947, -0.34458494186401367, 0.9997791647911072], "agreement_weight": 0.22954902052879333, "orthogonal_weight": -0.7388801574707031}, {"head": 3, "memory_to_active": 0.11146220564842224, "contribution_to_active": 0.6458656787872314, "mix_mean": 0.5764459371566772, "mix_low": 0.0007915496826171875, "mix_high": 0.002689361572265625, "read_cosine": 0.8120226860046387, "beta_mean": 0.008541031740605831, "nonfinite": 0.0, "weighted_memory_to_active": 0.0835239365696907, "mix_std": 0.25577789545059204, "beta_std": 0.00798760261386633, "beta_low": 0.68597412109375, "beta_high": 0.0, "orthogonal_to_active": 0.026008905842900276, "evidence_weights": [-0.18482419848442078, -0.3037933111190796, 0.5868982672691345], "agreement_weight": 0.20599856972694397, "orthogonal_weight": -0.6328766942024231}, {"head": 4, "memory_to_active": 1.109347939491272, "contribution_to_active": 0.16557113826274872, "mix_mean": 0.07022269070148468, "mix_low": 0.16727828979492188, "mix_high": 0.0, "read_cosine": 0.3480135202407837, "beta_mean": 0.024498796090483665, "nonfinite": 0.0, "weighted_memory_to_active": 0.14847321808338165, "mix_std": 0.09708791226148605, "beta_std": 0.03838745504617691, "beta_low": 0.4505615234375, "beta_high": 0.0, "orthogonal_to_active": 0.01716981828212738, "evidence_weights": [0.022297246381640434, -0.08438421785831451, 0.05571703240275383], "agreement_weight": 0.07894942164421082, "orthogonal_weight": -0.1271587610244751}, {"head": 5, "memory_to_active": 0.3570466637611389, "contribution_to_active": 0.6456811428070068, "mix_mean": 0.25775232911109924, "mix_low": 0.09601211547851562, "mix_high": 0.00012683868408203125, "read_cosine": 0.4867625832557678, "beta_mean": 0.07848753035068512, "nonfinite": 0.0, "weighted_memory_to_active": 0.19044508039951324, "mix_std": 0.2792792320251465, "beta_std": 0.09570588171482086, "beta_low": 0.12646484375, "beta_high": 0.0, "orthogonal_to_active": 0.29835814237594604, "evidence_weights": [-0.10215583443641663, -0.19225893914699554, -0.16948622465133667], "agreement_weight": -0.021213995292782784, "orthogonal_weight": 1.8540111780166626}]}, {"layer": 9, "memory_to_active": 0.9774581789970398, "contribution_to_active": 0.6512393951416016, "mix_mean": 0.33691543340682983, "mix_low": 0.1273997724056244, "mix_high": 0.0049610142596066, "read_cosine": 0.3369022309780121, "beta_mean": 0.05402752012014389, "nonfinite": 0.0, "weighted_memory_to_active": 0.36395686864852905, "mix_std": 0.21646374464035034, "beta_std": 0.04177742823958397, "beta_low": 0.5056865215301514, "beta_high": 0.0, "orthogonal_to_active": 0.037318311631679535, "output_influence": 0.5220186710357666, "residual_influence": 0.14489687979221344, "state_active": 0.04235990345478058, "state_memory": 0.01800759881734848, "state_peak": 49.7408332824707, "growth_active": 1.1765375137329102, "growth_memory": 1.3693311214447021, "heads": [{"head": 0, "memory_to_active": 0.0024375685025006533, "contribution_to_active": 0.560099720954895, "mix_mean": 0.6295552849769592, "mix_low": 0.00171661376953125, "mix_high": 0.021806716918945312, "read_cosine": 0.24217595160007477, "beta_mean": 5.456653525470756e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.001594836707226932, "mix_std": 0.29157260060310364, "beta_std": 2.3268228687811643e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015776418149471283, "evidence_weights": [-0.3005678951740265, -0.3088013529777527, 0.8130921125411987], "agreement_weight": 0.12421172112226486, "orthogonal_weight": -1.0295066833496094}, {"head": 1, "memory_to_active": 0.023384667932987213, "contribution_to_active": 0.6742914915084839, "mix_mean": 0.5337286591529846, "mix_low": 0.008337020874023438, "mix_high": 0.0018634796142578125, "read_cosine": 0.40215444564819336, "beta_mean": 7.045372331049293e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.016171716153621674, "mix_std": 0.2850952744483948, "beta_std": 0.00016514520393684506, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011635774746537209, "evidence_weights": [-0.23294216394424438, -0.2846272885799408, 1.1441453695297241], "agreement_weight": 0.42559510469436646, "orthogonal_weight": -0.8751798868179321}, {"head": 2, "memory_to_active": 0.2020813524723053, "contribution_to_active": 0.6410287022590637, "mix_mean": 0.5774046182632446, "mix_low": 0.0011844635009765625, "mix_high": 0.0050716400146484375, "read_cosine": 0.3251758813858032, "beta_mean": 0.0008304092916660011, "nonfinite": 0.0, "weighted_memory_to_active": 0.13173960149288177, "mix_std": 0.27002885937690735, "beta_std": 0.0052505009807646275, "beta_low": 0.99322509765625, "beta_high": 0.0, "orthogonal_to_active": 0.07502619922161102, "evidence_weights": [-0.19745290279388428, -0.27223145961761475, 0.7444469928741455], "agreement_weight": 0.29286083579063416, "orthogonal_weight": -0.6599389314651489}, {"head": 3, "memory_to_active": 1.2978423833847046, "contribution_to_active": 0.1688050776720047, "mix_mean": 0.0549469068646431, "mix_low": 0.3268165588378906, "mix_high": 3.814697265625e-05, "read_cosine": 0.3852730691432953, "beta_mean": 0.07905156910419464, "nonfinite": 0.0, "weighted_memory_to_active": 0.14327594637870789, "mix_std": 0.10722917318344116, "beta_std": 0.05998094007372856, "beta_low": 0.001953125, "beta_high": 0.0, "orthogonal_to_active": 0.00412158714607358, "evidence_weights": [0.06922382861375809, 0.000603673339355737, 0.05245096981525421], "agreement_weight": 0.09331008046865463, "orthogonal_weight": -0.0307259913533926}, {"head": 4, "memory_to_active": 2.841912269592285, "contribution_to_active": 1.3782687187194824, "mix_mean": 0.13983765244483948, "mix_low": 0.13993263244628906, "mix_high": 0.000812530517578125, "read_cosine": 0.15570509433746338, "beta_mean": 0.15019437670707703, "nonfinite": 0.0, "weighted_memory_to_active": 1.3588666915893555, "mix_std": 0.20157568156719208, "beta_std": 0.08992207795381546, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.031282130628824234, "evidence_weights": [-0.07444069534540176, -0.15642540156841278, -0.038691043853759766], "agreement_weight": -0.05746044963598251, "orthogonal_weight": -0.023826763033866882}, {"head": 5, "memory_to_active": 1.4970909357070923, "contribution_to_active": 0.48494237661361694, "mix_mean": 0.0860193595290184, "mix_low": 0.2864112854003906, "mix_high": 0.0001735687255859375, "read_cosine": 0.5109288692474365, "beta_mean": 0.09401284903287888, "nonfinite": 0.0, "weighted_memory_to_active": 0.5320922136306763, "mix_std": 0.1432807743549347, "beta_std": 0.0953226312994957, "beta_low": 0.0389404296875, "beta_high": 0.0, "orthogonal_to_active": 0.10026654601097107, "evidence_weights": [0.10780992358922958, -0.09851981699466705, 0.13050444424152374], "agreement_weight": 0.015010959468781948, "orthogonal_weight": 0.2760675251483917}]}, {"layer": 10, "memory_to_active": 0.21460673213005066, "contribution_to_active": 0.6654305458068848, "mix_mean": 0.5101668238639832, "mix_low": 0.01855580136179924, "mix_high": 0.005806764587759972, "read_cosine": 0.376802533864975, "beta_mean": 0.04722476005554199, "nonfinite": 0.0, "weighted_memory_to_active": 0.10261081904172897, "mix_std": 0.26823681592941284, "beta_std": 0.04911160469055176, "beta_low": 0.6537577509880066, "beta_high": 0.0, "orthogonal_to_active": 0.20136354863643646, "output_influence": 0.43621915578842163, "residual_influence": 0.15100938081741333, "state_active": 0.05570506677031517, "state_memory": 0.015566590242087841, "state_peak": 116.64588928222656, "growth_active": 1.0446152687072754, "growth_memory": 0.9835353493690491, "heads": [{"head": 0, "memory_to_active": 0.8060559630393982, "contribution_to_active": 1.753365159034729, "mix_mean": 0.5912035703659058, "mix_low": 0.07427597045898438, "mix_high": 0.01726818084716797, "read_cosine": 0.09980764985084534, "beta_mean": 0.2063581794500351, "nonfinite": 0.0, "weighted_memory_to_active": 0.4984111189842224, "mix_std": 0.39250171184539795, "beta_std": 0.22251084446907043, "beta_low": 0.13702392578125, "beta_high": 0.0, "orthogonal_to_active": 1.124406337738037, "evidence_weights": [-0.11163658648729324, -0.3926806151866913, -1.4859113693237305], "agreement_weight": 0.02687455154955387, "orthogonal_weight": 2.28224778175354}, {"head": 1, "memory_to_active": 0.012217727489769459, "contribution_to_active": 0.5979821681976318, "mix_mean": 0.6137153506278992, "mix_low": 0.0010318756103515625, "mix_high": 0.012223243713378906, "read_cosine": 0.28653067350387573, "beta_mean": 7.753677346045151e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006951397284865379, "mix_std": 0.2667887806892395, "beta_std": 0.00011032913607778028, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010313941165804863, "evidence_weights": [-0.25962594151496887, -0.20126943290233612, 0.5765587687492371], "agreement_weight": 0.5214561223983765, "orthogonal_weight": -1.6012907028198242}, {"head": 2, "memory_to_active": 0.012776849791407585, "contribution_to_active": 0.5998095870018005, "mix_mean": 0.6325637102127075, "mix_low": 0.0002079010009765625, "mix_high": 0.004424095153808594, "read_cosine": 0.0362018346786499, "beta_mean": 2.5433600967517123e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00559715460985899, "mix_std": 0.2528820335865021, "beta_std": 0.00010110344737768173, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002653846051543951, "evidence_weights": [-0.24936442077159882, -0.2800719141960144, 0.6222615838050842], "agreement_weight": -0.19215688109397888, "orthogonal_weight": -0.4544430375099182}, {"head": 3, "memory_to_active": 0.31398484110832214, "contribution_to_active": 0.09475032985210419, "mix_mean": 0.2663830816745758, "mix_low": 0.033661842346191406, "mix_high": 0.00033664703369140625, "read_cosine": 0.9123677015304565, "beta_mean": 0.0693659782409668, "nonfinite": 0.0, "weighted_memory_to_active": 0.036391813308000565, "mix_std": 0.22000150382518768, "beta_std": 0.06364192813634872, "beta_low": 0.00128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.03898899257183075, "evidence_weights": [-0.1054646298289299, -0.14699356257915497, 0.15441681444644928], "agreement_weight": 0.009158806875348091, "orthogonal_weight": 1.4307594299316406}, {"head": 4, "memory_to_active": 0.020176071673631668, "contribution_to_active": 0.5404545068740845, "mix_mean": 0.5638948678970337, "mix_low": 0.00038433074951171875, "mix_high": 0.0005807876586914062, "read_cosine": 0.06442949175834656, "beta_mean": 6.658431811956689e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011581662110984325, "mix_std": 0.24859537184238434, "beta_std": 0.0001439472835045308, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013941863551735878, "evidence_weights": [-0.181405708193779, -0.2599136531352997, 0.9125894904136658], "agreement_weight": 0.14330069720745087, "orthogonal_weight": -1.2639341354370117}, {"head": 5, "memory_to_active": 0.12242887914180756, "contribution_to_active": 0.40622150897979736, "mix_mean": 0.3932402431964874, "mix_low": 0.0017728805541992188, "mix_high": 7.62939453125e-06, "read_cosine": 0.861477792263031, "beta_mean": 0.007454857230186462, "nonfinite": 0.0, "weighted_memory_to_active": 0.05673171952366829, "mix_std": 0.22865132987499237, "beta_std": 0.00816149078309536, "beta_low": 0.78424072265625, "beta_high": 0.0, "orthogonal_to_active": 0.01787629909813404, "evidence_weights": [-0.1422504037618637, -0.15994977951049805, 0.27637845277786255], "agreement_weight": 0.08944640308618546, "orthogonal_weight": -0.6404924392700195}]}, {"layer": 11, "memory_to_active": 0.09763991832733154, "contribution_to_active": 0.5997174978256226, "mix_mean": 0.49679023027420044, "mix_low": 0.10674063861370087, "mix_high": 0.04730590432882309, "read_cosine": 0.3630886673927307, "beta_mean": 0.029649831354618073, "nonfinite": 0.0, "weighted_memory_to_active": 0.04864993691444397, "mix_std": 0.3236672282218933, "beta_std": 0.04272172227501869, "beta_low": 0.6310221552848816, "beta_high": 0.0, "orthogonal_to_active": 0.047490015625953674, "output_influence": 0.5316237211227417, "residual_influence": 0.13547229766845703, "state_active": 0.1867140829563141, "state_memory": 0.005271613597869873, "state_peak": 29.00406265258789, "growth_active": 1.0598567724227905, "growth_memory": 1.6576350927352905, "heads": [{"head": 0, "memory_to_active": 0.007425157818943262, "contribution_to_active": 0.5979028940200806, "mix_mean": 0.6293833255767822, "mix_low": 0.0010128021240234375, "mix_high": 0.007573127746582031, "read_cosine": 0.2901611030101776, "beta_mean": 9.118828165810555e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005112401209771633, "mix_std": 0.26017916202545166, "beta_std": 0.00352392066270113, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.0038603530265390873, "evidence_weights": [-0.16670066118240356, -0.368679940700531, 0.7661444544792175], "agreement_weight": 0.31904980540275574, "orthogonal_weight": -0.8029823303222656}, {"head": 1, "memory_to_active": 0.2252044379711151, "contribution_to_active": 0.36912429332733154, "mix_mean": 0.4114578068256378, "mix_low": 0.005695343017578125, "mix_high": 0.00017642974853515625, "read_cosine": 0.5538239479064941, "beta_mean": 0.016104500740766525, "nonfinite": 0.0, "weighted_memory_to_active": 0.09905485808849335, "mix_std": 0.26128053665161133, "beta_std": 0.03321880102157593, "beta_low": 0.64599609375, "beta_high": 0.0, "orthogonal_to_active": 0.04228593781590462, "evidence_weights": [-0.16624709963798523, -0.2172020971775055, 0.5598815679550171], "agreement_weight": 0.06954600661993027, "orthogonal_weight": -0.5421462655067444}, {"head": 2, "memory_to_active": 0.008219652809202671, "contribution_to_active": 0.9837498664855957, "mix_mean": 0.43695327639579773, "mix_low": 0.2834901809692383, "mix_high": 0.17421531677246094, "read_cosine": 0.1788155436515808, "beta_mean": 0.07412391155958176, "nonfinite": 0.0, "weighted_memory_to_active": 0.005963562056422234, "mix_std": 0.4482974410057068, "beta_std": 0.09768275916576385, "beta_low": 0.45391845703125, "beta_high": 0.0, "orthogonal_to_active": 0.045383814722299576, "evidence_weights": [0.036984119564294815, -0.41689586639404297, -1.313637614250183], "agreement_weight": 0.005455083679407835, "orthogonal_weight": 7.877318382263184}, {"head": 3, "memory_to_active": 0.010670362040400505, "contribution_to_active": 0.8348532915115356, "mix_mean": 0.40909379720687866, "mix_low": 0.3396139144897461, "mix_high": 0.0967245101928711, "read_cosine": 0.5331220030784607, "beta_mean": 0.061778370290994644, "nonfinite": 0.0, "weighted_memory_to_active": 0.006456955336034298, "mix_std": 0.44066399335861206, "beta_std": 0.08915652334690094, "beta_low": 0.4771728515625, "beta_high": 0.0, "orthogonal_to_active": 0.04063132032752037, "evidence_weights": [0.17193183302879333, -0.317270427942276, -0.20398856699466705], "agreement_weight": 0.051721397787332535, "orthogonal_weight": 8.207582473754883}, {"head": 4, "memory_to_active": 0.24798431992530823, "contribution_to_active": 0.42919158935546875, "mix_mean": 0.5604602098464966, "mix_low": 0.007924079895019531, "mix_high": 0.0010328292846679688, "read_cosine": 0.04582642763853073, "beta_mean": 0.01629304140806198, "nonfinite": 0.0, "weighted_memory_to_active": 0.13238179683685303, "mix_std": 0.27174827456474304, "beta_std": 0.018490996211767197, "beta_low": 0.49725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.1282932162284851, "evidence_weights": [-0.24287594854831696, -0.22973960638046265, 1.973878026008606], "agreement_weight": -0.19500525295734406, "orthogonal_weight": -0.9895393252372742}, {"head": 5, "memory_to_active": 0.08633561432361603, "contribution_to_active": 0.3834831118583679, "mix_mean": 0.5333928465843201, "mix_low": 0.0027074813842773438, "mix_high": 0.004113197326660156, "read_cosine": 0.5767828226089478, "beta_mean": 0.009507978335022926, "nonfinite": 0.0, "weighted_memory_to_active": 0.04293005168437958, "mix_std": 0.2598339319229126, "beta_std": 0.014257309958338737, "beta_low": 0.71209716796875, "beta_high": 0.0, "orthogonal_to_active": 0.02448543906211853, "evidence_weights": [-0.2129359394311905, -0.30719077587127686, 0.6489774584770203], "agreement_weight": 0.08591246604919434, "orthogonal_weight": -0.7573534846305847}]}, {"layer": 12, "memory_to_active": 0.22696463763713837, "contribution_to_active": 0.5634751915931702, "mix_mean": 0.5234265923500061, "mix_low": 0.008945147506892681, "mix_high": 0.005005518905818462, "read_cosine": 0.24834275245666504, "beta_mean": 0.016343962401151657, "nonfinite": 0.0, "weighted_memory_to_active": 0.11620107293128967, "mix_std": 0.259829580783844, "beta_std": 0.015307174064218998, "beta_low": 0.826446533203125, "beta_high": 0.0, "orthogonal_to_active": 0.06346103549003601, "output_influence": 0.47833728790283203, "residual_influence": 0.13202165067195892, "state_active": 0.0339105948805809, "state_memory": 0.009246865287423134, "state_peak": 57.46656799316406, "growth_active": 1.1250109672546387, "growth_memory": 1.1617681980133057, "heads": [{"head": 0, "memory_to_active": 0.05257141962647438, "contribution_to_active": 0.5683465003967285, "mix_mean": 0.5405795574188232, "mix_low": 0.00043773651123046875, "mix_high": 0.0027894973754882812, "read_cosine": 0.31687912344932556, "beta_mean": 0.0010658248793333769, "nonfinite": 0.0, "weighted_memory_to_active": 0.026709984987974167, "mix_std": 0.26917150616645813, "beta_std": 0.002228732220828533, "beta_low": 0.9874267578125, "beta_high": 0.0, "orthogonal_to_active": 0.023583529517054558, "evidence_weights": [-0.20432217419147491, -0.2745417356491089, 0.16420045495033264], "agreement_weight": 0.10721845179796219, "orthogonal_weight": -0.9491419792175293}, {"head": 1, "memory_to_active": 0.08004390448331833, "contribution_to_active": 0.535895586013794, "mix_mean": 0.545890212059021, "mix_low": 0.0004558563232421875, "mix_high": 0.00130462646484375, "read_cosine": 0.4787858724594116, "beta_mean": 0.000829619646538049, "nonfinite": 0.0, "weighted_memory_to_active": 0.045462265610694885, "mix_std": 0.2648198902606964, "beta_std": 0.006697780452668667, "beta_low": 0.99505615234375, "beta_high": 0.0, "orthogonal_to_active": 0.03641631454229355, "evidence_weights": [-0.20748509466648102, -0.27345314621925354, 1.0268226861953735], "agreement_weight": 0.18055538833141327, "orthogonal_weight": -0.9258612394332886}, {"head": 2, "memory_to_active": 0.6123449802398682, "contribution_to_active": 0.46722397208213806, "mix_mean": 0.308293879032135, "mix_low": 0.0262603759765625, "mix_high": 1.811981201171875e-05, "read_cosine": 0.47480136156082153, "beta_mean": 0.09487661719322205, "nonfinite": 0.0, "weighted_memory_to_active": 0.2361832857131958, "mix_std": 0.23082086443901062, "beta_std": 0.06794027239084244, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.2407502382993698, "evidence_weights": [-0.23002086579799652, -0.05339374765753746, -0.027244212105870247], "agreement_weight": 0.03131875768303871, "orthogonal_weight": 1.2856782674789429}, {"head": 3, "memory_to_active": 0.025802213698625565, "contribution_to_active": 0.7327820062637329, "mix_mean": 0.7076777219772339, "mix_low": 0.00017547607421875, "mix_high": 0.016138076782226562, "read_cosine": 0.10245406627655029, "beta_mean": 0.00019440085452515632, "nonfinite": 0.0, "weighted_memory_to_active": 0.017359010875225067, "mix_std": 0.23199832439422607, "beta_std": 0.0033619622699916363, "beta_low": 0.998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.016508065164089203, "evidence_weights": [-0.2920392155647278, -0.3409063518047333, 1.544883131980896], "agreement_weight": 0.4583425521850586, "orthogonal_weight": -0.9711906909942627}, {"head": 4, "memory_to_active": 0.5344746708869934, "contribution_to_active": 0.5813083052635193, "mix_mean": 0.41166096925735474, "mix_low": 0.019062042236328125, "mix_high": 0.003940582275390625, "read_cosine": -0.13289383053779602, "beta_mean": 0.0007459712796844542, "nonfinite": 0.0, "weighted_memory_to_active": 0.33582526445388794, "mix_std": 0.28585219383239746, "beta_std": 0.008213575929403305, "beta_low": 0.9818115234375, "beta_high": 0.0, "orthogonal_to_active": 0.036314480006694794, "evidence_weights": [-0.13958851993083954, -0.21641585230827332, -0.03973866626620293], "agreement_weight": -0.002178153023123741, "orthogonal_weight": -0.11396408826112747}, {"head": 5, "memory_to_active": 0.05655060335993767, "contribution_to_active": 0.49529457092285156, "mix_mean": 0.6264569163322449, "mix_low": 0.007279396057128906, "mix_high": 0.0058422088623046875, "read_cosine": 0.25002986192703247, "beta_mean": 0.0003513400733936578, "nonfinite": 0.0, "weighted_memory_to_active": 0.03566666319966316, "mix_std": 0.2763146758079529, "beta_std": 0.00340072438120842, "beta_low": 0.99468994140625, "beta_high": 0.0, "orthogonal_to_active": 0.027193564921617508, "evidence_weights": [-0.24165469408035278, -0.3164548873901367, 1.3122702836990356], "agreement_weight": 0.371351420879364, "orthogonal_weight": -0.8238608241081238}]}, {"layer": 13, "memory_to_active": 0.12323026359081268, "contribution_to_active": 0.5316926836967468, "mix_mean": 0.5291774272918701, "mix_low": 0.0051547689363360405, "mix_high": 0.007361730560660362, "read_cosine": 0.4244319498538971, "beta_mean": 0.017203159630298615, "nonfinite": 0.0, "weighted_memory_to_active": 0.05268235504627228, "mix_std": 0.2584538161754608, "beta_std": 0.021560318768024445, "beta_low": 0.7988485097885132, "beta_high": 0.0, "orthogonal_to_active": 0.0527389794588089, "output_influence": 0.512452244758606, "residual_influence": 0.16862556338310242, "state_active": 0.12106417119503021, "state_memory": 0.0424531064927578, "state_peak": 102.93822479248047, "growth_active": 0.9320924282073975, "growth_memory": 1.000361680984497, "heads": [{"head": 0, "memory_to_active": 0.02078484371304512, "contribution_to_active": 0.5784205198287964, "mix_mean": 0.5963259339332581, "mix_low": 0.0004749298095703125, "mix_high": 0.0032835006713867188, "read_cosine": 0.4170590043067932, "beta_mean": 7.845681830076501e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012988990172743797, "mix_std": 0.26977255940437317, "beta_std": 0.00015572032134514302, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012915046885609627, "evidence_weights": [-0.19975624978542328, -0.26819291710853577, 0.7825521230697632], "agreement_weight": 0.14575664699077606, "orthogonal_weight": -1.101470708847046}, {"head": 1, "memory_to_active": 0.01554892398416996, "contribution_to_active": 0.5954393148422241, "mix_mean": 0.559011697769165, "mix_low": 0.0019664764404296875, "mix_high": 0.0029754638671875, "read_cosine": 0.2077755331993103, "beta_mean": 5.530031194211915e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008864490315318108, "mix_std": 0.2760535776615143, "beta_std": 0.00011428755533415824, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011340216733515263, "evidence_weights": [-0.18485327064990997, -0.26701781153678894, 0.674349308013916], "agreement_weight": 0.402558833360672, "orthogonal_weight": -1.3300683498382568}, {"head": 2, "memory_to_active": 0.09470377117395401, "contribution_to_active": 0.4535561203956604, "mix_mean": 0.38153576850891113, "mix_low": 0.001514434814453125, "mix_high": 0.00012493133544921875, "read_cosine": 0.669959545135498, "beta_mean": 0.02069682627916336, "nonfinite": 0.0, "weighted_memory_to_active": 0.04339445382356644, "mix_std": 0.23018985986709595, "beta_std": 0.061720315366983414, "beta_low": 0.8016357421875, "beta_high": 0.0, "orthogonal_to_active": 0.024576764553785324, "evidence_weights": [-0.18910665810108185, -0.20467686653137207, 0.4136465787887573], "agreement_weight": 0.00825982354581356, "orthogonal_weight": -0.8331939578056335}, {"head": 3, "memory_to_active": 0.1493179053068161, "contribution_to_active": 0.6111150979995728, "mix_mean": 0.6502148509025574, "mix_low": 0.0206451416015625, "mix_high": 0.028092384338378906, "read_cosine": 0.034619566053152084, "beta_mean": 0.0010012205457314849, "nonfinite": 0.0, "weighted_memory_to_active": 0.1117316484451294, "mix_std": 0.2870098352432251, "beta_std": 0.008616045117378235, "beta_low": 0.99176025390625, "beta_high": 0.0, "orthogonal_to_active": 0.09895122796297073, "evidence_weights": [-0.3199682831764221, -0.2470276653766632, 1.757024884223938], "agreement_weight": 5.118095397949219, "orthogonal_weight": -0.9814483523368835}, {"head": 4, "memory_to_active": 0.4064975082874298, "contribution_to_active": 0.24921639263629913, "mix_mean": 0.3441314697265625, "mix_low": 0.0040607452392578125, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8896031379699707, "beta_mean": 0.08111930638551712, "nonfinite": 0.0, "weighted_memory_to_active": 0.10401631891727448, "mix_std": 0.21389788389205933, "beta_std": 0.05644629895687103, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.13971489667892456, "evidence_weights": [-0.09828854352235794, -0.23651105165481567, 0.47770562767982483], "agreement_weight": 0.005379660986363888, "orthogonal_weight": 2.2174432277679443}, {"head": 5, "memory_to_active": 0.052528586238622665, "contribution_to_active": 0.7024083733558655, "mix_mean": 0.643844723701477, "mix_low": 0.0022668838500976562, "mix_high": 0.0096893310546875, "read_cosine": 0.3275747299194336, "beta_mean": 0.0002678496821317822, "nonfinite": 0.0, "weighted_memory_to_active": 0.03509822115302086, "mix_std": 0.2737991511821747, "beta_std": 0.0023092501796782017, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.02893572486937046, "evidence_weights": [-0.2209145873785019, -0.3057404160499573, 0.8593469262123108], "agreement_weight": 0.16701215505599976, "orthogonal_weight": -0.9186355471611023}]}, {"layer": 14, "memory_to_active": 0.23499470949172974, "contribution_to_active": 0.413715124130249, "mix_mean": 0.42146533727645874, "mix_low": 0.02722819708287716, "mix_high": 0.0055667562410235405, "read_cosine": 0.6071051955223083, "beta_mean": 0.06954151391983032, "nonfinite": 0.0, "weighted_memory_to_active": 0.09853807836771011, "mix_std": 0.24471573531627655, "beta_std": 0.06416172534227371, "beta_low": 0.4790547788143158, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03929336369037628, "output_influence": 0.3342267870903015, "residual_influence": 0.12889514863491058, "state_active": 0.06016739457845688, "state_memory": 0.015834596008062363, "state_peak": 39.29060363769531, "growth_active": 1.0891951322555542, "growth_memory": 1.2292832136154175, "heads": [{"head": 0, "memory_to_active": 0.34024253487586975, "contribution_to_active": 0.1843101978302002, "mix_mean": 0.18419089913368225, "mix_low": 0.017095565795898438, "mix_high": 9.5367431640625e-07, "read_cosine": 0.8002622127532959, "beta_mean": 0.06650476157665253, "nonfinite": 0.0, "weighted_memory_to_active": 0.07327153533697128, "mix_std": 0.1656874120235443, "beta_std": 0.061720166355371475, "beta_low": 0.010498046875, "beta_high": 0.0, "orthogonal_to_active": 0.06889177113771439, "evidence_weights": [-0.1245114654302597, -0.03857162967324257, 0.4510677456855774], "agreement_weight": 0.026434866711497307, "orthogonal_weight": 1.570029377937317}, {"head": 1, "memory_to_active": 0.5028947591781616, "contribution_to_active": 0.29092177748680115, "mix_mean": 0.22973498702049255, "mix_low": 0.13036155700683594, "mix_high": 9.5367431640625e-06, "read_cosine": 0.840407133102417, "beta_mean": 0.29933440685272217, "nonfinite": 0.0, "weighted_memory_to_active": 0.2185382843017578, "mix_std": 0.25862908363342285, "beta_std": 0.2719288766384125, "beta_low": 0.0745849609375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.01403061207383871, "evidence_weights": [-0.011552420444786549, -0.20799146592617035, -0.11005919426679611], "agreement_weight": 0.055313125252723694, "orthogonal_weight": 0.1554046869277954}, {"head": 2, "memory_to_active": 0.1390913426876068, "contribution_to_active": 0.5659215450286865, "mix_mean": 0.5734000205993652, "mix_low": 0.0028314590454101562, "mix_high": 0.0028362274169921875, "read_cosine": 0.4070507884025574, "beta_mean": 0.0023377249017357826, "nonfinite": 0.0, "weighted_memory_to_active": 0.08823540061712265, "mix_std": 0.2657560110092163, "beta_std": 0.0031551229767501354, "beta_low": 0.97161865234375, "beta_high": 0.0, "orthogonal_to_active": 0.06700989603996277, "evidence_weights": [-0.143703430891037, -0.3158918619155884, 0.8918651342391968], "agreement_weight": 0.081024669110775, "orthogonal_weight": -0.8637077808380127}, {"head": 3, "memory_to_active": 0.042925797402858734, "contribution_to_active": 0.6088683605194092, "mix_mean": 0.6675350666046143, "mix_low": 0.0052204132080078125, "mix_high": 0.01983165740966797, "read_cosine": -0.048020437359809875, "beta_mean": 0.006821155548095703, "nonfinite": 0.0, "weighted_memory_to_active": 0.026804253458976746, "mix_std": 0.25812095403671265, "beta_std": 0.007611577864736319, "beta_low": 0.77972412109375, "beta_high": 0.0, "orthogonal_to_active": 0.019667375832796097, "evidence_weights": [-0.35364827513694763, -0.3132217526435852, 0.31907564401626587], "agreement_weight": 0.07577256858348846, "orthogonal_weight": -0.812510073184967}, {"head": 4, "memory_to_active": 0.15699902176856995, "contribution_to_active": 0.4932538866996765, "mix_mean": 0.5002578496932983, "mix_low": 0.004137992858886719, "mix_high": 0.008901596069335938, "read_cosine": 0.8880028128623962, "beta_mean": 0.040211111307144165, "nonfinite": 0.0, "weighted_memory_to_active": 0.09137918055057526, "mix_std": 0.2649831175804138, "beta_std": 0.029564127326011658, "beta_low": 0.07421875, "beta_high": 0.0, "orthogonal_to_active": 0.024759996682405472, "evidence_weights": [-0.21984270215034485, -0.24745289981365204, 0.4248405992984772], "agreement_weight": 0.12955889105796814, "orthogonal_weight": -0.6061824560165405}, {"head": 5, "memory_to_active": 0.22781474888324738, "contribution_to_active": 0.33901506662368774, "mix_mean": 0.3736732602119446, "mix_low": 0.0037221908569335938, "mix_high": 0.0018205642700195312, "read_cosine": 0.7549287676811218, "beta_mean": 0.0020399214699864388, "nonfinite": 0.0, "weighted_memory_to_active": 0.09299980103969574, "mix_std": 0.2551177740097046, "beta_std": 0.01099046878516674, "beta_low": 0.96368408203125, "beta_high": 0.0, "orthogonal_to_active": 0.04140052571892738, "evidence_weights": [-0.17970122396945953, -0.1714511215686798, 0.27907177805900574], "agreement_weight": 0.07957025617361069, "orthogonal_weight": -0.7132523655891418}]}, {"layer": 15, "memory_to_active": 0.0555916503071785, "contribution_to_active": 0.48426663875579834, "mix_mean": 0.4122719466686249, "mix_low": 0.008421103470027447, "mix_high": 0.0023371379356831312, "read_cosine": 0.6746782064437866, "beta_mean": 0.03766994923353195, "nonfinite": 0.0, "weighted_memory_to_active": 0.026912230998277664, "mix_std": 0.2841360569000244, "beta_std": 0.044801048934459686, "beta_low": 0.7045491933822632, "beta_high": 0.0, "orthogonal_to_active": 0.007858112454414368, "output_influence": 0.3044017553329468, "residual_influence": 0.17122003436088562, "state_active": 0.14074453711509705, "state_memory": 0.010736831463873386, "state_peak": 36.865787506103516, "growth_active": 1.2291501760482788, "growth_memory": 1.4519500732421875, "heads": [{"head": 0, "memory_to_active": 0.11087644100189209, "contribution_to_active": 0.48797833919525146, "mix_mean": 0.36962416768074036, "mix_low": 0.006869316101074219, "mix_high": 0.004960060119628906, "read_cosine": 0.604956865310669, "beta_mean": 0.04157278686761856, "nonfinite": 0.0, "weighted_memory_to_active": 0.05309957265853882, "mix_std": 0.27925676107406616, "beta_std": 0.030222712084650993, "beta_low": 0.06591796875, "beta_high": 0.0, "orthogonal_to_active": 0.028035402297973633, "evidence_weights": [-0.18406368792057037, -0.1869058459997177, 0.4184306263923645], "agreement_weight": 0.09355540573596954, "orthogonal_weight": -0.7359176874160767}, {"head": 1, "memory_to_active": 0.11324825137853622, "contribution_to_active": 0.4158500134944916, "mix_mean": 0.42685815691947937, "mix_low": 0.0016222000122070312, "mix_high": 0.004534721374511719, "read_cosine": 0.9229546785354614, "beta_mean": 0.0006680365186184645, "nonfinite": 0.0, "weighted_memory_to_active": 0.04687827453017235, "mix_std": 0.2836735248565674, "beta_std": 0.002119855722412467, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.00048788805725052953, "evidence_weights": [-0.15226687490940094, -0.16189129650592804, 0.19527675211429596], "agreement_weight": 0.12304508686065674, "orthogonal_weight": 0.02014864981174469}, {"head": 2, "memory_to_active": 0.005791449919342995, "contribution_to_active": 0.479495108127594, "mix_mean": 0.3414919972419739, "mix_low": 0.014975547790527344, "mix_high": 0.0001087188720703125, "read_cosine": 0.7915515303611755, "beta_mean": 8.949876064434648e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0028726300224661827, "mix_std": 0.2713809609413147, "beta_std": 0.00021276919869706035, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007198709063231945, "evidence_weights": [-0.20678064227104187, -0.2705067992210388, 0.14291200041770935], "agreement_weight": 0.010561459697782993, "orthogonal_weight": 0.6601817607879639}, {"head": 3, "memory_to_active": 0.0015519760781899095, "contribution_to_active": 0.3756738305091858, "mix_mean": 0.39360469579696655, "mix_low": 0.005504608154296875, "mix_high": 0.0002880096435546875, "read_cosine": 0.6616865396499634, "beta_mean": 8.782635632087477e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005128823686391115, "mix_std": 0.2870272397994995, "beta_std": 2.3575788873131387e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004065805987920612, "evidence_weights": [-0.2496311515569687, -0.2488446682691574, 0.20420509576797485], "agreement_weight": 0.04026773199439049, "orthogonal_weight": -0.7990381121635437}, {"head": 4, "memory_to_active": 0.0745043009519577, "contribution_to_active": 0.6142758727073669, "mix_mean": 0.3703467547893524, "mix_low": 0.011066436767578125, "mix_high": 0.0008363723754882812, "read_cosine": 0.7340282797813416, "beta_mean": 0.1814970225095749, "nonfinite": 0.0, "weighted_memory_to_active": 0.03993600606918335, "mix_std": 0.3075985610485077, "beta_std": 0.21962957084178925, "beta_low": 0.19757080078125, "beta_high": 0.0, "orthogonal_to_active": 0.006551190745085478, "evidence_weights": [-0.21437111496925354, -0.25421854853630066, -0.037918590009212494], "agreement_weight": 0.013047382235527039, "orthogonal_weight": 0.3338894844055176}, {"head": 5, "memory_to_active": 0.027577482163906097, "contribution_to_active": 0.5323265194892883, "mix_mean": 0.5717058777809143, "mix_low": 0.010488510131835938, "mix_high": 0.0032949447631835938, "read_cosine": 0.332891047000885, "beta_mean": 0.0021835591178387403, "nonfinite": 0.0, "weighted_memory_to_active": 0.018174011260271072, "mix_std": 0.2758792042732239, "beta_std": 0.01659781113266945, "beta_low": 0.9708251953125, "beta_high": 0.0, "orthogonal_to_active": 0.010947739705443382, "evidence_weights": [-0.2926141321659088, -0.3083384335041046, 0.7263196706771851], "agreement_weight": 0.28166663646698, "orthogonal_weight": 0.8030397891998291}]}]}, {"step": 48000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8797552249998262, "memory_to_active": 0.8123130202293396, "contribution_to_active": 0.5439720749855042, "mix_mean": 0.4018314480781555, "mix_low": 0.13879498839378357, "mix_high": 0.01013001799583435, "read_cosine": 0.35311806201934814, "beta_mean": 0.07179903984069824, "nonfinite": 0.0, "weighted_memory_to_active": 0.20428220927715302, "mix_std": 0.2215951383113861, "beta_std": 0.0629536435008049, "beta_low": 0.551346480846405, "beta_high": 2.8610231311176904e-05, "orthogonal_to_active": 0.05623864382505417, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0514323711395264, "normal_token_accuracy": 0.4505005478858948, "state_active": 0.061680279672145844, "state_memory": 0.01785656064748764, "state_peak": 127.25499725341797, "growth_active": 1.1162135601043701, "growth_memory": 1.3240020275115967, "output_influence": 0.411661297082901, "residual_influence": 0.13731999695301056, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1585135459899902, "no_memory_loss_delta": 0.603631854057312, "scrambled_loss_delta": 0.15637986361980438, "memory_only_loss_delta": 3.2914516925811768, "no_evidence_loss_delta": 0.32840171456336975, "no_agreement_loss_delta": 0.026444535702466965, "no_orthogonal_loss_delta": -0.009295908734202385}, {"index": 94, "tokens": 256, "normal_loss": 3.172990083694458, "no_memory_loss_delta": 0.5962448716163635, "scrambled_loss_delta": 0.3180316686630249, "memory_only_loss_delta": 3.2302498817443848, "no_evidence_loss_delta": 0.267223596572876, "no_agreement_loss_delta": 0.04220458120107651, "no_orthogonal_loss_delta": 0.05183829739689827}, {"index": 188, "tokens": 256, "normal_loss": 3.3700339794158936, "no_memory_loss_delta": 0.6806500554084778, "scrambled_loss_delta": 0.17057955265045166, "memory_only_loss_delta": 3.1288018226623535, "no_evidence_loss_delta": 0.34673213958740234, "no_agreement_loss_delta": 0.04440419748425484, "no_orthogonal_loss_delta": 0.03721998259425163}, {"index": 283, "tokens": 131, "normal_loss": 1.9820141792297363, "no_memory_loss_delta": 0.4068617522716522, "scrambled_loss_delta": 0.1805882602930069, "memory_only_loss_delta": 2.448568344116211, "no_evidence_loss_delta": 0.16862010955810547, "no_agreement_loss_delta": -0.00024678403860889375, "no_orthogonal_loss_delta": 0.0010826833313331008}], "agreement_weight_abs_mean": 0.19710873067378998, "agreement_weight_abs_max": 5.2075629234313965, "agreement_weight_mean": 0.17276699841022491, "orthogonal_weight_abs_mean": 0.8170132637023926, "orthogonal_weight_abs_max": 8.33381462097168, "orthogonal_weight_mean": -0.12423377484083176, "no_memory_loss": 3.646219491958618, "no_memory_loss_delta": 0.5947872400283813, "no_memory_token_accuracy_gain": 0.08008898794651031, "no_memory_token_win_fraction": 0.6618465185165405, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 44.83200712116872, "scrambled_loss": 3.261415481567383, "scrambled_loss_delta": 0.20998306572437286, "scrambled_token_accuracy_gain": 0.031145717948675156, "scrambled_token_win_fraction": 0.6418242454528809, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.940202725654647, "memory_only_loss": 6.156317234039307, "memory_only_loss_delta": 3.104884624481201, "memory_only_token_accuracy_gain": 0.27586206793785095, "memory_only_token_win_fraction": 0.906562864780426, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.51703094902969, "no_evidence_loss": 3.3443496227264404, "no_evidence_loss_delta": 0.2929174304008484, "no_evidence_token_accuracy_gain": 0.031145717948675156, "no_evidence_token_win_fraction": 0.6151279211044312, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 25.391625801050093, "no_agreement_loss": 3.083589553833008, "no_agreement_loss_delta": 0.03215719759464264, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.49721914529800415, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 3.16456528601042, "no_orthogonal_loss": 3.074303388595581, "no_orthogonal_loss_delta": 0.022870969027280807, "no_orthogonal_token_accuracy_gain": 0.008898776024580002, "no_orthogonal_token_win_fraction": 0.5695216655731201, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.2611410962272176, "short_no_memory_loss_delta": 0.5041618943214417, "long_context_memory_gain": 0.0906253457069397, "normal_context_loss_gain": 0.1503172218799591, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.430708169937134, "contribution_to_active": 1.122133493423462, "mix_mean": 0.10420265793800354, "mix_low": 0.5406062006950378, "mix_high": 0.0052032470703125, "read_cosine": 0.42493459582328796, "beta_mean": 0.22661054134368896, "nonfinite": 0.0, "weighted_memory_to_active": 1.1622556447982788, "mix_std": 0.08855444937944412, "beta_std": 0.15035618841648102, "beta_low": 0.0407816581428051, "beta_high": 0.0, "orthogonal_to_active": 0.04771960899233818, "output_influence": 0.2025265097618103, "residual_influence": 0.24104496836662292, "state_active": 0.06808116286993027, "state_memory": 0.044370971620082855, "state_peak": 45.29341125488281, "growth_active": 1.248889446258545, "growth_memory": 1.2696343660354614, "heads": [{"head": 0, "memory_to_active": 2.5401418209075928, "contribution_to_active": 0.45911508798599243, "mix_mean": 0.01853923313319683, "mix_low": 0.6316452026367188, "mix_high": 0.0, "read_cosine": 0.23545026779174805, "beta_mean": 0.2020757794380188, "nonfinite": 0.0, "weighted_memory_to_active": 0.572306215763092, "mix_std": 0.05756543576717377, "beta_std": 0.09840366989374161, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05370939522981644, "evidence_weights": [-0.010792916640639305, -0.05498373880982399, -0.0807092934846878], "agreement_weight": -0.1748352348804474, "orthogonal_weight": -0.10645211488008499}, {"head": 1, "memory_to_active": 6.0142974853515625, "contribution_to_active": 2.5244364738464355, "mix_mean": 0.017783764749765396, "mix_low": 0.6511354446411133, "mix_high": 0.0, "read_cosine": 0.4739643931388855, "beta_mean": 0.4086313247680664, "nonfinite": 0.0, "weighted_memory_to_active": 2.805169105529785, "mix_std": 0.054221540689468384, "beta_std": 0.22751322388648987, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.006477585062384605, "evidence_weights": [0.020160336047410965, -0.10524015873670578, 0.5171582698822021], "agreement_weight": 0.027420494705438614, "orthogonal_weight": -0.00479716993868351}, {"head": 2, "memory_to_active": 2.661191940307617, "contribution_to_active": 0.4787273406982422, "mix_mean": 0.013650268316268921, "mix_low": 0.686274528503418, "mix_high": 0.0, "read_cosine": 0.28374773263931274, "beta_mean": 0.28620773553848267, "nonfinite": 0.0, "weighted_memory_to_active": 0.557083010673523, "mix_std": 0.03198741376399994, "beta_std": 0.21074175834655762, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04595947265625, "evidence_weights": [0.06212816759943962, 0.057171665132045746, 0.09884036332368851], "agreement_weight": 0.010010812431573868, "orthogonal_weight": -0.1910889744758606}, {"head": 3, "memory_to_active": 7.202751636505127, "contribution_to_active": 2.7575650215148926, "mix_mean": 0.033872801810503006, "mix_low": 0.4777107238769531, "mix_high": 0.0, "read_cosine": 0.28595128655433655, "beta_mean": 0.34903982281684875, "nonfinite": 0.0, "weighted_memory_to_active": 2.9347548484802246, "mix_std": 0.07764464616775513, "beta_std": 0.25789523124694824, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1578650176525116, "evidence_weights": [-0.042918507009744644, -0.04702543094754219, -0.15797024965286255], "agreement_weight": -0.02116108313202858, "orthogonal_weight": -0.06220663711428642}, {"head": 4, "memory_to_active": 2.056823968887329, "contribution_to_active": 0.07057997584342957, "mix_mean": 0.008118653669953346, "mix_low": 0.7817964553833008, "mix_high": 0.0, "read_cosine": 0.32763105630874634, "beta_mean": 0.0907675102353096, "nonfinite": 0.0, "weighted_memory_to_active": 0.05003125220537186, "mix_std": 0.018745239824056625, "beta_std": 0.09102679789066315, "beta_low": 0.0225830078125, "beta_high": 0.0, "orthogonal_to_active": 0.005437952466309071, "evidence_weights": [0.14379312098026276, 0.10280927270650864, -0.26637083292007446], "agreement_weight": -0.12665048241615295, "orthogonal_weight": 0.09530734270811081}, {"head": 5, "memory_to_active": 0.10904058814048767, "contribution_to_active": 0.4423768222332001, "mix_mean": 0.5332512259483337, "mix_low": 0.015074729919433594, "mix_high": 0.031219482421875, "read_cosine": 0.9428627490997314, "beta_mean": 0.022941116243600845, "nonfinite": 0.0, "weighted_memory_to_active": 0.05418914183974266, "mix_std": 0.2911624312400818, "beta_std": 0.016556404531002045, "beta_low": 0.22210693359375, "beta_high": 0.0, "orthogonal_to_active": 0.016868222504854202, "evidence_weights": [-0.18810813128948212, -0.21956810355186462, 0.3645975589752197], "agreement_weight": 0.28935936093330383, "orthogonal_weight": -0.6937174201011658}]}, {"layer": 1, "memory_to_active": 2.2417593002319336, "contribution_to_active": 0.47727227210998535, "mix_mean": 0.2044333815574646, "mix_low": 0.3667111396789551, "mix_high": 0.0010418891906738281, "read_cosine": 0.29891860485076904, "beta_mean": 0.17369034886360168, "nonfinite": 0.0, "weighted_memory_to_active": 0.3240911364555359, "mix_std": 0.11767525970935822, "beta_std": 0.12720929086208344, "beta_low": 0.3375956416130066, "beta_high": 9.155274165095761e-05, "orthogonal_to_active": 0.045454904437065125, "output_influence": 0.22531211376190186, "residual_influence": 0.06027289479970932, "state_active": 0.030057618394494057, "state_memory": 0.03537360578775406, "state_peak": 39.77788543701172, "growth_active": 1.1812481880187988, "growth_memory": 1.4945608377456665, "heads": [{"head": 0, "memory_to_active": 0.03525253385305405, "contribution_to_active": 0.4722157120704651, "mix_mean": 0.49295032024383545, "mix_low": 0.008108139038085938, "mix_high": 0.0014324188232421875, "read_cosine": 0.0019949721172451973, "beta_mean": 0.00023967762535903603, "nonfinite": 0.0, "weighted_memory_to_active": 0.02166012115776539, "mix_std": 0.28350168466567993, "beta_std": 0.000374840572476387, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.020934946835041046, "evidence_weights": [-0.19967533648014069, -0.25990089774131775, 0.34595373272895813], "agreement_weight": 0.18184725940227509, "orthogonal_weight": -1.0133112668991089}, {"head": 1, "memory_to_active": 2.7825229167938232, "contribution_to_active": 0.2519003748893738, "mix_mean": 0.023492731153964996, "mix_low": 0.45845985412597656, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5116510987281799, "beta_mean": 0.23939669132232666, "nonfinite": 0.0, "weighted_memory_to_active": 0.2828960418701172, "mix_std": 0.04842260852456093, "beta_std": 0.168268620967865, "beta_low": 0.00079345703125, "beta_high": 0.000244140625, "orthogonal_to_active": 0.019192075356841087, "evidence_weights": [0.018560905009508133, -0.04983934760093689, 0.018806353211402893], "agreement_weight": -0.004584218841046095, "orthogonal_weight": 0.17414741218090057}, {"head": 2, "memory_to_active": 0.07934392243623734, "contribution_to_active": 0.5413196682929993, "mix_mean": 0.6478955745697021, "mix_low": 0.0006723403930664062, "mix_high": 0.004815101623535156, "read_cosine": 0.4759795665740967, "beta_mean": 0.0035291588865220547, "nonfinite": 0.0, "weighted_memory_to_active": 0.05770227313041687, "mix_std": 0.237501323223114, "beta_std": 0.005525307729840279, "beta_low": 0.9296875, "beta_high": 0.0, "orthogonal_to_active": 0.03563404455780983, "evidence_weights": [-0.284178763628006, -0.3524928689002991, 0.6766458749771118], "agreement_weight": 0.03457137197256088, "orthogonal_weight": -0.7295315861701965}, {"head": 3, "memory_to_active": 2.8742763996124268, "contribution_to_active": 0.15910735726356506, "mix_mean": 0.02268090471625328, "mix_low": 0.567601203918457, "mix_high": 0.0, "read_cosine": 0.2085529863834381, "beta_mean": 0.3219028413295746, "nonfinite": 0.0, "weighted_memory_to_active": 0.24838697910308838, "mix_std": 0.060491956770420074, "beta_std": 0.2505611777305603, "beta_low": 0.05194091796875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.07977558672428131, "evidence_weights": [0.03227357566356659, -0.021046949550509453, 0.06221843883395195], "agreement_weight": -0.0359218530356884, "orthogonal_weight": -0.44887369871139526}, {"head": 4, "memory_to_active": 3.644551992416382, "contribution_to_active": 1.3676104545593262, "mix_mean": 0.03305082395672798, "mix_low": 0.31797218322753906, "mix_high": 2.86102294921875e-06, "read_cosine": 0.3667178750038147, "beta_mean": 0.17664456367492676, "nonfinite": 0.0, "weighted_memory_to_active": 1.281617283821106, "mix_std": 0.05764676257967949, "beta_std": 0.11879278719425201, "beta_low": 0.0238037109375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.10429546236991882, "evidence_weights": [-0.033604878932237625, -5.4952455684542656e-05, 0.04437262564897537], "agreement_weight": -0.016590816900134087, "orthogonal_weight": 0.11159556359052658}, {"head": 5, "memory_to_active": 4.034607410430908, "contribution_to_active": 0.07147996127605438, "mix_mean": 0.006529982667416334, "mix_low": 0.8474531173706055, "mix_high": 0.0, "read_cosine": 0.22861513495445251, "beta_mean": 0.3004290461540222, "nonfinite": 0.0, "weighted_memory_to_active": 0.052283938974142075, "mix_std": 0.01848718896508217, "beta_std": 0.21973298490047455, "beta_low": 0.01934814453125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.012897282838821411, "evidence_weights": [0.16477882862091064, 0.08501556515693665, 0.1403830200433731], "agreement_weight": -0.12762078642845154, "orthogonal_weight": 0.23680220544338226}]}, {"layer": 2, "memory_to_active": 1.0377098321914673, "contribution_to_active": 0.3560716509819031, "mix_mean": 0.38981151580810547, "mix_low": 0.25808510184288025, "mix_high": 0.004751682281494141, "read_cosine": 0.3122416138648987, "beta_mean": 0.0988275557756424, "nonfinite": 0.0, "weighted_memory_to_active": 0.08844999969005585, "mix_std": 0.19249336421489716, "beta_std": 0.07573680579662323, "beta_low": 0.5699361562728882, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.019576285034418106, "output_influence": 0.38471561670303345, "residual_influence": 0.0986126959323883, "state_active": 0.03641919791698456, "state_memory": 0.01606239192187786, "state_peak": 28.07698631286621, "growth_active": 1.0245519876480103, "growth_memory": 1.5208457708358765, "heads": [{"head": 0, "memory_to_active": 0.009025916457176208, "contribution_to_active": 0.34273001551628113, "mix_mean": 0.6144203543663025, "mix_low": 0.000957489013671875, "mix_high": 0.010221481323242188, "read_cosine": 0.1440752148628235, "beta_mean": 6.671916344203055e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005549762398004532, "mix_std": 0.27826252579689026, "beta_std": 0.00010157444921787828, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005252332892268896, "evidence_weights": [-0.360099196434021, -0.21505217254161835, 0.13775619864463806], "agreement_weight": 0.15083961188793182, "orthogonal_weight": -0.9839290976524353}, {"head": 1, "memory_to_active": 3.034844398498535, "contribution_to_active": 0.0430428646504879, "mix_mean": 0.004265793599188328, "mix_low": 0.923614501953125, "mix_high": 0.0, "read_cosine": 0.16491767764091492, "beta_mean": 0.29089099168777466, "nonfinite": 0.0, "weighted_memory_to_active": 0.03137236088514328, "mix_std": 0.011779746040701866, "beta_std": 0.18820855021476746, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.0122427549213171, "evidence_weights": [0.14554952085018158, 0.13114413619041443, 0.4652760922908783], "agreement_weight": 0.030281227082014084, "orthogonal_weight": 0.4071180522441864}, {"head": 2, "memory_to_active": 0.00943597313016653, "contribution_to_active": 0.3978327810764313, "mix_mean": 0.5190644264221191, "mix_low": 0.001739501953125, "mix_high": 0.0026416778564453125, "read_cosine": 0.31895411014556885, "beta_mean": 2.1650679627782665e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035796158481389284, "mix_std": 0.28110912442207336, "beta_std": 3.416436811676249e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003827822220046073, "evidence_weights": [-0.24350318312644958, -0.28307628631591797, 0.5359686613082886], "agreement_weight": 0.01859390363097191, "orthogonal_weight": -0.12819015979766846}, {"head": 3, "memory_to_active": 0.14137734472751617, "contribution_to_active": 0.5331776738166809, "mix_mean": 0.5461055040359497, "mix_low": 0.00031757354736328125, "mix_high": 0.0073184967041015625, "read_cosine": 0.6601514220237732, "beta_mean": 0.02443500980734825, "nonfinite": 0.0, "weighted_memory_to_active": 0.08506236225366592, "mix_std": 0.2648504376411438, "beta_std": 0.03040251135826111, "beta_low": 0.391357421875, "beta_high": 0.0, "orthogonal_to_active": 0.05406899377703667, "evidence_weights": [-0.1956871747970581, -0.3044322431087494, 0.8298724293708801], "agreement_weight": 0.06054101511836052, "orthogonal_weight": -0.8912144303321838}, {"head": 4, "memory_to_active": 3.0197558403015137, "contribution_to_active": 0.300626277923584, "mix_mean": 0.018296051770448685, "mix_low": 0.6206083297729492, "mix_high": 0.0, "read_cosine": 0.20642125606536865, "beta_mean": 0.2775106728076935, "nonfinite": 0.0, "weighted_memory_to_active": 0.3980140686035156, "mix_std": 0.051928214728832245, "beta_std": 0.23560987412929535, "beta_low": 0.0277099609375, "beta_high": 0.000732421875, "orthogonal_to_active": 0.04517675191164017, "evidence_weights": [0.025138260796666145, 0.011824378743767738, -0.0020522535778582096], "agreement_weight": 0.005625042133033276, "orthogonal_weight": -0.13509218394756317}, {"head": 5, "memory_to_active": 0.011819716542959213, "contribution_to_active": 0.519020140171051, "mix_mean": 0.6367168426513672, "mix_low": 0.0012731552124023438, "mix_high": 0.008328437805175781, "read_cosine": 0.37892991304397583, "beta_mean": 4.0294587961398065e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007121786940842867, "mix_std": 0.2670300602912903, "beta_std": 6.415459210984409e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003340835974086076, "evidence_weights": [-0.27989867329597473, -0.2921195924282074, 0.96672123670578], "agreement_weight": 0.14443592727184296, "orthogonal_weight": -0.06456143409013748}]}, {"layer": 3, "memory_to_active": 0.49925023317337036, "contribution_to_active": 0.44711145758628845, "mix_mean": 0.45789122581481934, "mix_low": 0.06395642459392548, "mix_high": 0.00847482681274414, "read_cosine": 0.2204321324825287, "beta_mean": 0.029394853860139847, "nonfinite": 0.0, "weighted_memory_to_active": 0.13301798701286316, "mix_std": 0.262835830450058, "beta_std": 0.026557203382253647, "beta_low": 0.7391154170036316, "beta_high": 0.0, "orthogonal_to_active": 0.05626606196165085, "output_influence": 0.37877392768859863, "residual_influence": 0.08330831676721573, "state_active": 0.02393912337720394, "state_memory": 0.007583426311612129, "state_peak": 21.757963180541992, "growth_active": 1.0294748544692993, "growth_memory": 1.710183024406433, "heads": [{"head": 0, "memory_to_active": 0.41979050636291504, "contribution_to_active": 0.5709190368652344, "mix_mean": 0.5563057661056519, "mix_low": 0.0029888153076171875, "mix_high": 0.017670631408691406, "read_cosine": 0.2338058054447174, "beta_mean": 0.011472154408693314, "nonfinite": 0.0, "weighted_memory_to_active": 0.25041627883911133, "mix_std": 0.31487756967544556, "beta_std": 0.0152659323066473, "beta_low": 0.64739990234375, "beta_high": 0.0, "orthogonal_to_active": 0.17453640699386597, "evidence_weights": [-0.21076154708862305, -0.24392829835414886, 0.8910762667655945], "agreement_weight": 0.10439849644899368, "orthogonal_weight": -0.738734781742096}, {"head": 1, "memory_to_active": 0.021866796538233757, "contribution_to_active": 0.42194995284080505, "mix_mean": 0.5342615842819214, "mix_low": 0.006458282470703125, "mix_high": 0.009057998657226562, "read_cosine": 0.19206972420215607, "beta_mean": 0.00012575062282849103, "nonfinite": 0.0, "weighted_memory_to_active": 0.012315702624619007, "mix_std": 0.30625849962234497, "beta_std": 0.0002196931600337848, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009510318748652935, "evidence_weights": [-0.2551935613155365, -0.20786935091018677, 0.6022195219993591], "agreement_weight": 0.4399157464504242, "orthogonal_weight": -0.8506839275360107}, {"head": 2, "memory_to_active": 2.276674509048462, "contribution_to_active": 0.28446826338768005, "mix_mean": 0.04449169337749481, "mix_low": 0.34894466400146484, "mix_high": 0.0, "read_cosine": 0.21709391474723816, "beta_mean": 0.15716740489006042, "nonfinite": 0.0, "weighted_memory_to_active": 0.3336251676082611, "mix_std": 0.09433863312005997, "beta_std": 0.13339151442050934, "beta_low": 0.01898193359375, "beta_high": 0.0, "orthogonal_to_active": 0.006600077264010906, "evidence_weights": [-0.08403094112873077, 0.011244932189583778, -0.007247519679367542], "agreement_weight": -0.01705143041908741, "orthogonal_weight": -0.024819087237119675}, {"head": 3, "memory_to_active": 0.018215300515294075, "contribution_to_active": 0.49374818801879883, "mix_mean": 0.5705181360244751, "mix_low": 0.00772857666015625, "mix_high": 0.010501861572265625, "read_cosine": 0.29739129543304443, "beta_mean": 7.752641977276653e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01232052966952324, "mix_std": 0.2977716326713562, "beta_std": 0.00017654501425568014, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.003243078477680683, "evidence_weights": [-0.23900851607322693, -0.22841355204582214, 2.154886245727539], "agreement_weight": 1.8499252796173096, "orthogonal_weight": -0.3499140441417694}, {"head": 4, "memory_to_active": 0.02258072979748249, "contribution_to_active": 0.2644712030887604, "mix_mean": 0.36651188135147095, "mix_low": 0.014361381530761719, "mix_high": 0.0007181167602539062, "read_cosine": 0.19053582847118378, "beta_mean": 0.00013938026677351445, "nonfinite": 0.0, "weighted_memory_to_active": 0.01234443113207817, "mix_std": 0.284372478723526, "beta_std": 0.00017720968753565103, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011901549994945526, "evidence_weights": [-0.16204242408275604, -0.17869167029857635, 0.09028874337673187], "agreement_weight": -0.023727215826511383, "orthogonal_weight": -1.0243369340896606}, {"head": 5, "memory_to_active": 0.2363733947277069, "contribution_to_active": 0.6471118927001953, "mix_mean": 0.6752580404281616, "mix_low": 0.0032567977905273438, "mix_high": 0.012900352478027344, "read_cosine": 0.1916961669921875, "beta_mean": 0.007386904209852219, "nonfinite": 0.0, "weighted_memory_to_active": 0.1770857572555542, "mix_std": 0.27939608693122864, "beta_std": 0.01011230330914259, "beta_low": 0.76837158203125, "beta_high": 0.0, "orthogonal_to_active": 0.1318049281835556, "evidence_weights": [-0.2540903389453888, -0.31034618616104126, 0.48013368248939514], "agreement_weight": 0.14612489938735962, "orthogonal_weight": -0.7935934662818909}]}, {"layer": 4, "memory_to_active": 1.0700509548187256, "contribution_to_active": 0.44246169924736023, "mix_mean": 0.3596395254135132, "mix_low": 0.2734055519104004, "mix_high": 0.009045442566275597, "read_cosine": 0.3312942385673523, "beta_mean": 0.11469517648220062, "nonfinite": 0.0, "weighted_memory_to_active": 0.12369176745414734, "mix_std": 0.15603095293045044, "beta_std": 0.10539239645004272, "beta_low": 0.3818766474723816, "beta_high": 0.0, "orthogonal_to_active": 0.03965740650892258, "output_influence": 0.44342756271362305, "residual_influence": 0.1267586052417755, "state_active": 0.039399221539497375, "state_memory": 0.01829872652888298, "state_peak": 39.33489227294922, "growth_active": 1.2550854682922363, "growth_memory": 1.5211926698684692, "heads": [{"head": 0, "memory_to_active": 0.2165054827928543, "contribution_to_active": 0.7382230162620544, "mix_mean": 0.6823714971542358, "mix_low": 0.01312255859375, "mix_high": 0.025394439697265625, "read_cosine": 0.5034018754959106, "beta_mean": 0.2609251141548157, "nonfinite": 0.0, "weighted_memory_to_active": 0.17072536051273346, "mix_std": 0.343325138092041, "beta_std": 0.26872795820236206, "beta_low": 0.2369384765625, "beta_high": 0.0, "orthogonal_to_active": 0.06970943510532379, "evidence_weights": [-0.26836884021759033, -0.36972665786743164, 0.10321364551782608], "agreement_weight": 0.01102412398904562, "orthogonal_weight": 0.5242350697517395}, {"head": 1, "memory_to_active": 0.06306672096252441, "contribution_to_active": 0.6380854845046997, "mix_mean": 0.6698704361915588, "mix_low": 0.000148773193359375, "mix_high": 0.01226043701171875, "read_cosine": 0.6329997181892395, "beta_mean": 0.0009151348494924605, "nonfinite": 0.0, "weighted_memory_to_active": 0.04513702541589737, "mix_std": 0.24117694795131683, "beta_std": 0.00555072957649827, "beta_low": 0.9844970703125, "beta_high": 0.0, "orthogonal_to_active": 0.02534893900156021, "evidence_weights": [-0.23795871436595917, -0.351371705532074, 0.6798596382141113], "agreement_weight": 0.2526949644088745, "orthogonal_weight": -0.7925189733505249}, {"head": 2, "memory_to_active": 1.864715576171875, "contribution_to_active": 0.27567780017852783, "mix_mean": 0.02293930947780609, "mix_low": 0.549494743347168, "mix_high": 2.47955322265625e-05, "read_cosine": 0.2574825882911682, "beta_mean": 0.08068441599607468, "nonfinite": 0.0, "weighted_memory_to_active": 0.20135453343391418, "mix_std": 0.0497114472091198, "beta_std": 0.07913964986801147, "beta_low": 0.04791259765625, "beta_high": 0.0, "orthogonal_to_active": 0.032670244574546814, "evidence_weights": [0.05980457365512848, 0.017899027094244957, 0.07923070341348648], "agreement_weight": 0.140010267496109, "orthogonal_weight": -0.24155837297439575}, {"head": 3, "memory_to_active": 1.8954551219940186, "contribution_to_active": 0.12249623239040375, "mix_mean": 0.030299212783575058, "mix_low": 0.43003177642822266, "mix_high": 0.0, "read_cosine": 0.19695761799812317, "beta_mean": 0.13221272826194763, "nonfinite": 0.0, "weighted_memory_to_active": 0.18516936898231506, "mix_std": 0.055757537484169006, "beta_std": 0.11808004230260849, "beta_low": 0.02191162109375, "beta_high": 0.0, "orthogonal_to_active": 0.002629507565870881, "evidence_weights": [-0.02617405541241169, 0.05696196109056473, -0.2332153618335724], "agreement_weight": 0.06946703791618347, "orthogonal_weight": -0.02309728041291237}, {"head": 4, "memory_to_active": 0.012704676017165184, "contribution_to_active": 0.6341937780380249, "mix_mean": 0.7383220791816711, "mix_low": 8.487701416015625e-05, "mix_high": 0.016588211059570312, "read_cosine": 0.1569473147392273, "beta_mean": 2.639455851749517e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009008057415485382, "mix_std": 0.21523940563201904, "beta_std": 7.615523645654321e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0037517931777983904, "evidence_weights": [-0.33701005578041077, -0.3283722698688507, 0.5996660590171814], "agreement_weight": 0.3412916660308838, "orthogonal_weight": -0.4404119551181793}, {"head": 5, "memory_to_active": 2.3678581714630127, "contribution_to_active": 0.24609383940696716, "mix_mean": 0.014034546911716461, "mix_low": 0.6475505828857422, "mix_high": 4.76837158203125e-06, "read_cosine": 0.23997636139392853, "beta_mean": 0.2134072184562683, "nonfinite": 0.0, "weighted_memory_to_active": 0.13075627386569977, "mix_std": 0.030975159257650375, "beta_std": 0.16077986359596252, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10383452475070953, "evidence_weights": [0.049166783690452576, -0.028415445238351822, -0.3100089430809021], "agreement_weight": -0.07288572937250137, "orthogonal_weight": 0.9242234826087952}]}, {"layer": 5, "memory_to_active": 0.42477795481681824, "contribution_to_active": 0.5111746788024902, "mix_mean": 0.48597949743270874, "mix_low": 0.058883510529994965, "mix_high": 0.01720285415649414, "read_cosine": 0.08965010941028595, "beta_mean": 0.056449733674526215, "nonfinite": 0.0, "weighted_memory_to_active": 0.1347568929195404, "mix_std": 0.2546389698982239, "beta_std": 0.07056475430727005, "beta_low": 0.6489970088005066, "beta_high": 0.0, "orthogonal_to_active": 0.051563017070293427, "output_influence": 0.5240139961242676, "residual_influence": 0.14727306365966797, "state_active": 0.045471563935279846, "state_memory": 0.010841026902198792, "state_peak": 67.57171630859375, "growth_active": 1.1454569101333618, "growth_memory": 1.2317172288894653, "heads": [{"head": 0, "memory_to_active": 0.4141097664833069, "contribution_to_active": 0.4972175657749176, "mix_mean": 0.36616113781929016, "mix_low": 0.017498016357421875, "mix_high": 0.0010814666748046875, "read_cosine": -0.09320460259914398, "beta_mean": 0.021282097324728966, "nonfinite": 0.0, "weighted_memory_to_active": 0.11493417620658875, "mix_std": 0.2702271342277527, "beta_std": 0.03442368656396866, "beta_low": 0.45672607421875, "beta_high": 0.0, "orthogonal_to_active": 0.041156917810440063, "evidence_weights": [-0.13256484270095825, -0.16109023988246918, -0.04872946813702583], "agreement_weight": -0.023316366598010063, "orthogonal_weight": -0.3604077398777008}, {"head": 1, "memory_to_active": 0.012028470635414124, "contribution_to_active": 0.5298997163772583, "mix_mean": 0.6704455018043518, "mix_low": 0.0020685195922851562, "mix_high": 0.02889251708984375, "read_cosine": 0.20482797920703888, "beta_mean": 2.910411603807006e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006944413296878338, "mix_std": 0.2815517783164978, "beta_std": 6.080852108425461e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006397728808224201, "evidence_weights": [-0.2474551796913147, -0.28310951590538025, 0.8828951716423035], "agreement_weight": 0.6376025676727295, "orthogonal_weight": -0.9680962562561035}, {"head": 2, "memory_to_active": 0.009540125727653503, "contribution_to_active": 0.5542827248573303, "mix_mean": 0.6468852162361145, "mix_low": 0.00321197509765625, "mix_high": 0.021532058715820312, "read_cosine": -0.4712381958961487, "beta_mean": 7.23272460163571e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006169919855892658, "mix_std": 0.27124717831611633, "beta_std": 0.00011824497778434306, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023317995946854353, "evidence_weights": [-0.26072970032691956, -0.3440583646297455, 0.42717865109443665], "agreement_weight": -0.010351590812206268, "orthogonal_weight": -0.4721245765686035}, {"head": 3, "memory_to_active": 1.5971077680587769, "contribution_to_active": 0.21380536258220673, "mix_mean": 0.10497498512268066, "mix_low": 0.1367349624633789, "mix_high": 9.5367431640625e-07, "read_cosine": 0.16526103019714355, "beta_mean": 0.12186501175165176, "nonfinite": 0.0, "weighted_memory_to_active": 0.3329043984413147, "mix_std": 0.13884146511554718, "beta_std": 0.16540385782718658, "beta_low": 0.1640625, "beta_high": 0.0, "orthogonal_to_active": 0.13228754699230194, "evidence_weights": [-0.06896694749593735, -0.0936250388622284, -0.0056220972910523415], "agreement_weight": 0.037162117660045624, "orthogonal_weight": -0.47352850437164307}, {"head": 4, "memory_to_active": 0.4349687099456787, "contribution_to_active": 0.5907371044158936, "mix_mean": 0.33060795068740845, "mix_low": 0.19156932830810547, "mix_high": 0.007649421691894531, "read_cosine": 0.5493432283401489, "beta_mean": 0.19519227743148804, "nonfinite": 0.0, "weighted_memory_to_active": 0.2926430106163025, "mix_std": 0.3396095037460327, "beta_std": 0.22241772711277008, "beta_low": 0.27532958984375, "beta_high": 0.0, "orthogonal_to_active": 0.09275119006633759, "evidence_weights": [-0.11312669515609741, -0.271711140871048, -0.06283275038003922], "agreement_weight": 0.04177679866552353, "orthogonal_weight": 0.4940667152404785}, {"head": 5, "memory_to_active": 0.08091296255588531, "contribution_to_active": 0.6811054944992065, "mix_mean": 0.796802282333374, "mix_low": 0.0022182464599609375, "mix_high": 0.044060707092285156, "read_cosine": 0.18291117250919342, "beta_mean": 0.0002575357211753726, "nonfinite": 0.0, "weighted_memory_to_active": 0.054945360869169235, "mix_std": 0.22635678946971893, "beta_std": 0.0009641989599913359, "beta_low": 0.99786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.034452904015779495, "evidence_weights": [-0.3173989951610565, -0.33506137132644653, 1.9982566833496094], "agreement_weight": 0.44187965989112854, "orthogonal_weight": -0.6646785736083984}]}, {"layer": 6, "memory_to_active": 0.33519989252090454, "contribution_to_active": 0.6059402823448181, "mix_mean": 0.5780572891235352, "mix_low": 0.06030909717082977, "mix_high": 0.034276485443115234, "read_cosine": 0.28818967938423157, "beta_mean": 0.06463142484426498, "nonfinite": 0.0, "weighted_memory_to_active": 0.15849323570728302, "mix_std": 0.23573610186576843, "beta_std": 0.07415983080863953, "beta_low": 0.5207113027572632, "beta_high": 0.0, "orthogonal_to_active": 0.04192313924431801, "output_influence": 0.5216637849807739, "residual_influence": 0.16934777796268463, "state_active": 0.04556810483336449, "state_memory": 0.008403186686336994, "state_peak": 89.22936248779297, "growth_active": 1.1561754941940308, "growth_memory": 1.15707528591156, "heads": [{"head": 0, "memory_to_active": 0.1157701388001442, "contribution_to_active": 0.7011594772338867, "mix_mean": 0.7955517768859863, "mix_low": 0.00017547607421875, "mix_high": 0.036728858947753906, "read_cosine": 0.09725543111562729, "beta_mean": 0.01590432971715927, "nonfinite": 0.0, "weighted_memory_to_active": 0.09414982795715332, "mix_std": 0.2055538296699524, "beta_std": 0.023416198790073395, "beta_low": 0.559814453125, "beta_high": 0.0, "orthogonal_to_active": 0.0774146243929863, "evidence_weights": [-0.31999439001083374, -0.3442908823490143, 1.1334713697433472], "agreement_weight": 0.3916081488132477, "orthogonal_weight": -0.8484298586845398}, {"head": 1, "memory_to_active": 0.005274937953799963, "contribution_to_active": 0.5015170574188232, "mix_mean": 0.560051441192627, "mix_low": 0.0015897750854492188, "mix_high": 0.013205528259277344, "read_cosine": 0.5009867548942566, "beta_mean": 9.895755283650942e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002293881494551897, "mix_std": 0.28718191385269165, "beta_std": 1.6177711586351506e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012858235277235508, "evidence_weights": [-0.19840064644813538, -0.298074871301651, 1.1086114645004272], "agreement_weight": 0.05726473405957222, "orthogonal_weight": -0.7330518960952759}, {"head": 2, "memory_to_active": 0.99863201379776, "contribution_to_active": 0.08935990929603577, "mix_mean": 0.0348077267408371, "mix_low": 0.3504981994628906, "mix_high": 0.0, "read_cosine": 0.24264946579933167, "beta_mean": 0.04078182950615883, "nonfinite": 0.0, "weighted_memory_to_active": 0.06689700484275818, "mix_std": 0.05750174820423126, "beta_std": 0.041926056146621704, "beta_low": 0.06524658203125, "beta_high": 0.0, "orthogonal_to_active": 0.003752682823687792, "evidence_weights": [0.08388575166463852, -0.028743693605065346, 0.2955164611339569], "agreement_weight": 0.14379069209098816, "orthogonal_weight": -0.06992322951555252}, {"head": 3, "memory_to_active": 0.1573951542377472, "contribution_to_active": 0.6749967336654663, "mix_mean": 0.6500716209411621, "mix_low": 0.008774757385253906, "mix_high": 0.01894092559814453, "read_cosine": 0.4457628130912781, "beta_mean": 0.20204490423202515, "nonfinite": 0.0, "weighted_memory_to_active": 0.11096397042274475, "mix_std": 0.3257769048213959, "beta_std": 0.2345280796289444, "beta_low": 0.2225341796875, "beta_high": 0.0, "orthogonal_to_active": 0.05886775627732277, "evidence_weights": [-0.17793090641498566, -0.33268439769744873, 0.19415171444416046], "agreement_weight": -0.0038327511865645647, "orthogonal_weight": 0.6132444739341736}, {"head": 4, "memory_to_active": 0.6924116611480713, "contribution_to_active": 1.0585639476776123, "mix_mean": 0.7673948407173157, "mix_low": 0.000675201416015625, "mix_high": 0.13102149963378906, "read_cosine": 0.1707008183002472, "beta_mean": 0.12880992889404297, "nonfinite": 0.0, "weighted_memory_to_active": 0.646953284740448, "mix_std": 0.28496837615966797, "beta_std": 0.14455252885818481, "beta_low": 0.27703857421875, "beta_high": 0.0, "orthogonal_to_active": 0.08082498610019684, "evidence_weights": [-0.2300247848033905, -0.5852484107017517, -0.30225512385368347], "agreement_weight": -0.01622760109603405, "orthogonal_weight": 0.14280855655670166}, {"head": 5, "memory_to_active": 0.041715435683727264, "contribution_to_active": 0.6100442409515381, "mix_mean": 0.6604663133621216, "mix_low": 0.000141143798828125, "mix_high": 0.0057621002197265625, "read_cosine": 0.2717828154563904, "beta_mean": 0.0002376507909502834, "nonfinite": 0.0, "weighted_memory_to_active": 0.029701443389058113, "mix_std": 0.25343361496925354, "beta_std": 0.0005199283477850258, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.029392942786216736, "evidence_weights": [-0.24341312050819397, -0.3131021559238434, 0.4209640622138977], "agreement_weight": 0.17078940570354462, "orthogonal_weight": -1.0613794326782227}]}, {"layer": 7, "memory_to_active": 1.7687621116638184, "contribution_to_active": 0.2521163523197174, "mix_mean": 0.14480388164520264, "mix_low": 0.2624136805534363, "mix_high": 0.0009598732576705515, "read_cosine": 0.23410925269126892, "beta_mean": 0.12994641065597534, "nonfinite": 0.0, "weighted_memory_to_active": 0.21217644214630127, "mix_std": 0.14298111200332642, "beta_std": 0.10068641602993011, "beta_low": 0.2013346403837204, "beta_high": 0.00023396810865961015, "orthogonal_to_active": 0.07940225303173065, "output_influence": 0.2642820477485657, "residual_influence": 0.05950425937771797, "state_active": 0.013448942452669144, "state_memory": 0.025696685537695885, "state_peak": 29.32952308654785, "growth_active": 1.1217854022979736, "growth_memory": 1.296006441116333, "heads": [{"head": 0, "memory_to_active": 2.5691308975219727, "contribution_to_active": 0.20554403960704803, "mix_mean": 0.0379040464758873, "mix_low": 0.42383861541748047, "mix_high": 1.9073486328125e-06, "read_cosine": 0.3451976776123047, "beta_mean": 0.1570906639099121, "nonfinite": 0.0, "weighted_memory_to_active": 0.24847926199436188, "mix_std": 0.08017648011445999, "beta_std": 0.12237556278705597, "beta_low": 0.00335693359375, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.0016165341949090362, "evidence_weights": [0.030506912618875504, -0.08100245147943497, 0.057536836713552475], "agreement_weight": -0.03669717162847519, "orthogonal_weight": 0.009008216671645641}, {"head": 1, "memory_to_active": 1.005238652229309, "contribution_to_active": 0.3869432508945465, "mix_mean": 0.21337008476257324, "mix_low": 0.13259410858154297, "mix_high": 0.005146980285644531, "read_cosine": 0.3613928556442261, "beta_mean": 0.08471983671188354, "nonfinite": 0.0, "weighted_memory_to_active": 0.3377339839935303, "mix_std": 0.2429075837135315, "beta_std": 0.0644509494304657, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.03845958411693573, "evidence_weights": [-0.20920425653457642, -0.10222963243722916, -0.2310379594564438], "agreement_weight": 0.04639773443341255, "orthogonal_weight": 0.13636739552021027}, {"head": 2, "memory_to_active": 0.7970062494277954, "contribution_to_active": 0.326192706823349, "mix_mean": 0.16409078240394592, "mix_low": 0.053154945373535156, "mix_high": 0.0002155303955078125, "read_cosine": 0.11493609845638275, "beta_mean": 0.057016681879758835, "nonfinite": 0.0, "weighted_memory_to_active": 0.2621535062789917, "mix_std": 0.18688708543777466, "beta_std": 0.050263646990060806, "beta_low": 0.0577392578125, "beta_high": 0.0, "orthogonal_to_active": 0.11984071135520935, "evidence_weights": [-0.06396987289190292, -0.1464197188615799, 0.6743868589401245], "agreement_weight": 0.028301989659667015, "orthogonal_weight": -0.47364750504493713}, {"head": 3, "memory_to_active": 4.520260334014893, "contribution_to_active": 0.10281187295913696, "mix_mean": 0.03405782952904701, "mix_low": 0.33197784423828125, "mix_high": 0.0, "read_cosine": 0.11147773265838623, "beta_mean": 0.3997265696525574, "nonfinite": 0.0, "weighted_memory_to_active": 0.336903840303421, "mix_std": 0.050895947962999344, "beta_std": 0.2769448757171631, "beta_low": 0.0030517578125, "beta_high": 0.0009765625, "orthogonal_to_active": 0.285132497549057, "evidence_weights": [0.06903441250324249, -0.017625518143177032, 0.6979923844337463], "agreement_weight": 0.04606708139181137, "orthogonal_weight": -0.936969518661499}, {"head": 4, "memory_to_active": 1.6622401475906372, "contribution_to_active": 0.09080637991428375, "mix_mean": 0.017169319093227386, "mix_low": 0.6296262741088867, "mix_high": 0.0, "read_cosine": 0.2210400104522705, "beta_mean": 0.08077114820480347, "nonfinite": 0.0, "weighted_memory_to_active": 0.06218089163303375, "mix_std": 0.03901289403438568, "beta_std": 0.08923714607954025, "beta_low": 0.14508056640625, "beta_high": 0.0, "orthogonal_to_active": 0.010965235531330109, "evidence_weights": [0.14301730692386627, 0.01794712245464325, 0.25753989815711975], "agreement_weight": 0.03543468192219734, "orthogonal_weight": -0.21261811256408691}, {"head": 5, "memory_to_active": 0.05869608744978905, "contribution_to_active": 0.4003998339176178, "mix_mean": 0.4022311568260193, "mix_low": 0.0032901763916015625, "mix_high": 0.0003948211669921875, "read_cosine": 0.2506110668182373, "beta_mean": 0.0003535364521667361, "nonfinite": 0.0, "weighted_memory_to_active": 0.025607166811823845, "mix_std": 0.2580066919326782, "beta_std": 0.0008463251870125532, "beta_low": 0.9986572265625, "beta_high": 0.0, "orthogonal_to_active": 0.02039896324276924, "evidence_weights": [-0.153283953666687, -0.21109318733215332, 0.5569234490394592], "agreement_weight": 0.16710448265075684, "orthogonal_weight": -0.8687605857849121}]}, {"layer": 8, "memory_to_active": 0.29277878999710083, "contribution_to_active": 0.5434927940368652, "mix_mean": 0.44517260789871216, "mix_low": 0.041101619601249695, "mix_high": 0.0015435219975188375, "read_cosine": 0.44583895802497864, "beta_mean": 0.017145425081253052, "nonfinite": 0.0, "weighted_memory_to_active": 0.09162166714668274, "mix_std": 0.22426144778728485, "beta_std": 0.025443706661462784, "beta_low": 0.70965576171875, "beta_high": 0.0, "orthogonal_to_active": 0.06974470615386963, "output_influence": 0.4900107681751251, "residual_influence": 0.16184858977794647, "state_active": 0.04519113898277283, "state_memory": 0.007072828244417906, "state_peak": 38.769256591796875, "growth_active": 1.0463083982467651, "growth_memory": 1.2308372259140015, "heads": [{"head": 0, "memory_to_active": 0.0577663779258728, "contribution_to_active": 0.559442937374115, "mix_mean": 0.601064920425415, "mix_low": 0.00038051605224609375, "mix_high": 0.00115966796875, "read_cosine": 0.46451273560523987, "beta_mean": 0.0030945902690291405, "nonfinite": 0.0, "weighted_memory_to_active": 0.040890201926231384, "mix_std": 0.24589437246322632, "beta_std": 0.0100884810090065, "beta_low": 0.92706298828125, "beta_high": 0.0, "orthogonal_to_active": 0.026034435257315636, "evidence_weights": [-0.1709674447774887, -0.39019861817359924, 0.6550753116607666], "agreement_weight": 0.2026606649160385, "orthogonal_weight": -0.801234781742096}, {"head": 1, "memory_to_active": 0.1041426807641983, "contribution_to_active": 0.525374710559845, "mix_mean": 0.49750953912734985, "mix_low": 6.866455078125e-05, "mix_high": 0.0007677078247070312, "read_cosine": 0.1757543683052063, "beta_mean": 0.002000081352889538, "nonfinite": 0.0, "weighted_memory_to_active": 0.04879331588745117, "mix_std": 0.24409130215644836, "beta_std": 0.01104910857975483, "beta_low": 0.97381591796875, "beta_high": 0.0, "orthogonal_to_active": 0.03590317815542221, "evidence_weights": [-0.18972913920879364, -0.27080342173576355, 0.699856162071228], "agreement_weight": 0.07610276341438293, "orthogonal_weight": -0.7653502225875854}, {"head": 2, "memory_to_active": 0.019094502553343773, "contribution_to_active": 0.6776788830757141, "mix_mean": 0.6639996767044067, "mix_low": 0.0009899139404296875, "mix_high": 0.004131317138671875, "read_cosine": 0.41908037662506104, "beta_mean": 0.0001744173205224797, "nonfinite": 0.0, "weighted_memory_to_active": 0.014643687754869461, "mix_std": 0.2263522893190384, "beta_std": 0.0036420049145817757, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.009233297780156136, "evidence_weights": [-0.226384237408638, -0.34487250447273254, 1.0127670764923096], "agreement_weight": 0.24193274974822998, "orthogonal_weight": -0.7341198921203613}, {"head": 3, "memory_to_active": 0.10369285941123962, "contribution_to_active": 0.6677731275558472, "mix_mean": 0.5800625085830688, "mix_low": 0.0008134841918945312, "mix_high": 0.0030841827392578125, "read_cosine": 0.8288943767547607, "beta_mean": 0.008032426238059998, "nonfinite": 0.0, "weighted_memory_to_active": 0.07994896173477173, "mix_std": 0.25563859939575195, "beta_std": 0.007281729951500893, "beta_low": 0.71551513671875, "beta_high": 0.0, "orthogonal_to_active": 0.023475201800465584, "evidence_weights": [-0.1864023208618164, -0.3047749698162079, 0.5840762853622437], "agreement_weight": 0.21371595561504364, "orthogonal_weight": -0.632092297077179}, {"head": 4, "memory_to_active": 1.1126617193222046, "contribution_to_active": 0.19373512268066406, "mix_mean": 0.07556533068418503, "mix_low": 0.15234375, "mix_high": 0.0, "read_cosine": 0.31550711393356323, "beta_mean": 0.023310329765081406, "nonfinite": 0.0, "weighted_memory_to_active": 0.17550252377986908, "mix_std": 0.10161293298006058, "beta_std": 0.03932495415210724, "beta_low": 0.49969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.02374853566288948, "evidence_weights": [0.01950417459011078, -0.08718059211969376, 0.09433875232934952], "agreement_weight": 0.07499922066926956, "orthogonal_weight": -0.14465151727199554}, {"head": 5, "memory_to_active": 0.35931459069252014, "contribution_to_active": 0.6369520425796509, "mix_mean": 0.25283342599868774, "mix_low": 0.09201335906982422, "mix_high": 0.000118255615234375, "read_cosine": 0.4712847173213959, "beta_mean": 0.06626070290803909, "nonfinite": 0.0, "weighted_memory_to_active": 0.18995130062103271, "mix_std": 0.27197912335395813, "beta_std": 0.08127595484256744, "beta_low": 0.14190673828125, "beta_high": 0.0, "orthogonal_to_active": 0.3000735938549042, "evidence_weights": [-0.10355666279792786, -0.1858898252248764, -0.15953125059604645], "agreement_weight": -0.002475873799994588, "orthogonal_weight": 1.8537641763687134}]}, {"layer": 9, "memory_to_active": 0.9340654015541077, "contribution_to_active": 0.6730961799621582, "mix_mean": 0.3413160443305969, "mix_low": 0.12719187140464783, "mix_high": 0.005016803741455078, "read_cosine": 0.34959876537323, "beta_mean": 0.05073117837309837, "nonfinite": 0.0, "weighted_memory_to_active": 0.3845409154891968, "mix_std": 0.21719859540462494, "beta_std": 0.03746724873781204, "beta_low": 0.5126749873161316, "beta_high": 0.0, "orthogonal_to_active": 0.03545895591378212, "output_influence": 0.5134058594703674, "residual_influence": 0.1452200561761856, "state_active": 0.04236139357089996, "state_memory": 0.017510902136564255, "state_peak": 49.27664566040039, "growth_active": 1.204277753829956, "growth_memory": 1.3690898418426514, "heads": [{"head": 0, "memory_to_active": 0.0014096432132646441, "contribution_to_active": 0.5531368851661682, "mix_mean": 0.630227267742157, "mix_low": 0.0014057159423828125, "mix_high": 0.021678924560546875, "read_cosine": 0.27172043919563293, "beta_mean": 4.631434421753511e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0009114197455346584, "mix_std": 0.2908502519130707, "beta_std": 2.079606565530412e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009156572050414979, "evidence_weights": [-0.3006536364555359, -0.3090989291667938, 0.746469259262085], "agreement_weight": 0.11271274089813232, "orthogonal_weight": -1.0403138399124146}, {"head": 1, "memory_to_active": 0.02795199118554592, "contribution_to_active": 0.6752519011497498, "mix_mean": 0.5437291860580444, "mix_low": 0.0075531005859375, "mix_high": 0.0021610260009765625, "read_cosine": 0.4025067090988159, "beta_mean": 7.412418199237436e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.020066386088728905, "mix_std": 0.286526083946228, "beta_std": 0.0001813110284274444, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014009138569235802, "evidence_weights": [-0.2363177388906479, -0.28733494877815247, 1.1359031200408936], "agreement_weight": 0.4344322681427002, "orthogonal_weight": -0.869716227054596}, {"head": 2, "memory_to_active": 0.16176864504814148, "contribution_to_active": 0.645563006401062, "mix_mean": 0.5883175730705261, "mix_low": 0.0014677047729492188, "mix_high": 0.0052280426025390625, "read_cosine": 0.3564412593841553, "beta_mean": 0.000604685686994344, "nonfinite": 0.0, "weighted_memory_to_active": 0.1059284433722496, "mix_std": 0.2689681053161621, "beta_std": 0.003614824963733554, "beta_low": 0.9962158203125, "beta_high": 0.0, "orthogonal_to_active": 0.06047496199607849, "evidence_weights": [-0.19857613742351532, -0.2731231153011322, 0.7405806183815002], "agreement_weight": 0.3011592626571655, "orthogonal_weight": -0.6616502404212952}, {"head": 3, "memory_to_active": 1.2243810892105103, "contribution_to_active": 0.15608960390090942, "mix_mean": 0.053335994482040405, "mix_low": 0.33648681640625, "mix_high": 9.918212890625e-05, "read_cosine": 0.38915586471557617, "beta_mean": 0.05787012353539467, "nonfinite": 0.0, "weighted_memory_to_active": 0.13022013008594513, "mix_std": 0.10640230774879456, "beta_std": 0.04429047182202339, "beta_low": 0.007568359375, "beta_high": 0.0, "orthogonal_to_active": 0.0072494144551455975, "evidence_weights": [0.07373373955488205, 0.004394324496388435, 0.0457628071308136], "agreement_weight": 0.09367882460355759, "orthogonal_weight": -0.05931545048952103}, {"head": 4, "memory_to_active": 2.8268749713897705, "contribution_to_active": 1.5483450889587402, "mix_mean": 0.1397029012441635, "mix_low": 0.13736915588378906, "mix_high": 0.0007686614990234375, "read_cosine": 0.1257239133119583, "beta_mean": 0.16896414756774902, "nonfinite": 0.0, "weighted_memory_to_active": 1.5171074867248535, "mix_std": 0.19819366931915283, "beta_std": 0.09397080540657043, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03413810580968857, "evidence_weights": [-0.07581481337547302, -0.15775233507156372, -0.047069791704416275], "agreement_weight": -0.05848885327577591, "orthogonal_weight": -0.023314759135246277}, {"head": 5, "memory_to_active": 1.3620060682296753, "contribution_to_active": 0.46018993854522705, "mix_mean": 0.09258309006690979, "mix_low": 0.2788686752319336, "mix_high": 0.00016498565673828125, "read_cosine": 0.5520442724227905, "beta_mean": 0.07686933875083923, "nonfinite": 0.0, "weighted_memory_to_active": 0.5330115556716919, "mix_std": 0.15225112438201904, "beta_std": 0.08272526413202286, "beta_low": 0.072265625, "beta_high": 0.0, "orthogonal_to_active": 0.09596644341945648, "evidence_weights": [0.1066603735089302, -0.10205310583114624, 0.1338256448507309], "agreement_weight": 0.01725633814930916, "orthogonal_weight": 0.2738608121871948}]}, {"layer": 10, "memory_to_active": 0.20850393176078796, "contribution_to_active": 0.6563128232955933, "mix_mean": 0.5201014280319214, "mix_low": 0.017124971374869347, "mix_high": 0.007333914749324322, "read_cosine": 0.3799387812614441, "beta_mean": 0.044677309691905975, "nonfinite": 0.0, "weighted_memory_to_active": 0.09697447717189789, "mix_std": 0.27144792675971985, "beta_std": 0.04705890640616417, "beta_low": 0.6610006093978882, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.19123485684394836, "output_influence": 0.44793665409088135, "residual_influence": 0.15523743629455566, "state_active": 0.05406762287020683, "state_memory": 0.016344940289855003, "state_peak": 127.25499725341797, "growth_active": 1.0212477445602417, "growth_memory": 0.9802054762840271, "heads": [{"head": 0, "memory_to_active": 0.7327990531921387, "contribution_to_active": 1.6649165153503418, "mix_mean": 0.6003342270851135, "mix_low": 0.06595611572265625, "mix_high": 0.023932456970214844, "read_cosine": 0.11915741860866547, "beta_mean": 0.20412296056747437, "nonfinite": 0.0, "weighted_memory_to_active": 0.45773211121559143, "mix_std": 0.3951743543148041, "beta_std": 0.2156614512205124, "beta_low": 0.114013671875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.0656371116638184, "evidence_weights": [-0.11098439991474152, -0.39222389459609985, -1.5320959091186523], "agreement_weight": 0.028580453246831894, "orthogonal_weight": 2.351810932159424}, {"head": 1, "memory_to_active": 0.009938305243849754, "contribution_to_active": 0.601140022277832, "mix_mean": 0.6178619861602783, "mix_low": 0.001125335693359375, "mix_high": 0.013764381408691406, "read_cosine": 0.2889334559440613, "beta_mean": 6.150296394480392e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005555403884500265, "mix_std": 0.2661866843700409, "beta_std": 8.437519136350602e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008354809135198593, "evidence_weights": [-0.25968122482299805, -0.20191845297813416, 0.5723554491996765], "agreement_weight": 0.5117396116256714, "orthogonal_weight": -1.6262578964233398}, {"head": 2, "memory_to_active": 0.012495579198002815, "contribution_to_active": 0.6111792325973511, "mix_mean": 0.6408891677856445, "mix_low": 0.000274658203125, "mix_high": 0.0050945281982421875, "read_cosine": 0.009274247102439404, "beta_mean": 2.5104258384089917e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005355182569473982, "mix_std": 0.25433939695358276, "beta_std": 0.0001311264350079, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0024555965792387724, "evidence_weights": [-0.2526765763759613, -0.282135933637619, 0.6137787699699402], "agreement_weight": -0.19359081983566284, "orthogonal_weight": -0.43894293904304504}, {"head": 3, "memory_to_active": 0.3479563593864441, "contribution_to_active": 0.10133983194828033, "mix_mean": 0.29666799306869507, "mix_low": 0.03288745880126953, "mix_high": 0.0003757476806640625, "read_cosine": 0.9181448817253113, "beta_mean": 0.057614926248788834, "nonfinite": 0.0, "weighted_memory_to_active": 0.04003111645579338, "mix_std": 0.23425304889678955, "beta_std": 0.05867470055818558, "beta_low": 0.0081787109375, "beta_high": 0.0, "orthogonal_to_active": 0.041282523423433304, "evidence_weights": [-0.10406027734279633, -0.1454514116048813, 0.15851078927516937], "agreement_weight": 0.01883522979915142, "orthogonal_weight": 1.433051586151123}, {"head": 4, "memory_to_active": 0.013812299817800522, "contribution_to_active": 0.5447241067886353, "mix_mean": 0.5689942836761475, "mix_low": 0.0004634857177734375, "mix_high": 0.0008115768432617188, "read_cosine": 0.08467642962932587, "beta_mean": 4.6613749873358756e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007848504930734634, "mix_std": 0.2485995590686798, "beta_std": 9.363620483782142e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009634159505367279, "evidence_weights": [-0.18324986100196838, -0.26120537519454956, 0.9223695397377014], "agreement_weight": 0.14523041248321533, "orthogonal_weight": -1.2988955974578857}, {"head": 5, "memory_to_active": 0.13402201235294342, "contribution_to_active": 0.4145771265029907, "mix_mean": 0.3958606719970703, "mix_low": 0.0020427703857421875, "mix_high": 2.47955322265625e-05, "read_cosine": 0.8594461679458618, "beta_mean": 0.006192747037857771, "nonfinite": 0.0, "weighted_memory_to_active": 0.06532452255487442, "mix_std": 0.23013442754745483, "beta_std": 0.007708162069320679, "beta_low": 0.8438720703125, "beta_high": 0.0, "orthogonal_to_active": 0.02004491351544857, "evidence_weights": [-0.14205463230609894, -0.16013427078723907, 0.27344822883605957], "agreement_weight": 0.09050114452838898, "orthogonal_weight": -0.6442855596542358}]}, {"layer": 11, "memory_to_active": 0.09540554881095886, "contribution_to_active": 0.6055359840393066, "mix_mean": 0.4971546530723572, "mix_low": 0.1007644385099411, "mix_high": 0.045493289828300476, "read_cosine": 0.3656824231147766, "beta_mean": 0.02257545478641987, "nonfinite": 0.0, "weighted_memory_to_active": 0.0481598936021328, "mix_std": 0.32334139943122864, "beta_std": 0.03372516483068466, "beta_low": 0.6636962890625, "beta_high": 0.0, "orthogonal_to_active": 0.04639885574579239, "output_influence": 0.535495936870575, "residual_influence": 0.1372237354516983, "state_active": 0.19264501333236694, "state_memory": 0.004607123322784901, "state_peak": 32.6075553894043, "growth_active": 1.096727728843689, "growth_memory": 1.6013227701187134, "heads": [{"head": 0, "memory_to_active": 0.007466095499694347, "contribution_to_active": 0.5977157354354858, "mix_mean": 0.6316083073616028, "mix_low": 0.0011386871337890625, "mix_high": 0.007412910461425781, "read_cosine": 0.30217549204826355, "beta_mean": 8.016528590815142e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00510714715346694, "mix_std": 0.26050660014152527, "beta_std": 0.0035991566255688667, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.0038886209949851036, "evidence_weights": [-0.16911303997039795, -0.3705267608165741, 0.7697572112083435], "agreement_weight": 0.3143714368343353, "orthogonal_weight": -0.7985985279083252}, {"head": 1, "memory_to_active": 0.21158868074417114, "contribution_to_active": 0.387205570936203, "mix_mean": 0.4214218854904175, "mix_low": 0.0054607391357421875, "mix_high": 0.00032138824462890625, "read_cosine": 0.5646214485168457, "beta_mean": 0.013255422003567219, "nonfinite": 0.0, "weighted_memory_to_active": 0.0962996631860733, "mix_std": 0.26232701539993286, "beta_std": 0.028629153966903687, "beta_low": 0.70172119140625, "beta_high": 0.0, "orthogonal_to_active": 0.04076874628663063, "evidence_weights": [-0.16705679893493652, -0.2224445790052414, 0.5743314623832703], "agreement_weight": 0.07535156607627869, "orthogonal_weight": -0.5353769063949585}, {"head": 2, "memory_to_active": 0.007772747427225113, "contribution_to_active": 0.9812238216400146, "mix_mean": 0.44137004017829895, "mix_low": 0.26305484771728516, "mix_high": 0.17672348022460938, "read_cosine": 0.19509373605251312, "beta_mean": 0.055854544043540955, "nonfinite": 0.0, "weighted_memory_to_active": 0.005639711394906044, "mix_std": 0.44677960872650146, "beta_std": 0.07611330598592758, "beta_low": 0.48333740234375, "beta_high": 0.0, "orthogonal_to_active": 0.04354849085211754, "evidence_weights": [0.030915681272745132, -0.4176509976387024, -1.3060672283172607], "agreement_weight": 0.0003326025325804949, "orthogonal_weight": 8.015475273132324}, {"head": 3, "memory_to_active": 0.009548082947731018, "contribution_to_active": 0.8445913195610046, "mix_mean": 0.4056563377380371, "mix_low": 0.3239269256591797, "mix_high": 0.08374214172363281, "read_cosine": 0.5338118076324463, "beta_mean": 0.04387202858924866, "nonfinite": 0.0, "weighted_memory_to_active": 0.005816039629280567, "mix_std": 0.43758782744407654, "beta_std": 0.06642812490463257, "beta_low": 0.51416015625, "beta_high": 0.0, "orthogonal_to_active": 0.03513175621628761, "evidence_weights": [0.17521575093269348, -0.3141421973705292, -0.2020667940378189], "agreement_weight": 0.05159936472773552, "orthogonal_weight": 8.33381462097168}, {"head": 4, "memory_to_active": 0.25024282932281494, "contribution_to_active": 0.4402434229850769, "mix_mean": 0.5497350096702576, "mix_low": 0.008439064025878906, "mix_high": 0.0009403228759765625, "read_cosine": 0.0477481335401535, "beta_mean": 0.01409350149333477, "nonfinite": 0.0, "weighted_memory_to_active": 0.13321368396282196, "mix_std": 0.27268874645233154, "beta_std": 0.01523070503026247, "beta_low": 0.53033447265625, "beta_high": 0.0, "orthogonal_to_active": 0.12940713763237, "evidence_weights": [-0.2431749403476715, -0.2317012995481491, 1.9672777652740479], "agreement_weight": -0.19679106771945953, "orthogonal_weight": -0.9873098134994507}, {"head": 5, "memory_to_active": 0.08581489324569702, "contribution_to_active": 0.38223621249198914, "mix_mean": 0.5331360101699829, "mix_low": 0.0025663375854492188, "mix_high": 0.0038194656372070312, "read_cosine": 0.550644040107727, "beta_mean": 0.008297058753669262, "nonfinite": 0.0, "weighted_memory_to_active": 0.04288310557603836, "mix_std": 0.26015838980674744, "beta_std": 0.012350532226264477, "beta_low": 0.75286865234375, "beta_high": 0.0, "orthogonal_to_active": 0.025648361071944237, "evidence_weights": [-0.21354003250598907, -0.30866509675979614, 0.6475427150726318], "agreement_weight": 0.08261427283287048, "orthogonal_weight": -0.7564937472343445}]}, {"layer": 12, "memory_to_active": 0.23751932382583618, "contribution_to_active": 0.5731865167617798, "mix_mean": 0.5310360193252563, "mix_low": 0.008805434219539165, "mix_high": 0.005610784515738487, "read_cosine": 0.2243032455444336, "beta_mean": 0.012920058332383633, "nonfinite": 0.0, "weighted_memory_to_active": 0.12745270133018494, "mix_std": 0.26372456550598145, "beta_std": 0.014811289496719837, "beta_low": 0.8268331289291382, "beta_high": 0.0, "orthogonal_to_active": 0.0706631988286972, "output_influence": 0.48539358377456665, "residual_influence": 0.13489678502082825, "state_active": 0.03437424823641777, "state_memory": 0.008300626650452614, "state_peak": 48.316986083984375, "growth_active": 1.0937050580978394, "growth_memory": 1.1573437452316284, "heads": [{"head": 0, "memory_to_active": 0.04435032606124878, "contribution_to_active": 0.5746675133705139, "mix_mean": 0.5479797124862671, "mix_low": 0.0004730224609375, "mix_high": 0.0035247802734375, "read_cosine": 0.2757622003555298, "beta_mean": 0.0006681218510493636, "nonfinite": 0.0, "weighted_memory_to_active": 0.022929823026061058, "mix_std": 0.27008405327796936, "beta_std": 0.0013843434862792492, "beta_low": 0.9957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.02027944102883339, "evidence_weights": [-0.2054685354232788, -0.2756204307079315, 0.13989043235778809], "agreement_weight": 0.09803223609924316, "orthogonal_weight": -0.9494493007659912}, {"head": 1, "memory_to_active": 0.0907304659485817, "contribution_to_active": 0.5309208035469055, "mix_mean": 0.5427942276000977, "mix_low": 0.0007333755493164062, "mix_high": 0.0013904571533203125, "read_cosine": 0.43401551246643066, "beta_mean": 0.0007743973983451724, "nonfinite": 0.0, "weighted_memory_to_active": 0.05262446776032448, "mix_std": 0.2662128508090973, "beta_std": 0.007617124356329441, "beta_low": 0.9947509765625, "beta_high": 0.0, "orthogonal_to_active": 0.04379553720355034, "evidence_weights": [-0.2050754874944687, -0.27148106694221497, 1.023391604423523], "agreement_weight": 0.19238696992397308, "orthogonal_weight": -0.9320639967918396}, {"head": 2, "memory_to_active": 0.5724953413009644, "contribution_to_active": 0.4971528649330139, "mix_mean": 0.3395230770111084, "mix_low": 0.02476978302001953, "mix_high": 0.00045013427734375, "read_cosine": 0.4660123586654663, "beta_mean": 0.07440336048603058, "nonfinite": 0.0, "weighted_memory_to_active": 0.23678037524223328, "mix_std": 0.24695110321044922, "beta_std": 0.05831916257739067, "beta_low": 0.0037841796875, "beta_high": 0.0, "orthogonal_to_active": 0.2525325119495392, "evidence_weights": [-0.2298566997051239, -0.05492653325200081, -0.01901816949248314], "agreement_weight": 0.024568350985646248, "orthogonal_weight": 1.3047622442245483}, {"head": 3, "memory_to_active": 0.0326324962079525, "contribution_to_active": 0.7480334043502808, "mix_mean": 0.7096307277679443, "mix_low": 0.00019931793212890625, "mix_high": 0.018027305603027344, "read_cosine": 0.09672269225120544, "beta_mean": 0.0002928226604126394, "nonfinite": 0.0, "weighted_memory_to_active": 0.02426474541425705, "mix_std": 0.23426245152950287, "beta_std": 0.004466953221708536, "beta_low": 0.9970703125, "beta_high": 0.0, "orthogonal_to_active": 0.023158427327871323, "evidence_weights": [-0.292440801858902, -0.34199750423431396, 1.522817850112915], "agreement_weight": 0.47159644961357117, "orthogonal_weight": -0.9749119281768799}, {"head": 4, "memory_to_active": 0.5995462536811829, "contribution_to_active": 0.5991849303245544, "mix_mean": 0.4178628623485565, "mix_low": 0.019290924072265625, "mix_high": 0.004092216491699219, "read_cosine": -0.12125076353549957, "beta_mean": 0.0009300607489421964, "nonfinite": 0.0, "weighted_memory_to_active": 0.37250402569770813, "mix_std": 0.28816965222358704, "beta_std": 0.011282353661954403, "beta_low": 0.978271484375, "beta_high": 0.0, "orthogonal_to_active": 0.0412338487803936, "evidence_weights": [-0.14277751743793488, -0.21711337566375732, -0.040708787739276886], "agreement_weight": -0.0025451118126511574, "orthogonal_weight": -0.11541992425918579}, {"head": 5, "memory_to_active": 0.08536101877689362, "contribution_to_active": 0.48915958404541016, "mix_mean": 0.6284253597259521, "mix_low": 0.007366180419921875, "mix_high": 0.0061798095703125, "read_cosine": 0.19455738365650177, "beta_mean": 0.0004515948239713907, "nonfinite": 0.0, "weighted_memory_to_active": 0.055612798780202866, "mix_std": 0.27666717767715454, "beta_std": 0.005797797814011574, "beta_low": 0.99139404296875, "beta_high": 0.0, "orthogonal_to_active": 0.0429794117808342, "evidence_weights": [-0.24140217900276184, -0.3176577687263489, 1.3191276788711548], "agreement_weight": 0.3749539852142334, "orthogonal_weight": -0.8277335166931152}]}, {"layer": 13, "memory_to_active": 0.12026316672563553, "contribution_to_active": 0.5367880463600159, "mix_mean": 0.5315226316452026, "mix_low": 0.005452632904052734, "mix_high": 0.007912000641226768, "read_cosine": 0.4071357250213623, "beta_mean": 0.014564136043190956, "nonfinite": 0.0, "weighted_memory_to_active": 0.05328356474637985, "mix_std": 0.2619555592536926, "beta_std": 0.01754281483590603, "beta_low": 0.8044230341911316, "beta_high": 0.0, "orthogonal_to_active": 0.05479040741920471, "output_influence": 0.507881224155426, "residual_influence": 0.17164964973926544, "state_active": 0.11561031639575958, "state_memory": 0.0396113358438015, "state_peak": 94.2488021850586, "growth_active": 0.9168993830680847, "growth_memory": 0.9717170000076294, "heads": [{"head": 0, "memory_to_active": 0.017045387998223305, "contribution_to_active": 0.5810134410858154, "mix_mean": 0.5998767614364624, "mix_low": 0.0005159378051757812, "mix_high": 0.004109382629394531, "read_cosine": 0.4405105710029602, "beta_mean": 6.201335781952366e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010806011036038399, "mix_std": 0.269357830286026, "beta_std": 0.00012321563553996384, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010772749781608582, "evidence_weights": [-0.19917447865009308, -0.26742687821388245, 0.7675394415855408], "agreement_weight": 0.13719689846038818, "orthogonal_weight": -1.1183927059173584}, {"head": 1, "memory_to_active": 0.016423435881733894, "contribution_to_active": 0.5872705578804016, "mix_mean": 0.5529836416244507, "mix_low": 0.0018711090087890625, "mix_high": 0.003124237060546875, "read_cosine": 0.2005210816860199, "beta_mean": 6.520721944980323e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009216037578880787, "mix_std": 0.2767985165119171, "beta_std": 0.00011036251089535654, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01189032755792141, "evidence_weights": [-0.1833365112543106, -0.26529884338378906, 0.6650365591049194], "agreement_weight": 0.40445905923843384, "orthogonal_weight": -1.346251368522644}, {"head": 2, "memory_to_active": 0.07705827057361603, "contribution_to_active": 0.4771236181259155, "mix_mean": 0.3857211172580719, "mix_low": 0.0017118453979492188, "mix_high": 0.00016689300537109375, "read_cosine": 0.628326416015625, "beta_mean": 0.012098833918571472, "nonfinite": 0.0, "weighted_memory_to_active": 0.03624889254570007, "mix_std": 0.23243404924869537, "beta_std": 0.03876475989818573, "beta_low": 0.83056640625, "beta_high": 0.0, "orthogonal_to_active": 0.01996762305498123, "evidence_weights": [-0.1875249594449997, -0.20422929525375366, 0.4103284776210785], "agreement_weight": 0.008607568219304085, "orthogonal_weight": -0.8357180953025818}, {"head": 3, "memory_to_active": 0.15844780206680298, "contribution_to_active": 0.6024966239929199, "mix_mean": 0.6429682970046997, "mix_low": 0.021226882934570312, "mix_high": 0.028779983520507812, "read_cosine": -0.009820476174354553, "beta_mean": 0.0008644354529678822, "nonfinite": 0.0, "weighted_memory_to_active": 0.1168554499745369, "mix_std": 0.2916717529296875, "beta_std": 0.009296112693846226, "beta_low": 0.9962158203125, "beta_high": 0.0, "orthogonal_to_active": 0.10400819033384323, "evidence_weights": [-0.317626416683197, -0.24550668895244598, 1.7616082429885864], "agreement_weight": 5.2075629234313965, "orthogonal_weight": -0.9792971014976501}, {"head": 4, "memory_to_active": 0.4001961350440979, "contribution_to_active": 0.26928403973579407, "mix_mean": 0.36035454273223877, "mix_low": 0.004954338073730469, "mix_high": 0.00016689300537109375, "read_cosine": 0.8835716247558594, "beta_mean": 0.0740431696176529, "nonfinite": 0.0, "weighted_memory_to_active": 0.11071210354566574, "mix_std": 0.22607819736003876, "beta_std": 0.05429649353027344, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.15219172835350037, "evidence_weights": [-0.09386806190013885, -0.233533576130867, 0.4747845530509949], "agreement_weight": -0.0030567673966288567, "orthogonal_weight": 2.2199602127075195}, {"head": 5, "memory_to_active": 0.05240796506404877, "contribution_to_active": 0.7035400867462158, "mix_mean": 0.6472313404083252, "mix_low": 0.0024356842041015625, "mix_high": 0.011124610900878906, "read_cosine": 0.2997051179409027, "beta_mean": 0.0002511505736038089, "nonfinite": 0.0, "weighted_memory_to_active": 0.03586290031671524, "mix_std": 0.2753930985927582, "beta_std": 0.002665941836312413, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.029911790043115616, "evidence_weights": [-0.2223692387342453, -0.30629903078079224, 0.867102324962616], "agreement_weight": 0.1624358892440796, "orthogonal_weight": -0.9155586361885071}]}, {"layer": 14, "memory_to_active": 0.24231475591659546, "contribution_to_active": 0.4161149561405182, "mix_mean": 0.4260817766189575, "mix_low": 0.026803812012076378, "mix_high": 0.005831082817167044, "read_cosine": 0.6101929545402527, "beta_mean": 0.05466994270682335, "nonfinite": 0.0, "weighted_memory_to_active": 0.10217414796352386, "mix_std": 0.24882465600967407, "beta_std": 0.0563293993473053, "beta_low": 0.4884440302848816, "beta_high": 0.0, "orthogonal_to_active": 0.042716801166534424, "output_influence": 0.35600513219833374, "residual_influence": 0.1355590522289276, "state_active": 0.057266756892204285, "state_memory": 0.014989206567406654, "state_peak": 42.25604248046875, "growth_active": 1.075744867324829, "growth_memory": 1.2116080522537231, "heads": [{"head": 0, "memory_to_active": 0.36344578862190247, "contribution_to_active": 0.18603219091892242, "mix_mean": 0.20245158672332764, "mix_low": 0.01610851287841797, "mix_high": 5.53131103515625e-05, "read_cosine": 0.8022314310073853, "beta_mean": 0.06536266952753067, "nonfinite": 0.0, "weighted_memory_to_active": 0.07916149497032166, "mix_std": 0.18758080899715424, "beta_std": 0.0628649890422821, "beta_low": 0.0174560546875, "beta_high": 0.0, "orthogonal_to_active": 0.07980801165103912, "evidence_weights": [-0.12120384722948074, -0.0368872806429863, 0.44270989298820496], "agreement_weight": 0.02249976061284542, "orthogonal_weight": 1.5893573760986328}, {"head": 1, "memory_to_active": 0.512834906578064, "contribution_to_active": 0.28699183464050293, "mix_mean": 0.22939816117286682, "mix_low": 0.128875732421875, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8431762456893921, "beta_mean": 0.20791177451610565, "nonfinite": 0.0, "weighted_memory_to_active": 0.22614891827106476, "mix_std": 0.2595711052417755, "beta_std": 0.22154682874679565, "beta_low": 0.1279296875, "beta_high": 0.0, "orthogonal_to_active": 0.014533756300807, "evidence_weights": [-0.011221586726605892, -0.20648585259914398, -0.1165662482380867], "agreement_weight": 0.05538777634501457, "orthogonal_weight": 0.1553625762462616}, {"head": 2, "memory_to_active": 0.13737080991268158, "contribution_to_active": 0.573930025100708, "mix_mean": 0.578700602054596, "mix_low": 0.0026111602783203125, "mix_high": 0.0032052993774414062, "read_cosine": 0.41686323285102844, "beta_mean": 0.0020449915900826454, "nonfinite": 0.0, "weighted_memory_to_active": 0.08998134732246399, "mix_std": 0.2680130898952484, "beta_std": 0.002721719676628709, "beta_low": 0.98150634765625, "beta_high": 0.0, "orthogonal_to_active": 0.06816539168357849, "evidence_weights": [-0.14109103381633759, -0.3131231367588043, 0.8924287557601929], "agreement_weight": 0.08214250952005386, "orthogonal_weight": -0.8669149279594421}, {"head": 3, "memory_to_active": 0.053342945873737335, "contribution_to_active": 0.621931254863739, "mix_mean": 0.6694798469543457, "mix_low": 0.005236625671386719, "mix_high": 0.020623207092285156, "read_cosine": -0.013079933822154999, "beta_mean": 0.007447430863976479, "nonfinite": 0.0, "weighted_memory_to_active": 0.03368896618485451, "mix_std": 0.2576422095298767, "beta_std": 0.00862216204404831, "beta_low": 0.76312255859375, "beta_high": 0.0, "orthogonal_to_active": 0.024489132687449455, "evidence_weights": [-0.351655513048172, -0.3119535446166992, 0.2994951903820038], "agreement_weight": 0.0777812972664833, "orthogonal_weight": -0.8058739900588989}, {"head": 4, "memory_to_active": 0.1436370611190796, "contribution_to_active": 0.493490606546402, "mix_mean": 0.5039961934089661, "mix_low": 0.004471778869628906, "mix_high": 0.008977890014648438, "read_cosine": 0.872236967086792, "beta_mean": 0.04351870343089104, "nonfinite": 0.0, "weighted_memory_to_active": 0.08377458900213242, "mix_std": 0.26465725898742676, "beta_std": 0.0313001424074173, "beta_low": 0.06707763671875, "beta_high": 0.0, "orthogonal_to_active": 0.023575924336910248, "evidence_weights": [-0.21920403838157654, -0.2465703785419464, 0.4294985234737396], "agreement_weight": 0.12949612736701965, "orthogonal_weight": -0.5960801839828491}, {"head": 5, "memory_to_active": 0.24325698614120483, "contribution_to_active": 0.33431389927864075, "mix_mean": 0.3724643588066101, "mix_low": 0.0035190582275390625, "mix_high": 0.0021190643310546875, "read_cosine": 0.7397298216819763, "beta_mean": 0.001734076882712543, "nonfinite": 0.0, "weighted_memory_to_active": 0.10028951615095139, "mix_std": 0.2554834187030792, "beta_std": 0.01092054694890976, "beta_low": 0.97357177734375, "beta_high": 0.0, "orthogonal_to_active": 0.04572856053709984, "evidence_weights": [-0.17775340378284454, -0.17092016339302063, 0.28299275040626526], "agreement_weight": 0.07904322445392609, "orthogonal_weight": -0.7197083830833435}]}, {"layer": 15, "memory_to_active": 0.05793902277946472, "contribution_to_active": 0.48474395275115967, "mix_mean": 0.41209888458251953, "mix_low": 0.009104093536734581, "mix_high": 0.0023825964890420437, "read_cosine": 0.6674282550811768, "beta_mean": 0.03725508600473404, "nonfinite": 0.0, "weighted_memory_to_active": 0.02737482450902462, "mix_std": 0.2838221490383148, "beta_std": 0.04421696811914444, "beta_low": 0.7144674062728882, "beta_high": 0.0, "orthogonal_to_active": 0.007247858680784702, "output_influence": 0.30573567748069763, "residual_influence": 0.16936200857162476, "state_active": 0.14298301935195923, "state_memory": 0.010637998580932617, "state_peak": 35.50688552856445, "growth_active": 1.2418371438980103, "growth_memory": 1.460693359375, "heads": [{"head": 0, "memory_to_active": 0.10388867557048798, "contribution_to_active": 0.4869692325592041, "mix_mean": 0.3754642605781555, "mix_low": 0.006730079650878906, "mix_high": 0.005421638488769531, "read_cosine": 0.6319800019264221, "beta_mean": 0.03308047726750374, "nonfinite": 0.0, "weighted_memory_to_active": 0.047369956970214844, "mix_std": 0.2788974642753601, "beta_std": 0.026852738112211227, "beta_low": 0.12994384765625, "beta_high": 0.0, "orthogonal_to_active": 0.023728949949145317, "evidence_weights": [-0.1874132752418518, -0.18933936953544617, 0.4182387590408325], "agreement_weight": 0.09215935319662094, "orthogonal_weight": -0.744213342666626}, {"head": 1, "memory_to_active": 0.13887089490890503, "contribution_to_active": 0.40716540813446045, "mix_mean": 0.42557424306869507, "mix_low": 0.0018625259399414062, "mix_high": 0.004014015197753906, "read_cosine": 0.924375057220459, "beta_mean": 0.0008902702247723937, "nonfinite": 0.0, "weighted_memory_to_active": 0.05791419371962547, "mix_std": 0.28408581018447876, "beta_std": 0.0028195383492857218, "beta_low": 0.98907470703125, "beta_high": 0.0, "orthogonal_to_active": 0.0009407253237441182, "evidence_weights": [-0.14999715983867645, -0.1597071886062622, 0.19283343851566315], "agreement_weight": 0.12415174394845963, "orthogonal_weight": 0.03293633460998535}, {"head": 2, "memory_to_active": 0.008911201730370522, "contribution_to_active": 0.4831213057041168, "mix_mean": 0.3398740589618683, "mix_low": 0.016074180603027344, "mix_high": 0.000133514404296875, "read_cosine": 0.7994089126586914, "beta_mean": 0.0001307129714405164, "nonfinite": 0.0, "weighted_memory_to_active": 0.004510004539042711, "mix_std": 0.271428644657135, "beta_std": 0.0003152699500788003, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0011654943227767944, "evidence_weights": [-0.20895618200302124, -0.2712189555168152, 0.15486609935760498], "agreement_weight": 0.012205063365399837, "orthogonal_weight": 0.6726181507110596}, {"head": 3, "memory_to_active": 0.0028198249638080597, "contribution_to_active": 0.379728764295578, "mix_mean": 0.39601069688796997, "mix_low": 0.0057373046875, "mix_high": 0.00040721893310546875, "read_cosine": 0.637784481048584, "beta_mean": 1.630702354304958e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0009756202343851328, "mix_std": 0.2865014672279358, "beta_std": 4.891786375083029e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007181565160863101, "evidence_weights": [-0.249790757894516, -0.24741783738136292, 0.20663321018218994], "agreement_weight": 0.054301295429468155, "orthogonal_weight": -0.7580968737602234}, {"head": 4, "memory_to_active": 0.06789194792509079, "contribution_to_active": 0.6344552040100098, "mix_mean": 0.3751397728919983, "mix_low": 0.013728141784667969, "mix_high": 0.0011396408081054688, "read_cosine": 0.6748418807983398, "beta_mean": 0.18767105042934418, "nonfinite": 0.0, "weighted_memory_to_active": 0.037284351885318756, "mix_std": 0.30656546354293823, "beta_std": 0.22333437204360962, "beta_low": 0.195556640625, "beta_high": 0.0, "orthogonal_to_active": 0.0067522525787353516, "evidence_weights": [-0.2145703136920929, -0.25528281927108765, -0.024446887895464897], "agreement_weight": 0.01457239780575037, "orthogonal_weight": 0.3451237082481384}, {"head": 5, "memory_to_active": 0.025251604616642, "contribution_to_active": 0.5170236825942993, "mix_mean": 0.5605302453041077, "mix_low": 0.010492324829101562, "mix_high": 0.0031795501708984375, "read_cosine": 0.3361789286136627, "beta_mean": 0.001741686137393117, "nonfinite": 0.0, "weighted_memory_to_active": 0.01619480550289154, "mix_std": 0.2754540741443634, "beta_std": 0.011930973269045353, "beta_low": 0.9722900390625, "beta_high": 0.0, "orthogonal_to_active": 0.01018157321959734, "evidence_weights": [-0.2894267141819, -0.3053385615348816, 0.6990700364112854], "agreement_weight": 0.2871286869049072, "orthogonal_weight": 0.8025892972946167}]}]}, {"step": 49000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8791123289993266, "memory_to_active": 0.8131982684135437, "contribution_to_active": 0.5507321357727051, "mix_mean": 0.4022054076194763, "mix_low": 0.13666918873786926, "mix_high": 0.010207315906882286, "read_cosine": 0.3494802713394165, "beta_mean": 0.07191735506057739, "nonfinite": 0.0, "weighted_memory_to_active": 0.2055475413799286, "mix_std": 0.22145044803619385, "beta_std": 0.06276851892471313, "beta_low": 0.5509580373764038, "beta_high": 3.751119220396504e-05, "orthogonal_to_active": 0.06062670424580574, "samples": 4, "scored_tokens": 899, "normal_loss": 3.109217405319214, "normal_token_accuracy": 0.4271412789821625, "state_active": 0.06076040863990784, "state_memory": 0.01792106404900551, "state_peak": 120.79269409179688, "growth_active": 1.1099659204483032, "growth_memory": 1.3200081586837769, "output_influence": 0.4095522165298462, "residual_influence": 0.13673369586467743, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.155182123184204, "no_memory_loss_delta": 0.6080322265625, "scrambled_loss_delta": 0.1746489256620407, "memory_only_loss_delta": 3.2023322582244873, "no_evidence_loss_delta": 0.3612201511859894, "no_agreement_loss_delta": 0.03341224044561386, "no_orthogonal_loss_delta": 0.0176535751670599}, {"index": 94, "tokens": 256, "normal_loss": 3.2569580078125, "no_memory_loss_delta": 0.5957286357879639, "scrambled_loss_delta": 0.32681804895401, "memory_only_loss_delta": 3.44179368019104, "no_evidence_loss_delta": 0.2595807611942291, "no_agreement_loss_delta": 0.03725145757198334, "no_orthogonal_loss_delta": 0.058331314474344254}, {"index": 188, "tokens": 256, "normal_loss": 3.4761805534362793, "no_memory_loss_delta": 0.7842641472816467, "scrambled_loss_delta": 0.15690293908119202, "memory_only_loss_delta": 3.197371006011963, "no_evidence_loss_delta": 0.4187803268432617, "no_agreement_loss_delta": 0.05512985587120056, "no_orthogonal_loss_delta": 0.05021068453788757}, {"index": 283, "tokens": 131, "normal_loss": 2.0135583877563477, "no_memory_loss_delta": 0.22496116161346436, "scrambled_loss_delta": 0.17176753282546997, "memory_only_loss_delta": 2.6712236404418945, "no_evidence_loss_delta": 0.08861050009727478, "no_agreement_loss_delta": -0.017169862985610962, "no_orthogonal_loss_delta": 0.012762757949531078}], "agreement_weight_abs_mean": 0.19944703578948975, "agreement_weight_abs_max": 5.304490089416504, "agreement_weight_mean": 0.17529146373271942, "orthogonal_weight_abs_mean": 0.823998212814331, "orthogonal_weight_abs_max": 8.470503807067871, "orthogonal_weight_mean": -0.12316670268774033, "no_memory_loss": 3.7081096172332764, "no_memory_loss_delta": 0.5988925099372864, "no_memory_token_accuracy_gain": 0.0767519474029541, "no_memory_token_win_fraction": 0.6651835441589355, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 45.058022378028355, "scrambled_loss": 3.3217246532440186, "scrambled_loss_delta": 0.21250751614570618, "scrambled_token_accuracy_gain": 0.023359287530183792, "scrambled_token_win_fraction": 0.6329254508018494, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.144576091285852, "memory_only_loss": 6.300933837890625, "memory_only_loss_delta": 3.1917171478271484, "memory_only_token_accuracy_gain": 0.26362624764442444, "memory_only_token_win_fraction": 0.9121245741844177, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.88987665847628, "no_evidence_loss": 3.418161392211914, "no_evidence_loss_delta": 0.30894413590431213, "no_evidence_token_accuracy_gain": 0.02447163499891758, "no_evidence_token_win_fraction": 0.6095662117004395, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 26.577821448331168, "no_agreement_loss": 3.1425366401672363, "no_agreement_loss_delta": 0.033319130539894104, "no_agreement_token_accuracy_gain": 0.0, "no_agreement_token_win_fraction": 0.49388208985328674, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 3.27701622499543, "no_orthogonal_loss": 3.14701247215271, "no_orthogonal_loss_delta": 0.03779531642794609, "no_orthogonal_token_accuracy_gain": -0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5817574858665466, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 3.7089987417215475, "short_no_memory_loss_delta": 0.5143299698829651, "long_context_memory_gain": 0.08456254005432129, "normal_context_loss_gain": 0.15560752153396606, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.526742935180664, "contribution_to_active": 1.1572872400283813, "mix_mean": 0.10429443418979645, "mix_low": 0.5419796705245972, "mix_high": 0.005056063644587994, "read_cosine": 0.42350736260414124, "beta_mean": 0.2324124276638031, "nonfinite": 0.0, "weighted_memory_to_active": 1.1966125965118408, "mix_std": 0.08817595988512039, "beta_std": 0.14285269379615784, "beta_low": 0.04156494140625, "beta_high": 0.0, "orthogonal_to_active": 0.044457025825977325, "output_influence": 0.20300185680389404, "residual_influence": 0.2426203340291977, "state_active": 0.0668339878320694, "state_memory": 0.04526929929852486, "state_peak": 44.21054458618164, "growth_active": 1.2592873573303223, "growth_memory": 1.2721033096313477, "heads": [{"head": 0, "memory_to_active": 2.654350757598877, "contribution_to_active": 0.44721439480781555, "mix_mean": 0.018915195018053055, "mix_low": 0.6354236602783203, "mix_high": 0.0, "read_cosine": 0.23490124940872192, "beta_mean": 0.1998371183872223, "nonfinite": 0.0, "weighted_memory_to_active": 0.5587379932403564, "mix_std": 0.059089552611112595, "beta_std": 0.09558281302452087, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.053559064865112305, "evidence_weights": [-0.009316092357039452, -0.05341449752449989, -0.07604622840881348], "agreement_weight": -0.18540745973587036, "orthogonal_weight": -0.10777200758457184}, {"head": 1, "memory_to_active": 6.295459270477295, "contribution_to_active": 2.688157320022583, "mix_mean": 0.01854303851723671, "mix_low": 0.6212339401245117, "mix_high": 0.0, "read_cosine": 0.45613712072372437, "beta_mean": 0.45211878418922424, "nonfinite": 0.0, "weighted_memory_to_active": 2.9795968532562256, "mix_std": 0.055310823023319244, "beta_std": 0.23018111288547516, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013830376556143165, "evidence_weights": [0.024076158180832863, -0.10215383023023605, 0.5460054278373718], "agreement_weight": 0.03009517304599285, "orthogonal_weight": -0.0009917609859257936}, {"head": 2, "memory_to_active": 2.6561832427978516, "contribution_to_active": 0.4817774295806885, "mix_mean": 0.013337322510778904, "mix_low": 0.6920223236083984, "mix_high": 0.0, "read_cosine": 0.29030728340148926, "beta_mean": 0.2561616897583008, "nonfinite": 0.0, "weighted_memory_to_active": 0.5607524514198303, "mix_std": 0.031048456206917763, "beta_std": 0.17524154484272003, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.043676331639289856, "evidence_weights": [0.06333375722169876, 0.059520311653614044, 0.11136486381292343], "agreement_weight": 0.01100658904761076, "orthogonal_weight": -0.18376237154006958}, {"head": 3, "memory_to_active": 7.278952121734619, "contribution_to_active": 2.792840003967285, "mix_mean": 0.032289065420627594, "mix_low": 0.48647403717041016, "mix_high": 0.0, "read_cosine": 0.2815771698951721, "beta_mean": 0.3442603349685669, "nonfinite": 0.0, "weighted_memory_to_active": 2.9691240787506104, "mix_std": 0.07546544075012207, "beta_std": 0.2412448227405548, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.144197016954422, "evidence_weights": [-0.04272906482219696, -0.04719473421573639, -0.1566314697265625], "agreement_weight": -0.012555811554193497, "orthogonal_weight": -0.056389253586530685}, {"head": 4, "memory_to_active": 2.1626062393188477, "contribution_to_active": 0.07851345837116241, "mix_mean": 0.007288886234164238, "mix_low": 0.8015413284301758, "mix_high": 0.0, "read_cosine": 0.34056419134140015, "beta_mean": 0.1170525774359703, "nonfinite": 0.0, "weighted_memory_to_active": 0.0526723675429821, "mix_std": 0.01699981838464737, "beta_std": 0.09554818272590637, "beta_low": 0.01220703125, "beta_high": 0.0, "orthogonal_to_active": 0.0058173746801912785, "evidence_weights": [0.1501036286354065, 0.10860541462898254, -0.255694180727005], "agreement_weight": -0.14440584182739258, "orthogonal_weight": 0.0955728068947792}, {"head": 5, "memory_to_active": 0.11290545761585236, "contribution_to_active": 0.45522069931030273, "mix_mean": 0.5353930592536926, "mix_low": 0.0151824951171875, "mix_high": 0.030336380004882812, "read_cosine": 0.9375569224357605, "beta_mean": 0.02504408359527588, "nonfinite": 0.0, "weighted_memory_to_active": 0.05879213660955429, "mix_std": 0.29114168882369995, "beta_std": 0.019317639991641045, "beta_low": 0.2371826171875, "beta_high": 0.0, "orthogonal_to_active": 0.018109317868947983, "evidence_weights": [-0.1860787272453308, -0.219816654920578, 0.37155237793922424], "agreement_weight": 0.2902510166168213, "orthogonal_weight": -0.6856456995010376}]}, {"layer": 1, "memory_to_active": 2.302440881729126, "contribution_to_active": 0.4594433307647705, "mix_mean": 0.20359230041503906, "mix_low": 0.36807727813720703, "mix_high": 0.0008273124694824219, "read_cosine": 0.3089057505130768, "beta_mean": 0.17827370762825012, "nonfinite": 0.0, "weighted_memory_to_active": 0.29447558522224426, "mix_std": 0.11662005633115768, "beta_std": 0.12892268598079681, "beta_low": 0.3358663022518158, "beta_high": 0.00017293293785769492, "orthogonal_to_active": 0.04165869206190109, "output_influence": 0.2268950194120407, "residual_influence": 0.06045585125684738, "state_active": 0.030406765639781952, "state_memory": 0.035414502024650574, "state_peak": 39.22768020629883, "growth_active": 1.1856697797775269, "growth_memory": 1.4870920181274414, "heads": [{"head": 0, "memory_to_active": 0.04517284408211708, "contribution_to_active": 0.47001951932907104, "mix_mean": 0.4906828999519348, "mix_low": 0.0067653656005859375, "mix_high": 0.0012912750244140625, "read_cosine": 0.04908597469329834, "beta_mean": 0.0002942131250165403, "nonfinite": 0.0, "weighted_memory_to_active": 0.024965643882751465, "mix_std": 0.28151634335517883, "beta_std": 0.0004762948665302247, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.023958036676049232, "evidence_weights": [-0.20142483711242676, -0.26083335280418396, 0.2983432412147522], "agreement_weight": 0.17203757166862488, "orthogonal_weight": -1.0065654516220093}, {"head": 1, "memory_to_active": 2.909278392791748, "contribution_to_active": 0.25489047169685364, "mix_mean": 0.023851266130805016, "mix_low": 0.44452857971191406, "mix_high": 0.0, "read_cosine": 0.5186291933059692, "beta_mean": 0.2902030348777771, "nonfinite": 0.0, "weighted_memory_to_active": 0.2832794785499573, "mix_std": 0.04694001376628876, "beta_std": 0.18764647841453552, "beta_low": 0.000244140625, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.01883781887590885, "evidence_weights": [0.01669461652636528, -0.05043000355362892, 0.03093011863529682], "agreement_weight": -0.0052279517985880375, "orthogonal_weight": 0.1706317663192749}, {"head": 2, "memory_to_active": 0.08236657828092575, "contribution_to_active": 0.548969566822052, "mix_mean": 0.6462764143943787, "mix_low": 0.0005273818969726562, "mix_high": 0.0036668777465820312, "read_cosine": 0.5514448285102844, "beta_mean": 0.004200804512947798, "nonfinite": 0.0, "weighted_memory_to_active": 0.05793097987771034, "mix_std": 0.23631899058818817, "beta_std": 0.006062692496925592, "beta_low": 0.912353515625, "beta_high": 0.0, "orthogonal_to_active": 0.03554011508822441, "evidence_weights": [-0.2829292416572571, -0.3551434576511383, 0.6899850964546204], "agreement_weight": 0.03497644513845444, "orthogonal_weight": -0.7363177537918091}, {"head": 3, "memory_to_active": 2.763465166091919, "contribution_to_active": 0.1409308910369873, "mix_mean": 0.021685197949409485, "mix_low": 0.588104248046875, "mix_high": 3.814697265625e-06, "read_cosine": 0.21089908480644226, "beta_mean": 0.30585235357284546, "nonfinite": 0.0, "weighted_memory_to_active": 0.1774888038635254, "mix_std": 0.059696029871702194, "beta_std": 0.24822618067264557, "beta_low": 0.06103515625, "beta_high": 0.000244140625, "orthogonal_to_active": 0.06424836814403534, "evidence_weights": [0.04016556218266487, -0.012081271968781948, 0.0567348338663578], "agreement_weight": -0.02998809888958931, "orthogonal_weight": -0.45778408646583557}, {"head": 4, "memory_to_active": 3.688535690307617, "contribution_to_active": 1.255184292793274, "mix_mean": 0.03281574323773384, "mix_low": 0.30984020233154297, "mix_high": 1.9073486328125e-06, "read_cosine": 0.31221070885658264, "beta_mean": 0.17252370715141296, "nonfinite": 0.0, "weighted_memory_to_active": 1.1594375371932983, "mix_std": 0.055552221834659576, "beta_std": 0.11677898466587067, "beta_low": 0.0169677734375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.09090068936347961, "evidence_weights": [-0.032607950270175934, -0.002371768467128277, 0.04305043816566467], "agreement_weight": -0.013088978826999664, "orthogonal_weight": 0.10928544402122498}, {"head": 5, "memory_to_active": 4.325826644897461, "contribution_to_active": 0.08666520565748215, "mix_mean": 0.006242314353585243, "mix_low": 0.8586978912353516, "mix_high": 0.0, "read_cosine": 0.21116456389427185, "beta_mean": 0.2965680956840515, "nonfinite": 0.0, "weighted_memory_to_active": 0.06375110894441605, "mix_std": 0.01969670131802559, "beta_std": 0.21434548497200012, "beta_low": 0.02459716796875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.016467131674289703, "evidence_weights": [0.17046351730823517, 0.08977101743221283, 0.1395369917154312], "agreement_weight": -0.11873798817396164, "orthogonal_weight": 0.2384154498577118}]}, {"layer": 2, "memory_to_active": 0.9817739725112915, "contribution_to_active": 0.3457491397857666, "mix_mean": 0.3856583833694458, "mix_low": 0.2535567581653595, "mix_high": 0.0038011870346963406, "read_cosine": 0.3117903470993042, "beta_mean": 0.08957747370004654, "nonfinite": 0.0, "weighted_memory_to_active": 0.08445405960083008, "mix_std": 0.1899426281452179, "beta_std": 0.07397529482841492, "beta_low": 0.5631002187728882, "beta_high": 0.0002543131704442203, "orthogonal_to_active": 0.020080052316188812, "output_influence": 0.36690977215766907, "residual_influence": 0.09425093233585358, "state_active": 0.03522813320159912, "state_memory": 0.016159787774086, "state_peak": 31.822999954223633, "growth_active": 0.963371217250824, "growth_memory": 1.5714764595031738, "heads": [{"head": 0, "memory_to_active": 0.011560032144188881, "contribution_to_active": 0.3476070165634155, "mix_mean": 0.5959955453872681, "mix_low": 0.0010633468627929688, "mix_high": 0.0070972442626953125, "read_cosine": 0.12665946781635284, "beta_mean": 6.981594924582168e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007034884765744209, "mix_std": 0.27708178758621216, "beta_std": 0.00011756426829379052, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007084273267537355, "evidence_weights": [-0.35833340883255005, -0.21106885373592377, 0.11810413002967834], "agreement_weight": 0.1311018019914627, "orthogonal_weight": -1.0386958122253418}, {"head": 1, "memory_to_active": 2.669221878051758, "contribution_to_active": 0.03551989048719406, "mix_mean": 0.004086673259735107, "mix_low": 0.9209184646606445, "mix_high": 9.5367431640625e-07, "read_cosine": 0.1530398428440094, "beta_mean": 0.23767447471618652, "nonfinite": 0.0, "weighted_memory_to_active": 0.026881802827119827, "mix_std": 0.009306448511779308, "beta_std": 0.16445258259773254, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.009427432902157307, "evidence_weights": [0.14466769993305206, 0.1306265890598297, 0.47770068049430847], "agreement_weight": 0.029537811875343323, "orthogonal_weight": 0.3973838686943054}, {"head": 2, "memory_to_active": 0.014610226266086102, "contribution_to_active": 0.41321998834609985, "mix_mean": 0.5190765857696533, "mix_low": 0.0013713836669921875, "mix_high": 0.0021495819091796875, "read_cosine": 0.33944687247276306, "beta_mean": 3.717991785379127e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005640820600092411, "mix_std": 0.2755061388015747, "beta_std": 5.9525038523133844e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005769514245912433, "evidence_weights": [-0.24544093012809753, -0.2866259217262268, 0.5296177268028259], "agreement_weight": 0.021500857546925545, "orthogonal_weight": -0.12498175352811813}, {"head": 3, "memory_to_active": 0.16559100151062012, "contribution_to_active": 0.5196211338043213, "mix_mean": 0.5460267066955566, "mix_low": 0.0003833770751953125, "mix_high": 0.007045745849609375, "read_cosine": 0.6518672108650208, "beta_mean": 0.03035077452659607, "nonfinite": 0.0, "weighted_memory_to_active": 0.10017582774162292, "mix_std": 0.26330679655075073, "beta_std": 0.03876696899533272, "beta_low": 0.3428955078125, "beta_high": 0.0, "orthogonal_to_active": 0.06423717737197876, "evidence_weights": [-0.19686730206012726, -0.3068241775035858, 0.828887939453125], "agreement_weight": 0.05023931339383125, "orthogonal_weight": -0.888181746006012}, {"head": 4, "memory_to_active": 3.0170042514801025, "contribution_to_active": 0.2411515861749649, "mix_mean": 0.019030356779694557, "mix_low": 0.5964632034301758, "mix_high": 0.0, "read_cosine": 0.21065160632133484, "beta_mean": 0.26929038763046265, "nonfinite": 0.0, "weighted_memory_to_active": 0.3596080541610718, "mix_std": 0.0495242103934288, "beta_std": 0.24038589000701904, "beta_low": 0.03466796875, "beta_high": 0.00152587890625, "orthogonal_to_active": 0.038715656846761703, "evidence_weights": [0.020875096321105957, 0.006871077232062817, 0.011915748938918114], "agreement_weight": 0.024832244962453842, "orthogonal_weight": -0.141717791557312}, {"head": 5, "memory_to_active": 0.012656526640057564, "contribution_to_active": 0.517375111579895, "mix_mean": 0.6297342777252197, "mix_low": 0.001140594482421875, "mix_high": 0.0065135955810546875, "read_cosine": 0.3890770375728607, "beta_mean": 4.21882068621926e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0073829516768455505, "mix_std": 0.26493039727211, "beta_std": 6.922744796611369e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004388213565107435, "evidence_weights": [-0.28028395771980286, -0.29300564527511597, 0.9777657985687256], "agreement_weight": 0.15255635976791382, "orthogonal_weight": -0.08011136949062347}]}, {"layer": 3, "memory_to_active": 0.5008351802825928, "contribution_to_active": 0.45231449604034424, "mix_mean": 0.4659546911716461, "mix_low": 0.0670628547668457, "mix_high": 0.009215196594595909, "read_cosine": 0.20830528438091278, "beta_mean": 0.029508570209145546, "nonfinite": 0.0, "weighted_memory_to_active": 0.12952235341072083, "mix_std": 0.2619505524635315, "beta_std": 0.025736164301633835, "beta_low": 0.7505493760108948, "beta_high": 0.0, "orthogonal_to_active": 0.05270424485206604, "output_influence": 0.37796953320503235, "residual_influence": 0.08327881991863251, "state_active": 0.024077659472823143, "state_memory": 0.0073517304845154285, "state_peak": 22.059507369995117, "growth_active": 1.0460635423660278, "growth_memory": 1.6816176176071167, "heads": [{"head": 0, "memory_to_active": 0.379477322101593, "contribution_to_active": 0.5771034955978394, "mix_mean": 0.5609277486801147, "mix_low": 0.002399444580078125, "mix_high": 0.021340370178222656, "read_cosine": 0.24922120571136475, "beta_mean": 0.010602341964840889, "nonfinite": 0.0, "weighted_memory_to_active": 0.22904792428016663, "mix_std": 0.31427693367004395, "beta_std": 0.012705069035291672, "beta_low": 0.653076171875, "beta_high": 0.0, "orthogonal_to_active": 0.15789291262626648, "evidence_weights": [-0.2097041755914688, -0.24273981153964996, 0.9079393744468689], "agreement_weight": 0.1109418049454689, "orthogonal_weight": -0.7386787533760071}, {"head": 1, "memory_to_active": 0.01995987445116043, "contribution_to_active": 0.4146926999092102, "mix_mean": 0.5399390459060669, "mix_low": 0.005932807922363281, "mix_high": 0.007622718811035156, "read_cosine": 0.19620153307914734, "beta_mean": 0.00010824771015904844, "nonfinite": 0.0, "weighted_memory_to_active": 0.01099342666566372, "mix_std": 0.30426549911499023, "beta_std": 0.00022555486066266894, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008466565050184727, "evidence_weights": [-0.25791487097740173, -0.20984841883182526, 0.6298525333404541], "agreement_weight": 0.4561358392238617, "orthogonal_weight": -0.8508598804473877}, {"head": 2, "memory_to_active": 2.32647967338562, "contribution_to_active": 0.2915344834327698, "mix_mean": 0.042335785925388336, "mix_low": 0.37271690368652344, "mix_high": 0.0, "read_cosine": 0.19118261337280273, "beta_mean": 0.16002027690410614, "nonfinite": 0.0, "weighted_memory_to_active": 0.3378026783466339, "mix_std": 0.0939410924911499, "beta_std": 0.13204112648963928, "beta_low": 0.0169677734375, "beta_high": 0.0, "orthogonal_to_active": 0.00252408254891634, "evidence_weights": [-0.07748230546712875, 0.016445374116301537, 0.000359936966560781], "agreement_weight": -0.009237767197191715, "orthogonal_weight": -0.00954253040254116}, {"head": 3, "memory_to_active": 0.020341861993074417, "contribution_to_active": 0.4948839247226715, "mix_mean": 0.5739181637763977, "mix_low": 0.006556510925292969, "mix_high": 0.011303901672363281, "read_cosine": 0.29708436131477356, "beta_mean": 9.039692668011412e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01381007581949234, "mix_std": 0.29487624764442444, "beta_std": 0.00022010912653058767, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0037132943980395794, "evidence_weights": [-0.24081525206565857, -0.22997014224529266, 2.148092031478882], "agreement_weight": 1.8875677585601807, "orthogonal_weight": -0.3539658188819885}, {"head": 4, "memory_to_active": 0.01948033832013607, "contribution_to_active": 0.29498291015625, "mix_mean": 0.3942813277244568, "mix_low": 0.011902809143066406, "mix_high": 0.0008592605590820312, "read_cosine": 0.18438655138015747, "beta_mean": 0.00013016504817642272, "nonfinite": 0.0, "weighted_memory_to_active": 0.01093290001153946, "mix_std": 0.2878183424472809, "beta_std": 0.00018476651166565716, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010452312417328358, "evidence_weights": [-0.17331060767173767, -0.18626758456230164, 0.12207575142383575], "agreement_weight": -0.012109905481338501, "orthogonal_weight": -1.021652102470398}, {"head": 5, "memory_to_active": 0.23927170038223267, "contribution_to_active": 0.6406893730163574, "mix_mean": 0.6843259334564209, "mix_low": 0.00286865234375, "mix_high": 0.014164924621582031, "read_cosine": 0.1317552924156189, "beta_mean": 0.006099975202232599, "nonfinite": 0.0, "weighted_memory_to_active": 0.17454707622528076, "mix_std": 0.2765250504016876, "beta_std": 0.009040355682373047, "beta_low": 0.83331298828125, "beta_high": 0.0, "orthogonal_to_active": 0.1331762969493866, "evidence_weights": [-0.253520131111145, -0.31188565492630005, 0.48806676268577576], "agreement_weight": 0.14134778082370758, "orthogonal_weight": -0.7921698689460754}]}, {"layer": 4, "memory_to_active": 1.016324520111084, "contribution_to_active": 0.46806663274765015, "mix_mean": 0.3606974184513092, "mix_low": 0.26983770728111267, "mix_high": 0.009986400604248047, "read_cosine": 0.3355150520801544, "beta_mean": 0.11397748440504074, "nonfinite": 0.0, "weighted_memory_to_active": 0.14744415879249573, "mix_std": 0.1605384647846222, "beta_std": 0.10375049710273743, "beta_low": 0.38043212890625, "beta_high": 0.0, "orthogonal_to_active": 0.04752679914236069, "output_influence": 0.4511029124259949, "residual_influence": 0.1270894706249237, "state_active": 0.0397520549595356, "state_memory": 0.018348725512623787, "state_peak": 41.848350524902344, "growth_active": 1.27688467502594, "growth_memory": 1.539778470993042, "heads": [{"head": 0, "memory_to_active": 0.19822314381599426, "contribution_to_active": 0.7434519529342651, "mix_mean": 0.6786543130874634, "mix_low": 0.012969017028808594, "mix_high": 0.027146339416503906, "read_cosine": 0.5104091167449951, "beta_mean": 0.2665843963623047, "nonfinite": 0.0, "weighted_memory_to_active": 0.1583625227212906, "mix_std": 0.3479096591472626, "beta_std": 0.27047568559646606, "beta_low": 0.2242431640625, "beta_high": 0.0, "orthogonal_to_active": 0.0653761625289917, "evidence_weights": [-0.26637497544288635, -0.36789530515670776, 0.08205338567495346], "agreement_weight": 0.005285820458084345, "orthogonal_weight": 0.5268707871437073}, {"head": 1, "memory_to_active": 0.06039693206548691, "contribution_to_active": 0.6423536539077759, "mix_mean": 0.6756739616394043, "mix_low": 0.0001239776611328125, "mix_high": 0.016855239868164062, "read_cosine": 0.6638056039810181, "beta_mean": 0.0010031926212832332, "nonfinite": 0.0, "weighted_memory_to_active": 0.04332394152879715, "mix_std": 0.2448386549949646, "beta_std": 0.006319655105471611, "beta_low": 0.98345947265625, "beta_high": 0.0, "orthogonal_to_active": 0.022968877106904984, "evidence_weights": [-0.2392498105764389, -0.3543343245983124, 0.6692174673080444], "agreement_weight": 0.25232964754104614, "orthogonal_weight": -0.7807361483573914}, {"head": 2, "memory_to_active": 1.846889853477478, "contribution_to_active": 0.2879284620285034, "mix_mean": 0.025345992296934128, "mix_low": 0.5427303314208984, "mix_high": 2.288818359375e-05, "read_cosine": 0.2405758649110794, "beta_mean": 0.07843072712421417, "nonfinite": 0.0, "weighted_memory_to_active": 0.273691862821579, "mix_std": 0.05903294309973717, "beta_std": 0.07313992083072662, "beta_low": 0.0438232421875, "beta_high": 0.0, "orthogonal_to_active": 0.045572079718112946, "evidence_weights": [0.05754154920578003, 0.011882112361490726, 0.09442773461341858], "agreement_weight": 0.14112567901611328, "orthogonal_weight": -0.2605648636817932}, {"head": 3, "memory_to_active": 1.7519859075546265, "contribution_to_active": 0.1485520750284195, "mix_mean": 0.031992584466934204, "mix_low": 0.4154233932495117, "mix_high": 0.0, "read_cosine": 0.1749904751777649, "beta_mean": 0.12560880184173584, "nonfinite": 0.0, "weighted_memory_to_active": 0.2341383695602417, "mix_std": 0.05802324786782265, "beta_std": 0.11346397548913956, "beta_low": 0.03106689453125, "beta_high": 0.0, "orthogonal_to_active": 0.004262060392647982, "evidence_weights": [-0.02971784584224224, 0.051103848963975906, -0.23007464408874512], "agreement_weight": 0.07658287137746811, "orthogonal_weight": -0.0290409903973341}, {"head": 4, "memory_to_active": 0.009378921240568161, "contribution_to_active": 0.6570690274238586, "mix_mean": 0.7379469871520996, "mix_low": 0.0002193450927734375, "mix_high": 0.01588916778564453, "read_cosine": 0.17892254889011383, "beta_mean": 2.1225887394393794e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00699811615049839, "mix_std": 0.2192285805940628, "beta_std": 4.985962368664332e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029158492106944323, "evidence_weights": [-0.3376314043998718, -0.3279738426208496, 0.5745684504508972], "agreement_weight": 0.33894112706184387, "orthogonal_weight": -0.4472283124923706}, {"head": 5, "memory_to_active": 2.2310714721679688, "contribution_to_active": 0.3290444314479828, "mix_mean": 0.014570645987987518, "mix_low": 0.6475601196289062, "mix_high": 4.76837158203125e-06, "read_cosine": 0.24438676238059998, "beta_mean": 0.21221649646759033, "nonfinite": 0.0, "weighted_memory_to_active": 0.16815003752708435, "mix_std": 0.034197621047496796, "beta_std": 0.15905383229255676, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.1440657377243042, "evidence_weights": [0.04795747250318527, -0.03050483576953411, -0.3034554421901703], "agreement_weight": -0.07837113738059998, "orthogonal_weight": 0.9335198998451233}]}, {"layer": 5, "memory_to_active": 0.40396952629089355, "contribution_to_active": 0.5218075513839722, "mix_mean": 0.4917045533657074, "mix_low": 0.05790122598409653, "mix_high": 0.0163218192756176, "read_cosine": 0.040430761873722076, "beta_mean": 0.05552637577056885, "nonfinite": 0.0, "weighted_memory_to_active": 0.13328543305397034, "mix_std": 0.256540447473526, "beta_std": 0.07120557874441147, "beta_low": 0.649871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.050673823803663254, "output_influence": 0.5283121466636658, "residual_influence": 0.1469275802373886, "state_active": 0.04525846242904663, "state_memory": 0.011122643947601318, "state_peak": 67.44657135009766, "growth_active": 1.145242691040039, "growth_memory": 1.2584364414215088, "heads": [{"head": 0, "memory_to_active": 0.3621906340122223, "contribution_to_active": 0.5584561824798584, "mix_mean": 0.38212722539901733, "mix_low": 0.019486427307128906, "mix_high": 0.0010852813720703125, "read_cosine": -0.25181615352630615, "beta_mean": 0.02624850906431675, "nonfinite": 0.0, "weighted_memory_to_active": 0.10965172946453094, "mix_std": 0.2733146846294403, "beta_std": 0.04771476611495018, "beta_low": 0.4459228515625, "beta_high": 0.0, "orthogonal_to_active": 0.03644733130931854, "evidence_weights": [-0.13616779446601868, -0.1627095490694046, -0.035144224762916565], "agreement_weight": -0.01793806068599224, "orthogonal_weight": -0.3638371527194977}, {"head": 1, "memory_to_active": 0.010211249813437462, "contribution_to_active": 0.5393257141113281, "mix_mean": 0.680100679397583, "mix_low": 0.0018672943115234375, "mix_high": 0.03003978729248047, "read_cosine": 0.15021640062332153, "beta_mean": 2.37811964325374e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005941377021372318, "mix_std": 0.27979740500450134, "beta_std": 5.251486800261773e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005539377219974995, "evidence_weights": [-0.2501579821109772, -0.2856890559196472, 0.8824670910835266], "agreement_weight": 0.6418855786323547, "orthogonal_weight": -0.9757544994354248}, {"head": 2, "memory_to_active": 0.013665303587913513, "contribution_to_active": 0.568458616733551, "mix_mean": 0.648676872253418, "mix_low": 0.0029458999633789062, "mix_high": 0.018472671508789062, "read_cosine": -0.5424631834030151, "beta_mean": 9.920805314322934e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00838382262736559, "mix_std": 0.27039816975593567, "beta_std": 0.00014805266982875764, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003276812843978405, "evidence_weights": [-0.26160869002342224, -0.3413412868976593, 0.39850836992263794], "agreement_weight": -0.022994007915258408, "orthogonal_weight": -0.4938530921936035}, {"head": 3, "memory_to_active": 1.5004568099975586, "contribution_to_active": 0.1978289783000946, "mix_mean": 0.10199479013681412, "mix_low": 0.14761638641357422, "mix_high": 0.0, "read_cosine": 0.1560390293598175, "beta_mean": 0.11908119171857834, "nonfinite": 0.0, "weighted_memory_to_active": 0.30636143684387207, "mix_std": 0.13802869617938995, "beta_std": 0.16536636650562286, "beta_low": 0.179443359375, "beta_high": 0.0, "orthogonal_to_active": 0.12128600478172302, "evidence_weights": [-0.06546977907419205, -0.08983945101499557, 0.02658093348145485], "agreement_weight": 0.04236404225230217, "orthogonal_weight": -0.47743135690689087}, {"head": 4, "memory_to_active": 0.43642473220825195, "contribution_to_active": 0.5911551713943481, "mix_mean": 0.34529876708984375, "mix_low": 0.17360496520996094, "mix_high": 0.006707191467285156, "read_cosine": 0.5614098310470581, "beta_mean": 0.18741339445114136, "nonfinite": 0.0, "weighted_memory_to_active": 0.30118459463119507, "mix_std": 0.3468908965587616, "beta_std": 0.21265870332717896, "beta_low": 0.27685546875, "beta_high": 0.0, "orthogonal_to_active": 0.09460987150669098, "evidence_weights": [-0.116721510887146, -0.2735057771205902, -0.05604736879467964], "agreement_weight": 0.03718320280313492, "orthogonal_weight": 0.4915163218975067}, {"head": 5, "memory_to_active": 0.10086826235055923, "contribution_to_active": 0.6756203770637512, "mix_mean": 0.7920288443565369, "mix_low": 0.0018863677978515625, "mix_high": 0.0416259765625, "read_cosine": 0.16919857263565063, "beta_mean": 0.0002921869745478034, "nonfinite": 0.0, "weighted_memory_to_active": 0.06818971782922745, "mix_std": 0.23081272840499878, "beta_std": 0.0012930273078382015, "beta_low": 0.99700927734375, "beta_high": 0.0, "orthogonal_to_active": 0.04288353770971298, "evidence_weights": [-0.3155372440814972, -0.33312347531318665, 1.9899678230285645], "agreement_weight": 0.4589066207408905, "orthogonal_weight": -0.6661767363548279}]}, {"layer": 6, "memory_to_active": 0.31242769956588745, "contribution_to_active": 0.6123416423797607, "mix_mean": 0.580208420753479, "mix_low": 0.055815860629081726, "mix_high": 0.03561004251241684, "read_cosine": 0.29789263010025024, "beta_mean": 0.060966383665800095, "nonfinite": 0.0, "weighted_memory_to_active": 0.15219131112098694, "mix_std": 0.23776492476463318, "beta_std": 0.07072465121746063, "beta_low": 0.5245768427848816, "beta_high": 0.0, "orthogonal_to_active": 0.038735270500183105, "output_influence": 0.5325677394866943, "residual_influence": 0.1760781705379486, "state_active": 0.04591088742017746, "state_memory": 0.00809574592858553, "state_peak": 90.99340057373047, "growth_active": 1.2072590589523315, "growth_memory": 1.1852517127990723, "heads": [{"head": 0, "memory_to_active": 0.10005304217338562, "contribution_to_active": 0.6997057199478149, "mix_mean": 0.7957701683044434, "mix_low": 0.00032138824462890625, "mix_high": 0.03846454620361328, "read_cosine": 0.13716751337051392, "beta_mean": 0.01644189842045307, "nonfinite": 0.0, "weighted_memory_to_active": 0.0813770666718483, "mix_std": 0.20878911018371582, "beta_std": 0.02464481256902218, "beta_low": 0.552490234375, "beta_high": 0.0, "orthogonal_to_active": 0.06545637547969818, "evidence_weights": [-0.32097381353378296, -0.3457411229610443, 1.160384178161621], "agreement_weight": 0.41290760040283203, "orthogonal_weight": -0.8426679968833923}, {"head": 1, "memory_to_active": 0.005002246238291264, "contribution_to_active": 0.538676381111145, "mix_mean": 0.5639928579330444, "mix_low": 0.0017261505126953125, "mix_high": 0.01384735107421875, "read_cosine": 0.5273997783660889, "beta_mean": 9.97661663859617e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002329114591702819, "mix_std": 0.2902904152870178, "beta_std": 1.6336411135853268e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012827973114326596, "evidence_weights": [-0.19943006336688995, -0.29886114597320557, 1.1018264293670654], "agreement_weight": 0.08225247263908386, "orthogonal_weight": -0.7410575151443481}, {"head": 2, "memory_to_active": 0.9057779312133789, "contribution_to_active": 0.09114639461040497, "mix_mean": 0.03773820400238037, "mix_low": 0.3256034851074219, "mix_high": 1.9073486328125e-06, "read_cosine": 0.24854542315006256, "beta_mean": 0.03522980958223343, "nonfinite": 0.0, "weighted_memory_to_active": 0.07027764618396759, "mix_std": 0.06008961424231529, "beta_std": 0.037141602486371994, "beta_low": 0.10479736328125, "beta_high": 0.0, "orthogonal_to_active": 0.0036917722318321466, "evidence_weights": [0.0810004472732544, -0.03252255544066429, 0.2994140386581421], "agreement_weight": 0.1479203701019287, "orthogonal_weight": -0.06289117783308029}, {"head": 3, "memory_to_active": 0.1360868662595749, "contribution_to_active": 0.6766737699508667, "mix_mean": 0.6561213731765747, "mix_low": 0.005816459655761719, "mix_high": 0.022556304931640625, "read_cosine": 0.47174322605133057, "beta_mean": 0.18860316276550293, "nonfinite": 0.0, "weighted_memory_to_active": 0.09661035239696503, "mix_std": 0.32404327392578125, "beta_std": 0.22327972948551178, "beta_low": 0.21429443359375, "beta_high": 0.0, "orthogonal_to_active": 0.05138116329908371, "evidence_weights": [-0.17979252338409424, -0.3347388803958893, 0.1946209967136383], "agreement_weight": -0.009730760008096695, "orthogonal_weight": 0.612949550151825}, {"head": 4, "memory_to_active": 0.6904663443565369, "contribution_to_active": 1.0634653568267822, "mix_mean": 0.7654834985733032, "mix_low": 0.00079345703125, "mix_high": 0.1332845687866211, "read_cosine": 0.13815519213676453, "beta_mean": 0.12530586123466492, "nonfinite": 0.0, "weighted_memory_to_active": 0.6357792615890503, "mix_std": 0.2890389859676361, "beta_std": 0.13870644569396973, "beta_low": 0.2762451171875, "beta_high": 0.0, "orthogonal_to_active": 0.08409392833709717, "evidence_weights": [-0.22784769535064697, -0.5826059579849243, -0.29907840490341187], "agreement_weight": -0.019597087055444717, "orthogonal_weight": 0.14820298552513123}, {"head": 5, "memory_to_active": 0.0371796190738678, "contribution_to_active": 0.6043819785118103, "mix_mean": 0.6621441841125488, "mix_low": 0.0006341934204101562, "mix_high": 0.005505561828613281, "read_cosine": 0.26434457302093506, "beta_mean": 0.0002075664815492928, "nonfinite": 0.0, "weighted_memory_to_active": 0.026774374768137932, "mix_std": 0.2543380856513977, "beta_std": 0.0005589632201008499, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.02650555968284607, "evidence_weights": [-0.24233393371105194, -0.31263673305511475, 0.4343554377555847], "agreement_weight": 0.17070910334587097, "orthogonal_weight": -1.0560052394866943}]}, {"layer": 7, "memory_to_active": 1.7257319688796997, "contribution_to_active": 0.24801364541053772, "mix_mean": 0.14990876615047455, "mix_low": 0.25230392813682556, "mix_high": 0.0009144147625193, "read_cosine": 0.22916279733181, "beta_mean": 0.1262105107307434, "nonfinite": 0.0, "weighted_memory_to_active": 0.2063230574131012, "mix_std": 0.14534896612167358, "beta_std": 0.09847279638051987, "beta_low": 0.1996358335018158, "beta_high": 0.0001627604360692203, "orthogonal_to_active": 0.08492109179496765, "output_influence": 0.2549024820327759, "residual_influence": 0.058267608284950256, "state_active": 0.012786859646439552, "state_memory": 0.024776119738817215, "state_peak": 28.238039016723633, "growth_active": 1.082627296447754, "growth_memory": 1.2891652584075928, "heads": [{"head": 0, "memory_to_active": 2.4618635177612305, "contribution_to_active": 0.1768178641796112, "mix_mean": 0.036727674305438995, "mix_low": 0.43462562561035156, "mix_high": 9.5367431640625e-07, "read_cosine": 0.34082216024398804, "beta_mean": 0.15135830640792847, "nonfinite": 0.0, "weighted_memory_to_active": 0.20010538399219513, "mix_std": 0.07669345289468765, "beta_std": 0.11585956811904907, "beta_low": 0.0029296875, "beta_high": 0.00048828125, "orthogonal_to_active": 0.003081542206928134, "evidence_weights": [0.031018933281302452, -0.0785512775182724, 0.0500115267932415], "agreement_weight": -0.03272975608706474, "orthogonal_weight": 0.018173126503825188}, {"head": 1, "memory_to_active": 0.9976540803909302, "contribution_to_active": 0.3901606798171997, "mix_mean": 0.21536076068878174, "mix_low": 0.13280391693115234, "mix_high": 0.0048007965087890625, "read_cosine": 0.3751266598701477, "beta_mean": 0.09579157829284668, "nonfinite": 0.0, "weighted_memory_to_active": 0.32875296473503113, "mix_std": 0.24368596076965332, "beta_std": 0.06713688373565674, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04015852510929108, "evidence_weights": [-0.21228346228599548, -0.1047886461019516, -0.23545141518115997], "agreement_weight": 0.04557456076145172, "orthogonal_weight": 0.1445222645998001}, {"head": 2, "memory_to_active": 0.7326104640960693, "contribution_to_active": 0.3345278799533844, "mix_mean": 0.17434459924697876, "mix_low": 0.04854774475097656, "mix_high": 0.00043010711669921875, "read_cosine": 0.09582427144050598, "beta_mean": 0.050340477377176285, "nonfinite": 0.0, "weighted_memory_to_active": 0.23497767746448517, "mix_std": 0.1941288560628891, "beta_std": 0.04319462925195694, "beta_low": 0.06390380859375, "beta_high": 0.0, "orthogonal_to_active": 0.109932079911232, "evidence_weights": [-0.06844077259302139, -0.15091444551944733, 0.6872149705886841], "agreement_weight": 0.026207614690065384, "orthogonal_weight": -0.4801803231239319}, {"head": 3, "memory_to_active": 4.307618141174316, "contribution_to_active": 0.10660992562770844, "mix_mean": 0.038308143615722656, "mix_low": 0.303863525390625, "mix_high": 0.0, "read_cosine": 0.11382215470075607, "beta_mean": 0.37258198857307434, "nonfinite": 0.0, "weighted_memory_to_active": 0.3510887920856476, "mix_std": 0.05726287141442299, "beta_std": 0.2668136954307556, "beta_low": 0.006103515625, "beta_high": 0.00048828125, "orthogonal_to_active": 0.3020162582397461, "evidence_weights": [0.06354455649852753, -0.025174498558044434, 0.7101466655731201], "agreement_weight": 0.05716138705611229, "orthogonal_weight": -0.939373254776001}, {"head": 4, "memory_to_active": 1.7525326013565063, "contribution_to_active": 0.08552823960781097, "mix_mean": 0.019029825925827026, "mix_low": 0.5908327102661133, "mix_high": 0.0, "read_cosine": 0.22250699996948242, "beta_mean": 0.08673228323459625, "nonfinite": 0.0, "weighted_memory_to_active": 0.07354481518268585, "mix_std": 0.04186803847551346, "beta_std": 0.09670373797416687, "beta_low": 0.127197265625, "beta_high": 0.0, "orthogonal_to_active": 0.012405700981616974, "evidence_weights": [0.14057187736034393, 0.008270751684904099, 0.2556006908416748], "agreement_weight": 0.0372844859957695, "orthogonal_weight": -0.20904138684272766}, {"head": 5, "memory_to_active": 0.10211305320262909, "contribution_to_active": 0.3944372534751892, "mix_mean": 0.41568154096603394, "mix_low": 0.0031499862670898438, "mix_high": 0.00025463104248046875, "read_cosine": 0.2268744707107544, "beta_mean": 0.0004584052076097578, "nonfinite": 0.0, "weighted_memory_to_active": 0.04946863278746605, "mix_std": 0.2584545314311981, "beta_std": 0.0011282542254775763, "beta_low": 0.9976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.04193240404129028, "evidence_weights": [-0.15726971626281738, -0.21532054245471954, 0.56145840883255], "agreement_weight": 0.18512098491191864, "orthogonal_weight": -0.9096252918243408}]}, {"layer": 8, "memory_to_active": 0.2649231255054474, "contribution_to_active": 0.5332804918289185, "mix_mean": 0.44357550144195557, "mix_low": 0.03568172827363014, "mix_high": 0.0014050803147256374, "read_cosine": 0.439983993768692, "beta_mean": 0.015953337773680687, "nonfinite": 0.0, "weighted_memory_to_active": 0.08654139935970306, "mix_std": 0.22502672672271729, "beta_std": 0.023851338773965836, "beta_low": 0.723114013671875, "beta_high": 0.0, "orthogonal_to_active": 0.06625475734472275, "output_influence": 0.48160433769226074, "residual_influence": 0.16203245520591736, "state_active": 0.04497617855668068, "state_memory": 0.00670796912163496, "state_peak": 41.29527282714844, "growth_active": 1.0561531782150269, "growth_memory": 1.2177138328552246, "heads": [{"head": 0, "memory_to_active": 0.05320797115564346, "contribution_to_active": 0.5644709467887878, "mix_mean": 0.6025051474571228, "mix_low": 0.0004024505615234375, "mix_high": 0.0013990402221679688, "read_cosine": 0.44581252336502075, "beta_mean": 0.002844716189429164, "nonfinite": 0.0, "weighted_memory_to_active": 0.03698450326919556, "mix_std": 0.24567243456840515, "beta_std": 0.010007675737142563, "beta_low": 0.9327392578125, "beta_high": 0.0, "orthogonal_to_active": 0.023813240230083466, "evidence_weights": [-0.17015093564987183, -0.39005759358406067, 0.6538137793540955], "agreement_weight": 0.20208324491977692, "orthogonal_weight": -0.7976941466331482}, {"head": 1, "memory_to_active": 0.09303098917007446, "contribution_to_active": 0.5312498807907104, "mix_mean": 0.4980636239051819, "mix_low": 6.866455078125e-05, "mix_high": 0.0007905960083007812, "read_cosine": 0.17638152837753296, "beta_mean": 0.0016775333788245916, "nonfinite": 0.0, "weighted_memory_to_active": 0.042116429656744, "mix_std": 0.2428044229745865, "beta_std": 0.009637732990086079, "beta_low": 0.9808349609375, "beta_high": 0.0, "orthogonal_to_active": 0.03036528080701828, "evidence_weights": [-0.18992266058921814, -0.27138251066207886, 0.6965947151184082], "agreement_weight": 0.06858889758586884, "orthogonal_weight": -0.7528001666069031}, {"head": 2, "memory_to_active": 0.017415113747119904, "contribution_to_active": 0.6649990677833557, "mix_mean": 0.6573052406311035, "mix_low": 0.000942230224609375, "mix_high": 0.0026979446411132812, "read_cosine": 0.4095519185066223, "beta_mean": 0.00017312626005150378, "nonfinite": 0.0, "weighted_memory_to_active": 0.013432747684419155, "mix_std": 0.22949136793613434, "beta_std": 0.0036927128676325083, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00844577793031931, "evidence_weights": [-0.22590692341327667, -0.3442210257053375, 1.0171629190444946], "agreement_weight": 0.24880532920360565, "orthogonal_weight": -0.7277763485908508}, {"head": 3, "memory_to_active": 0.10750244557857513, "contribution_to_active": 0.6418640613555908, "mix_mean": 0.5730894804000854, "mix_low": 0.0008640289306640625, "mix_high": 0.0034637451171875, "read_cosine": 0.8112839460372925, "beta_mean": 0.008321953006088734, "nonfinite": 0.0, "weighted_memory_to_active": 0.08376659452915192, "mix_std": 0.25759804248809814, "beta_std": 0.007245258893817663, "beta_low": 0.700439453125, "beta_high": 0.0, "orthogonal_to_active": 0.025719597935676575, "evidence_weights": [-0.18464118242263794, -0.30201438069343567, 0.5791273713111877], "agreement_weight": 0.21141387522220612, "orthogonal_weight": -0.6337911486625671}, {"head": 4, "memory_to_active": 0.9553835988044739, "contribution_to_active": 0.18543365597724915, "mix_mean": 0.08557280898094177, "mix_low": 0.1272106170654297, "mix_high": 0.0, "read_cosine": 0.30606889724731445, "beta_mean": 0.016887137666344643, "nonfinite": 0.0, "weighted_memory_to_active": 0.16030140221118927, "mix_std": 0.11080904304981232, "beta_std": 0.031945452094078064, "beta_low": 0.59869384765625, "beta_high": 0.0, "orthogonal_to_active": 0.023038187995553017, "evidence_weights": [0.008347135968506336, -0.09737847000360489, 0.12757490575313568], "agreement_weight": 0.06786396354436874, "orthogonal_weight": -0.15406177937984467}, {"head": 5, "memory_to_active": 0.3629985749721527, "contribution_to_active": 0.6116655468940735, "mix_mean": 0.2449166625738144, "mix_low": 0.08460235595703125, "mix_high": 7.915496826171875e-05, "read_cosine": 0.49080491065979004, "beta_mean": 0.06581556051969528, "nonfinite": 0.0, "weighted_memory_to_active": 0.18264667689800262, "mix_std": 0.2637850344181061, "beta_std": 0.08057918399572372, "beta_low": 0.12603759765625, "beta_high": 0.0, "orthogonal_to_active": 0.2861464321613312, "evidence_weights": [-0.1068853810429573, -0.18153706192970276, -0.1440798044204712], "agreement_weight": -0.005351037252694368, "orthogonal_weight": 1.857292890548706}]}, {"layer": 9, "memory_to_active": 0.9822544455528259, "contribution_to_active": 0.663232684135437, "mix_mean": 0.3426925539970398, "mix_low": 0.12282960116863251, "mix_high": 0.005530516617000103, "read_cosine": 0.3503957986831665, "beta_mean": 0.048782557249069214, "nonfinite": 0.0, "weighted_memory_to_active": 0.3784603178501129, "mix_std": 0.21932362020015717, "beta_std": 0.03948739543557167, "beta_low": 0.5087484121322632, "beta_high": 0.0, "orthogonal_to_active": 0.03860599175095558, "output_influence": 0.5176612138748169, "residual_influence": 0.14536523818969727, "state_active": 0.04021947830915451, "state_memory": 0.01738225668668747, "state_peak": 52.170509338378906, "growth_active": 1.15402352809906, "growth_memory": 1.3258516788482666, "heads": [{"head": 0, "memory_to_active": 0.0009451685473322868, "contribution_to_active": 0.5581703782081604, "mix_mean": 0.6294615864753723, "mix_low": 0.0017576217651367188, "mix_high": 0.023311614990234375, "read_cosine": 0.30747684836387634, "beta_mean": 2.501834387658164e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005885140271857381, "mix_std": 0.29255515336990356, "beta_std": 7.086879122653045e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005892036133445799, "evidence_weights": [-0.30182474851608276, -0.3095286786556244, 0.7302249670028687], "agreement_weight": 0.11199049651622772, "orthogonal_weight": -1.0593844652175903}, {"head": 1, "memory_to_active": 0.01967078074812889, "contribution_to_active": 0.673611044883728, "mix_mean": 0.5481763482093811, "mix_low": 0.007884025573730469, "mix_high": 0.00244903564453125, "read_cosine": 0.3928226828575134, "beta_mean": 5.0233447836944833e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014263682067394257, "mix_std": 0.2853319048881531, "beta_std": 0.00010624949936755002, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010001609101891518, "evidence_weights": [-0.23887963593006134, -0.29130783677101135, 1.1377288103103638], "agreement_weight": 0.42510077357292175, "orthogonal_weight": -0.8540788888931274}, {"head": 2, "memory_to_active": 0.20944446325302124, "contribution_to_active": 0.6408982276916504, "mix_mean": 0.5852180123329163, "mix_low": 0.001712799072265625, "mix_high": 0.006153106689453125, "read_cosine": 0.3132462501525879, "beta_mean": 0.0007627659942954779, "nonfinite": 0.0, "weighted_memory_to_active": 0.13849571347236633, "mix_std": 0.27097511291503906, "beta_std": 0.004919811617583036, "beta_low": 0.9947509765625, "beta_high": 0.0, "orthogonal_to_active": 0.08030900359153748, "evidence_weights": [-0.1984402984380722, -0.2738667130470276, 0.7558086514472961], "agreement_weight": 0.30288878083229065, "orthogonal_weight": -0.6657950282096863}, {"head": 3, "memory_to_active": 1.177916407585144, "contribution_to_active": 0.17156976461410522, "mix_mean": 0.060393162071704865, "mix_low": 0.3103199005126953, "mix_high": 8.296966552734375e-05, "read_cosine": 0.39367884397506714, "beta_mean": 0.06429574638605118, "nonfinite": 0.0, "weighted_memory_to_active": 0.14140105247497559, "mix_std": 0.11541002243757248, "beta_std": 0.05276339501142502, "beta_low": 0.005615234375, "beta_high": 0.0, "orthogonal_to_active": 0.01063570100814104, "evidence_weights": [0.06225554272532463, -0.007369477301836014, 0.037610385566949844], "agreement_weight": 0.08178573101758957, "orthogonal_weight": -0.08112291246652603}, {"head": 4, "memory_to_active": 2.9859366416931152, "contribution_to_active": 1.4209492206573486, "mix_mean": 0.13852229714393616, "mix_low": 0.13892459869384766, "mix_high": 0.0009298324584960938, "read_cosine": 0.1738397777080536, "beta_mean": 0.14029432833194733, "nonfinite": 0.0, "weighted_memory_to_active": 1.4080376625061035, "mix_std": 0.1973283290863037, "beta_std": 0.08979643881320953, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.025955403223633766, "evidence_weights": [-0.07663366943597794, -0.15713363885879517, -0.033001530915498734], "agreement_weight": -0.060452356934547424, "orthogonal_weight": -0.01931772008538246}, {"head": 5, "memory_to_active": 1.4996131658554077, "contribution_to_active": 0.5141973495483398, "mix_mean": 0.09438367933034897, "mix_low": 0.2763786315917969, "mix_high": 0.00025653839111328125, "read_cosine": 0.5213102102279663, "beta_mean": 0.08728975802659988, "nonfinite": 0.0, "weighted_memory_to_active": 0.5679752826690674, "mix_std": 0.15434116125106812, "beta_std": 0.08933137357234955, "beta_low": 0.0521240234375, "beta_high": 0.0, "orthogonal_to_active": 0.10414503514766693, "evidence_weights": [0.10771487653255463, -0.10272403061389923, 0.13333529233932495], "agreement_weight": 0.01813168078660965, "orthogonal_weight": 0.2713159918785095}]}, {"layer": 10, "memory_to_active": 0.247755229473114, "contribution_to_active": 0.7379178404808044, "mix_mean": 0.5117419958114624, "mix_low": 0.020137470215559006, "mix_high": 0.005216916557401419, "read_cosine": 0.36670970916748047, "beta_mean": 0.043325573205947876, "nonfinite": 0.0, "weighted_memory_to_active": 0.12271369993686676, "mix_std": 0.26767587661743164, "beta_std": 0.04588504880666733, "beta_low": 0.661651611328125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.261554479598999, "output_influence": 0.4359126091003418, "residual_influence": 0.15069474279880524, "state_active": 0.052160732448101044, "state_memory": 0.015595297329127789, "state_peak": 120.79269409179688, "growth_active": 1.0229344367980957, "growth_memory": 0.9754095077514648, "heads": [{"head": 0, "memory_to_active": 0.9752256870269775, "contribution_to_active": 2.201073169708252, "mix_mean": 0.5770151615142822, "mix_low": 0.08351707458496094, "mix_high": 0.012434005737304688, "read_cosine": 0.08635147660970688, "beta_mean": 0.19582217931747437, "nonfinite": 0.0, "weighted_memory_to_active": 0.6182640790939331, "mix_std": 0.3931785821914673, "beta_std": 0.20940810441970825, "beta_low": 0.125244140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.482668399810791, "evidence_weights": [-0.1074339896440506, -0.3899790942668915, -1.5678645372390747], "agreement_weight": 0.025544162839651108, "orthogonal_weight": 2.419630765914917}, {"head": 1, "memory_to_active": 0.014332270249724388, "contribution_to_active": 0.5918135643005371, "mix_mean": 0.6143901348114014, "mix_low": 0.0013370513916015625, "mix_high": 0.012613296508789062, "read_cosine": 0.2560015916824341, "beta_mean": 8.66409536683932e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007518906611949205, "mix_std": 0.2672986090183258, "beta_std": 0.00014732536510564387, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011500701308250427, "evidence_weights": [-0.25776469707489014, -0.20040249824523926, 0.5653272867202759], "agreement_weight": 0.5320408940315247, "orthogonal_weight": -1.6121132373809814}, {"head": 2, "memory_to_active": 0.009864809922873974, "contribution_to_active": 0.6023902893066406, "mix_mean": 0.642824649810791, "mix_low": 0.0002040863037109375, "mix_high": 0.0054874420166015625, "read_cosine": 0.01637028530240059, "beta_mean": 2.126599065377377e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0043120067566633224, "mix_std": 0.25034499168395996, "beta_std": 7.640970579814166e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019227842567488551, "evidence_weights": [-0.25508981943130493, -0.28322097659111023, 0.6410283446311951], "agreement_weight": -0.17841660976409912, "orthogonal_weight": -0.42894530296325684}, {"head": 3, "memory_to_active": 0.33829766511917114, "contribution_to_active": 0.09665876626968384, "mix_mean": 0.26981595158576965, "mix_low": 0.033682823181152344, "mix_high": 0.00015163421630859375, "read_cosine": 0.9171304702758789, "beta_mean": 0.05775078386068344, "nonfinite": 0.0, "weighted_memory_to_active": 0.03723333403468132, "mix_std": 0.22251369059085846, "beta_std": 0.05795317515730858, "beta_low": 0.00787353515625, "beta_high": 0.0, "orthogonal_to_active": 0.03921707719564438, "evidence_weights": [-0.1047573909163475, -0.14648814499378204, 0.15773062407970428], "agreement_weight": 0.017452063038945198, "orthogonal_weight": 1.427774429321289}, {"head": 4, "memory_to_active": 0.020259255543351173, "contribution_to_active": 0.5413214564323425, "mix_mean": 0.5750545859336853, "mix_low": 0.00030231475830078125, "mix_high": 0.0006132125854492188, "read_cosine": 0.07624326646327972, "beta_mean": 6.888398638693616e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011636045761406422, "mix_std": 0.2470703125, "beta_std": 0.00014057912630960345, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014726979658007622, "evidence_weights": [-0.18524925410747528, -0.26248738169670105, 0.9276984930038452], "agreement_weight": 0.13471460342407227, "orthogonal_weight": -1.3340520858764648}, {"head": 5, "memory_to_active": 0.12855161726474762, "contribution_to_active": 0.39424973726272583, "mix_mean": 0.39135122299194336, "mix_low": 0.001781463623046875, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8481611013412476, "beta_mean": 0.0062036579474806786, "nonfinite": 0.0, "weighted_memory_to_active": 0.0573178194463253, "mix_std": 0.22564899921417236, "beta_std": 0.007584720384329557, "beta_low": 0.8367919921875, "beta_high": 0.0, "orthogonal_to_active": 0.019290922209620476, "evidence_weights": [-0.14153186976909637, -0.1605600118637085, 0.2718255817890167], "agreement_weight": 0.090177021920681, "orthogonal_weight": -0.6449070572853088}]}, {"layer": 11, "memory_to_active": 0.09512867033481598, "contribution_to_active": 0.6090375781059265, "mix_mean": 0.5073348879814148, "mix_low": 0.09309260547161102, "mix_high": 0.04803387448191643, "read_cosine": 0.37478822469711304, "beta_mean": 0.025857560336589813, "nonfinite": 0.0, "weighted_memory_to_active": 0.047953829169273376, "mix_std": 0.32345372438430786, "beta_std": 0.037559181451797485, "beta_low": 0.656494140625, "beta_high": 0.0, "orthogonal_to_active": 0.04695324972271919, "output_influence": 0.5360131859779358, "residual_influence": 0.1350688338279724, "state_active": 0.18645335733890533, "state_memory": 0.0045951372012495995, "state_peak": 33.284767150878906, "growth_active": 1.1144821643829346, "growth_memory": 1.5932774543762207, "heads": [{"head": 0, "memory_to_active": 0.010424371808767319, "contribution_to_active": 0.602892279624939, "mix_mean": 0.6432303190231323, "mix_low": 0.0010318756103515625, "mix_high": 0.007622718811035156, "read_cosine": 0.25591036677360535, "beta_mean": 9.016013063956052e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007245467510074377, "mix_std": 0.2591249942779541, "beta_std": 0.0038833064027130604, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.005602733232080936, "evidence_weights": [-0.17013084888458252, -0.3712122440338135, 0.7466439604759216], "agreement_weight": 0.3129129111766815, "orthogonal_weight": -0.803483247756958}, {"head": 1, "memory_to_active": 0.2051720917224884, "contribution_to_active": 0.38110679388046265, "mix_mean": 0.4257429242134094, "mix_low": 0.0060024261474609375, "mix_high": 0.00033664703369140625, "read_cosine": 0.5179144144058228, "beta_mean": 0.012685919180512428, "nonfinite": 0.0, "weighted_memory_to_active": 0.0947510227560997, "mix_std": 0.26311859488487244, "beta_std": 0.027971472591161728, "beta_low": 0.732421875, "beta_high": 0.0, "orthogonal_to_active": 0.04345613718032837, "evidence_weights": [-0.1660347729921341, -0.22631849348545074, 0.5709663033485413], "agreement_weight": 0.07099363952875137, "orthogonal_weight": -0.5528523325920105}, {"head": 2, "memory_to_active": 0.008376013487577438, "contribution_to_active": 0.9804311394691467, "mix_mean": 0.44661644101142883, "mix_low": 0.2498455047607422, "mix_high": 0.17612457275390625, "read_cosine": 0.2118912935256958, "beta_mean": 0.06842909753322601, "nonfinite": 0.0, "weighted_memory_to_active": 0.006064353510737419, "mix_std": 0.44544321298599243, "beta_std": 0.08910498023033142, "beta_low": 0.44207763671875, "beta_high": 0.0, "orthogonal_to_active": 0.04737275838851929, "evidence_weights": [0.03848207741975784, -0.4114820063114166, -1.3155145645141602], "agreement_weight": -0.00187020655721426, "orthogonal_weight": 8.155734062194824}, {"head": 3, "memory_to_active": 0.009559710510075092, "contribution_to_active": 0.8659898042678833, "mix_mean": 0.4169387221336365, "mix_low": 0.2921438217163086, "mix_high": 0.09816360473632812, "read_cosine": 0.5861886739730835, "beta_mean": 0.051460906863212585, "nonfinite": 0.0, "weighted_memory_to_active": 0.0059839882887899876, "mix_std": 0.43962395191192627, "beta_std": 0.0772717073559761, "beta_low": 0.49066162109375, "beta_high": 0.0, "orthogonal_to_active": 0.03387738764286041, "evidence_weights": [0.18092814087867737, -0.3102903664112091, -0.1881193071603775], "agreement_weight": 0.04914291948080063, "orthogonal_weight": 8.470503807067871}, {"head": 4, "memory_to_active": 0.24312277138233185, "contribution_to_active": 0.43556538224220276, "mix_mean": 0.5624406933784485, "mix_low": 0.007550239562988281, "mix_high": 0.001369476318359375, "read_cosine": 0.03607940301299095, "beta_mean": 0.014159655198454857, "nonfinite": 0.0, "weighted_memory_to_active": 0.12639287114143372, "mix_std": 0.2724274694919586, "beta_std": 0.015983518213033676, "beta_low": 0.53485107421875, "beta_high": 0.0, "orthogonal_to_active": 0.12451623380184174, "evidence_weights": [-0.24399706721305847, -0.23371824622154236, 1.9659470319747925], "agreement_weight": -0.20125789940357208, "orthogonal_weight": -0.9952248930931091}, {"head": 5, "memory_to_active": 0.09411704540252686, "contribution_to_active": 0.38823989033699036, "mix_mean": 0.5490400791168213, "mix_low": 0.0019817352294921875, "mix_high": 0.004586219787597656, "read_cosine": 0.6407451033592224, "beta_mean": 0.00831962376832962, "nonfinite": 0.0, "weighted_memory_to_active": 0.04728526249527931, "mix_std": 0.2609841823577881, "beta_std": 0.011140106245875359, "beta_low": 0.739501953125, "beta_high": 0.0, "orthogonal_to_active": 0.026894252747297287, "evidence_weights": [-0.21405477821826935, -0.3111163079738617, 0.6547020673751831], "agreement_weight": 0.08220706880092621, "orthogonal_weight": -0.7836863994598389}]}, {"layer": 12, "memory_to_active": 0.23047462105751038, "contribution_to_active": 0.5673030018806458, "mix_mean": 0.5232807397842407, "mix_low": 0.009596983902156353, "mix_high": 0.005094210617244244, "read_cosine": 0.2189498096704483, "beta_mean": 0.013286536559462547, "nonfinite": 0.0, "weighted_memory_to_active": 0.12333269417285919, "mix_std": 0.2621711790561676, "beta_std": 0.015099043026566505, "beta_low": 0.827850341796875, "beta_high": 0.0, "orthogonal_to_active": 0.06887181103229523, "output_influence": 0.4795796871185303, "residual_influence": 0.13365718722343445, "state_active": 0.033082637935876846, "state_memory": 0.008924435824155807, "state_peak": 54.508182525634766, "growth_active": 1.0587491989135742, "growth_memory": 1.1543482542037964, "heads": [{"head": 0, "memory_to_active": 0.04693106561899185, "contribution_to_active": 0.5909233093261719, "mix_mean": 0.5501975417137146, "mix_low": 0.0005035400390625, "mix_high": 0.0036401748657226562, "read_cosine": 0.30010974407196045, "beta_mean": 0.0007883887737989426, "nonfinite": 0.0, "weighted_memory_to_active": 0.024123163893818855, "mix_std": 0.26908132433891296, "beta_std": 0.001492984825745225, "beta_low": 0.9959716796875, "beta_high": 0.0, "orthogonal_to_active": 0.021804817020893097, "evidence_weights": [-0.2065626084804535, -0.27614155411720276, 0.12483774125576019], "agreement_weight": 0.10423891991376877, "orthogonal_weight": -0.9714967012405396}, {"head": 1, "memory_to_active": 0.1063426285982132, "contribution_to_active": 0.5288265943527222, "mix_mean": 0.5414303541183472, "mix_low": 0.0005817413330078125, "mix_high": 0.0012502670288085938, "read_cosine": 0.4029005467891693, "beta_mean": 0.0008776531904004514, "nonfinite": 0.0, "weighted_memory_to_active": 0.0631984993815422, "mix_std": 0.2670835554599762, "beta_std": 0.010314988903701305, "beta_low": 0.994873046875, "beta_high": 0.0, "orthogonal_to_active": 0.05340897664427757, "evidence_weights": [-0.20501016080379486, -0.2715579569339752, 1.0183887481689453], "agreement_weight": 0.19687658548355103, "orthogonal_weight": -0.9301060438156128}, {"head": 2, "memory_to_active": 0.5754892826080322, "contribution_to_active": 0.476094126701355, "mix_mean": 0.3043826222419739, "mix_low": 0.02959918975830078, "mix_high": 0.000102996826171875, "read_cosine": 0.44620755314826965, "beta_mean": 0.07663044333457947, "nonfinite": 0.0, "weighted_memory_to_active": 0.2379525899887085, "mix_std": 0.2357722669839859, "beta_std": 0.06060972064733505, "beta_low": 0.00439453125, "beta_high": 0.0, "orthogonal_to_active": 0.24699965119361877, "evidence_weights": [-0.22617225348949432, -0.05226624757051468, -0.009710459038615227], "agreement_weight": 0.025900252163410187, "orthogonal_weight": 1.305106520652771}, {"head": 3, "memory_to_active": 0.0281735397875309, "contribution_to_active": 0.742141842842102, "mix_mean": 0.7033462524414062, "mix_low": 0.00022983551025390625, "mix_high": 0.016307830810546875, "read_cosine": 0.07765422761440277, "beta_mean": 0.00023848767159506679, "nonfinite": 0.0, "weighted_memory_to_active": 0.020512474700808525, "mix_std": 0.2361852079629898, "beta_std": 0.004462129436433315, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.019602924585342407, "evidence_weights": [-0.29086369276046753, -0.34126734733581543, 1.5409362316131592], "agreement_weight": 0.4692395031452179, "orthogonal_weight": -0.9747443199157715}, {"head": 4, "memory_to_active": 0.556846022605896, "contribution_to_active": 0.5835010409355164, "mix_mean": 0.4172114133834839, "mix_low": 0.01943206787109375, "mix_high": 0.00396728515625, "read_cosine": -0.12417209148406982, "beta_mean": 0.0007955841720104218, "nonfinite": 0.0, "weighted_memory_to_active": 0.3500456213951111, "mix_std": 0.2863742709159851, "beta_std": 0.009474722668528557, "beta_low": 0.98077392578125, "beta_high": 0.0, "orthogonal_to_active": 0.03734266012907028, "evidence_weights": [-0.14523939788341522, -0.21795745193958282, -0.04571737349033356], "agreement_weight": 0.0005414187908172607, "orthogonal_weight": -0.11109618842601776}, {"head": 5, "memory_to_active": 0.06906513124704361, "contribution_to_active": 0.482330858707428, "mix_mean": 0.623116135597229, "mix_low": 0.007235527038574219, "mix_high": 0.0052967071533203125, "read_cosine": 0.21099889278411865, "beta_mean": 0.00038866643444634974, "nonfinite": 0.0, "weighted_memory_to_active": 0.04416374862194061, "mix_std": 0.27853041887283325, "beta_std": 0.004239700268954039, "beta_low": 0.9930419921875, "beta_high": 0.0, "orthogonal_to_active": 0.03407178819179535, "evidence_weights": [-0.23757082223892212, -0.3156808614730835, 1.333199381828308], "agreement_weight": 0.37734803557395935, "orthogonal_weight": -0.8264849185943604}]}, {"layer": 13, "memory_to_active": 0.12922289967536926, "contribution_to_active": 0.5319741368293762, "mix_mean": 0.5291985869407654, "mix_low": 0.005875269882380962, "mix_high": 0.00790262222290039, "read_cosine": 0.40541696548461914, "beta_mean": 0.014256575144827366, "nonfinite": 0.0, "weighted_memory_to_active": 0.05700845271348953, "mix_std": 0.2606610059738159, "beta_std": 0.019711367785930634, "beta_low": 0.7997843623161316, "beta_high": 0.0, "orthogonal_to_active": 0.057303108274936676, "output_influence": 0.5061896443367004, "residual_influence": 0.16914507746696472, "state_active": 0.11623896658420563, "state_memory": 0.04110327363014221, "state_peak": 100.84131622314453, "growth_active": 0.9216964244842529, "growth_memory": 0.9840460419654846, "heads": [{"head": 0, "memory_to_active": 0.024536289274692535, "contribution_to_active": 0.5795280933380127, "mix_mean": 0.6012929081916809, "mix_low": 0.00048542022705078125, "mix_high": 0.003444671630859375, "read_cosine": 0.4134220480918884, "beta_mean": 9.501638123765588e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015495408326387405, "mix_std": 0.2677032947540283, "beta_std": 0.0002091064234264195, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015546494163572788, "evidence_weights": [-0.19914016127586365, -0.2670995891094208, 0.7737160325050354], "agreement_weight": 0.14826752245426178, "orthogonal_weight": -1.113486886024475}, {"head": 1, "memory_to_active": 0.016262903809547424, "contribution_to_active": 0.5937721729278564, "mix_mean": 0.5609291791915894, "mix_low": 0.0018482208251953125, "mix_high": 0.0032262802124023438, "read_cosine": 0.19264736771583557, "beta_mean": 6.137949822004884e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009197406470775604, "mix_std": 0.2750362455844879, "beta_std": 0.00011848672875203192, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011992866173386574, "evidence_weights": [-0.1837058663368225, -0.26603731513023376, 0.6504254937171936], "agreement_weight": 0.39566904306411743, "orthogonal_weight": -1.352625846862793}, {"head": 2, "memory_to_active": 0.09732604026794434, "contribution_to_active": 0.46934986114501953, "mix_mean": 0.3798961341381073, "mix_low": 0.0017309188842773438, "mix_high": 0.000202178955078125, "read_cosine": 0.6374239921569824, "beta_mean": 0.01705704815685749, "nonfinite": 0.0, "weighted_memory_to_active": 0.04362139478325844, "mix_std": 0.23271624743938446, "beta_std": 0.0518965870141983, "beta_low": 0.80938720703125, "beta_high": 0.0, "orthogonal_to_active": 0.02357051707804203, "evidence_weights": [-0.18713872134685516, -0.20466801524162292, 0.41376936435699463], "agreement_weight": 0.009009546600282192, "orthogonal_weight": -0.8361505270004272}, {"head": 3, "memory_to_active": 0.16606798768043518, "contribution_to_active": 0.5838906168937683, "mix_mean": 0.6409999132156372, "mix_low": 0.023572921752929688, "mix_high": 0.030031204223632812, "read_cosine": -0.002659205812960863, "beta_mean": 0.0011902274563908577, "nonfinite": 0.0, "weighted_memory_to_active": 0.12171141803264618, "mix_std": 0.29512107372283936, "beta_std": 0.012368460185825825, "beta_low": 0.98822021484375, "beta_high": 0.0, "orthogonal_to_active": 0.10980621725320816, "evidence_weights": [-0.31826528906822205, -0.24596233665943146, 1.6834794282913208], "agreement_weight": 5.304490089416504, "orthogonal_weight": -0.9881972670555115}, {"head": 4, "memory_to_active": 0.4015010893344879, "contribution_to_active": 0.2639201879501343, "mix_mean": 0.3456454873085022, "mix_low": 0.004924774169921875, "mix_high": 1.9073486328125e-06, "read_cosine": 0.885178804397583, "beta_mean": 0.06679916381835938, "nonfinite": 0.0, "weighted_memory_to_active": 0.10749877244234085, "mix_std": 0.21965813636779785, "beta_std": 0.050902869552373886, "beta_low": 0.0023193359375, "beta_high": 0.0, "orthogonal_to_active": 0.1456531137228012, "evidence_weights": [-0.09510602056980133, -0.23520778119564056, 0.47597536444664], "agreement_weight": 0.002247855067253113, "orthogonal_weight": 2.217801332473755}, {"head": 5, "memory_to_active": 0.06964309513568878, "contribution_to_active": 0.7013839483261108, "mix_mean": 0.6464278101921082, "mix_low": 0.002689361572265625, "mix_high": 0.010509490966796875, "read_cosine": 0.3064887225627899, "beta_mean": 0.00033661676570773125, "nonfinite": 0.0, "weighted_memory_to_active": 0.04452630877494812, "mix_std": 0.273730993270874, "beta_std": 0.002772704930976033, "beta_low": 0.998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.037249427288770676, "evidence_weights": [-0.2212904840707779, -0.3046565055847168, 0.8901905417442322], "agreement_weight": 0.16065408289432526, "orthogonal_weight": -0.9163332581520081}]}, {"layer": 14, "memory_to_active": 0.23262347280979156, "contribution_to_active": 0.4193717837333679, "mix_mean": 0.4255524277687073, "mix_low": 0.024103641510009766, "mix_high": 0.005862077232450247, "read_cosine": 0.607574462890625, "beta_mean": 0.05948670208454132, "nonfinite": 0.0, "weighted_memory_to_active": 0.09998136013746262, "mix_std": 0.24489013850688934, "beta_std": 0.058391060680150986, "beta_low": 0.4921163022518158, "beta_high": 0.0, "orthogonal_to_active": 0.04173210635781288, "output_influence": 0.349409818649292, "residual_influence": 0.13325512409210205, "state_active": 0.0545894019305706, "state_memory": 0.015192602761089802, "state_peak": 39.78756332397461, "growth_active": 1.0383965969085693, "growth_memory": 1.2243397235870361, "heads": [{"head": 0, "memory_to_active": 0.349761426448822, "contribution_to_active": 0.19199709594249725, "mix_mean": 0.19710414111614227, "mix_low": 0.01590251922607422, "mix_high": 1.239776611328125e-05, "read_cosine": 0.77995765209198, "beta_mean": 0.06082608550786972, "nonfinite": 0.0, "weighted_memory_to_active": 0.07924934476613998, "mix_std": 0.18256154656410217, "beta_std": 0.05946268513798714, "beta_low": 0.02337646484375, "beta_high": 0.0, "orthogonal_to_active": 0.08100065588951111, "evidence_weights": [-0.11995083093643188, -0.03792960196733475, 0.4426078796386719], "agreement_weight": 0.02331899292767048, "orthogonal_weight": 1.6080495119094849}, {"head": 1, "memory_to_active": 0.4858326017856598, "contribution_to_active": 0.29794031381607056, "mix_mean": 0.2190714329481125, "mix_low": 0.11266326904296875, "mix_high": 9.5367431640625e-06, "read_cosine": 0.8706468343734741, "beta_mean": 0.24453656375408173, "nonfinite": 0.0, "weighted_memory_to_active": 0.22047603130340576, "mix_std": 0.24577496945858002, "beta_std": 0.24119022488594055, "beta_low": 0.096435546875, "beta_high": 0.0, "orthogonal_to_active": 0.012275421991944313, "evidence_weights": [-0.008042517118155956, -0.20224207639694214, -0.10324973613023758], "agreement_weight": 0.05583639815449715, "orthogonal_weight": 0.15624821186065674}, {"head": 2, "memory_to_active": 0.13001658022403717, "contribution_to_active": 0.581527829170227, "mix_mean": 0.590266227722168, "mix_low": 0.0028200149536132812, "mix_high": 0.0036096572875976562, "read_cosine": 0.42903852462768555, "beta_mean": 0.0019605718553066254, "nonfinite": 0.0, "weighted_memory_to_active": 0.08623459935188293, "mix_std": 0.266007661819458, "beta_std": 0.002599583240225911, "beta_low": 0.9827880859375, "beta_high": 0.0, "orthogonal_to_active": 0.06538572907447815, "evidence_weights": [-0.14431239664554596, -0.3172094225883484, 0.8950666189193726], "agreement_weight": 0.08692768961191177, "orthogonal_weight": -0.8722619414329529}, {"head": 3, "memory_to_active": 0.04813764989376068, "contribution_to_active": 0.6101517081260681, "mix_mean": 0.6648732423782349, "mix_low": 0.005290985107421875, "mix_high": 0.020180702209472656, "read_cosine": -0.03918785601854324, "beta_mean": 0.006006188690662384, "nonfinite": 0.0, "weighted_memory_to_active": 0.0295957550406456, "mix_std": 0.25751298666000366, "beta_std": 0.007455066312104464, "beta_low": 0.82818603515625, "beta_high": 0.0, "orthogonal_to_active": 0.022130385041236877, "evidence_weights": [-0.34972190856933594, -0.31066250801086426, 0.3031860888004303], "agreement_weight": 0.07344222068786621, "orthogonal_weight": -0.8080787658691406}, {"head": 4, "memory_to_active": 0.16331902146339417, "contribution_to_active": 0.49694347381591797, "mix_mean": 0.5080017447471619, "mix_low": 0.0041751861572265625, "mix_high": 0.009282112121582031, "read_cosine": 0.8680732250213623, "beta_mean": 0.04177074134349823, "nonfinite": 0.0, "weighted_memory_to_active": 0.09658947587013245, "mix_std": 0.2619316279888153, "beta_std": 0.029446857050061226, "beta_low": 0.05572509765625, "beta_high": 0.0, "orthogonal_to_active": 0.028682637959718704, "evidence_weights": [-0.218906968832016, -0.246835857629776, 0.42921751737594604], "agreement_weight": 0.132847398519516, "orthogonal_weight": -0.6081124544143677}, {"head": 5, "memory_to_active": 0.21867352724075317, "contribution_to_active": 0.337670236825943, "mix_mean": 0.37399768829345703, "mix_low": 0.0037698745727539062, "mix_high": 0.0020780563354492188, "read_cosine": 0.7369182109832764, "beta_mean": 0.0018200445920228958, "nonfinite": 0.0, "weighted_memory_to_active": 0.08774293959140778, "mix_std": 0.25555193424224854, "beta_std": 0.010191937908530235, "beta_low": 0.9661865234375, "beta_high": 0.0, "orthogonal_to_active": 0.04091779887676239, "evidence_weights": [-0.17759257555007935, -0.17176441848278046, 0.28631827235221863], "agreement_weight": 0.07909853756427765, "orthogonal_weight": -0.7290358543395996}]}, {"layer": 15, "memory_to_active": 0.05854372680187225, "contribution_to_active": 0.4845733940601349, "mix_mean": 0.4098910987377167, "mix_low": 0.008854389190673828, "mix_high": 0.0025393168907612562, "read_cosine": 0.6723548769950867, "beta_mean": 0.043275874108076096, "nonfinite": 0.0, "weighted_memory_to_active": 0.028460325673222542, "mix_std": 0.2831228971481323, "beta_std": 0.04867151752114296, "beta_low": 0.6999715566635132, "beta_high": 0.0, "orthogonal_to_active": 0.007994773797690868, "output_influence": 0.30480414628982544, "residual_influence": 0.16955168545246124, "state_active": 0.14419102668762207, "state_memory": 0.0106975007802248, "state_peak": 34.95370101928711, "growth_active": 1.2266143560409546, "growth_memory": 1.3602229356765747, "heads": [{"head": 0, "memory_to_active": 0.10408297181129456, "contribution_to_active": 0.47909459471702576, "mix_mean": 0.3723881244659424, "mix_low": 0.007109642028808594, "mix_high": 0.0045337677001953125, "read_cosine": 0.603480875492096, "beta_mean": 0.04507322609424591, "nonfinite": 0.0, "weighted_memory_to_active": 0.049747563898563385, "mix_std": 0.277099609375, "beta_std": 0.035296302288770676, "beta_low": 0.0672607421875, "beta_high": 0.0, "orthogonal_to_active": 0.028027167543768883, "evidence_weights": [-0.1866893321275711, -0.18852803111076355, 0.4249458611011505], "agreement_weight": 0.09141813963651657, "orthogonal_weight": -0.752987802028656}, {"head": 1, "memory_to_active": 0.12264694273471832, "contribution_to_active": 0.4161036014556885, "mix_mean": 0.43170106410980225, "mix_low": 0.0014848709106445312, "mix_high": 0.0058155059814453125, "read_cosine": 0.9180403351783752, "beta_mean": 0.0006980045582167804, "nonfinite": 0.0, "weighted_memory_to_active": 0.05134034901857376, "mix_std": 0.2851826548576355, "beta_std": 0.0020837222691625357, "beta_low": 0.993408203125, "beta_high": 0.0, "orthogonal_to_active": 0.0008565657190047204, "evidence_weights": [-0.151432603597641, -0.16077463328838348, 0.1903853565454483], "agreement_weight": 0.12617182731628418, "orthogonal_weight": 0.03259805217385292}, {"head": 2, "memory_to_active": 0.004435545299202204, "contribution_to_active": 0.4739578068256378, "mix_mean": 0.3380674719810486, "mix_low": 0.014865875244140625, "mix_high": 5.91278076171875e-05, "read_cosine": 0.786858081817627, "beta_mean": 6.051623131497763e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002274983562529087, "mix_std": 0.26937970519065857, "beta_std": 0.0001365005155093968, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006271625170484185, "evidence_weights": [-0.2065967321395874, -0.2692359983921051, 0.14740605652332306], "agreement_weight": 0.009776531718671322, "orthogonal_weight": 0.6823344826698303}, {"head": 3, "memory_to_active": 0.0021939356811344624, "contribution_to_active": 0.39770418405532837, "mix_mean": 0.3942221999168396, "mix_low": 0.0059833526611328125, "mix_high": 0.0002994537353515625, "read_cosine": 0.6191689372062683, "beta_mean": 1.280395645153476e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007855969597585499, "mix_std": 0.2863945960998535, "beta_std": 3.24359061778523e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000557475839741528, "evidence_weights": [-0.2547164559364319, -0.24887599050998688, 0.20649847388267517], "agreement_weight": 0.05098843574523926, "orthogonal_weight": -0.7603774666786194}, {"head": 4, "memory_to_active": 0.09091055393218994, "contribution_to_active": 0.6190356016159058, "mix_mean": 0.35986727476119995, "mix_low": 0.013915061950683594, "mix_high": 0.0008716583251953125, "read_cosine": 0.7818599939346313, "beta_mean": 0.21139054000377655, "nonfinite": 0.0, "weighted_memory_to_active": 0.049018654972314835, "mix_std": 0.3064733147621155, "beta_std": 0.23616358637809753, "beta_low": 0.1680908203125, "beta_high": 0.0, "orthogonal_to_active": 0.007047588005661964, "evidence_weights": [-0.210531547665596, -0.2524895966053009, -0.017245391383767128], "agreement_weight": 0.01195189543068409, "orthogonal_weight": 0.3524162173271179}, {"head": 5, "memory_to_active": 0.026992402970790863, "contribution_to_active": 0.5215444564819336, "mix_mean": 0.5631003379821777, "mix_low": 0.009767532348632812, "mix_high": 0.0036563873291015625, "read_cosine": 0.3247208595275879, "beta_mean": 0.0024201355408877134, "nonfinite": 0.0, "weighted_memory_to_active": 0.017594804987311363, "mix_std": 0.27420738339424133, "beta_std": 0.01831653341650963, "beta_low": 0.9710693359375, "beta_high": 0.0, "orthogonal_to_active": 0.010852683335542679, "evidence_weights": [-0.2895708382129669, -0.30471041798591614, 0.6918067932128906], "agreement_weight": 0.2934769093990326, "orthogonal_weight": 0.7960616946220398}]}]}, {"step": 50000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8945464369971887, "memory_to_active": 0.8111111521720886, "contribution_to_active": 0.5523838996887207, "mix_mean": 0.4045941233634949, "mix_low": 0.1369156539440155, "mix_high": 0.010128488764166832, "read_cosine": 0.350768506526947, "beta_mean": 0.07260415703058243, "nonfinite": 0.0, "weighted_memory_to_active": 0.21118953824043274, "mix_std": 0.22286579012870789, "beta_std": 0.0634402483701706, "beta_low": 0.5489718317985535, "beta_high": 2.2252401322475635e-05, "orthogonal_to_active": 0.05771967023611069, "samples": 4, "scored_tokens": 899, "normal_loss": 3.154865026473999, "normal_token_accuracy": 0.4282536208629608, "state_active": 0.061361100524663925, "state_memory": 0.01825188845396042, "state_peak": 120.72775268554688, "growth_active": 1.1319222450256348, "growth_memory": 1.3570773601531982, "output_influence": 0.41461992263793945, "residual_influence": 0.13930776715278625, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.271021604537964, "no_memory_loss_delta": 0.4913191497325897, "scrambled_loss_delta": 0.12154730409383774, "memory_only_loss_delta": 2.9596781730651855, "no_evidence_loss_delta": 0.2927956283092499, "no_agreement_loss_delta": 0.0353635735809803, "no_orthogonal_loss_delta": -0.009306992404162884}, {"index": 94, "tokens": 256, "normal_loss": 3.268099546432495, "no_memory_loss_delta": 0.6370645761489868, "scrambled_loss_delta": 0.32377099990844727, "memory_only_loss_delta": 3.063676595687866, "no_evidence_loss_delta": 0.30916038155555725, "no_agreement_loss_delta": 0.04331636428833008, "no_orthogonal_loss_delta": 0.0636560320854187}, {"index": 188, "tokens": 256, "normal_loss": 3.47717547416687, "no_memory_loss_delta": 0.728430449962616, "scrambled_loss_delta": 0.12727604806423187, "memory_only_loss_delta": 2.912777900695801, "no_evidence_loss_delta": 0.40662434697151184, "no_agreement_loss_delta": 0.053956277668476105, "no_orthogonal_loss_delta": 0.06580532342195511}, {"index": 283, "tokens": 131, "normal_loss": 2.0767295360565186, "no_memory_loss_delta": 0.2422139048576355, "scrambled_loss_delta": 0.20980118215084076, "memory_only_loss_delta": 2.395066022872925, "no_evidence_loss_delta": 0.08594770729541779, "no_agreement_loss_delta": -0.002689952263608575, "no_orthogonal_loss_delta": -0.000468552578240633}], "agreement_weight_abs_mean": 0.20193922519683838, "agreement_weight_abs_max": 5.385779857635498, "agreement_weight_mean": 0.1764681190252304, "orthogonal_weight_abs_mean": 0.831978976726532, "orthogonal_weight_abs_max": 8.613619804382324, "orthogonal_weight_mean": -0.12168671935796738, "no_memory_loss": 3.7189078330993652, "no_memory_loss_delta": 0.5640426874160767, "no_memory_token_accuracy_gain": 0.06229143589735031, "no_memory_token_win_fraction": 0.6896551847457886, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 43.109549440255726, "scrambled_loss": 3.3484890460968018, "scrambled_loss_delta": 0.19362415373325348, "scrambled_token_accuracy_gain": 0.03337041288614273, "scrambled_token_win_fraction": 0.6162402629852295, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 17.603246880060937, "memory_only_loss": 6.04852819442749, "memory_only_loss_delta": 2.8936636447906494, "memory_only_token_accuracy_gain": 0.253615140914917, "memory_only_token_win_fraction": 0.8954393863677979, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 94.4627026365298, "no_evidence_loss": 3.4545934200286865, "no_evidence_loss_delta": 0.29972827434539795, "no_evidence_token_accuracy_gain": 0.04338153451681137, "no_evidence_token_win_fraction": 0.6273637413978577, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 25.898045265075474, "no_agreement_loss": 3.192242383956909, "no_agreement_loss_delta": 0.037377625703811646, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.5094549655914307, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 3.6687704827779126, "no_orthogonal_loss": 3.189012050628662, "no_orthogonal_loss_delta": 0.034146975725889206, "no_orthogonal_token_accuracy_gain": 0.01223581749945879, "no_orthogonal_token_win_fraction": 0.5817574858665466, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 3.357054747164962, "short_no_memory_loss_delta": 0.5056391358375549, "long_context_memory_gain": 0.05840355157852173, "normal_context_loss_gain": 0.15755586326122284, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.491569995880127, "contribution_to_active": 1.1825027465820312, "mix_mean": 0.10464383661746979, "mix_low": 0.543016791343689, "mix_high": 0.005230903625488281, "read_cosine": 0.42584115266799927, "beta_mean": 0.21840651333332062, "nonfinite": 0.0, "weighted_memory_to_active": 1.2244489192962646, "mix_std": 0.08931339532136917, "beta_std": 0.14352843165397644, "beta_low": 0.043914794921875, "beta_high": 0.0, "orthogonal_to_active": 0.04513135924935341, "output_influence": 0.21672847867012024, "residual_influence": 0.25416097044944763, "state_active": 0.06762136518955231, "state_memory": 0.04467565193772316, "state_peak": 43.5335693359375, "growth_active": 1.2766412496566772, "growth_memory": 1.2695540189743042, "heads": [{"head": 0, "memory_to_active": 2.585735321044922, "contribution_to_active": 0.47345542907714844, "mix_mean": 0.020802535116672516, "mix_low": 0.6181392669677734, "mix_high": 9.5367431640625e-07, "read_cosine": 0.2283533811569214, "beta_mean": 0.19350115954875946, "nonfinite": 0.0, "weighted_memory_to_active": 0.5893323421478271, "mix_std": 0.06323995441198349, "beta_std": 0.09668596088886261, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0604858472943306, "evidence_weights": [-0.011631377972662449, -0.05573480203747749, -0.061803627759218216], "agreement_weight": -0.17930492758750916, "orthogonal_weight": -0.11326730996370316}, {"head": 1, "memory_to_active": 6.2132568359375, "contribution_to_active": 2.7275593280792236, "mix_mean": 0.017378922551870346, "mix_low": 0.6621780395507812, "mix_high": 0.0, "read_cosine": 0.4433631896972656, "beta_mean": 0.43076419830322266, "nonfinite": 0.0, "weighted_memory_to_active": 3.0371296405792236, "mix_std": 0.05475369840860367, "beta_std": 0.22231078147888184, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.015537220053374767, "evidence_weights": [0.027291730046272278, -0.09905125945806503, 0.5550130009651184], "agreement_weight": 0.028212735429406166, "orthogonal_weight": -0.01093321293592453}, {"head": 2, "memory_to_active": 2.7372725009918213, "contribution_to_active": 0.49051180481910706, "mix_mean": 0.01274622417986393, "mix_low": 0.6929521560668945, "mix_high": 0.0, "read_cosine": 0.3098088800907135, "beta_mean": 0.2723439335823059, "nonfinite": 0.0, "weighted_memory_to_active": 0.5697342157363892, "mix_std": 0.029970664530992508, "beta_std": 0.2010945826768875, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.042470335960388184, "evidence_weights": [0.06649783253669739, 0.061775628477334976, 0.09942205995321274], "agreement_weight": 0.012534985318779945, "orthogonal_weight": -0.18616056442260742}, {"head": 3, "memory_to_active": 7.417752265930176, "contribution_to_active": 2.8784961700439453, "mix_mean": 0.0336112454533577, "mix_low": 0.4766073226928711, "mix_high": 0.0, "read_cosine": 0.2820056080818176, "beta_mean": 0.3153768479824066, "nonfinite": 0.0, "weighted_memory_to_active": 3.042935371398926, "mix_std": 0.07815588265657425, "beta_std": 0.26213538646698, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12951816618442535, "evidence_weights": [-0.04387516528367996, -0.04813874512910843, -0.16067606210708618], "agreement_weight": -0.004938886966556311, "orthogonal_weight": -0.04961904510855675}, {"head": 4, "memory_to_active": 1.8830640316009521, "contribution_to_active": 0.06907375901937485, "mix_mean": 0.007781324442476034, "mix_low": 0.7929782867431641, "mix_high": 0.0, "read_cosine": 0.35067057609558105, "beta_mean": 0.07438992708921432, "nonfinite": 0.0, "weighted_memory_to_active": 0.04849676042795181, "mix_std": 0.019496304914355278, "beta_std": 0.058568380773067474, "beta_low": 0.0201416015625, "beta_high": 0.0, "orthogonal_to_active": 0.004642045125365257, "evidence_weights": [0.149628683924675, 0.10760119557380676, -0.23300282657146454], "agreement_weight": -0.15452894568443298, "orthogonal_weight": 0.08507516235113144}, {"head": 5, "memory_to_active": 0.11233648657798767, "contribution_to_active": 0.45591989159584045, "mix_mean": 0.5355427265167236, "mix_low": 0.015245437622070312, "mix_high": 0.03138446807861328, "read_cosine": 0.9408453702926636, "beta_mean": 0.024062909185886383, "nonfinite": 0.0, "weighted_memory_to_active": 0.059065431356430054, "mix_std": 0.290263831615448, "beta_std": 0.020375516265630722, "beta_low": 0.24334716796875, "beta_high": 0.0, "orthogonal_to_active": 0.018134526908397675, "evidence_weights": [-0.18278922140598297, -0.21865715086460114, 0.3736502528190613], "agreement_weight": 0.2908257842063904, "orthogonal_weight": -0.6898491382598877}]}, {"layer": 1, "memory_to_active": 2.2752575874328613, "contribution_to_active": 0.5035822987556458, "mix_mean": 0.2080712765455246, "mix_low": 0.36397045850753784, "mix_high": 0.0008746783714741468, "read_cosine": 0.29872235655784607, "beta_mean": 0.17357246577739716, "nonfinite": 0.0, "weighted_memory_to_active": 0.3467443585395813, "mix_std": 0.11715342104434967, "beta_std": 0.1278993785381317, "beta_low": 0.3407389521598816, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.047477900981903076, "output_influence": 0.22948992252349854, "residual_influence": 0.06038574129343033, "state_active": 0.03035980463027954, "state_memory": 0.03621016815304756, "state_peak": 40.28367233276367, "growth_active": 1.1949011087417603, "growth_memory": 1.513974666595459, "heads": [{"head": 0, "memory_to_active": 0.045009151101112366, "contribution_to_active": 0.48458874225616455, "mix_mean": 0.5035416483879089, "mix_low": 0.00682830810546875, "mix_high": 0.0013980865478515625, "read_cosine": 0.024433990940451622, "beta_mean": 0.0003063298063352704, "nonfinite": 0.0, "weighted_memory_to_active": 0.02575678564608097, "mix_std": 0.2817624807357788, "beta_std": 0.00043761898996308446, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.024350149556994438, "evidence_weights": [-0.20528505742549896, -0.2637678384780884, 0.30460745096206665], "agreement_weight": 0.18486636877059937, "orthogonal_weight": -0.9943129420280457}, {"head": 1, "memory_to_active": 3.0549206733703613, "contribution_to_active": 0.3211621940135956, "mix_mean": 0.0241555068641901, "mix_low": 0.4616508483886719, "mix_high": 0.0, "read_cosine": 0.4797104597091675, "beta_mean": 0.2988544702529907, "nonfinite": 0.0, "weighted_memory_to_active": 0.352230966091156, "mix_std": 0.04974504932761192, "beta_std": 0.1989600658416748, "beta_low": 0.00177001953125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.024066261947155, "evidence_weights": [0.014523315243422985, -0.05199360474944115, 0.04596399515867233], "agreement_weight": -0.004732145462185144, "orthogonal_weight": 0.17935068905353546}, {"head": 2, "memory_to_active": 0.06463581323623657, "contribution_to_active": 0.5360051989555359, "mix_mean": 0.6573917865753174, "mix_low": 0.0005855560302734375, "mix_high": 0.0038442611694335938, "read_cosine": 0.514487624168396, "beta_mean": 0.002797394758090377, "nonfinite": 0.0, "weighted_memory_to_active": 0.04495095834136009, "mix_std": 0.23266787827014923, "beta_std": 0.004327674396336079, "beta_low": 0.95556640625, "beta_high": 0.0, "orthogonal_to_active": 0.028010062873363495, "evidence_weights": [-0.2843089997768402, -0.35885587334632874, 0.6964809894561768], "agreement_weight": 0.03358921408653259, "orthogonal_weight": -0.7382341027259827}, {"head": 3, "memory_to_active": 2.908616542816162, "contribution_to_active": 0.15667451918125153, "mix_mean": 0.021429285407066345, "mix_low": 0.5936088562011719, "mix_high": 2.86102294921875e-06, "read_cosine": 0.2015398144721985, "beta_mean": 0.3182007074356079, "nonfinite": 0.0, "weighted_memory_to_active": 0.245819091796875, "mix_std": 0.05868832767009735, "beta_std": 0.2540057897567749, "beta_low": 0.04510498046875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.08452159911394119, "evidence_weights": [0.04109766334295273, -0.01233705971390009, 0.033526938408613205], "agreement_weight": -0.035757649689912796, "orthogonal_weight": -0.4690123200416565}, {"head": 4, "memory_to_active": 3.6614606380462646, "contribution_to_active": 1.436084508895874, "mix_mean": 0.03417150676250458, "mix_low": 0.30779075622558594, "mix_high": 9.5367431640625e-07, "read_cosine": 0.33904922008514404, "beta_mean": 0.1649174839258194, "nonfinite": 0.0, "weighted_memory_to_active": 1.346874713897705, "mix_std": 0.058978427201509476, "beta_std": 0.11555998027324677, "beta_low": 0.01654052734375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.10745318233966827, "evidence_weights": [-0.03441372886300087, -0.004533365834504366, 0.07330582290887833], "agreement_weight": -0.016012800857424736, "orthogonal_weight": 0.10676810890436172}, {"head": 5, "memory_to_active": 3.9169015884399414, "contribution_to_active": 0.08697875589132309, "mix_mean": 0.007737831678241491, "mix_low": 0.8133583068847656, "mix_high": 1.9073486328125e-06, "read_cosine": 0.23311303555965424, "beta_mean": 0.2563583254814148, "nonfinite": 0.0, "weighted_memory_to_active": 0.06483357399702072, "mix_std": 0.021078314632177353, "beta_std": 0.19410507380962372, "beta_low": 0.02545166015625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.016466116532683372, "evidence_weights": [0.15454135835170746, 0.07503212243318558, 0.13793189823627472], "agreement_weight": -0.13245058059692383, "orthogonal_weight": 0.25976285338401794}]}, {"layer": 2, "memory_to_active": 1.0109875202178955, "contribution_to_active": 0.3514551520347595, "mix_mean": 0.3880773186683655, "mix_low": 0.25321802496910095, "mix_high": 0.003673553466796875, "read_cosine": 0.3111085295677185, "beta_mean": 0.09032965451478958, "nonfinite": 0.0, "weighted_memory_to_active": 0.09035991132259369, "mix_std": 0.18987879157066345, "beta_std": 0.07024028897285461, "beta_low": 0.5630391836166382, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.021341366693377495, "output_influence": 0.37187764048576355, "residual_influence": 0.09687726199626923, "state_active": 0.0359952449798584, "state_memory": 0.015954487025737762, "state_peak": 31.923330307006836, "growth_active": 1.01650071144104, "growth_memory": 1.5787118673324585, "heads": [{"head": 0, "memory_to_active": 0.009584720246493816, "contribution_to_active": 0.332287073135376, "mix_mean": 0.5936030149459839, "mix_low": 0.0009870529174804688, "mix_high": 0.00579833984375, "read_cosine": 0.14544105529785156, "beta_mean": 7.583177648484707e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005729292053729296, "mix_std": 0.2749103903770447, "beta_std": 0.00011007604916812852, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005626034922897816, "evidence_weights": [-0.35750770568847656, -0.20978201925754547, 0.12893353402614594], "agreement_weight": 0.1226041167974472, "orthogonal_weight": -1.017690658569336}, {"head": 1, "memory_to_active": 2.8440017700195312, "contribution_to_active": 0.03616756200790405, "mix_mean": 0.003911497071385384, "mix_low": 0.9216651916503906, "mix_high": 0.0, "read_cosine": 0.16446733474731445, "beta_mean": 0.2441038191318512, "nonfinite": 0.0, "weighted_memory_to_active": 0.026317492127418518, "mix_std": 0.008760887198150158, "beta_std": 0.15308335423469543, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.010053051635622978, "evidence_weights": [0.15043173730373383, 0.13502947986125946, 0.5096520781517029], "agreement_weight": 0.03490042686462402, "orthogonal_weight": 0.39354953169822693}, {"head": 2, "memory_to_active": 0.0136586744338274, "contribution_to_active": 0.4088088870048523, "mix_mean": 0.5272120833396912, "mix_low": 0.0012722015380859375, "mix_high": 0.0025396347045898438, "read_cosine": 0.30316561460494995, "beta_mean": 3.060287781408988e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0052387649193406105, "mix_std": 0.27566900849342346, "beta_std": 4.04595339205116e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005387527053244412, "evidence_weights": [-0.24708695709705353, -0.2875814139842987, 0.5327553749084473], "agreement_weight": -0.015662984922528267, "orthogonal_weight": -0.1228717491030693}, {"head": 3, "memory_to_active": 0.15756724774837494, "contribution_to_active": 0.5226858854293823, "mix_mean": 0.5585265159606934, "mix_low": 0.0003147125244140625, "mix_high": 0.0076618194580078125, "read_cosine": 0.6285334825515747, "beta_mean": 0.03068896010518074, "nonfinite": 0.0, "weighted_memory_to_active": 0.09537895768880844, "mix_std": 0.2633383870124817, "beta_std": 0.03844314068555832, "beta_low": 0.347900390625, "beta_high": 0.0, "orthogonal_to_active": 0.06400895118713379, "evidence_weights": [-0.20134088397026062, -0.3132602572441101, 0.8316110968589783], "agreement_weight": 0.053057797253131866, "orthogonal_weight": -0.8811335563659668}, {"head": 4, "memory_to_active": 3.031489372253418, "contribution_to_active": 0.2937469482421875, "mix_mean": 0.01962525211274624, "mix_low": 0.5941429138183594, "mix_high": 0.0, "read_cosine": 0.2147957682609558, "beta_mean": 0.2670432925224304, "nonfinite": 0.0, "weighted_memory_to_active": 0.40395843982696533, "mix_std": 0.050615761429071426, "beta_std": 0.22971509397029877, "beta_low": 0.029296875, "beta_high": 0.000732421875, "orthogonal_to_active": 0.04735010117292404, "evidence_weights": [0.02122032269835472, 0.007380936294794083, -0.0026769330725073814], "agreement_weight": 0.009153660386800766, "orthogonal_weight": -0.14234742522239685}, {"head": 5, "memory_to_active": 0.009622816927731037, "contribution_to_active": 0.5150344967842102, "mix_mean": 0.6255854964256287, "mix_low": 0.0009260177612304688, "mix_high": 0.006041526794433594, "read_cosine": 0.410247802734375, "beta_mean": 3.5416273021837696e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005536474287509918, "mix_std": 0.26597830653190613, "beta_std": 4.958878707839176e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004712966037914157, "evidence_weights": [-0.2765240967273712, -0.2911307215690613, 1.01156485080719], "agreement_weight": 0.1624898910522461, "orthogonal_weight": -0.11525649577379227}]}, {"layer": 3, "memory_to_active": 0.4665513038635254, "contribution_to_active": 0.458859384059906, "mix_mean": 0.46825987100601196, "mix_low": 0.06057993695139885, "mix_high": 0.008997282013297081, "read_cosine": 0.20277494192123413, "beta_mean": 0.026410456746816635, "nonfinite": 0.0, "weighted_memory_to_active": 0.12985362112522125, "mix_std": 0.2620115578174591, "beta_std": 0.025104273110628128, "beta_low": 0.7509359121322632, "beta_high": 0.0, "orthogonal_to_active": 0.05375199392437935, "output_influence": 0.37531134486198425, "residual_influence": 0.08249007165431976, "state_active": 0.02381901443004608, "state_memory": 0.007101806811988354, "state_peak": 23.049205780029297, "growth_active": 1.006657361984253, "growth_memory": 1.729482650756836, "heads": [{"head": 0, "memory_to_active": 0.4111036956310272, "contribution_to_active": 0.5797673463821411, "mix_mean": 0.5620916485786438, "mix_low": 0.002803802490234375, "mix_high": 0.021364212036132812, "read_cosine": 0.21227185428142548, "beta_mean": 0.011752136051654816, "nonfinite": 0.0, "weighted_memory_to_active": 0.24398095905780792, "mix_std": 0.31305962800979614, "beta_std": 0.015637028962373734, "beta_low": 0.64013671875, "beta_high": 0.0, "orthogonal_to_active": 0.1721068024635315, "evidence_weights": [-0.21095912158489227, -0.24475820362567902, 0.9121102094650269], "agreement_weight": 0.11517186462879181, "orthogonal_weight": -0.7429182529449463}, {"head": 1, "memory_to_active": 0.01351867988705635, "contribution_to_active": 0.4156993627548218, "mix_mean": 0.5316897034645081, "mix_low": 0.00566864013671875, "mix_high": 0.0064239501953125, "read_cosine": 0.1883459985256195, "beta_mean": 7.555878983112052e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007312100846320391, "mix_std": 0.30270665884017944, "beta_std": 0.00016869392129592597, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0056932708248496056, "evidence_weights": [-0.25531888008117676, -0.20931817591190338, 0.6410576701164246], "agreement_weight": 0.4673278331756592, "orthogonal_weight": -0.8552935123443604}, {"head": 2, "memory_to_active": 2.093899965286255, "contribution_to_active": 0.3028750419616699, "mix_mean": 0.049349769949913025, "mix_low": 0.3356132507324219, "mix_high": 0.0, "read_cosine": 0.21121464669704437, "beta_mean": 0.1404852569103241, "nonfinite": 0.0, "weighted_memory_to_active": 0.3316314220428467, "mix_std": 0.10368052870035172, "beta_std": 0.1245986670255661, "beta_low": 0.020751953125, "beta_high": 0.0, "orthogonal_to_active": 0.00016647552547510713, "evidence_weights": [-0.0848071351647377, 0.009576650336384773, 0.009815382771193981], "agreement_weight": -0.010409773327410221, "orthogonal_weight": -0.0006285138661041856}, {"head": 3, "memory_to_active": 0.022286128252744675, "contribution_to_active": 0.4850507974624634, "mix_mean": 0.5707952976226807, "mix_low": 0.0071258544921875, "mix_high": 0.010558128356933594, "read_cosine": 0.2920725345611572, "beta_mean": 0.00010718663543229923, "nonfinite": 0.0, "weighted_memory_to_active": 0.014934462495148182, "mix_std": 0.295700341463089, "beta_std": 0.000252872530836612, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00404129596427083, "evidence_weights": [-0.2406805157661438, -0.22864177823066711, 2.139314651489258], "agreement_weight": 1.9188883304595947, "orthogonal_weight": -0.3585667014122009}, {"head": 4, "memory_to_active": 0.02234908938407898, "contribution_to_active": 0.31071901321411133, "mix_mean": 0.40295034646987915, "mix_low": 0.009489059448242188, "mix_high": 0.000865936279296875, "read_cosine": 0.17384976148605347, "beta_mean": 0.00014868928701616824, "nonfinite": 0.0, "weighted_memory_to_active": 0.012991632334887981, "mix_std": 0.28432339429855347, "beta_std": 0.00022176966012921184, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012407032772898674, "evidence_weights": [-0.1790255457162857, -0.18841366469860077, 0.1319492906332016], "agreement_weight": -0.028559485450387, "orthogonal_weight": -1.01239812374115}, {"head": 5, "memory_to_active": 0.23615022003650665, "contribution_to_active": 0.6590445041656494, "mix_mean": 0.6926822662353516, "mix_low": 0.0027790069580078125, "mix_high": 0.014771461486816406, "read_cosine": 0.13889479637145996, "beta_mean": 0.005893920548260212, "nonfinite": 0.0, "weighted_memory_to_active": 0.16827113926410675, "mix_std": 0.2725987434387207, "beta_std": 0.009746589697897434, "beta_low": 0.8447265625, "beta_high": 0.0, "orthogonal_to_active": 0.12809708714485168, "evidence_weights": [-0.25696584582328796, -0.31715357303619385, 0.4753768742084503], "agreement_weight": 0.1471158117055893, "orthogonal_weight": -0.7917575836181641}]}, {"layer": 4, "memory_to_active": 1.0311050415039062, "contribution_to_active": 0.440643310546875, "mix_mean": 0.3605443835258484, "mix_low": 0.2779940068721771, "mix_high": 0.009264628402888775, "read_cosine": 0.3289806842803955, "beta_mean": 0.11633097380399704, "nonfinite": 0.0, "weighted_memory_to_active": 0.13178615272045135, "mix_std": 0.15647563338279724, "beta_std": 0.1051655188202858, "beta_low": 0.3773091733455658, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.038337092846632004, "output_influence": 0.4527842700481415, "residual_influence": 0.12920665740966797, "state_active": 0.039540454745292664, "state_memory": 0.018294550478458405, "state_peak": 36.00987243652344, "growth_active": 1.2693966627120972, "growth_memory": 1.5709508657455444, "heads": [{"head": 0, "memory_to_active": 0.20445291697978973, "contribution_to_active": 0.7372679114341736, "mix_mean": 0.6781517863273621, "mix_low": 0.013721466064453125, "mix_high": 0.023079872131347656, "read_cosine": 0.5272660255432129, "beta_mean": 0.2672760486602783, "nonfinite": 0.0, "weighted_memory_to_active": 0.1638094186782837, "mix_std": 0.34556934237480164, "beta_std": 0.2705572545528412, "beta_low": 0.22149658203125, "beta_high": 0.0, "orthogonal_to_active": 0.06589190661907196, "evidence_weights": [-0.267181932926178, -0.36876434087753296, 0.056674499064683914], "agreement_weight": 0.010853899642825127, "orthogonal_weight": 0.5367056727409363}, {"head": 1, "memory_to_active": 0.054204702377319336, "contribution_to_active": 0.6461105346679688, "mix_mean": 0.6761150360107422, "mix_low": 0.0001010894775390625, "mix_high": 0.014677047729492188, "read_cosine": 0.6457709074020386, "beta_mean": 0.0009265111293643713, "nonfinite": 0.0, "weighted_memory_to_active": 0.03907698765397072, "mix_std": 0.2444155067205429, "beta_std": 0.005694008432328701, "beta_low": 0.98345947265625, "beta_high": 0.0, "orthogonal_to_active": 0.021372778341174126, "evidence_weights": [-0.23857203125953674, -0.3549725413322449, 0.6697653532028198], "agreement_weight": 0.26195183396339417, "orthogonal_weight": -0.7975513935089111}, {"head": 2, "memory_to_active": 1.86043381690979, "contribution_to_active": 0.26720041036605835, "mix_mean": 0.024007046595215797, "mix_low": 0.557612419128418, "mix_high": 1.621246337890625e-05, "read_cosine": 0.237687349319458, "beta_mean": 0.07819762825965881, "nonfinite": 0.0, "weighted_memory_to_active": 0.26957231760025024, "mix_std": 0.05471618473529816, "beta_std": 0.0704965740442276, "beta_low": 0.02996826171875, "beta_high": 0.0, "orthogonal_to_active": 0.04613644629716873, "evidence_weights": [0.06200212985277176, 0.013486894778907299, 0.10853654146194458], "agreement_weight": 0.12906989455223083, "orthogonal_weight": -0.26988455653190613}, {"head": 3, "memory_to_active": 1.7303105592727661, "contribution_to_active": 0.13503064215183258, "mix_mean": 0.030300933867692947, "mix_low": 0.4328145980834961, "mix_high": 0.0, "read_cosine": 0.18886394798755646, "beta_mean": 0.13646620512008667, "nonfinite": 0.0, "weighted_memory_to_active": 0.19391781091690063, "mix_std": 0.05300527811050415, "beta_std": 0.12147711217403412, "beta_low": 0.02886962890625, "beta_high": 0.0, "orthogonal_to_active": 0.003624006174504757, "evidence_weights": [-0.02649724669754505, 0.05469273030757904, -0.2462778240442276], "agreement_weight": 0.07350728660821915, "orthogonal_weight": -0.02898702397942543}, {"head": 4, "memory_to_active": 0.010609382763504982, "contribution_to_active": 0.6535816788673401, "mix_mean": 0.7417564392089844, "mix_low": 9.34600830078125e-05, "mix_high": 0.017813682556152344, "read_cosine": 0.14597484469413757, "beta_mean": 2.2719617845723405e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007847771979868412, "mix_std": 0.21468402445316315, "beta_std": 6.487923383247107e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003544615115970373, "evidence_weights": [-0.34051594138145447, -0.3294275403022766, 0.5589717626571655], "agreement_weight": 0.34123915433883667, "orthogonal_weight": -0.47970402240753174}, {"head": 5, "memory_to_active": 2.3266184329986572, "contribution_to_active": 0.20466865599155426, "mix_mean": 0.012934987433254719, "mix_low": 0.6636209487915039, "mix_high": 9.5367431640625e-07, "read_cosine": 0.22832104563713074, "beta_mean": 0.21509671211242676, "nonfinite": 0.0, "weighted_memory_to_active": 0.1164926066994667, "mix_std": 0.02646341547369957, "beta_std": 0.1627032458782196, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08945281058549881, "evidence_weights": [0.05094742402434349, -0.027311431244015694, -0.29693564772605896], "agreement_weight": -0.07341797649860382, "orthogonal_weight": 0.9394866824150085}]}, {"layer": 5, "memory_to_active": 0.40344828367233276, "contribution_to_active": 0.5140080451965332, "mix_mean": 0.4947531521320343, "mix_low": 0.056918781250715256, "mix_high": 0.01695362851023674, "read_cosine": 0.09157227724790573, "beta_mean": 0.053821973502635956, "nonfinite": 0.0, "weighted_memory_to_active": 0.1357836127281189, "mix_std": 0.2560323476791382, "beta_std": 0.06961563229560852, "beta_low": 0.6436361074447632, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05191628634929657, "output_influence": 0.5337405800819397, "residual_influence": 0.15123535692691803, "state_active": 0.04332399740815163, "state_memory": 0.010745223611593246, "state_peak": 66.021484375, "growth_active": 1.1063997745513916, "growth_memory": 1.258486270904541, "heads": [{"head": 0, "memory_to_active": 0.3692101836204529, "contribution_to_active": 0.5007956027984619, "mix_mean": 0.39268338680267334, "mix_low": 0.017281532287597656, "mix_high": 0.0011873245239257812, "read_cosine": -0.03635506331920624, "beta_mean": 0.032796360552310944, "nonfinite": 0.0, "weighted_memory_to_active": 0.11730825901031494, "mix_std": 0.2772577404975891, "beta_std": 0.05430226027965546, "beta_low": 0.37957763671875, "beta_high": 0.0, "orthogonal_to_active": 0.04394640028476715, "evidence_weights": [-0.13761860132217407, -0.1626274436712265, -0.010882746428251266], "agreement_weight": -0.02127298340201378, "orthogonal_weight": -0.3791375458240509}, {"head": 1, "memory_to_active": 0.01159657258540392, "contribution_to_active": 0.5419149398803711, "mix_mean": 0.6889122724533081, "mix_low": 0.0007686614990234375, "mix_high": 0.03253459930419922, "read_cosine": 0.2024582326412201, "beta_mean": 2.9126465960871428e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007097349036484957, "mix_std": 0.27479302883148193, "beta_std": 6.268643483053893e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006662285886704922, "evidence_weights": [-0.2548642158508301, -0.2911829650402069, 0.8763515949249268], "agreement_weight": 0.653283953666687, "orthogonal_weight": -0.9848629832267761}, {"head": 2, "memory_to_active": 0.01091031450778246, "contribution_to_active": 0.5662137866020203, "mix_mean": 0.6517820954322815, "mix_low": 0.0026636123657226562, "mix_high": 0.01933574676513672, "read_cosine": -0.48425620794296265, "beta_mean": 7.834753341739997e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006755338050425053, "mix_std": 0.2677933871746063, "beta_std": 0.00011816263577202335, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002942063147202134, "evidence_weights": [-0.2583286762237549, -0.33798521757125854, 0.37585094571113586], "agreement_weight": -0.035644277930259705, "orthogonal_weight": -0.5279617309570312}, {"head": 3, "memory_to_active": 1.5069955587387085, "contribution_to_active": 0.21886686980724335, "mix_mean": 0.10684666782617569, "mix_low": 0.1379871368408203, "mix_high": 6.67572021484375e-06, "read_cosine": 0.15566718578338623, "beta_mean": 0.10125696659088135, "nonfinite": 0.0, "weighted_memory_to_active": 0.34288960695266724, "mix_std": 0.14423280954360962, "beta_std": 0.1469961702823639, "beta_low": 0.201416015625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1280032992362976, "evidence_weights": [-0.06765377521514893, -0.09260422736406326, 0.017095908522605896], "agreement_weight": 0.04206658527255058, "orthogonal_weight": -0.481110543012619}, {"head": 4, "memory_to_active": 0.4303146302700043, "contribution_to_active": 0.5803147554397583, "mix_mean": 0.3349914252758026, "mix_low": 0.18061351776123047, "mix_high": 0.006665229797363281, "read_cosine": 0.549389660358429, "beta_mean": 0.18850943446159363, "nonfinite": 0.0, "weighted_memory_to_active": 0.2791398763656616, "mix_std": 0.3429187536239624, "beta_std": 0.21498338878154755, "beta_low": 0.283447265625, "beta_high": 0.0, "orthogonal_to_active": 0.09164117276668549, "evidence_weights": [-0.11738936603069305, -0.27198460698127747, -0.060459088534116745], "agreement_weight": 0.038864802569150925, "orthogonal_weight": 0.501961350440979}, {"head": 5, "memory_to_active": 0.09166240692138672, "contribution_to_active": 0.6759419441223145, "mix_mean": 0.7933028936386108, "mix_low": 0.0021982192993164062, "mix_high": 0.0419921875, "read_cosine": 0.1625298410654068, "beta_mean": 0.00026158030959777534, "nonfinite": 0.0, "weighted_memory_to_active": 0.06151125207543373, "mix_std": 0.2291984111070633, "beta_std": 0.0012311004102230072, "beta_low": 0.99737548828125, "beta_high": 0.0, "orthogonal_to_active": 0.0383024699985981, "evidence_weights": [-0.31326743960380554, -0.33264029026031494, 1.9797453880310059], "agreement_weight": 0.4566107988357544, "orthogonal_weight": -0.6591057181358337}]}, {"layer": 6, "memory_to_active": 0.3323368430137634, "contribution_to_active": 0.6170272827148438, "mix_mean": 0.578070878982544, "mix_low": 0.05477190017700195, "mix_high": 0.03258037567138672, "read_cosine": 0.30055785179138184, "beta_mean": 0.06660228967666626, "nonfinite": 0.0, "weighted_memory_to_active": 0.1643545776605606, "mix_std": 0.23922617733478546, "beta_std": 0.07533999532461166, "beta_low": 0.523223876953125, "beta_high": 0.0, "orthogonal_to_active": 0.043319184333086014, "output_influence": 0.5322926044464111, "residual_influence": 0.17566058039665222, "state_active": 0.046800076961517334, "state_memory": 0.00865219347178936, "state_peak": 93.77326202392578, "growth_active": 1.1906923055648804, "growth_memory": 1.215434193611145, "heads": [{"head": 0, "memory_to_active": 0.12778592109680176, "contribution_to_active": 0.7217841148376465, "mix_mean": 0.8073088526725769, "mix_low": 0.0003108978271484375, "mix_high": 0.04477500915527344, "read_cosine": 0.20826014876365662, "beta_mean": 0.017193876206874847, "nonfinite": 0.0, "weighted_memory_to_active": 0.10498218238353729, "mix_std": 0.20508691668510437, "beta_std": 0.026046190410852432, "beta_low": 0.54620361328125, "beta_high": 0.0, "orthogonal_to_active": 0.08224672824144363, "evidence_weights": [-0.3229565918445587, -0.34907233715057373, 1.1689945459365845], "agreement_weight": 0.43529197573661804, "orthogonal_weight": -0.8370842933654785}, {"head": 1, "memory_to_active": 0.004633225500583649, "contribution_to_active": 0.5064951181411743, "mix_mean": 0.5666146278381348, "mix_low": 0.00142669677734375, "mix_high": 0.0136566162109375, "read_cosine": 0.5290721654891968, "beta_mean": 1.0390129318693653e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002055585151538253, "mix_std": 0.2889355421066284, "beta_std": 2.2622298274654895e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011317518074065447, "evidence_weights": [-0.19787612557411194, -0.2970609962940216, 1.1029868125915527], "agreement_weight": 0.08423690497875214, "orthogonal_weight": -0.7224817872047424}, {"head": 2, "memory_to_active": 0.9570736289024353, "contribution_to_active": 0.1028466448187828, "mix_mean": 0.03895530104637146, "mix_low": 0.31662654876708984, "mix_high": 3.814697265625e-06, "read_cosine": 0.24019920825958252, "beta_mean": 0.03797813504934311, "nonfinite": 0.0, "weighted_memory_to_active": 0.08353697508573532, "mix_std": 0.06415225565433502, "beta_std": 0.03798610344529152, "beta_low": 0.09735107421875, "beta_high": 0.0, "orthogonal_to_active": 0.00524565065279603, "evidence_weights": [0.0806984156370163, -0.0338561087846756, 0.31226789951324463], "agreement_weight": 0.14491324126720428, "orthogonal_weight": -0.0777745470404625}, {"head": 3, "memory_to_active": 0.15462854504585266, "contribution_to_active": 0.6627799272537231, "mix_mean": 0.641238808631897, "mix_low": 0.009077072143554688, "mix_high": 0.017996788024902344, "read_cosine": 0.4433637261390686, "beta_mean": 0.21506835520267487, "nonfinite": 0.0, "weighted_memory_to_active": 0.10644156485795975, "mix_std": 0.32717329263687134, "beta_std": 0.24521619081497192, "beta_low": 0.20709228515625, "beta_high": 0.0, "orthogonal_to_active": 0.05741141363978386, "evidence_weights": [-0.17670582234859467, -0.3322433531284332, 0.19071035087108612], "agreement_weight": -0.010072167031466961, "orthogonal_weight": 0.6178022623062134}, {"head": 4, "memory_to_active": 0.7159745693206787, "contribution_to_active": 1.0962755680084229, "mix_mean": 0.7498728632926941, "mix_low": 0.0009918212890625, "mix_high": 0.11382293701171875, "read_cosine": 0.12353550642728806, "beta_mean": 0.1291736662387848, "nonfinite": 0.0, "weighted_memory_to_active": 0.6649706959724426, "mix_std": 0.2991221249103546, "beta_std": 0.14232079684734344, "beta_low": 0.28900146484375, "beta_high": 0.0, "orthogonal_to_active": 0.08980323374271393, "evidence_weights": [-0.22652798891067505, -0.5806015729904175, -0.31117451190948486], "agreement_weight": -0.02243032492697239, "orthogonal_weight": 0.1523672193288803}, {"head": 5, "memory_to_active": 0.03392500802874565, "contribution_to_active": 0.6119822263717651, "mix_mean": 0.6644346117973328, "mix_low": 0.0001983642578125, "mix_high": 0.005227088928222656, "read_cosine": 0.25891631841659546, "beta_mean": 0.0001892984437290579, "nonfinite": 0.0, "weighted_memory_to_active": 0.024140415713191032, "mix_std": 0.25088685750961304, "beta_std": 0.0004480422066990286, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.024076325818896294, "evidence_weights": [-0.24244071543216705, -0.3129955232143402, 0.41563841700553894], "agreement_weight": 0.17237026989459991, "orthogonal_weight": -1.0648030042648315}]}, {"layer": 7, "memory_to_active": 1.7286590337753296, "contribution_to_active": 0.24422097206115723, "mix_mean": 0.14763057231903076, "mix_low": 0.26095661520957947, "mix_high": 0.0007748603820800781, "read_cosine": 0.2261015921831131, "beta_mean": 0.13630768656730652, "nonfinite": 0.0, "weighted_memory_to_active": 0.1905285120010376, "mix_std": 0.14222833514213562, "beta_std": 0.10228365659713745, "beta_low": 0.1956482082605362, "beta_high": 8.138021075865254e-05, "orthogonal_to_active": 0.07272309064865112, "output_influence": 0.2640910744667053, "residual_influence": 0.060743849724531174, "state_active": 0.014110736548900604, "state_memory": 0.025212423875927925, "state_peak": 28.055477142333984, "growth_active": 1.1846319437026978, "growth_memory": 1.286422848701477, "heads": [{"head": 0, "memory_to_active": 2.5492258071899414, "contribution_to_active": 0.19459106028079987, "mix_mean": 0.03717338666319847, "mix_low": 0.4234809875488281, "mix_high": 2.86102294921875e-06, "read_cosine": 0.3285331726074219, "beta_mean": 0.19957008957862854, "nonfinite": 0.0, "weighted_memory_to_active": 0.22260919213294983, "mix_std": 0.07803423702716827, "beta_std": 0.14737506210803986, "beta_low": 0.0028076171875, "beta_high": 0.000244140625, "orthogonal_to_active": 0.002447432605549693, "evidence_weights": [0.030894527211785316, -0.07830403745174408, 0.04712864011526108], "agreement_weight": -0.027505774050951004, "orthogonal_weight": 0.01350775733590126}, {"head": 1, "memory_to_active": 0.9967317581176758, "contribution_to_active": 0.37353095412254333, "mix_mean": 0.20739242434501648, "mix_low": 0.1242055892944336, "mix_high": 0.0040378570556640625, "read_cosine": 0.37787550687789917, "beta_mean": 0.11366073042154312, "nonfinite": 0.0, "weighted_memory_to_active": 0.3217105567455292, "mix_std": 0.2358185052871704, "beta_std": 0.07067225873470306, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.038537200540304184, "evidence_weights": [-0.21130184829235077, -0.10293865948915482, -0.23463869094848633], "agreement_weight": 0.04649448022246361, "orthogonal_weight": 0.14535664021968842}, {"head": 2, "memory_to_active": 0.7298093438148499, "contribution_to_active": 0.33226317167282104, "mix_mean": 0.17404696345329285, "mix_low": 0.047110557556152344, "mix_high": 0.00027942657470703125, "read_cosine": 0.07303687185049057, "beta_mean": 0.05477867275476456, "nonfinite": 0.0, "weighted_memory_to_active": 0.23219549655914307, "mix_std": 0.1924719363451004, "beta_std": 0.04729669541120529, "beta_low": 0.0509033203125, "beta_high": 0.0, "orthogonal_to_active": 0.11196593195199966, "evidence_weights": [-0.07081833481788635, -0.15264180302619934, 0.6925227046012878], "agreement_weight": 0.026228701695799828, "orthogonal_weight": -0.49594077467918396}, {"head": 3, "memory_to_active": 4.417392730712891, "contribution_to_active": 0.08943948149681091, "mix_mean": 0.031421925872564316, "mix_low": 0.3683891296386719, "mix_high": 0.0, "read_cosine": 0.08191318809986115, "beta_mean": 0.3703933656215668, "nonfinite": 0.0, "weighted_memory_to_active": 0.29749417304992676, "mix_std": 0.04778609797358513, "beta_std": 0.26743078231811523, "beta_low": 0.0052490234375, "beta_high": 0.000244140625, "orthogonal_to_active": 0.25901246070861816, "evidence_weights": [0.07279524952173233, -0.016095081344246864, 0.6981253623962402], "agreement_weight": 0.04656999930739403, "orthogonal_weight": -0.9444578886032104}, {"head": 4, "memory_to_active": 1.6412467956542969, "contribution_to_active": 0.07164986431598663, "mix_mean": 0.018807917833328247, "mix_low": 0.5989418029785156, "mix_high": 0.0, "read_cosine": 0.21238866448402405, "beta_mean": 0.07923322916030884, "nonfinite": 0.0, "weighted_memory_to_active": 0.05264619365334511, "mix_std": 0.04110807180404663, "beta_std": 0.0805022269487381, "beta_low": 0.114990234375, "beta_high": 0.0, "orthogonal_to_active": 0.010184040293097496, "evidence_weights": [0.14431969821453094, 0.008855515159666538, 0.23444965481758118], "agreement_weight": 0.03459210321307182, "orthogonal_weight": -0.20656362175941467}, {"head": 5, "memory_to_active": 0.03754761070013046, "contribution_to_active": 0.40385133028030396, "mix_mean": 0.4169408082962036, "mix_low": 0.0036115646362304688, "mix_high": 0.00032901763916015625, "read_cosine": 0.2828621566295624, "beta_mean": 0.00020988742471672595, "nonfinite": 0.0, "weighted_memory_to_active": 0.016515515744686127, "mix_std": 0.258151113986969, "beta_std": 0.00042490914347581565, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.014191489666700363, "evidence_weights": [-0.16037976741790771, -0.2183016538619995, 0.5640775561332703], "agreement_weight": 0.18699786067008972, "orthogonal_weight": -0.932918906211853}]}, {"layer": 8, "memory_to_active": 0.2623453736305237, "contribution_to_active": 0.5376936197280884, "mix_mean": 0.4479440450668335, "mix_low": 0.031384944915771484, "mix_high": 0.0013893445720896125, "read_cosine": 0.4330958127975464, "beta_mean": 0.017537403851747513, "nonfinite": 0.0, "weighted_memory_to_active": 0.09438486397266388, "mix_std": 0.22932438552379608, "beta_std": 0.024462539702653885, "beta_low": 0.7054036855697632, "beta_high": 0.0, "orthogonal_to_active": 0.07108399271965027, "output_influence": 0.48447099328041077, "residual_influence": 0.1632061004638672, "state_active": 0.045699700713157654, "state_memory": 0.00709665147587657, "state_peak": 41.043087005615234, "growth_active": 1.1199041604995728, "growth_memory": 1.2441645860671997, "heads": [{"head": 0, "memory_to_active": 0.062326960265636444, "contribution_to_active": 0.5663940906524658, "mix_mean": 0.6004891991615295, "mix_low": 0.00047016143798828125, "mix_high": 0.0015163421630859375, "read_cosine": 0.4629549980163574, "beta_mean": 0.0030935085378587246, "nonfinite": 0.0, "weighted_memory_to_active": 0.04401101917028427, "mix_std": 0.24773722887039185, "beta_std": 0.009910634718835354, "beta_low": 0.9261474609375, "beta_high": 0.0, "orthogonal_to_active": 0.028115857392549515, "evidence_weights": [-0.16922743618488312, -0.3894852101802826, 0.6530283093452454], "agreement_weight": 0.20573563873767853, "orthogonal_weight": -0.801792323589325}, {"head": 1, "memory_to_active": 0.0895870178937912, "contribution_to_active": 0.5297005772590637, "mix_mean": 0.4979809820652008, "mix_low": 7.43865966796875e-05, "mix_high": 0.0010318756103515625, "read_cosine": 0.17200088500976562, "beta_mean": 0.0017374466406181455, "nonfinite": 0.0, "weighted_memory_to_active": 0.04319240152835846, "mix_std": 0.24288006126880646, "beta_std": 0.006989680230617523, "beta_low": 0.97540283203125, "beta_high": 0.0, "orthogonal_to_active": 0.031053554266691208, "evidence_weights": [-0.1910778433084488, -0.2729564607143402, 0.7012478709220886], "agreement_weight": 0.06544594466686249, "orthogonal_weight": -0.7591419816017151}, {"head": 2, "memory_to_active": 0.019154518842697144, "contribution_to_active": 0.6709545254707336, "mix_mean": 0.6579199433326721, "mix_low": 0.0011167526245117188, "mix_high": 0.0030031204223632812, "read_cosine": 0.41515493392944336, "beta_mean": 0.00015501913730986416, "nonfinite": 0.0, "weighted_memory_to_active": 0.014897511340677738, "mix_std": 0.2302384227514267, "beta_std": 0.0021357631776481867, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00928820576518774, "evidence_weights": [-0.22570648789405823, -0.3433261215686798, 1.0149415731430054], "agreement_weight": 0.24614472687244415, "orthogonal_weight": -0.7249028086662292}, {"head": 3, "memory_to_active": 0.12066195905208588, "contribution_to_active": 0.6209388971328735, "mix_mean": 0.5754092931747437, "mix_low": 0.0011091232299804688, "mix_high": 0.0026798248291015625, "read_cosine": 0.7686277627944946, "beta_mean": 0.008804536424577236, "nonfinite": 0.0, "weighted_memory_to_active": 0.09328899532556534, "mix_std": 0.25850552320480347, "beta_std": 0.008044038899242878, "beta_low": 0.67974853515625, "beta_high": 0.0, "orthogonal_to_active": 0.03217637911438942, "evidence_weights": [-0.18769308924674988, -0.3051910400390625, 0.576208770275116], "agreement_weight": 0.21246081590652466, "orthogonal_weight": -0.6317539811134338}, {"head": 4, "memory_to_active": 0.9149972200393677, "contribution_to_active": 0.20777875185012817, "mix_mean": 0.10367285460233688, "mix_low": 0.098785400390625, "mix_high": 0.0, "read_cosine": 0.28753238916397095, "beta_mean": 0.021356821060180664, "nonfinite": 0.0, "weighted_memory_to_active": 0.18240879476070404, "mix_std": 0.12779146432876587, "beta_std": 0.04064095765352249, "beta_low": 0.56634521484375, "beta_high": 0.0, "orthogonal_to_active": 0.027812935411930084, "evidence_weights": [-0.005105124786496162, -0.11104831844568253, 0.1584366112947464], "agreement_weight": 0.07831267267465591, "orthogonal_weight": -0.16169732809066772}, {"head": 5, "memory_to_active": 0.36734455823898315, "contribution_to_active": 0.6303945779800415, "mix_mean": 0.25219210982322693, "mix_low": 0.08675384521484375, "mix_high": 0.0001049041748046875, "read_cosine": 0.49230384826660156, "beta_mean": 0.0700770914554596, "nonfinite": 0.0, "weighted_memory_to_active": 0.1885104775428772, "mix_std": 0.26879364252090454, "beta_std": 0.07905415445566177, "beta_low": 0.0848388671875, "beta_high": 0.0, "orthogonal_to_active": 0.29805701971054077, "evidence_weights": [-0.11062923073768616, -0.1784108281135559, -0.14143149554729462], "agreement_weight": -0.014097058214247227, "orthogonal_weight": 1.8581417798995972}]}, {"layer": 9, "memory_to_active": 0.9795918464660645, "contribution_to_active": 0.6854341626167297, "mix_mean": 0.34737178683280945, "mix_low": 0.1197468489408493, "mix_high": 0.005701859947293997, "read_cosine": 0.3491658568382263, "beta_mean": 0.055278223007917404, "nonfinite": 0.0, "weighted_memory_to_active": 0.39046624302864075, "mix_std": 0.22153401374816895, "beta_std": 0.04124043881893158, "beta_low": 0.5096029043197632, "beta_high": 0.0, "orthogonal_to_active": 0.0359763503074646, "output_influence": 0.5271484851837158, "residual_influence": 0.14806437492370605, "state_active": 0.04008949548006058, "state_memory": 0.018217064440250397, "state_peak": 52.11840057373047, "growth_active": 1.1538676023483276, "growth_memory": 1.3757044076919556, "heads": [{"head": 0, "memory_to_active": 0.0012252131709828973, "contribution_to_active": 0.5696350932121277, "mix_mean": 0.6379941701889038, "mix_low": 0.0013141632080078125, "mix_high": 0.02425384521484375, "read_cosine": 0.3002840578556061, "beta_mean": 4.024479039799189e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007685496238991618, "mix_std": 0.29065200686454773, "beta_std": 1.4502691556117497e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007723470334894955, "evidence_weights": [-0.30283060669898987, -0.31061625480651855, 0.7093129754066467], "agreement_weight": 0.10165564715862274, "orthogonal_weight": -1.0536123514175415}, {"head": 1, "memory_to_active": 0.025685925036668777, "contribution_to_active": 0.6898188591003418, "mix_mean": 0.5572288036346436, "mix_low": 0.0076236724853515625, "mix_high": 0.0026998519897460938, "read_cosine": 0.389422744512558, "beta_mean": 7.147264841478318e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.019190777093172073, "mix_std": 0.2874394357204437, "beta_std": 0.0002124111051671207, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.013344844803214073, "evidence_weights": [-0.24089165031909943, -0.29545101523399353, 1.1083170175552368], "agreement_weight": 0.42057329416275024, "orthogonal_weight": -0.8536856174468994}, {"head": 2, "memory_to_active": 0.1650351583957672, "contribution_to_active": 0.6389456987380981, "mix_mean": 0.5838480591773987, "mix_low": 0.0015354156494140625, "mix_high": 0.006030082702636719, "read_cosine": 0.366102933883667, "beta_mean": 0.0007339350413531065, "nonfinite": 0.0, "weighted_memory_to_active": 0.11002816259860992, "mix_std": 0.2710843086242676, "beta_std": 0.004071792587637901, "beta_low": 0.995361328125, "beta_high": 0.0, "orthogonal_to_active": 0.06353681534528732, "evidence_weights": [-0.19582737982273102, -0.27191320061683655, 0.7449467778205872], "agreement_weight": 0.30501651763916016, "orthogonal_weight": -0.6727564334869385}, {"head": 3, "memory_to_active": 1.2284584045410156, "contribution_to_active": 0.18255598843097687, "mix_mean": 0.06291677802801132, "mix_low": 0.2986316680908203, "mix_high": 2.765655517578125e-05, "read_cosine": 0.3770548701286316, "beta_mean": 0.07627413421869278, "nonfinite": 0.0, "weighted_memory_to_active": 0.1759064495563507, "mix_std": 0.11671672761440277, "beta_std": 0.05900639668107033, "beta_low": 0.0045166015625, "beta_high": 0.0, "orthogonal_to_active": 0.017056265845894814, "evidence_weights": [0.05910162627696991, -0.010580603964626789, 0.06815778464078903], "agreement_weight": 0.08667062968015671, "orthogonal_weight": -0.10622378438711166}, {"head": 4, "memory_to_active": 3.03439998626709, "contribution_to_active": 1.5608866214752197, "mix_mean": 0.14302074909210205, "mix_low": 0.14165878295898438, "mix_high": 0.00096893310546875, "read_cosine": 0.14449851214885712, "beta_mean": 0.15986105799674988, "nonfinite": 0.0, "weighted_memory_to_active": 1.5152682065963745, "mix_std": 0.20394445955753326, "beta_std": 0.08687231689691544, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.024932602420449257, "evidence_weights": [-0.0793435350060463, -0.15837226808071136, -0.029170768335461617], "agreement_weight": -0.05958881601691246, "orthogonal_weight": -0.017103299498558044}, {"head": 5, "memory_to_active": 1.422746181488037, "contribution_to_active": 0.4707622230052948, "mix_mean": 0.09922205656766891, "mix_low": 0.2677173614501953, "mix_high": 0.0002307891845703125, "read_cosine": 0.5176318883895874, "beta_mean": 0.09472469985485077, "nonfinite": 0.0, "weighted_memory_to_active": 0.5216351747512817, "mix_std": 0.15936709940433502, "beta_std": 0.09726519882678986, "beta_low": 0.05780029296875, "beta_high": 0.0, "orthogonal_to_active": 0.09621521830558777, "evidence_weights": [0.10410633683204651, -0.1075194850564003, 0.1358138769865036], "agreement_weight": 0.01715482957661152, "orthogonal_weight": 0.2692428529262543}]}, {"layer": 10, "memory_to_active": 0.21019065380096436, "contribution_to_active": 0.6660352945327759, "mix_mean": 0.516574501991272, "mix_low": 0.018468856811523438, "mix_high": 0.005596637725830078, "read_cosine": 0.3820120692253113, "beta_mean": 0.046148017048835754, "nonfinite": 0.0, "weighted_memory_to_active": 0.09795880317687988, "mix_std": 0.2719633877277374, "beta_std": 0.04718594253063202, "beta_low": 0.6624654531478882, "beta_high": 0.0, "orthogonal_to_active": 0.2035023719072342, "output_influence": 0.4403340816497803, "residual_influence": 0.15367376804351807, "state_active": 0.054433465003967285, "state_memory": 0.01522328145802021, "state_peak": 120.72775268554688, "growth_active": 1.075547695159912, "growth_memory": 1.0047721862792969, "heads": [{"head": 0, "memory_to_active": 0.7313377261161804, "contribution_to_active": 1.7267858982086182, "mix_mean": 0.5826215744018555, "mix_low": 0.07360649108886719, "mix_high": 0.013867378234863281, "read_cosine": 0.11617761850357056, "beta_mean": 0.2056603878736496, "nonfinite": 0.0, "weighted_memory_to_active": 0.4588505029678345, "mix_std": 0.3917909264564514, "beta_std": 0.21683673560619354, "beta_low": 0.14154052734375, "beta_high": 0.0, "orthogonal_to_active": 1.1305780410766602, "evidence_weights": [-0.10526255518198013, -0.3888195753097534, -1.5747897624969482], "agreement_weight": 0.017577646300196648, "orthogonal_weight": 2.4904990196228027}, {"head": 1, "memory_to_active": 0.00743070850148797, "contribution_to_active": 0.5954552292823792, "mix_mean": 0.60943204164505, "mix_low": 0.001743316650390625, "mix_high": 0.013784408569335938, "read_cosine": 0.29727303981781006, "beta_mean": 5.076852539787069e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004118934739381075, "mix_std": 0.2695518732070923, "beta_std": 6.73106696922332e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006151692476123571, "evidence_weights": [-0.25489723682403564, -0.19841714203357697, 0.5543932914733887], "agreement_weight": 0.5240471363067627, "orthogonal_weight": -1.6127800941467285}, {"head": 2, "memory_to_active": 0.014233998954296112, "contribution_to_active": 0.597017765045166, "mix_mean": 0.6372528672218323, "mix_low": 0.0004825592041015625, "mix_high": 0.004734039306640625, "read_cosine": 0.03566114977002144, "beta_mean": 2.836769272107631e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006049643270671368, "mix_std": 0.25448697805404663, "beta_std": 0.00014574980014003813, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00272757769562304, "evidence_weights": [-0.25474974513053894, -0.28241661190986633, 0.6549980640411377], "agreement_weight": -0.17856770753860474, "orthogonal_weight": -0.4295920431613922}, {"head": 3, "memory_to_active": 0.3530254065990448, "contribution_to_active": 0.11043111234903336, "mix_mean": 0.3007000982761383, "mix_low": 0.03218650817871094, "mix_high": 0.0005645751953125, "read_cosine": 0.9131369590759277, "beta_mean": 0.06466638296842575, "nonfinite": 0.0, "weighted_memory_to_active": 0.04244265332818031, "mix_std": 0.23774945735931396, "beta_std": 0.058766044676303864, "beta_low": 0.002197265625, "beta_high": 0.0, "orthogonal_to_active": 0.04680056869983673, "evidence_weights": [-0.10336132347583771, -0.1459726095199585, 0.16178563237190247], "agreement_weight": 0.01246462669223547, "orthogonal_weight": 1.4183226823806763}, {"head": 4, "memory_to_active": 0.018970990553498268, "contribution_to_active": 0.55306077003479, "mix_mean": 0.574258029460907, "mix_low": 0.0004711151123046875, "mix_high": 0.0006237030029296875, "read_cosine": 0.07479885220527649, "beta_mean": 6.529864913318306e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011027744971215725, "mix_std": 0.24982398748397827, "beta_std": 0.00012264832912478596, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013973088003695011, "evidence_weights": [-0.18455706536769867, -0.26206502318382263, 0.9384536743164062], "agreement_weight": 0.14203707873821259, "orthogonal_weight": -1.3385469913482666}, {"head": 5, "memory_to_active": 0.13614502549171448, "contribution_to_active": 0.41346102952957153, "mix_mean": 0.395182341337204, "mix_low": 0.002323150634765625, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8550246953964233, "beta_mean": 0.006416873075067997, "nonfinite": 0.0, "weighted_memory_to_active": 0.06526334583759308, "mix_std": 0.22837699949741364, "beta_std": 0.007177160121500492, "beta_low": 0.83111572265625, "beta_high": 0.0, "orthogonal_to_active": 0.020783191546797752, "evidence_weights": [-0.14165416359901428, -0.1619042605161667, 0.2716119885444641], "agreement_weight": 0.08983933180570602, "orthogonal_weight": -0.6488149166107178}]}, {"layer": 11, "memory_to_active": 0.09053324908018112, "contribution_to_active": 0.6035642623901367, "mix_mean": 0.5016434192657471, "mix_low": 0.10051202774047852, "mix_high": 0.0479329451918602, "read_cosine": 0.3644454777240753, "beta_mean": 0.02857937477529049, "nonfinite": 0.0, "weighted_memory_to_active": 0.04592118784785271, "mix_std": 0.3246763050556183, "beta_std": 0.04250720515847206, "beta_low": 0.6558329463005066, "beta_high": 0.0, "orthogonal_to_active": 0.046461161226034164, "output_influence": 0.5322220325469971, "residual_influence": 0.1371898204088211, "state_active": 0.18108458817005157, "state_memory": 0.004886520095169544, "state_peak": 32.890323638916016, "growth_active": 1.0912922620773315, "growth_memory": 1.6876530647277832, "heads": [{"head": 0, "memory_to_active": 0.008140244521200657, "contribution_to_active": 0.5980351567268372, "mix_mean": 0.6327028274536133, "mix_low": 0.0011625289916992188, "mix_high": 0.008280754089355469, "read_cosine": 0.29020148515701294, "beta_mean": 8.171149966074154e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0056249406188726425, "mix_std": 0.2613268494606018, "beta_std": 0.003279551165178418, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.004350613337010145, "evidence_weights": [-0.16882646083831787, -0.36755046248435974, 0.7556474208831787], "agreement_weight": 0.3254133462905884, "orthogonal_weight": -0.8085343837738037}, {"head": 1, "memory_to_active": 0.1980195939540863, "contribution_to_active": 0.3780856728553772, "mix_mean": 0.41917675733566284, "mix_low": 0.0064449310302734375, "mix_high": 0.0003204345703125, "read_cosine": 0.5249380469322205, "beta_mean": 0.01499709952622652, "nonfinite": 0.0, "weighted_memory_to_active": 0.08867501467466354, "mix_std": 0.2649059295654297, "beta_std": 0.03466428443789482, "beta_low": 0.6895751953125, "beta_high": 0.0, "orthogonal_to_active": 0.040490444749593735, "evidence_weights": [-0.1642436534166336, -0.2289004623889923, 0.5802794694900513], "agreement_weight": 0.07293612509965897, "orthogonal_weight": -0.5621910095214844}, {"head": 2, "memory_to_active": 0.007883450016379356, "contribution_to_active": 0.9820187091827393, "mix_mean": 0.44022732973098755, "mix_low": 0.27140235900878906, "mix_high": 0.1799306869506836, "read_cosine": 0.1896716207265854, "beta_mean": 0.07477184385061264, "nonfinite": 0.0, "weighted_memory_to_active": 0.0055470094084739685, "mix_std": 0.44781675934791565, "beta_std": 0.10020706802606583, "beta_low": 0.47991943359375, "beta_high": 0.0, "orthogonal_to_active": 0.04422026127576828, "evidence_weights": [0.040083806961774826, -0.4069228172302246, -1.2715932130813599], "agreement_weight": 0.012882508337497711, "orthogonal_weight": 8.308536529541016}, {"head": 3, "memory_to_active": 0.010793531313538551, "contribution_to_active": 0.8357104063034058, "mix_mean": 0.40931811928749084, "mix_low": 0.3150339126586914, "mix_high": 0.0936441421508789, "read_cosine": 0.556014895439148, "beta_mean": 0.05705280974507332, "nonfinite": 0.0, "weighted_memory_to_active": 0.006434950046241283, "mix_std": 0.4390966296195984, "beta_std": 0.08702170848846436, "beta_low": 0.5198974609375, "beta_high": 0.0, "orthogonal_to_active": 0.04110667109489441, "evidence_weights": [0.17929872870445251, -0.31244122982025146, -0.17842498421669006], "agreement_weight": 0.04836764186620712, "orthogonal_weight": 8.613619804382324}, {"head": 4, "memory_to_active": 0.23142708837985992, "contribution_to_active": 0.4468041956424713, "mix_mean": 0.5677657127380371, "mix_low": 0.0065364837646484375, "mix_high": 0.001499176025390625, "read_cosine": 0.0551464706659317, "beta_mean": 0.015955287963151932, "nonfinite": 0.0, "weighted_memory_to_active": 0.12567465007305145, "mix_std": 0.2734713554382324, "beta_std": 0.017739830538630486, "beta_low": 0.4935302734375, "beta_high": 0.0, "orthogonal_to_active": 0.12135924398899078, "evidence_weights": [-0.24758531153202057, -0.23805856704711914, 1.9465279579162598], "agreement_weight": -0.19765731692314148, "orthogonal_weight": -0.9892690181732178}, {"head": 5, "memory_to_active": 0.08693557977676392, "contribution_to_active": 0.38073092699050903, "mix_mean": 0.5406694412231445, "mix_low": 0.0024919509887695312, "mix_high": 0.003922462463378906, "read_cosine": 0.5707002282142639, "beta_mean": 0.008617501705884933, "nonfinite": 0.0, "weighted_memory_to_active": 0.04357057064771652, "mix_std": 0.2614402770996094, "beta_std": 0.012130780145525932, "beta_low": 0.75244140625, "beta_high": 0.0, "orthogonal_to_active": 0.027239711955189705, "evidence_weights": [-0.21151390671730042, -0.3094072937965393, 0.6542595028877258], "agreement_weight": 0.08117043226957321, "orthogonal_weight": -0.7945793867111206}]}, {"layer": 12, "memory_to_active": 0.27518999576568604, "contribution_to_active": 0.5872533917427063, "mix_mean": 0.5310298800468445, "mix_low": 0.009379546158015728, "mix_high": 0.0050258636474609375, "read_cosine": 0.2343553900718689, "beta_mean": 0.016758102923631668, "nonfinite": 0.0, "weighted_memory_to_active": 0.15140363574028015, "mix_std": 0.26417914032936096, "beta_std": 0.016750101000070572, "beta_low": 0.8236491084098816, "beta_high": 0.0, "orthogonal_to_active": 0.0819428414106369, "output_influence": 0.48415714502334595, "residual_influence": 0.135369211435318, "state_active": 0.03431351110339165, "state_memory": 0.009648618288338184, "state_peak": 56.49919509887695, "growth_active": 1.1110879182815552, "growth_memory": 1.2162377834320068, "heads": [{"head": 0, "memory_to_active": 0.051674578338861465, "contribution_to_active": 0.5754398703575134, "mix_mean": 0.5518437623977661, "mix_low": 0.0005979537963867188, "mix_high": 0.00334930419921875, "read_cosine": 0.3036077618598938, "beta_mean": 0.0008123959414660931, "nonfinite": 0.0, "weighted_memory_to_active": 0.02562083676457405, "mix_std": 0.2687961459159851, "beta_std": 0.0015953887486830354, "beta_low": 0.99481201171875, "beta_high": 0.0, "orthogonal_to_active": 0.023297883570194244, "evidence_weights": [-0.20717737078666687, -0.2757149040699005, 0.11202602833509445], "agreement_weight": 0.09987100958824158, "orthogonal_weight": -0.9704962968826294}, {"head": 1, "memory_to_active": 0.11063437908887863, "contribution_to_active": 0.5370970964431763, "mix_mean": 0.5503585934638977, "mix_low": 0.0006628036499023438, "mix_high": 0.0014257431030273438, "read_cosine": 0.4172849655151367, "beta_mean": 0.0010443073697388172, "nonfinite": 0.0, "weighted_memory_to_active": 0.0676458328962326, "mix_std": 0.266025185585022, "beta_std": 0.00889307539910078, "beta_low": 0.9898681640625, "beta_high": 0.0, "orthogonal_to_active": 0.05641922354698181, "evidence_weights": [-0.20616811513900757, -0.27224472165107727, 1.0319528579711914], "agreement_weight": 0.20403891801834106, "orthogonal_weight": -0.9299963712692261}, {"head": 2, "memory_to_active": 0.6273826360702515, "contribution_to_active": 0.5383098125457764, "mix_mean": 0.32926684617996216, "mix_low": 0.028037071228027344, "mix_high": 0.0006227493286132812, "read_cosine": 0.5040376782417297, "beta_mean": 0.09602151811122894, "nonfinite": 0.0, "weighted_memory_to_active": 0.26765719056129456, "mix_std": 0.25428834557533264, "beta_std": 0.06434887647628784, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.28381139039993286, "evidence_weights": [-0.2252577245235443, -0.05297808349132538, 0.0029486489947885275], "agreement_weight": 0.02164575457572937, "orthogonal_weight": 1.3167388439178467}, {"head": 3, "memory_to_active": 0.03712209314107895, "contribution_to_active": 0.7363876104354858, "mix_mean": 0.7036954760551453, "mix_low": 0.0001888275146484375, "mix_high": 0.013852119445800781, "read_cosine": 0.11670656502246857, "beta_mean": 0.0003545027575455606, "nonfinite": 0.0, "weighted_memory_to_active": 0.026644961908459663, "mix_std": 0.23145335912704468, "beta_std": 0.004889076575636864, "beta_low": 0.996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.02535407617688179, "evidence_weights": [-0.28980961441993713, -0.34005045890808105, 1.5420726537704468], "agreement_weight": 0.48278406262397766, "orthogonal_weight": -0.9739944338798523}, {"head": 4, "memory_to_active": 0.7200119495391846, "contribution_to_active": 0.6381392478942871, "mix_mean": 0.41636091470718384, "mix_low": 0.019556045532226562, "mix_high": 0.004485130310058594, "read_cosine": -0.12907269597053528, "beta_mean": 0.0017243374604731798, "nonfinite": 0.0, "weighted_memory_to_active": 0.4508339762687683, "mix_std": 0.2868967652320862, "beta_std": 0.014674043282866478, "beta_low": 0.97369384765625, "beta_high": 0.0, "orthogonal_to_active": 0.04904117062687874, "evidence_weights": [-0.14636054635047913, -0.21750375628471375, -0.04737969860434532], "agreement_weight": 0.0048567247577011585, "orthogonal_weight": -0.11327114701271057}, {"head": 5, "memory_to_active": 0.10431432723999023, "contribution_to_active": 0.49814653396606445, "mix_mean": 0.6346536874771118, "mix_low": 0.0072345733642578125, "mix_high": 0.006420135498046875, "read_cosine": 0.19356803596019745, "beta_mean": 0.0005915508372709155, "nonfinite": 0.0, "weighted_memory_to_active": 0.0700189396739006, "mix_std": 0.27761492133140564, "beta_std": 0.006100143771618605, "beta_low": 0.98651123046875, "beta_high": 0.0, "orthogonal_to_active": 0.05373328551650047, "evidence_weights": [-0.23876437544822693, -0.31900739669799805, 1.3381260633468628], "agreement_weight": 0.37598925828933716, "orthogonal_weight": -0.8247295022010803}]}, {"layer": 13, "memory_to_active": 0.12683023512363434, "contribution_to_active": 0.534564733505249, "mix_mean": 0.53607177734375, "mix_low": 0.005956967826932669, "mix_high": 0.009226163849234581, "read_cosine": 0.40559056401252747, "beta_mean": 0.016695287078619003, "nonfinite": 0.0, "weighted_memory_to_active": 0.05727623403072357, "mix_std": 0.2638372778892517, "beta_std": 0.02003295347094536, "beta_low": 0.8026733994483948, "beta_high": 0.0, "orthogonal_to_active": 0.05853773280978203, "output_influence": 0.5278332233428955, "residual_influence": 0.1740172803401947, "state_active": 0.11783894896507263, "state_memory": 0.04275960847735405, "state_peak": 99.4694595336914, "growth_active": 0.9498520493507385, "growth_memory": 1.0374733209609985, "heads": [{"head": 0, "memory_to_active": 0.01879262737929821, "contribution_to_active": 0.5792385339736938, "mix_mean": 0.603121280670166, "mix_low": 0.0005064010620117188, "mix_high": 0.004305839538574219, "read_cosine": 0.40653276443481445, "beta_mean": 6.651642615906894e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011742649599909782, "mix_std": 0.2698822617530823, "beta_std": 0.0001432399294571951, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011882573366165161, "evidence_weights": [-0.19927752017974854, -0.2680550515651703, 0.7737826704978943], "agreement_weight": 0.15419092774391174, "orthogonal_weight": -1.1178796291351318}, {"head": 1, "memory_to_active": 0.014010947197675705, "contribution_to_active": 0.5877022743225098, "mix_mean": 0.5504758954048157, "mix_low": 0.0022916793823242188, "mix_high": 0.0030078887939453125, "read_cosine": 0.22919008135795593, "beta_mean": 5.6090677389875054e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007839694619178772, "mix_std": 0.2774354815483093, "beta_std": 9.42845072131604e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010210257023572922, "evidence_weights": [-0.18115805089473724, -0.26332026720046997, 0.6302794218063354], "agreement_weight": 0.39654871821403503, "orthogonal_weight": -1.359914779663086}, {"head": 2, "memory_to_active": 0.08960732072591782, "contribution_to_active": 0.4732362926006317, "mix_mean": 0.38478296995162964, "mix_low": 0.001499176025390625, "mix_high": 0.0001049041748046875, "read_cosine": 0.6207188963890076, "beta_mean": 0.015979645773768425, "nonfinite": 0.0, "weighted_memory_to_active": 0.04397488757967949, "mix_std": 0.23286336660385132, "beta_std": 0.05118069052696228, "beta_low": 0.82672119140625, "beta_high": 0.0, "orthogonal_to_active": 0.026161683723330498, "evidence_weights": [-0.18742705881595612, -0.2057664692401886, 0.418840616941452], "agreement_weight": 0.008543165400624275, "orthogonal_weight": -0.8427683711051941}, {"head": 3, "memory_to_active": 0.17015884816646576, "contribution_to_active": 0.5797699093818665, "mix_mean": 0.6534633636474609, "mix_low": 0.024776458740234375, "mix_high": 0.03477668762207031, "read_cosine": -0.010515658184885979, "beta_mean": 0.0011771984864026308, "nonfinite": 0.0, "weighted_memory_to_active": 0.12795843183994293, "mix_std": 0.2928486466407776, "beta_std": 0.011883175931870937, "beta_low": 0.9898681640625, "beta_high": 0.0, "orthogonal_to_active": 0.11395247280597687, "evidence_weights": [-0.31981825828552246, -0.24777910113334656, 1.688542127609253], "agreement_weight": 5.385779857635498, "orthogonal_weight": -0.9795842170715332}, {"head": 4, "memory_to_active": 0.41268330812454224, "contribution_to_active": 0.2809402048587799, "mix_mean": 0.37473878264427185, "mix_low": 0.0042572021484375, "mix_high": 0.00013828277587890625, "read_cosine": 0.8865079283714294, "beta_mean": 0.08263863623142242, "nonfinite": 0.0, "weighted_memory_to_active": 0.1157045066356659, "mix_std": 0.2355388104915619, "beta_std": 0.053988441824913025, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.15812364220619202, "evidence_weights": [-0.09291016310453415, -0.23339784145355225, 0.4743509292602539], "agreement_weight": 0.008042932488024235, "orthogonal_weight": 2.2208497524261475}, {"head": 5, "memory_to_active": 0.055728353559970856, "contribution_to_active": 0.7065011262893677, "mix_mean": 0.6498483419418335, "mix_low": 0.002410888671875, "mix_high": 0.01302337646484375, "read_cosine": 0.30110928416252136, "beta_mean": 0.0002536210813559592, "nonfinite": 0.0, "weighted_memory_to_active": 0.03643721342086792, "mix_std": 0.27445507049560547, "beta_std": 0.0029078759253025055, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.03089575096964836, "evidence_weights": [-0.22181637585163116, -0.3047000467777252, 0.8867501020431519], "agreement_weight": 0.16599243879318237, "orthogonal_weight": -0.9256986379623413}]}, {"layer": 14, "memory_to_active": 0.2363220751285553, "contribution_to_active": 0.4225499927997589, "mix_mean": 0.4316635727882385, "mix_low": 0.024731477722525597, "mix_high": 0.006106535904109478, "read_cosine": 0.5978094339370728, "beta_mean": 0.0629776120185852, "nonfinite": 0.0, "weighted_memory_to_active": 0.0999593734741211, "mix_std": 0.2526070773601532, "beta_std": 0.06091504171490669, "beta_low": 0.4775594472885132, "beta_high": 0.0, "orthogonal_to_active": 0.044473692774772644, "output_influence": 0.35674136877059937, "residual_influence": 0.13827863335609436, "state_active": 0.058074597269296646, "state_memory": 0.01640772819519043, "state_peak": 42.823486328125, "growth_active": 1.0987783670425415, "growth_memory": 1.3047033548355103, "heads": [{"head": 0, "memory_to_active": 0.37047064304351807, "contribution_to_active": 0.21360346674919128, "mix_mean": 0.22701573371887207, "mix_low": 0.012393951416015625, "mix_high": 7.915496826171875e-05, "read_cosine": 0.815365195274353, "beta_mean": 0.07145240157842636, "nonfinite": 0.0, "weighted_memory_to_active": 0.09107008576393127, "mix_std": 0.20650964975357056, "beta_std": 0.061294157058000565, "beta_low": 0.007080078125, "beta_high": 0.0, "orthogonal_to_active": 0.09230085462331772, "evidence_weights": [-0.11848761141300201, -0.03871341422200203, 0.44930407404899597], "agreement_weight": 0.022538965567946434, "orthogonal_weight": 1.6275885105133057}, {"head": 1, "memory_to_active": 0.4802606999874115, "contribution_to_active": 0.28036874532699585, "mix_mean": 0.22578293085098267, "mix_low": 0.11919975280761719, "mix_high": 1.1444091796875e-05, "read_cosine": 0.8580363988876343, "beta_mean": 0.2527622878551483, "nonfinite": 0.0, "weighted_memory_to_active": 0.2043120115995407, "mix_std": 0.25507983565330505, "beta_std": 0.25121936202049255, "beta_low": 0.10003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.013295977376401424, "evidence_weights": [-0.003995803650468588, -0.19739846885204315, -0.10480210930109024], "agreement_weight": 0.05561613664031029, "orthogonal_weight": 0.1678769588470459}, {"head": 2, "memory_to_active": 0.12131372094154358, "contribution_to_active": 0.5822257995605469, "mix_mean": 0.5896590352058411, "mix_low": 0.0030078887939453125, "mix_high": 0.0037260055541992188, "read_cosine": 0.42308148741722107, "beta_mean": 0.002011724514886737, "nonfinite": 0.0, "weighted_memory_to_active": 0.08088435232639313, "mix_std": 0.2689271867275238, "beta_std": 0.0028992905281484127, "beta_low": 0.98077392578125, "beta_high": 0.0, "orthogonal_to_active": 0.062187887728214264, "evidence_weights": [-0.1426631510257721, -0.3159177601337433, 0.9042028188705444], "agreement_weight": 0.0895434245467186, "orthogonal_weight": -0.8827294707298279}, {"head": 3, "memory_to_active": 0.05704011023044586, "contribution_to_active": 0.6227275729179382, "mix_mean": 0.660957396030426, "mix_low": 0.005684852600097656, "mix_high": 0.021065711975097656, "read_cosine": -0.12953540682792664, "beta_mean": 0.008047204464673996, "nonfinite": 0.0, "weighted_memory_to_active": 0.03364967182278633, "mix_std": 0.2637919783592224, "beta_std": 0.010735699906945229, "beta_low": 0.75616455078125, "beta_high": 0.0, "orthogonal_to_active": 0.025628812611103058, "evidence_weights": [-0.34762445092201233, -0.31038764119148254, 0.30728983879089355], "agreement_weight": 0.07199260592460632, "orthogonal_weight": -0.8115409016609192}, {"head": 4, "memory_to_active": 0.1491740196943283, "contribution_to_active": 0.5009517073631287, "mix_mean": 0.5108287334442139, "mix_low": 0.0039844512939453125, "mix_high": 0.009694099426269531, "read_cosine": 0.8651367425918579, "beta_mean": 0.041729629039764404, "nonfinite": 0.0, "weighted_memory_to_active": 0.08862122893333435, "mix_std": 0.2636534571647644, "beta_std": 0.029372500255703926, "beta_low": 0.0545654296875, "beta_high": 0.0, "orthogonal_to_active": 0.026603959500789642, "evidence_weights": [-0.21877175569534302, -0.24698974192142487, 0.4298114776611328], "agreement_weight": 0.13180097937583923, "orthogonal_weight": -0.613063633441925}, {"head": 5, "memory_to_active": 0.2396731972694397, "contribution_to_active": 0.33542248606681824, "mix_mean": 0.3757374882698059, "mix_low": 0.0041179656982421875, "mix_high": 0.0020627975463867188, "read_cosine": 0.7547717690467834, "beta_mean": 0.0018624188378453255, "nonfinite": 0.0, "weighted_memory_to_active": 0.10121891647577286, "mix_std": 0.2576802968978882, "beta_std": 0.009969225153326988, "beta_low": 0.96673583984375, "beta_high": 0.0, "orthogonal_to_active": 0.04682467132806778, "evidence_weights": [-0.1772671937942505, -0.1733541339635849, 0.292080819606781], "agreement_weight": 0.07832677662372589, "orthogonal_weight": -0.7341786623001099}]}, {"layer": 15, "memory_to_active": 0.056859344244003296, "contribution_to_active": 0.48874861001968384, "mix_mean": 0.41115522384643555, "mix_low": 0.009043216705322266, "mix_high": 0.0027265548706054688, "read_cosine": 0.6601622700691223, "beta_mean": 0.03591054305434227, "nonfinite": 0.0, "weighted_memory_to_active": 0.02780274488031864, "mix_std": 0.2854114770889282, "beta_std": 0.042772579938173294, "beta_low": 0.7079163193702698, "beta_high": 0.0, "orthogonal_to_active": 0.007538260892033577, "output_influence": 0.30469536781311035, "residual_influence": 0.1683647185564041, "state_active": 0.14867258071899414, "state_memory": 0.010944236069917679, "state_peak": 32.89730453491211, "growth_active": 1.2646039724349976, "growth_memory": 1.419513463973999, "heads": [{"head": 0, "memory_to_active": 0.1166127622127533, "contribution_to_active": 0.4859398305416107, "mix_mean": 0.3840771019458771, "mix_low": 0.0062408447265625, "mix_high": 0.005496978759765625, "read_cosine": 0.6591302156448364, "beta_mean": 0.0410337969660759, "nonfinite": 0.0, "weighted_memory_to_active": 0.05555395781993866, "mix_std": 0.28237399458885193, "beta_std": 0.029518824070692062, "beta_low": 0.07269287109375, "beta_high": 0.0, "orthogonal_to_active": 0.026259370148181915, "evidence_weights": [-0.19028709828853607, -0.19278782606124878, 0.42800238728523254], "agreement_weight": 0.09446097910404205, "orthogonal_weight": -0.7456660270690918}, {"head": 1, "memory_to_active": 0.1132047027349472, "contribution_to_active": 0.4247565269470215, "mix_mean": 0.42764803767204285, "mix_low": 0.0015716552734375, "mix_high": 0.005684852600097656, "read_cosine": 0.9245955944061279, "beta_mean": 0.000689625390805304, "nonfinite": 0.0, "weighted_memory_to_active": 0.04900002479553223, "mix_std": 0.28669658303260803, "beta_std": 0.0020934154745191336, "beta_low": 0.9935302734375, "beta_high": 0.0, "orthogonal_to_active": 0.0008041274268180132, "evidence_weights": [-0.1496688425540924, -0.15886126458644867, 0.18834595382213593], "agreement_weight": 0.12471350282430649, "orthogonal_weight": 0.03380797430872917}, {"head": 2, "memory_to_active": 0.007155352272093296, "contribution_to_active": 0.4686511158943176, "mix_mean": 0.3389953076839447, "mix_low": 0.01653003692626953, "mix_high": 9.059906005859375e-05, "read_cosine": 0.7509948015213013, "beta_mean": 0.00010248579201288521, "nonfinite": 0.0, "weighted_memory_to_active": 0.0036397939547896385, "mix_std": 0.2709902226924896, "beta_std": 0.00022576807532459497, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010263603180646896, "evidence_weights": [-0.2049562931060791, -0.2677515149116516, 0.14925487339496613], "agreement_weight": 0.009729873389005661, "orthogonal_weight": 0.6980306506156921}, {"head": 3, "memory_to_active": 0.001746167428791523, "contribution_to_active": 0.3938063979148865, "mix_mean": 0.39053672552108765, "mix_low": 0.0059871673583984375, "mix_high": 0.00035190582275390625, "read_cosine": 0.5662791132926941, "beta_mean": 1.0152745744562708e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005852043977938592, "mix_std": 0.28826576471328735, "beta_std": 2.274879443575628e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004638696846086532, "evidence_weights": [-0.2584269046783447, -0.24670839309692383, 0.19083431363105774], "agreement_weight": 0.039984576404094696, "orthogonal_weight": -0.8008792400360107}, {"head": 4, "memory_to_active": 0.07595240324735641, "contribution_to_active": 0.6270710825920105, "mix_mean": 0.36391234397888184, "mix_low": 0.013483047485351562, "mix_high": 0.0009860992431640625, "read_cosine": 0.7383931875228882, "beta_mean": 0.17141567170619965, "nonfinite": 0.0, "weighted_memory_to_active": 0.041464127600193024, "mix_std": 0.30602824687957764, "beta_std": 0.20925191044807434, "beta_low": 0.2119140625, "beta_high": 0.0, "orthogonal_to_active": 0.00657358905300498, "evidence_weights": [-0.2132222205400467, -0.25405651330947876, -0.015885477885603905], "agreement_weight": 0.013670772314071655, "orthogonal_weight": 0.3388243317604065}, {"head": 5, "memory_to_active": 0.026484670117497444, "contribution_to_active": 0.5322664976119995, "mix_mean": 0.5617618560791016, "mix_low": 0.010446548461914062, "mix_high": 0.0037488937377929688, "read_cosine": 0.32158035039901733, "beta_mean": 0.0022115083411335945, "nonfinite": 0.0, "weighted_memory_to_active": 0.01657336950302124, "mix_std": 0.2781140208244324, "beta_std": 0.015522802248597145, "beta_low": 0.9693603515625, "beta_high": 0.0, "orthogonal_to_active": 0.010102246887981892, "evidence_weights": [-0.2887584865093231, -0.3043225109577179, 0.7080798149108887], "agreement_weight": 0.29110071063041687, "orthogonal_weight": 0.8002525568008423}]}]}, {"step": 51000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 36.29628659399987, "memory_to_active": 0.8162270188331604, "contribution_to_active": 0.5587980151176453, "mix_mean": 0.40541958808898926, "mix_low": 0.1346203237771988, "mix_high": 0.010016649961471558, "read_cosine": 0.35143113136291504, "beta_mean": 0.07220365107059479, "nonfinite": 0.0, "weighted_memory_to_active": 0.2115698754787445, "mix_std": 0.22189337015151978, "beta_std": 0.0627608671784401, "beta_low": 0.5545768737792969, "beta_high": 1.780192178557627e-05, "orthogonal_to_active": 0.06288579106330872, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1079797744750977, "normal_token_accuracy": 0.4482758641242981, "state_active": 0.06158438324928284, "state_memory": 0.01798310875892639, "state_peak": 115.90574645996094, "growth_active": 1.1355695724487305, "growth_memory": 1.3281221389770508, "output_influence": 0.4182112514972687, "residual_influence": 0.14012686908245087, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.240947723388672, "no_memory_loss_delta": 0.5615907907485962, "scrambled_loss_delta": 0.15448889136314392, "memory_only_loss_delta": 3.41973876953125, "no_evidence_loss_delta": 0.34665969014167786, "no_agreement_loss_delta": 0.02373373694717884, "no_orthogonal_loss_delta": -0.0019951416179537773}, {"index": 94, "tokens": 256, "normal_loss": 3.182199001312256, "no_memory_loss_delta": 0.5781567096710205, "scrambled_loss_delta": 0.3964749574661255, "memory_only_loss_delta": 3.3989477157592773, "no_evidence_loss_delta": 0.2666285037994385, "no_agreement_loss_delta": 0.0319882296025753, "no_orthogonal_loss_delta": 0.027613701298832893}, {"index": 188, "tokens": 256, "normal_loss": 3.466902732849121, "no_memory_loss_delta": 0.6131789088249207, "scrambled_loss_delta": 0.10663515329360962, "memory_only_loss_delta": 2.9708101749420166, "no_evidence_loss_delta": 0.33188867568969727, "no_agreement_loss_delta": 0.052376601845026016, "no_orthogonal_loss_delta": 0.032271258533000946}, {"index": 283, "tokens": 131, "normal_loss": 2.0016870498657227, "no_memory_loss_delta": 0.3497370779514313, "scrambled_loss_delta": 0.2741890251636505, "memory_only_loss_delta": 2.7963600158691406, "no_evidence_loss_delta": 0.16266056895256042, "no_agreement_loss_delta": -0.0068267290480434895, "no_orthogonal_loss_delta": -0.00656261807307601}], "agreement_weight_abs_mean": 0.20429611206054688, "agreement_weight_abs_max": 5.462803363800049, "agreement_weight_mean": 0.17886406183242798, "orthogonal_weight_abs_mean": 0.8381659984588623, "orthogonal_weight_abs_max": 8.751544952392578, "orthogonal_weight_mean": -0.11913210153579712, "no_memory_loss": 3.658107042312622, "no_memory_loss_delta": 0.5501275658607483, "no_memory_token_accuracy_gain": 0.0678531676530838, "no_memory_token_win_fraction": 0.656284749507904, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 42.31237840245164, "scrambled_loss": 3.3351926803588867, "scrambled_loss_delta": 0.227212592959404, "scrambled_token_accuracy_gain": 0.03448275849223137, "scrambled_token_win_fraction": 0.6062291264533997, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.32486194499313, "memory_only_loss": 6.303123474121094, "memory_only_loss_delta": 3.195143938064575, "memory_only_token_accuracy_gain": 0.3014460504055023, "memory_only_token_win_fraction": 0.8998887538909912, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.90393708419369, "no_evidence_loss": 3.400831460952759, "no_evidence_loss_delta": 0.29285186529159546, "no_evidence_token_accuracy_gain": 0.032258063554763794, "no_evidence_token_win_fraction": 0.6117908954620361, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 25.38673393447843, "no_agreement_loss": 3.1377673149108887, "no_agreement_loss_delta": 0.029787462204694748, "no_agreement_token_accuracy_gain": 0.006674082484096289, "no_agreement_token_win_fraction": 0.5094549655914307, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.9348188177139276, "no_orthogonal_loss": 3.1235084533691406, "no_orthogonal_loss_delta": 0.015528467483818531, "no_orthogonal_token_accuracy_gain": -0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5595105886459351, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 1.540852248918001, "short_no_memory_loss_delta": 0.540931761264801, "long_context_memory_gain": 0.009195804595947266, "normal_context_loss_gain": 0.12282517552375793, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.4348878860473633, "contribution_to_active": 1.1469597816467285, "mix_mean": 0.10531328618526459, "mix_low": 0.5306710004806519, "mix_high": 0.005263329017907381, "read_cosine": 0.43228209018707275, "beta_mean": 0.2246513068675995, "nonfinite": 0.0, "weighted_memory_to_active": 1.1804447174072266, "mix_std": 0.09035076200962067, "beta_std": 0.14312998950481415, "beta_low": 0.0561319999396801, "beta_high": 0.0, "orthogonal_to_active": 0.04003559798002243, "output_influence": 0.20781376957893372, "residual_influence": 0.24660006165504456, "state_active": 0.06808483600616455, "state_memory": 0.04476197436451912, "state_peak": 44.331764221191406, "growth_active": 1.2558130025863647, "growth_memory": 1.2634578943252563, "heads": [{"head": 0, "memory_to_active": 2.772057294845581, "contribution_to_active": 0.5142350792884827, "mix_mean": 0.020972013473510742, "mix_low": 0.6009140014648438, "mix_high": 1.9073486328125e-06, "read_cosine": 0.2352173626422882, "beta_mean": 0.2055487036705017, "nonfinite": 0.0, "weighted_memory_to_active": 0.6340993642807007, "mix_std": 0.06227573752403259, "beta_std": 0.1109357550740242, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06297098100185394, "evidence_weights": [-0.010371328331530094, -0.05547511950135231, -0.05593517795205116], "agreement_weight": -0.18428407609462738, "orthogonal_weight": -0.11203952878713608}, {"head": 1, "memory_to_active": 6.571188449859619, "contribution_to_active": 2.682616710662842, "mix_mean": 0.018246036022901535, "mix_low": 0.6416225433349609, "mix_high": 0.0, "read_cosine": 0.4543823003768921, "beta_mean": 0.4522489905357361, "nonfinite": 0.0, "weighted_memory_to_active": 2.951022148132324, "mix_std": 0.0534636527299881, "beta_std": 0.20634567737579346, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004904145607724786, "evidence_weights": [0.02526325173676014, -0.10040441155433655, 0.5679432153701782], "agreement_weight": 0.028192302212119102, "orthogonal_weight": -0.0003442458692006767}, {"head": 2, "memory_to_active": 2.4138729572296143, "contribution_to_active": 0.44751787185668945, "mix_mean": 0.014154970645904541, "mix_low": 0.6666975021362305, "mix_high": 0.0, "read_cosine": 0.32492899894714355, "beta_mean": 0.2868829667568207, "nonfinite": 0.0, "weighted_memory_to_active": 0.5253043174743652, "mix_std": 0.0331520177423954, "beta_std": 0.1912102848291397, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.045641303062438965, "evidence_weights": [0.0625384971499443, 0.05762607231736183, 0.12440671026706696], "agreement_weight": 0.010642493143677711, "orthogonal_weight": -0.2039203643798828}, {"head": 3, "memory_to_active": 6.811145305633545, "contribution_to_active": 2.720107316970825, "mix_mean": 0.033185217529535294, "mix_low": 0.48557472229003906, "mix_high": 0.0, "read_cosine": 0.31043368577957153, "beta_mean": 0.3049485981464386, "nonfinite": 0.0, "weighted_memory_to_active": 2.871244430541992, "mix_std": 0.07900920510292053, "beta_std": 0.2506704032421112, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11117096245288849, "evidence_weights": [-0.043249018490314484, -0.045761365443468094, -0.15740197896957397], "agreement_weight": -0.015590631403028965, "orthogonal_weight": -0.045636989176273346}, {"head": 4, "memory_to_active": 1.9325942993164062, "contribution_to_active": 0.06054771691560745, "mix_mean": 0.008996414951980114, "mix_low": 0.7741012573242188, "mix_high": 0.0, "read_cosine": 0.32481902837753296, "beta_mean": 0.07682514935731888, "nonfinite": 0.0, "weighted_memory_to_active": 0.045208610594272614, "mix_std": 0.024530678987503052, "beta_std": 0.0809689611196518, "beta_low": 0.04888916015625, "beta_high": 0.0, "orthogonal_to_active": 0.004176014568656683, "evidence_weights": [0.14623717963695526, 0.10411491990089417, -0.20892028510570526], "agreement_weight": -0.12293773889541626, "orthogonal_weight": 0.08416812121868134}, {"head": 5, "memory_to_active": 0.10846735537052155, "contribution_to_active": 0.45673343539237976, "mix_mean": 0.5363250970840454, "mix_low": 0.015115737915039062, "mix_high": 0.03157806396484375, "read_cosine": 0.9439108967781067, "beta_mean": 0.021453367546200752, "nonfinite": 0.0, "weighted_memory_to_active": 0.0557892806828022, "mix_std": 0.2896732985973358, "beta_std": 0.018648797646164894, "beta_low": 0.28790283203125, "beta_high": 0.0, "orthogonal_to_active": 0.0157638993114233, "evidence_weights": [-0.18183356523513794, -0.21878761053085327, 0.37706393003463745], "agreement_weight": 0.2921139597892761, "orthogonal_weight": -0.6882069110870361}]}, {"layer": 1, "memory_to_active": 2.3546206951141357, "contribution_to_active": 0.49053871631622314, "mix_mean": 0.20767638087272644, "mix_low": 0.3636406362056732, "mix_high": 0.0008522669668309391, "read_cosine": 0.3008479177951813, "beta_mean": 0.1803693175315857, "nonfinite": 0.0, "weighted_memory_to_active": 0.33371368050575256, "mix_std": 0.11624066531658173, "beta_std": 0.12535244226455688, "beta_low": 0.3421122431755066, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.04535219818353653, "output_influence": 0.23368118703365326, "residual_influence": 0.062439825385808945, "state_active": 0.031085854396224022, "state_memory": 0.03543198108673096, "state_peak": 36.77922058105469, "growth_active": 1.2589739561080933, "growth_memory": 1.476992130279541, "heads": [{"head": 0, "memory_to_active": 0.040835440158843994, "contribution_to_active": 0.4858251214027405, "mix_mean": 0.5040794610977173, "mix_low": 0.007061958312988281, "mix_high": 0.0014438629150390625, "read_cosine": 0.04027342051267624, "beta_mean": 0.0002832756727002561, "nonfinite": 0.0, "weighted_memory_to_active": 0.022502664476633072, "mix_std": 0.2807115316390991, "beta_std": 0.0004302310408093035, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.020997321233153343, "evidence_weights": [-0.20553503930568695, -0.2647351026535034, 0.29378169775009155], "agreement_weight": 0.18052983283996582, "orthogonal_weight": -0.9906957149505615}, {"head": 1, "memory_to_active": 3.031914234161377, "contribution_to_active": 0.2879103124141693, "mix_mean": 0.024960560724139214, "mix_low": 0.43871021270751953, "mix_high": 1.9073486328125e-06, "read_cosine": 0.48451241850852966, "beta_mean": 0.2727552056312561, "nonfinite": 0.0, "weighted_memory_to_active": 0.3167935311794281, "mix_std": 0.050514768809080124, "beta_std": 0.17712369561195374, "beta_low": 0.000732421875, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.021836278960108757, "evidence_weights": [0.01050583366304636, -0.054280515760183334, 0.05239739641547203], "agreement_weight": -0.0048004379495978355, "orthogonal_weight": 0.1769891381263733}, {"head": 2, "memory_to_active": 0.06153900548815727, "contribution_to_active": 0.5494109392166138, "mix_mean": 0.6556597948074341, "mix_low": 0.0005655288696289062, "mix_high": 0.0036640167236328125, "read_cosine": 0.4922115206718445, "beta_mean": 0.0028772330842912197, "nonfinite": 0.0, "weighted_memory_to_active": 0.04202966019511223, "mix_std": 0.23373344540596008, "beta_std": 0.005019203759729862, "beta_low": 0.96221923828125, "beta_high": 0.0, "orthogonal_to_active": 0.026738503947854042, "evidence_weights": [-0.28598451614379883, -0.3637315332889557, 0.6734742522239685], "agreement_weight": 0.03724170848727226, "orthogonal_weight": -0.7552914023399353}, {"head": 3, "memory_to_active": 2.8247358798980713, "contribution_to_active": 0.16155490279197693, "mix_mean": 0.020562563091516495, "mix_low": 0.6042842864990234, "mix_high": 2.86102294921875e-06, "read_cosine": 0.20509465038776398, "beta_mean": 0.31975889205932617, "nonfinite": 0.0, "weighted_memory_to_active": 0.2608673572540283, "mix_std": 0.056352853775024414, "beta_std": 0.2581331133842468, "beta_low": 0.05328369140625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.08709221333265305, "evidence_weights": [0.04529763013124466, -0.010760988108813763, 0.03223327174782753], "agreement_weight": -0.02750960923731327, "orthogonal_weight": -0.47449547052383423}, {"head": 4, "memory_to_active": 3.8480844497680664, "contribution_to_active": 1.377335786819458, "mix_mean": 0.03357860445976257, "mix_low": 0.3135080337524414, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3432837128639221, "beta_mean": 0.16761890053749084, "nonfinite": 0.0, "weighted_memory_to_active": 1.296526551246643, "mix_std": 0.058187227696180344, "beta_std": 0.10848396271467209, "beta_low": 0.01922607421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1001187115907669, "evidence_weights": [-0.03386761248111725, -0.005867908708751202, 0.11025947332382202], "agreement_weight": -0.017523786053061485, "orthogonal_weight": 0.10291271656751633}, {"head": 5, "memory_to_active": 4.320613861083984, "contribution_to_active": 0.08119509369134903, "mix_mean": 0.00721737602725625, "mix_low": 0.817713737487793, "mix_high": 0.0, "read_cosine": 0.23971155285835266, "beta_mean": 0.31892237067222595, "nonfinite": 0.0, "weighted_memory_to_active": 0.06356237828731537, "mix_std": 0.017944183200597763, "beta_std": 0.20292438566684723, "beta_low": 0.0172119140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.015330149792134762, "evidence_weights": [0.1570155769586563, 0.07685790956020355, 0.14517417550086975], "agreement_weight": -0.11811420321464539, "orthogonal_weight": 0.2549777925014496}]}, {"layer": 2, "memory_to_active": 0.9848276972770691, "contribution_to_active": 0.3893057405948639, "mix_mean": 0.3909580409526825, "mix_low": 0.24786347150802612, "mix_high": 0.004590034484863281, "read_cosine": 0.31388285756111145, "beta_mean": 0.07811758667230606, "nonfinite": 0.0, "weighted_memory_to_active": 0.11521513760089874, "mix_std": 0.1928626000881195, "beta_std": 0.06506095081567764, "beta_low": 0.5770975947380066, "beta_high": 7.120767986634746e-05, "orthogonal_to_active": 0.024796927347779274, "output_influence": 0.4223513901233673, "residual_influence": 0.11048269271850586, "state_active": 0.038505688309669495, "state_memory": 0.01546221412718296, "state_peak": 29.30569076538086, "growth_active": 1.0471444129943848, "growth_memory": 1.5897510051727295, "heads": [{"head": 0, "memory_to_active": 0.012549594976007938, "contribution_to_active": 0.36822962760925293, "mix_mean": 0.6158905625343323, "mix_low": 0.0008039474487304688, "mix_high": 0.0096282958984375, "read_cosine": 0.13900336623191833, "beta_mean": 8.482385601382703e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008120566606521606, "mix_std": 0.2757910192012787, "beta_std": 0.00013551062147598714, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008037816733121872, "evidence_weights": [-0.3639538288116455, -0.21384434401988983, 0.1471693217754364], "agreement_weight": 0.12360595911741257, "orthogonal_weight": -1.026306390762329}, {"head": 1, "memory_to_active": 2.818876266479492, "contribution_to_active": 0.05508483201265335, "mix_mean": 0.005096951499581337, "mix_low": 0.8940343856811523, "mix_high": 9.5367431640625e-07, "read_cosine": 0.14986568689346313, "beta_mean": 0.21257491409778595, "nonfinite": 0.0, "weighted_memory_to_active": 0.04174548014998436, "mix_std": 0.012196552939713001, "beta_std": 0.1373644322156906, "beta_low": 0.0015869140625, "beta_high": 0.0, "orthogonal_to_active": 0.01511400006711483, "evidence_weights": [0.13695912063121796, 0.12243751436471939, 0.5286368131637573], "agreement_weight": 0.05351271107792854, "orthogonal_weight": 0.3909550607204437}, {"head": 2, "memory_to_active": 0.00992625206708908, "contribution_to_active": 0.4303056597709656, "mix_mean": 0.5274852514266968, "mix_low": 0.001300811767578125, "mix_high": 0.0033597946166992188, "read_cosine": 0.35745060443878174, "beta_mean": 3.10441610054113e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003880538046360016, "mix_std": 0.27883198857307434, "beta_std": 5.918558963458054e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003707912692334503, "evidence_weights": [-0.2464745193719864, -0.28740954399108887, 0.547985315322876], "agreement_weight": -0.011906247586011887, "orthogonal_weight": -0.10960007458925247}, {"head": 3, "memory_to_active": 0.12685073912143707, "contribution_to_active": 0.5333841443061829, "mix_mean": 0.5449999570846558, "mix_low": 0.0004100799560546875, "mix_high": 0.006342887878417969, "read_cosine": 0.6463426351547241, "beta_mean": 0.024601565673947334, "nonfinite": 0.0, "weighted_memory_to_active": 0.07853290438652039, "mix_std": 0.2644011080265045, "beta_std": 0.031347911804914474, "beta_low": 0.42431640625, "beta_high": 0.0, "orthogonal_to_active": 0.05184450000524521, "evidence_weights": [-0.19626620411872864, -0.3087959885597229, 0.8196446299552917], "agreement_weight": 0.056423384696245193, "orthogonal_weight": -0.8851220011711121}, {"head": 4, "memory_to_active": 2.930830955505371, "contribution_to_active": 0.4290417730808258, "mix_mean": 0.021481823176145554, "mix_low": 0.5895977020263672, "mix_high": 0.0, "read_cosine": 0.20643647015094757, "beta_mean": 0.2313765436410904, "nonfinite": 0.0, "weighted_memory_to_active": 0.5534833669662476, "mix_std": 0.059461720287799835, "beta_std": 0.2214105874300003, "beta_low": 0.03668212890625, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.07285775244235992, "evidence_weights": [0.021816913038492203, 0.007847935892641544, 0.00812071654945612], "agreement_weight": 0.019275587052106857, "orthogonal_weight": -0.14720742404460907}, {"head": 5, "memory_to_active": 0.009932445362210274, "contribution_to_active": 0.5197882056236267, "mix_mean": 0.6307934522628784, "mix_low": 0.001033782958984375, "mix_high": 0.008208274841308594, "read_cosine": 0.3841983675956726, "beta_mean": 3.65880478057079e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00552794523537159, "mix_std": 0.26649320125579834, "beta_std": 4.806337528862059e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005566893960349262, "evidence_weights": [-0.2771502435207367, -0.29213106632232666, 1.0380032062530518], "agreement_weight": 0.16616763174533844, "orthogonal_weight": -0.1296112835407257}]}, {"layer": 3, "memory_to_active": 0.5023881196975708, "contribution_to_active": 0.4539734721183777, "mix_mean": 0.47474926710128784, "mix_low": 0.06017240136861801, "mix_high": 0.009087562561035156, "read_cosine": 0.20095479488372803, "beta_mean": 0.02968650311231613, "nonfinite": 0.0, "weighted_memory_to_active": 0.13531336188316345, "mix_std": 0.2622467279434204, "beta_std": 0.026690827682614326, "beta_low": 0.7638041377067566, "beta_high": 0.0, "orthogonal_to_active": 0.05763206630945206, "output_influence": 0.3879045844078064, "residual_influence": 0.08658706396818161, "state_active": 0.024906598031520844, "state_memory": 0.007430230733007193, "state_peak": 22.94158935546875, "growth_active": 1.0436644554138184, "growth_memory": 1.5934547185897827, "heads": [{"head": 0, "memory_to_active": 0.4223872423171997, "contribution_to_active": 0.5706917643547058, "mix_mean": 0.570382833480835, "mix_low": 0.0029201507568359375, "mix_high": 0.019990921020507812, "read_cosine": 0.26153701543807983, "beta_mean": 0.01083130482584238, "nonfinite": 0.0, "weighted_memory_to_active": 0.25401949882507324, "mix_std": 0.31173259019851685, "beta_std": 0.014776376076042652, "beta_low": 0.6763916015625, "beta_high": 0.0, "orthogonal_to_active": 0.178893581032753, "evidence_weights": [-0.21169471740722656, -0.24510431289672852, 0.9317004084587097], "agreement_weight": 0.11896003782749176, "orthogonal_weight": -0.748030960559845}, {"head": 1, "memory_to_active": 0.01219177059829235, "contribution_to_active": 0.4177585244178772, "mix_mean": 0.5498156547546387, "mix_low": 0.005291938781738281, "mix_high": 0.008252143859863281, "read_cosine": 0.19299110770225525, "beta_mean": 6.915793346706778e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007068020291626453, "mix_std": 0.30180948972702026, "beta_std": 0.00014791174908168614, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005390377715229988, "evidence_weights": [-0.25850698351860046, -0.2127988636493683, 0.6637816429138184], "agreement_weight": 0.47674447298049927, "orthogonal_weight": -0.8412349224090576}, {"head": 2, "memory_to_active": 2.2828402519226074, "contribution_to_active": 0.2906007766723633, "mix_mean": 0.0488300696015358, "mix_low": 0.33350372314453125, "mix_high": 0.0, "read_cosine": 0.1997506320476532, "beta_mean": 0.16236886382102966, "nonfinite": 0.0, "weighted_memory_to_active": 0.33816927671432495, "mix_std": 0.09903492033481598, "beta_std": 0.13764116168022156, "beta_low": 0.01922607421875, "beta_high": 0.0, "orthogonal_to_active": 0.004909733310341835, "evidence_weights": [-0.0903509259223938, 0.0026756813749670982, 0.022008853033185005], "agreement_weight": -0.00907889474183321, "orthogonal_weight": -0.018674058839678764}, {"head": 3, "memory_to_active": 0.02546168677508831, "contribution_to_active": 0.4987809658050537, "mix_mean": 0.5758794546127319, "mix_low": 0.007109642028808594, "mix_high": 0.010967254638671875, "read_cosine": 0.30151844024658203, "beta_mean": 0.00012309884186834097, "nonfinite": 0.0, "weighted_memory_to_active": 0.01721329055726528, "mix_std": 0.29638707637786865, "beta_std": 0.00028379797004163265, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004396055359393358, "evidence_weights": [-0.24129529297351837, -0.22810471057891846, 2.147606611251831], "agreement_weight": 1.9253097772598267, "orthogonal_weight": -0.34033137559890747}, {"head": 4, "memory_to_active": 0.02047131396830082, "contribution_to_active": 0.31231772899627686, "mix_mean": 0.41034865379333496, "mix_low": 0.009164810180664062, "mix_high": 0.0015668869018554688, "read_cosine": 0.15310828387737274, "beta_mean": 0.00011048218584619462, "nonfinite": 0.0, "weighted_memory_to_active": 0.011541246436536312, "mix_std": 0.2894611656665802, "beta_std": 0.0001514885516371578, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010825597681105137, "evidence_weights": [-0.18326546251773834, -0.19043704867362976, 0.120427206158638], "agreement_weight": -0.03342907875776291, "orthogonal_weight": -0.9925369620323181}, {"head": 5, "memory_to_active": 0.250976026058197, "contribution_to_active": 0.633691132068634, "mix_mean": 0.6932388544082642, "mix_low": 0.00304412841796875, "mix_high": 0.0137481689453125, "read_cosine": 0.09682323038578033, "beta_mean": 0.0046161022037267685, "nonfinite": 0.0, "weighted_memory_to_active": 0.1838688999414444, "mix_std": 0.2750549614429474, "beta_std": 0.0071442145854234695, "beta_low": 0.88726806640625, "beta_high": 0.0, "orthogonal_to_active": 0.14137700200080872, "evidence_weights": [-0.25960713624954224, -0.32115933299064636, 0.4773813486099243], "agreement_weight": 0.1514262855052948, "orthogonal_weight": -0.7967697381973267}]}, {"layer": 4, "memory_to_active": 1.0507503747940063, "contribution_to_active": 0.4519793391227722, "mix_mean": 0.3616402745246887, "mix_low": 0.26757049560546875, "mix_high": 0.010326862335205078, "read_cosine": 0.3265167772769928, "beta_mean": 0.1185450479388237, "nonfinite": 0.0, "weighted_memory_to_active": 0.13767792284488678, "mix_std": 0.16057412326335907, "beta_std": 0.10765781998634338, "beta_low": 0.3761596977710724, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.04448547959327698, "output_influence": 0.44916263222694397, "residual_influence": 0.13002322614192963, "state_active": 0.03903460502624512, "state_memory": 0.018045805394649506, "state_peak": 37.37887191772461, "growth_active": 1.2557874917984009, "growth_memory": 1.5548889636993408, "heads": [{"head": 0, "memory_to_active": 0.19642499089241028, "contribution_to_active": 0.7529184222221375, "mix_mean": 0.6815764904022217, "mix_low": 0.01361846923828125, "mix_high": 0.03151702880859375, "read_cosine": 0.5111777186393738, "beta_mean": 0.2648165822029114, "nonfinite": 0.0, "weighted_memory_to_active": 0.16268786787986755, "mix_std": 0.34732186794281006, "beta_std": 0.2667350471019745, "beta_low": 0.22314453125, "beta_high": 0.0, "orthogonal_to_active": 0.0683777928352356, "evidence_weights": [-0.2670706808567047, -0.3678739368915558, 0.04362557455897331], "agreement_weight": 0.002664674073457718, "orthogonal_weight": 0.5414662957191467}, {"head": 1, "memory_to_active": 0.05405905842781067, "contribution_to_active": 0.6416383385658264, "mix_mean": 0.6761202812194824, "mix_low": 0.00011348724365234375, "mix_high": 0.014004707336425781, "read_cosine": 0.6450320482254028, "beta_mean": 0.0008748545660637319, "nonfinite": 0.0, "weighted_memory_to_active": 0.0387762077152729, "mix_std": 0.2428349256515503, "beta_std": 0.0048338077031075954, "beta_low": 0.98419189453125, "beta_high": 0.0, "orthogonal_to_active": 0.021027032285928726, "evidence_weights": [-0.23946282267570496, -0.3563959002494812, 0.6665197610855103], "agreement_weight": 0.26307743787765503, "orthogonal_weight": -0.7884916663169861}, {"head": 2, "memory_to_active": 1.9525249004364014, "contribution_to_active": 0.25952214002609253, "mix_mean": 0.026719702407717705, "mix_low": 0.5291824340820312, "mix_high": 2.002716064453125e-05, "read_cosine": 0.23911616206169128, "beta_mean": 0.08513865619897842, "nonfinite": 0.0, "weighted_memory_to_active": 0.26708805561065674, "mix_std": 0.06055816635489464, "beta_std": 0.08389125764369965, "beta_low": 0.033447265625, "beta_high": 0.0, "orthogonal_to_active": 0.04990948736667633, "evidence_weights": [0.05932317301630974, 0.008596744388341904, 0.10504980385303497], "agreement_weight": 0.12327190488576889, "orthogonal_weight": -0.2928200364112854}, {"head": 3, "memory_to_active": 1.7755694389343262, "contribution_to_active": 0.15309584140777588, "mix_mean": 0.03672274947166443, "mix_low": 0.39833831787109375, "mix_high": 0.0, "read_cosine": 0.180661141872406, "beta_mean": 0.14236479997634888, "nonfinite": 0.0, "weighted_memory_to_active": 0.2049497663974762, "mix_std": 0.06596416234970093, "beta_std": 0.12461394816637039, "beta_low": 0.01611328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.0018393909558653831, "evidence_weights": [-0.02879098802804947, 0.04947919026017189, -0.25765174627304077], "agreement_weight": 0.07256820797920227, "orthogonal_weight": -0.012588411569595337}, {"head": 4, "memory_to_active": 0.015308249741792679, "contribution_to_active": 0.6353261470794678, "mix_mean": 0.7353786826133728, "mix_low": 7.82012939453125e-05, "mix_high": 0.01641559600830078, "read_cosine": 0.15739694237709045, "beta_mean": 2.9867225748603232e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011370368301868439, "mix_std": 0.21656745672225952, "beta_std": 6.956983997952193e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00519246980547905, "evidence_weights": [-0.33665475249290466, -0.32557374238967896, 0.5495638847351074], "agreement_weight": 0.3521369397640228, "orthogonal_weight": -0.49267855286598206}, {"head": 5, "memory_to_active": 2.310615301132202, "contribution_to_active": 0.26937511563301086, "mix_mean": 0.013323643244802952, "mix_low": 0.6640920639038086, "mix_high": 3.814697265625e-06, "read_cosine": 0.22571663558483124, "beta_mean": 0.21804550290107727, "nonfinite": 0.0, "weighted_memory_to_active": 0.14119526743888855, "mix_std": 0.03019816428422928, "beta_std": 0.16580328345298767, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1205667033791542, "evidence_weights": [0.053621262311935425, -0.02419394440948963, -0.27900731563568115], "agreement_weight": -0.0741603672504425, "orthogonal_weight": 0.9490177631378174}]}, {"layer": 5, "memory_to_active": 0.45385974645614624, "contribution_to_active": 0.5210119485855103, "mix_mean": 0.4929324686527252, "mix_low": 0.05999438464641571, "mix_high": 0.017460983246564865, "read_cosine": 0.09752452373504639, "beta_mean": 0.05522850155830383, "nonfinite": 0.0, "weighted_memory_to_active": 0.14008919894695282, "mix_std": 0.2589283585548401, "beta_std": 0.07118310034275055, "beta_low": 0.64013671875, "beta_high": 0.0, "orthogonal_to_active": 0.05425722524523735, "output_influence": 0.5331324338912964, "residual_influence": 0.15408658981323242, "state_active": 0.04370597004890442, "state_memory": 0.011397704482078552, "state_peak": 60.48475646972656, "growth_active": 1.1600053310394287, "growth_memory": 1.3140006065368652, "heads": [{"head": 0, "memory_to_active": 0.382096529006958, "contribution_to_active": 0.5196362733840942, "mix_mean": 0.3957293629646301, "mix_low": 0.017122268676757812, "mix_high": 0.0012235641479492188, "read_cosine": -0.0381026417016983, "beta_mean": 0.03156481683254242, "nonfinite": 0.0, "weighted_memory_to_active": 0.12229014188051224, "mix_std": 0.2797130048274994, "beta_std": 0.05290458723902702, "beta_low": 0.390625, "beta_high": 0.0, "orthogonal_to_active": 0.04722500592470169, "evidence_weights": [-0.13805636763572693, -0.1635298877954483, 0.014251665212213993], "agreement_weight": -0.026733385398983955, "orthogonal_weight": -0.3890458643436432}, {"head": 1, "memory_to_active": 0.01190551184117794, "contribution_to_active": 0.5431089401245117, "mix_mean": 0.6878583431243896, "mix_low": 0.0017871856689453125, "mix_high": 0.03407478332519531, "read_cosine": 0.17930647730827332, "beta_mean": 2.8804966859752312e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007256141863763332, "mix_std": 0.27689117193222046, "beta_std": 5.761177453678101e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006893077865242958, "evidence_weights": [-0.25178638100624084, -0.28878656029701233, 0.8634520173072815], "agreement_weight": 0.6531897783279419, "orthogonal_weight": -0.9921162724494934}, {"head": 2, "memory_to_active": 0.010475965216755867, "contribution_to_active": 0.5916415452957153, "mix_mean": 0.6511770486831665, "mix_low": 0.0031900405883789062, "mix_high": 0.021806716918945312, "read_cosine": -0.4274420142173767, "beta_mean": 9.614392183721066e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006711828056722879, "mix_std": 0.2697872221469879, "beta_std": 0.00015845599409658462, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028929379768669605, "evidence_weights": [-0.26080650091171265, -0.3391727805137634, 0.3376705050468445], "agreement_weight": -0.044373758137226105, "orthogonal_weight": -0.5335100293159485}, {"head": 3, "memory_to_active": 1.7915312051773071, "contribution_to_active": 0.2151969075202942, "mix_mean": 0.11155954003334045, "mix_low": 0.1402111053466797, "mix_high": 4.76837158203125e-06, "read_cosine": 0.14938285946846008, "beta_mean": 0.11724594235420227, "nonfinite": 0.0, "weighted_memory_to_active": 0.356723427772522, "mix_std": 0.14838828146457672, "beta_std": 0.16130027174949646, "beta_low": 0.1641845703125, "beta_high": 0.0, "orthogonal_to_active": 0.13648755848407745, "evidence_weights": [-0.06886187195777893, -0.095838263630867, 0.04517427086830139], "agreement_weight": 0.04224827140569687, "orthogonal_weight": -0.47972679138183594}, {"head": 4, "memory_to_active": 0.4376052916049957, "contribution_to_active": 0.5859270095825195, "mix_mean": 0.32404178380966187, "mix_low": 0.19540977478027344, "mix_high": 0.008586883544921875, "read_cosine": 0.5405621528625488, "beta_mean": 0.18218551576137543, "nonfinite": 0.0, "weighted_memory_to_active": 0.28738290071487427, "mix_std": 0.3436964154243469, "beta_std": 0.2115742266178131, "beta_low": 0.2882080078125, "beta_high": 0.0, "orthogonal_to_active": 0.09462889283895493, "evidence_weights": [-0.11672306060791016, -0.2679695785045624, -0.07000356912612915], "agreement_weight": 0.04503713548183441, "orthogonal_weight": 0.4982011020183563}, {"head": 5, "memory_to_active": 0.08954401314258575, "contribution_to_active": 0.6705607175827026, "mix_mean": 0.7872285842895508, "mix_low": 0.0022459030151367188, "mix_high": 0.039069175720214844, "read_cosine": 0.18144020438194275, "beta_mean": 0.0002497791429050267, "nonfinite": 0.0, "weighted_memory_to_active": 0.060170698910951614, "mix_std": 0.23509398102760315, "beta_std": 0.0011034236522391438, "beta_low": 0.997802734375, "beta_high": 0.0, "orthogonal_to_active": 0.03741585090756416, "evidence_weights": [-0.3123334050178528, -0.33203718066215515, 1.9495526552200317], "agreement_weight": 0.4555409848690033, "orthogonal_weight": -0.6573934555053711}]}, {"layer": 6, "memory_to_active": 0.31087443232536316, "contribution_to_active": 0.6201295852661133, "mix_mean": 0.5837873220443726, "mix_low": 0.05339972302317619, "mix_high": 0.03690401837229729, "read_cosine": 0.2933182120323181, "beta_mean": 0.0652976930141449, "nonfinite": 0.0, "weighted_memory_to_active": 0.15884706377983093, "mix_std": 0.23592394590377808, "beta_std": 0.07620302587747574, "beta_low": 0.5269063711166382, "beta_high": 0.0, "orthogonal_to_active": 0.04591734707355499, "output_influence": 0.5362183451652527, "residual_influence": 0.17743511497974396, "state_active": 0.04606951028108597, "state_memory": 0.008018785156309605, "state_peak": 91.91654205322266, "growth_active": 1.2075084447860718, "growth_memory": 1.1667759418487549, "heads": [{"head": 0, "memory_to_active": 0.13652828335762024, "contribution_to_active": 0.7130221128463745, "mix_mean": 0.7989366054534912, "mix_low": 0.00031566619873046875, "mix_high": 0.038712501525878906, "read_cosine": 0.14440540969371796, "beta_mean": 0.018849892541766167, "nonfinite": 0.0, "weighted_memory_to_active": 0.11057349294424057, "mix_std": 0.20497167110443115, "beta_std": 0.034352801740169525, "beta_low": 0.57098388671875, "beta_high": 0.0, "orthogonal_to_active": 0.08878153562545776, "evidence_weights": [-0.3205999732017517, -0.3476233184337616, 1.157751202583313], "agreement_weight": 0.4487016499042511, "orthogonal_weight": -0.8335407376289368}, {"head": 1, "memory_to_active": 0.006354180630296469, "contribution_to_active": 0.5302470326423645, "mix_mean": 0.578801155090332, "mix_low": 0.0011110305786132812, "mix_high": 0.014496803283691406, "read_cosine": 0.49810779094696045, "beta_mean": 1.500144207966514e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002985571278259158, "mix_std": 0.28511181473731995, "beta_std": 2.7526197300176136e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015804875874891877, "evidence_weights": [-0.20068728923797607, -0.301428884267807, 1.1044635772705078], "agreement_weight": 0.08582571893930435, "orthogonal_weight": -0.6930199265480042}, {"head": 2, "memory_to_active": 0.859143853187561, "contribution_to_active": 0.0988597497344017, "mix_mean": 0.03974241763353348, "mix_low": 0.3112783432006836, "mix_high": 0.0, "read_cosine": 0.253251314163208, "beta_mean": 0.03496689721941948, "nonfinite": 0.0, "weighted_memory_to_active": 0.07659493386745453, "mix_std": 0.06367893517017365, "beta_std": 0.045448072254657745, "beta_low": 0.13104248046875, "beta_high": 0.0, "orthogonal_to_active": 0.005399500019848347, "evidence_weights": [0.08126552402973175, -0.03418755903840065, 0.29792946577072144], "agreement_weight": 0.13394148647785187, "orthogonal_weight": -0.08370690047740936}, {"head": 3, "memory_to_active": 0.1454249918460846, "contribution_to_active": 0.6749532222747803, "mix_mean": 0.6515734195709229, "mix_low": 0.00629425048828125, "mix_high": 0.022012710571289062, "read_cosine": 0.4631842076778412, "beta_mean": 0.21389679610729218, "nonfinite": 0.0, "weighted_memory_to_active": 0.10493107885122299, "mix_std": 0.3256557285785675, "beta_std": 0.24086791276931763, "beta_low": 0.18975830078125, "beta_high": 0.0, "orthogonal_to_active": 0.05635439604520798, "evidence_weights": [-0.17452602088451385, -0.33062341809272766, 0.18452855944633484], "agreement_weight": -0.008031228557229042, "orthogonal_weight": 0.6213257312774658}, {"head": 4, "memory_to_active": 0.6774744987487793, "contribution_to_active": 1.0843615531921387, "mix_mean": 0.7631558179855347, "mix_low": 0.00122833251953125, "mix_high": 0.14068126678466797, "read_cosine": 0.1349080204963684, "beta_mean": 0.12383250892162323, "nonfinite": 0.0, "weighted_memory_to_active": 0.6289050579071045, "mix_std": 0.28853490948677063, "beta_std": 0.13600029051303864, "beta_low": 0.2698974609375, "beta_high": 0.0, "orthogonal_to_active": 0.09449145942926407, "evidence_weights": [-0.22551845014095306, -0.578830361366272, -0.33108341693878174], "agreement_weight": -0.02114218659698963, "orthogonal_weight": 0.16736742854118347}, {"head": 5, "memory_to_active": 0.04032079502940178, "contribution_to_active": 0.6193336248397827, "mix_mean": 0.6705142259597778, "mix_low": 0.00017070770263671875, "mix_high": 0.005520820617675781, "read_cosine": 0.2660524249076843, "beta_mean": 0.0002250351244583726, "nonfinite": 0.0, "weighted_memory_to_active": 0.029092196375131607, "mix_std": 0.247590571641922, "beta_std": 0.0005215306300669909, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.028896696865558624, "evidence_weights": [-0.24395082890987396, -0.31552785634994507, 0.40706419944763184], "agreement_weight": 0.18025411665439606, "orthogonal_weight": -1.0647462606430054}]}, {"layer": 7, "memory_to_active": 1.7153034210205078, "contribution_to_active": 0.25008994340896606, "mix_mean": 0.1548900306224823, "mix_low": 0.2550336718559265, "mix_high": 0.0008640289306640625, "read_cosine": 0.24002769589424133, "beta_mean": 0.13302460312843323, "nonfinite": 0.0, "weighted_memory_to_active": 0.2015613317489624, "mix_std": 0.1465175747871399, "beta_std": 0.1000041663646698, "beta_low": 0.1990356594324112, "beta_high": 8.138021075865254e-05, "orthogonal_to_active": 0.08371511101722717, "output_influence": 0.26377537846565247, "residual_influence": 0.06163204461336136, "state_active": 0.013708574697375298, "state_memory": 0.02536649815738201, "state_peak": 30.33223533630371, "growth_active": 1.1896723508834839, "growth_memory": 1.3012197017669678, "heads": [{"head": 0, "memory_to_active": 2.4013872146606445, "contribution_to_active": 0.17141079902648926, "mix_mean": 0.0387079194188118, "mix_low": 0.41056346893310547, "mix_high": 0.0, "read_cosine": 0.36687347292900085, "beta_mean": 0.1869487762451172, "nonfinite": 0.0, "weighted_memory_to_active": 0.2031617909669876, "mix_std": 0.08089178800582886, "beta_std": 0.14064785838127136, "beta_low": 0.00439453125, "beta_high": 0.000244140625, "orthogonal_to_active": 0.0022077157627791166, "evidence_weights": [0.03068200685083866, -0.07679115980863571, 0.0408967025578022], "agreement_weight": -0.031810030341148376, "orthogonal_weight": 0.013585778884589672}, {"head": 1, "memory_to_active": 0.9653044939041138, "contribution_to_active": 0.3941328525543213, "mix_mean": 0.2179652452468872, "mix_low": 0.1294231414794922, "mix_high": 0.0044155120849609375, "read_cosine": 0.3611224889755249, "beta_mean": 0.08955767750740051, "nonfinite": 0.0, "weighted_memory_to_active": 0.32467663288116455, "mix_std": 0.2424754500389099, "beta_std": 0.06107564643025398, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04198794066905975, "evidence_weights": [-0.21100355684757233, -0.10171186923980713, -0.2311948537826538], "agreement_weight": 0.04039660841226578, "orthogonal_weight": 0.152562215924263}, {"head": 2, "memory_to_active": 0.7041423320770264, "contribution_to_active": 0.3386002779006958, "mix_mean": 0.1903354823589325, "mix_low": 0.03771209716796875, "mix_high": 0.0003070831298828125, "read_cosine": 0.11829017102718353, "beta_mean": 0.051198624074459076, "nonfinite": 0.0, "weighted_memory_to_active": 0.2282772809267044, "mix_std": 0.20148812234401703, "beta_std": 0.044443465769290924, "beta_low": 0.06146240234375, "beta_high": 0.0, "orthogonal_to_active": 0.11122547090053558, "evidence_weights": [-0.07583533972501755, -0.15760333836078644, 0.712683916091919], "agreement_weight": 0.020988235250115395, "orthogonal_weight": -0.5001363754272461}, {"head": 3, "memory_to_active": 4.492860794067383, "contribution_to_active": 0.10237966477870941, "mix_mean": 0.03655264899134636, "mix_low": 0.3239116668701172, "mix_high": 0.0, "read_cosine": 0.0936727449297905, "beta_mean": 0.3927082419395447, "nonfinite": 0.0, "weighted_memory_to_active": 0.35650837421417236, "mix_std": 0.055439628660678864, "beta_std": 0.2712066173553467, "beta_low": 0.00506591796875, "beta_high": 0.000244140625, "orthogonal_to_active": 0.31010574102401733, "evidence_weights": [0.06827716529369354, -0.022695764899253845, 0.7126595973968506], "agreement_weight": 0.05438125878572464, "orthogonal_weight": -0.9431546926498413}, {"head": 4, "memory_to_active": 1.6708518266677856, "contribution_to_active": 0.08715295791625977, "mix_mean": 0.017796438187360764, "mix_low": 0.6256017684936523, "mix_high": 0.0, "read_cosine": 0.22911997139453888, "beta_mean": 0.07741709053516388, "nonfinite": 0.0, "weighted_memory_to_active": 0.06898801028728485, "mix_std": 0.0415128730237484, "beta_std": 0.08199071884155273, "beta_low": 0.1236572265625, "beta_high": 0.0, "orthogonal_to_active": 0.012495298869907856, "evidence_weights": [0.1512337028980255, 0.01658923737704754, 0.22082673013210297], "agreement_weight": 0.04332231357693672, "orthogonal_weight": -0.20396950840950012}, {"head": 5, "memory_to_active": 0.05727294459939003, "contribution_to_active": 0.4068629741668701, "mix_mean": 0.4279823899269104, "mix_low": 0.0029897689819335938, "mix_high": 0.000461578369140625, "read_cosine": 0.27108728885650635, "beta_mean": 0.00031720715924166143, "nonfinite": 0.0, "weighted_memory_to_active": 0.027755914255976677, "mix_std": 0.2572975158691406, "beta_std": 0.0006606535753235221, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.024268459528684616, "evidence_weights": [-0.1621595323085785, -0.2199370414018631, 0.5568636059761047], "agreement_weight": 0.18754050135612488, "orthogonal_weight": -0.9468806982040405}]}, {"layer": 8, "memory_to_active": 0.2381955087184906, "contribution_to_active": 0.5366516709327698, "mix_mean": 0.447862833738327, "mix_low": 0.027073225006461143, "mix_high": 0.0014278094749897718, "read_cosine": 0.44007110595703125, "beta_mean": 0.016103368252515793, "nonfinite": 0.0, "weighted_memory_to_active": 0.0901690274477005, "mix_std": 0.2284362018108368, "beta_std": 0.022909872233867645, "beta_low": 0.721832275390625, "beta_high": 0.0, "orthogonal_to_active": 0.07338105887174606, "output_influence": 0.4856811761856079, "residual_influence": 0.16155412793159485, "state_active": 0.047001294791698456, "state_memory": 0.006617508828639984, "state_peak": 41.47425079345703, "growth_active": 1.1272289752960205, "growth_memory": 1.196947455406189, "heads": [{"head": 0, "memory_to_active": 0.06208895146846771, "contribution_to_active": 0.5605273842811584, "mix_mean": 0.5965544581413269, "mix_low": 0.0004711151123046875, "mix_high": 0.0012788772583007812, "read_cosine": 0.48472923040390015, "beta_mean": 0.00245469156652689, "nonfinite": 0.0, "weighted_memory_to_active": 0.043905287981033325, "mix_std": 0.2476317286491394, "beta_std": 0.008355778641998768, "beta_low": 0.9444580078125, "beta_high": 0.0, "orthogonal_to_active": 0.02746601216495037, "evidence_weights": [-0.16847269237041473, -0.38948166370391846, 0.6475296020507812], "agreement_weight": 0.21669252216815948, "orthogonal_weight": -0.8068140149116516}, {"head": 1, "memory_to_active": 0.09681443870067596, "contribution_to_active": 0.524688184261322, "mix_mean": 0.49761924147605896, "mix_low": 0.000148773193359375, "mix_high": 0.0009002685546875, "read_cosine": 0.17083150148391724, "beta_mean": 0.001435504062101245, "nonfinite": 0.0, "weighted_memory_to_active": 0.044035784900188446, "mix_std": 0.23893213272094727, "beta_std": 0.008837307803332806, "beta_low": 0.98211669921875, "beta_high": 0.0, "orthogonal_to_active": 0.03225230425596237, "evidence_weights": [-0.19036929309368134, -0.2721381187438965, 0.6963698863983154], "agreement_weight": 0.07656698673963547, "orthogonal_weight": -0.7613250017166138}, {"head": 2, "memory_to_active": 0.014649299904704094, "contribution_to_active": 0.6651633977890015, "mix_mean": 0.6623917818069458, "mix_low": 0.0008440017700195312, "mix_high": 0.0032835006713867188, "read_cosine": 0.4091804027557373, "beta_mean": 0.00013461634807754308, "nonfinite": 0.0, "weighted_memory_to_active": 0.011265214532613754, "mix_std": 0.2271466851234436, "beta_std": 0.0021590895485132933, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0070052118971943855, "evidence_weights": [-0.22604243457317352, -0.3441755473613739, 1.021672248840332], "agreement_weight": 0.24791868031024933, "orthogonal_weight": -0.718651533126831}, {"head": 3, "memory_to_active": 0.10807318985462189, "contribution_to_active": 0.6309255361557007, "mix_mean": 0.5669758319854736, "mix_low": 0.000888824462890625, "mix_high": 0.0030260086059570312, "read_cosine": 0.7960419654846191, "beta_mean": 0.00888141430914402, "nonfinite": 0.0, "weighted_memory_to_active": 0.08289873600006104, "mix_std": 0.25700509548187256, "beta_std": 0.007909970358014107, "beta_low": 0.6729736328125, "beta_high": 0.0, "orthogonal_to_active": 0.027390671893954277, "evidence_weights": [-0.18525321781635284, -0.30338582396507263, 0.5706105828285217], "agreement_weight": 0.21457648277282715, "orthogonal_weight": -0.6407122015953064}, {"head": 4, "memory_to_active": 0.7691128253936768, "contribution_to_active": 0.1899520456790924, "mix_mean": 0.10976196825504303, "mix_low": 0.08255672454833984, "mix_high": 0.0, "read_cosine": 0.3001846373081207, "beta_mean": 0.016055632382631302, "nonfinite": 0.0, "weighted_memory_to_active": 0.16145512461662292, "mix_std": 0.13183993101119995, "beta_std": 0.03286007046699524, "beta_low": 0.65325927734375, "beta_high": 0.0, "orthogonal_to_active": 0.029314981773495674, "evidence_weights": [-0.012789981439709663, -0.11442264169454575, 0.1923811286687851], "agreement_weight": 0.0755685567855835, "orthogonal_weight": -0.19218304753303528}, {"head": 5, "memory_to_active": 0.3784342408180237, "contribution_to_active": 0.6486532092094421, "mix_mean": 0.2538735568523407, "mix_low": 0.0775299072265625, "mix_high": 7.82012939453125e-05, "read_cosine": 0.479458749294281, "beta_mean": 0.06765834987163544, "nonfinite": 0.0, "weighted_memory_to_active": 0.19745397567749023, "mix_std": 0.2680616080760956, "beta_std": 0.07733701169490814, "beta_low": 0.0782470703125, "beta_high": 0.0, "orthogonal_to_active": 0.31685715913772583, "evidence_weights": [-0.11816368997097015, -0.17762404680252075, -0.13843810558319092], "agreement_weight": -0.01507748942822218, "orthogonal_weight": 1.8685519695281982}]}, {"layer": 9, "memory_to_active": 0.9959661960601807, "contribution_to_active": 0.6968730688095093, "mix_mean": 0.34818360209465027, "mix_low": 0.11306620389223099, "mix_high": 0.005167802330106497, "read_cosine": 0.35023725032806396, "beta_mean": 0.04947880282998085, "nonfinite": 0.0, "weighted_memory_to_active": 0.40868890285491943, "mix_std": 0.22180159389972687, "beta_std": 0.038575101643800735, "beta_low": 0.5128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.039477113634347916, "output_influence": 0.5311166048049927, "residual_influence": 0.1492345631122589, "state_active": 0.04451512545347214, "state_memory": 0.017379652708768845, "state_peak": 54.51637649536133, "growth_active": 1.2450963258743286, "growth_memory": 1.3337994813919067, "heads": [{"head": 0, "memory_to_active": 0.001093965140171349, "contribution_to_active": 0.5594008564949036, "mix_mean": 0.6347522735595703, "mix_low": 0.0013666152954101562, "mix_high": 0.022240638732910156, "read_cosine": 0.28941109776496887, "beta_mean": 3.4233071346534416e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007055996684357524, "mix_std": 0.29026249051094055, "beta_std": 1.5163630450842902e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007225294248200953, "evidence_weights": [-0.30264559388160706, -0.30994337797164917, 0.6819168925285339], "agreement_weight": 0.11105538159608841, "orthogonal_weight": -1.0692301988601685}, {"head": 1, "memory_to_active": 0.02124757133424282, "contribution_to_active": 0.6763932704925537, "mix_mean": 0.5545148849487305, "mix_low": 0.007221221923828125, "mix_high": 0.0018224716186523438, "read_cosine": 0.41109246015548706, "beta_mean": 5.8039629948325455e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015242583118379116, "mix_std": 0.2849216163158417, "beta_std": 0.00011728860408766195, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010696789249777794, "evidence_weights": [-0.23881052434444427, -0.2938453257083893, 1.1056636571884155], "agreement_weight": 0.4259306490421295, "orthogonal_weight": -0.8520534634590149}, {"head": 2, "memory_to_active": 0.18390503525733948, "contribution_to_active": 0.6441121697425842, "mix_mean": 0.587278425693512, "mix_low": 0.0016794204711914062, "mix_high": 0.005848884582519531, "read_cosine": 0.3341493010520935, "beta_mean": 0.0006567347445525229, "nonfinite": 0.0, "weighted_memory_to_active": 0.12648478150367737, "mix_std": 0.27101996541023254, "beta_std": 0.004534909967333078, "beta_low": 0.99493408203125, "beta_high": 0.0, "orthogonal_to_active": 0.07370609045028687, "evidence_weights": [-0.19821786880493164, -0.27427905797958374, 0.7643454670906067], "agreement_weight": 0.3114234507083893, "orthogonal_weight": -0.6778221726417542}, {"head": 3, "memory_to_active": 1.2003098726272583, "contribution_to_active": 0.1742946207523346, "mix_mean": 0.06781156361103058, "mix_low": 0.28088951110839844, "mix_high": 5.7220458984375e-05, "read_cosine": 0.394010066986084, "beta_mean": 0.06573899835348129, "nonfinite": 0.0, "weighted_memory_to_active": 0.16064290702342987, "mix_std": 0.1248769536614418, "beta_std": 0.0540023073554039, "beta_low": 0.006591796875, "beta_high": 0.0, "orthogonal_to_active": 0.01960419863462448, "evidence_weights": [0.05425162985920906, -0.01692144386470318, 0.09522206336259842], "agreement_weight": 0.09685447067022324, "orthogonal_weight": -0.13013117015361786}, {"head": 4, "memory_to_active": 3.1009438037872314, "contribution_to_active": 1.5792381763458252, "mix_mean": 0.1431795358657837, "mix_low": 0.13286304473876953, "mix_high": 0.000873565673828125, "read_cosine": 0.16959312558174133, "beta_mean": 0.1532488763332367, "nonfinite": 0.0, "weighted_memory_to_active": 1.551802158355713, "mix_std": 0.2020076960325241, "beta_std": 0.0855422168970108, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02336989715695381, "evidence_weights": [-0.07908674329519272, -0.15701718628406525, -0.02970561571419239], "agreement_weight": -0.05772653594613075, "orthogonal_weight": -0.01576397754251957}, {"head": 5, "memory_to_active": 1.4682964086532593, "contribution_to_active": 0.547799289226532, "mix_mean": 0.10156470537185669, "mix_low": 0.2543773651123047, "mix_high": 0.000164031982421875, "read_cosine": 0.5031672716140747, "beta_mean": 0.07716674357652664, "nonfinite": 0.0, "weighted_memory_to_active": 0.5972553491592407, "mix_std": 0.15772081911563873, "beta_std": 0.08723871409893036, "beta_low": 0.07537841796875, "beta_high": 0.0, "orthogonal_to_active": 0.10876317322254181, "evidence_weights": [0.10217560827732086, -0.11034470796585083, 0.13939034938812256], "agreement_weight": 0.017094474285840988, "orthogonal_weight": 0.2609172463417053}]}, {"layer": 10, "memory_to_active": 0.25670552253723145, "contribution_to_active": 0.759863018989563, "mix_mean": 0.5137821435928345, "mix_low": 0.02078835293650627, "mix_high": 0.005281607620418072, "read_cosine": 0.3691466450691223, "beta_mean": 0.04477256163954735, "nonfinite": 0.0, "weighted_memory_to_active": 0.12375490367412567, "mix_std": 0.26718270778656006, "beta_std": 0.04673928767442703, "beta_low": 0.66009521484375, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.2757185399532318, "output_influence": 0.44964519143104553, "residual_influence": 0.15311282873153687, "state_active": 0.05205809324979782, "state_memory": 0.015617545694112778, "state_peak": 115.90574645996094, "growth_active": 1.039198875427246, "growth_memory": 0.9751800894737244, "heads": [{"head": 0, "memory_to_active": 1.01335608959198, "contribution_to_active": 2.28120493888855, "mix_mean": 0.5760970115661621, "mix_low": 0.084869384765625, "mix_high": 0.011720657348632812, "read_cosine": 0.0820000171661377, "beta_mean": 0.20428681373596191, "nonfinite": 0.0, "weighted_memory_to_active": 0.6164693832397461, "mix_std": 0.3917423486709595, "beta_std": 0.21932312846183777, "beta_low": 0.1414794921875, "beta_high": 0.0001220703125, "orthogonal_to_active": 1.5656300783157349, "evidence_weights": [-0.10250554233789444, -0.38672274351119995, -1.6242247819900513], "agreement_weight": 0.02419174276292324, "orthogonal_weight": 2.558640241622925}, {"head": 1, "memory_to_active": 0.014132336713373661, "contribution_to_active": 0.6055964231491089, "mix_mean": 0.6214004158973694, "mix_low": 0.0015821456909179688, "mix_high": 0.013851165771484375, "read_cosine": 0.2694552540779114, "beta_mean": 9.926095663104206e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00819880235940218, "mix_std": 0.26512497663497925, "beta_std": 0.00013140993542037904, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012265211902558804, "evidence_weights": [-0.25402289628982544, -0.197873055934906, 0.5588364005088806], "agreement_weight": 0.5406143665313721, "orthogonal_weight": -1.6094563007354736}, {"head": 2, "memory_to_active": 0.015672041103243828, "contribution_to_active": 0.6149777173995972, "mix_mean": 0.6513365507125854, "mix_low": 0.0001926422119140625, "mix_high": 0.0050373077392578125, "read_cosine": 0.04088140279054642, "beta_mean": 3.280527380411513e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006815020926296711, "mix_std": 0.2481347769498825, "beta_std": 0.00014261092292144895, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030962293967604637, "evidence_weights": [-0.2584857940673828, -0.2837371230125427, 0.6402884721755981], "agreement_weight": -0.19082298874855042, "orthogonal_weight": -0.4351459741592407}, {"head": 3, "memory_to_active": 0.3490343987941742, "contribution_to_active": 0.09725014865398407, "mix_mean": 0.26206913590431213, "mix_low": 0.03609466552734375, "mix_high": 0.0002155303955078125, "read_cosine": 0.9119691848754883, "beta_mean": 0.057407915592193604, "nonfinite": 0.0, "weighted_memory_to_active": 0.038445428013801575, "mix_std": 0.22015094757080078, "beta_std": 0.052586063742637634, "beta_low": 0.00421142578125, "beta_high": 0.0, "orthogonal_to_active": 0.04182840511202812, "evidence_weights": [-0.10284057259559631, -0.1451132744550705, 0.1588573455810547], "agreement_weight": 0.01402996201068163, "orthogonal_weight": 1.415257215499878}, {"head": 4, "memory_to_active": 0.016932113096117973, "contribution_to_active": 0.5446515679359436, "mix_mean": 0.5747096538543701, "mix_low": 0.0002307891845703125, "mix_high": 0.0008459091186523438, "read_cosine": 0.0407949760556221, "beta_mean": 5.729562326450832e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00962390098720789, "mix_std": 0.24916554987430573, "beta_std": 0.00012563489144667983, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012346007861196995, "evidence_weights": [-0.1832042634487152, -0.2600783109664917, 0.9305154085159302], "agreement_weight": 0.14211654663085938, "orthogonal_weight": -1.3539303541183472}, {"head": 5, "memory_to_active": 0.1311061978340149, "contribution_to_active": 0.4154972434043884, "mix_mean": 0.3970801830291748, "mix_low": 0.0017604827880859375, "mix_high": 1.9073486328125e-05, "read_cosine": 0.8697788715362549, "beta_mean": 0.006751279346644878, "nonfinite": 0.0, "weighted_memory_to_active": 0.0629768818616867, "mix_std": 0.2287776619195938, "beta_std": 0.008126871660351753, "beta_low": 0.81488037109375, "beta_high": 0.0, "orthogonal_to_active": 0.0191451758146286, "evidence_weights": [-0.14044557511806488, -0.16081993281841278, 0.26804235577583313], "agreement_weight": 0.08981607854366302, "orthogonal_weight": -0.6517888307571411}]}, {"layer": 11, "memory_to_active": 0.08943627774715424, "contribution_to_active": 0.6089110374450684, "mix_mean": 0.5092675089836121, "mix_low": 0.10440842807292938, "mix_high": 0.04053942486643791, "read_cosine": 0.35969939827919006, "beta_mean": 0.027140704914927483, "nonfinite": 0.0, "weighted_memory_to_active": 0.045844435691833496, "mix_std": 0.3234875500202179, "beta_std": 0.041332878172397614, "beta_low": 0.6754049062728882, "beta_high": 0.0, "orthogonal_to_active": 0.04566839709877968, "output_influence": 0.5388990640640259, "residual_influence": 0.13839450478553772, "state_active": 0.18983162939548492, "state_memory": 0.004419893026351929, "state_peak": 31.610607147216797, "growth_active": 1.065228819847107, "growth_memory": 1.608154058456421, "heads": [{"head": 0, "memory_to_active": 0.0062910933047533035, "contribution_to_active": 0.5996224284172058, "mix_mean": 0.6426085233688354, "mix_low": 0.0009546279907226562, "mix_high": 0.008572578430175781, "read_cosine": 0.28262943029403687, "beta_mean": 7.158119115047157e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004357452038675547, "mix_std": 0.2577359080314636, "beta_std": 0.0036093806847929955, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.003433842211961746, "evidence_weights": [-0.17034025490283966, -0.36729303002357483, 0.7502562403678894], "agreement_weight": 0.33145463466644287, "orthogonal_weight": -0.8173352479934692}, {"head": 1, "memory_to_active": 0.19320622086524963, "contribution_to_active": 0.3835012912750244, "mix_mean": 0.4354550242424011, "mix_low": 0.006066322326660156, "mix_high": 0.0003757476806640625, "read_cosine": 0.5254914164543152, "beta_mean": 0.013495503924787045, "nonfinite": 0.0, "weighted_memory_to_active": 0.08851108700037003, "mix_std": 0.26827090978622437, "beta_std": 0.0370606891810894, "beta_low": 0.772705078125, "beta_high": 0.0, "orthogonal_to_active": 0.04094577953219414, "evidence_weights": [-0.16261766850948334, -0.23073117434978485, 0.5854297280311584], "agreement_weight": 0.07080407440662384, "orthogonal_weight": -0.5658150315284729}, {"head": 2, "memory_to_active": 0.008573556318879128, "contribution_to_active": 0.9806762337684631, "mix_mean": 0.4314233064651489, "mix_low": 0.2871532440185547, "mix_high": 0.14590167999267578, "read_cosine": 0.1884213089942932, "beta_mean": 0.07037079334259033, "nonfinite": 0.0, "weighted_memory_to_active": 0.006082418840378523, "mix_std": 0.44476526975631714, "beta_std": 0.09433906525373459, "beta_low": 0.47283935546875, "beta_high": 0.0, "orthogonal_to_active": 0.049655623733997345, "evidence_weights": [0.047683559358119965, -0.4018113613128662, -1.3143635988235474], "agreement_weight": 0.0027671733405441046, "orthogonal_weight": 8.457582473754883}, {"head": 3, "memory_to_active": 0.008366276510059834, "contribution_to_active": 0.8482438325881958, "mix_mean": 0.4075062870979309, "mix_low": 0.3231515884399414, "mix_high": 0.08183097839355469, "read_cosine": 0.5091701745986938, "beta_mean": 0.056185849010944366, "nonfinite": 0.0, "weighted_memory_to_active": 0.00509306276217103, "mix_std": 0.4370463788509369, "beta_std": 0.08397669345140457, "beta_low": 0.50787353515625, "beta_high": 0.0, "orthogonal_to_active": 0.032350409775972366, "evidence_weights": [0.18083323538303375, -0.3111833333969116, -0.18318651616573334], "agreement_weight": 0.046800754964351654, "orthogonal_weight": 8.751544952392578}, {"head": 4, "memory_to_active": 0.2245965301990509, "contribution_to_active": 0.4338831901550293, "mix_mean": 0.581873893737793, "mix_low": 0.006595611572265625, "mix_high": 0.0015916824340820312, "read_cosine": 0.08056916296482086, "beta_mean": 0.014423244632780552, "nonfinite": 0.0, "weighted_memory_to_active": 0.12225566804409027, "mix_std": 0.2726080119609833, "beta_std": 0.016912169754505157, "beta_low": 0.54193115234375, "beta_high": 0.0, "orthogonal_to_active": 0.11801126599311829, "evidence_weights": [-0.24721543490886688, -0.2373715043067932, 1.9399336576461792], "agreement_weight": -0.20568662881851196, "orthogonal_weight": -0.981793999671936}, {"head": 5, "memory_to_active": 0.0955839604139328, "contribution_to_active": 0.40753912925720215, "mix_mean": 0.5567376017570496, "mix_low": 0.002529144287109375, "mix_high": 0.004963874816894531, "read_cosine": 0.5719148516654968, "beta_mean": 0.008297249674797058, "nonfinite": 0.0, "weighted_memory_to_active": 0.048766929656267166, "mix_std": 0.2604988217353821, "beta_std": 0.01209927722811699, "beta_low": 0.75714111328125, "beta_high": 0.0, "orthogonal_to_active": 0.02961343713104725, "evidence_weights": [-0.21434178948402405, -0.3126834034919739, 0.6535224914550781], "agreement_weight": 0.08280625939369202, "orthogonal_weight": -0.7927694320678711}]}, {"layer": 12, "memory_to_active": 0.25441208481788635, "contribution_to_active": 0.5756714344024658, "mix_mean": 0.5318176746368408, "mix_low": 0.009447574615478516, "mix_high": 0.005647977814078331, "read_cosine": 0.21825724840164185, "beta_mean": 0.014389175921678543, "nonfinite": 0.0, "weighted_memory_to_active": 0.13484343886375427, "mix_std": 0.26069119572639465, "beta_std": 0.015950774773955345, "beta_low": 0.8258260488510132, "beta_high": 0.0, "orthogonal_to_active": 0.07222101092338562, "output_influence": 0.48911699652671814, "residual_influence": 0.13636045157909393, "state_active": 0.0331730954349041, "state_memory": 0.008874085731804371, "state_peak": 55.32060623168945, "growth_active": 1.127022624015808, "growth_memory": 1.205636978149414, "heads": [{"head": 0, "memory_to_active": 0.0487917885184288, "contribution_to_active": 0.5845497846603394, "mix_mean": 0.5612736344337463, "mix_low": 0.0007200241088867188, "mix_high": 0.003994941711425781, "read_cosine": 0.29724884033203125, "beta_mean": 0.0008135994430631399, "nonfinite": 0.0, "weighted_memory_to_active": 0.026036493480205536, "mix_std": 0.2679866850376129, "beta_std": 0.001539462711662054, "beta_low": 0.9952392578125, "beta_high": 0.0, "orthogonal_to_active": 0.0236502718180418, "evidence_weights": [-0.20792008936405182, -0.27669286727905273, 0.10596278309822083], "agreement_weight": 0.10853908210992813, "orthogonal_weight": -0.9824385046958923}, {"head": 1, "memory_to_active": 0.10651621222496033, "contribution_to_active": 0.5400288105010986, "mix_mean": 0.5550720691680908, "mix_low": 0.0005865097045898438, "mix_high": 0.0017719268798828125, "read_cosine": 0.3687572181224823, "beta_mean": 0.0007768018404021859, "nonfinite": 0.0, "weighted_memory_to_active": 0.06667323410511017, "mix_std": 0.2660257816314697, "beta_std": 0.010413508862257004, "beta_low": 0.995361328125, "beta_high": 0.0, "orthogonal_to_active": 0.057228341698646545, "evidence_weights": [-0.20731617510318756, -0.2734290659427643, 1.0350830554962158], "agreement_weight": 0.20577889680862427, "orthogonal_weight": -0.9299702644348145}, {"head": 2, "memory_to_active": 0.6047530174255371, "contribution_to_active": 0.461885005235672, "mix_mean": 0.3030974268913269, "mix_low": 0.03138923645019531, "mix_high": 3.24249267578125e-05, "read_cosine": 0.4915737807750702, "beta_mean": 0.08283772319555283, "nonfinite": 0.0, "weighted_memory_to_active": 0.23005566000938416, "mix_std": 0.23290428519248962, "beta_std": 0.06187748908996582, "beta_low": 0.002197265625, "beta_high": 0.0, "orthogonal_to_active": 0.24343140423297882, "evidence_weights": [-0.22661978006362915, -0.05614800006151199, -0.004303965251892805], "agreement_weight": 0.024732282385230064, "orthogonal_weight": 1.3245031833648682}, {"head": 3, "memory_to_active": 0.028955003246665, "contribution_to_active": 0.7452402114868164, "mix_mean": 0.7094030380249023, "mix_low": 0.000179290771484375, "mix_high": 0.017142295837402344, "read_cosine": 0.07852868735790253, "beta_mean": 0.0003437297709751874, "nonfinite": 0.0, "weighted_memory_to_active": 0.021947162225842476, "mix_std": 0.23109638690948486, "beta_std": 0.005080738570541143, "beta_low": 0.995361328125, "beta_high": 0.0, "orthogonal_to_active": 0.0209943987429142, "evidence_weights": [-0.28955191373825073, -0.339394748210907, 1.5413483381271362], "agreement_weight": 0.4794224202632904, "orthogonal_weight": -0.9740562438964844}, {"head": 4, "memory_to_active": 0.6492657661437988, "contribution_to_active": 0.6241781711578369, "mix_mean": 0.4277803301811218, "mix_low": 0.01713848114013672, "mix_high": 0.0044956207275390625, "read_cosine": -0.13149482011795044, "beta_mean": 0.0010844068601727486, "nonfinite": 0.0, "weighted_memory_to_active": 0.4053207337856293, "mix_std": 0.28948885202407837, "beta_std": 0.011637250892817974, "beta_low": 0.976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.04251330345869064, "evidence_weights": [-0.1494954526424408, -0.21883581578731537, -0.06130515784025192], "agreement_weight": 0.0018406982999294996, "orthogonal_weight": -0.10949863493442535}, {"head": 5, "memory_to_active": 0.08819052577018738, "contribution_to_active": 0.49814659357070923, "mix_mean": 0.634279191493988, "mix_low": 0.006671905517578125, "mix_high": 0.006450653076171875, "read_cosine": 0.20492970943450928, "beta_mean": 0.0004787938669323921, "nonfinite": 0.0, "weighted_memory_to_active": 0.05902734771370888, "mix_std": 0.27664509415626526, "beta_std": 0.0051561943255364895, "beta_low": 0.989990234375, "beta_high": 0.0, "orthogonal_to_active": 0.04550836607813835, "evidence_weights": [-0.23678988218307495, -0.31802257895469666, 1.3280744552612305], "agreement_weight": 0.3849634528160095, "orthogonal_weight": -0.8247609734535217}]}, {"layer": 13, "memory_to_active": 0.12222342193126678, "contribution_to_active": 0.5329710245132446, "mix_mean": 0.5331811308860779, "mix_low": 0.005132993217557669, "mix_high": 0.008516470901668072, "read_cosine": 0.4066888093948364, "beta_mean": 0.014470178633928299, "nonfinite": 0.0, "weighted_memory_to_active": 0.053962502628564835, "mix_std": 0.2586428225040436, "beta_std": 0.01773453876376152, "beta_low": 0.8067525625228882, "beta_high": 0.0, "orthogonal_to_active": 0.05474364012479782, "output_influence": 0.5130822062492371, "residual_influence": 0.17104408144950867, "state_active": 0.11666832864284515, "state_memory": 0.042010433971881866, "state_peak": 98.5899658203125, "growth_active": 0.9269719123840332, "growth_memory": 1.0327955484390259, "heads": [{"head": 0, "memory_to_active": 0.018329240381717682, "contribution_to_active": 0.5822787880897522, "mix_mean": 0.6036217212677002, "mix_low": 0.0004749298095703125, "mix_high": 0.003972053527832031, "read_cosine": 0.4230247139930725, "beta_mean": 6.466203194577247e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011455286294221878, "mix_std": 0.2689618766307831, "beta_std": 0.00014670453674625605, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011604578234255314, "evidence_weights": [-0.19819052517414093, -0.26771029829978943, 0.7595051527023315], "agreement_weight": 0.14772853255271912, "orthogonal_weight": -1.1253339052200317}, {"head": 1, "memory_to_active": 0.010366877540946007, "contribution_to_active": 0.5963885188102722, "mix_mean": 0.5641645789146423, "mix_low": 0.0018606185913085938, "mix_high": 0.0038127899169921875, "read_cosine": 0.22411920130252838, "beta_mean": 4.019935295218602e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005822763778269291, "mix_std": 0.2778439521789551, "beta_std": 7.07971048541367e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007707771845161915, "evidence_weights": [-0.18516439199447632, -0.2667441964149475, 0.6382368206977844], "agreement_weight": 0.4015737473964691, "orthogonal_weight": -1.375344157218933}, {"head": 2, "memory_to_active": 0.0763845443725586, "contribution_to_active": 0.4764954447746277, "mix_mean": 0.38490474224090576, "mix_low": 0.0014896392822265625, "mix_high": 0.00018787384033203125, "read_cosine": 0.5884910821914673, "beta_mean": 0.011493945494294167, "nonfinite": 0.0, "weighted_memory_to_active": 0.03532256931066513, "mix_std": 0.23361122608184814, "beta_std": 0.03934841603040695, "beta_low": 0.84716796875, "beta_high": 0.0, "orthogonal_to_active": 0.02070011757314205, "evidence_weights": [-0.1857639104127884, -0.20500904321670532, 0.4127293825149536], "agreement_weight": 0.008526311255991459, "orthogonal_weight": -0.8445292115211487}, {"head": 3, "memory_to_active": 0.1669095903635025, "contribution_to_active": 0.5889265537261963, "mix_mean": 0.6625952124595642, "mix_low": 0.01892375946044922, "mix_high": 0.030149459838867188, "read_cosine": 0.006541770417243242, "beta_mean": 0.0009467586642131209, "nonfinite": 0.0, "weighted_memory_to_active": 0.1261579990386963, "mix_std": 0.28702032566070557, "beta_std": 0.01123224850744009, "beta_low": 0.99371337890625, "beta_high": 0.0, "orthogonal_to_active": 0.11114157736301422, "evidence_weights": [-0.3199478089809418, -0.24856851994991302, 1.6516221761703491], "agreement_weight": 5.462803363800049, "orthogonal_weight": -0.9773157835006714}, {"head": 4, "memory_to_active": 0.4036960303783417, "contribution_to_active": 0.24859854578971863, "mix_mean": 0.32601821422576904, "mix_low": 0.005992889404296875, "mix_high": 1.811981201171875e-05, "read_cosine": 0.8827431797981262, "beta_mean": 0.07400411367416382, "nonfinite": 0.0, "weighted_memory_to_active": 0.10384303331375122, "mix_std": 0.2137981355190277, "beta_std": 0.05277358368039131, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.142457515001297, "evidence_weights": [-0.09126895666122437, -0.2325303852558136, 0.4697670638561249], "agreement_weight": 0.006080486346036196, "orthogonal_weight": 2.220958948135376}, {"head": 5, "memory_to_active": 0.057654231786727905, "contribution_to_active": 0.7051383256912231, "mix_mean": 0.6577820777893066, "mix_low": 0.002056121826171875, "mix_high": 0.012958526611328125, "read_cosine": 0.31521284580230713, "beta_mean": 0.0002713885041885078, "nonfinite": 0.0, "weighted_memory_to_active": 0.041173361241817474, "mix_std": 0.2706214487552643, "beta_std": 0.0028354802634567022, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.034850288182497025, "evidence_weights": [-0.2227421998977661, -0.3051428198814392, 0.8824245929718018], "agreement_weight": 0.16321688890457153, "orthogonal_weight": -0.9333937168121338}]}, {"layer": 14, "memory_to_active": 0.23819243907928467, "contribution_to_active": 0.41753461956977844, "mix_mean": 0.42033952474594116, "mix_low": 0.026387693360447884, "mix_high": 0.0057080588303506374, "read_cosine": 0.6111112833023071, "beta_mean": 0.060996539890766144, "nonfinite": 0.0, "weighted_memory_to_active": 0.09759068489074707, "mix_std": 0.2435559630393982, "beta_std": 0.05870705097913742, "beta_low": 0.4899088740348816, "beta_high": 0.0, "orthogonal_to_active": 0.040930818766355515, "output_influence": 0.34870824217796326, "residual_influence": 0.134578675031662, "state_active": 0.05420824512839317, "state_memory": 0.016181854531168938, "state_peak": 49.034629821777344, "growth_active": 1.0484330654144287, "growth_memory": 1.2515270709991455, "heads": [{"head": 0, "memory_to_active": 0.4031566381454468, "contribution_to_active": 0.1900704950094223, "mix_mean": 0.17694687843322754, "mix_low": 0.019263267517089844, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8133095502853394, "beta_mean": 0.07341991364955902, "nonfinite": 0.0, "weighted_memory_to_active": 0.08591882139444351, "mix_std": 0.17208437621593475, "beta_std": 0.06395374238491058, "beta_low": 0.007080078125, "beta_high": 0.0, "orthogonal_to_active": 0.08256708830595016, "evidence_weights": [-0.11674550175666809, -0.03961522877216339, 0.44391560554504395], "agreement_weight": 0.022991759702563286, "orthogonal_weight": 1.644122838973999}, {"head": 1, "memory_to_active": 0.4927327036857605, "contribution_to_active": 0.2930658757686615, "mix_mean": 0.2162371128797531, "mix_low": 0.12455940246582031, "mix_high": 9.5367431640625e-06, "read_cosine": 0.856736421585083, "beta_mean": 0.24315619468688965, "nonfinite": 0.0, "weighted_memory_to_active": 0.21740321815013885, "mix_std": 0.2477072924375534, "beta_std": 0.23873776197433472, "beta_low": 0.10638427734375, "beta_high": 0.0, "orthogonal_to_active": 0.013684914447367191, "evidence_weights": [-0.007585715968161821, -0.19992847740650177, -0.09994896501302719], "agreement_weight": 0.05505194887518883, "orthogonal_weight": 0.16941122710704803}, {"head": 2, "memory_to_active": 0.1157413050532341, "contribution_to_active": 0.5757808089256287, "mix_mean": 0.587796688079834, "mix_low": 0.0029020309448242188, "mix_high": 0.0041599273681640625, "read_cosine": 0.40040600299835205, "beta_mean": 0.001856196322478354, "nonfinite": 0.0, "weighted_memory_to_active": 0.07601208239793777, "mix_std": 0.2659758925437927, "beta_std": 0.0025712803471833467, "beta_low": 0.98333740234375, "beta_high": 0.0, "orthogonal_to_active": 0.05975484475493431, "evidence_weights": [-0.14434587955474854, -0.318830668926239, 0.9119396805763245], "agreement_weight": 0.08441023528575897, "orthogonal_weight": -0.8879832029342651}, {"head": 3, "memory_to_active": 0.05388019233942032, "contribution_to_active": 0.615731418132782, "mix_mean": 0.6652480363845825, "mix_low": 0.004195213317871094, "mix_high": 0.019887924194335938, "read_cosine": -0.021979473531246185, "beta_mean": 0.006476328708231449, "nonfinite": 0.0, "weighted_memory_to_active": 0.03414858132600784, "mix_std": 0.2587522864341736, "beta_std": 0.008419695310294628, "beta_low": 0.80438232421875, "beta_high": 0.0, "orthogonal_to_active": 0.02520047500729561, "evidence_weights": [-0.34785041213035583, -0.31037887930870056, 0.28479239344596863], "agreement_weight": 0.07078313827514648, "orthogonal_weight": -0.8190047144889832}, {"head": 4, "memory_to_active": 0.15242978930473328, "contribution_to_active": 0.4951072037220001, "mix_mean": 0.5107307434082031, "mix_low": 0.003353118896484375, "mix_high": 0.008146286010742188, "read_cosine": 0.8750371336936951, "beta_mean": 0.03941410034894943, "nonfinite": 0.0, "weighted_memory_to_active": 0.08860628306865692, "mix_std": 0.26023226976394653, "beta_std": 0.02785327285528183, "beta_low": 0.06414794921875, "beta_high": 0.0, "orthogonal_to_active": 0.02467913180589676, "evidence_weights": [-0.21908508241176605, -0.2471417486667633, 0.43060511350631714], "agreement_weight": 0.13215073943138123, "orthogonal_weight": -0.5980873107910156}, {"head": 5, "memory_to_active": 0.21121396124362946, "contribution_to_active": 0.33545178174972534, "mix_mean": 0.3650776147842407, "mix_low": 0.0040531158447265625, "mix_high": 0.0020360946655273438, "read_cosine": 0.7431577444076538, "beta_mean": 0.0016564715187996626, "nonfinite": 0.0, "weighted_memory_to_active": 0.08345513045787811, "mix_std": 0.25658366084098816, "beta_std": 0.010706533677875996, "beta_low": 0.97412109375, "beta_high": 0.0, "orthogonal_to_active": 0.03969844430685043, "evidence_weights": [-0.1743321418762207, -0.17036226391792297, 0.2957662045955658], "agreement_weight": 0.0777411013841629, "orthogonal_weight": -0.7365065813064575}]}, {"layer": 15, "memory_to_active": 0.0569889098405838, "contribution_to_active": 0.4883040487766266, "mix_mean": 0.4103318750858307, "mix_low": 0.009275118820369244, "mix_high": 0.002628167625516653, "read_cosine": 0.6623317003250122, "beta_mean": 0.04298652708530426, "nonfinite": 0.0, "weighted_memory_to_active": 0.027401622384786606, "mix_std": 0.2828512489795685, "beta_std": 0.04694214463233948, "beta_low": 0.6992086172103882, "beta_high": 0.0, "orthogonal_to_active": 0.007840135134756565, "output_influence": 0.30109065771102905, "residual_influence": 0.16846418380737305, "state_active": 0.14279267191886902, "state_memory": 0.010713575407862663, "state_peak": 32.89379119873047, "growth_active": 1.1713635921478271, "growth_memory": 1.3853710889816284, "heads": [{"head": 0, "memory_to_active": 0.11390060186386108, "contribution_to_active": 0.47802481055259705, "mix_mean": 0.3785427212715149, "mix_low": 0.008556365966796875, "mix_high": 0.0052509307861328125, "read_cosine": 0.6327338814735413, "beta_mean": 0.04300890117883682, "nonfinite": 0.0, "weighted_memory_to_active": 0.05425469949841499, "mix_std": 0.2828220725059509, "beta_std": 0.030320128425955772, "beta_low": 0.05718994140625, "beta_high": 0.0, "orthogonal_to_active": 0.027939777821302414, "evidence_weights": [-0.19036880135536194, -0.19327116012573242, 0.4322426915168762], "agreement_weight": 0.09001190960407257, "orthogonal_weight": -0.7464104890823364}, {"head": 1, "memory_to_active": 0.1183938980102539, "contribution_to_active": 0.4167121648788452, "mix_mean": 0.43179357051849365, "mix_low": 0.0015125274658203125, "mix_high": 0.0055179595947265625, "read_cosine": 0.9172627925872803, "beta_mean": 0.0006609464762732387, "nonfinite": 0.0, "weighted_memory_to_active": 0.04901862516999245, "mix_std": 0.28443169593811035, "beta_std": 0.0022104179952293634, "beta_low": 0.99432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.0011572293005883694, "evidence_weights": [-0.14814883470535278, -0.15722310543060303, 0.18350264430046082], "agreement_weight": 0.12500743567943573, "orthogonal_weight": 0.0453527458012104}, {"head": 2, "memory_to_active": 0.005188828334212303, "contribution_to_active": 0.4798705577850342, "mix_mean": 0.3326810598373413, "mix_low": 0.016518592834472656, "mix_high": 4.57763671875e-05, "read_cosine": 0.7920669913291931, "beta_mean": 7.247306348290294e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002632566262036562, "mix_std": 0.2658906579017639, "beta_std": 0.00019844960479531437, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0007038943003863096, "evidence_weights": [-0.20622792840003967, -0.26785752177238464, 0.1456642746925354], "agreement_weight": 0.008781195618212223, "orthogonal_weight": 0.7177206873893738}, {"head": 3, "memory_to_active": 0.0017310322728008032, "contribution_to_active": 0.39028042554855347, "mix_mean": 0.39644259214401245, "mix_low": 0.006056785583496094, "mix_high": 0.00040912628173828125, "read_cosine": 0.5809623599052429, "beta_mean": 1.0001176633522846e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.000579184852540493, "mix_std": 0.28582867980003357, "beta_std": 2.4045311874942854e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00047634041402488947, "evidence_weights": [-0.2612534165382385, -0.24846142530441284, 0.1939510703086853], "agreement_weight": 0.03886719420552254, "orthogonal_weight": -0.7993730306625366}, {"head": 4, "memory_to_active": 0.07716092467308044, "contribution_to_active": 0.6404929161071777, "mix_mean": 0.36117202043533325, "mix_low": 0.012797355651855469, "mix_high": 0.0009717941284179688, "read_cosine": 0.7149097919464111, "beta_mean": 0.21240244805812836, "nonfinite": 0.0, "weighted_memory_to_active": 0.04108604043722153, "mix_std": 0.30146288871765137, "beta_std": 0.23621809482574463, "beta_low": 0.171142578125, "beta_high": 0.0, "orthogonal_to_active": 0.006598378531634808, "evidence_weights": [-0.21241919696331024, -0.25487637519836426, -0.014063412323594093], "agreement_weight": 0.013166517950594425, "orthogonal_weight": 0.3304789066314697}, {"head": 5, "memory_to_active": 0.02555815503001213, "contribution_to_active": 0.5244433283805847, "mix_mean": 0.5613592267036438, "mix_low": 0.010209083557128906, "mix_high": 0.0035734176635742188, "read_cosine": 0.33605414628982544, "beta_mean": 0.0017644015606492758, "nonfinite": 0.0, "weighted_memory_to_active": 0.016838623210787773, "mix_std": 0.2766714096069336, "beta_std": 0.012681735679507256, "beta_low": 0.97265625, "beta_high": 0.0, "orthogonal_to_active": 0.010165192186832428, "evidence_weights": [-0.2904948890209198, -0.30541521310806274, 0.6924915313720703], "agreement_weight": 0.2932029068470001, "orthogonal_weight": 0.7950572967529297}]}]}, {"step": 52000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9290832870001395, "memory_to_active": 0.7971161007881165, "contribution_to_active": 0.5520987510681152, "mix_mean": 0.4071801006793976, "mix_low": 0.13402456045150757, "mix_high": 0.010819465853273869, "read_cosine": 0.3510204255580902, "beta_mean": 0.07057031989097595, "nonfinite": 0.0, "weighted_memory_to_active": 0.20527131855487823, "mix_std": 0.2219206690788269, "beta_std": 0.061252035200595856, "beta_low": 0.5574480891227722, "beta_high": 2.6702880859375e-05, "orthogonal_to_active": 0.05722516030073166, "samples": 4, "scored_tokens": 899, "normal_loss": 3.1256914138793945, "normal_token_accuracy": 0.442714124917984, "state_active": 0.06356494873762131, "state_memory": 0.017699789255857468, "state_peak": 128.8436737060547, "growth_active": 1.1274244785308838, "growth_memory": 1.3424224853515625, "output_influence": 0.41766107082366943, "residual_influence": 0.14059315621852875, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2727837562561035, "no_memory_loss_delta": 0.5596213340759277, "scrambled_loss_delta": 0.12379226088523865, "memory_only_loss_delta": 3.6054065227508545, "no_evidence_loss_delta": 0.35466527938842773, "no_agreement_loss_delta": 0.02615056000649929, "no_orthogonal_loss_delta": -0.010780450887978077}, {"index": 94, "tokens": 256, "normal_loss": 3.203094005584717, "no_memory_loss_delta": 0.5778753757476807, "scrambled_loss_delta": 0.3467807173728943, "memory_only_loss_delta": 3.6968345642089844, "no_evidence_loss_delta": 0.26128536462783813, "no_agreement_loss_delta": 0.040559716522693634, "no_orthogonal_loss_delta": 0.04640434682369232}, {"index": 188, "tokens": 256, "normal_loss": 3.455014705657959, "no_memory_loss_delta": 0.6909564137458801, "scrambled_loss_delta": 0.15537743270397186, "memory_only_loss_delta": 3.2034032344818115, "no_evidence_loss_delta": 0.3885909914970398, "no_agreement_loss_delta": 0.0513426698744297, "no_orthogonal_loss_delta": 0.04714733734726906}, {"index": 283, "tokens": 131, "normal_loss": 2.043419599533081, "no_memory_loss_delta": 0.28462374210357666, "scrambled_loss_delta": 0.22913067042827606, "memory_only_loss_delta": 2.63236403465271, "no_evidence_loss_delta": 0.14121103286743164, "no_agreement_loss_delta": 0.010390154086053371, "no_orthogonal_loss_delta": 0.008558559231460094}], "agreement_weight_abs_mean": 0.2076931595802307, "agreement_weight_abs_max": 5.54857873916626, "agreement_weight_mean": 0.18111921846866608, "orthogonal_weight_abs_mean": 0.8457328677177429, "orthogonal_weight_abs_max": 8.891709327697754, "orthogonal_weight_mean": -0.11885372549295425, "no_memory_loss": 3.687837600708008, "no_memory_loss_delta": 0.5621465444564819, "no_memory_token_accuracy_gain": 0.06674082577228546, "no_memory_token_win_fraction": 0.6796440482139587, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 43.00157467751626, "scrambled_loss": 3.3373258113861084, "scrambled_loss_delta": 0.21163451671600342, "scrambled_token_accuracy_gain": 0.015572858974337578, "scrambled_token_win_fraction": 0.628476083278656, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.07395853226576, "memory_only_loss": 6.500868797302246, "memory_only_loss_delta": 3.3751776218414307, "memory_only_token_accuracy_gain": 0.2769744098186493, "memory_only_token_win_fraction": 0.8998887538909912, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.57879590337681, "no_evidence_loss": 3.4323222637176514, "no_evidence_loss_delta": 0.30663105845451355, "no_evidence_token_accuracy_gain": 0.03337041288614273, "no_evidence_token_win_fraction": 0.6229143738746643, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 26.407793694935783, "no_agreement_loss": 3.1608221530914307, "no_agreement_loss_delta": 0.03513088449835777, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.5250278115272522, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.4520958252539353, "no_orthogonal_loss": 3.150508403778076, "no_orthogonal_loss_delta": 0.02481713704764843, "no_orthogonal_token_accuracy_gain": 0.0055617354810237885, "no_orthogonal_token_win_fraction": 0.5817574858665466, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.4511723614076595, "short_no_memory_loss_delta": 0.5217980742454529, "long_context_memory_gain": 0.04034847021102905, "normal_context_loss_gain": 0.15263044834136963, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.48305606842041, "contribution_to_active": 1.1659257411956787, "mix_mean": 0.10453544557094574, "mix_low": 0.5306897163391113, "mix_high": 0.0053653717041015625, "read_cosine": 0.4288938343524933, "beta_mean": 0.21606816351413727, "nonfinite": 0.0, "weighted_memory_to_active": 1.2026941776275635, "mix_std": 0.0910421684384346, "beta_std": 0.13779044151306152, "beta_low": 0.0518086776137352, "beta_high": 0.0, "orthogonal_to_active": 0.04185297340154648, "output_influence": 0.21415995061397552, "residual_influence": 0.2524622678756714, "state_active": 0.06420385837554932, "state_memory": 0.04449940100312233, "state_peak": 44.68097686767578, "growth_active": 1.23541259765625, "growth_memory": 1.288525938987732, "heads": [{"head": 0, "memory_to_active": 2.5910680294036865, "contribution_to_active": 0.4734508693218231, "mix_mean": 0.02143161930143833, "mix_low": 0.6082086563110352, "mix_high": 0.0, "read_cosine": 0.22588828206062317, "beta_mean": 0.20402324199676514, "nonfinite": 0.0, "weighted_memory_to_active": 0.5920530557632446, "mix_std": 0.0650084987282753, "beta_std": 0.09700183570384979, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05656778812408447, "evidence_weights": [-0.009982500225305557, -0.054721880704164505, -0.06133485212922096], "agreement_weight": -0.18995222449302673, "orthogonal_weight": -0.10818180441856384}, {"head": 1, "memory_to_active": 6.397928237915039, "contribution_to_active": 2.744924545288086, "mix_mean": 0.019503770396113396, "mix_low": 0.6150369644165039, "mix_high": 0.0, "read_cosine": 0.45058709383010864, "beta_mean": 0.4187583923339844, "nonfinite": 0.0, "weighted_memory_to_active": 3.0374608039855957, "mix_std": 0.05501703545451164, "beta_std": 0.1961674839258194, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007938692579045892, "evidence_weights": [0.01998773030936718, -0.10184437036514282, 0.5803492665290833], "agreement_weight": 0.027561059221625328, "orthogonal_weight": -0.0005683572962880135}, {"head": 2, "memory_to_active": 2.5074596405029297, "contribution_to_active": 0.4619564414024353, "mix_mean": 0.015148564241826534, "mix_low": 0.6621007919311523, "mix_high": 0.0, "read_cosine": 0.3363933563232422, "beta_mean": 0.2333049476146698, "nonfinite": 0.0, "weighted_memory_to_active": 0.5490365624427795, "mix_std": 0.033389925956726074, "beta_std": 0.1901993304491043, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04420871660113335, "evidence_weights": [0.06004505604505539, 0.05352173000574112, 0.13719923794269562], "agreement_weight": 0.01411953754723072, "orthogonal_weight": -0.20520584285259247}, {"head": 3, "memory_to_active": 7.396502494812012, "contribution_to_active": 2.767268419265747, "mix_mean": 0.030950220301747322, "mix_low": 0.5098953247070312, "mix_high": 0.0, "read_cosine": 0.2876642346382141, "beta_mean": 0.33966881036758423, "nonfinite": 0.0, "weighted_memory_to_active": 2.917665481567383, "mix_std": 0.0742553099989891, "beta_std": 0.2519126236438751, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.12690319120883942, "evidence_weights": [-0.04248294234275818, -0.04337579384446144, -0.15152105689048767], "agreement_weight": -0.016355233266949654, "orthogonal_weight": -0.04962967336177826}, {"head": 4, "memory_to_active": 1.888758659362793, "contribution_to_active": 0.0940549373626709, "mix_mean": 0.009961673989892006, "mix_low": 0.773838996887207, "mix_high": 0.0, "read_cosine": 0.3298206925392151, "beta_mean": 0.07641208916902542, "nonfinite": 0.0, "weighted_memory_to_active": 0.05981733649969101, "mix_std": 0.02981480024755001, "beta_std": 0.07239416241645813, "beta_low": 0.0601806640625, "beta_high": 0.0, "orthogonal_to_active": 0.005208061542361975, "evidence_weights": [0.14497196674346924, 0.10484939813613892, -0.20837676525115967], "agreement_weight": -0.1306600719690323, "orthogonal_weight": 0.07234752923250198}, {"head": 5, "memory_to_active": 0.1166173666715622, "contribution_to_active": 0.4538993239402771, "mix_mean": 0.5302168726921082, "mix_low": 0.015057563781738281, "mix_high": 0.032192230224609375, "read_cosine": 0.9430091977119446, "beta_mean": 0.024241436272859573, "nonfinite": 0.0, "weighted_memory_to_active": 0.060131583362817764, "mix_std": 0.28876739740371704, "beta_std": 0.019067145884037018, "beta_low": 0.25067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.017436185851693153, "evidence_weights": [-0.17832060158252716, -0.2151166796684265, 0.37580209970474243], "agreement_weight": 0.2887403070926666, "orthogonal_weight": -0.6879773736000061}]}, {"layer": 1, "memory_to_active": 2.2235565185546875, "contribution_to_active": 0.4624229371547699, "mix_mean": 0.21185752749443054, "mix_low": 0.3724042773246765, "mix_high": 0.0011048316955566406, "read_cosine": 0.2944280505180359, "beta_mean": 0.1720338761806488, "nonfinite": 0.0, "weighted_memory_to_active": 0.30475693941116333, "mix_std": 0.11616303026676178, "beta_std": 0.121048703789711, "beta_low": 0.3383992612361908, "beta_high": 0.00014241537428461015, "orthogonal_to_active": 0.04539372771978378, "output_influence": 0.2392595261335373, "residual_influence": 0.06269785761833191, "state_active": 0.028442122042179108, "state_memory": 0.03547225147485733, "state_peak": 34.996665954589844, "growth_active": 1.1982028484344482, "growth_memory": 1.4829678535461426, "heads": [{"head": 0, "memory_to_active": 0.05299269035458565, "contribution_to_active": 0.48608583211898804, "mix_mean": 0.512665867805481, "mix_low": 0.005957603454589844, "mix_high": 0.001708984375, "read_cosine": 0.020507263019680977, "beta_mean": 0.0003375723899807781, "nonfinite": 0.0, "weighted_memory_to_active": 0.032768286764621735, "mix_std": 0.28316837549209595, "beta_std": 0.0006031546508893371, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.030527979135513306, "evidence_weights": [-0.2069467157125473, -0.2663741111755371, 0.3212377429008484], "agreement_weight": 0.1779416799545288, "orthogonal_weight": -0.98906010389328}, {"head": 1, "memory_to_active": 2.8787291049957275, "contribution_to_active": 0.3133030831813812, "mix_mean": 0.024052225053310394, "mix_low": 0.4673166275024414, "mix_high": 9.5367431640625e-07, "read_cosine": 0.49735909700393677, "beta_mean": 0.2758353352546692, "nonfinite": 0.0, "weighted_memory_to_active": 0.3566269874572754, "mix_std": 0.04882228374481201, "beta_std": 0.18443825840950012, "beta_low": 0.001953125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.02086137980222702, "evidence_weights": [0.014943222515285015, -0.05155208706855774, 0.05846058577299118], "agreement_weight": -0.0009292092290706933, "orthogonal_weight": 0.17499616742134094}, {"head": 2, "memory_to_active": 0.07194524258375168, "contribution_to_active": 0.5601279735565186, "mix_mean": 0.6738536953926086, "mix_low": 0.0005044937133789062, "mix_high": 0.004918098449707031, "read_cosine": 0.4452781677246094, "beta_mean": 0.0033604344353079796, "nonfinite": 0.0, "weighted_memory_to_active": 0.05073945224285126, "mix_std": 0.23056340217590332, "beta_std": 0.005729110445827246, "beta_low": 0.94097900390625, "beta_high": 0.0, "orthogonal_to_active": 0.03345198556780815, "evidence_weights": [-0.2887810468673706, -0.36941710114479065, 0.6482493281364441], "agreement_weight": 0.03900386765599251, "orthogonal_weight": -0.7749947905540466}, {"head": 3, "memory_to_active": 2.84317684173584, "contribution_to_active": 0.1675788313150406, "mix_mean": 0.02098550647497177, "mix_low": 0.6091766357421875, "mix_high": 0.0, "read_cosine": 0.20987701416015625, "beta_mean": 0.31677621603012085, "nonfinite": 0.0, "weighted_memory_to_active": 0.2452544867992401, "mix_std": 0.05835139378905296, "beta_std": 0.24552147090435028, "beta_low": 0.05572509765625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.08638712763786316, "evidence_weights": [0.043723948299884796, -0.012305899523198605, 0.0035061296075582504], "agreement_weight": -0.02891283482313156, "orthogonal_weight": -0.4796206057071686}, {"head": 4, "memory_to_active": 3.6254827976226807, "contribution_to_active": 1.171707272529602, "mix_mean": 0.0323060005903244, "mix_low": 0.32989025115966797, "mix_high": 9.5367431640625e-07, "read_cosine": 0.31238001585006714, "beta_mean": 0.1735403835773468, "nonfinite": 0.0, "weighted_memory_to_active": 1.0897235870361328, "mix_std": 0.05663181468844414, "beta_std": 0.11009599268436432, "beta_low": 0.00958251953125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.08673390746116638, "evidence_weights": [-0.028195464983582497, -0.0008673674892634153, 0.07731509208679199], "agreement_weight": -0.01946365088224411, "orthogonal_weight": 0.1065712571144104}, {"head": 5, "memory_to_active": 3.869011878967285, "contribution_to_active": 0.07573463022708893, "mix_mean": 0.007281898986548185, "mix_low": 0.8215799331665039, "mix_high": 0.0, "read_cosine": 0.28116685152053833, "beta_mean": 0.26235324144363403, "nonfinite": 0.0, "weighted_memory_to_active": 0.053428761661052704, "mix_std": 0.01944083906710148, "beta_std": 0.17990422248840332, "beta_low": 0.0225830078125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.014399946667253971, "evidence_weights": [0.15653477609157562, 0.07761895656585693, 0.12312574684619904], "agreement_weight": -0.13398052752017975, "orthogonal_weight": 0.2653573453426361}]}, {"layer": 2, "memory_to_active": 1.0337457656860352, "contribution_to_active": 0.3634248971939087, "mix_mean": 0.3896826505661011, "mix_low": 0.2477935254573822, "mix_high": 0.004034201614558697, "read_cosine": 0.3078245520591736, "beta_mean": 0.08805061876773834, "nonfinite": 0.0, "weighted_memory_to_active": 0.09325510263442993, "mix_std": 0.19213765859603882, "beta_std": 0.07084283232688904, "beta_low": 0.5648295283317566, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.024002239108085632, "output_influence": 0.3782525658607483, "residual_influence": 0.10142707824707031, "state_active": 0.03386592119932175, "state_memory": 0.016222376376390457, "state_peak": 25.03512191772461, "growth_active": 0.9686323404312134, "growth_memory": 1.5904176235198975, "heads": [{"head": 0, "memory_to_active": 0.020603954792022705, "contribution_to_active": 0.341248095035553, "mix_mean": 0.6069815754890442, "mix_low": 0.0009613037109375, "mix_high": 0.0077114105224609375, "read_cosine": 0.1029878705739975, "beta_mean": 0.00011706328223226592, "nonfinite": 0.0, "weighted_memory_to_active": 0.012686770409345627, "mix_std": 0.2778029441833496, "beta_std": 0.00025021861074492335, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01291314885020256, "evidence_weights": [-0.36407986283302307, -0.2126106321811676, 0.15021799504756927], "agreement_weight": 0.1328938901424408, "orthogonal_weight": -1.054369568824768}, {"head": 1, "memory_to_active": 2.9681427478790283, "contribution_to_active": 0.0549829825758934, "mix_mean": 0.0048189908266067505, "mix_low": 0.8997478485107422, "mix_high": 0.0, "read_cosine": 0.15021194517612457, "beta_mean": 0.24308151006698608, "nonfinite": 0.0, "weighted_memory_to_active": 0.04138525202870369, "mix_std": 0.010944384150207043, "beta_std": 0.15680155158042908, "beta_low": 0.0008544921875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.014615416526794434, "evidence_weights": [0.13935299217700958, 0.1254662722349167, 0.5251343846321106], "agreement_weight": 0.03368891030550003, "orthogonal_weight": 0.3761976957321167}, {"head": 2, "memory_to_active": 0.010058869607746601, "contribution_to_active": 0.4454962909221649, "mix_mean": 0.5263542532920837, "mix_low": 0.0013885498046875, "mix_high": 0.0027618408203125, "read_cosine": 0.3861798048019409, "beta_mean": 3.2683783501852304e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004329306073486805, "mix_std": 0.277747243642807, "beta_std": 5.687447264790535e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003505420754663646, "evidence_weights": [-0.2474813610315323, -0.2890709340572357, 0.5657946467399597], "agreement_weight": -0.010120023973286152, "orthogonal_weight": -0.10170891880989075}, {"head": 3, "memory_to_active": 0.14974334836006165, "contribution_to_active": 0.5171682834625244, "mix_mean": 0.5536855459213257, "mix_low": 0.0002689361572265625, "mix_high": 0.006473541259765625, "read_cosine": 0.6245559453964233, "beta_mean": 0.029201343655586243, "nonfinite": 0.0, "weighted_memory_to_active": 0.08976408839225769, "mix_std": 0.26026082038879395, "beta_std": 0.03717944398522377, "beta_low": 0.35198974609375, "beta_high": 0.0, "orthogonal_to_active": 0.06086938828229904, "evidence_weights": [-0.1987188309431076, -0.3124856650829315, 0.8199411034584045], "agreement_weight": 0.05340028181672096, "orthogonal_weight": -0.8917195200920105}, {"head": 4, "memory_to_active": 3.0403478145599365, "contribution_to_active": 0.2988031506538391, "mix_mean": 0.02149251475930214, "mix_low": 0.5832738876342773, "mix_high": 0.0, "read_cosine": 0.20922324061393738, "beta_mean": 0.2558290958404541, "nonfinite": 0.0, "weighted_memory_to_active": 0.4035940170288086, "mix_std": 0.05863092467188835, "beta_std": 0.2307012528181076, "beta_low": 0.0361328125, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.054283469915390015, "evidence_weights": [0.02027442492544651, 0.006190740969032049, 0.00942071434110403], "agreement_weight": 0.017882386222481728, "orthogonal_weight": -0.15577763319015503}, {"head": 5, "memory_to_active": 0.013577153906226158, "contribution_to_active": 0.5228503942489624, "mix_mean": 0.6247628331184387, "mix_low": 0.0011205673217773438, "mix_high": 0.007258415222167969, "read_cosine": 0.3737884759902954, "beta_mean": 4.1944625991163775e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007771151605993509, "mix_std": 0.2674395740032196, "beta_std": 6.759674579370767e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009814579971134663, "evidence_weights": [-0.2788340151309967, -0.2906569540500641, 1.0280835628509521], "agreement_weight": 0.16583794355392456, "orthogonal_weight": -0.152536079287529}]}, {"layer": 3, "memory_to_active": 0.4647955894470215, "contribution_to_active": 0.46737605333328247, "mix_mean": 0.4800121784210205, "mix_low": 0.06026872247457504, "mix_high": 0.010107677429914474, "read_cosine": 0.2060094028711319, "beta_mean": 0.029372474178671837, "nonfinite": 0.0, "weighted_memory_to_active": 0.13454709947109222, "mix_std": 0.2628480792045593, "beta_std": 0.026596836745738983, "beta_low": 0.7544759511947632, "beta_high": 0.0, "orthogonal_to_active": 0.055176835507154465, "output_influence": 0.40160316228866577, "residual_influence": 0.08975693583488464, "state_active": 0.02209492214024067, "state_memory": 0.007422241382300854, "state_peak": 23.92116928100586, "growth_active": 0.9959210753440857, "growth_memory": 1.7058709859848022, "heads": [{"head": 0, "memory_to_active": 0.3999224603176117, "contribution_to_active": 0.5900852084159851, "mix_mean": 0.5834625363349915, "mix_low": 0.0021963119506835938, "mix_high": 0.02386951446533203, "read_cosine": 0.2400258481502533, "beta_mean": 0.011490719392895699, "nonfinite": 0.0, "weighted_memory_to_active": 0.24643458425998688, "mix_std": 0.30746230483055115, "beta_std": 0.014008679427206516, "beta_low": 0.636962890625, "beta_high": 0.0, "orthogonal_to_active": 0.1723077893257141, "evidence_weights": [-0.2153199315071106, -0.24932141602039337, 0.9442428350448608], "agreement_weight": 0.1204083263874054, "orthogonal_weight": -0.7365337014198303}, {"head": 1, "memory_to_active": 0.018337706103920937, "contribution_to_active": 0.4306417405605316, "mix_mean": 0.5517959594726562, "mix_low": 0.004900932312011719, "mix_high": 0.008296012878417969, "read_cosine": 0.18662786483764648, "beta_mean": 0.00010098100028699264, "nonfinite": 0.0, "weighted_memory_to_active": 0.010491260327398777, "mix_std": 0.30271482467651367, "beta_std": 0.00026064086705446243, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008199908770620823, "evidence_weights": [-0.2608276605606079, -0.21352508664131165, 0.6560728549957275], "agreement_weight": 0.4833531677722931, "orthogonal_weight": -0.864378809928894}, {"head": 2, "memory_to_active": 2.080822229385376, "contribution_to_active": 0.3065333962440491, "mix_mean": 0.05176485329866409, "mix_low": 0.3356313705444336, "mix_high": 9.5367431640625e-07, "read_cosine": 0.2050691694021225, "beta_mean": 0.15906068682670593, "nonfinite": 0.0, "weighted_memory_to_active": 0.34764236211776733, "mix_std": 0.10676351189613342, "beta_std": 0.13477155566215515, "beta_low": 0.0142822265625, "beta_high": 0.0, "orthogonal_to_active": 0.0022404477931559086, "evidence_weights": [-0.09391651302576065, -0.0007144251721911132, 0.007996507920324802], "agreement_weight": -0.016642851755023003, "orthogonal_weight": -0.008109251037240028}, {"head": 3, "memory_to_active": 0.025258708745241165, "contribution_to_active": 0.5083534717559814, "mix_mean": 0.5732125043869019, "mix_low": 0.007466316223144531, "mix_high": 0.010007858276367188, "read_cosine": 0.30197858810424805, "beta_mean": 9.791616321308538e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.017518287524580956, "mix_std": 0.2952960133552551, "beta_std": 0.00023548403987661004, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004455734044313431, "evidence_weights": [-0.24085889756679535, -0.22564992308616638, 2.1527483463287354], "agreement_weight": 1.9527380466461182, "orthogonal_weight": -0.3439711034297943}, {"head": 4, "memory_to_active": 0.022252148017287254, "contribution_to_active": 0.3358800411224365, "mix_mean": 0.4191330075263977, "mix_low": 0.008795738220214844, "mix_high": 0.0013561248779296875, "read_cosine": 0.16519729793071747, "beta_mean": 0.00011698272282956168, "nonfinite": 0.0, "weighted_memory_to_active": 0.013623511418700218, "mix_std": 0.29023534059524536, "beta_std": 0.00015371953486464918, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012662762776017189, "evidence_weights": [-0.1888570785522461, -0.1937929093837738, 0.11670433729887009], "agreement_weight": -0.038565583527088165, "orthogonal_weight": -0.9979231953620911}, {"head": 5, "memory_to_active": 0.24218013882637024, "contribution_to_active": 0.6327623724937439, "mix_mean": 0.7007039785385132, "mix_low": 0.0026216506958007812, "mix_high": 0.01711559295654297, "read_cosine": 0.1371576189994812, "beta_mean": 0.005367554258555174, "nonfinite": 0.0, "weighted_memory_to_active": 0.17157255113124847, "mix_std": 0.2746163606643677, "beta_std": 0.010150935500860214, "beta_low": 0.8756103515625, "beta_high": 0.0, "orthogonal_to_active": 0.1311943680047989, "evidence_weights": [-0.2601158916950226, -0.3247877061367035, 0.48937687277793884], "agreement_weight": 0.153973788022995, "orthogonal_weight": -0.7969990968704224}]}, {"layer": 4, "memory_to_active": 0.9920797944068909, "contribution_to_active": 0.4499755799770355, "mix_mean": 0.36277735233306885, "mix_low": 0.27091535925865173, "mix_high": 0.010065874084830284, "read_cosine": 0.3325962424278259, "beta_mean": 0.11310034990310669, "nonfinite": 0.0, "weighted_memory_to_active": 0.12250004708766937, "mix_std": 0.15774431824684143, "beta_std": 0.10368751734495163, "beta_low": 0.3812662959098816, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04147430509328842, "output_influence": 0.463174045085907, "residual_influence": 0.1301879733800888, "state_active": 0.03894355148077011, "state_memory": 0.01751849800348282, "state_peak": 35.71353530883789, "growth_active": 1.2586085796356201, "growth_memory": 1.5366405248641968, "heads": [{"head": 0, "memory_to_active": 0.18570929765701294, "contribution_to_active": 0.7619850635528564, "mix_mean": 0.682455837726593, "mix_low": 0.011284828186035156, "mix_high": 0.02620983123779297, "read_cosine": 0.4807736277580261, "beta_mean": 0.2523602247238159, "nonfinite": 0.0, "weighted_memory_to_active": 0.15354324877262115, "mix_std": 0.34244638681411743, "beta_std": 0.2624068260192871, "beta_low": 0.2418212890625, "beta_high": 0.0, "orthogonal_to_active": 0.06903985142707825, "evidence_weights": [-0.2694723904132843, -0.37029120326042175, 0.03935634344816208], "agreement_weight": 0.012020885944366455, "orthogonal_weight": 0.557273268699646}, {"head": 1, "memory_to_active": 0.05508147180080414, "contribution_to_active": 0.6501398086547852, "mix_mean": 0.6834220886230469, "mix_low": 0.00012683868408203125, "mix_high": 0.01600933074951172, "read_cosine": 0.657852292060852, "beta_mean": 0.0009156740270555019, "nonfinite": 0.0, "weighted_memory_to_active": 0.039937522262334824, "mix_std": 0.24237103760242462, "beta_std": 0.006544368341565132, "beta_low": 0.9840087890625, "beta_high": 0.0, "orthogonal_to_active": 0.021586235612630844, "evidence_weights": [-0.24070148169994354, -0.35811957716941833, 0.664110541343689], "agreement_weight": 0.2716931998729706, "orthogonal_weight": -0.7887540459632874}, {"head": 2, "memory_to_active": 1.7614483833312988, "contribution_to_active": 0.2803521454334259, "mix_mean": 0.029020292684435844, "mix_low": 0.5050821304321289, "mix_high": 2.09808349609375e-05, "read_cosine": 0.25866714119911194, "beta_mean": 0.07673962414264679, "nonfinite": 0.0, "weighted_memory_to_active": 0.2377091944217682, "mix_std": 0.06405843049287796, "beta_std": 0.07127220928668976, "beta_low": 0.04034423828125, "beta_high": 0.0, "orthogonal_to_active": 0.049252815544605255, "evidence_weights": [0.053508032113313675, -0.0015758974477648735, 0.14450068771839142], "agreement_weight": 0.13202281296253204, "orthogonal_weight": -0.3079570233821869}, {"head": 3, "memory_to_active": 1.7756643295288086, "contribution_to_active": 0.1199350655078888, "mix_mean": 0.028047330677509308, "mix_low": 0.45322132110595703, "mix_high": 0.0, "read_cosine": 0.17705649137496948, "beta_mean": 0.13926023244857788, "nonfinite": 0.0, "weighted_memory_to_active": 0.1638038158416748, "mix_std": 0.050746794790029526, "beta_std": 0.12179307639598846, "beta_low": 0.0213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.0012236922048032284, "evidence_weights": [-0.018322350457310677, 0.05937251076102257, -0.27530166506767273], "agreement_weight": 0.0716186985373497, "orthogonal_weight": -0.0107162119820714}, {"head": 4, "memory_to_active": 0.007853914052248001, "contribution_to_active": 0.6490929126739502, "mix_mean": 0.7401196956634521, "mix_low": 0.00011539459228515625, "mix_high": 0.01815032958984375, "read_cosine": 0.174085333943367, "beta_mean": 1.8188904505223036e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005638170056045055, "mix_std": 0.21716853976249695, "beta_std": 4.103148967260495e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026778937317430973, "evidence_weights": [-0.3389149010181427, -0.32909485697746277, 0.5039724707603455], "agreement_weight": 0.33341753482818604, "orthogonal_weight": -0.50799959897995}, {"head": 5, "memory_to_active": 2.1667213439941406, "contribution_to_active": 0.23834842443466187, "mix_mean": 0.013598775491118431, "mix_low": 0.6556615829467773, "mix_high": 4.76837158203125e-06, "read_cosine": 0.24714235961437225, "beta_mean": 0.20930816233158112, "nonfinite": 0.0, "weighted_memory_to_active": 0.13436834514141083, "mix_std": 0.029674626886844635, "beta_std": 0.16006752848625183, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.10506531596183777, "evidence_weights": [0.05070028081536293, -0.02768542431294918, -0.27587881684303284], "agreement_weight": -0.08113442361354828, "orthogonal_weight": 0.9583696126937866}]}, {"layer": 5, "memory_to_active": 0.4356469511985779, "contribution_to_active": 0.525501012802124, "mix_mean": 0.49405601620674133, "mix_low": 0.05604235827922821, "mix_high": 0.017505645751953125, "read_cosine": 0.06659339368343353, "beta_mean": 0.05190179869532585, "nonfinite": 0.0, "weighted_memory_to_active": 0.15180423855781555, "mix_std": 0.25732138752937317, "beta_std": 0.06925663352012634, "beta_low": 0.6534017324447632, "beta_high": 0.0, "orthogonal_to_active": 0.05777747929096222, "output_influence": 0.5344663262367249, "residual_influence": 0.1513349711894989, "state_active": 0.03925555944442749, "state_memory": 0.011190414428710938, "state_peak": 58.739105224609375, "growth_active": 1.0842205286026, "growth_memory": 1.2916288375854492, "heads": [{"head": 0, "memory_to_active": 0.3597364127635956, "contribution_to_active": 0.5235912799835205, "mix_mean": 0.3982980251312256, "mix_low": 0.016698837280273438, "mix_high": 0.001682281494140625, "read_cosine": -0.10833413898944855, "beta_mean": 0.028076963499188423, "nonfinite": 0.0, "weighted_memory_to_active": 0.12032659351825714, "mix_std": 0.2786293625831604, "beta_std": 0.05100952833890915, "beta_low": 0.42724609375, "beta_high": 0.0, "orthogonal_to_active": 0.04540885612368584, "evidence_weights": [-0.140081524848938, -0.16514016687870026, 0.03366202861070633], "agreement_weight": -0.02409680001437664, "orthogonal_weight": -0.38446125388145447}, {"head": 1, "memory_to_active": 0.010003265924751759, "contribution_to_active": 0.5431824922561646, "mix_mean": 0.682553768157959, "mix_low": 0.0019006729125976562, "mix_high": 0.033789634704589844, "read_cosine": 0.1644608974456787, "beta_mean": 2.4447923351544887e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0057584550231695175, "mix_std": 0.2780160903930664, "beta_std": 5.243459600023925e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005483619868755341, "evidence_weights": [-0.25257354974746704, -0.2914137542247772, 0.872298538684845], "agreement_weight": 0.6605809330940247, "orthogonal_weight": -0.9965424537658691}, {"head": 2, "memory_to_active": 0.010906579904258251, "contribution_to_active": 0.5870857238769531, "mix_mean": 0.653908371925354, "mix_low": 0.0026035308837890625, "mix_high": 0.022319793701171875, "read_cosine": -0.49896588921546936, "beta_mean": 7.755540718790144e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006966135464608669, "mix_std": 0.26824817061424255, "beta_std": 0.0001269132480956614, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003291415749117732, "evidence_weights": [-0.26075196266174316, -0.3389846086502075, 0.3022654056549072], "agreement_weight": -0.055823374539613724, "orthogonal_weight": -0.5839061737060547}, {"head": 3, "memory_to_active": 1.6680399179458618, "contribution_to_active": 0.2576110064983368, "mix_mean": 0.12147024273872375, "mix_low": 0.11836814880371094, "mix_high": 9.5367431640625e-07, "read_cosine": 0.14149637520313263, "beta_mean": 0.10214667022228241, "nonfinite": 0.0, "weighted_memory_to_active": 0.41255903244018555, "mix_std": 0.15197953581809998, "beta_std": 0.1529669612646103, "beta_low": 0.201171875, "beta_high": 0.0, "orthogonal_to_active": 0.15184058248996735, "evidence_weights": [-0.07522393763065338, -0.10443372279405594, 0.054415013641119], "agreement_weight": 0.05014749988913536, "orthogonal_weight": -0.4849265515804291}, {"head": 4, "memory_to_active": 0.46296775341033936, "contribution_to_active": 0.5674024224281311, "mix_mean": 0.31651273369789124, "mix_low": 0.19463539123535156, "mix_high": 0.0062694549560546875, "read_cosine": 0.5445973873138428, "beta_mean": 0.1807873696088791, "nonfinite": 0.0, "weighted_memory_to_active": 0.29650813341140747, "mix_std": 0.33577755093574524, "beta_std": 0.21001943945884705, "beta_low": 0.29583740234375, "beta_high": 0.0, "orthogonal_to_active": 0.09836677461862564, "evidence_weights": [-0.11840824037790298, -0.2687935531139374, -0.05038052424788475], "agreement_weight": 0.04177913814783096, "orthogonal_weight": 0.5060449838638306}, {"head": 5, "memory_to_active": 0.10222770273685455, "contribution_to_active": 0.6741328835487366, "mix_mean": 0.7915929555892944, "mix_low": 0.0020475387573242188, "mix_high": 0.04097175598144531, "read_cosine": 0.15630567073822021, "beta_mean": 0.00029776181327179074, "nonfinite": 0.0, "weighted_memory_to_active": 0.06870700418949127, "mix_std": 0.23127765953540802, "beta_std": 0.0013644741848111153, "beta_low": 0.99615478515625, "beta_high": 0.0, "orthogonal_to_active": 0.042273588478565216, "evidence_weights": [-0.3139376938343048, -0.3324164152145386, 1.9354755878448486], "agreement_weight": 0.45478129386901855, "orthogonal_weight": -0.6481276154518127}]}, {"layer": 6, "memory_to_active": 0.31012168526649475, "contribution_to_active": 0.6209084391593933, "mix_mean": 0.5894486904144287, "mix_low": 0.05459276959300041, "mix_high": 0.03736225888133049, "read_cosine": 0.30689141154289246, "beta_mean": 0.06683699041604996, "nonfinite": 0.0, "weighted_memory_to_active": 0.1539861410856247, "mix_std": 0.2335541844367981, "beta_std": 0.075888991355896, "beta_low": 0.535125732421875, "beta_high": 0.0, "orthogonal_to_active": 0.043570585548877716, "output_influence": 0.5486732721328735, "residual_influence": 0.18015365302562714, "state_active": 0.044544585049152374, "state_memory": 0.00791836529970169, "state_peak": 88.11679077148438, "growth_active": 1.1884006261825562, "growth_memory": 1.2136653661727905, "heads": [{"head": 0, "memory_to_active": 0.11211137473583221, "contribution_to_active": 0.7234083414077759, "mix_mean": 0.8089972138404846, "mix_low": 0.00026416778564453125, "mix_high": 0.04313945770263672, "read_cosine": 0.24090057611465454, "beta_mean": 0.017107537016272545, "nonfinite": 0.0, "weighted_memory_to_active": 0.09124556183815002, "mix_std": 0.20023995637893677, "beta_std": 0.02934373915195465, "beta_low": 0.572021484375, "beta_high": 0.0, "orthogonal_to_active": 0.06957321614027023, "evidence_weights": [-0.31973958015441895, -0.3479154109954834, 1.169299840927124], "agreement_weight": 0.4657057821750641, "orthogonal_weight": -0.8301375508308411}, {"head": 1, "memory_to_active": 0.005640949588268995, "contribution_to_active": 0.524602472782135, "mix_mean": 0.5755469799041748, "mix_low": 0.001125335693359375, "mix_high": 0.014716148376464844, "read_cosine": 0.5097631216049194, "beta_mean": 1.3605316780740395e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0026097502559423447, "mix_std": 0.28750473260879517, "beta_std": 3.400662535568699e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014764374354854226, "evidence_weights": [-0.19917620718479156, -0.3005909323692322, 1.1402919292449951], "agreement_weight": 0.10207019746303558, "orthogonal_weight": -0.737740159034729}, {"head": 2, "memory_to_active": 0.8824158906936646, "contribution_to_active": 0.09376765042543411, "mix_mean": 0.037356991320848465, "mix_low": 0.32036685943603516, "mix_high": 0.0, "read_cosine": 0.23472781479358673, "beta_mean": 0.02971254102885723, "nonfinite": 0.0, "weighted_memory_to_active": 0.0688222125172615, "mix_std": 0.05965191870927811, "beta_std": 0.029953770339488983, "beta_low": 0.14959716796875, "beta_high": 0.0, "orthogonal_to_active": 0.005205885507166386, "evidence_weights": [0.08052551746368408, -0.034595683217048645, 0.29079729318618774], "agreement_weight": 0.1479814499616623, "orthogonal_weight": -0.09312973916530609}, {"head": 3, "memory_to_active": 0.15305277705192566, "contribution_to_active": 0.6782117486000061, "mix_mean": 0.6684043407440186, "mix_low": 0.00467681884765625, "mix_high": 0.026590347290039062, "read_cosine": 0.44714105129241943, "beta_mean": 0.2221130132675171, "nonfinite": 0.0, "weighted_memory_to_active": 0.10880067944526672, "mix_std": 0.32097241282463074, "beta_std": 0.2507385015487671, "beta_low": 0.19488525390625, "beta_high": 0.0, "orthogonal_to_active": 0.059232447296381, "evidence_weights": [-0.17820365726947784, -0.33357205986976624, 0.1780993491411209], "agreement_weight": -0.0039940085262060165, "orthogonal_weight": 0.6152565479278564}, {"head": 4, "memory_to_active": 0.6692869067192078, "contribution_to_active": 1.0826268196105957, "mix_mean": 0.7724589109420776, "mix_low": 0.0006818771362304688, "mix_high": 0.13315677642822266, "read_cosine": 0.13363702595233917, "beta_mean": 0.13187803328037262, "nonfinite": 0.0, "weighted_memory_to_active": 0.6252830624580383, "mix_std": 0.2842695116996765, "beta_std": 0.14483857154846191, "beta_low": 0.29443359375, "beta_high": 0.0, "orthogonal_to_active": 0.09889402985572815, "evidence_weights": [-0.22227853536605835, -0.5747115612030029, -0.34176185727119446], "agreement_weight": -0.01926170289516449, "orthogonal_weight": 0.1758987009525299}, {"head": 5, "memory_to_active": 0.03822214901447296, "contribution_to_active": 0.6228335499763489, "mix_mean": 0.6739277243614197, "mix_low": 0.00044155120849609375, "mix_high": 0.0065708160400390625, "read_cosine": 0.27517861127853394, "beta_mean": 0.0001972168538486585, "nonfinite": 0.0, "weighted_memory_to_active": 0.027155574411153793, "mix_std": 0.24868649244308472, "beta_std": 0.0004253398801665753, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.02704145386815071, "evidence_weights": [-0.2420988529920578, -0.31383809447288513, 0.41298753023147583], "agreement_weight": 0.1827385276556015, "orthogonal_weight": -1.0686432123184204}]}, {"layer": 7, "memory_to_active": 1.7001062631607056, "contribution_to_active": 0.24981719255447388, "mix_mean": 0.15128488838672638, "mix_low": 0.2561896741390228, "mix_high": 0.000810305355116725, "read_cosine": 0.23717233538627625, "beta_mean": 0.13769137859344482, "nonfinite": 0.0, "weighted_memory_to_active": 0.19636164605617523, "mix_std": 0.14235594868659973, "beta_std": 0.09827108681201935, "beta_low": 0.2065022885799408, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.07507534325122833, "output_influence": 0.25835201144218445, "residual_influence": 0.059093765914440155, "state_active": 0.0135275237262249, "state_memory": 0.024837728589773178, "state_peak": 25.89046859741211, "growth_active": 1.1480717658996582, "growth_memory": 1.2788430452346802, "heads": [{"head": 0, "memory_to_active": 2.3206729888916016, "contribution_to_active": 0.19697445631027222, "mix_mean": 0.03639920428395271, "mix_low": 0.4300212860107422, "mix_high": 0.0, "read_cosine": 0.33847254514694214, "beta_mean": 0.1538659930229187, "nonfinite": 0.0, "weighted_memory_to_active": 0.2341470867395401, "mix_std": 0.07674643397331238, "beta_std": 0.12267822027206421, "beta_low": 0.00982666015625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.001699115033261478, "evidence_weights": [0.0361722894012928, -0.07106618583202362, 0.0516061894595623], "agreement_weight": -0.029123611748218536, "orthogonal_weight": 0.009752525947988033}, {"head": 1, "memory_to_active": 0.980048418045044, "contribution_to_active": 0.3902440369129181, "mix_mean": 0.2157265841960907, "mix_low": 0.12089824676513672, "mix_high": 0.004260063171386719, "read_cosine": 0.3599124252796173, "beta_mean": 0.09084029495716095, "nonfinite": 0.0, "weighted_memory_to_active": 0.32349610328674316, "mix_std": 0.23900432884693146, "beta_std": 0.06136737018823624, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04305369406938553, "evidence_weights": [-0.21481573581695557, -0.10464707016944885, -0.23369663953781128], "agreement_weight": 0.048325128853321075, "orthogonal_weight": 0.15609155595302582}, {"head": 2, "memory_to_active": 0.6795887351036072, "contribution_to_active": 0.3292432129383087, "mix_mean": 0.1823357790708542, "mix_low": 0.03925323486328125, "mix_high": 0.00028896331787109375, "read_cosine": 0.10761841386556625, "beta_mean": 0.04338241368532181, "nonfinite": 0.0, "weighted_memory_to_active": 0.2159314900636673, "mix_std": 0.19476601481437683, "beta_std": 0.04185255616903305, "beta_low": 0.1129150390625, "beta_high": 0.0, "orthogonal_to_active": 0.10644681006669998, "evidence_weights": [-0.07504957914352417, -0.15751059353351593, 0.7198019027709961], "agreement_weight": 0.022685419768095016, "orthogonal_weight": -0.5044638514518738}, {"head": 3, "memory_to_active": 4.437889575958252, "contribution_to_active": 0.0935106873512268, "mix_mean": 0.03319746255874634, "mix_low": 0.33328819274902344, "mix_high": 0.0, "read_cosine": 0.10047458112239838, "beta_mean": 0.454342246055603, "nonfinite": 0.0, "weighted_memory_to_active": 0.3060920834541321, "mix_std": 0.04832456260919571, "beta_std": 0.2740066945552826, "beta_low": 0.00079345703125, "beta_high": 0.000732421875, "orthogonal_to_active": 0.26041439175605774, "evidence_weights": [0.07291360944509506, -0.01984390802681446, 0.7067990899085999], "agreement_weight": 0.05514035373926163, "orthogonal_weight": -0.9374237656593323}, {"head": 4, "memory_to_active": 1.7215981483459473, "contribution_to_active": 0.08409444987773895, "mix_mean": 0.018023556098341942, "mix_low": 0.6111516952514648, "mix_high": 0.0, "read_cosine": 0.2396925985813141, "beta_mean": 0.0834512710571289, "nonfinite": 0.0, "weighted_memory_to_active": 0.06832057237625122, "mix_std": 0.040139101445674896, "beta_std": 0.08908487856388092, "beta_low": 0.11614990234375, "beta_high": 0.0, "orthogonal_to_active": 0.01229896955192089, "evidence_weights": [0.14886416494846344, 0.011850135400891304, 0.20552648603916168], "agreement_weight": 0.041240058839321136, "orthogonal_weight": -0.21819819509983063}, {"head": 5, "memory_to_active": 0.06083855405449867, "contribution_to_active": 0.4048362672328949, "mix_mean": 0.42202675342559814, "mix_low": 0.00252532958984375, "mix_high": 0.00031280517578125, "read_cosine": 0.2768633961677551, "beta_mean": 0.0002660758909769356, "nonfinite": 0.0, "weighted_memory_to_active": 0.03018259070813656, "mix_std": 0.25515517592430115, "beta_std": 0.0006367713795043528, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.026539037004113197, "evidence_weights": [-0.16092263162136078, -0.2192607969045639, 0.5516321659088135], "agreement_weight": 0.19014261662960052, "orthogonal_weight": -0.9566770792007446}]}, {"layer": 8, "memory_to_active": 0.20333462953567505, "contribution_to_active": 0.5393073558807373, "mix_mean": 0.4530545473098755, "mix_low": 0.02519957348704338, "mix_high": 0.0014526050072163343, "read_cosine": 0.44691282510757446, "beta_mean": 0.015763333067297935, "nonfinite": 0.0, "weighted_memory_to_active": 0.07937600463628769, "mix_std": 0.2307305783033371, "beta_std": 0.022231079638004303, "beta_low": 0.7328898906707764, "beta_high": 0.0, "orthogonal_to_active": 0.06773491203784943, "output_influence": 0.4914814829826355, "residual_influence": 0.16696706414222717, "state_active": 0.045437365770339966, "state_memory": 0.0064149475656449795, "state_peak": 38.023799896240234, "growth_active": 1.1038646697998047, "growth_memory": 1.229067325592041, "heads": [{"head": 0, "memory_to_active": 0.05431032553315163, "contribution_to_active": 0.5691453814506531, "mix_mean": 0.6035138964653015, "mix_low": 0.0004978179931640625, "mix_high": 0.0013723373413085938, "read_cosine": 0.4800192415714264, "beta_mean": 0.0032599784899502993, "nonfinite": 0.0, "weighted_memory_to_active": 0.03845766931772232, "mix_std": 0.24751496315002441, "beta_std": 0.011289898306131363, "beta_low": 0.92706298828125, "beta_high": 0.0, "orthogonal_to_active": 0.02411145344376564, "evidence_weights": [-0.1680534929037094, -0.38978058099746704, 0.6489675641059875], "agreement_weight": 0.2142297625541687, "orthogonal_weight": -0.7961692214012146}, {"head": 1, "memory_to_active": 0.0771651491522789, "contribution_to_active": 0.5446518659591675, "mix_mean": 0.5076651573181152, "mix_low": 6.4849853515625e-05, "mix_high": 0.0010509490966796875, "read_cosine": 0.17055247724056244, "beta_mean": 0.001367358025163412, "nonfinite": 0.0, "weighted_memory_to_active": 0.036832939833402634, "mix_std": 0.24033425748348236, "beta_std": 0.00711592985317111, "beta_low": 0.98260498046875, "beta_high": 0.0, "orthogonal_to_active": 0.02680746465921402, "evidence_weights": [-0.19413171708583832, -0.2754714787006378, 0.6854893565177917], "agreement_weight": 0.06843195855617523, "orthogonal_weight": -0.7588306069374084}, {"head": 2, "memory_to_active": 0.011802276596426964, "contribution_to_active": 0.6670089960098267, "mix_mean": 0.6623934507369995, "mix_low": 0.0008363723754882812, "mix_high": 0.003063201904296875, "read_cosine": 0.41722312569618225, "beta_mean": 0.00012508255895227194, "nonfinite": 0.0, "weighted_memory_to_active": 0.009245653636753559, "mix_std": 0.22773444652557373, "beta_std": 0.0028579519130289555, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005713053047657013, "evidence_weights": [-0.2250431329011917, -0.3428930938243866, 1.0152184963226318], "agreement_weight": 0.24876780807971954, "orthogonal_weight": -0.7168750762939453}, {"head": 3, "memory_to_active": 0.11782842129468918, "contribution_to_active": 0.631758987903595, "mix_mean": 0.575094997882843, "mix_low": 0.0006380081176757812, "mix_high": 0.0030651092529296875, "read_cosine": 0.8167693614959717, "beta_mean": 0.008711319416761398, "nonfinite": 0.0, "weighted_memory_to_active": 0.08766718208789825, "mix_std": 0.257416307926178, "beta_std": 0.007717430591583252, "beta_low": 0.67254638671875, "beta_high": 0.0, "orthogonal_to_active": 0.027331627905368805, "evidence_weights": [-0.18461523950099945, -0.30149880051612854, 0.5660427212715149], "agreement_weight": 0.21938739717006683, "orthogonal_weight": -0.6323977112770081}, {"head": 4, "memory_to_active": 0.5908725261688232, "contribution_to_active": 0.19094394147396088, "mix_mean": 0.12463200092315674, "mix_low": 0.06638336181640625, "mix_high": 9.5367431640625e-07, "read_cosine": 0.30779680609703064, "beta_mean": 0.010766001418232918, "nonfinite": 0.0, "weighted_memory_to_active": 0.11477860808372498, "mix_std": 0.14401136338710785, "beta_std": 0.021929912269115448, "beta_low": 0.72314453125, "beta_high": 0.0, "orthogonal_to_active": 0.02223488688468933, "evidence_weights": [-0.022269325330853462, -0.12152988463640213, 0.24437813460826874], "agreement_weight": 0.07178650796413422, "orthogonal_weight": -0.2064645141363144}, {"head": 5, "memory_to_active": 0.3680291771888733, "contribution_to_active": 0.6323346495628357, "mix_mean": 0.2450275421142578, "mix_low": 0.08277702331542969, "mix_high": 0.00016307830810546875, "read_cosine": 0.4891158938407898, "beta_mean": 0.07035025209188461, "nonfinite": 0.0, "weighted_memory_to_active": 0.18927396833896637, "mix_std": 0.2673720717430115, "beta_std": 0.08247536420822144, "beta_low": 0.092041015625, "beta_high": 0.0, "orthogonal_to_active": 0.30021095275878906, "evidence_weights": [-0.11905337870121002, -0.17200101912021637, -0.14035190641880035], "agreement_weight": -0.01466891635209322, "orthogonal_weight": 1.8660720586776733}]}, {"layer": 9, "memory_to_active": 0.998609185218811, "contribution_to_active": 0.7122616767883301, "mix_mean": 0.34892839193344116, "mix_low": 0.10950756072998047, "mix_high": 0.006062190048396587, "read_cosine": 0.3391164541244507, "beta_mean": 0.05127909034490585, "nonfinite": 0.0, "weighted_memory_to_active": 0.41665756702423096, "mix_std": 0.22353851795196533, "beta_std": 0.04099627956748009, "beta_low": 0.5103861689567566, "beta_high": 0.0, "orthogonal_to_active": 0.04118776693940163, "output_influence": 0.5347504615783691, "residual_influence": 0.15008893609046936, "state_active": 0.04241965711116791, "state_memory": 0.017759619280695915, "state_peak": 53.9073371887207, "growth_active": 1.1811728477478027, "growth_memory": 1.374403715133667, "heads": [{"head": 0, "memory_to_active": 0.0016393080586567521, "contribution_to_active": 0.5666087865829468, "mix_mean": 0.6388524770736694, "mix_low": 0.001373291015625, "mix_high": 0.025595664978027344, "read_cosine": 0.2913505434989929, "beta_mean": 5.412096015788848e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0010508504929021, "mix_std": 0.2908805310726166, "beta_std": 2.243518792965915e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010926106479018927, "evidence_weights": [-0.3027297854423523, -0.3090895414352417, 0.670466423034668], "agreement_weight": 0.11998306214809418, "orthogonal_weight": -1.0870999097824097}, {"head": 1, "memory_to_active": 0.020744871348142624, "contribution_to_active": 0.679267168045044, "mix_mean": 0.5438418388366699, "mix_low": 0.007500648498535156, "mix_high": 0.0025014877319335938, "read_cosine": 0.4104960560798645, "beta_mean": 5.0485654355725273e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015440361574292183, "mix_std": 0.28606829047203064, "beta_std": 0.0001221786078531295, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010569106787443161, "evidence_weights": [-0.23622846603393555, -0.2920825779438019, 1.1017582416534424], "agreement_weight": 0.4308273196220398, "orthogonal_weight": -0.857387363910675}, {"head": 2, "memory_to_active": 0.18471522629261017, "contribution_to_active": 0.6486651301383972, "mix_mean": 0.5923318862915039, "mix_low": 0.0016965866088867188, "mix_high": 0.007030487060546875, "read_cosine": 0.34581243991851807, "beta_mean": 0.0007031996501609683, "nonfinite": 0.0, "weighted_memory_to_active": 0.12440018355846405, "mix_std": 0.26888740062713623, "beta_std": 0.004294992424547672, "beta_low": 0.9952392578125, "beta_high": 0.0, "orthogonal_to_active": 0.07279226928949356, "evidence_weights": [-0.1986178308725357, -0.27456018328666687, 0.767405092716217], "agreement_weight": 0.3121575117111206, "orthogonal_weight": -0.6759417653083801}, {"head": 3, "memory_to_active": 1.1101950407028198, "contribution_to_active": 0.186320498585701, "mix_mean": 0.07097971439361572, "mix_low": 0.26731204986572266, "mix_high": 0.00017833709716796875, "read_cosine": 0.370441198348999, "beta_mean": 0.060024444013834, "nonfinite": 0.0, "weighted_memory_to_active": 0.16318002343177795, "mix_std": 0.12765152752399445, "beta_std": 0.04855624586343765, "beta_low": 0.010498046875, "beta_high": 0.0, "orthogonal_to_active": 0.024112312123179436, "evidence_weights": [0.05052962526679039, -0.020875880494713783, 0.11127348989248276], "agreement_weight": 0.09148641675710678, "orthogonal_weight": -0.15786221623420715}, {"head": 4, "memory_to_active": 3.0896897315979004, "contribution_to_active": 1.6204956769943237, "mix_mean": 0.1410631537437439, "mix_low": 0.1349630355834961, "mix_high": 0.0009098052978515625, "read_cosine": 0.1313856840133667, "beta_mean": 0.151217520236969, "nonfinite": 0.0, "weighted_memory_to_active": 1.5886735916137695, "mix_std": 0.20426802337169647, "beta_std": 0.09119825065135956, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02901957370340824, "evidence_weights": [-0.0800103172659874, -0.1573195904493332, -0.030303817242383957], "agreement_weight": -0.06000283360481262, "orthogonal_weight": -0.018978899344801903}, {"head": 5, "memory_to_active": 1.5846705436706543, "contribution_to_active": 0.5722125768661499, "mix_mean": 0.10650114715099335, "mix_low": 0.2441997528076172, "mix_high": 0.00015735626220703125, "read_cosine": 0.4852127730846405, "beta_mean": 0.09567347913980484, "nonfinite": 0.0, "weighted_memory_to_active": 0.6072003245353699, "mix_std": 0.16347533464431763, "beta_std": 0.10178355872631073, "beta_low": 0.05657958984375, "beta_high": 0.0, "orthogonal_to_active": 0.10954072326421738, "evidence_weights": [0.10064683109521866, -0.11290017515420914, 0.14020554721355438], "agreement_weight": 0.012983744964003563, "orthogonal_weight": 0.2538659870624542}]}, {"layer": 10, "memory_to_active": 0.2068205028772354, "contribution_to_active": 0.6817175149917603, "mix_mean": 0.5177625417709351, "mix_low": 0.018993377685546875, "mix_high": 0.006566843017935753, "read_cosine": 0.37749749422073364, "beta_mean": 0.042185790836811066, "nonfinite": 0.0, "weighted_memory_to_active": 0.09650884568691254, "mix_std": 0.2688278257846832, "beta_std": 0.04516927897930145, "beta_low": 0.6699117422103882, "beta_high": 0.0, "orthogonal_to_active": 0.21344566345214844, "output_influence": 0.44850796461105347, "residual_influence": 0.15473207831382751, "state_active": 0.052970051765441895, "state_memory": 0.014913375489413738, "state_peak": 128.8436737060547, "growth_active": 1.0299983024597168, "growth_memory": 1.0115387439727783, "heads": [{"head": 0, "memory_to_active": 0.7556443214416504, "contribution_to_active": 1.8102753162384033, "mix_mean": 0.5843181610107422, "mix_low": 0.07206153869628906, "mix_high": 0.016145706176757812, "read_cosine": 0.11892469227313995, "beta_mean": 0.19128850102424622, "nonfinite": 0.0, "weighted_memory_to_active": 0.4620812237262726, "mix_std": 0.39208412170410156, "beta_std": 0.20887695252895355, "beta_low": 0.1551513671875, "beta_high": 0.0, "orthogonal_to_active": 1.2008050680160522, "evidence_weights": [-0.10143157839775085, -0.3883092701435089, -1.6278961896896362], "agreement_weight": 0.03712264075875282, "orthogonal_weight": 2.6268296241760254}, {"head": 1, "memory_to_active": 0.010600199922919273, "contribution_to_active": 0.6043763160705566, "mix_mean": 0.6239079236984253, "mix_low": 0.0013704299926757812, "mix_high": 0.016577720642089844, "read_cosine": 0.2757118344306946, "beta_mean": 6.777698581572622e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005877497605979443, "mix_std": 0.2677273452281952, "beta_std": 9.816634701564908e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008808179758489132, "evidence_weights": [-0.25462910532951355, -0.19822536408901215, 0.5698623061180115], "agreement_weight": 0.5457524657249451, "orthogonal_weight": -1.6165164709091187}, {"head": 2, "memory_to_active": 0.01972052827477455, "contribution_to_active": 0.615782618522644, "mix_mean": 0.6492255330085754, "mix_low": 0.00016307830810546875, "mix_high": 0.005364418029785156, "read_cosine": 0.019633276388049126, "beta_mean": 3.950883183279075e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008766605518758297, "mix_std": 0.2514854967594147, "beta_std": 0.0001618497190065682, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0037266910076141357, "evidence_weights": [-0.2575324475765228, -0.2821309268474579, 0.6458930373191833], "agreement_weight": -0.19005513191223145, "orthogonal_weight": -0.4087808132171631}, {"head": 3, "memory_to_active": 0.3188301622867584, "contribution_to_active": 0.09265600889921188, "mix_mean": 0.271375447511673, "mix_low": 0.03773784637451172, "mix_high": 0.00036144256591796875, "read_cosine": 0.9165185689926147, "beta_mean": 0.055900972336530685, "nonfinite": 0.0, "weighted_memory_to_active": 0.03501331806182861, "mix_std": 0.22532770037651062, "beta_std": 0.05528092756867409, "beta_low": 0.0089111328125, "beta_high": 0.0, "orthogonal_to_active": 0.03815755248069763, "evidence_weights": [-0.10415253043174744, -0.14735792577266693, 0.15690040588378906], "agreement_weight": 0.009728502482175827, "orthogonal_weight": 1.410614252090454}, {"head": 4, "memory_to_active": 0.014293716289103031, "contribution_to_active": 0.5470887422561646, "mix_mean": 0.5763431787490845, "mix_low": 0.00027751922607421875, "mix_high": 0.0009450912475585938, "read_cosine": 0.06839442253112793, "beta_mean": 5.140122084412724e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008162370882928371, "mix_std": 0.24715226888656616, "beta_std": 0.000130774118588306, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.010610289871692657, "evidence_weights": [-0.1833401769399643, -0.2598588168621063, 0.9395838379859924], "agreement_weight": 0.14714714884757996, "orthogonal_weight": -1.369439959526062}, {"head": 5, "memory_to_active": 0.12183408439159393, "contribution_to_active": 0.42012566328048706, "mix_mean": 0.40140464901924133, "mix_low": 0.002349853515625, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8658021092414856, "beta_mean": 0.005766562186181545, "nonfinite": 0.0, "weighted_memory_to_active": 0.059152014553546906, "mix_std": 0.22919002175331116, "beta_std": 0.006467003840953112, "beta_low": 0.85546875, "beta_high": 0.0, "orthogonal_to_active": 0.018566245213150978, "evidence_weights": [-0.14104753732681274, -0.1617686003446579, 0.26928672194480896], "agreement_weight": 0.09073760360479355, "orthogonal_weight": -0.6531304717063904}]}, {"layer": 11, "memory_to_active": 0.08829759806394577, "contribution_to_active": 0.6131246089935303, "mix_mean": 0.5157029628753662, "mix_low": 0.0935114324092865, "mix_high": 0.0520223006606102, "read_cosine": 0.34785741567611694, "beta_mean": 0.025241848081350327, "nonfinite": 0.0, "weighted_memory_to_active": 0.04577770084142685, "mix_std": 0.3258379101753235, "beta_std": 0.03713987022638321, "beta_low": 0.670501708984375, "beta_high": 0.0, "orthogonal_to_active": 0.044649552553892136, "output_influence": 0.534944474697113, "residual_influence": 0.1384001076221466, "state_active": 0.2136693149805069, "state_memory": 0.004337115213274956, "state_peak": 28.156023025512695, "growth_active": 1.1718223094940186, "growth_memory": 1.64612877368927, "heads": [{"head": 0, "memory_to_active": 0.006075526587665081, "contribution_to_active": 0.6035792231559753, "mix_mean": 0.6389025449752808, "mix_low": 0.0010213851928710938, "mix_high": 0.008324623107910156, "read_cosine": 0.30345436930656433, "beta_mean": 5.6090728321578354e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0044984836131334305, "mix_std": 0.26134321093559265, "beta_std": 0.002583023626357317, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.003528689267113805, "evidence_weights": [-0.17138484120368958, -0.36702200770378113, 0.7549031972885132], "agreement_weight": 0.3371778726577759, "orthogonal_weight": -0.8150404095649719}, {"head": 1, "memory_to_active": 0.19071617722511292, "contribution_to_active": 0.39309436082839966, "mix_mean": 0.4398467242717743, "mix_low": 0.007145881652832031, "mix_high": 0.00047206878662109375, "read_cosine": 0.48275381326675415, "beta_mean": 0.009548387490212917, "nonfinite": 0.0, "weighted_memory_to_active": 0.08616430312395096, "mix_std": 0.2692074775695801, "beta_std": 0.022729218006134033, "beta_low": 0.79656982421875, "beta_high": 0.0, "orthogonal_to_active": 0.04161931946873665, "evidence_weights": [-0.16205541789531708, -0.23478290438652039, 0.5943530201911926], "agreement_weight": 0.07116223126649857, "orthogonal_weight": -0.575721025466919}, {"head": 2, "memory_to_active": 0.006750340573489666, "contribution_to_active": 0.9858279228210449, "mix_mean": 0.4518534541130066, "mix_low": 0.25130271911621094, "mix_high": 0.18682575225830078, "read_cosine": 0.17400668561458588, "beta_mean": 0.06601691991090775, "nonfinite": 0.0, "weighted_memory_to_active": 0.004775368608534336, "mix_std": 0.4464486837387085, "beta_std": 0.08896070718765259, "beta_low": 0.4781494140625, "beta_high": 0.0, "orthogonal_to_active": 0.03972741961479187, "evidence_weights": [0.03777068853378296, -0.408528208732605, -1.301506519317627], "agreement_weight": 0.007308472413569689, "orthogonal_weight": 8.596402168273926}, {"head": 3, "memory_to_active": 0.007589476648718119, "contribution_to_active": 0.8669723272323608, "mix_mean": 0.42373183369636536, "mix_low": 0.29143619537353516, "mix_high": 0.10841178894042969, "read_cosine": 0.5686018466949463, "beta_mean": 0.05120767280459404, "nonfinite": 0.0, "weighted_memory_to_active": 0.004945735447108746, "mix_std": 0.4417363405227661, "beta_std": 0.07818452268838882, "beta_low": 0.51593017578125, "beta_high": 0.0, "orthogonal_to_active": 0.02813904732465744, "evidence_weights": [0.18628688156604767, -0.31136566400527954, -0.16289164125919342], "agreement_weight": 0.050607409328222275, "orthogonal_weight": 8.891709327697754}, {"head": 4, "memory_to_active": 0.22661276161670685, "contribution_to_active": 0.4415382742881775, "mix_mean": 0.591334342956543, "mix_low": 0.00701904296875, "mix_high": 0.0018215179443359375, "read_cosine": 0.059710126370191574, "beta_mean": 0.015342095866799355, "nonfinite": 0.0, "weighted_memory_to_active": 0.12674486637115479, "mix_std": 0.2732338011264801, "beta_std": 0.017140470445156097, "beta_low": 0.498779296875, "beta_high": 0.0, "orthogonal_to_active": 0.1227702647447586, "evidence_weights": [-0.24816714227199554, -0.23939666152000427, 1.9439876079559326], "agreement_weight": -0.21180596947669983, "orthogonal_weight": -0.985118567943573}, {"head": 5, "memory_to_active": 0.09204129129648209, "contribution_to_active": 0.38773518800735474, "mix_mean": 0.5485485792160034, "mix_low": 0.003143310546875, "mix_high": 0.006278038024902344, "read_cosine": 0.49861741065979004, "beta_mean": 0.009279897436499596, "nonfinite": 0.0, "weighted_memory_to_active": 0.0475374236702919, "mix_std": 0.2630579471588135, "beta_std": 0.013241255655884743, "beta_low": 0.733642578125, "beta_high": 0.0, "orthogonal_to_active": 0.032112546265125275, "evidence_weights": [-0.21213111281394958, -0.31189456582069397, 0.6453912258148193], "agreement_weight": 0.084743931889534, "orthogonal_weight": -0.8143600821495056}]}, {"layer": 12, "memory_to_active": 0.22145605087280273, "contribution_to_active": 0.566299319267273, "mix_mean": 0.5325179100036621, "mix_low": 0.009121894836425781, "mix_high": 0.005647182464599609, "read_cosine": 0.2342555820941925, "beta_mean": 0.012673266232013702, "nonfinite": 0.0, "weighted_memory_to_active": 0.11776223033666611, "mix_std": 0.2619268000125885, "beta_std": 0.014022252522408962, "beta_low": 0.8276265859603882, "beta_high": 0.0, "orthogonal_to_active": 0.06871013343334198, "output_influence": 0.4849380552768707, "residual_influence": 0.13667047023773193, "state_active": 0.03410005196928978, "state_memory": 0.008626537397503853, "state_peak": 53.28779602050781, "growth_active": 1.113998293876648, "growth_memory": 1.2533975839614868, "heads": [{"head": 0, "memory_to_active": 0.05213577300310135, "contribution_to_active": 0.5838073492050171, "mix_mean": 0.5600261688232422, "mix_low": 0.00057220458984375, "mix_high": 0.0037441253662109375, "read_cosine": 0.3124361038208008, "beta_mean": 0.0007700661662966013, "nonfinite": 0.0, "weighted_memory_to_active": 0.026662662625312805, "mix_std": 0.266069233417511, "beta_std": 0.0013579553924500942, "beta_low": 0.99615478515625, "beta_high": 0.0, "orthogonal_to_active": 0.024494212120771408, "evidence_weights": [-0.2073747217655182, -0.27565881609916687, 0.07959754019975662], "agreement_weight": 0.10315506905317307, "orthogonal_weight": -0.987925112247467}, {"head": 1, "memory_to_active": 0.1057204082608223, "contribution_to_active": 0.5469667315483093, "mix_mean": 0.5572986602783203, "mix_low": 0.0004444122314453125, "mix_high": 0.001499176025390625, "read_cosine": 0.4270305037498474, "beta_mean": 0.0009707798599265516, "nonfinite": 0.0, "weighted_memory_to_active": 0.06216800585389137, "mix_std": 0.26681947708129883, "beta_std": 0.010041738860309124, "beta_low": 0.99127197265625, "beta_high": 0.0, "orthogonal_to_active": 0.05192387476563454, "evidence_weights": [-0.206636443734169, -0.27305468916893005, 1.040570855140686], "agreement_weight": 0.21460294723510742, "orthogonal_weight": -0.937744140625}, {"head": 2, "memory_to_active": 0.5751088261604309, "contribution_to_active": 0.4666634202003479, "mix_mean": 0.3116950988769531, "mix_low": 0.02937030792236328, "mix_high": 0.0001373291015625, "read_cosine": 0.5090377926826477, "beta_mean": 0.07298722863197327, "nonfinite": 0.0, "weighted_memory_to_active": 0.2390538454055786, "mix_std": 0.236385315656662, "beta_std": 0.05597483739256859, "beta_low": 0.00537109375, "beta_high": 0.0, "orthogonal_to_active": 0.23981115221977234, "evidence_weights": [-0.223188117146492, -0.05610257387161255, 0.0005001076497137547], "agreement_weight": 0.02842147834599018, "orthogonal_weight": 1.3225313425064087}, {"head": 3, "memory_to_active": 0.029015542939305305, "contribution_to_active": 0.7386235594749451, "mix_mean": 0.7050706148147583, "mix_low": 0.000164031982421875, "mix_high": 0.018075942993164062, "read_cosine": 0.09491010010242462, "beta_mean": 0.0002640283782966435, "nonfinite": 0.0, "weighted_memory_to_active": 0.020844891667366028, "mix_std": 0.23448607325553894, "beta_std": 0.004252253100275993, "beta_low": 0.99725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.020083609968423843, "evidence_weights": [-0.2896389067173004, -0.3385322093963623, 1.5519304275512695], "agreement_weight": 0.48020297288894653, "orthogonal_weight": -0.9817149639129639}, {"head": 4, "memory_to_active": 0.4845741391181946, "contribution_to_active": 0.5608350038528442, "mix_mean": 0.4306510090827942, "mix_low": 0.01687145233154297, "mix_high": 0.004511833190917969, "read_cosine": -0.14550268650054932, "beta_mean": 0.000636272132396698, "nonfinite": 0.0, "weighted_memory_to_active": 0.3020481467247009, "mix_std": 0.28971976041793823, "beta_std": 0.008304647170007229, "beta_low": 0.984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.032755739986896515, "evidence_weights": [-0.14954110980033875, -0.21826496720314026, -0.06437579542398453], "agreement_weight": 0.00518358638510108, "orthogonal_weight": -0.11319053173065186}, {"head": 5, "memory_to_active": 0.08218143880367279, "contribution_to_active": 0.5009000301361084, "mix_mean": 0.6303656101226807, "mix_low": 0.0073089599609375, "mix_high": 0.0059146881103515625, "read_cosine": 0.20762169361114502, "beta_mean": 0.0004112245514988899, "nonfinite": 0.0, "weighted_memory_to_active": 0.05579579249024391, "mix_std": 0.2780809700489044, "beta_std": 0.00420207716524601, "beta_low": 0.9915771484375, "beta_high": 0.0, "orthogonal_to_active": 0.04319222271442413, "evidence_weights": [-0.23446707427501678, -0.3168729841709137, 1.3135944604873657], "agreement_weight": 0.38743525743484497, "orthogonal_weight": -0.8281663060188293}]}, {"layer": 13, "memory_to_active": 0.11541909724473953, "contribution_to_active": 0.521781325340271, "mix_mean": 0.5249119997024536, "mix_low": 0.005824883934110403, "mix_high": 0.006700675003230572, "read_cosine": 0.3939175009727478, "beta_mean": 0.013692911714315414, "nonfinite": 0.0, "weighted_memory_to_active": 0.048721522092819214, "mix_std": 0.25779569149017334, "beta_std": 0.016564032062888145, "beta_low": 0.8066508173942566, "beta_high": 0.0, "orthogonal_to_active": 0.05137629806995392, "output_influence": 0.5003601312637329, "residual_influence": 0.16762515902519226, "state_active": 0.1219695657491684, "state_memory": 0.040485695004463196, "state_peak": 100.56133270263672, "growth_active": 0.9182093739509583, "growth_memory": 1.0200785398483276, "heads": [{"head": 0, "memory_to_active": 0.027391081675887108, "contribution_to_active": 0.5775721073150635, "mix_mean": 0.5961495041847229, "mix_low": 0.0005941390991210938, "mix_high": 0.0033092498779296875, "read_cosine": 0.3911358118057251, "beta_mean": 0.00011080486729042605, "nonfinite": 0.0, "weighted_memory_to_active": 0.016850940883159637, "mix_std": 0.26908594369888306, "beta_std": 0.00029196799732744694, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01703845150768757, "evidence_weights": [-0.19674575328826904, -0.2657637894153595, 0.7695450782775879], "agreement_weight": 0.15675050020217896, "orthogonal_weight": -1.116132140159607}, {"head": 1, "memory_to_active": 0.01401265524327755, "contribution_to_active": 0.5937079787254333, "mix_mean": 0.5606050491333008, "mix_low": 0.0018634796142578125, "mix_high": 0.0036668777465820312, "read_cosine": 0.19909989833831787, "beta_mean": 5.297725147102028e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007907770574092865, "mix_std": 0.2745510935783386, "beta_std": 0.00010052011930383742, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010533828288316727, "evidence_weights": [-0.18371500074863434, -0.26500388979911804, 0.6380739212036133], "agreement_weight": 0.4064912497997284, "orthogonal_weight": -1.3828734159469604}, {"head": 2, "memory_to_active": 0.07418692111968994, "contribution_to_active": 0.4661463797092438, "mix_mean": 0.3759898543357849, "mix_low": 0.0016326904296875, "mix_high": 0.00018024444580078125, "read_cosine": 0.6243013143539429, "beta_mean": 0.010931525379419327, "nonfinite": 0.0, "weighted_memory_to_active": 0.035092227160930634, "mix_std": 0.23022012412548065, "beta_std": 0.03745156526565552, "beta_low": 0.844482421875, "beta_high": 0.0, "orthogonal_to_active": 0.020501593127846718, "evidence_weights": [-0.1856764405965805, -0.20548729598522186, 0.40841180086135864], "agreement_weight": 0.0063639250583946705, "orthogonal_weight": -0.8362756371498108}, {"head": 3, "memory_to_active": 0.1449902206659317, "contribution_to_active": 0.5498433709144592, "mix_mean": 0.6375590562820435, "mix_low": 0.02317047119140625, "mix_high": 0.022617340087890625, "read_cosine": -0.038602426648139954, "beta_mean": 0.0008604678441770375, "nonfinite": 0.0, "weighted_memory_to_active": 0.10466332733631134, "mix_std": 0.28920599818229675, "beta_std": 0.008020627312362194, "beta_low": 0.99505615234375, "beta_high": 0.0, "orthogonal_to_active": 0.0936722457408905, "evidence_weights": [-0.31675001978874207, -0.24456191062927246, 1.661324143409729], "agreement_weight": 5.54857873916626, "orthogonal_weight": -0.9770472049713135}, {"head": 4, "memory_to_active": 0.38179290294647217, "contribution_to_active": 0.24898187816143036, "mix_mean": 0.33577024936676025, "mix_low": 0.004820823669433594, "mix_high": 0.0, "read_cosine": 0.8764258027076721, "beta_mean": 0.06998284161090851, "nonfinite": 0.0, "weighted_memory_to_active": 0.09488113224506378, "mix_std": 0.20914876461029053, "beta_std": 0.05144074186682701, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.13858212530612946, "evidence_weights": [-0.09006775915622711, -0.23447740077972412, 0.46884191036224365], "agreement_weight": 0.009693772532045841, "orthogonal_weight": 2.214353322982788}, {"head": 5, "memory_to_active": 0.05014076083898544, "contribution_to_active": 0.6944360136985779, "mix_mean": 0.6433980464935303, "mix_low": 0.0028676986694335938, "mix_high": 0.010430335998535156, "read_cosine": 0.3111446797847748, "beta_mean": 0.00021885722526349127, "nonfinite": 0.0, "weighted_memory_to_active": 0.03293372318148613, "mix_std": 0.27456220984458923, "beta_std": 0.002078765071928501, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0279295165091753, "evidence_weights": [-0.22100326418876648, -0.30213403701782227, 0.8718701004981995], "agreement_weight": 0.16007296741008759, "orthogonal_weight": -0.9296550154685974}]}, {"layer": 14, "memory_to_active": 0.22394070029258728, "contribution_to_active": 0.4110603332519531, "mix_mean": 0.42315763235092163, "mix_low": 0.025827568024396896, "mix_high": 0.005631924141198397, "read_cosine": 0.6206645965576172, "beta_mean": 0.05630868673324585, "nonfinite": 0.0, "weighted_memory_to_active": 0.09391942620277405, "mix_std": 0.24424321949481964, "beta_std": 0.05576709285378456, "beta_low": 0.498565673828125, "beta_high": 0.0, "orthogonal_to_active": 0.03740048035979271, "output_influence": 0.34446606040000916, "residual_influence": 0.1323360949754715, "state_active": 0.06110832095146179, "state_memory": 0.014983262866735458, "state_peak": 45.51605987548828, "growth_active": 1.145276665687561, "growth_memory": 1.2957994937896729, "heads": [{"head": 0, "memory_to_active": 0.3259628117084503, "contribution_to_active": 0.17158061265945435, "mix_mean": 0.18126556277275085, "mix_low": 0.020868301391601562, "mix_high": 0.0, "read_cosine": 0.8145560026168823, "beta_mean": 0.057998739182949066, "nonfinite": 0.0, "weighted_memory_to_active": 0.06654585152864456, "mix_std": 0.17609688639640808, "beta_std": 0.05345027148723602, "beta_low": 0.0174560546875, "beta_high": 0.0, "orthogonal_to_active": 0.06459340453147888, "evidence_weights": [-0.11385796219110489, -0.03934271261096001, 0.435955673456192], "agreement_weight": 0.024299325421452522, "orthogonal_weight": 1.655240774154663}, {"head": 1, "memory_to_active": 0.487261563539505, "contribution_to_active": 0.2906944751739502, "mix_mean": 0.2189735621213913, "mix_low": 0.118194580078125, "mix_high": 0.0, "read_cosine": 0.8608068823814392, "beta_mean": 0.23123201727867126, "nonfinite": 0.0, "weighted_memory_to_active": 0.2159714698791504, "mix_std": 0.24746301770210266, "beta_std": 0.23474116623401642, "beta_low": 0.11285400390625, "beta_high": 0.0, "orthogonal_to_active": 0.013705301098525524, "evidence_weights": [-0.0041275243274867535, -0.19752155244350433, -0.09362533688545227], "agreement_weight": 0.05802810564637184, "orthogonal_weight": 0.1705753654241562}, {"head": 2, "memory_to_active": 0.12665820121765137, "contribution_to_active": 0.5797175765037537, "mix_mean": 0.587811291217804, "mix_low": 0.0030117034912109375, "mix_high": 0.003253936767578125, "read_cosine": 0.41811442375183105, "beta_mean": 0.0018692833837121725, "nonfinite": 0.0, "weighted_memory_to_active": 0.0829399973154068, "mix_std": 0.26649904251098633, "beta_std": 0.0024726386182010174, "beta_low": 0.983154296875, "beta_high": 0.0, "orthogonal_to_active": 0.0638035237789154, "evidence_weights": [-0.14244985580444336, -0.3169397711753845, 0.9226239919662476], "agreement_weight": 0.08316905051469803, "orthogonal_weight": -0.880174458026886}, {"head": 3, "memory_to_active": 0.047098852694034576, "contribution_to_active": 0.5952007174491882, "mix_mean": 0.6704959869384766, "mix_low": 0.004558563232421875, "mix_high": 0.01999378204345703, "read_cosine": -0.020916670560836792, "beta_mean": 0.006207088008522987, "nonfinite": 0.0, "weighted_memory_to_active": 0.029456667602062225, "mix_std": 0.25608542561531067, "beta_std": 0.008215007372200489, "beta_low": 0.81829833984375, "beta_high": 0.0, "orthogonal_to_active": 0.021395981311798096, "evidence_weights": [-0.3478155732154846, -0.3108455538749695, 0.2698608934879303], "agreement_weight": 0.0722513422369957, "orthogonal_weight": -0.8161762356758118}, {"head": 4, "memory_to_active": 0.15939341485500336, "contribution_to_active": 0.49904632568359375, "mix_mean": 0.5066460371017456, "mix_low": 0.004132270812988281, "mix_high": 0.00849151611328125, "read_cosine": 0.8896901607513428, "beta_mean": 0.03913215547800064, "nonfinite": 0.0, "weighted_memory_to_active": 0.09390540421009064, "mix_std": 0.2625344693660736, "beta_std": 0.027956247329711914, "beta_low": 0.0806884765625, "beta_high": 0.0, "orthogonal_to_active": 0.02490326389670372, "evidence_weights": [-0.2158481925725937, -0.2434002012014389, 0.42656707763671875], "agreement_weight": 0.1327451914548874, "orthogonal_weight": -0.6099427938461304}, {"head": 5, "memory_to_active": 0.19726930558681488, "contribution_to_active": 0.33012205362319946, "mix_mean": 0.37375327944755554, "mix_low": 0.004199981689453125, "mix_high": 0.00205230712890625, "read_cosine": 0.7617365717887878, "beta_mean": 0.0014128413749858737, "nonfinite": 0.0, "weighted_memory_to_active": 0.07469718158245087, "mix_std": 0.2567804753780365, "beta_std": 0.007767210248857737, "beta_low": 0.97894287109375, "beta_high": 0.0, "orthogonal_to_active": 0.036001384258270264, "evidence_weights": [-0.17565926909446716, -0.1721898913383484, 0.3005158305168152], "agreement_weight": 0.07573816925287247, "orthogonal_weight": -0.7388545870780945}]}, {"layer": 15, "memory_to_active": 0.05287124216556549, "contribution_to_active": 0.48267561197280884, "mix_mean": 0.4151903986930847, "mix_low": 0.007510344497859478, "mix_high": 0.0026715598069131374, "read_cosine": 0.6756957769393921, "beta_mean": 0.03692459315061569, "nonfinite": 0.0, "weighted_memory_to_active": 0.02571246027946472, "mix_std": 0.284663587808609, "beta_std": 0.0447595939040184, "beta_low": 0.7168274521827698, "beta_high": 0.0, "orthogonal_to_active": 0.00677420012652874, "output_influence": 0.3051874041557312, "residual_influence": 0.17555606365203857, "state_active": 0.16048681735992432, "state_memory": 0.010594801977276802, "state_peak": 33.81733322143555, "growth_active": 1.2969788312911987, "growth_memory": 1.2597873210906982, "heads": [{"head": 0, "memory_to_active": 0.09462351351976395, "contribution_to_active": 0.49423229694366455, "mix_mean": 0.3882905840873718, "mix_low": 0.0064296722412109375, "mix_high": 0.006285667419433594, "read_cosine": 0.6622315645217896, "beta_mean": 0.03310157358646393, "nonfinite": 0.0, "weighted_memory_to_active": 0.04763330891728401, "mix_std": 0.28462207317352295, "beta_std": 0.02697180211544037, "beta_low": 0.135498046875, "beta_high": 0.0, "orthogonal_to_active": 0.021552706137299538, "evidence_weights": [-0.19138845801353455, -0.19392043352127075, 0.4348990321159363], "agreement_weight": 0.09266842901706696, "orthogonal_weight": -0.7569580078125}, {"head": 1, "memory_to_active": 0.11063414812088013, "contribution_to_active": 0.4102482199668884, "mix_mean": 0.4256264269351959, "mix_low": 0.0016994476318359375, "mix_high": 0.0049457550048828125, "read_cosine": 0.9284448623657227, "beta_mean": 0.0006065073539502919, "nonfinite": 0.0, "weighted_memory_to_active": 0.04571957886219025, "mix_std": 0.28688642382621765, "beta_std": 0.0018420862033963203, "beta_low": 0.9947509765625, "beta_high": 0.0, "orthogonal_to_active": 0.0011173976818099618, "evidence_weights": [-0.14636634290218353, -0.1556597650051117, 0.1791476458311081], "agreement_weight": 0.12542259693145752, "orthogonal_weight": 0.049911148846149445}, {"head": 2, "memory_to_active": 0.005645143799483776, "contribution_to_active": 0.4822452664375305, "mix_mean": 0.34359031915664673, "mix_low": 0.011998176574707031, "mix_high": 9.5367431640625e-05, "read_cosine": 0.7717930674552917, "beta_mean": 8.702396735316142e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002872304292395711, "mix_std": 0.26812034845352173, "beta_std": 0.00018067657947540283, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007632679189555347, "evidence_weights": [-0.2089066207408905, -0.26812365651130676, 0.13762639462947845], "agreement_weight": 0.008617833256721497, "orthogonal_weight": 0.7174580693244934}, {"head": 3, "memory_to_active": 0.001607500365935266, "contribution_to_active": 0.3937567174434662, "mix_mean": 0.4034796953201294, "mix_low": 0.005335807800292969, "mix_high": 0.0004138946533203125, "read_cosine": 0.630818247795105, "beta_mean": 9.090209459827747e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.000538797932676971, "mix_std": 0.29034945368766785, "beta_std": 1.8911214283434674e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004522843228187412, "evidence_weights": [-0.261134535074234, -0.24593377113342285, 0.19707421958446503], "agreement_weight": 0.04817240685224533, "orthogonal_weight": -0.8021408319473267}, {"head": 4, "memory_to_active": 0.07929347455501556, "contribution_to_active": 0.5968921780586243, "mix_mean": 0.3674995005130768, "mix_low": 0.009114265441894531, "mix_high": 0.0008754730224609375, "read_cosine": 0.7421002388000488, "beta_mean": 0.1850452721118927, "nonfinite": 0.0, "weighted_memory_to_active": 0.0413968525826931, "mix_std": 0.3038955628871918, "beta_std": 0.21957039833068848, "beta_low": 0.20367431640625, "beta_high": 0.0, "orthogonal_to_active": 0.00687661487609148, "evidence_weights": [-0.2089061737060547, -0.2525385618209839, -0.007085273042321205], "agreement_weight": 0.013383766636252403, "orthogonal_weight": 0.3538290858268738}, {"head": 5, "memory_to_active": 0.025423649698495865, "contribution_to_active": 0.5186788439750671, "mix_mean": 0.5626558065414429, "mix_low": 0.010484695434570312, "mix_high": 0.0034132003784179688, "read_cosine": 0.3187864124774933, "beta_mean": 0.0026980694383382797, "nonfinite": 0.0, "weighted_memory_to_active": 0.01611390896141529, "mix_std": 0.2741076946258545, "beta_std": 0.019973676651716232, "beta_low": 0.967041015625, "beta_high": 0.0, "orthogonal_to_active": 0.009882926940917969, "evidence_weights": [-0.28720393776893616, -0.3031159043312073, 0.6923196315765381], "agreement_weight": 0.30438101291656494, "orthogonal_weight": 0.786647617816925}]}]}, {"step": 53000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.9051603230000183, "memory_to_active": 0.8115896582603455, "contribution_to_active": 0.5575436353683472, "mix_mean": 0.4078674912452698, "mix_low": 0.1338222622871399, "mix_high": 0.01032158825546503, "read_cosine": 0.3511311411857605, "beta_mean": 0.07195674628019333, "nonfinite": 0.0, "weighted_memory_to_active": 0.21092361211776733, "mix_std": 0.22251629829406738, "beta_std": 0.06293120980262756, "beta_low": 0.5577304363250732, "beta_high": 2.86102294921875e-05, "orthogonal_to_active": 0.06076742708683014, "samples": 4, "scored_tokens": 899, "normal_loss": 3.115450382232666, "normal_token_accuracy": 0.44160178303718567, "state_active": 0.06326642632484436, "state_memory": 0.018208729103207588, "state_peak": 123.58769989013672, "growth_active": 1.1239346265792847, "growth_memory": 1.3168456554412842, "output_influence": 0.42040762305259705, "residual_influence": 0.14209002256393433, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.274468421936035, "no_memory_loss_delta": 0.48778197169303894, "scrambled_loss_delta": 0.1287621706724167, "memory_only_loss_delta": 3.640528917312622, "no_evidence_loss_delta": 0.281352698802948, "no_agreement_loss_delta": 0.03187629207968712, "no_orthogonal_loss_delta": -0.002282259054481983}, {"index": 94, "tokens": 256, "normal_loss": 3.2445273399353027, "no_memory_loss_delta": 0.6587991714477539, "scrambled_loss_delta": 0.36390578746795654, "memory_only_loss_delta": 3.4525763988494873, "no_evidence_loss_delta": 0.3147328197956085, "no_agreement_loss_delta": 0.03988398611545563, "no_orthogonal_loss_delta": 0.07753070443868637}, {"index": 188, "tokens": 256, "normal_loss": 3.4099416732788086, "no_memory_loss_delta": 0.6869189143180847, "scrambled_loss_delta": 0.13388212025165558, "memory_only_loss_delta": 3.3911962509155273, "no_evidence_loss_delta": 0.349808007478714, "no_agreement_loss_delta": 0.03796935826539993, "no_orthogonal_loss_delta": 0.03634896129369736}, {"index": 283, "tokens": 131, "normal_loss": 1.976959228515625, "no_memory_loss_delta": 0.46005603671073914, "scrambled_loss_delta": 0.24748879671096802, "memory_only_loss_delta": 2.822542667388916, "no_evidence_loss_delta": 0.21515034139156342, "no_agreement_loss_delta": -0.0045426590368151665, "no_orthogonal_loss_delta": -0.016310203820466995}], "agreement_weight_abs_mean": 0.20963680744171143, "agreement_weight_abs_max": 5.633092880249023, "agreement_weight_mean": 0.18381071090698242, "orthogonal_weight_abs_mean": 0.8520500063896179, "orthogonal_weight_abs_max": 9.022316932678223, "orthogonal_weight_mean": -0.1164267435669899, "no_memory_loss": 3.7045974731445312, "no_memory_loss_delta": 0.5891472697257996, "no_memory_token_accuracy_gain": 0.0767519474029541, "no_memory_token_win_fraction": 0.6718575954437256, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 44.51998220114427, "scrambled_loss": 3.3299310207366943, "scrambled_loss_delta": 0.21448038518428802, "scrambled_token_accuracy_gain": 0.022246941924095154, "scrambled_token_win_fraction": 0.6129032373428345, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.303936003909413, "memory_only_loss": 6.51226282119751, "memory_only_loss_delta": 3.3968124389648438, "memory_only_token_accuracy_gain": 0.2714126706123352, "memory_only_token_win_fraction": 0.8998887538909912, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.65201809791321, "no_evidence_loss": 3.4161548614501953, "no_evidence_loss_delta": 0.30070459842681885, "no_evidence_token_accuracy_gain": 0.04004449397325516, "no_evidence_token_win_fraction": 0.6129032373428345, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 25.970357482140038, "no_agreement_loss": 3.1460349559783936, "no_agreement_loss_delta": 0.03058476187288761, "no_agreement_token_accuracy_gain": 0.0, "no_agreement_token_win_fraction": 0.47385984659194946, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 3.012178011136003, "no_orthogonal_loss": 3.1448521614074707, "no_orthogonal_loss_delta": 0.029401889070868492, "no_orthogonal_token_accuracy_gain": -0.006674082484096289, "no_orthogonal_token_win_fraction": 0.5873192548751831, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 2.8973858755153965, "short_no_memory_loss_delta": 0.533109724521637, "long_context_memory_gain": 0.0560375452041626, "normal_context_loss_gain": 0.14556752145290375, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.5499799251556396, "contribution_to_active": 1.2072772979736328, "mix_mean": 0.105815589427948, "mix_low": 0.5312544703483582, "mix_high": 0.0052310628816485405, "read_cosine": 0.4248914122581482, "beta_mean": 0.22451478242874146, "nonfinite": 0.0, "weighted_memory_to_active": 1.2390331029891968, "mix_std": 0.09073644876480103, "beta_std": 0.1426626294851303, "beta_low": 0.0623881034553051, "beta_high": 0.0, "orthogonal_to_active": 0.03977398946881294, "output_influence": 0.21472638845443726, "residual_influence": 0.2521441578865051, "state_active": 0.06414234638214111, "state_memory": 0.04548804089426994, "state_peak": 46.48468780517578, "growth_active": 1.2179768085479736, "growth_memory": 1.2747913599014282, "heads": [{"head": 0, "memory_to_active": 2.689542531967163, "contribution_to_active": 0.507498562335968, "mix_mean": 0.0209854356944561, "mix_low": 0.6138362884521484, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21893428266048431, "beta_mean": 0.2118138074874878, "nonfinite": 0.0, "weighted_memory_to_active": 0.6216011643409729, "mix_std": 0.06247855722904205, "beta_std": 0.11223739385604858, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.059954844415187836, "evidence_weights": [-0.009703694842755795, -0.05444607138633728, -0.06986202299594879], "agreement_weight": -0.19430144131183624, "orthogonal_weight": -0.10814541578292847}, {"head": 1, "memory_to_active": 6.409363746643066, "contribution_to_active": 2.751797914505005, "mix_mean": 0.0201213750988245, "mix_low": 0.6222085952758789, "mix_high": 0.0, "read_cosine": 0.4240233898162842, "beta_mean": 0.4765079617500305, "nonfinite": 0.0, "weighted_memory_to_active": 3.0284807682037354, "mix_std": 0.05774020776152611, "beta_std": 0.20258508622646332, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012882850132882595, "evidence_weights": [0.021650822833180428, -0.10166706889867783, 0.576893150806427], "agreement_weight": 0.027709340676665306, "orthogonal_weight": 0.008911198005080223}, {"head": 2, "memory_to_active": 2.3245279788970947, "contribution_to_active": 0.418809175491333, "mix_mean": 0.014148067682981491, "mix_low": 0.6670770645141602, "mix_high": 0.0, "read_cosine": 0.32926470041275024, "beta_mean": 0.2389616072177887, "nonfinite": 0.0, "weighted_memory_to_active": 0.5019736289978027, "mix_std": 0.03201325982809067, "beta_std": 0.18402883410453796, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03778665140271187, "evidence_weights": [0.06443648785352707, 0.056661780923604965, 0.1394004076719284], "agreement_weight": 0.016097035259008408, "orthogonal_weight": -0.20514880120754242}, {"head": 3, "memory_to_active": 7.8362908363342285, "contribution_to_active": 3.0237808227539062, "mix_mean": 0.0308532752096653, "mix_low": 0.5118427276611328, "mix_high": 0.0, "read_cosine": 0.31029558181762695, "beta_mean": 0.32218047976493835, "nonfinite": 0.0, "weighted_memory_to_active": 3.1693692207336426, "mix_std": 0.07502991706132889, "beta_std": 0.23472651839256287, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10755594819784164, "evidence_weights": [-0.04265840724110603, -0.04285498708486557, -0.1535397469997406], "agreement_weight": -0.015667175874114037, "orthogonal_weight": -0.03994042053818703}, {"head": 4, "memory_to_active": 1.9270410537719727, "contribution_to_active": 0.07784448564052582, "mix_mean": 0.010481094941496849, "mix_low": 0.7571821212768555, "mix_high": 0.0, "read_cosine": 0.3205143213272095, "beta_mean": 0.0762876570224762, "nonfinite": 0.0, "weighted_memory_to_active": 0.054464131593704224, "mix_std": 0.030155520886182785, "beta_std": 0.1048462986946106, "beta_low": 0.093505859375, "beta_high": 0.0, "orthogonal_to_active": 0.004646795801818371, "evidence_weights": [0.142604261636734, 0.1026645079255104, -0.16537423431873322], "agreement_weight": -0.13951988518238068, "orthogonal_weight": 0.07075046002864838}, {"head": 5, "memory_to_active": 0.11311233788728714, "contribution_to_active": 0.4639317989349365, "mix_mean": 0.5383042693138123, "mix_low": 0.015379905700683594, "mix_high": 0.03138542175292969, "read_cosine": 0.9463163018226624, "beta_mean": 0.02133706584572792, "nonfinite": 0.0, "weighted_memory_to_active": 0.05830932408571243, "mix_std": 0.2870011329650879, "beta_std": 0.017551567405462265, "beta_low": 0.28082275390625, "beta_high": 0.0, "orthogonal_to_active": 0.015816830098628998, "evidence_weights": [-0.18253298103809357, -0.21916460990905762, 0.3793164789676666], "agreement_weight": 0.2916722595691681, "orthogonal_weight": -0.6725156307220459}]}, {"layer": 1, "memory_to_active": 2.192261219024658, "contribution_to_active": 0.44155603647232056, "mix_mean": 0.20756995677947998, "mix_low": 0.3673340678215027, "mix_high": 0.0009667078847996891, "read_cosine": 0.3043580651283264, "beta_mean": 0.1683661788702011, "nonfinite": 0.0, "weighted_memory_to_active": 0.2816929519176483, "mix_std": 0.11551684141159058, "beta_std": 0.12277361750602722, "beta_low": 0.3489176630973816, "beta_high": 0.00010172526526730508, "orthogonal_to_active": 0.0433463379740715, "output_influence": 0.23557254672050476, "residual_influence": 0.061651214957237244, "state_active": 0.02999185398221016, "state_memory": 0.03525504097342491, "state_peak": 37.59505081176758, "growth_active": 1.2101510763168335, "growth_memory": 1.4917552471160889, "heads": [{"head": 0, "memory_to_active": 0.05848246067762375, "contribution_to_active": 0.4852476716041565, "mix_mean": 0.5013454556465149, "mix_low": 0.006558418273925781, "mix_high": 0.0017118453979492188, "read_cosine": 0.026600608602166176, "beta_mean": 0.00034172425512224436, "nonfinite": 0.0, "weighted_memory_to_active": 0.03700179606676102, "mix_std": 0.282296359539032, "beta_std": 0.0005737671162933111, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.03496228903532028, "evidence_weights": [-0.2054366171360016, -0.26703521609306335, 0.3474613130092621], "agreement_weight": 0.19828830659389496, "orthogonal_weight": -1.00314199924469}, {"head": 1, "memory_to_active": 2.782151937484741, "contribution_to_active": 0.24194800853729248, "mix_mean": 0.024152837693691254, "mix_low": 0.4534568786621094, "mix_high": 1.9073486328125e-06, "read_cosine": 0.5041588544845581, "beta_mean": 0.2668737769126892, "nonfinite": 0.0, "weighted_memory_to_active": 0.27676093578338623, "mix_std": 0.04720078781247139, "beta_std": 0.17638787627220154, "beta_low": 0.00103759765625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.019173815846443176, "evidence_weights": [0.014329180121421814, -0.05355631560087204, 0.051903821527957916], "agreement_weight": -0.0016798394499346614, "orthogonal_weight": 0.17416317760944366}, {"head": 2, "memory_to_active": 0.060548607259988785, "contribution_to_active": 0.5567585825920105, "mix_mean": 0.6601250171661377, "mix_low": 0.0005359649658203125, "mix_high": 0.004086494445800781, "read_cosine": 0.47532010078430176, "beta_mean": 0.0024255765601992607, "nonfinite": 0.0, "weighted_memory_to_active": 0.04165439307689667, "mix_std": 0.23447981476783752, "beta_std": 0.004522187635302544, "beta_low": 0.9708251953125, "beta_high": 0.0, "orthogonal_to_active": 0.027021929621696472, "evidence_weights": [-0.28452885150909424, -0.36838310956954956, 0.664989709854126], "agreement_weight": 0.038194380700588226, "orthogonal_weight": -0.7824471592903137}, {"head": 3, "memory_to_active": 2.9158682823181152, "contribution_to_active": 0.15326499938964844, "mix_mean": 0.020909268409013748, "mix_low": 0.6019392013549805, "mix_high": 0.0, "read_cosine": 0.202309250831604, "beta_mean": 0.3229342997074127, "nonfinite": 0.0, "weighted_memory_to_active": 0.21439191699028015, "mix_std": 0.05683920159935951, "beta_std": 0.2541050910949707, "beta_low": 0.05474853515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08112072944641113, "evidence_weights": [0.045512862503528595, -0.01101668830960989, -0.017177239060401917], "agreement_weight": -0.026047969236969948, "orthogonal_weight": -0.47100043296813965}, {"head": 4, "memory_to_active": 3.641084909439087, "contribution_to_active": 1.133967399597168, "mix_mean": 0.031703367829322815, "mix_low": 0.31770992279052734, "mix_high": 0.0, "read_cosine": 0.34669986367225647, "beta_mean": 0.17184913158416748, "nonfinite": 0.0, "weighted_memory_to_active": 1.0655734539031982, "mix_std": 0.0547294020652771, "beta_std": 0.11742118000984192, "beta_low": 0.034423828125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.08266708254814148, "evidence_weights": [-0.028993763029575348, -0.00039435955113731325, 0.07981736958026886], "agreement_weight": -0.01225592102855444, "orthogonal_weight": 0.10694033652544022}, {"head": 5, "memory_to_active": 3.695430040359497, "contribution_to_active": 0.07814974337816238, "mix_mean": 0.007183812092989683, "mix_low": 0.8238039016723633, "mix_high": 0.0, "read_cosine": 0.2710596024990082, "beta_mean": 0.245772585272789, "nonfinite": 0.0, "weighted_memory_to_active": 0.054775040596723557, "mix_std": 0.017555424943566322, "beta_std": 0.18363159894943237, "beta_low": 0.03265380859375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.015132183209061623, "evidence_weights": [0.15787066519260406, 0.07866154611110687, 0.1495051234960556], "agreement_weight": -0.11607827991247177, "orthogonal_weight": 0.2825314998626709}]}, {"layer": 2, "memory_to_active": 1.0366616249084473, "contribution_to_active": 0.3487313687801361, "mix_mean": 0.39602166414260864, "mix_low": 0.2465057373046875, "mix_high": 0.0045140585862100124, "read_cosine": 0.2933257818222046, "beta_mean": 0.0945327952504158, "nonfinite": 0.0, "weighted_memory_to_active": 0.08294133841991425, "mix_std": 0.19162653386592865, "beta_std": 0.0720803290605545, "beta_low": 0.5603841543197632, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.022476961836218834, "output_influence": 0.37259751558303833, "residual_influence": 0.09907115250825882, "state_active": 0.038771338760852814, "state_memory": 0.016554202884435654, "state_peak": 31.847501754760742, "growth_active": 1.0568078756332397, "growth_memory": 1.5528197288513184, "heads": [{"head": 0, "memory_to_active": 0.012753116898238659, "contribution_to_active": 0.3246859908103943, "mix_mean": 0.617303192615509, "mix_low": 0.0006475448608398438, "mix_high": 0.009011268615722656, "read_cosine": 0.12400532513856888, "beta_mean": 8.592929953010753e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008103636093437672, "mix_std": 0.2760913372039795, "beta_std": 0.00015384869766421616, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008134759031236172, "evidence_weights": [-0.36581388115882874, -0.21430005133152008, 0.11822659522294998], "agreement_weight": 0.12694908678531647, "orthogonal_weight": -1.0454317331314087}, {"head": 1, "memory_to_active": 2.9741885662078857, "contribution_to_active": 0.04647846519947052, "mix_mean": 0.005079346708953381, "mix_low": 0.8923835754394531, "mix_high": 0.0, "read_cosine": 0.14318212866783142, "beta_mean": 0.24654743075370789, "nonfinite": 0.0, "weighted_memory_to_active": 0.03527846187353134, "mix_std": 0.01125936210155487, "beta_std": 0.1526418775320053, "beta_low": 0.0006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.011954424902796745, "evidence_weights": [0.13806723058223724, 0.121867835521698, 0.5413186550140381], "agreement_weight": 0.028998197987675667, "orthogonal_weight": 0.36752817034721375}, {"head": 2, "memory_to_active": 0.010073717683553696, "contribution_to_active": 0.44717687368392944, "mix_mean": 0.5385090112686157, "mix_low": 0.0011930465698242188, "mix_high": 0.0033521652221679688, "read_cosine": 0.32417213916778564, "beta_mean": 2.7514499379321933e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00400264747440815, "mix_std": 0.27802789211273193, "beta_std": 4.238892870489508e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003208643465768546, "evidence_weights": [-0.25161078572273254, -0.2925027906894684, 0.5863360166549683], "agreement_weight": -0.011945233680307865, "orthogonal_weight": -0.09102902561426163}, {"head": 3, "memory_to_active": 0.16129785776138306, "contribution_to_active": 0.5124711990356445, "mix_mean": 0.5584367513656616, "mix_low": 0.0002307891845703125, "mix_high": 0.007046699523925781, "read_cosine": 0.5878337621688843, "beta_mean": 0.03202032297849655, "nonfinite": 0.0, "weighted_memory_to_active": 0.09977854788303375, "mix_std": 0.26152294874191284, "beta_std": 0.040296610444784164, "beta_low": 0.33642578125, "beta_high": 0.0, "orthogonal_to_active": 0.06887008249759674, "evidence_weights": [-0.2003302127122879, -0.31575965881347656, 0.8261193633079529], "agreement_weight": 0.05564115568995476, "orthogonal_weight": -0.8932267427444458}, {"head": 4, "memory_to_active": 3.045314073562622, "contribution_to_active": 0.24433164298534393, "mix_mean": 0.021183054894208908, "mix_low": 0.5836391448974609, "mix_high": 7.62939453125e-06, "read_cosine": 0.20509864389896393, "beta_mean": 0.28847503662109375, "nonfinite": 0.0, "weighted_memory_to_active": 0.34096479415893555, "mix_std": 0.056030988693237305, "beta_std": 0.23927149176597595, "beta_low": 0.0252685546875, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.0442420095205307, "evidence_weights": [0.021422192454338074, 0.00647358363494277, 0.015872182324528694], "agreement_weight": 0.014269404113292694, "orthogonal_weight": -0.1552785485982895}, {"head": 5, "memory_to_active": 0.016341732814908028, "contribution_to_active": 0.5172441005706787, "mix_mean": 0.6356184482574463, "mix_low": 0.0009403228759765625, "mix_high": 0.007666587829589844, "read_cosine": 0.37566256523132324, "beta_mean": 4.0501676267012954e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009519890882074833, "mix_std": 0.2668266296386719, "beta_std": 7.574402843602002e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013396138092502952, "evidence_weights": [-0.2812182903289795, -0.29197943210601807, 1.0453852415084839], "agreement_weight": 0.15305522084236145, "orthogonal_weight": -0.16517314314842224}]}, {"layer": 3, "memory_to_active": 0.4576868712902069, "contribution_to_active": 0.4660719633102417, "mix_mean": 0.4886946976184845, "mix_low": 0.05849504470825195, "mix_high": 0.01101652905344963, "read_cosine": 0.20555928349494934, "beta_mean": 0.027082184329628944, "nonfinite": 0.0, "weighted_memory_to_active": 0.13401523232460022, "mix_std": 0.26349666714668274, "beta_std": 0.026237444952130318, "beta_low": 0.7577921748161316, "beta_high": 0.0, "orthogonal_to_active": 0.0542171485722065, "output_influence": 0.3991434872150421, "residual_influence": 0.0895954817533493, "state_active": 0.02458794228732586, "state_memory": 0.007431133650243282, "state_peak": 21.152692794799805, "growth_active": 1.025760293006897, "growth_memory": 1.6379382610321045, "heads": [{"head": 0, "memory_to_active": 0.39217859506607056, "contribution_to_active": 0.5923512578010559, "mix_mean": 0.5914813280105591, "mix_low": 0.0022096633911132812, "mix_high": 0.025636672973632812, "read_cosine": 0.24886538088321686, "beta_mean": 0.009642954915761948, "nonfinite": 0.0, "weighted_memory_to_active": 0.24354156851768494, "mix_std": 0.3073473274707794, "beta_std": 0.010959362611174583, "beta_low": 0.67724609375, "beta_high": 0.0, "orthogonal_to_active": 0.1719689965248108, "evidence_weights": [-0.21783387660980225, -0.251572847366333, 0.9503236413002014], "agreement_weight": 0.12243722379207611, "orthogonal_weight": -0.7489954829216003}, {"head": 1, "memory_to_active": 0.013696737587451935, "contribution_to_active": 0.4068780541419983, "mix_mean": 0.5515425205230713, "mix_low": 0.005236625671386719, "mix_high": 0.008544921875, "read_cosine": 0.17642097175121307, "beta_mean": 7.382913463516161e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007634800858795643, "mix_std": 0.3043433129787445, "beta_std": 0.0002096843090839684, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006025799084454775, "evidence_weights": [-0.2606607973575592, -0.21266745030879974, 0.649429976940155], "agreement_weight": 0.4851100742816925, "orthogonal_weight": -0.8690028190612793}, {"head": 2, "memory_to_active": 2.0736989974975586, "contribution_to_active": 0.30396565794944763, "mix_mean": 0.0518970713019371, "mix_low": 0.3273906707763672, "mix_high": 0.0, "read_cosine": 0.21017354726791382, "beta_mean": 0.14685028791427612, "nonfinite": 0.0, "weighted_memory_to_active": 0.3613792061805725, "mix_std": 0.10826513916254044, "beta_std": 0.13586357235908508, "beta_low": 0.0206298828125, "beta_high": 0.0, "orthogonal_to_active": 0.005857521668076515, "evidence_weights": [-0.09219995886087418, 0.0008627797942608595, 0.009013774804770947], "agreement_weight": -0.007336881943047047, "orthogonal_weight": -0.020929064601659775}, {"head": 3, "memory_to_active": 0.01663838140666485, "contribution_to_active": 0.512852132320404, "mix_mean": 0.5942664742469788, "mix_low": 0.00528717041015625, "mix_high": 0.013345718383789062, "read_cosine": 0.29794639348983765, "beta_mean": 7.846593507565558e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011587212793529034, "mix_std": 0.2953683137893677, "beta_std": 0.00016656090156175196, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002991052344441414, "evidence_weights": [-0.24822208285331726, -0.2308797985315323, 2.1670870780944824], "agreement_weight": 1.9710352420806885, "orthogonal_weight": -0.344705194234848}, {"head": 4, "memory_to_active": 0.015906132757663727, "contribution_to_active": 0.33608344197273254, "mix_mean": 0.4374086558818817, "mix_low": 0.008162498474121094, "mix_high": 0.002353668212890625, "read_cosine": 0.1823480874300003, "beta_mean": 9.777466766536236e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009152961894869804, "mix_std": 0.2908816933631897, "beta_std": 0.0001401025365339592, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008671276271343231, "evidence_weights": [-0.19635257124900818, -0.19792325794696808, 0.10437969863414764], "agreement_weight": -0.03735706955194473, "orthogonal_weight": -1.002401351928711}, {"head": 5, "memory_to_active": 0.23400241136550903, "contribution_to_active": 0.6443009972572327, "mix_mean": 0.7055718898773193, "mix_low": 0.0026836395263671875, "mix_high": 0.016218185424804688, "read_cosine": 0.11760125309228897, "beta_mean": 0.005749784409999847, "nonfinite": 0.0, "weighted_memory_to_active": 0.17079561948776245, "mix_std": 0.27477407455444336, "beta_std": 0.010085392743349075, "beta_low": 0.84893798828125, "beta_high": 0.0, "orthogonal_to_active": 0.12978824973106384, "evidence_weights": [-0.26049596071243286, -0.329100638628006, 0.4841543138027191], "agreement_weight": 0.1594386249780655, "orthogonal_weight": -0.7924656867980957}]}, {"layer": 4, "memory_to_active": 1.094421625137329, "contribution_to_active": 0.4643915891647339, "mix_mean": 0.3670629858970642, "mix_low": 0.2678740918636322, "mix_high": 0.010807832702994347, "read_cosine": 0.3201846778392792, "beta_mean": 0.11849255114793777, "nonfinite": 0.0, "weighted_memory_to_active": 0.13850198686122894, "mix_std": 0.15956953167915344, "beta_std": 0.11176621913909912, "beta_low": 0.3828938901424408, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.0467197559773922, "output_influence": 0.47158390283584595, "residual_influence": 0.13800746202468872, "state_active": 0.04007915407419205, "state_memory": 0.01760799251496792, "state_peak": 37.313961029052734, "growth_active": 1.2423499822616577, "growth_memory": 1.4920895099639893, "heads": [{"head": 0, "memory_to_active": 0.18894484639167786, "contribution_to_active": 0.7630012035369873, "mix_mean": 0.684943675994873, "mix_low": 0.012746810913085938, "mix_high": 0.025437355041503906, "read_cosine": 0.487988144159317, "beta_mean": 0.273744136095047, "nonfinite": 0.0, "weighted_memory_to_active": 0.1572733372449875, "mix_std": 0.3443918824195862, "beta_std": 0.2747201919555664, "beta_low": 0.2442626953125, "beta_high": 0.0, "orthogonal_to_active": 0.0701138973236084, "evidence_weights": [-0.26875969767570496, -0.37001150846481323, 0.019162887707352638], "agreement_weight": 0.00010325774201191962, "orthogonal_weight": 0.5688445568084717}, {"head": 1, "memory_to_active": 0.05615080147981644, "contribution_to_active": 0.6534897089004517, "mix_mean": 0.6930574178695679, "mix_low": 0.00013446807861328125, "mix_high": 0.017578125, "read_cosine": 0.6374887228012085, "beta_mean": 0.0010402349289506674, "nonfinite": 0.0, "weighted_memory_to_active": 0.0410304069519043, "mix_std": 0.24081268906593323, "beta_std": 0.006940718740224838, "beta_low": 0.9810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.022112902253866196, "evidence_weights": [-0.24264009296894073, -0.3603311777114868, 0.6586247086524963], "agreement_weight": 0.28496333956718445, "orthogonal_weight": -0.7759368419647217}, {"head": 2, "memory_to_active": 1.8611030578613281, "contribution_to_active": 0.2947339415550232, "mix_mean": 0.030279293656349182, "mix_low": 0.5041036605834961, "mix_high": 2.384185791015625e-05, "read_cosine": 0.24360108375549316, "beta_mean": 0.0735732689499855, "nonfinite": 0.0, "weighted_memory_to_active": 0.2663564383983612, "mix_std": 0.06611620634794235, "beta_std": 0.07593165338039398, "beta_low": 0.0582275390625, "beta_high": 0.0, "orthogonal_to_active": 0.060681331902742386, "evidence_weights": [0.05269550904631615, -0.004202892538160086, 0.16770517826080322], "agreement_weight": 0.140961155295372, "orthogonal_weight": -0.32908734679222107}, {"head": 3, "memory_to_active": 2.1170642375946045, "contribution_to_active": 0.15464864671230316, "mix_mean": 0.031595177948474884, "mix_low": 0.42704105377197266, "mix_high": 0.0, "read_cosine": 0.18689832091331482, "beta_mean": 0.15830431878566742, "nonfinite": 0.0, "weighted_memory_to_active": 0.20450659096240997, "mix_std": 0.05723611265420914, "beta_std": 0.13775372505187988, "beta_low": 0.0137939453125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.00020204478641971946, "evidence_weights": [-0.022492526099085808, 0.051544301211833954, -0.27138400077819824], "agreement_weight": 0.0817529484629631, "orthogonal_weight": -0.0014117676764726639}, {"head": 4, "memory_to_active": 0.015787944197654724, "contribution_to_active": 0.6492007374763489, "mix_mean": 0.7487691640853882, "mix_low": 6.580352783203125e-05, "mix_high": 0.021800994873046875, "read_cosine": 0.12582406401634216, "beta_mean": 2.9103553970344365e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011837530881166458, "mix_std": 0.21589446067810059, "beta_std": 9.12628456717357e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005819729529321194, "evidence_weights": [-0.33874499797821045, -0.3288978934288025, 0.5197017788887024], "agreement_weight": 0.3446943759918213, "orthogonal_weight": -0.5318709015846252}, {"head": 5, "memory_to_active": 2.3274784088134766, "contribution_to_active": 0.27127522230148315, "mix_mean": 0.013733191415667534, "mix_low": 0.6631526947021484, "mix_high": 6.67572021484375e-06, "read_cosine": 0.23930758237838745, "beta_mean": 0.20426425337791443, "nonfinite": 0.0, "weighted_memory_to_active": 0.15000760555267334, "mix_std": 0.03296579793095589, "beta_std": 0.1751597374677658, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12138861417770386, "evidence_weights": [0.05209110304713249, -0.0260433629155159, -0.2686639428138733], "agreement_weight": -0.07742787897586823, "orthogonal_weight": 0.9651889204978943}]}, {"layer": 5, "memory_to_active": 0.4340343177318573, "contribution_to_active": 0.5338600277900696, "mix_mean": 0.5003910064697266, "mix_low": 0.06076844781637192, "mix_high": 0.018707118928432465, "read_cosine": 0.08617192506790161, "beta_mean": 0.05608540400862694, "nonfinite": 0.0, "weighted_memory_to_active": 0.1464582085609436, "mix_std": 0.2581138610839844, "beta_std": 0.07186993956565857, "beta_low": 0.6497396230697632, "beta_high": 0.0, "orthogonal_to_active": 0.05853051692247391, "output_influence": 0.5457181930541992, "residual_influence": 0.1575118899345398, "state_active": 0.04318214952945709, "state_memory": 0.011123789474368095, "state_peak": 63.8366584777832, "growth_active": 1.1202589273452759, "growth_memory": 1.2353981733322144, "heads": [{"head": 0, "memory_to_active": 0.39840781688690186, "contribution_to_active": 0.5241439938545227, "mix_mean": 0.41602498292922974, "mix_low": 0.017217636108398438, "mix_high": 0.0020570755004882812, "read_cosine": -0.03099672682583332, "beta_mean": 0.02533797360956669, "nonfinite": 0.0, "weighted_memory_to_active": 0.1256905347108841, "mix_std": 0.2848787307739258, "beta_std": 0.0417889766395092, "beta_low": 0.41796875, "beta_high": 0.0, "orthogonal_to_active": 0.04881305992603302, "evidence_weights": [-0.1452559232711792, -0.1688602864742279, 0.03137153014540672], "agreement_weight": -0.020310543477535248, "orthogonal_weight": -0.3913463056087494}, {"head": 1, "memory_to_active": 0.009686438366770744, "contribution_to_active": 0.544165849685669, "mix_mean": 0.6844826936721802, "mix_low": 0.0011529922485351562, "mix_high": 0.03450775146484375, "read_cosine": 0.19350233674049377, "beta_mean": 2.147727172996383e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006045152433216572, "mix_std": 0.27762115001678467, "beta_std": 5.001543468097225e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0058234455063939095, "evidence_weights": [-0.2501233220100403, -0.289364755153656, 0.8809607625007629], "agreement_weight": 0.6649085879325867, "orthogonal_weight": -0.9967565536499023}, {"head": 2, "memory_to_active": 0.012090669013559818, "contribution_to_active": 0.6168166995048523, "mix_mean": 0.6676557064056396, "mix_low": 0.002536773681640625, "mix_high": 0.025869369506835938, "read_cosine": -0.49420005083084106, "beta_mean": 8.916815568227321e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007959555834531784, "mix_std": 0.26658377051353455, "beta_std": 0.00015582976629957557, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00386698916554451, "evidence_weights": [-0.26621362566947937, -0.34282249212265015, 0.25868597626686096], "agreement_weight": -0.04971012845635414, "orthogonal_weight": -0.5940160751342773}, {"head": 3, "memory_to_active": 1.645740270614624, "contribution_to_active": 0.2473633587360382, "mix_mean": 0.11650986969470978, "mix_low": 0.1341533660888672, "mix_high": 0.0, "read_cosine": 0.1366906613111496, "beta_mean": 0.10292800515890121, "nonfinite": 0.0, "weighted_memory_to_active": 0.3913801312446594, "mix_std": 0.1563323587179184, "beta_std": 0.1562541425228119, "beta_low": 0.19879150390625, "beta_high": 0.0, "orthogonal_to_active": 0.16192689538002014, "evidence_weights": [-0.0707961842417717, -0.09995590150356293, 0.03408109396696091], "agreement_weight": 0.05169924348592758, "orthogonal_weight": -0.48926326632499695}, {"head": 4, "memory_to_active": 0.46321746706962585, "contribution_to_active": 0.586618959903717, "mix_mean": 0.31724560260772705, "mix_low": 0.20746326446533203, "mix_high": 0.005913734436035156, "read_cosine": 0.5378196239471436, "beta_mean": 0.2079312801361084, "nonfinite": 0.0, "weighted_memory_to_active": 0.29654258489608765, "mix_std": 0.33675214648246765, "beta_std": 0.23209494352340698, "beta_low": 0.28314208984375, "beta_high": 0.0, "orthogonal_to_active": 0.0986362174153328, "evidence_weights": [-0.11511339992284775, -0.26345375180244446, -0.04905670881271362], "agreement_weight": 0.03931635245680809, "orthogonal_weight": 0.5078753232955933}, {"head": 5, "memory_to_active": 0.075063057243824, "contribution_to_active": 0.6840510964393616, "mix_mean": 0.8004269599914551, "mix_low": 0.002086639404296875, "mix_high": 0.04389476776123047, "read_cosine": 0.17421571910381317, "beta_mean": 0.0002045144501607865, "nonfinite": 0.0, "weighted_memory_to_active": 0.051131196320056915, "mix_std": 0.22651486098766327, "beta_std": 0.0008757042232900858, "beta_low": 0.99853515625, "beta_high": 0.0, "orthogonal_to_active": 0.03211648389697075, "evidence_weights": [-0.3132873773574829, -0.33196842670440674, 1.9357939958572388], "agreement_weight": 0.45747432112693787, "orthogonal_weight": -0.6644967198371887}]}, {"layer": 6, "memory_to_active": 0.2952795624732971, "contribution_to_active": 0.6185368299484253, "mix_mean": 0.5810076594352722, "mix_low": 0.05116447061300278, "mix_high": 0.03083149716258049, "read_cosine": 0.31466829776763916, "beta_mean": 0.061555273830890656, "nonfinite": 0.0, "weighted_memory_to_active": 0.14917045831680298, "mix_std": 0.2398340404033661, "beta_std": 0.07143445312976837, "beta_low": 0.540924072265625, "beta_high": 0.0, "orthogonal_to_active": 0.04260872304439545, "output_influence": 0.5322439670562744, "residual_influence": 0.17652785778045654, "state_active": 0.047737207263708115, "state_memory": 0.007950968109071255, "state_peak": 95.33102416992188, "growth_active": 1.2186274528503418, "growth_memory": 1.171389102935791, "heads": [{"head": 0, "memory_to_active": 0.11177927255630493, "contribution_to_active": 0.7006967663764954, "mix_mean": 0.8019438982009888, "mix_low": 0.00032520294189453125, "mix_high": 0.04206371307373047, "read_cosine": 0.23541359603405, "beta_mean": 0.01608511060476303, "nonfinite": 0.0, "weighted_memory_to_active": 0.09028149396181107, "mix_std": 0.20903974771499634, "beta_std": 0.026943132281303406, "beta_low": 0.593017578125, "beta_high": 0.0, "orthogonal_to_active": 0.06897878646850586, "evidence_weights": [-0.317548006772995, -0.3469192385673523, 1.1883398294448853], "agreement_weight": 0.4939807951450348, "orthogonal_weight": -0.8204665780067444}, {"head": 1, "memory_to_active": 0.005253669340163469, "contribution_to_active": 0.5324177145957947, "mix_mean": 0.5777080655097961, "mix_low": 0.0013875961303710938, "mix_high": 0.014225006103515625, "read_cosine": 0.5337134003639221, "beta_mean": 1.4241079043131322e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0025076207239180803, "mix_std": 0.2868903875350952, "beta_std": 2.832473546732217e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014072590274736285, "evidence_weights": [-0.1992076337337494, -0.30050668120384216, 1.1242460012435913], "agreement_weight": 0.0852830559015274, "orthogonal_weight": -0.7457624077796936}, {"head": 2, "memory_to_active": 0.8225888013839722, "contribution_to_active": 0.10154880583286285, "mix_mean": 0.04213482141494751, "mix_low": 0.2944002151489258, "mix_high": 0.0, "read_cosine": 0.23875398933887482, "beta_mean": 0.029496721923351288, "nonfinite": 0.0, "weighted_memory_to_active": 0.0704084262251854, "mix_std": 0.06837723404169083, "beta_std": 0.030636006966233253, "beta_low": 0.1490478515625, "beta_high": 0.0, "orthogonal_to_active": 0.006177372299134731, "evidence_weights": [0.07324964553117752, -0.04309742525219917, 0.305691123008728], "agreement_weight": 0.14318504929542542, "orthogonal_weight": -0.104904904961586}, {"head": 3, "memory_to_active": 0.1396358758211136, "contribution_to_active": 0.6753787994384766, "mix_mean": 0.6431105136871338, "mix_low": 0.009619712829589844, "mix_high": 0.017365455627441406, "read_cosine": 0.4695204496383667, "beta_mean": 0.19848623871803284, "nonfinite": 0.0, "weighted_memory_to_active": 0.09867452085018158, "mix_std": 0.3253556191921234, "beta_std": 0.23174753785133362, "beta_low": 0.20947265625, "beta_high": 0.0, "orthogonal_to_active": 0.05299033224582672, "evidence_weights": [-0.17550580203533173, -0.3299371898174286, 0.17060090601444244], "agreement_weight": -0.005565797910094261, "orthogonal_weight": 0.6163696646690369}, {"head": 4, "memory_to_active": 0.6660670042037964, "contribution_to_active": 1.0716400146484375, "mix_mean": 0.7407367825508118, "mix_low": 0.0009441375732421875, "mix_high": 0.10410690307617188, "read_cosine": 0.12951715290546417, "beta_mean": 0.12510187923908234, "nonfinite": 0.0, "weighted_memory_to_active": 0.614211916923523, "mix_std": 0.3029262125492096, "beta_std": 0.13893142342567444, "beta_low": 0.2940673828125, "beta_high": 0.0, "orthogonal_to_active": 0.10737840086221695, "evidence_weights": [-0.22234636545181274, -0.5747606754302979, -0.35107821226119995], "agreement_weight": -0.018901482224464417, "orthogonal_weight": 0.1956082433462143}, {"head": 5, "memory_to_active": 0.02635270543396473, "contribution_to_active": 0.6295385956764221, "mix_mean": 0.6804119944572449, "mix_low": 0.00030994415283203125, "mix_high": 0.007227897644042969, "read_cosine": 0.2810911238193512, "beta_mean": 0.00014745391672477126, "nonfinite": 0.0, "weighted_memory_to_active": 0.018938720226287842, "mix_std": 0.24641507863998413, "beta_std": 0.0003202997613698244, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01872018352150917, "evidence_weights": [-0.24446789920330048, -0.3172304332256317, 0.4000025689601898], "agreement_weight": 0.19053855538368225, "orthogonal_weight": -1.058046579360962}]}, {"layer": 7, "memory_to_active": 1.7144089937210083, "contribution_to_active": 0.2675517201423645, "mix_mean": 0.15720143914222717, "mix_low": 0.25521454215049744, "mix_high": 0.0009856224060058594, "read_cosine": 0.2361854910850525, "beta_mean": 0.13462716341018677, "nonfinite": 0.0, "weighted_memory_to_active": 0.22348631918430328, "mix_std": 0.1467445343732834, "beta_std": 0.1012253388762474, "beta_low": 0.2041829526424408, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.08797634392976761, "output_influence": 0.27809327840805054, "residual_influence": 0.0641392394900322, "state_active": 0.014238270930945873, "state_memory": 0.025993920862674713, "state_peak": 30.2036075592041, "growth_active": 1.1681647300720215, "growth_memory": 1.2827808856964111, "heads": [{"head": 0, "memory_to_active": 2.4577081203460693, "contribution_to_active": 0.24712689220905304, "mix_mean": 0.03816230967640877, "mix_low": 0.4164905548095703, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3301253616809845, "beta_mean": 0.19040700793266296, "nonfinite": 0.0, "weighted_memory_to_active": 0.2910216748714447, "mix_std": 0.0799114853143692, "beta_std": 0.14565928280353546, "beta_low": 0.00201416015625, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.0020480582024902105, "evidence_weights": [0.03176764026284218, -0.07424192130565643, 0.030027151107788086], "agreement_weight": -0.03251447156071663, "orthogonal_weight": 0.009588093496859074}, {"head": 1, "memory_to_active": 0.9915428161621094, "contribution_to_active": 0.39812588691711426, "mix_mean": 0.22071433067321777, "mix_low": 0.12749195098876953, "mix_high": 0.00524139404296875, "read_cosine": 0.37089699506759644, "beta_mean": 0.09859417378902435, "nonfinite": 0.0, "weighted_memory_to_active": 0.33980196714401245, "mix_std": 0.24556320905685425, "beta_std": 0.06790407001972198, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04203503578901291, "evidence_weights": [-0.21497651934623718, -0.10405905544757843, -0.2433314174413681], "agreement_weight": 0.04390151426196098, "orthogonal_weight": 0.1476898491382599}, {"head": 2, "memory_to_active": 0.7039135098457336, "contribution_to_active": 0.3506183624267578, "mix_mean": 0.1897810995578766, "mix_low": 0.040287017822265625, "mix_high": 0.0003108978271484375, "read_cosine": 0.10717220604419708, "beta_mean": 0.04505037143826485, "nonfinite": 0.0, "weighted_memory_to_active": 0.25096169114112854, "mix_std": 0.20160570740699768, "beta_std": 0.04140277951955795, "beta_low": 0.09375, "beta_high": 0.0, "orthogonal_to_active": 0.12421073019504547, "evidence_weights": [-0.0754748061299324, -0.1586223691701889, 0.7316684722900391], "agreement_weight": 0.019543129950761795, "orthogonal_weight": -0.5100471377372742}, {"head": 3, "memory_to_active": 4.581642150878906, "contribution_to_active": 0.10325533151626587, "mix_mean": 0.034792639315128326, "mix_low": 0.33080196380615234, "mix_high": 0.0, "read_cosine": 0.08788769692182541, "beta_mean": 0.4001583158969879, "nonfinite": 0.0, "weighted_memory_to_active": 0.37470948696136475, "mix_std": 0.05275258421897888, "beta_std": 0.2714505195617676, "beta_low": 0.0040283203125, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.3318352699279785, "evidence_weights": [0.07129798829555511, -0.021981816738843918, 0.6970602869987488], "agreement_weight": 0.04594637081027031, "orthogonal_weight": -0.9493520855903625}, {"head": 4, "memory_to_active": 1.51301109790802, "contribution_to_active": 0.08532906323671341, "mix_mean": 0.019178209826350212, "mix_low": 0.6138830184936523, "mix_high": 0.0, "read_cosine": 0.24018608033657074, "beta_mean": 0.07335276901721954, "nonfinite": 0.0, "weighted_memory_to_active": 0.06621058285236359, "mix_std": 0.044212035834789276, "beta_std": 0.08050079643726349, "beta_low": 0.12554931640625, "beta_high": 0.0, "orthogonal_to_active": 0.011641339398920536, "evidence_weights": [0.14884208142757416, 0.005515382159501314, 0.18440306186676025], "agreement_weight": 0.04007790610194206, "orthogonal_weight": -0.22383670508861542}, {"head": 5, "memory_to_active": 0.038635317236185074, "contribution_to_active": 0.4208548069000244, "mix_mean": 0.44058001041412354, "mix_low": 0.0023326873779296875, "mix_high": 0.0003604888916015625, "read_cosine": 0.2808445692062378, "beta_mean": 0.00020034253248013556, "nonfinite": 0.0, "weighted_memory_to_active": 0.018212513998150826, "mix_std": 0.25642216205596924, "beta_std": 0.00043453677790239453, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.01608758419752121, "evidence_weights": [-0.1655014455318451, -0.2233644723892212, 0.5377334356307983], "agreement_weight": 0.1829325556755066, "orthogonal_weight": -0.9613727331161499}]}, {"layer": 8, "memory_to_active": 0.20899179577827454, "contribution_to_active": 0.5531618595123291, "mix_mean": 0.46154913306236267, "mix_low": 0.020336151123046875, "mix_high": 0.0014778773766011, "read_cosine": 0.42976975440979004, "beta_mean": 0.01676398515701294, "nonfinite": 0.0, "weighted_memory_to_active": 0.09113094210624695, "mix_std": 0.2336256355047226, "beta_std": 0.023520750924944878, "beta_low": 0.748291015625, "beta_high": 0.0, "orthogonal_to_active": 0.07457457482814789, "output_influence": 0.49779996275901794, "residual_influence": 0.16833187639713287, "state_active": 0.04324297979474068, "state_memory": 0.006374286022037268, "state_peak": 43.37117004394531, "growth_active": 1.0769776105880737, "growth_memory": 1.1175285577774048, "heads": [{"head": 0, "memory_to_active": 0.05658096820116043, "contribution_to_active": 0.5739120244979858, "mix_mean": 0.609873354434967, "mix_low": 0.0004405975341796875, "mix_high": 0.0015306472778320312, "read_cosine": 0.46355384588241577, "beta_mean": 0.002764839446172118, "nonfinite": 0.0, "weighted_memory_to_active": 0.04006721079349518, "mix_std": 0.24667347967624664, "beta_std": 0.009544739499688148, "beta_low": 0.93310546875, "beta_high": 0.0, "orthogonal_to_active": 0.02538095973432064, "evidence_weights": [-0.17072881758213043, -0.3911074995994568, 0.641183614730835], "agreement_weight": 0.22216753661632538, "orthogonal_weight": -0.8017538189888}, {"head": 1, "memory_to_active": 0.07785994559526443, "contribution_to_active": 0.5440150499343872, "mix_mean": 0.5034729838371277, "mix_low": 2.86102294921875e-05, "mix_high": 0.0009508132934570312, "read_cosine": 0.15264618396759033, "beta_mean": 0.0012114893179386854, "nonfinite": 0.0, "weighted_memory_to_active": 0.03596540167927742, "mix_std": 0.23821544647216797, "beta_std": 0.006387622095644474, "beta_low": 0.987548828125, "beta_high": 0.0, "orthogonal_to_active": 0.025773845613002777, "evidence_weights": [-0.19329781830310822, -0.27426010370254517, 0.6935514807701111], "agreement_weight": 0.07240566611289978, "orthogonal_weight": -0.7514680624008179}, {"head": 2, "memory_to_active": 0.017372580245137215, "contribution_to_active": 0.6713787317276001, "mix_mean": 0.6678057312965393, "mix_low": 0.0007686614990234375, "mix_high": 0.0030727386474609375, "read_cosine": 0.38653457164764404, "beta_mean": 0.0001646165328565985, "nonfinite": 0.0, "weighted_memory_to_active": 0.013446666300296783, "mix_std": 0.22706584632396698, "beta_std": 0.0028668930754065514, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.008418533951044083, "evidence_weights": [-0.22746995091438293, -0.345982164144516, 1.027956485748291], "agreement_weight": 0.26292815804481506, "orthogonal_weight": -0.7146684527397156}, {"head": 3, "memory_to_active": 0.1053692102432251, "contribution_to_active": 0.6370209455490112, "mix_mean": 0.5745200514793396, "mix_low": 0.0007839202880859375, "mix_high": 0.0030994415283203125, "read_cosine": 0.8112555742263794, "beta_mean": 0.007259586360305548, "nonfinite": 0.0, "weighted_memory_to_active": 0.08227472007274628, "mix_std": 0.2566669285297394, "beta_std": 0.00658710952848196, "beta_low": 0.765625, "beta_high": 0.0, "orthogonal_to_active": 0.02602999657392502, "evidence_weights": [-0.18601664900779724, -0.3024196922779083, 0.5639576315879822], "agreement_weight": 0.21921272575855255, "orthogonal_weight": -0.6305078268051147}, {"head": 4, "memory_to_active": 0.6022748351097107, "contribution_to_active": 0.22656714916229248, "mix_mean": 0.1465354859828949, "mix_low": 0.046929359436035156, "mix_high": 9.5367431640625e-07, "read_cosine": 0.26071566343307495, "beta_mean": 0.011420303955674171, "nonfinite": 0.0, "weighted_memory_to_active": 0.1662522703409195, "mix_std": 0.15700072050094604, "beta_std": 0.027210483327507973, "beta_low": 0.73468017578125, "beta_high": 0.0, "orthogonal_to_active": 0.03287968412041664, "evidence_weights": [-0.03836223483085632, -0.1356465220451355, 0.2865601181983948], "agreement_weight": 0.07236840575933456, "orthogonal_weight": -0.21412122249603271}, {"head": 5, "memory_to_active": 0.3944932520389557, "contribution_to_active": 0.6660771369934082, "mix_mean": 0.26708710193634033, "mix_low": 0.07306575775146484, "mix_high": 0.00021266937255859375, "read_cosine": 0.5039125680923462, "beta_mean": 0.07776306569576263, "nonfinite": 0.0, "weighted_memory_to_active": 0.20877933502197266, "mix_std": 0.27613136172294617, "beta_std": 0.08852764964103699, "beta_low": 0.06884765625, "beta_high": 0.0, "orthogonal_to_active": 0.32896438241004944, "evidence_weights": [-0.12432259321212769, -0.17081743478775024, -0.13716089725494385], "agreement_weight": -0.009273658506572247, "orthogonal_weight": 1.872517704963684}]}, {"layer": 9, "memory_to_active": 1.012895107269287, "contribution_to_active": 0.7030140161514282, "mix_mean": 0.3491954207420349, "mix_low": 0.1075485572218895, "mix_high": 0.0051930746994912624, "read_cosine": 0.3436095118522644, "beta_mean": 0.05188050493597984, "nonfinite": 0.0, "weighted_memory_to_active": 0.41556909680366516, "mix_std": 0.22233228385448456, "beta_std": 0.04113635793328285, "beta_low": 0.5116475820541382, "beta_high": 0.0, "orthogonal_to_active": 0.04377271607518196, "output_influence": 0.526159942150116, "residual_influence": 0.15020224452018738, "state_active": 0.04303547367453575, "state_memory": 0.017601095139980316, "state_peak": 51.383121490478516, "growth_active": 1.1581300497055054, "growth_memory": 1.3292768001556396, "heads": [{"head": 0, "memory_to_active": 0.001392203732393682, "contribution_to_active": 0.5602151155471802, "mix_mean": 0.6330115795135498, "mix_low": 0.0014219284057617188, "mix_high": 0.021816253662109375, "read_cosine": 0.2790783941745758, "beta_mean": 4.6862282943038736e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0008824581746011972, "mix_std": 0.28997087478637695, "beta_std": 1.7255399143323302e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000955266528762877, "evidence_weights": [-0.3029387593269348, -0.3094182014465332, 0.6449403166770935], "agreement_weight": 0.12096104025840759, "orthogonal_weight": -1.1206464767456055}, {"head": 1, "memory_to_active": 0.022487729787826538, "contribution_to_active": 0.6860624551773071, "mix_mean": 0.556238055229187, "mix_low": 0.0066967010498046875, "mix_high": 0.0020036697387695312, "read_cosine": 0.4159091114997864, "beta_mean": 6.125918298494071e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.016597913578152657, "mix_std": 0.2846568822860718, "beta_std": 0.00012534833513200283, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011458925902843475, "evidence_weights": [-0.24011504650115967, -0.2941281795501709, 1.0882025957107544], "agreement_weight": 0.44514000415802, "orthogonal_weight": -0.8549613356590271}, {"head": 2, "memory_to_active": 0.2043132781982422, "contribution_to_active": 0.642230212688446, "mix_mean": 0.584735095500946, "mix_low": 0.0020732879638671875, "mix_high": 0.006275177001953125, "read_cosine": 0.34485748410224915, "beta_mean": 0.0008182195015251637, "nonfinite": 0.0, "weighted_memory_to_active": 0.13837897777557373, "mix_std": 0.26907023787498474, "beta_std": 0.004395162686705589, "beta_low": 0.99334716796875, "beta_high": 0.0, "orthogonal_to_active": 0.0797906219959259, "evidence_weights": [-0.20009563863277435, -0.27461743354797363, 0.7537415027618408], "agreement_weight": 0.31671321392059326, "orthogonal_weight": -0.6764010787010193}, {"head": 3, "memory_to_active": 1.1806185245513916, "contribution_to_active": 0.17036330699920654, "mix_mean": 0.07308827340602875, "mix_low": 0.25438594818115234, "mix_high": 5.817413330078125e-05, "read_cosine": 0.3979622423648834, "beta_mean": 0.0569801889359951, "nonfinite": 0.0, "weighted_memory_to_active": 0.15262728929519653, "mix_std": 0.12498864531517029, "beta_std": 0.043857842683792114, "beta_low": 0.01202392578125, "beta_high": 0.0, "orthogonal_to_active": 0.023922370746731758, "evidence_weights": [0.04568607360124588, -0.02645423635840416, 0.10444145649671555], "agreement_weight": 0.0863780751824379, "orthogonal_weight": -0.1679198294878006}, {"head": 4, "memory_to_active": 3.0594420433044434, "contribution_to_active": 1.575692057609558, "mix_mean": 0.14135810732841492, "mix_low": 0.13394451141357422, "mix_high": 0.0008831024169921875, "read_cosine": 0.16722434759140015, "beta_mean": 0.16038665175437927, "nonfinite": 0.0, "weighted_memory_to_active": 1.5714844465255737, "mix_std": 0.20303277671337128, "beta_std": 0.09406411647796631, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03798656910657883, "evidence_weights": [-0.07949844747781754, -0.15601418912410736, -0.027737705036997795], "agreement_weight": -0.060110513120889664, "orthogonal_weight": -0.025205375626683235}, {"head": 5, "memory_to_active": 1.6091164350509644, "contribution_to_active": 0.5835208892822266, "mix_mean": 0.10674113035202026, "mix_low": 0.24676895141601562, "mix_high": 0.0001220703125, "read_cosine": 0.4566253423690796, "beta_mean": 0.09303203225135803, "nonfinite": 0.0, "weighted_memory_to_active": 0.6134434938430786, "mix_std": 0.16227427124977112, "beta_std": 0.10435841232538223, "beta_low": 0.06451416015625, "beta_high": 0.0, "orthogonal_to_active": 0.10852253437042236, "evidence_weights": [0.10039970278739929, -0.11560973525047302, 0.14603634178638458], "agreement_weight": 0.014350122772157192, "orthogonal_weight": 0.24426355957984924}]}, {"layer": 10, "memory_to_active": 0.2121274471282959, "contribution_to_active": 0.6887805461883545, "mix_mean": 0.513460636138916, "mix_low": 0.02036396786570549, "mix_high": 0.005534013733267784, "read_cosine": 0.3738458752632141, "beta_mean": 0.045961007475852966, "nonfinite": 0.0, "weighted_memory_to_active": 0.10009787976741791, "mix_std": 0.2676122188568115, "beta_std": 0.047415412962436676, "beta_low": 0.6469116806983948, "beta_high": 0.0, "orthogonal_to_active": 0.2255646288394928, "output_influence": 0.44421476125717163, "residual_influence": 0.15322290360927582, "state_active": 0.056411515921354294, "state_memory": 0.01539621315896511, "state_peak": 123.58769989013672, "growth_active": 1.0200129747390747, "growth_memory": 1.0038502216339111, "heads": [{"head": 0, "memory_to_active": 0.7829854488372803, "contribution_to_active": 1.873855471611023, "mix_mean": 0.5776650309562683, "mix_low": 0.08231639862060547, "mix_high": 0.0152587890625, "read_cosine": 0.10169006884098053, "beta_mean": 0.20097073912620544, "nonfinite": 0.0, "weighted_memory_to_active": 0.4735475480556488, "mix_std": 0.3914233446121216, "beta_std": 0.2117270976305008, "beta_low": 0.13970947265625, "beta_high": 0.0, "orthogonal_to_active": 1.2630174160003662, "evidence_weights": [-0.09602954238653183, -0.38359856605529785, -1.6539108753204346], "agreement_weight": 0.02249077707529068, "orthogonal_weight": 2.690662145614624}, {"head": 1, "memory_to_active": 0.01225876435637474, "contribution_to_active": 0.603804349899292, "mix_mean": 0.6152469515800476, "mix_low": 0.0013189315795898438, "mix_high": 0.013051986694335938, "read_cosine": 0.2578783929347992, "beta_mean": 8.814946340862662e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0068184928968548775, "mix_std": 0.26569968461990356, "beta_std": 0.00014587020268663764, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010385045781731606, "evidence_weights": [-0.25522977113723755, -0.19893376529216766, 0.5728546380996704], "agreement_weight": 0.5502395629882812, "orthogonal_weight": -1.624588966369629}, {"head": 2, "memory_to_active": 0.01711231842637062, "contribution_to_active": 0.6099477410316467, "mix_mean": 0.6375929713249207, "mix_low": 0.00042057037353515625, "mix_high": 0.00405120849609375, "read_cosine": 0.030592577531933784, "beta_mean": 3.5498407669365406e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006953796371817589, "mix_std": 0.2528701424598694, "beta_std": 0.00014270975952968001, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003166504669934511, "evidence_weights": [-0.25561386346817017, -0.2788830101490021, 0.6494314670562744], "agreement_weight": -0.19977623224258423, "orthogonal_weight": -0.4264809787273407}, {"head": 3, "memory_to_active": 0.30346208810806274, "contribution_to_active": 0.09749030321836472, "mix_mean": 0.27336347103118896, "mix_low": 0.035991668701171875, "mix_high": 6.67572021484375e-05, "read_cosine": 0.9092440009117126, "beta_mean": 0.06623515486717224, "nonfinite": 0.0, "weighted_memory_to_active": 0.03793573006987572, "mix_std": 0.22387230396270752, "beta_std": 0.06359344720840454, "beta_low": 0.00579833984375, "beta_high": 0.0, "orthogonal_to_active": 0.03876878693699837, "evidence_weights": [-0.10442622750997543, -0.14759626984596252, 0.1661584973335266], "agreement_weight": 0.015915662050247192, "orthogonal_weight": 1.4032098054885864}, {"head": 4, "memory_to_active": 0.0232694149017334, "contribution_to_active": 0.54156494140625, "mix_mean": 0.5802655816078186, "mix_low": 0.00030422210693359375, "mix_high": 0.0007715225219726562, "read_cosine": 0.0901743546128273, "beta_mean": 6.806316378060728e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013389780186116695, "mix_std": 0.24513772130012512, "beta_std": 0.00012002154107904062, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.017599163576960564, "evidence_weights": [-0.18623104691505432, -0.26273488998413086, 0.9742629528045654], "agreement_weight": 0.14930693805217743, "orthogonal_weight": -1.3856697082519531}, {"head": 5, "memory_to_active": 0.1336766630411148, "contribution_to_active": 0.4060203433036804, "mix_mean": 0.39662981033325195, "mix_low": 0.0018320083618164062, "mix_high": 3.814697265625e-06, "read_cosine": 0.8534958362579346, "beta_mean": 0.008368430659174919, "nonfinite": 0.0, "weighted_memory_to_active": 0.061941951513290405, "mix_std": 0.2266702651977539, "beta_std": 0.008763321675360203, "beta_low": 0.7359619140625, "beta_high": 0.0, "orthogonal_to_active": 0.020450880751013756, "evidence_weights": [-0.1420464664697647, -0.1632290482521057, 0.270869642496109], "agreement_weight": 0.09177515655755997, "orthogonal_weight": -0.6564871668815613}]}, {"layer": 11, "memory_to_active": 0.09304463118314743, "contribution_to_active": 0.6051344871520996, "mix_mean": 0.5076237916946411, "mix_low": 0.10159716010093689, "mix_high": 0.047687213867902756, "read_cosine": 0.33465898036956787, "beta_mean": 0.026501961052417755, "nonfinite": 0.0, "weighted_memory_to_active": 0.04706646129488945, "mix_std": 0.3231446146965027, "beta_std": 0.03888307139277458, "beta_low": 0.665191650390625, "beta_high": 0.0, "orthogonal_to_active": 0.04860527813434601, "output_influence": 0.5390795469284058, "residual_influence": 0.13732969760894775, "state_active": 0.1990935057401657, "state_memory": 0.004479501396417618, "state_peak": 31.13401222229004, "growth_active": 1.104737639427185, "growth_memory": 1.628163456916809, "heads": [{"head": 0, "memory_to_active": 0.004548404365777969, "contribution_to_active": 0.5935667157173157, "mix_mean": 0.6388508081436157, "mix_low": 0.0010061264038085938, "mix_high": 0.007936477661132812, "read_cosine": 0.2803111970424652, "beta_mean": 6.661726365564391e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0030136120039969683, "mix_std": 0.25809869170188904, "beta_std": 0.0032585097942501307, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.002377294236794114, "evidence_weights": [-0.1718376874923706, -0.36717113852500916, 0.7533407807350159], "agreement_weight": 0.341054767370224, "orthogonal_weight": -0.8294658064842224}, {"head": 1, "memory_to_active": 0.1841643750667572, "contribution_to_active": 0.37845274806022644, "mix_mean": 0.4259156584739685, "mix_low": 0.007084846496582031, "mix_high": 0.00023555755615234375, "read_cosine": 0.4747929871082306, "beta_mean": 0.00951116532087326, "nonfinite": 0.0, "weighted_memory_to_active": 0.08095952123403549, "mix_std": 0.2633363604545593, "beta_std": 0.0224016010761261, "beta_low": 0.80364990234375, "beta_high": 0.0, "orthogonal_to_active": 0.03930883854627609, "evidence_weights": [-0.15905845165252686, -0.23534344136714935, 0.5811258554458618], "agreement_weight": 0.06836948543787003, "orthogonal_weight": -0.5773633122444153}, {"head": 2, "memory_to_active": 0.007481152191758156, "contribution_to_active": 0.9831626415252686, "mix_mean": 0.4428749680519104, "mix_low": 0.2751932144165039, "mix_high": 0.17342758178710938, "read_cosine": 0.18191733956336975, "beta_mean": 0.06897290050983429, "nonfinite": 0.0, "weighted_memory_to_active": 0.005235209129750729, "mix_std": 0.4465491473674774, "beta_std": 0.0912334993481636, "beta_low": 0.464111328125, "beta_high": 0.0, "orthogonal_to_active": 0.044068798422813416, "evidence_weights": [0.045649830251932144, -0.3991435170173645, -1.3079931735992432], "agreement_weight": 0.009488707408308983, "orthogonal_weight": 8.726021766662598}, {"head": 3, "memory_to_active": 0.00925540179014206, "contribution_to_active": 0.8414648771286011, "mix_mean": 0.4142077565193176, "mix_low": 0.31692028045654297, "mix_high": 0.09848403930664062, "read_cosine": 0.5201840400695801, "beta_mean": 0.05508429929614067, "nonfinite": 0.0, "weighted_memory_to_active": 0.005643090233206749, "mix_std": 0.439696729183197, "beta_std": 0.08268684893846512, "beta_low": 0.5050048828125, "beta_high": 0.0, "orthogonal_to_active": 0.037693850696086884, "evidence_weights": [0.19086740911006927, -0.3071072995662689, -0.1767633855342865], "agreement_weight": 0.050668735057115555, "orthogonal_weight": 9.022316932678223}, {"head": 4, "memory_to_active": 0.24938952922821045, "contribution_to_active": 0.43962764739990234, "mix_mean": 0.5757738351821899, "mix_low": 0.006434440612792969, "mix_high": 0.001453399658203125, "read_cosine": 0.028236903250217438, "beta_mean": 0.014283694326877594, "nonfinite": 0.0, "weighted_memory_to_active": 0.13397546112537384, "mix_std": 0.27116191387176514, "beta_std": 0.01558440551161766, "beta_low": 0.517333984375, "beta_high": 0.0, "orthogonal_to_active": 0.13309717178344727, "evidence_weights": [-0.24369937181472778, -0.2364114224910736, 1.9417954683303833], "agreement_weight": -0.20387175679206848, "orthogonal_weight": -1.0093885660171509}, {"head": 5, "memory_to_active": 0.10342886298894882, "contribution_to_active": 0.3945320248603821, "mix_mean": 0.5481197834014893, "mix_low": 0.0029439926147460938, "mix_high": 0.004586219787597656, "read_cosine": 0.5225112438201904, "beta_mean": 0.011093082837760448, "nonfinite": 0.0, "weighted_memory_to_active": 0.053571853786706924, "mix_std": 0.26002469658851624, "beta_std": 0.018133554607629776, "beta_low": 0.7012939453125, "beta_high": 0.0, "orthogonal_to_active": 0.03508572652935982, "evidence_weights": [-0.21128882467746735, -0.31209543347358704, 0.6446079015731812], "agreement_weight": 0.0871870368719101, "orthogonal_weight": -0.8218134045600891}]}, {"layer": 12, "memory_to_active": 0.26088595390319824, "contribution_to_active": 0.5791593790054321, "mix_mean": 0.5283946990966797, "mix_low": 0.008917014114558697, "mix_high": 0.005142371170222759, "read_cosine": 0.22772617638111115, "beta_mean": 0.01436470914632082, "nonfinite": 0.0, "weighted_memory_to_active": 0.13958857953548431, "mix_std": 0.26007261872291565, "beta_std": 0.016589131206274033, "beta_low": 0.8246257305145264, "beta_high": 0.0, "orthogonal_to_active": 0.07647263258695602, "output_influence": 0.49195390939712524, "residual_influence": 0.13777796924114227, "state_active": 0.035561610013246536, "state_memory": 0.008898930624127388, "state_peak": 54.262115478515625, "growth_active": 1.136089563369751, "growth_memory": 1.1901021003723145, "heads": [{"head": 0, "memory_to_active": 0.049287572503089905, "contribution_to_active": 0.5834494233131409, "mix_mean": 0.5592195987701416, "mix_low": 0.00041103363037109375, "mix_high": 0.002971649169921875, "read_cosine": 0.28834250569343567, "beta_mean": 0.0009597025346010923, "nonfinite": 0.0, "weighted_memory_to_active": 0.025604695081710815, "mix_std": 0.2663387954235077, "beta_std": 0.002216908847913146, "beta_low": 0.988525390625, "beta_high": 0.0, "orthogonal_to_active": 0.023813216015696526, "evidence_weights": [-0.2084648162126541, -0.27637091279029846, 0.06941983848810196], "agreement_weight": 0.10218933969736099, "orthogonal_weight": -0.9928579330444336}, {"head": 1, "memory_to_active": 0.11431808024644852, "contribution_to_active": 0.5383584499359131, "mix_mean": 0.5498103499412537, "mix_low": 0.0003643035888671875, "mix_high": 0.0013608932495117188, "read_cosine": 0.4278629422187805, "beta_mean": 0.0009623272926546633, "nonfinite": 0.0, "weighted_memory_to_active": 0.07039482891559601, "mix_std": 0.26418817043304443, "beta_std": 0.010299425572156906, "beta_low": 0.99322509765625, "beta_high": 0.0, "orthogonal_to_active": 0.05882719159126282, "evidence_weights": [-0.20613183081150055, -0.2735339105129242, 1.0458322763442993], "agreement_weight": 0.22135679423809052, "orthogonal_weight": -0.9353155493736267}, {"head": 2, "memory_to_active": 0.5991650223731995, "contribution_to_active": 0.4925153851509094, "mix_mean": 0.30537065863609314, "mix_low": 0.02795124053955078, "mix_high": 0.0001201629638671875, "read_cosine": 0.49892666935920715, "beta_mean": 0.08228837698698044, "nonfinite": 0.0, "weighted_memory_to_active": 0.24630200862884521, "mix_std": 0.236590176820755, "beta_std": 0.0628863275051117, "beta_low": 0.00421142578125, "beta_high": 0.0, "orthogonal_to_active": 0.2574079632759094, "evidence_weights": [-0.22052063047885895, -0.05545997619628906, -0.002534222323447466], "agreement_weight": 0.024438349530100822, "orthogonal_weight": 1.3268636465072632}, {"head": 3, "memory_to_active": 0.03815152123570442, "contribution_to_active": 0.7504921555519104, "mix_mean": 0.7087127566337585, "mix_low": 0.00016498565673828125, "mix_high": 0.016536712646484375, "read_cosine": 0.10727498680353165, "beta_mean": 0.00034076895099133253, "nonfinite": 0.0, "weighted_memory_to_active": 0.026805981993675232, "mix_std": 0.22887197136878967, "beta_std": 0.006098954007029533, "beta_low": 0.99639892578125, "beta_high": 0.0, "orthogonal_to_active": 0.025741489604115486, "evidence_weights": [-0.2919910252094269, -0.34145379066467285, 1.578131914138794], "agreement_weight": 0.4897709786891937, "orthogonal_weight": -0.9830516576766968}, {"head": 4, "memory_to_active": 0.6702200174331665, "contribution_to_active": 0.6218631267547607, "mix_mean": 0.42184877395629883, "mix_low": 0.01740550994873047, "mix_high": 0.0048675537109375, "read_cosine": -0.14887119829654694, "beta_mean": 0.0011399140348657966, "nonfinite": 0.0, "weighted_memory_to_active": 0.40447479486465454, "mix_std": 0.2867485284805298, "beta_std": 0.012114974670112133, "beta_low": 0.9766845703125, "beta_high": 0.0, "orthogonal_to_active": 0.04326264560222626, "evidence_weights": [-0.15142247080802917, -0.219119131565094, -0.0768112912774086], "agreement_weight": 0.005426687188446522, "orthogonal_weight": -0.11153602600097656}, {"head": 5, "memory_to_active": 0.094173364341259, "contribution_to_active": 0.48827776312828064, "mix_mean": 0.6254056692123413, "mix_low": 0.007205009460449219, "mix_high": 0.00499725341796875, "read_cosine": 0.19282108545303345, "beta_mean": 0.0004971581511199474, "nonfinite": 0.0, "weighted_memory_to_active": 0.0639491081237793, "mix_std": 0.27769792079925537, "beta_std": 0.005918193608522415, "beta_low": 0.98870849609375, "beta_high": 0.0, "orthogonal_to_active": 0.04978322237730026, "evidence_weights": [-0.23230373859405518, -0.31562668085098267, 1.3148268461227417], "agreement_weight": 0.3923415541648865, "orthogonal_weight": -0.8292791843414307}]}, {"layer": 13, "memory_to_active": 0.13338035345077515, "contribution_to_active": 0.5300803780555725, "mix_mean": 0.5290106534957886, "mix_low": 0.005793253891170025, "mix_high": 0.008314291946589947, "read_cosine": 0.427461177110672, "beta_mean": 0.01426781713962555, "nonfinite": 0.0, "weighted_memory_to_active": 0.05993933975696564, "mix_std": 0.2594276964664459, "beta_std": 0.018058324232697487, "beta_low": 0.8105672597885132, "beta_high": 0.0, "orthogonal_to_active": 0.05980408191680908, "output_influence": 0.5143930315971375, "residual_influence": 0.17160093784332275, "state_active": 0.12322007864713669, "state_memory": 0.04411649703979492, "state_peak": 101.06124114990234, "growth_active": 0.9195012450218201, "growth_memory": 1.0367896556854248, "heads": [{"head": 0, "memory_to_active": 0.025334537029266357, "contribution_to_active": 0.5803790092468262, "mix_mean": 0.6015946865081787, "mix_low": 0.0005025863647460938, "mix_high": 0.0034799575805664062, "read_cosine": 0.4400184750556946, "beta_mean": 0.00010520264186197892, "nonfinite": 0.0, "weighted_memory_to_active": 0.016099577769637108, "mix_std": 0.26860445737838745, "beta_std": 0.00023388079716823995, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015875600278377533, "evidence_weights": [-0.19785557687282562, -0.2677336633205414, 0.767643928527832], "agreement_weight": 0.14891663193702698, "orthogonal_weight": -1.107900857925415}, {"head": 1, "memory_to_active": 0.021750353276729584, "contribution_to_active": 0.5930609703063965, "mix_mean": 0.5559102296829224, "mix_low": 0.0019445419311523438, "mix_high": 0.0036420822143554688, "read_cosine": 0.20554162561893463, "beta_mean": 8.599944703746587e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011850075796246529, "mix_std": 0.27532869577407837, "beta_std": 0.0001679041306488216, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015781687572598457, "evidence_weights": [-0.18377387523651123, -0.2647210359573364, 0.6453043818473816], "agreement_weight": 0.42484623193740845, "orthogonal_weight": -1.3833738565444946}, {"head": 2, "memory_to_active": 0.08664052188396454, "contribution_to_active": 0.4696672856807709, "mix_mean": 0.38427889347076416, "mix_low": 0.00154876708984375, "mix_high": 0.00014591217041015625, "read_cosine": 0.6937594413757324, "beta_mean": 0.011338132433593273, "nonfinite": 0.0, "weighted_memory_to_active": 0.0418073907494545, "mix_std": 0.2315385639667511, "beta_std": 0.042271438986063004, "beta_low": 0.87261962890625, "beta_high": 0.0, "orthogonal_to_active": 0.022672738879919052, "evidence_weights": [-0.18697628378868103, -0.20785515010356903, 0.41015490889549255], "agreement_weight": 0.008055202662944794, "orthogonal_weight": -0.8355520367622375}, {"head": 3, "memory_to_active": 0.20115019381046295, "contribution_to_active": 0.5880640745162964, "mix_mean": 0.6484496593475342, "mix_low": 0.02278900146484375, "mix_high": 0.0324554443359375, "read_cosine": 0.016896888613700867, "beta_mean": 0.001073028426617384, "nonfinite": 0.0, "weighted_memory_to_active": 0.15066833794116974, "mix_std": 0.2929335832595825, "beta_std": 0.011743206530809402, "beta_low": 0.99102783203125, "beta_high": 0.0, "orthogonal_to_active": 0.1314321756362915, "evidence_weights": [-0.3164448142051697, -0.24449437856674194, 1.665152668952942], "agreement_weight": 5.633092880249023, "orthogonal_weight": -0.9712687134742737}, {"head": 4, "memory_to_active": 0.40537792444229126, "contribution_to_active": 0.24691306054592133, "mix_mean": 0.33469638228416443, "mix_low": 0.0053768157958984375, "mix_high": 9.5367431640625e-06, "read_cosine": 0.8861550092697144, "beta_mean": 0.07272730022668839, "nonfinite": 0.0, "weighted_memory_to_active": 0.09901124984025955, "mix_std": 0.21669770777225494, "beta_std": 0.05135570466518402, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.1392296850681305, "evidence_weights": [-0.08769533783197403, -0.23139937222003937, 0.4631067216396332], "agreement_weight": 0.005839204415678978, "orthogonal_weight": 2.2215120792388916}, {"head": 5, "memory_to_active": 0.0600285530090332, "contribution_to_active": 0.7023974657058716, "mix_mean": 0.6491342782974243, "mix_low": 0.002597808837890625, "mix_high": 0.010152816772460938, "read_cosine": 0.32239559292793274, "beta_mean": 0.0002772419829852879, "nonfinite": 0.0, "weighted_memory_to_active": 0.040199391543865204, "mix_std": 0.2714630961418152, "beta_std": 0.00257780565880239, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.03383258357644081, "evidence_weights": [-0.22258779406547546, -0.3039970099925995, 0.8899213075637817], "agreement_weight": 0.173116996884346, "orthogonal_weight": -0.9254828691482544}]}, {"layer": 14, "memory_to_active": 0.2334078550338745, "contribution_to_active": 0.4217262864112854, "mix_mean": 0.4236077666282654, "mix_low": 0.028906505554914474, "mix_high": 0.005906741134822369, "read_cosine": 0.6176167726516724, "beta_mean": 0.06568256765604019, "nonfinite": 0.0, "weighted_memory_to_active": 0.09840484708547592, "mix_std": 0.24403943121433258, "beta_std": 0.062215130776166916, "beta_low": 0.4939982295036316, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.04081355407834053, "output_influence": 0.35359102487564087, "residual_influence": 0.13801589608192444, "state_active": 0.05769610404968262, "state_memory": 0.016336608678102493, "state_peak": 40.736446380615234, "growth_active": 1.0802465677261353, "growth_memory": 1.2543879747390747, "heads": [{"head": 0, "memory_to_active": 0.3731042444705963, "contribution_to_active": 0.19225825369358063, "mix_mean": 0.1806580275297165, "mix_low": 0.023941993713378906, "mix_high": 9.5367431640625e-06, "read_cosine": 0.8180826306343079, "beta_mean": 0.06935437023639679, "nonfinite": 0.0, "weighted_memory_to_active": 0.08085552603006363, "mix_std": 0.1768237054347992, "beta_std": 0.06279917061328888, "beta_low": 0.01177978515625, "beta_high": 0.0, "orthogonal_to_active": 0.07587367296218872, "evidence_weights": [-0.1154286190867424, -0.04408865049481392, 0.4434010088443756], "agreement_weight": 0.022911498323082924, "orthogonal_weight": 1.6723216772079468}, {"head": 1, "memory_to_active": 0.47087562084198, "contribution_to_active": 0.29172283411026, "mix_mean": 0.21328431367874146, "mix_low": 0.1335735321044922, "mix_high": 2.193450927734375e-05, "read_cosine": 0.8482056260108948, "beta_mean": 0.2733941674232483, "nonfinite": 0.0, "weighted_memory_to_active": 0.20757551491260529, "mix_std": 0.24376888573169708, "beta_std": 0.2566814720630646, "beta_low": 0.09417724609375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.014469956047832966, "evidence_weights": [-0.003938388545066118, -0.1948530524969101, -0.090367391705513], "agreement_weight": 0.056203968822956085, "orthogonal_weight": 0.18584394454956055}, {"head": 2, "memory_to_active": 0.125030979514122, "contribution_to_active": 0.5814948081970215, "mix_mean": 0.5885812640190125, "mix_low": 0.0029163360595703125, "mix_high": 0.003962516784667969, "read_cosine": 0.4111687242984772, "beta_mean": 0.0022280761040747166, "nonfinite": 0.0, "weighted_memory_to_active": 0.08142586052417755, "mix_std": 0.26697659492492676, "beta_std": 0.003068420337513089, "beta_low": 0.9716796875, "beta_high": 0.0, "orthogonal_to_active": 0.06361709535121918, "evidence_weights": [-0.13913582265377045, -0.31497588753700256, 0.9159348011016846], "agreement_weight": 0.08025526255369186, "orthogonal_weight": -0.8884409666061401}, {"head": 3, "memory_to_active": 0.04734800010919571, "contribution_to_active": 0.6221615076065063, "mix_mean": 0.6691283583641052, "mix_low": 0.004630088806152344, "mix_high": 0.020499229431152344, "read_cosine": -0.013653039932250977, "beta_mean": 0.0052680764347314835, "nonfinite": 0.0, "weighted_memory_to_active": 0.031095314770936966, "mix_std": 0.2564329504966736, "beta_std": 0.007187037728726864, "beta_low": 0.85894775390625, "beta_high": 0.0, "orthogonal_to_active": 0.02210458368062973, "evidence_weights": [-0.34851643443107605, -0.31256103515625, 0.2815626263618469], "agreement_weight": 0.07306312024593353, "orthogonal_weight": -0.8235687017440796}, {"head": 4, "memory_to_active": 0.16685540974140167, "contribution_to_active": 0.49826106429100037, "mix_mean": 0.512611448764801, "mix_low": 0.004057884216308594, "mix_high": 0.008863449096679688, "read_cosine": 0.8872734904289246, "beta_mean": 0.04172114282846451, "nonfinite": 0.0, "weighted_memory_to_active": 0.09865046292543411, "mix_std": 0.2627248167991638, "beta_std": 0.03010031022131443, "beta_low": 0.06658935546875, "beta_high": 0.0, "orthogonal_to_active": 0.026730084791779518, "evidence_weights": [-0.218115895986557, -0.24558675289154053, 0.4298393428325653], "agreement_weight": 0.1310453861951828, "orthogonal_weight": -0.6118410229682922}, {"head": 5, "memory_to_active": 0.2172328531742096, "contribution_to_active": 0.34445908665657043, "mix_mean": 0.3773830533027649, "mix_low": 0.004319190979003906, "mix_high": 0.0020837783813476562, "read_cosine": 0.7546228766441345, "beta_mean": 0.002129557076841593, "nonfinite": 0.0, "weighted_memory_to_active": 0.09082641452550888, "mix_std": 0.25750958919525146, "beta_std": 0.013454379513859749, "beta_low": 0.9608154296875, "beta_high": 0.0, "orthogonal_to_active": 0.04208594560623169, "evidence_weights": [-0.17804673314094543, -0.17371845245361328, 0.30397236347198486], "agreement_weight": 0.0776500403881073, "orthogonal_weight": -0.7386239171028137}]}, {"layer": 15, "memory_to_active": 0.0559675469994545, "contribution_to_active": 0.49166423082351685, "mix_mean": 0.4092729091644287, "mix_low": 0.00908263586461544, "mix_high": 0.002829392906278372, "read_cosine": 0.6780651807785034, "beta_mean": 0.030629029497504234, "nonfinite": 0.0, "weighted_memory_to_active": 0.027680937200784683, "mix_std": 0.2843678891658783, "beta_std": 0.03903127461671829, "beta_low": 0.7152303457260132, "beta_high": 0.0, "orthogonal_to_active": 0.007021602243185043, "output_influence": 0.3096502721309662, "residual_influence": 0.178310364484787, "state_active": 0.1512712836265564, "state_memory": 0.010731445625424385, "state_peak": 38.540618896484375, "growth_active": 1.227161169052124, "growth_memory": 1.3704696893692017, "heads": [{"head": 0, "memory_to_active": 0.10076883435249329, "contribution_to_active": 0.48353785276412964, "mix_mean": 0.38033393025398254, "mix_low": 0.007714271545410156, "mix_high": 0.0057926177978515625, "read_cosine": 0.6265410780906677, "beta_mean": 0.03844527527689934, "nonfinite": 0.0, "weighted_memory_to_active": 0.05019570514559746, "mix_std": 0.28122425079345703, "beta_std": 0.03268235921859741, "beta_low": 0.10125732421875, "beta_high": 0.0, "orthogonal_to_active": 0.02373368665575981, "evidence_weights": [-0.1917935460805893, -0.19454999268054962, 0.4377893805503845], "agreement_weight": 0.093276247382164, "orthogonal_weight": -0.7572354674339294}, {"head": 1, "memory_to_active": 0.12870612740516663, "contribution_to_active": 0.4128988981246948, "mix_mean": 0.4302525222301483, "mix_low": 0.001850128173828125, "mix_high": 0.005999565124511719, "read_cosine": 0.9243968725204468, "beta_mean": 0.0007308778003789485, "nonfinite": 0.0, "weighted_memory_to_active": 0.05358058214187622, "mix_std": 0.2877163290977478, "beta_std": 0.0026587247848510742, "beta_low": 0.99273681640625, "beta_high": 0.0, "orthogonal_to_active": 0.0015387299936264753, "evidence_weights": [-0.14712008833885193, -0.1556825339794159, 0.17841878533363342], "agreement_weight": 0.12637394666671753, "orthogonal_weight": 0.05707180127501488}, {"head": 2, "memory_to_active": 0.004365528933703899, "contribution_to_active": 0.49646973609924316, "mix_mean": 0.3306492269039154, "mix_low": 0.014832496643066406, "mix_high": 8.96453857421875e-05, "read_cosine": 0.7894119024276733, "beta_mean": 5.6182128901127726e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00234029907733202, "mix_std": 0.2674245238304138, "beta_std": 9.852467337623239e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00062507763504982, "evidence_weights": [-0.20676757395267487, -0.26720547676086426, 0.12895916402339935], "agreement_weight": 0.009861893020570278, "orthogonal_weight": 0.7178363800048828}, {"head": 3, "memory_to_active": 0.001609002472832799, "contribution_to_active": 0.3806975483894348, "mix_mean": 0.3869103193283081, "mix_low": 0.0060405731201171875, "mix_high": 0.00023937225341796875, "read_cosine": 0.6127246618270874, "beta_mean": 8.743811122258194e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005266491789370775, "mix_std": 0.28776079416275024, "beta_std": 1.8331089449930005e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00040296942461282015, "evidence_weights": [-0.26301276683807373, -0.24359244108200073, 0.20832479000091553], "agreement_weight": 0.047199588268995285, "orthogonal_weight": -0.7436913847923279}, {"head": 4, "memory_to_active": 0.07554684579372406, "contribution_to_active": 0.6574227809906006, "mix_mean": 0.36537492275238037, "mix_low": 0.014081001281738281, "mix_high": 0.00144195556640625, "read_cosine": 0.7832337021827698, "beta_mean": 0.14265400171279907, "nonfinite": 0.0, "weighted_memory_to_active": 0.04387017339468002, "mix_std": 0.3069518804550171, "beta_std": 0.1854400932788849, "beta_low": 0.22515869140625, "beta_high": 0.0, "orthogonal_to_active": 0.00636414485052228, "evidence_weights": [-0.20908772945404053, -0.2534780204296112, -0.005135831423103809], "agreement_weight": 0.014128044247627258, "orthogonal_weight": 0.34870561957359314}, {"head": 5, "memory_to_active": 0.024808917194604874, "contribution_to_active": 0.5189585089683533, "mix_mean": 0.5621166229248047, "mix_low": 0.009977340698242188, "mix_high": 0.0034132003784179688, "read_cosine": 0.33208274841308594, "beta_mean": 0.0018790821777656674, "nonfinite": 0.0, "weighted_memory_to_active": 0.015572210773825645, "mix_std": 0.2751295566558838, "beta_std": 0.013289613649249077, "beta_low": 0.97222900390625, "beta_high": 0.0, "orthogonal_to_active": 0.009465005248785019, "evidence_weights": [-0.2898441255092621, -0.3063915967941284, 0.7031051516532898], "agreement_weight": 0.3048420548439026, "orthogonal_weight": 0.7987809777259827}]}]}, {"step": 54000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.766391046001445, "memory_to_active": 0.8038839101791382, "contribution_to_active": 0.5538539886474609, "mix_mean": 0.41034621000289917, "mix_low": 0.1338222771883011, "mix_high": 0.010333488695323467, "read_cosine": 0.34396517276763916, "beta_mean": 0.07114843279123306, "nonfinite": 0.0, "weighted_memory_to_active": 0.2042839378118515, "mix_std": 0.22262252867221832, "beta_std": 0.06228096783161163, "beta_low": 0.5611381530761719, "beta_high": 1.843770223786123e-05, "orthogonal_to_active": 0.0589633509516716, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0868380069732666, "normal_token_accuracy": 0.4605116844177246, "state_active": 0.06418168544769287, "state_memory": 0.01794843003153801, "state_peak": 121.12355041503906, "growth_active": 1.1333496570587158, "growth_memory": 1.3300292491912842, "output_influence": 0.4210817217826843, "residual_influence": 0.14226189255714417, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.219113826751709, "no_memory_loss_delta": 0.5752263069152832, "scrambled_loss_delta": 0.14853867888450623, "memory_only_loss_delta": 3.343950033187866, "no_evidence_loss_delta": 0.3513089120388031, "no_agreement_loss_delta": 0.04779580980539322, "no_orthogonal_loss_delta": 0.01537550799548626}, {"index": 94, "tokens": 256, "normal_loss": 3.2330117225646973, "no_memory_loss_delta": 0.5941968560218811, "scrambled_loss_delta": 0.32277610898017883, "memory_only_loss_delta": 3.4653992652893066, "no_evidence_loss_delta": 0.2974947988986969, "no_agreement_loss_delta": 0.053228914737701416, "no_orthogonal_loss_delta": 0.033246442675590515}, {"index": 188, "tokens": 256, "normal_loss": 3.384143114089966, "no_memory_loss_delta": 0.7313593626022339, "scrambled_loss_delta": 0.16614043712615967, "memory_only_loss_delta": 3.0814361572265625, "no_evidence_loss_delta": 0.38421908020973206, "no_agreement_loss_delta": 0.0505729578435421, "no_orthogonal_loss_delta": 0.04756638780236244}, {"index": 283, "tokens": 131, "normal_loss": 1.961700677871704, "no_memory_loss_delta": 0.32142403721809387, "scrambled_loss_delta": 0.19364193081855774, "memory_only_loss_delta": 2.640004873275757, "no_evidence_loss_delta": 0.13403841853141785, "no_agreement_loss_delta": 0.008641519583761692, "no_orthogonal_loss_delta": -0.026840897276997566}], "agreement_weight_abs_mean": 0.21249359846115112, "agreement_weight_abs_max": 5.717923164367676, "agreement_weight_mean": 0.1858624815940857, "orthogonal_weight_abs_mean": 0.858016312122345, "orthogonal_weight_abs_max": 9.164284706115723, "orthogonal_weight_mean": -0.11364889144897461, "no_memory_loss": 3.6749439239501953, "no_memory_loss_delta": 0.5881054997444153, "no_memory_token_accuracy_gain": 0.08898776769638062, "no_memory_token_win_fraction": 0.6551724076271057, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 44.46215466777052, "scrambled_loss": 3.2965774536132812, "scrambled_loss_delta": 0.20973928272724152, "scrambled_token_accuracy_gain": 0.031145717948675156, "scrambled_token_win_fraction": 0.6351501941680908, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.92043931643456, "memory_only_loss": 6.288041591644287, "memory_only_loss_delta": 3.2012033462524414, "memory_only_token_accuracy_gain": 0.30589544773101807, "memory_only_token_win_fraction": 0.8965517282485962, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.92868175661711, "no_evidence_loss": 3.4005343914031982, "no_evidence_loss_delta": 0.3136962056159973, "no_evidence_token_accuracy_gain": 0.046718575060367584, "no_evidence_token_win_fraction": 0.6273637413978577, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 26.925901054882335, "no_agreement_loss": 3.1312663555145264, "no_agreement_loss_delta": 0.04442830756306648, "no_agreement_token_accuracy_gain": 0.015572858974337578, "no_agreement_token_win_fraction": 0.5094549655914307, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 4.345582538163303, "no_orthogonal_loss": 3.1103172302246094, "no_orthogonal_loss_delta": 0.023479482159018517, "no_orthogonal_token_accuracy_gain": 0.01001112349331379, "no_orthogonal_token_win_fraction": 0.57953280210495, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.3205983832465638, "short_no_memory_loss_delta": 0.5251581072807312, "long_context_memory_gain": 0.06294739246368408, "normal_context_loss_gain": 0.13855305314064026, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.4732248783111572, "contribution_to_active": 1.1833252906799316, "mix_mean": 0.10576917231082916, "mix_low": 0.5372046232223511, "mix_high": 0.005594412796199322, "read_cosine": 0.4212959408760071, "beta_mean": 0.2115599513053894, "nonfinite": 0.0, "weighted_memory_to_active": 1.21560537815094, "mix_std": 0.08946487307548523, "beta_std": 0.14519912004470825, "beta_low": 0.0632222518324852, "beta_high": 0.0, "orthogonal_to_active": 0.04460245370864868, "output_influence": 0.21074742078781128, "residual_influence": 0.2470972239971161, "state_active": 0.06515708565711975, "state_memory": 0.04503971338272095, "state_peak": 48.42584228515625, "growth_active": 1.2258647680282593, "growth_memory": 1.2913520336151123, "heads": [{"head": 0, "memory_to_active": 2.6528637409210205, "contribution_to_active": 0.4651888608932495, "mix_mean": 0.01910930499434471, "mix_low": 0.6335077285766602, "mix_high": 2.86102294921875e-06, "read_cosine": 0.21329720318317413, "beta_mean": 0.2036118358373642, "nonfinite": 0.0, "weighted_memory_to_active": 0.5800322890281677, "mix_std": 0.05850525200366974, "beta_std": 0.10309118032455444, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06040224805474281, "evidence_weights": [-0.005709330551326275, -0.04960884898900986, -0.06319563090801239], "agreement_weight": -0.19773630797863007, "orthogonal_weight": -0.11773088574409485}, {"head": 1, "memory_to_active": 6.324731826782227, "contribution_to_active": 2.8100881576538086, "mix_mean": 0.020060501992702484, "mix_low": 0.6110973358154297, "mix_high": 0.0, "read_cosine": 0.431184321641922, "beta_mean": 0.44059106707572937, "nonfinite": 0.0, "weighted_memory_to_active": 3.077817440032959, "mix_std": 0.05620510131120682, "beta_std": 0.22583100199699402, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02627859078347683, "evidence_weights": [0.021028775721788406, -0.10101896524429321, 0.5914870500564575], "agreement_weight": 0.024761812761425972, "orthogonal_weight": 0.018480241298675537}, {"head": 2, "memory_to_active": 2.5263586044311523, "contribution_to_active": 0.4766260087490082, "mix_mean": 0.014252522960305214, "mix_low": 0.6619892120361328, "mix_high": 0.0, "read_cosine": 0.31849387288093567, "beta_mean": 0.22635699808597565, "nonfinite": 0.0, "weighted_memory_to_active": 0.5586232542991638, "mix_std": 0.03150274232029915, "beta_std": 0.1932900846004486, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04295225068926811, "evidence_weights": [0.06430907547473907, 0.05526137724518776, 0.15206991136074066], "agreement_weight": 0.009758736938238144, "orthogonal_weight": -0.1949702948331833}, {"head": 3, "memory_to_active": 7.46101188659668, "contribution_to_active": 2.821990966796875, "mix_mean": 0.030170917510986328, "mix_low": 0.5308027267456055, "mix_high": 0.0, "read_cosine": 0.2911352813243866, "beta_mean": 0.3200068473815918, "nonfinite": 0.0, "weighted_memory_to_active": 2.9732859134674072, "mix_std": 0.07336734980344772, "beta_std": 0.24827302992343903, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11934157460927963, "evidence_weights": [-0.041258327662944794, -0.04103869944810867, -0.1444224864244461], "agreement_weight": -0.012323427945375443, "orthogonal_weight": -0.046967241913080215}, {"head": 4, "memory_to_active": 1.764004111289978, "contribution_to_active": 0.06491563469171524, "mix_mean": 0.009683479554951191, "mix_low": 0.7714357376098633, "mix_high": 0.0, "read_cosine": 0.32755953073501587, "beta_mean": 0.05777835100889206, "nonfinite": 0.0, "weighted_memory_to_active": 0.0463547557592392, "mix_std": 0.030506126582622528, "beta_std": 0.08196272701025009, "beta_low": 0.0943603515625, "beta_high": 0.0, "orthogonal_to_active": 0.0031098031904548407, "evidence_weights": [0.14572955667972565, 0.10507047921419144, -0.16382473707199097], "agreement_weight": -0.14348657429218292, "orthogonal_weight": 0.057935480028390884}, {"head": 5, "memory_to_active": 0.11037794500589371, "contribution_to_active": 0.461141437292099, "mix_mean": 0.5413382649421692, "mix_low": 0.014394760131835938, "mix_high": 0.03356361389160156, "read_cosine": 0.9461051225662231, "beta_mean": 0.021014496684074402, "nonfinite": 0.0, "weighted_memory_to_active": 0.057517893612384796, "mix_std": 0.28670263290405273, "beta_std": 0.018746674060821533, "beta_low": 0.28497314453125, "beta_high": 0.0, "orthogonal_to_active": 0.015530252829194069, "evidence_weights": [-0.18158705532550812, -0.21852239966392517, 0.3809817433357239], "agreement_weight": 0.2913118302822113, "orthogonal_weight": -0.6663339138031006}]}, {"layer": 1, "memory_to_active": 2.320798635482788, "contribution_to_active": 0.4729368984699249, "mix_mean": 0.20966872572898865, "mix_low": 0.3572613596916199, "mix_high": 0.0009770393371582031, "read_cosine": 0.29190847277641296, "beta_mean": 0.18150794506072998, "nonfinite": 0.0, "weighted_memory_to_active": 0.3139995336532593, "mix_std": 0.11483081430196762, "beta_std": 0.13239088654518127, "beta_low": 0.3440144956111908, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.04630621522665024, "output_influence": 0.2426624894142151, "residual_influence": 0.06537504494190216, "state_active": 0.028110861778259277, "state_memory": 0.03552402928471565, "state_peak": 36.29005432128906, "growth_active": 1.1629329919815063, "growth_memory": 1.464905858039856, "heads": [{"head": 0, "memory_to_active": 0.05600976198911667, "contribution_to_active": 0.48189279437065125, "mix_mean": 0.5062273144721985, "mix_low": 0.005967140197753906, "mix_high": 0.0017414093017578125, "read_cosine": -0.008650255389511585, "beta_mean": 0.00032266799826174974, "nonfinite": 0.0, "weighted_memory_to_active": 0.035779282450675964, "mix_std": 0.27981486916542053, "beta_std": 0.0006102685001678765, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.035061340779066086, "evidence_weights": [-0.20776404440402985, -0.26913610100746155, 0.3748842775821686], "agreement_weight": 0.2084778994321823, "orthogonal_weight": -1.028069019317627}, {"head": 1, "memory_to_active": 2.8662774562835693, "contribution_to_active": 0.2644139230251312, "mix_mean": 0.02391858398914337, "mix_low": 0.44975852966308594, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5071778893470764, "beta_mean": 0.2851797640323639, "nonfinite": 0.0, "weighted_memory_to_active": 0.2903014123439789, "mix_std": 0.046209175139665604, "beta_std": 0.19308863580226898, "beta_low": 0.00323486328125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.021686755120754242, "evidence_weights": [0.015335118398070335, -0.054334405809640884, 0.04444780573248863], "agreement_weight": -0.006286474876105785, "orthogonal_weight": 0.17957039177417755}, {"head": 2, "memory_to_active": 0.07069358974695206, "contribution_to_active": 0.5593194961547852, "mix_mean": 0.6653217077255249, "mix_low": 0.0005197525024414062, "mix_high": 0.004119873046875, "read_cosine": 0.48709791898727417, "beta_mean": 0.003271793946623802, "nonfinite": 0.0, "weighted_memory_to_active": 0.05076940357685089, "mix_std": 0.23302005231380463, "beta_std": 0.006601270288228989, "beta_low": 0.9449462890625, "beta_high": 0.0, "orthogonal_to_active": 0.033218059688806534, "evidence_weights": [-0.28463974595069885, -0.3703559935092926, 0.6632669568061829], "agreement_weight": 0.03989043086767197, "orthogonal_weight": -0.7725328803062439}, {"head": 3, "memory_to_active": 2.8866195678710938, "contribution_to_active": 0.16272751986980438, "mix_mean": 0.023040641099214554, "mix_low": 0.5613899230957031, "mix_high": 0.0, "read_cosine": 0.19392521679401398, "beta_mean": 0.3213757276535034, "nonfinite": 0.0, "weighted_memory_to_active": 0.24704144895076752, "mix_std": 0.058489248156547546, "beta_std": 0.2550407946109772, "beta_low": 0.0589599609375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08374977111816406, "evidence_weights": [0.03892543539404869, -0.01850360445678234, 0.0008953113574534655], "agreement_weight": -0.02798030711710453, "orthogonal_weight": -0.4734753370285034}, {"head": 4, "memory_to_active": 3.8908486366271973, "contribution_to_active": 1.281670331954956, "mix_mean": 0.03214603289961815, "mix_low": 0.3178672790527344, "mix_high": 0.0, "read_cosine": 0.28530174493789673, "beta_mean": 0.1831163913011551, "nonfinite": 0.0, "weighted_memory_to_active": 1.1964737176895142, "mix_std": 0.05420289561152458, "beta_std": 0.12845836579799652, "beta_low": 0.03033447265625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08554799854755402, "evidence_weights": [-0.029191946610808372, -0.0015482025919482112, 0.09515269845724106], "agreement_weight": -0.01679687388241291, "orthogonal_weight": 0.10105949640274048}, {"head": 5, "memory_to_active": 4.1543426513671875, "contribution_to_active": 0.08759719133377075, "mix_mean": 0.007358030881732702, "mix_low": 0.8080654144287109, "mix_high": 0.0, "read_cosine": 0.2865982949733734, "beta_mean": 0.2957812249660492, "nonfinite": 0.0, "weighted_memory_to_active": 0.06363194435834885, "mix_std": 0.017248626798391342, "beta_std": 0.21054594218730927, "beta_low": 0.02728271484375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.018573354929685593, "evidence_weights": [0.1543356031179428, 0.07551930844783783, 0.12066546827554703], "agreement_weight": -0.11525989323854446, "orthogonal_weight": 0.29746344685554504}]}, {"layer": 2, "memory_to_active": 1.0636059045791626, "contribution_to_active": 0.3526078462600708, "mix_mean": 0.3957027494907379, "mix_low": 0.24929270148277283, "mix_high": 0.004080136772245169, "read_cosine": 0.30485376715660095, "beta_mean": 0.08411939442157745, "nonfinite": 0.0, "weighted_memory_to_active": 0.07921207696199417, "mix_std": 0.19036680459976196, "beta_std": 0.0672050416469574, "beta_low": 0.57818603515625, "beta_high": 7.120767986634746e-05, "orthogonal_to_active": 0.019960161298513412, "output_influence": 0.376322478055954, "residual_influence": 0.10114972293376923, "state_active": 0.03892152011394501, "state_memory": 0.015743885189294815, "state_peak": 25.593294143676758, "growth_active": 1.0480051040649414, "growth_memory": 1.607839822769165, "heads": [{"head": 0, "memory_to_active": 0.01545646507292986, "contribution_to_active": 0.3590405285358429, "mix_mean": 0.6191668510437012, "mix_low": 0.000713348388671875, "mix_high": 0.008074760437011719, "read_cosine": 0.11889831721782684, "beta_mean": 9.045127080753446e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010022778995335102, "mix_std": 0.2737361192703247, "beta_std": 0.0001610633044037968, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010414479300379753, "evidence_weights": [-0.36880579590797424, -0.2172664999961853, 0.143469899892807], "agreement_weight": 0.12473250180482864, "orthogonal_weight": -1.0752813816070557}, {"head": 1, "memory_to_active": 3.2544023990631104, "contribution_to_active": 0.03880881890654564, "mix_mean": 0.0045339008793234825, "mix_low": 0.9093399047851562, "mix_high": 0.0, "read_cosine": 0.12961731851100922, "beta_mean": 0.24447737634181976, "nonfinite": 0.0, "weighted_memory_to_active": 0.029934026300907135, "mix_std": 0.010111035779118538, "beta_std": 0.15662896633148193, "beta_low": 0.001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.009460103698074818, "evidence_weights": [0.14078719913959503, 0.12269512563943863, 0.5595324039459229], "agreement_weight": 0.02034153789281845, "orthogonal_weight": 0.3531094193458557}, {"head": 2, "memory_to_active": 0.009271093644201756, "contribution_to_active": 0.433008074760437, "mix_mean": 0.5320877432823181, "mix_low": 0.001323699951171875, "mix_high": 0.0025854110717773438, "read_cosine": 0.2974282503128052, "beta_mean": 2.4698509150766768e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038171415217220783, "mix_std": 0.2778260111808777, "beta_std": 3.980036126449704e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00030635346774943173, "evidence_weights": [-0.25050050020217896, -0.2913598120212555, 0.5801100134849548], "agreement_weight": -0.018179332837462425, "orthogonal_weight": -0.09616205096244812}, {"head": 3, "memory_to_active": 0.1412537395954132, "contribution_to_active": 0.5364567637443542, "mix_mean": 0.5670941472053528, "mix_low": 0.00023365020751953125, "mix_high": 0.0067348480224609375, "read_cosine": 0.6514992117881775, "beta_mean": 0.023100344464182854, "nonfinite": 0.0, "weighted_memory_to_active": 0.09029335528612137, "mix_std": 0.2603306770324707, "beta_std": 0.0292048417031765, "beta_low": 0.43035888671875, "beta_high": 0.0, "orthogonal_to_active": 0.05913052707910538, "evidence_weights": [-0.20169182121753693, -0.3164709508419037, 0.8150607943534851], "agreement_weight": 0.06095414236187935, "orthogonal_weight": -0.8960896730422974}, {"head": 4, "memory_to_active": 2.9501168727874756, "contribution_to_active": 0.21601669490337372, "mix_mean": 0.02043277956545353, "mix_low": 0.5830564498901367, "mix_high": 0.0, "read_cosine": 0.2023695856332779, "beta_mean": 0.236978679895401, "nonfinite": 0.0, "weighted_memory_to_active": 0.3345942199230194, "mix_std": 0.052105143666267395, "beta_std": 0.21713367104530334, "beta_low": 0.03704833984375, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.03965818136930466, "evidence_weights": [0.021726001054048538, 0.007021215744316578, 0.03285091742873192], "agreement_weight": 0.024155044928193092, "orthogonal_weight": -0.15922722220420837}, {"head": 5, "memory_to_active": 0.011134395375847816, "contribution_to_active": 0.5323158502578735, "mix_mean": 0.6309010982513428, "mix_low": 0.0010890960693359375, "mix_high": 0.0070858001708984375, "read_cosine": 0.4293099343776703, "beta_mean": 4.481024734559469e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006610944867134094, "mix_std": 0.26809176802635193, "beta_std": 6.191412830958143e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007913218578323722, "evidence_weights": [-0.27844926714897156, -0.2898286283016205, 1.032738208770752], "agreement_weight": 0.1646614521741867, "orthogonal_weight": -0.1643105149269104}]}, {"layer": 3, "memory_to_active": 0.46764183044433594, "contribution_to_active": 0.4593390226364136, "mix_mean": 0.48821237683296204, "mix_low": 0.0627598762512207, "mix_high": 0.010535717010498047, "read_cosine": 0.199318528175354, "beta_mean": 0.028932934626936913, "nonfinite": 0.0, "weighted_memory_to_active": 0.12127359211444855, "mix_std": 0.2596921920776367, "beta_std": 0.025906365364789963, "beta_low": 0.7572022080421448, "beta_high": 0.0, "orthogonal_to_active": 0.05420884117484093, "output_influence": 0.40055274963378906, "residual_influence": 0.0894402340054512, "state_active": 0.023422464728355408, "state_memory": 0.007329033687710762, "state_peak": 22.6108341217041, "growth_active": 1.0537583827972412, "growth_memory": 1.697339415550232, "heads": [{"head": 0, "memory_to_active": 0.4033789038658142, "contribution_to_active": 0.5833210349082947, "mix_mean": 0.5805501937866211, "mix_low": 0.0023345947265625, "mix_high": 0.02233409881591797, "read_cosine": 0.24903401732444763, "beta_mean": 0.011483538895845413, "nonfinite": 0.0, "weighted_memory_to_active": 0.24516397714614868, "mix_std": 0.30875471234321594, "beta_std": 0.013999348506331444, "beta_low": 0.642333984375, "beta_high": 0.0, "orthogonal_to_active": 0.17325499653816223, "evidence_weights": [-0.21490231156349182, -0.2487160861492157, 0.9450815916061401], "agreement_weight": 0.110469289124012, "orthogonal_weight": -0.7499196529388428}, {"head": 1, "memory_to_active": 0.01264730840921402, "contribution_to_active": 0.42357325553894043, "mix_mean": 0.5576307773590088, "mix_low": 0.004660606384277344, "mix_high": 0.008304595947265625, "read_cosine": 0.1804666668176651, "beta_mean": 6.976416625548154e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007313667330890894, "mix_std": 0.2999054193496704, "beta_std": 0.0001673554361332208, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005820753984153271, "evidence_weights": [-0.26437312364578247, -0.21535582840442657, 0.6531729102134705], "agreement_weight": 0.49029892683029175, "orthogonal_weight": -0.8656417727470398}, {"head": 2, "memory_to_active": 2.115447759628296, "contribution_to_active": 0.24205473065376282, "mix_mean": 0.04571283608675003, "mix_low": 0.3537254333496094, "mix_high": 0.0, "read_cosine": 0.1955849528312683, "beta_mean": 0.1567080318927765, "nonfinite": 0.0, "weighted_memory_to_active": 0.2761053442955017, "mix_std": 0.0937180146574974, "beta_std": 0.12995930016040802, "beta_low": 0.01861572265625, "beta_high": 0.0, "orthogonal_to_active": 0.004394529853016138, "evidence_weights": [-0.08762819319963455, 0.005882550030946732, 0.00813988409936428], "agreement_weight": -0.01169560942798853, "orthogonal_weight": -0.01997038722038269}, {"head": 3, "memory_to_active": 0.02344176359474659, "contribution_to_active": 0.5205765962600708, "mix_mean": 0.5973129272460938, "mix_low": 0.0052642822265625, "mix_high": 0.0119476318359375, "read_cosine": 0.29093050956726074, "beta_mean": 9.424159361515194e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01616603694856167, "mix_std": 0.29294660687446594, "beta_std": 0.00022425469069276005, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004393074195832014, "evidence_weights": [-0.2516334652900696, -0.23324204981327057, 2.1661734580993652], "agreement_weight": 1.9987080097198486, "orthogonal_weight": -0.356391042470932}, {"head": 4, "memory_to_active": 0.0132994893938303, "contribution_to_active": 0.3463495671749115, "mix_mean": 0.4373771548271179, "mix_low": 0.007990837097167969, "mix_high": 0.0013055801391601562, "read_cosine": 0.16012562811374664, "beta_mean": 8.550616621505469e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008335860446095467, "mix_std": 0.288000226020813, "beta_std": 0.00014124528388492763, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007873181253671646, "evidence_weights": [-0.19924215972423553, -0.19903579354286194, 0.10392240434885025], "agreement_weight": -0.04789528623223305, "orthogonal_weight": -1.015028715133667}, {"head": 5, "memory_to_active": 0.2376357614994049, "contribution_to_active": 0.6401587724685669, "mix_mean": 0.7106902599334717, "mix_low": 0.0025835037231445312, "mix_high": 0.01932239532470703, "read_cosine": 0.1197693794965744, "beta_mean": 0.005156512372195721, "nonfinite": 0.0, "weighted_memory_to_active": 0.17455661296844482, "mix_std": 0.2748282551765442, "beta_std": 0.010946691036224365, "beta_low": 0.88226318359375, "beta_high": 0.0, "orthogonal_to_active": 0.12951651215553284, "evidence_weights": [-0.26205146312713623, -0.33107730746269226, 0.4706532955169678], "agreement_weight": 0.1584140956401825, "orthogonal_weight": -0.7716544270515442}]}, {"layer": 4, "memory_to_active": 1.0076966285705566, "contribution_to_active": 0.46228450536727905, "mix_mean": 0.3631223440170288, "mix_low": 0.2642582356929779, "mix_high": 0.009228706359863281, "read_cosine": 0.318659245967865, "beta_mean": 0.11437679082155228, "nonfinite": 0.0, "weighted_memory_to_active": 0.13301917910575867, "mix_std": 0.1595436930656433, "beta_std": 0.10507264733314514, "beta_low": 0.3830668330192566, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04731544852256775, "output_influence": 0.47060322761535645, "residual_influence": 0.1375044584274292, "state_active": 0.040341567248106, "state_memory": 0.017664268612861633, "state_peak": 38.20669174194336, "growth_active": 1.2535964250564575, "growth_memory": 1.548780918121338, "heads": [{"head": 0, "memory_to_active": 0.1870311051607132, "contribution_to_active": 0.7660641670227051, "mix_mean": 0.6783519983291626, "mix_low": 0.013452529907226562, "mix_high": 0.022699356079101562, "read_cosine": 0.4808502197265625, "beta_mean": 0.2500581741333008, "nonfinite": 0.0, "weighted_memory_to_active": 0.1551080346107483, "mix_std": 0.34707361459732056, "beta_std": 0.2539485991001129, "beta_low": 0.2413330078125, "beta_high": 0.0, "orthogonal_to_active": 0.0720859169960022, "evidence_weights": [-0.2682546079158783, -0.370692640542984, 0.014573912136256695], "agreement_weight": 0.006527172401547432, "orthogonal_weight": 0.5834956765174866}, {"head": 1, "memory_to_active": 0.05688082426786423, "contribution_to_active": 0.6465029716491699, "mix_mean": 0.6857975125312805, "mix_low": 0.00010585784912109375, "mix_high": 0.015819549560546875, "read_cosine": 0.6137059330940247, "beta_mean": 0.0010837665759027004, "nonfinite": 0.0, "weighted_memory_to_active": 0.04137846827507019, "mix_std": 0.24137276411056519, "beta_std": 0.006247951649129391, "beta_low": 0.98077392578125, "beta_high": 0.0, "orthogonal_to_active": 0.02257639728486538, "evidence_weights": [-0.24108323454856873, -0.3598466217517853, 0.6490199565887451], "agreement_weight": 0.27875953912734985, "orthogonal_weight": -0.7641407251358032}, {"head": 2, "memory_to_active": 1.6848193407058716, "contribution_to_active": 0.2991974949836731, "mix_mean": 0.029045404866337776, "mix_low": 0.49860572814941406, "mix_high": 1.430511474609375e-05, "read_cosine": 0.2351188063621521, "beta_mean": 0.06733827292919159, "nonfinite": 0.0, "weighted_memory_to_active": 0.23435699939727783, "mix_std": 0.06254379451274872, "beta_std": 0.06945442408323288, "beta_low": 0.06195068359375, "beta_high": 0.0, "orthogonal_to_active": 0.055763088166713715, "evidence_weights": [0.053593531250953674, -0.005400775000452995, 0.17899176478385925], "agreement_weight": 0.13630124926567078, "orthogonal_weight": -0.33271926641464233}, {"head": 3, "memory_to_active": 1.8067359924316406, "contribution_to_active": 0.13807688653469086, "mix_mean": 0.03132244944572449, "mix_low": 0.43683338165283203, "mix_high": 0.0, "read_cosine": 0.17781946063041687, "beta_mean": 0.15868380665779114, "nonfinite": 0.0, "weighted_memory_to_active": 0.21203354001045227, "mix_std": 0.057880617678165436, "beta_std": 0.12998445332050323, "beta_low": 0.01434326171875, "beta_high": 0.0, "orthogonal_to_active": 0.0010538563365116715, "evidence_weights": [-0.020949460566043854, 0.051445916295051575, -0.2871031165122986], "agreement_weight": 0.07586313784122467, "orthogonal_weight": -0.008247160352766514}, {"head": 4, "memory_to_active": 0.010079377330839634, "contribution_to_active": 0.6455813646316528, "mix_mean": 0.7391469478607178, "mix_low": 0.0001277923583984375, "mix_high": 0.016834259033203125, "read_cosine": 0.16386254131793976, "beta_mean": 2.4061519070528448e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007345984689891338, "mix_std": 0.2147376537322998, "beta_std": 4.9887996283359826e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003694874234497547, "evidence_weights": [-0.33448970317840576, -0.3250601887702942, 0.5065667033195496], "agreement_weight": 0.33804506063461304, "orthogonal_weight": -0.540130615234375}, {"head": 5, "memory_to_active": 2.300633430480957, "contribution_to_active": 0.2782841920852661, "mix_mean": 0.015069721266627312, "mix_low": 0.6364240646362305, "mix_high": 4.76837158203125e-06, "read_cosine": 0.24059849977493286, "beta_mean": 0.20907264947891235, "nonfinite": 0.0, "weighted_memory_to_active": 0.147892028093338, "mix_std": 0.033653777092695236, "beta_std": 0.17075057327747345, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1287185400724411, "evidence_weights": [0.050947196781635284, -0.026383008807897568, -0.24789713323116302], "agreement_weight": -0.08505726605653763, "orthogonal_weight": 0.9830778241157532}]}, {"layer": 5, "memory_to_active": 0.4220382571220398, "contribution_to_active": 0.522627592086792, "mix_mean": 0.4938472509384155, "mix_low": 0.057775817811489105, "mix_high": 0.01607004925608635, "read_cosine": 0.06419232487678528, "beta_mean": 0.05459197610616684, "nonfinite": 0.0, "weighted_memory_to_active": 0.14413286745548248, "mix_std": 0.257641077041626, "beta_std": 0.07162129878997803, "beta_low": 0.64825439453125, "beta_high": 0.0, "orthogonal_to_active": 0.059442732483148575, "output_influence": 0.5306532382965088, "residual_influence": 0.15192563831806183, "state_active": 0.04343709722161293, "state_memory": 0.010719104669988155, "state_peak": 66.78485870361328, "growth_active": 1.1149959564208984, "growth_memory": 1.2959085702896118, "heads": [{"head": 0, "memory_to_active": 0.3189921975135803, "contribution_to_active": 0.5088694095611572, "mix_mean": 0.40093356370925903, "mix_low": 0.01605224609375, "mix_high": 0.0010519027709960938, "read_cosine": -0.06818187236785889, "beta_mean": 0.027590574696660042, "nonfinite": 0.0, "weighted_memory_to_active": 0.1003599613904953, "mix_std": 0.2756747901439667, "beta_std": 0.04680296406149864, "beta_low": 0.4168701171875, "beta_high": 0.0, "orthogonal_to_active": 0.04063005372881889, "evidence_weights": [-0.14542631804943085, -0.1681285947561264, 0.02302352897822857], "agreement_weight": -0.026905430480837822, "orthogonal_weight": -0.40847155451774597}, {"head": 1, "memory_to_active": 0.010325721465051174, "contribution_to_active": 0.5182031393051147, "mix_mean": 0.6761248111724854, "mix_low": 0.0016326904296875, "mix_high": 0.031378746032714844, "read_cosine": 0.1725747138261795, "beta_mean": 2.474281245667953e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006044433917850256, "mix_std": 0.27792584896087646, "beta_std": 4.8425332352053374e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005829222965985537, "evidence_weights": [-0.24902816116809845, -0.28917795419692993, 0.8951815366744995], "agreement_weight": 0.6684194207191467, "orthogonal_weight": -1.0016674995422363}, {"head": 2, "memory_to_active": 0.012186117470264435, "contribution_to_active": 0.5835635662078857, "mix_mean": 0.6508691310882568, "mix_low": 0.002292633056640625, "mix_high": 0.01976776123046875, "read_cosine": -0.5597308874130249, "beta_mean": 9.126339864451438e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0074288793839514256, "mix_std": 0.267861008644104, "beta_std": 0.0001483106316300109, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00359859736636281, "evidence_weights": [-0.2629152238368988, -0.33719778060913086, 0.25265246629714966], "agreement_weight": -0.03930559381842613, "orthogonal_weight": -0.6162087917327881}, {"head": 3, "memory_to_active": 1.6521323919296265, "contribution_to_active": 0.25296294689178467, "mix_mean": 0.12130829691886902, "mix_low": 0.12517452239990234, "mix_high": 5.7220458984375e-06, "read_cosine": 0.12964430451393127, "beta_mean": 0.11575828492641449, "nonfinite": 0.0, "weighted_memory_to_active": 0.3927486538887024, "mix_std": 0.15880562365055084, "beta_std": 0.16957822442054749, "beta_low": 0.19384765625, "beta_high": 0.0, "orthogonal_to_active": 0.16484209895133972, "evidence_weights": [-0.07522626221179962, -0.10531143844127655, 0.022114858031272888], "agreement_weight": 0.040219586342573166, "orthogonal_weight": -0.4926423132419586}, {"head": 4, "memory_to_active": 0.4312900900840759, "contribution_to_active": 0.583856463432312, "mix_mean": 0.3196024000644684, "mix_low": 0.1994647979736328, "mix_high": 0.0072154998779296875, "read_cosine": 0.5448848009109497, "beta_mean": 0.18375517427921295, "nonfinite": 0.0, "weighted_memory_to_active": 0.28412437438964844, "mix_std": 0.33668363094329834, "beta_std": 0.21154578030109406, "beta_low": 0.28369140625, "beta_high": 0.0, "orthogonal_to_active": 0.09612119197845459, "evidence_weights": [-0.11624251306056976, -0.26323366165161133, -0.049157727509737015], "agreement_weight": 0.0387512668967247, "orthogonal_weight": 0.5188828706741333}, {"head": 5, "memory_to_active": 0.10730291903018951, "contribution_to_active": 0.6883100271224976, "mix_mean": 0.7942452430725098, "mix_low": 0.0020380020141601562, "mix_high": 0.03700065612792969, "read_cosine": 0.1659628450870514, "beta_mean": 0.0003318047383800149, "nonfinite": 0.0, "weighted_memory_to_active": 0.0740908533334732, "mix_std": 0.22889547049999237, "beta_std": 0.0016040876507759094, "beta_low": 0.9951171875, "beta_high": 0.0, "orthogonal_to_active": 0.04563521593809128, "evidence_weights": [-0.3136914074420929, -0.3336656391620636, 1.9392145872116089], "agreement_weight": 0.4719231128692627, "orthogonal_weight": -0.6561977863311768}]}, {"layer": 6, "memory_to_active": 0.2988186776638031, "contribution_to_active": 0.623183012008667, "mix_mean": 0.5864993333816528, "mix_low": 0.05308564752340317, "mix_high": 0.03405841439962387, "read_cosine": 0.30017268657684326, "beta_mean": 0.06135588884353638, "nonfinite": 0.0, "weighted_memory_to_active": 0.14985069632530212, "mix_std": 0.23603220283985138, "beta_std": 0.070108562707901, "beta_low": 0.545989990234375, "beta_high": 0.0, "orthogonal_to_active": 0.04422813281416893, "output_influence": 0.5306165814399719, "residual_influence": 0.1798689067363739, "state_active": 0.04787538945674896, "state_memory": 0.008128401823341846, "state_peak": 95.59095001220703, "growth_active": 1.2547740936279297, "growth_memory": 1.1924936771392822, "heads": [{"head": 0, "memory_to_active": 0.12099315226078033, "contribution_to_active": 0.7406498193740845, "mix_mean": 0.8170733451843262, "mix_low": 0.00030040740966796875, "mix_high": 0.05341529846191406, "read_cosine": 0.19109664857387543, "beta_mean": 0.014775194227695465, "nonfinite": 0.0, "weighted_memory_to_active": 0.10003355890512466, "mix_std": 0.20098832249641418, "beta_std": 0.029262419790029526, "beta_low": 0.66375732421875, "beta_high": 0.0, "orthogonal_to_active": 0.0762578696012497, "evidence_weights": [-0.3182940185070038, -0.34885576367378235, 1.1982769966125488], "agreement_weight": 0.521153450012207, "orthogonal_weight": -0.812166690826416}, {"head": 1, "memory_to_active": 0.006166732870042324, "contribution_to_active": 0.5176272392272949, "mix_mean": 0.5796884894371033, "mix_low": 0.00084686279296875, "mix_high": 0.01598644256591797, "read_cosine": 0.5024488568305969, "beta_mean": 1.4647781426901929e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002777505200356245, "mix_std": 0.28651881217956543, "beta_std": 2.4970615413622e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015308881411328912, "evidence_weights": [-0.1978025883436203, -0.29963308572769165, 1.132622480392456], "agreement_weight": 0.06941715627908707, "orthogonal_weight": -0.7218688130378723}, {"head": 2, "memory_to_active": 0.8489660620689392, "contribution_to_active": 0.09226352721452713, "mix_mean": 0.04000844806432724, "mix_low": 0.30985164642333984, "mix_high": 0.0, "read_cosine": 0.2480432391166687, "beta_mean": 0.03127574175596237, "nonfinite": 0.0, "weighted_memory_to_active": 0.06804481893777847, "mix_std": 0.06400631368160248, "beta_std": 0.028599917888641357, "beta_low": 0.1116943359375, "beta_high": 0.0, "orthogonal_to_active": 0.006131397094577551, "evidence_weights": [0.0762791782617569, -0.03990323841571808, 0.32028454542160034], "agreement_weight": 0.14885447919368744, "orthogonal_weight": -0.1110793948173523}, {"head": 3, "memory_to_active": 0.13601115345954895, "contribution_to_active": 0.6550089716911316, "mix_mean": 0.653638482093811, "mix_low": 0.006468772888183594, "mix_high": 0.019197463989257812, "read_cosine": 0.45509445667266846, "beta_mean": 0.2027035504579544, "nonfinite": 0.0, "weighted_memory_to_active": 0.09603338688611984, "mix_std": 0.3238404393196106, "beta_std": 0.23093107342720032, "beta_low": 0.21209716796875, "beta_high": 0.0, "orthogonal_to_active": 0.05168641358613968, "evidence_weights": [-0.17309312522411346, -0.3279672861099243, 0.1763588786125183], "agreement_weight": -0.003641598392277956, "orthogonal_weight": 0.6131743788719177}, {"head": 4, "memory_to_active": 0.654191792011261, "contribution_to_active": 1.1045947074890137, "mix_mean": 0.751989483833313, "mix_low": 0.0008478164672851562, "mix_high": 0.1082315444946289, "read_cosine": 0.109757199883461, "beta_mean": 0.11921100318431854, "nonfinite": 0.0, "weighted_memory_to_active": 0.613117516040802, "mix_std": 0.2917352318763733, "beta_std": 0.13146470487117767, "beta_low": 0.28857421875, "beta_high": 0.0, "orthogonal_to_active": 0.11096221208572388, "evidence_weights": [-0.22040966153144836, -0.5729846358299255, -0.37951990962028503], "agreement_weight": -0.016778206452727318, "orthogonal_weight": 0.20469793677330017}, {"head": 5, "memory_to_active": 0.02658315747976303, "contribution_to_active": 0.6289539337158203, "mix_mean": 0.6765977144241333, "mix_low": 0.0001983642578125, "mix_high": 0.007519721984863281, "read_cosine": 0.2945956289768219, "beta_mean": 0.0001551444292999804, "nonfinite": 0.0, "weighted_memory_to_active": 0.01909736916422844, "mix_std": 0.24910405278205872, "beta_std": 0.0003682471578940749, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.018800023943185806, "evidence_weights": [-0.24231749773025513, -0.31545254588127136, 0.3927682638168335], "agreement_weight": 0.18455617129802704, "orthogonal_weight": -1.057656168937683}]}, {"layer": 7, "memory_to_active": 1.7643593549728394, "contribution_to_active": 0.27430325746536255, "mix_mean": 0.1561828851699829, "mix_low": 0.25931549072265625, "mix_high": 0.001080671907402575, "read_cosine": 0.23415540158748627, "beta_mean": 0.1326858401298523, "nonfinite": 0.0, "weighted_memory_to_active": 0.2228509783744812, "mix_std": 0.14454679191112518, "beta_std": 0.09943411499261856, "beta_low": 0.1992696225643158, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0815264880657196, "output_influence": 0.2803395092487335, "residual_influence": 0.0652451366186142, "state_active": 0.01327238418161869, "state_memory": 0.026328055188059807, "state_peak": 31.455236434936523, "growth_active": 1.12613844871521, "growth_memory": 1.285470962524414, "heads": [{"head": 0, "memory_to_active": 2.6685478687286377, "contribution_to_active": 0.2777450382709503, "mix_mean": 0.0352281779050827, "mix_low": 0.4398355484008789, "mix_high": 0.0, "read_cosine": 0.3325899839401245, "beta_mean": 0.18537451326847076, "nonfinite": 0.0, "weighted_memory_to_active": 0.3446468114852905, "mix_std": 0.07536125183105469, "beta_std": 0.13479450345039368, "beta_low": 0.0035400390625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.0022117430344223976, "evidence_weights": [0.03562743961811066, -0.07272756099700928, 0.022795433178544044], "agreement_weight": -0.027825361117720604, "orthogonal_weight": 0.010131600312888622}, {"head": 1, "memory_to_active": 0.9846957325935364, "contribution_to_active": 0.387347936630249, "mix_mean": 0.21917960047721863, "mix_low": 0.12079238891601562, "mix_high": 0.0055866241455078125, "read_cosine": 0.38431602716445923, "beta_mean": 0.09343579411506653, "nonfinite": 0.0, "weighted_memory_to_active": 0.3321702480316162, "mix_std": 0.24521438777446747, "beta_std": 0.06324975937604904, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04085950925946236, "evidence_weights": [-0.21597127616405487, -0.10332310199737549, -0.2594725787639618], "agreement_weight": 0.049209658056497574, "orthogonal_weight": 0.1461523026227951}, {"head": 2, "memory_to_active": 0.6463824510574341, "contribution_to_active": 0.351545125246048, "mix_mean": 0.18717412650585175, "mix_low": 0.037461280822753906, "mix_high": 0.00031280517578125, "read_cosine": 0.10013555735349655, "beta_mean": 0.0458221361041069, "nonfinite": 0.0, "weighted_memory_to_active": 0.20828942954540253, "mix_std": 0.19799092411994934, "beta_std": 0.0398915559053421, "beta_low": 0.073486328125, "beta_high": 0.0, "orthogonal_to_active": 0.1060958132147789, "evidence_weights": [-0.07487843930721283, -0.1593506932258606, 0.7704043388366699], "agreement_weight": 0.019883055239915848, "orthogonal_weight": -0.5261498093605042}, {"head": 3, "memory_to_active": 4.645927429199219, "contribution_to_active": 0.10486659407615662, "mix_mean": 0.033346083015203476, "mix_low": 0.3502674102783203, "mix_high": 0.0, "read_cosine": 0.08940112590789795, "beta_mean": 0.3898048996925354, "nonfinite": 0.0, "weighted_memory_to_active": 0.3585125505924225, "mix_std": 0.05069717392325401, "beta_std": 0.2756865918636322, "beta_low": 0.00445556640625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.30794596672058105, "evidence_weights": [0.07376734167337418, -0.021813493221998215, 0.7011057138442993], "agreement_weight": 0.04604144021868706, "orthogonal_weight": -0.9421419501304626}, {"head": 4, "memory_to_active": 1.5969197750091553, "contribution_to_active": 0.09512120485305786, "mix_mean": 0.019029144197702408, "mix_low": 0.6050901412963867, "mix_high": 0.0, "read_cosine": 0.23300683498382568, "beta_mean": 0.08145835995674133, "nonfinite": 0.0, "weighted_memory_to_active": 0.07199908047914505, "mix_std": 0.042411670088768005, "beta_std": 0.08247748762369156, "beta_low": 0.11431884765625, "beta_high": 0.0, "orthogonal_to_active": 0.012745313346385956, "evidence_weights": [0.1460779458284378, 0.0025145416148006916, 0.1808774322271347], "agreement_weight": 0.0408160500228405, "orthogonal_weight": -0.2406798005104065}, {"head": 5, "memory_to_active": 0.04368199408054352, "contribution_to_active": 0.42919349670410156, "mix_mean": 0.4431402087211609, "mix_low": 0.0024461746215820312, "mix_high": 0.0005846023559570312, "read_cosine": 0.2654828429222107, "beta_mean": 0.0002192863612435758, "nonfinite": 0.0, "weighted_memory_to_active": 0.021487778052687645, "mix_std": 0.2556053400039673, "beta_std": 0.0005047809681855142, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.019300552085042, "evidence_weights": [-0.16979371011257172, -0.22655197978019714, 0.5434820652008057], "agreement_weight": 0.18988622725009918, "orthogonal_weight": -0.9678669571876526}]}, {"layer": 8, "memory_to_active": 0.1928933560848236, "contribution_to_active": 0.5490554571151733, "mix_mean": 0.46445345878601074, "mix_low": 0.017621200531721115, "mix_high": 0.0015281041851267219, "read_cosine": 0.42132875323295593, "beta_mean": 0.014907944947481155, "nonfinite": 0.0, "weighted_memory_to_active": 0.08195538073778152, "mix_std": 0.2351122796535492, "beta_std": 0.021331992000341415, "beta_low": 0.76507568359375, "beta_high": 0.0, "orthogonal_to_active": 0.07068037986755371, "output_influence": 0.5001422762870789, "residual_influence": 0.17137309908866882, "state_active": 0.04585131257772446, "state_memory": 0.005908103659749031, "state_peak": 42.20747375488281, "growth_active": 1.1038962602615356, "growth_memory": 1.152705192565918, "heads": [{"head": 0, "memory_to_active": 0.05756048485636711, "contribution_to_active": 0.5700782537460327, "mix_mean": 0.6094329357147217, "mix_low": 0.0005407333374023438, "mix_high": 0.0015592575073242188, "read_cosine": 0.47203385829925537, "beta_mean": 0.0028386563062667847, "nonfinite": 0.0, "weighted_memory_to_active": 0.04069357365369797, "mix_std": 0.24729867279529572, "beta_std": 0.010131987743079662, "beta_low": 0.93487548828125, "beta_high": 0.0, "orthogonal_to_active": 0.025787658989429474, "evidence_weights": [-0.1683296412229538, -0.38951578736305237, 0.6416126489639282], "agreement_weight": 0.23591609299182892, "orthogonal_weight": -0.806000292301178}, {"head": 1, "memory_to_active": 0.09755970537662506, "contribution_to_active": 0.5489816069602966, "mix_mean": 0.5077314972877502, "mix_low": 4.00543212890625e-05, "mix_high": 0.0011472702026367188, "read_cosine": 0.04486652463674545, "beta_mean": 0.0013374325353652239, "nonfinite": 0.0, "weighted_memory_to_active": 0.04364211484789848, "mix_std": 0.23678205907344818, "beta_std": 0.00940291490405798, "beta_low": 0.98651123046875, "beta_high": 0.0, "orthogonal_to_active": 0.03170498460531235, "evidence_weights": [-0.1948114037513733, -0.2759134769439697, 0.7015844583511353], "agreement_weight": 0.07077696919441223, "orthogonal_weight": -0.751739501953125}, {"head": 2, "memory_to_active": 0.012347863055765629, "contribution_to_active": 0.6771034598350525, "mix_mean": 0.6702814102172852, "mix_low": 0.0005855560302734375, "mix_high": 0.0034847259521484375, "read_cosine": 0.4337295889854431, "beta_mean": 0.00011571783397812396, "nonfinite": 0.0, "weighted_memory_to_active": 0.009537157602608204, "mix_std": 0.22563926875591278, "beta_std": 0.0026031460147351027, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005877193063497543, "evidence_weights": [-0.22798661887645721, -0.347074419260025, 1.015170693397522], "agreement_weight": 0.27691900730133057, "orthogonal_weight": -0.7240757942199707}, {"head": 3, "memory_to_active": 0.09593096375465393, "contribution_to_active": 0.6341751217842102, "mix_mean": 0.5682640671730042, "mix_low": 0.000762939453125, "mix_high": 0.0028600692749023438, "read_cosine": 0.8030099868774414, "beta_mean": 0.007114796433597803, "nonfinite": 0.0, "weighted_memory_to_active": 0.07287053763866425, "mix_std": 0.2591094672679901, "beta_std": 0.006976793520152569, "beta_low": 0.786865234375, "beta_high": 0.0, "orthogonal_to_active": 0.02330271527171135, "evidence_weights": [-0.18293723464012146, -0.29970255494117737, 0.5620110034942627], "agreement_weight": 0.2261696606874466, "orthogonal_weight": -0.6375917196273804}, {"head": 4, "memory_to_active": 0.5221457481384277, "contribution_to_active": 0.22145652770996094, "mix_mean": 0.16585923731327057, "mix_low": 0.034300804138183594, "mix_high": 9.5367431640625e-07, "read_cosine": 0.276510626077652, "beta_mean": 0.007719904650002718, "nonfinite": 0.0, "weighted_memory_to_active": 0.1295335292816162, "mix_std": 0.16934308409690857, "beta_std": 0.020221825689077377, "beta_low": 0.8162841796875, "beta_high": 0.0, "orthogonal_to_active": 0.02751009352505207, "evidence_weights": [-0.048998091369867325, -0.1427711546421051, 0.3261222243309021], "agreement_weight": 0.06895408034324646, "orthogonal_weight": -0.2279735803604126}, {"head": 5, "memory_to_active": 0.37181538343429565, "contribution_to_active": 0.6425374746322632, "mix_mean": 0.26515135169029236, "mix_low": 0.06949710845947266, "mix_high": 0.0001163482666015625, "read_cosine": 0.4978218674659729, "beta_mean": 0.07032115757465363, "nonfinite": 0.0, "weighted_memory_to_active": 0.1954553872346878, "mix_std": 0.2725011110305786, "beta_std": 0.07865528017282486, "beta_low": 0.06597900390625, "beta_high": 0.0, "orthogonal_to_active": 0.30989962816238403, "evidence_weights": [-0.1241358295083046, -0.16627882421016693, -0.12786217033863068], "agreement_weight": -0.01603914052248001, "orthogonal_weight": 1.8715646266937256}]}, {"layer": 9, "memory_to_active": 0.9198369979858398, "contribution_to_active": 0.6760450601577759, "mix_mean": 0.3547478914260864, "mix_low": 0.10425027459859848, "mix_high": 0.005491415970027447, "read_cosine": 0.33995258808135986, "beta_mean": 0.05398111045360565, "nonfinite": 0.0, "weighted_memory_to_active": 0.3748883903026581, "mix_std": 0.22363325953483582, "beta_std": 0.03998424857854843, "beta_low": 0.5135905146598816, "beta_high": 0.0, "orthogonal_to_active": 0.04272095859050751, "output_influence": 0.5345017910003662, "residual_influence": 0.15149790048599243, "state_active": 0.04430413991212845, "state_memory": 0.017384685575962067, "state_peak": 50.589942932128906, "growth_active": 1.176580548286438, "growth_memory": 1.331325650215149, "heads": [{"head": 0, "memory_to_active": 0.0025402638129889965, "contribution_to_active": 0.5689775943756104, "mix_mean": 0.6409258842468262, "mix_low": 0.0013647079467773438, "mix_high": 0.022296905517578125, "read_cosine": 0.2404193878173828, "beta_mean": 1.0189680324401706e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00157757883425802, "mix_std": 0.2872607111930847, "beta_std": 4.323966641095467e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017162105068564415, "evidence_weights": [-0.30510029196739197, -0.3113371729850769, 0.6118343472480774], "agreement_weight": 0.12552015483379364, "orthogonal_weight": -1.1122630834579468}, {"head": 1, "memory_to_active": 0.022196132689714432, "contribution_to_active": 0.6891892552375793, "mix_mean": 0.5616183280944824, "mix_low": 0.008021354675292969, "mix_high": 0.0025501251220703125, "read_cosine": 0.4109417796134949, "beta_mean": 6.201986252563074e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.016545940190553665, "mix_std": 0.2852185368537903, "beta_std": 0.00015155911387410015, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.011359469965100288, "evidence_weights": [-0.24142637848854065, -0.29545679688453674, 1.1053682565689087], "agreement_weight": 0.45699596405029297, "orthogonal_weight": -0.8555533289909363}, {"head": 2, "memory_to_active": 0.18985256552696228, "contribution_to_active": 0.6451664566993713, "mix_mean": 0.5934776067733765, "mix_low": 0.0015993118286132812, "mix_high": 0.006931304931640625, "read_cosine": 0.3797990381717682, "beta_mean": 0.0007595572387799621, "nonfinite": 0.0, "weighted_memory_to_active": 0.13054662942886353, "mix_std": 0.2692420780658722, "beta_std": 0.004427005536854267, "beta_low": 0.9949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.07460840046405792, "evidence_weights": [-0.20081877708435059, -0.2754121422767639, 0.7572115063667297], "agreement_weight": 0.3193010091781616, "orthogonal_weight": -0.678111732006073}, {"head": 3, "memory_to_active": 1.1047639846801758, "contribution_to_active": 0.18177586793899536, "mix_mean": 0.076801598072052, "mix_low": 0.2519054412841797, "mix_high": 0.00010395050048828125, "read_cosine": 0.3770163059234619, "beta_mean": 0.05682064965367317, "nonfinite": 0.0, "weighted_memory_to_active": 0.1545049399137497, "mix_std": 0.1299578994512558, "beta_std": 0.04641248285770416, "beta_low": 0.02001953125, "beta_high": 0.0, "orthogonal_to_active": 0.029509268701076508, "evidence_weights": [0.04259605333209038, -0.03043062798678875, 0.12106140702962875], "agreement_weight": 0.09500891715288162, "orthogonal_weight": -0.20132994651794434}, {"head": 4, "memory_to_active": 2.610687732696533, "contribution_to_active": 1.3938814401626587, "mix_mean": 0.14460401237010956, "mix_low": 0.1311798095703125, "mix_high": 0.0009260177612304688, "read_cosine": 0.1360502988100052, "beta_mean": 0.1850852519273758, "nonfinite": 0.0, "weighted_memory_to_active": 1.324988603591919, "mix_std": 0.203894704580307, "beta_std": 0.09968437999486923, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.034855958074331284, "evidence_weights": [-0.07946394383907318, -0.1557178944349289, -0.02226223610341549], "agreement_weight": -0.06150291487574577, "orthogonal_weight": -0.02693408727645874}, {"head": 5, "memory_to_active": 1.588981032371521, "contribution_to_active": 0.5772800445556641, "mix_mean": 0.11105997860431671, "mix_low": 0.2314310073852539, "mix_high": 0.00014019012451171875, "read_cosine": 0.49548855423927307, "beta_mean": 0.08114896714687347, "nonfinite": 0.0, "weighted_memory_to_active": 0.621166467666626, "mix_std": 0.1662255823612213, "beta_std": 0.08918680250644684, "beta_low": 0.06658935546875, "beta_high": 0.0, "orthogonal_to_active": 0.10427642613649368, "evidence_weights": [0.10073024034500122, -0.11860522627830505, 0.14711886644363403], "agreement_weight": 0.018201926723122597, "orthogonal_weight": 0.239461287856102}]}, {"layer": 10, "memory_to_active": 0.20836088061332703, "contribution_to_active": 0.6696664690971375, "mix_mean": 0.5145657062530518, "mix_low": 0.02079598233103752, "mix_high": 0.005041917786002159, "read_cosine": 0.3768504559993744, "beta_mean": 0.043708402663469315, "nonfinite": 0.0, "weighted_memory_to_active": 0.09390342235565186, "mix_std": 0.2687574625015259, "beta_std": 0.04522249102592468, "beta_low": 0.6594747304916382, "beta_high": 0.0, "orthogonal_to_active": 0.21338391304016113, "output_influence": 0.44295743107795715, "residual_influence": 0.15416043996810913, "state_active": 0.053783878684043884, "state_memory": 0.014607774093747139, "state_peak": 121.12355041503906, "growth_active": 1.0572293996810913, "growth_memory": 1.0046882629394531, "heads": [{"head": 0, "memory_to_active": 0.7226235866546631, "contribution_to_active": 1.761042833328247, "mix_mean": 0.5777990818023682, "mix_low": 0.0907297134399414, "mix_high": 0.011183738708496094, "read_cosine": 0.11060836166143417, "beta_mean": 0.1927449256181717, "nonfinite": 0.0, "weighted_memory_to_active": 0.4359515905380249, "mix_std": 0.3922244906425476, "beta_std": 0.20589478313922882, "beta_low": 0.15106201171875, "beta_high": 0.0, "orthogonal_to_active": 1.1908284425735474, "evidence_weights": [-0.09518597275018692, -0.3839086890220642, -1.6910581588745117], "agreement_weight": 0.027977144345641136, "orthogonal_weight": 2.7637691497802734}, {"head": 1, "memory_to_active": 0.010826854035258293, "contribution_to_active": 0.5973532199859619, "mix_mean": 0.6149302124977112, "mix_low": 0.0015420913696289062, "mix_high": 0.013880729675292969, "read_cosine": 0.2744576334953308, "beta_mean": 7.078352791722864e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005986778065562248, "mix_std": 0.26623958349227905, "beta_std": 9.56882577156648e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008984645828604698, "evidence_weights": [-0.2535427510738373, -0.19747000932693481, 0.5678080916404724], "agreement_weight": 0.5566157698631287, "orthogonal_weight": -1.6272908449172974}, {"head": 2, "memory_to_active": 0.023378560319542885, "contribution_to_active": 0.6019542813301086, "mix_mean": 0.6403095722198486, "mix_low": 0.00018215179443359375, "mix_high": 0.0038661956787109375, "read_cosine": 0.017840664833784103, "beta_mean": 4.23021410824731e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010366663336753845, "mix_std": 0.2513020634651184, "beta_std": 0.00018634075240697712, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004464847967028618, "evidence_weights": [-0.2583265006542206, -0.27977314591407776, 0.6188227534294128], "agreement_weight": -0.19865502417087555, "orthogonal_weight": -0.4133046865463257}, {"head": 3, "memory_to_active": 0.3486327528953552, "contribution_to_active": 0.10515933483839035, "mix_mean": 0.2897162139415741, "mix_low": 0.030303955078125, "mix_high": 0.0001316070556640625, "read_cosine": 0.907238781452179, "beta_mean": 0.06247431039810181, "nonfinite": 0.0, "weighted_memory_to_active": 0.04112580418586731, "mix_std": 0.22971287369728088, "beta_std": 0.05789502337574959, "beta_low": 0.0023193359375, "beta_high": 0.0, "orthogonal_to_active": 0.04430122300982475, "evidence_weights": [-0.10318583995103836, -0.1472548097372055, 0.16327005624771118], "agreement_weight": 0.012409556657075882, "orthogonal_weight": 1.4044424295425415}, {"head": 4, "memory_to_active": 0.01583224907517433, "contribution_to_active": 0.5426592230796814, "mix_mean": 0.575467586517334, "mix_low": 0.00027561187744140625, "mix_high": 0.001186370849609375, "read_cosine": 0.09610769152641296, "beta_mean": 4.721860750578344e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009036650881171227, "mix_std": 0.24729737639427185, "beta_std": 8.221511234296486e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011895021423697472, "evidence_weights": [-0.18601585924625397, -0.26134777069091797, 0.9757856726646423], "agreement_weight": 0.15206775069236755, "orthogonal_weight": -1.3905372619628906}, {"head": 5, "memory_to_active": 0.12887117266654968, "contribution_to_active": 0.4098300039768219, "mix_mean": 0.3891713619232178, "mix_low": 0.0017423629760742188, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8548495173454285, "beta_mean": 0.006870857439935207, "nonfinite": 0.0, "weighted_memory_to_active": 0.060953013598918915, "mix_std": 0.22576844692230225, "beta_std": 0.007180884480476379, "beta_low": 0.80352783203125, "beta_high": 0.0, "orthogonal_to_active": 0.01982918009161949, "evidence_weights": [-0.14027076959609985, -0.16123470664024353, 0.26896047592163086], "agreement_weight": 0.09221301972866058, "orthogonal_weight": -0.6572859287261963}]}, {"layer": 11, "memory_to_active": 0.08546335995197296, "contribution_to_active": 0.6127741932868958, "mix_mean": 0.5086447596549988, "mix_low": 0.10950390994548798, "mix_high": 0.04958359897136688, "read_cosine": 0.34289300441741943, "beta_mean": 0.025148890912532806, "nonfinite": 0.0, "weighted_memory_to_active": 0.0431169793009758, "mix_std": 0.3253670930862427, "beta_std": 0.03724834322929382, "beta_low": 0.6851806640625, "beta_high": 0.0, "orthogonal_to_active": 0.04399064928293228, "output_influence": 0.5513407588005066, "residual_influence": 0.14199940860271454, "state_active": 0.21550089120864868, "state_memory": 0.004187607206404209, "state_peak": 33.768070220947266, "growth_active": 1.1222127676010132, "growth_memory": 1.6001853942871094, "heads": [{"head": 0, "memory_to_active": 0.007748920936137438, "contribution_to_active": 0.6024043560028076, "mix_mean": 0.6411774158477783, "mix_low": 0.001033782958984375, "mix_high": 0.009245872497558594, "read_cosine": 0.2682349383831024, "beta_mean": 9.049816435435787e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005360408220440149, "mix_std": 0.26093626022338867, "beta_std": 0.0036311710719019175, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.004184182733297348, "evidence_weights": [-0.1733483076095581, -0.36660876870155334, 0.7458001971244812], "agreement_weight": 0.34723567962646484, "orthogonal_weight": -0.8224292993545532}, {"head": 1, "memory_to_active": 0.1833854764699936, "contribution_to_active": 0.39228832721710205, "mix_mean": 0.44473955035209656, "mix_low": 0.005646705627441406, "mix_high": 0.00041484832763671875, "read_cosine": 0.5288568735122681, "beta_mean": 0.009567486122250557, "nonfinite": 0.0, "weighted_memory_to_active": 0.08045292645692825, "mix_std": 0.2673095166683197, "beta_std": 0.021661922335624695, "beta_low": 0.79412841796875, "beta_high": 0.0, "orthogonal_to_active": 0.039363596588373184, "evidence_weights": [-0.15961690247058868, -0.23986725509166718, 0.5931957364082336], "agreement_weight": 0.07234136015176773, "orthogonal_weight": -0.5883744955062866}, {"head": 2, "memory_to_active": 0.00637770164757967, "contribution_to_active": 0.9857691526412964, "mix_mean": 0.42922815680503845, "mix_low": 0.3056621551513672, "mix_high": 0.17828845977783203, "read_cosine": 0.1721227914094925, "beta_mean": 0.06445921957492828, "nonfinite": 0.0, "weighted_memory_to_active": 0.004767930135130882, "mix_std": 0.44932541251182556, "beta_std": 0.08753879368305206, "beta_low": 0.484130859375, "beta_high": 0.0, "orthogonal_to_active": 0.040935590863227844, "evidence_weights": [0.04288049787282944, -0.3962109386920929, -1.3280457258224487], "agreement_weight": 0.00924935843795538, "orthogonal_weight": 8.862029075622559}, {"head": 3, "memory_to_active": 0.008343977853655815, "contribution_to_active": 0.8595850467681885, "mix_mean": 0.4076130986213684, "mix_low": 0.33536529541015625, "mix_high": 0.10345935821533203, "read_cosine": 0.5276027917861938, "beta_mean": 0.05532459914684296, "nonfinite": 0.0, "weighted_memory_to_active": 0.005250344984233379, "mix_std": 0.44163957238197327, "beta_std": 0.0842335969209671, "beta_low": 0.5125732421875, "beta_high": 0.0, "orthogonal_to_active": 0.03332297503948212, "evidence_weights": [0.19121180474758148, -0.30615267157554626, -0.17232570052146912], "agreement_weight": 0.05171434208750725, "orthogonal_weight": 9.164284706115723}, {"head": 4, "memory_to_active": 0.2163453847169876, "contribution_to_active": 0.4437662959098816, "mix_mean": 0.5776622891426086, "mix_low": 0.0064830780029296875, "mix_high": 0.0011196136474609375, "read_cosine": 0.04700905457139015, "beta_mean": 0.012682314030826092, "nonfinite": 0.0, "weighted_memory_to_active": 0.11778764426708221, "mix_std": 0.2732813358306885, "beta_std": 0.013937026262283325, "beta_low": 0.56646728515625, "beta_high": 0.0, "orthogonal_to_active": 0.11585408449172974, "evidence_weights": [-0.24330919981002808, -0.23719578981399536, 1.9440723657608032], "agreement_weight": -0.20494350790977478, "orthogonal_weight": -0.996345043182373}, {"head": 5, "memory_to_active": 0.09057869762182236, "contribution_to_active": 0.39283156394958496, "mix_mean": 0.5514476895332336, "mix_low": 0.0028324127197265625, "mix_high": 0.004973411560058594, "read_cosine": 0.5135313272476196, "beta_mean": 0.008769205771386623, "nonfinite": 0.0, "weighted_memory_to_active": 0.04508260637521744, "mix_std": 0.259710431098938, "beta_std": 0.01248753909021616, "beta_low": 0.7542724609375, "beta_high": 0.0, "orthogonal_to_active": 0.030283458530902863, "evidence_weights": [-0.20920073986053467, -0.3110213279724121, 0.6448960900306702], "agreement_weight": 0.08786767721176147, "orthogonal_weight": -0.8319123983383179}]}, {"layer": 12, "memory_to_active": 0.23028044402599335, "contribution_to_active": 0.5733211040496826, "mix_mean": 0.5367422103881836, "mix_low": 0.008994102478027344, "mix_high": 0.005462170112878084, "read_cosine": 0.22523373365402222, "beta_mean": 0.013932833448052406, "nonfinite": 0.0, "weighted_memory_to_active": 0.11910320073366165, "mix_std": 0.263493150472641, "beta_std": 0.014881124719977379, "beta_low": 0.8253886699676514, "beta_high": 0.0, "orthogonal_to_active": 0.07202548533678055, "output_influence": 0.49068185687065125, "residual_influence": 0.13543610274791718, "state_active": 0.03603136166930199, "state_memory": 0.00864300038665533, "state_peak": 59.16693115234375, "growth_active": 1.1902875900268555, "growth_memory": 1.2066664695739746, "heads": [{"head": 0, "memory_to_active": 0.05667421594262123, "contribution_to_active": 0.5970113277435303, "mix_mean": 0.5692428350448608, "mix_low": 0.000457763671875, "mix_high": 0.004502296447753906, "read_cosine": 0.27334827184677124, "beta_mean": 0.0011349038686603308, "nonfinite": 0.0, "weighted_memory_to_active": 0.030641542747616768, "mix_std": 0.2676053047180176, "beta_std": 0.002754604211077094, "beta_low": 0.98431396484375, "beta_high": 0.0, "orthogonal_to_active": 0.028312379494309425, "evidence_weights": [-0.2099021077156067, -0.27681177854537964, 0.06986541301012039], "agreement_weight": 0.09876702725887299, "orthogonal_weight": -0.9849076867103577}, {"head": 1, "memory_to_active": 0.12112146615982056, "contribution_to_active": 0.5394463539123535, "mix_mean": 0.5552933812141418, "mix_low": 0.00032711029052734375, "mix_high": 0.00193023681640625, "read_cosine": 0.39311158657073975, "beta_mean": 0.0009793495992198586, "nonfinite": 0.0, "weighted_memory_to_active": 0.07379526644945145, "mix_std": 0.2655537724494934, "beta_std": 0.01106985192745924, "beta_low": 0.99285888671875, "beta_high": 0.0, "orthogonal_to_active": 0.06265965104103088, "evidence_weights": [-0.20480985939502716, -0.2726441025733948, 1.0426195859909058], "agreement_weight": 0.21983182430267334, "orthogonal_weight": -0.933659017086029}, {"head": 2, "memory_to_active": 0.5888592600822449, "contribution_to_active": 0.49430736899375916, "mix_mean": 0.32676783204078674, "mix_low": 0.027139663696289062, "mix_high": 0.000270843505859375, "read_cosine": 0.5301616787910461, "beta_mean": 0.08017756044864655, "nonfinite": 0.0, "weighted_memory_to_active": 0.2352893352508545, "mix_std": 0.2480788230895996, "beta_std": 0.05828703194856644, "beta_low": 0.0025634765625, "beta_high": 0.0, "orthogonal_to_active": 0.256011039018631, "evidence_weights": [-0.2184447944164276, -0.05501341447234154, 0.0032352954149246216], "agreement_weight": 0.018152760341763496, "orthogonal_weight": 1.3307468891143799}, {"head": 3, "memory_to_active": 0.02920362539589405, "contribution_to_active": 0.7451559901237488, "mix_mean": 0.7129473686218262, "mix_low": 0.00015354156494140625, "mix_high": 0.01647472381591797, "read_cosine": 0.0931658148765564, "beta_mean": 0.00025346409529447556, "nonfinite": 0.0, "weighted_memory_to_active": 0.021457623690366745, "mix_std": 0.2304525077342987, "beta_std": 0.005388089455664158, "beta_low": 0.99835205078125, "beta_high": 0.0, "orthogonal_to_active": 0.020604442805051804, "evidence_weights": [-0.2914329767227173, -0.3406195342540741, 1.6107101440429688], "agreement_weight": 0.5021221041679382, "orthogonal_weight": -0.9784390926361084}, {"head": 4, "memory_to_active": 0.5249130725860596, "contribution_to_active": 0.5712528824806213, "mix_mean": 0.4326763451099396, "mix_low": 0.01833820343017578, "mix_high": 0.004691123962402344, "read_cosine": -0.15757638216018677, "beta_mean": 0.0007445358205586672, "nonfinite": 0.0, "weighted_memory_to_active": 0.3126087188720703, "mix_std": 0.290865957736969, "beta_std": 0.008252612315118313, "beta_low": 0.9786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.03309635818004608, "evidence_weights": [-0.15185968577861786, -0.21821914613246918, -0.0861695259809494], "agreement_weight": 0.005715346895158291, "orthogonal_weight": -0.11065790802240372}, {"head": 5, "memory_to_active": 0.0609109103679657, "contribution_to_active": 0.492752343416214, "mix_mean": 0.6235251426696777, "mix_low": 0.007548332214355469, "mix_high": 0.0049037933349609375, "read_cosine": 0.21919134259223938, "beta_mean": 0.0003071902319788933, "nonfinite": 0.0, "weighted_memory_to_active": 0.0408267118036747, "mix_std": 0.2784025967121124, "beta_std": 0.003534550778567791, "beta_low": 0.99560546875, "beta_high": 0.0, "orthogonal_to_active": 0.03146901726722717, "evidence_weights": [-0.23212583363056183, -0.3158571422100067, 1.3060798645019531], "agreement_weight": 0.39222201704978943, "orthogonal_weight": -0.8265274167060852}]}, {"layer": 13, "memory_to_active": 0.11649251729249954, "contribution_to_active": 0.5253903865814209, "mix_mean": 0.5390608310699463, "mix_low": 0.0059940023347735405, "mix_high": 0.007511934265494347, "read_cosine": 0.39227133989334106, "beta_mean": 0.016287729144096375, "nonfinite": 0.0, "weighted_memory_to_active": 0.050197795033454895, "mix_std": 0.2611100673675537, "beta_std": 0.01828903704881668, "beta_low": 0.80572509765625, "beta_high": 0.0, "orthogonal_to_active": 0.05248758941888809, "output_influence": 0.5111901760101318, "residual_influence": 0.17202357947826385, "state_active": 0.1246287152171135, "state_memory": 0.04350557550787926, "state_peak": 97.392333984375, "growth_active": 0.9220706224441528, "growth_memory": 1.0033454895019531, "heads": [{"head": 0, "memory_to_active": 0.012165714055299759, "contribution_to_active": 0.5878838300704956, "mix_mean": 0.6124230623245239, "mix_low": 0.000431060791015625, "mix_high": 0.003930091857910156, "read_cosine": 0.43155866861343384, "beta_mean": 4.3329171603545547e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007823947817087173, "mix_std": 0.2667713165283203, "beta_std": 8.941470878198743e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007732424419373274, "evidence_weights": [-0.19824135303497314, -0.26829057931900024, 0.7771363258361816], "agreement_weight": 0.1555965393781662, "orthogonal_weight": -1.1022900342941284}, {"head": 1, "memory_to_active": 0.010669193230569363, "contribution_to_active": 0.598534107208252, "mix_mean": 0.5679032802581787, "mix_low": 0.00197601318359375, "mix_high": 0.00444793701171875, "read_cosine": 0.20719614624977112, "beta_mean": 4.2130908695980906e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006033254787325859, "mix_std": 0.2761516869068146, "beta_std": 7.862508937250823e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007951208390295506, "evidence_weights": [-0.18394945561885834, -0.26520535349845886, 0.6308024525642395], "agreement_weight": 0.43356025218963623, "orthogonal_weight": -1.3644897937774658}, {"head": 2, "memory_to_active": 0.08481109142303467, "contribution_to_active": 0.46294382214546204, "mix_mean": 0.38871634006500244, "mix_low": 0.0014362335205078125, "mix_high": 0.00013637542724609375, "read_cosine": 0.6050562858581543, "beta_mean": 0.01219380833208561, "nonfinite": 0.0, "weighted_memory_to_active": 0.041029080748558044, "mix_std": 0.2338232398033142, "beta_std": 0.04153535142540932, "beta_low": 0.8424072265625, "beta_high": 0.0, "orthogonal_to_active": 0.02587907947599888, "evidence_weights": [-0.18419043719768524, -0.20659829676151276, 0.4073265790939331], "agreement_weight": 0.0061376821249723434, "orthogonal_weight": -0.8444280028343201}, {"head": 3, "memory_to_active": 0.13583038747310638, "contribution_to_active": 0.5282530784606934, "mix_mean": 0.6426651477813721, "mix_low": 0.02396678924560547, "mix_high": 0.024538040161132812, "read_cosine": -0.0677640438079834, "beta_mean": 0.000868667964823544, "nonfinite": 0.0, "weighted_memory_to_active": 0.09651807695627213, "mix_std": 0.28998813033103943, "beta_std": 0.008347980678081512, "beta_low": 0.99273681640625, "beta_high": 0.0, "orthogonal_to_active": 0.08856064081192017, "evidence_weights": [-0.3190889060497284, -0.247467502951622, 1.65060555934906], "agreement_weight": 5.717923164367676, "orthogonal_weight": -0.9759815335273743}, {"head": 4, "memory_to_active": 0.3953259289264679, "contribution_to_active": 0.2681298851966858, "mix_mean": 0.3677626848220825, "mix_low": 0.0056610107421875, "mix_high": 5.7220458984375e-06, "read_cosine": 0.883449137210846, "beta_mean": 0.08432215452194214, "nonfinite": 0.0, "weighted_memory_to_active": 0.10910796374082565, "mix_std": 0.22891992330551147, "beta_std": 0.057339757680892944, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.15076933801174164, "evidence_weights": [-0.08488813787698746, -0.22847935557365417, 0.46509110927581787], "agreement_weight": 0.004892776720225811, "orthogonal_weight": 2.2226951122283936}, {"head": 5, "memory_to_active": 0.060152795165777206, "contribution_to_active": 0.7065973281860352, "mix_mean": 0.6548943519592285, "mix_low": 0.0024929046630859375, "mix_high": 0.012013435363769531, "read_cosine": 0.29413163661956787, "beta_mean": 0.00025627645663917065, "nonfinite": 0.0, "weighted_memory_to_active": 0.04067440330982208, "mix_std": 0.2710060775279999, "beta_std": 0.0023430949077010155, "beta_low": 0.99896240234375, "beta_high": 0.0, "orthogonal_to_active": 0.03403284773230553, "evidence_weights": [-0.22152850031852722, -0.30185285210609436, 0.8812040090560913], "agreement_weight": 0.17654426395893097, "orthogonal_weight": -0.9121896624565125}]}, {"layer": 14, "memory_to_active": 0.2296130210161209, "contribution_to_active": 0.4201395809650421, "mix_mean": 0.43135184049606323, "mix_low": 0.024414222687482834, "mix_high": 0.0063069663010537624, "read_cosine": 0.6018097996711731, "beta_mean": 0.0640047937631607, "nonfinite": 0.0, "weighted_memory_to_active": 0.09561322629451752, "mix_std": 0.24665407836437225, "beta_std": 0.05976657569408417, "beta_low": 0.49249267578125, "beta_high": 0.0, "orthogonal_to_active": 0.04278067871928215, "output_influence": 0.3594440817832947, "residual_influence": 0.13786935806274414, "state_active": 0.05637039616703987, "state_memory": 0.015628596767783165, "state_peak": 38.92337417602539, "growth_active": 1.0845543146133423, "growth_memory": 1.2448341846466064, "heads": [{"head": 0, "memory_to_active": 0.37169334292411804, "contribution_to_active": 0.20584192872047424, "mix_mean": 0.2074936032295227, "mix_low": 0.01471710205078125, "mix_high": 3.814697265625e-06, "read_cosine": 0.8032245635986328, "beta_mean": 0.07504003494977951, "nonfinite": 0.0, "weighted_memory_to_active": 0.08844946324825287, "mix_std": 0.18673156201839447, "beta_std": 0.06475162506103516, "beta_low": 0.003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.09160488098859787, "evidence_weights": [-0.11117690801620483, -0.04166039451956749, 0.4373994767665863], "agreement_weight": 0.024331998080015182, "orthogonal_weight": 1.681739330291748}, {"head": 1, "memory_to_active": 0.4711681604385376, "contribution_to_active": 0.2876718044281006, "mix_mean": 0.22025875747203827, "mix_low": 0.11562538146972656, "mix_high": 7.62939453125e-06, "read_cosine": 0.8452979326248169, "beta_mean": 0.2623976469039917, "nonfinite": 0.0, "weighted_memory_to_active": 0.2006547451019287, "mix_std": 0.24810707569122314, "beta_std": 0.24922031164169312, "beta_low": 0.091064453125, "beta_high": 0.0, "orthogonal_to_active": 0.014639787375926971, "evidence_weights": [-0.006800713483244181, -0.1973106414079666, -0.09616754949092865], "agreement_weight": 0.056654103100299835, "orthogonal_weight": 0.18116961419582367}, {"head": 2, "memory_to_active": 0.12711378931999207, "contribution_to_active": 0.5814967155456543, "mix_mean": 0.5908795595169067, "mix_low": 0.0030450820922851562, "mix_high": 0.004370689392089844, "read_cosine": 0.414719820022583, "beta_mean": 0.0019875764846801758, "nonfinite": 0.0, "weighted_memory_to_active": 0.0827300176024437, "mix_std": 0.2672596573829651, "beta_std": 0.0027704122476279736, "beta_low": 0.98272705078125, "beta_high": 0.0, "orthogonal_to_active": 0.06374408304691315, "evidence_weights": [-0.14066912233829498, -0.3167142868041992, 0.9133955836296082], "agreement_weight": 0.08293009549379349, "orthogonal_weight": -0.8779700398445129}, {"head": 3, "memory_to_active": 0.05098060518503189, "contribution_to_active": 0.596264123916626, "mix_mean": 0.6703397035598755, "mix_low": 0.0045871734619140625, "mix_high": 0.02219390869140625, "read_cosine": -0.10455139726400375, "beta_mean": 0.00556415319442749, "nonfinite": 0.0, "weighted_memory_to_active": 0.032405510544776917, "mix_std": 0.25940650701522827, "beta_std": 0.007790869101881981, "beta_low": 0.84600830078125, "beta_high": 0.0, "orthogonal_to_active": 0.02441597357392311, "evidence_weights": [-0.34723639488220215, -0.31147223711013794, 0.2882288992404938], "agreement_weight": 0.07087469846010208, "orthogonal_weight": -0.8311535716056824}, {"head": 4, "memory_to_active": 0.147796630859375, "contribution_to_active": 0.509401798248291, "mix_mean": 0.5198323726654053, "mix_low": 0.004239082336425781, "mix_high": 0.009556770324707031, "read_cosine": 0.9025911092758179, "beta_mean": 0.03740623593330383, "nonfinite": 0.0, "weighted_memory_to_active": 0.08868372440338135, "mix_std": 0.2612687647342682, "beta_std": 0.025891277939081192, "beta_low": 0.06005859375, "beta_high": 0.0, "orthogonal_to_active": 0.022432750090956688, "evidence_weights": [-0.2189532369375229, -0.2457806020975113, 0.4281249940395355], "agreement_weight": 0.13481038808822632, "orthogonal_weight": -0.6141058206558228}, {"head": 5, "memory_to_active": 0.2089255005121231, "contribution_to_active": 0.3401610851287842, "mix_mean": 0.37930727005004883, "mix_low": 0.004271507263183594, "mix_high": 0.001708984375, "read_cosine": 0.7495764493942261, "beta_mean": 0.0016331131337210536, "nonfinite": 0.0, "weighted_memory_to_active": 0.08075589686632156, "mix_std": 0.25715091824531555, "beta_std": 0.008174916729331017, "beta_low": 0.971435546875, "beta_high": 0.0, "orthogonal_to_active": 0.03984658792614937, "evidence_weights": [-0.17740364372730255, -0.17466475069522858, 0.3100298047065735], "agreement_weight": 0.07480438798666, "orthogonal_weight": -0.7464744448661804}]}, {"layer": 15, "memory_to_active": 0.06101769953966141, "contribution_to_active": 0.48466378450393677, "mix_mean": 0.41696810722351074, "mix_low": 0.00862884521484375, "mix_high": 0.002784570213407278, "read_cosine": 0.6685464978218079, "beta_mean": 0.03727252781391144, "nonfinite": 0.0, "weighted_memory_to_active": 0.02982032299041748, "mix_std": 0.28571441769599915, "beta_std": 0.04283367842435837, "beta_low": 0.7120768427848816, "beta_high": 0.0, "orthogonal_to_active": 0.007753526791930199, "output_influence": 0.304551899433136, "residual_influence": 0.17422419786453247, "state_active": 0.14989794790744781, "state_memory": 0.010833053849637508, "state_peak": 33.87553024291992, "growth_active": 1.2366969585418701, "growth_memory": 1.3526250123977661, "heads": [{"head": 0, "memory_to_active": 0.11736100912094116, "contribution_to_active": 0.4733075201511383, "mix_mean": 0.3919026553630829, "mix_low": 0.006785392761230469, "mix_high": 0.005626678466796875, "read_cosine": 0.6466278433799744, "beta_mean": 0.03762274608016014, "nonfinite": 0.0, "weighted_memory_to_active": 0.05552662909030914, "mix_std": 0.2845761477947235, "beta_std": 0.029939454048871994, "beta_low": 0.110107421875, "beta_high": 0.0, "orthogonal_to_active": 0.025802765041589737, "evidence_weights": [-0.19105227291584015, -0.19427502155303955, 0.44611191749572754], "agreement_weight": 0.09521936625242233, "orthogonal_weight": -0.7633525729179382}, {"head": 1, "memory_to_active": 0.13784931600093842, "contribution_to_active": 0.41867315769195557, "mix_mean": 0.44008490443229675, "mix_low": 0.0013074874877929688, "mix_high": 0.006318092346191406, "read_cosine": 0.9235390424728394, "beta_mean": 0.0007716891705058515, "nonfinite": 0.0, "weighted_memory_to_active": 0.05866581201553345, "mix_std": 0.2879791855812073, "beta_std": 0.0027535194531083107, "beta_low": 0.9910888671875, "beta_high": 0.0, "orthogonal_to_active": 0.0013996989000588655, "evidence_weights": [-0.1476524919271469, -0.1565539538860321, 0.17767581343650818], "agreement_weight": 0.12665154039859772, "orthogonal_weight": 0.047919657081365585}, {"head": 2, "memory_to_active": 0.0044257459230721, "contribution_to_active": 0.4768722653388977, "mix_mean": 0.33072584867477417, "mix_low": 0.01631641387939453, "mix_high": 6.4849853515625e-05, "read_cosine": 0.7553960084915161, "beta_mean": 6.385360029526055e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021898667328059673, "mix_std": 0.2707277834415436, "beta_std": 0.00011525193986017257, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006043451139703393, "evidence_weights": [-0.20418286323547363, -0.26383838057518005, 0.11832135915756226], "agreement_weight": 0.009905542246997356, "orthogonal_weight": 0.7309376001358032}, {"head": 3, "memory_to_active": 0.001581012038514018, "contribution_to_active": 0.3746660351753235, "mix_mean": 0.3932216763496399, "mix_low": 0.006091117858886719, "mix_high": 0.000244140625, "read_cosine": 0.5957409143447876, "beta_mean": 8.615872502559796e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005181118031032383, "mix_std": 0.2879579961299896, "beta_std": 2.047996167675592e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004033837467432022, "evidence_weights": [-0.26379117369651794, -0.24220266938209534, 0.1946433186531067], "agreement_weight": 0.0469798743724823, "orthogonal_weight": -0.7544556260108948}, {"head": 4, "memory_to_active": 0.07675720006227493, "contribution_to_active": 0.64109867811203, "mix_mean": 0.3722861409187317, "mix_low": 0.011157989501953125, "mix_high": 0.0009260177612304688, "read_cosine": 0.7590250372886658, "beta_mean": 0.18300461769104004, "nonfinite": 0.0, "weighted_memory_to_active": 0.043656088411808014, "mix_std": 0.3069615066051483, "beta_std": 0.2089090198278427, "beta_low": 0.19854736328125, "beta_high": 0.0, "orthogonal_to_active": 0.006629941053688526, "evidence_weights": [-0.20480869710445404, -0.2510322034358978, -0.009304479695856571], "agreement_weight": 0.013624160550534725, "orthogonal_weight": 0.34550586342811584}, {"head": 5, "memory_to_active": 0.02813190221786499, "contribution_to_active": 0.5233651399612427, "mix_mean": 0.57358717918396, "mix_low": 0.010114669799804688, "mix_high": 0.0035276412963867188, "read_cosine": 0.3309502601623535, "beta_mean": 0.0021636360324919224, "nonfinite": 0.0, "weighted_memory_to_active": 0.018365422263741493, "mix_std": 0.2760838270187378, "beta_std": 0.015264326706528664, "beta_low": 0.97271728515625, "beta_high": 0.0, "orthogonal_to_active": 0.0116810267791152, "evidence_weights": [-0.2880696654319763, -0.30418020486831665, 0.7153688073158264], "agreement_weight": 0.29782232642173767, "orthogonal_weight": 0.8161357641220093}]}]}, {"step": 55000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7689335939976445, "memory_to_active": 0.8027336597442627, "contribution_to_active": 0.5603603720664978, "mix_mean": 0.41228151321411133, "mix_low": 0.13254475593566895, "mix_high": 0.010309766046702862, "read_cosine": 0.34900426864624023, "beta_mean": 0.07195474952459335, "nonfinite": 0.0, "weighted_memory_to_active": 0.2050609439611435, "mix_std": 0.2231031209230423, "beta_std": 0.06255805492401123, "beta_low": 0.5634657740592957, "beta_high": 2.6702880859375e-05, "orthogonal_to_active": 0.062009043991565704, "samples": 4, "scored_tokens": 899, "normal_loss": 3.079821825027466, "normal_token_accuracy": 0.44160178303718567, "state_active": 0.06397893279790878, "state_memory": 0.017938334494829178, "state_peak": 118.92684936523438, "growth_active": 1.134282112121582, "growth_memory": 1.317430019378662, "output_influence": 0.42553311586380005, "residual_influence": 0.14494840800762177, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2684335708618164, "no_memory_loss_delta": 0.6322418451309204, "scrambled_loss_delta": 0.1499270349740982, "memory_only_loss_delta": 3.414492130279541, "no_evidence_loss_delta": 0.36017948389053345, "no_agreement_loss_delta": 0.029563136398792267, "no_orthogonal_loss_delta": 3.059953451156616e-05}, {"index": 94, "tokens": 256, "normal_loss": 3.1617021560668945, "no_memory_loss_delta": 0.6023858189582825, "scrambled_loss_delta": 0.30775976181030273, "memory_only_loss_delta": 3.5510928630828857, "no_evidence_loss_delta": 0.26058679819107056, "no_agreement_loss_delta": 0.033344656229019165, "no_orthogonal_loss_delta": 0.041032809764146805}, {"index": 188, "tokens": 256, "normal_loss": 3.3523406982421875, "no_memory_loss_delta": 0.8413465619087219, "scrambled_loss_delta": 0.20060782134532928, "memory_only_loss_delta": 3.1955223083496094, "no_evidence_loss_delta": 0.4412372410297394, "no_agreement_loss_delta": 0.05540355294942856, "no_orthogonal_loss_delta": 0.05131375417113304}, {"index": 283, "tokens": 131, "normal_loss": 2.0186703205108643, "no_memory_loss_delta": 0.35541409254074097, "scrambled_loss_delta": 0.2558232247829437, "memory_only_loss_delta": 2.8651797771453857, "no_evidence_loss_delta": 0.1327596753835678, "no_agreement_loss_delta": 0.015246037393808365, "no_orthogonal_loss_delta": -0.01521065179258585}], "agreement_weight_abs_mean": 0.21426886320114136, "agreement_weight_abs_max": 5.807581901550293, "agreement_weight_mean": 0.1880425661802292, "orthogonal_weight_abs_mean": 0.864582359790802, "orthogonal_weight_abs_max": 9.285462379455566, "orthogonal_weight_mean": -0.11186466366052628, "no_memory_loss": 3.7227680683135986, "no_memory_loss_delta": 0.6429462432861328, "no_memory_token_accuracy_gain": 0.08898776769638062, "no_memory_token_win_fraction": 0.6840934157371521, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 47.42588214257306, "scrambled_loss": 3.304556369781494, "scrambled_loss_delta": 0.22473442554473877, "scrambled_token_accuracy_gain": 0.02002224698662758, "scrambled_token_win_fraction": 0.6229143738746643, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.127168756872457, "memory_only_loss": 6.390814781188965, "memory_only_loss_delta": 3.310992479324341, "memory_only_token_accuracy_gain": 0.2747497260570526, "memory_only_token_win_fraction": 0.9010010957717896, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.35200498151693, "no_evidence_loss": 3.4015841484069824, "no_evidence_loss_delta": 0.32176247239112854, "no_evidence_token_accuracy_gain": 0.04338153451681137, "no_evidence_token_win_fraction": 0.6240267157554626, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 27.512965339667, "no_agreement_loss": 3.1157338619232178, "no_agreement_loss_delta": 0.035912055522203445, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.5228031277656555, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.527486799952021, "no_orthogonal_loss": 3.103910446166992, "no_orthogonal_loss_delta": 0.024088943377137184, "no_orthogonal_token_accuracy_gain": 0.006674082484096289, "no_orthogonal_token_win_fraction": 0.5884315967559814, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.3801120528620623, "short_no_memory_loss_delta": 0.5242871642112732, "long_context_memory_gain": 0.11865907907485962, "normal_context_loss_gain": 0.14305056631565094, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.463223934173584, "contribution_to_active": 1.1763715744018555, "mix_mean": 0.10531789064407349, "mix_low": 0.5266509056091309, "mix_high": 0.004999956116080284, "read_cosine": 0.41758471727371216, "beta_mean": 0.22269806265830994, "nonfinite": 0.0, "weighted_memory_to_active": 1.207998514175415, "mix_std": 0.09172216802835464, "beta_std": 0.1405937820672989, "beta_low": 0.064910888671875, "beta_high": 0.0, "orthogonal_to_active": 0.043505191802978516, "output_influence": 0.22520923614501953, "residual_influence": 0.2623478174209595, "state_active": 0.06725122779607773, "state_memory": 0.045286450535058975, "state_peak": 51.05006408691406, "growth_active": 1.2750883102416992, "growth_memory": 1.2872134447097778, "heads": [{"head": 0, "memory_to_active": 2.6904549598693848, "contribution_to_active": 0.5272393822669983, "mix_mean": 0.020379066467285156, "mix_low": 0.6242589950561523, "mix_high": 2.86102294921875e-06, "read_cosine": 0.20924606919288635, "beta_mean": 0.20018306374549866, "nonfinite": 0.0, "weighted_memory_to_active": 0.6614542603492737, "mix_std": 0.06527451425790787, "beta_std": 0.09767084568738937, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06615839153528214, "evidence_weights": [-0.008204342797398567, -0.052464600652456284, -0.05095855891704559], "agreement_weight": -0.19809558987617493, "orthogonal_weight": -0.11417128145694733}, {"head": 1, "memory_to_active": 6.3010711669921875, "contribution_to_active": 2.7426810264587402, "mix_mean": 0.019996732473373413, "mix_low": 0.6019821166992188, "mix_high": 0.0, "read_cosine": 0.4578532874584198, "beta_mean": 0.4781467318534851, "nonfinite": 0.0, "weighted_memory_to_active": 3.0079309940338135, "mix_std": 0.05489937588572502, "beta_std": 0.21940575540065765, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00802189577370882, "evidence_weights": [0.020142456516623497, -0.10394728928804398, 0.5973455309867859], "agreement_weight": 0.02698294073343277, "orthogonal_weight": 0.005593727342784405}, {"head": 2, "memory_to_active": 2.5896384716033936, "contribution_to_active": 0.5318806171417236, "mix_mean": 0.014925455674529076, "mix_low": 0.6663808822631836, "mix_high": 0.0, "read_cosine": 0.3074060082435608, "beta_mean": 0.23151348531246185, "nonfinite": 0.0, "weighted_memory_to_active": 0.6258797645568848, "mix_std": 0.034560106694698334, "beta_std": 0.17887623608112335, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.05369959771633148, "evidence_weights": [0.06278061121702194, 0.05325298011302948, 0.16804130375385284], "agreement_weight": 0.012294492684304714, "orthogonal_weight": -0.21073795855045319}, {"head": 3, "memory_to_active": 7.2396626472473145, "contribution_to_active": 2.7029566764831543, "mix_mean": 0.031193219125270844, "mix_low": 0.4952859878540039, "mix_high": 0.0, "read_cosine": 0.2767564654350281, "beta_mean": 0.34449708461761475, "nonfinite": 0.0, "weighted_memory_to_active": 2.84135365486145, "mix_std": 0.07393723726272583, "beta_std": 0.26507318019866943, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11188622564077377, "evidence_weights": [-0.042523160576820374, -0.04160444065928459, -0.1553078591823578], "agreement_weight": -0.013265424408018589, "orthogonal_weight": -0.046441320329904556}, {"head": 4, "memory_to_active": 1.8541065454483032, "contribution_to_active": 0.07937870919704437, "mix_mean": 0.01022346317768097, "mix_low": 0.7574520111083984, "mix_high": 0.0, "read_cosine": 0.3253090977668762, "beta_mean": 0.0602165162563324, "nonfinite": 0.0, "weighted_memory_to_active": 0.05457912012934685, "mix_std": 0.03219303488731384, "beta_std": 0.061828263103961945, "beta_low": 0.0894775390625, "beta_high": 0.0, "orthogonal_to_active": 0.003797783050686121, "evidence_weights": [0.1398814469575882, 0.1009286642074585, -0.16110169887542725], "agreement_weight": -0.14194932579994202, "orthogonal_weight": 0.059837792068719864}, {"head": 5, "memory_to_active": 0.10440897941589355, "contribution_to_active": 0.4740924835205078, "mix_mean": 0.5351893901824951, "mix_low": 0.014545440673828125, "mix_high": 0.029996871948242188, "read_cosine": 0.9289371967315674, "beta_mean": 0.021631496027112007, "nonfinite": 0.0, "weighted_memory_to_active": 0.05679343268275261, "mix_std": 0.2894687354564667, "beta_std": 0.0207083597779274, "beta_low": 0.29986572265625, "beta_high": 0.0, "orthogonal_to_active": 0.01746724173426628, "evidence_weights": [-0.18044881522655487, -0.21803982555866241, 0.3843197226524353], "agreement_weight": 0.2908002436161041, "orthogonal_weight": -0.675674557685852}]}, {"layer": 1, "memory_to_active": 2.304574728012085, "contribution_to_active": 0.4628196060657501, "mix_mean": 0.20973359048366547, "mix_low": 0.36980709433555603, "mix_high": 0.0011404355755075812, "read_cosine": 0.31482696533203125, "beta_mean": 0.18633726239204407, "nonfinite": 0.0, "weighted_memory_to_active": 0.29975202679634094, "mix_std": 0.11410292983055115, "beta_std": 0.13614864647388458, "beta_low": 0.3482971489429474, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.03900061920285225, "output_influence": 0.2411535382270813, "residual_influence": 0.06589813530445099, "state_active": 0.03283950686454773, "state_memory": 0.03527399152517319, "state_peak": 38.9359245300293, "growth_active": 1.3130906820297241, "growth_memory": 1.4463999271392822, "heads": [{"head": 0, "memory_to_active": 0.022334199398756027, "contribution_to_active": 0.49344706535339355, "mix_mean": 0.5127655267715454, "mix_low": 0.0069866180419921875, "mix_high": 0.0022363662719726562, "read_cosine": 0.03492479398846626, "beta_mean": 0.00016622734256088734, "nonfinite": 0.0, "weighted_memory_to_active": 0.01300862804055214, "mix_std": 0.27973508834838867, "beta_std": 0.0002383348037255928, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012442545965313911, "evidence_weights": [-0.2090579718351364, -0.26991844177246094, 0.3943612575531006], "agreement_weight": 0.19552668929100037, "orthogonal_weight": -1.023733377456665}, {"head": 1, "memory_to_active": 2.8437962532043457, "contribution_to_active": 0.2768940031528473, "mix_mean": 0.023012228310108185, "mix_low": 0.47428035736083984, "mix_high": 0.0, "read_cosine": 0.5128958821296692, "beta_mean": 0.27800771594047546, "nonfinite": 0.0, "weighted_memory_to_active": 0.3125058114528656, "mix_std": 0.0451308935880661, "beta_std": 0.19446934759616852, "beta_low": 0.00201416015625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.019287612289190292, "evidence_weights": [0.01821232959628105, -0.052811648696660995, 0.04884716868400574], "agreement_weight": -0.000504413852468133, "orthogonal_weight": 0.17875738441944122}, {"head": 2, "memory_to_active": 0.054987113922834396, "contribution_to_active": 0.551140308380127, "mix_mean": 0.6633809208869934, "mix_low": 0.000522613525390625, "mix_high": 0.0046062469482421875, "read_cosine": 0.5091365575790405, "beta_mean": 0.002497569192200899, "nonfinite": 0.0, "weighted_memory_to_active": 0.03754381090402603, "mix_std": 0.23352757096290588, "beta_std": 0.004506996832787991, "beta_low": 0.9677734375, "beta_high": 0.0, "orthogonal_to_active": 0.025134922936558723, "evidence_weights": [-0.2822261452674866, -0.3694544732570648, 0.6650896668434143], "agreement_weight": 0.04051080346107483, "orthogonal_weight": -0.7909334897994995}, {"head": 3, "memory_to_active": 2.935457706451416, "contribution_to_active": 0.14840713143348694, "mix_mean": 0.022007418796420097, "mix_low": 0.5701990127563477, "mix_high": 0.0, "read_cosine": 0.19110232591629028, "beta_mean": 0.31761273741722107, "nonfinite": 0.0, "weighted_memory_to_active": 0.22168727219104767, "mix_std": 0.05662195384502411, "beta_std": 0.25905805826187134, "beta_low": 0.06610107421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08502250164747238, "evidence_weights": [0.03879343718290329, -0.017486654222011566, 0.010061103850603104], "agreement_weight": -0.03318162262439728, "orthogonal_weight": -0.46642568707466125}, {"head": 4, "memory_to_active": 3.830014228820801, "contribution_to_active": 1.240121841430664, "mix_mean": 0.030887022614479065, "mix_low": 0.3272438049316406, "mix_high": 0.0, "read_cosine": 0.32528796792030334, "beta_mean": 0.17916488647460938, "nonfinite": 0.0, "weighted_memory_to_active": 1.169478416442871, "mix_std": 0.053227778524160385, "beta_std": 0.1284831464290619, "beta_low": 0.03228759765625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.07824118435382843, "evidence_weights": [-0.028008049353957176, -0.0005045895813964307, 0.1141326054930687], "agreement_weight": -0.013603976927697659, "orthogonal_weight": 0.09447778761386871}, {"head": 5, "memory_to_active": 4.1408586502075195, "contribution_to_active": 0.06690728664398193, "mix_mean": 0.0063483999110758305, "mix_low": 0.8396100997924805, "mix_high": 0.0, "read_cosine": 0.3156143128871918, "beta_mean": 0.34057438373565674, "nonfinite": 0.0, "weighted_memory_to_active": 0.04428822547197342, "mix_std": 0.01637430302798748, "beta_std": 0.2301359623670578, "beta_low": 0.0216064453125, "beta_high": 0.000244140625, "orthogonal_to_active": 0.013874941505491734, "evidence_weights": [0.16396895051002502, 0.08439556509256363, 0.12272859364748001], "agreement_weight": -0.11783605068922043, "orthogonal_weight": 0.3113097846508026}]}, {"layer": 2, "memory_to_active": 1.035170555114746, "contribution_to_active": 0.35130614042282104, "mix_mean": 0.39464476704597473, "mix_low": 0.24974632263183594, "mix_high": 0.003982385154813528, "read_cosine": 0.30599066615104675, "beta_mean": 0.0826374888420105, "nonfinite": 0.0, "weighted_memory_to_active": 0.08492515981197357, "mix_std": 0.18923327326774597, "beta_std": 0.06846018880605698, "beta_low": 0.580902099609375, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.02122809924185276, "output_influence": 0.38311538100242615, "residual_influence": 0.10024997591972351, "state_active": 0.03600939363241196, "state_memory": 0.01530481781810522, "state_peak": 23.592668533325195, "growth_active": 1.0237457752227783, "growth_memory": 1.5503374338150024, "heads": [{"head": 0, "memory_to_active": 0.01597605273127556, "contribution_to_active": 0.32875677943229675, "mix_mean": 0.5975072383880615, "mix_low": 0.0008554458618164062, "mix_high": 0.006413459777832031, "read_cosine": 0.1162806823849678, "beta_mean": 9.17347933864221e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010316633619368076, "mix_std": 0.2751915156841278, "beta_std": 0.00024861760903149843, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010489999316632748, "evidence_weights": [-0.36293813586235046, -0.20999696850776672, 0.15026140213012695], "agreement_weight": 0.13320237398147583, "orthogonal_weight": -1.0491306781768799}, {"head": 1, "memory_to_active": 3.074333667755127, "contribution_to_active": 0.04320903122425079, "mix_mean": 0.004293282050639391, "mix_low": 0.913945198059082, "mix_high": 0.0, "read_cosine": 0.12866848707199097, "beta_mean": 0.23841151595115662, "nonfinite": 0.0, "weighted_memory_to_active": 0.034239232540130615, "mix_std": 0.009416201151907444, "beta_std": 0.15501706302165985, "beta_low": 0.001708984375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.01097869873046875, "evidence_weights": [0.1455673724412918, 0.1262633055448532, 0.5826371312141418], "agreement_weight": 0.03177441284060478, "orthogonal_weight": 0.36370211839675903}, {"head": 2, "memory_to_active": 0.00996145699173212, "contribution_to_active": 0.3978824019432068, "mix_mean": 0.5323483943939209, "mix_low": 0.0013093948364257812, "mix_high": 0.0026445388793945312, "read_cosine": 0.33253782987594604, "beta_mean": 2.9150585760362446e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003902928903698921, "mix_std": 0.27543365955352783, "beta_std": 4.382884071674198e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00035118887899443507, "evidence_weights": [-0.25023284554481506, -0.29168304800987244, 0.5690525770187378], "agreement_weight": -0.007422820199280977, "orthogonal_weight": -0.10425551235675812}, {"head": 3, "memory_to_active": 0.13374455273151398, "contribution_to_active": 0.5411005616188049, "mix_mean": 0.5713397264480591, "mix_low": 0.00022125244140625, "mix_high": 0.0065784454345703125, "read_cosine": 0.6689913868904114, "beta_mean": 0.02317645587027073, "nonfinite": 0.0, "weighted_memory_to_active": 0.08307298272848129, "mix_std": 0.2590549886226654, "beta_std": 0.030504409223794937, "beta_low": 0.44512939453125, "beta_high": 0.0, "orthogonal_to_active": 0.05438406020402908, "evidence_weights": [-0.20331154763698578, -0.31817588210105896, 0.8039522171020508], "agreement_weight": 0.055378787219524384, "orthogonal_weight": -0.8967023491859436}, {"head": 4, "memory_to_active": 2.96565580368042, "contribution_to_active": 0.2760530710220337, "mix_mean": 0.020129172131419182, "mix_low": 0.5812644958496094, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19157633185386658, "beta_mean": 0.23407310247421265, "nonfinite": 0.0, "weighted_memory_to_active": 0.371631383895874, "mix_std": 0.05105379968881607, "beta_std": 0.22489047050476074, "beta_low": 0.03857421875, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.05035809054970741, "evidence_weights": [0.022032873705029488, 0.0053019048646092415, 0.0427100770175457], "agreement_weight": 0.020971689373254776, "orthogonal_weight": -0.15705084800720215}, {"head": 5, "memory_to_active": 0.011351484805345535, "contribution_to_active": 0.5208349227905273, "mix_mean": 0.6422507762908936, "mix_low": 0.0008821487426757812, "mix_high": 0.008256912231445312, "read_cosine": 0.39788922667503357, "beta_mean": 4.2989515350200236e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006387726869434118, "mix_std": 0.26524943113327026, "beta_std": 5.6760596635285765e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008065405418165028, "evidence_weights": [-0.283080130815506, -0.29274049401283264, 1.0310386419296265], "agreement_weight": 0.15824829041957855, "orthogonal_weight": -0.16139915585517883}]}, {"layer": 3, "memory_to_active": 0.4550110399723053, "contribution_to_active": 0.4615294337272644, "mix_mean": 0.4876401424407959, "mix_low": 0.05616983026266098, "mix_high": 0.010125796310603619, "read_cosine": 0.1990671008825302, "beta_mean": 0.02716640569269657, "nonfinite": 0.0, "weighted_memory_to_active": 0.13523612916469574, "mix_std": 0.2609002888202667, "beta_std": 0.026520119979977608, "beta_low": 0.7638143301010132, "beta_high": 0.0, "orthogonal_to_active": 0.05512295663356781, "output_influence": 0.4064335823059082, "residual_influence": 0.0930200070142746, "state_active": 0.024110324680805206, "state_memory": 0.007330988068133593, "state_peak": 23.151227951049805, "growth_active": 1.024488925933838, "growth_memory": 1.705501675605774, "heads": [{"head": 0, "memory_to_active": 0.38178399205207825, "contribution_to_active": 0.5855590105056763, "mix_mean": 0.5899677276611328, "mix_low": 0.002063751220703125, "mix_high": 0.02489185333251953, "read_cosine": 0.26147401332855225, "beta_mean": 0.011325621977448463, "nonfinite": 0.0, "weighted_memory_to_active": 0.2419431507587433, "mix_std": 0.3047499656677246, "beta_std": 0.014747824519872665, "beta_low": 0.65753173828125, "beta_high": 0.0, "orthogonal_to_active": 0.17185792326927185, "evidence_weights": [-0.21998806297779083, -0.25378674268722534, 0.9486073851585388], "agreement_weight": 0.10905152559280396, "orthogonal_weight": -0.7521809935569763}, {"head": 1, "memory_to_active": 0.014764649793505669, "contribution_to_active": 0.41547834873199463, "mix_mean": 0.5551053285598755, "mix_low": 0.005946159362792969, "mix_high": 0.007602691650390625, "read_cosine": 0.1639404594898224, "beta_mean": 6.783391290809959e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008573977276682854, "mix_std": 0.2982053756713867, "beta_std": 0.0001612867636140436, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006922990083694458, "evidence_weights": [-0.26424694061279297, -0.21462400257587433, 0.6471370458602905], "agreement_weight": 0.49375104904174805, "orthogonal_weight": -0.8756584525108337}, {"head": 2, "memory_to_active": 2.0499749183654785, "contribution_to_active": 0.29653942584991455, "mix_mean": 0.0521748922765255, "mix_low": 0.3142280578613281, "mix_high": 0.0, "read_cosine": 0.21713802218437195, "beta_mean": 0.14689277112483978, "nonfinite": 0.0, "weighted_memory_to_active": 0.3565307855606079, "mix_std": 0.10780692100524902, "beta_std": 0.13362734019756317, "beta_low": 0.02313232421875, "beta_high": 0.0, "orthogonal_to_active": 0.00501627754420042, "evidence_weights": [-0.09368574619293213, 0.0014787771506235003, 7.024468504823744e-05], "agreement_weight": -0.008099437691271305, "orthogonal_weight": -0.01837713085114956}, {"head": 3, "memory_to_active": 0.022942079231142998, "contribution_to_active": 0.5116198658943176, "mix_mean": 0.5824530720710754, "mix_low": 0.006134986877441406, "mix_high": 0.010592460632324219, "read_cosine": 0.2974001169204712, "beta_mean": 0.00010187475709244609, "nonfinite": 0.0, "weighted_memory_to_active": 0.015241106040775776, "mix_std": 0.29247230291366577, "beta_std": 0.0002306994138052687, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0039375340566039085, "evidence_weights": [-0.2491624504327774, -0.22936546802520752, 2.160074472427368], "agreement_weight": 2.0159928798675537, "orthogonal_weight": -0.34903156757354736}, {"head": 4, "memory_to_active": 0.0211188867688179, "contribution_to_active": 0.32295116782188416, "mix_mean": 0.4437824487686157, "mix_low": 0.006468772888183594, "mix_high": 0.0018644332885742188, "read_cosine": 0.1346331685781479, "beta_mean": 9.24782725633122e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013208298943936825, "mix_std": 0.28928491473197937, "beta_std": 0.00013605077401734889, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012652505189180374, "evidence_weights": [-0.20412006974220276, -0.20246100425720215, 0.12516570091247559], "agreement_weight": -0.048832766711711884, "orthogonal_weight": -1.0400103330612183}, {"head": 5, "memory_to_active": 0.23948147892951965, "contribution_to_active": 0.6370287537574768, "mix_mean": 0.7023573517799377, "mix_low": 0.0021772384643554688, "mix_high": 0.01580333709716797, "read_cosine": 0.11981675028800964, "beta_mean": 0.004517842084169388, "nonfinite": 0.0, "weighted_memory_to_active": 0.17591936886310577, "mix_std": 0.27288222312927246, "beta_std": 0.01021752879023552, "beta_low": 0.90228271484375, "beta_high": 0.0, "orthogonal_to_active": 0.1303505152463913, "evidence_weights": [-0.26023462414741516, -0.3304778039455414, 0.48661360144615173], "agreement_weight": 0.1771375983953476, "orthogonal_weight": -0.7759723663330078}]}, {"layer": 4, "memory_to_active": 1.088573932647705, "contribution_to_active": 0.46386390924453735, "mix_mean": 0.3636542856693268, "mix_low": 0.2612017095088959, "mix_high": 0.008778095245361328, "read_cosine": 0.3108943998813629, "beta_mean": 0.11808481067419052, "nonfinite": 0.0, "weighted_memory_to_active": 0.13555096089839935, "mix_std": 0.15918108820915222, "beta_std": 0.10887861996889114, "beta_low": 0.383087158203125, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.045574694871902466, "output_influence": 0.4654931128025055, "residual_influence": 0.13463984429836273, "state_active": 0.03880889713764191, "state_memory": 0.017989041283726692, "state_peak": 38.40578842163086, "growth_active": 1.2290139198303223, "growth_memory": 1.530357003211975, "heads": [{"head": 0, "memory_to_active": 0.21406513452529907, "contribution_to_active": 0.759016215801239, "mix_mean": 0.6744692325592041, "mix_low": 0.013095855712890625, "mix_high": 0.018596649169921875, "read_cosine": 0.47880735993385315, "beta_mean": 0.2655906677246094, "nonfinite": 0.0, "weighted_memory_to_active": 0.17993877828121185, "mix_std": 0.3454754948616028, "beta_std": 0.270428329706192, "beta_low": 0.24774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.08247851580381393, "evidence_weights": [-0.26602378487586975, -0.36893513798713684, 0.019659703597426414], "agreement_weight": 0.003788465866819024, "orthogonal_weight": 0.6012492775917053}, {"head": 1, "memory_to_active": 0.05908409506082535, "contribution_to_active": 0.6467770338058472, "mix_mean": 0.6877512335777283, "mix_low": 0.00012111663818359375, "mix_high": 0.015116691589355469, "read_cosine": 0.5846518278121948, "beta_mean": 0.001130163436755538, "nonfinite": 0.0, "weighted_memory_to_active": 0.042750075459480286, "mix_std": 0.2417883425951004, "beta_std": 0.007501149084419012, "beta_low": 0.98089599609375, "beta_high": 0.0, "orthogonal_to_active": 0.02394125983119011, "evidence_weights": [-0.24148625135421753, -0.3616907298564911, 0.6374916434288025], "agreement_weight": 0.28723883628845215, "orthogonal_weight": -0.7668494582176208}, {"head": 2, "memory_to_active": 1.7678277492523193, "contribution_to_active": 0.3027351200580597, "mix_mean": 0.030243312940001488, "mix_low": 0.4802370071411133, "mix_high": 9.5367431640625e-06, "read_cosine": 0.2179427593946457, "beta_mean": 0.07474222779273987, "nonfinite": 0.0, "weighted_memory_to_active": 0.223004549741745, "mix_std": 0.06225050985813141, "beta_std": 0.07614871859550476, "beta_low": 0.0489501953125, "beta_high": 0.0, "orthogonal_to_active": 0.05829606205224991, "evidence_weights": [0.052332554012537, -0.00933240819722414, 0.18931511044502258], "agreement_weight": 0.12267068028450012, "orthogonal_weight": -0.34879589080810547}, {"head": 3, "memory_to_active": 2.1706783771514893, "contribution_to_active": 0.1574448049068451, "mix_mean": 0.032999590039253235, "mix_low": 0.41675472259521484, "mix_high": 0.0, "read_cosine": 0.18503938615322113, "beta_mean": 0.15176023542881012, "nonfinite": 0.0, "weighted_memory_to_active": 0.2146206945180893, "mix_std": 0.05859091132879257, "beta_std": 0.13340939581394196, "beta_low": 0.02093505859375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.000935798161663115, "evidence_weights": [-0.022714955732226372, 0.049745697528123856, -0.2747447192668915], "agreement_weight": 0.08065328001976013, "orthogonal_weight": -0.006232216954231262}, {"head": 4, "memory_to_active": 0.005948178935796022, "contribution_to_active": 0.6518455743789673, "mix_mean": 0.742903470993042, "mix_low": 9.632110595703125e-05, "mix_high": 0.0189208984375, "read_cosine": 0.1782013177871704, "beta_mean": 1.816876101656817e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004420297686010599, "mix_std": 0.2171851098537445, "beta_std": 2.9372920835157856e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024072404485195875, "evidence_weights": [-0.33329710364341736, -0.32540956139564514, 0.4879859387874603], "agreement_weight": 0.34693536162376404, "orthogonal_weight": -0.5844874978065491}, {"head": 5, "memory_to_active": 2.3138394355773926, "contribution_to_active": 0.265364408493042, "mix_mean": 0.013558631762862206, "mix_low": 0.6569051742553711, "mix_high": 2.47955322265625e-05, "read_cosine": 0.2207237333059311, "beta_mean": 0.2152673900127411, "nonfinite": 0.0, "weighted_memory_to_active": 0.14857134222984314, "mix_std": 0.02979608066380024, "beta_std": 0.1657547652721405, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.10538926720619202, "evidence_weights": [0.05486241355538368, -0.022397516295313835, -0.2579042613506317], "agreement_weight": -0.0843883752822876, "orthogonal_weight": 0.9826096296310425}]}, {"layer": 5, "memory_to_active": 0.4046687185764313, "contribution_to_active": 0.5375756025314331, "mix_mean": 0.5079560279846191, "mix_low": 0.055875301361083984, "mix_high": 0.01946369931101799, "read_cosine": 0.08223879337310791, "beta_mean": 0.051564399152994156, "nonfinite": 0.0, "weighted_memory_to_active": 0.14259080588817596, "mix_std": 0.2583509683609009, "beta_std": 0.06776770204305649, "beta_low": 0.66387939453125, "beta_high": 0.0, "orthogonal_to_active": 0.060080692172050476, "output_influence": 0.5498113036155701, "residual_influence": 0.15845440328121185, "state_active": 0.04187055304646492, "state_memory": 0.010706133209168911, "state_peak": 62.25232696533203, "growth_active": 1.1198450326919556, "growth_memory": 1.258184790611267, "heads": [{"head": 0, "memory_to_active": 0.2728722393512726, "contribution_to_active": 0.5479878783226013, "mix_mean": 0.42749497294425964, "mix_low": 0.018880844116210938, "mix_high": 0.0018138885498046875, "read_cosine": -0.0775449126958847, "beta_mean": 0.02078372612595558, "nonfinite": 0.0, "weighted_memory_to_active": 0.08735392242670059, "mix_std": 0.2822740077972412, "beta_std": 0.035062141716480255, "beta_low": 0.47576904296875, "beta_high": 0.0, "orthogonal_to_active": 0.03552211821079254, "evidence_weights": [-0.1497056782245636, -0.17091940343379974, 0.028959501534700394], "agreement_weight": -0.023078197613358498, "orthogonal_weight": -0.41445958614349365}, {"head": 1, "memory_to_active": 0.009384594857692719, "contribution_to_active": 0.5523107051849365, "mix_mean": 0.6912945508956909, "mix_low": 0.0015935897827148438, "mix_high": 0.03407478332519531, "read_cosine": 0.2325238585472107, "beta_mean": 2.2187319700606167e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00538235530257225, "mix_std": 0.27453362941741943, "beta_std": 4.72040701424703e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005104581825435162, "evidence_weights": [-0.24859488010406494, -0.28718870878219604, 0.8953832387924194], "agreement_weight": 0.674744188785553, "orthogonal_weight": -0.9942566752433777}, {"head": 2, "memory_to_active": 0.013034515082836151, "contribution_to_active": 0.6032531261444092, "mix_mean": 0.6687799692153931, "mix_low": 0.0022068023681640625, "mix_high": 0.026479721069335938, "read_cosine": -0.5099343657493591, "beta_mean": 9.275894990423694e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008060704916715622, "mix_std": 0.26568809151649475, "beta_std": 0.00014097311941441149, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004243436269462109, "evidence_weights": [-0.2664177715778351, -0.33934733271598816, 0.22411130368709564], "agreement_weight": -0.044977959245443344, "orthogonal_weight": -0.6446735262870789}, {"head": 3, "memory_to_active": 1.5714499950408936, "contribution_to_active": 0.24149611592292786, "mix_mean": 0.123555988073349, "mix_low": 0.11500835418701172, "mix_high": 9.5367431640625e-07, "read_cosine": 0.1492859274148941, "beta_mean": 0.1028149425983429, "nonfinite": 0.0, "weighted_memory_to_active": 0.38127756118774414, "mix_std": 0.1575949788093567, "beta_std": 0.15583647787570953, "beta_low": 0.20440673828125, "beta_high": 0.0, "orthogonal_to_active": 0.1656649261713028, "evidence_weights": [-0.07793883979320526, -0.10702846199274063, 0.046712953597307205], "agreement_weight": 0.04781722277402878, "orthogonal_weight": -0.5076565742492676}, {"head": 4, "memory_to_active": 0.463206022977829, "contribution_to_active": 0.5936568975448608, "mix_mean": 0.33294790983200073, "mix_low": 0.19588947296142578, "mix_high": 0.008072853088378906, "read_cosine": 0.5300256013870239, "beta_mean": 0.1853943169116974, "nonfinite": 0.0, "weighted_memory_to_active": 0.3048103153705597, "mix_std": 0.34605246782302856, "beta_std": 0.21424946188926697, "beta_low": 0.30682373046875, "beta_high": 0.0, "orthogonal_to_active": 0.10808505117893219, "evidence_weights": [-0.11818858981132507, -0.26448526978492737, -0.051737792789936066], "agreement_weight": 0.03547237068414688, "orthogonal_weight": 0.5253255367279053}, {"head": 5, "memory_to_active": 0.09806476533412933, "contribution_to_active": 0.6867486238479614, "mix_mean": 0.8036624789237976, "mix_low": 0.0016727447509765625, "mix_high": 0.046339988708496094, "read_cosine": 0.1690765917301178, "beta_mean": 0.00027842604322358966, "nonfinite": 0.0, "weighted_memory_to_active": 0.06865984946489334, "mix_std": 0.22396241128444672, "beta_std": 0.0012699132785201073, "beta_low": 0.99627685546875, "beta_high": 0.0, "orthogonal_to_active": 0.041864022612571716, "evidence_weights": [-0.3127424120903015, -0.33417534828186035, 1.9316695928573608], "agreement_weight": 0.4646156430244446, "orthogonal_weight": -0.6491867899894714}]}, {"layer": 6, "memory_to_active": 0.2968711256980896, "contribution_to_active": 0.6239414811134338, "mix_mean": 0.5886460542678833, "mix_low": 0.05151557922363281, "mix_high": 0.034955501556396484, "read_cosine": 0.31636565923690796, "beta_mean": 0.06411345303058624, "nonfinite": 0.0, "weighted_memory_to_active": 0.14870256185531616, "mix_std": 0.236001655459404, "beta_std": 0.07253660261631012, "beta_low": 0.5465087890625, "beta_high": 0.0, "orthogonal_to_active": 0.043364446610212326, "output_influence": 0.550728440284729, "residual_influence": 0.18157869577407837, "state_active": 0.04576139152050018, "state_memory": 0.007717744912952185, "state_peak": 94.68304443359375, "growth_active": 1.1982804536819458, "growth_memory": 1.140255331993103, "heads": [{"head": 0, "memory_to_active": 0.10518398880958557, "contribution_to_active": 0.7358826398849487, "mix_mean": 0.8114612102508545, "mix_low": 0.00019073486328125, "mix_high": 0.05070018768310547, "read_cosine": 0.2820243537425995, "beta_mean": 0.014462312683463097, "nonfinite": 0.0, "weighted_memory_to_active": 0.08870278298854828, "mix_std": 0.20412790775299072, "beta_std": 0.02668529376387596, "beta_low": 0.64007568359375, "beta_high": 0.0, "orthogonal_to_active": 0.06560675799846649, "evidence_weights": [-0.3156796991825104, -0.34815922379493713, 1.1981478929519653], "agreement_weight": 0.5396401286125183, "orthogonal_weight": -0.8107251524925232}, {"head": 1, "memory_to_active": 0.004833160899579525, "contribution_to_active": 0.5052958726882935, "mix_mean": 0.5813419818878174, "mix_low": 0.0009202957153320312, "mix_high": 0.013217926025390625, "read_cosine": 0.46801504492759705, "beta_mean": 1.027080634230515e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002110365778207779, "mix_std": 0.2850440442562103, "beta_std": 1.72062536876183e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011662684846669436, "evidence_weights": [-0.19639544188976288, -0.2996313273906708, 1.140020728111267], "agreement_weight": 0.05619344487786293, "orthogonal_weight": -0.7075133323669434}, {"head": 2, "memory_to_active": 0.8408167362213135, "contribution_to_active": 0.09875498712062836, "mix_mean": 0.04129209369421005, "mix_low": 0.3015890121459961, "mix_high": 0.0, "read_cosine": 0.2649301290512085, "beta_mean": 0.029568418860435486, "nonfinite": 0.0, "weighted_memory_to_active": 0.06863068044185638, "mix_std": 0.0671001598238945, "beta_std": 0.027934882789850235, "beta_low": 0.135009765625, "beta_high": 0.0, "orthogonal_to_active": 0.006652003154158592, "evidence_weights": [0.07500060647726059, -0.04226155951619148, 0.31824859976768494], "agreement_weight": 0.14860408008098602, "orthogonal_weight": -0.12300335615873337}, {"head": 3, "memory_to_active": 0.1370992809534073, "contribution_to_active": 0.6892569065093994, "mix_mean": 0.6636273860931396, "mix_low": 0.005734443664550781, "mix_high": 0.02515697479248047, "read_cosine": 0.4503418505191803, "beta_mean": 0.21579232811927795, "nonfinite": 0.0, "weighted_memory_to_active": 0.0980004295706749, "mix_std": 0.32376283407211304, "beta_std": 0.2441025823354721, "beta_low": 0.20458984375, "beta_high": 0.0, "orthogonal_to_active": 0.05417918041348457, "evidence_weights": [-0.17343638837337494, -0.32618263363838196, 0.17929832637310028], "agreement_weight": -0.0016871602274477482, "orthogonal_weight": 0.6273248791694641}, {"head": 4, "memory_to_active": 0.6652522087097168, "contribution_to_active": 1.086456060409546, "mix_mean": 0.7617976665496826, "mix_low": 0.0005598068237304688, "mix_high": 0.11414718627929688, "read_cosine": 0.14558622241020203, "beta_mean": 0.12470497190952301, "nonfinite": 0.0, "weighted_memory_to_active": 0.6145867109298706, "mix_std": 0.28848549723625183, "beta_std": 0.13616818189620972, "beta_low": 0.29949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.112716905772686, "evidence_weights": [-0.21649375557899475, -0.5677006840705872, -0.3808687925338745], "agreement_weight": -0.02422347664833069, "orthogonal_weight": 0.20473603904247284}, {"head": 5, "memory_to_active": 0.028041284531354904, "contribution_to_active": 0.6280022859573364, "mix_mean": 0.6723560094833374, "mix_low": 9.918212890625e-05, "mix_high": 0.006510734558105469, "read_cosine": 0.28729623556137085, "beta_mean": 0.00014239017036743462, "nonfinite": 0.0, "weighted_memory_to_active": 0.02018430270254612, "mix_std": 0.24748939275741577, "beta_std": 0.00031147064873948693, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.019865551963448524, "evidence_weights": [-0.23977762460708618, -0.3143640160560608, 0.38717538118362427], "agreement_weight": 0.18005692958831787, "orthogonal_weight": -1.0568088293075562}]}, {"layer": 7, "memory_to_active": 1.7128263711929321, "contribution_to_active": 0.2631884813308716, "mix_mean": 0.1580374836921692, "mix_low": 0.2491704821586609, "mix_high": 0.0011302630882710218, "read_cosine": 0.22225694358348846, "beta_mean": 0.1312432587146759, "nonfinite": 0.0, "weighted_memory_to_active": 0.20382685959339142, "mix_std": 0.14591194689273834, "beta_std": 0.09759383648633957, "beta_low": 0.202606201171875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.08125732839107513, "output_influence": 0.27281537652015686, "residual_influence": 0.06368037313222885, "state_active": 0.01339625846594572, "state_memory": 0.024528302252292633, "state_peak": 28.32799530029297, "growth_active": 1.1528931856155396, "growth_memory": 1.3153563737869263, "heads": [{"head": 0, "memory_to_active": 2.4883511066436768, "contribution_to_active": 0.21966633200645447, "mix_mean": 0.03549424558877945, "mix_low": 0.4364805221557617, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3307251036167145, "beta_mean": 0.15155348181724548, "nonfinite": 0.0, "weighted_memory_to_active": 0.25124257802963257, "mix_std": 0.0753638744354248, "beta_std": 0.11708410084247589, "beta_low": 0.00439453125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0008272202685475349, "evidence_weights": [0.04017086327075958, -0.07098101824522018, 0.03729701414704323], "agreement_weight": -0.035009391605854034, "orthogonal_weight": 0.00431935815140605}, {"head": 1, "memory_to_active": 0.9694069623947144, "contribution_to_active": 0.3997907042503357, "mix_mean": 0.22398579120635986, "mix_low": 0.12113475799560547, "mix_high": 0.006291389465332031, "read_cosine": 0.3743875324726105, "beta_mean": 0.0910947322845459, "nonfinite": 0.0, "weighted_memory_to_active": 0.3328285813331604, "mix_std": 0.24819880723953247, "beta_std": 0.06133482977747917, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.044377900660037994, "evidence_weights": [-0.21463866531848907, -0.10088373720645905, -0.26712530851364136], "agreement_weight": 0.03580128774046898, "orthogonal_weight": 0.1580297350883484}, {"head": 2, "memory_to_active": 0.6691792011260986, "contribution_to_active": 0.34527748823165894, "mix_mean": 0.192348450422287, "mix_low": 0.034806251525878906, "mix_high": 0.00026416778564453125, "read_cosine": 0.10006703436374664, "beta_mean": 0.04647449404001236, "nonfinite": 0.0, "weighted_memory_to_active": 0.2041832059621811, "mix_std": 0.20033231377601624, "beta_std": 0.042587533593177795, "beta_low": 0.097412109375, "beta_high": 0.0, "orthogonal_to_active": 0.10707543790340424, "evidence_weights": [-0.07709645479917526, -0.16066865622997284, 0.7665402293205261], "agreement_weight": 0.016376374289393425, "orthogonal_weight": -0.5383448004722595}, {"head": 3, "memory_to_active": 4.516059875488281, "contribution_to_active": 0.11130110919475555, "mix_mean": 0.03687122464179993, "mix_low": 0.3082151412963867, "mix_high": 0.0, "read_cosine": 0.06576446443796158, "beta_mean": 0.41704225540161133, "nonfinite": 0.0, "weighted_memory_to_active": 0.35177525877952576, "mix_std": 0.05386323481798172, "beta_std": 0.27680110931396484, "beta_low": 0.00274658203125, "beta_high": 0.0006103515625, "orthogonal_to_active": 0.29949766397476196, "evidence_weights": [0.06832980364561081, -0.031065037474036217, 0.7116124629974365], "agreement_weight": 0.05223650485277176, "orthogonal_weight": -0.9339590072631836}, {"head": 4, "memory_to_active": 1.5776647329330444, "contribution_to_active": 0.08420832455158234, "mix_mean": 0.01949736662209034, "mix_low": 0.5919294357299805, "mix_high": 0.0, "read_cosine": 0.1982938051223755, "beta_mean": 0.08104856312274933, "nonfinite": 0.0, "weighted_memory_to_active": 0.057242345064878464, "mix_std": 0.04214148595929146, "beta_std": 0.08713051676750183, "beta_low": 0.111572265625, "beta_high": 0.0, "orthogonal_to_active": 0.01266123354434967, "evidence_weights": [0.14485719799995422, 0.002241660375148058, 0.1720941662788391], "agreement_weight": 0.03792782500386238, "orthogonal_weight": -0.25887855887413025}, {"head": 5, "memory_to_active": 0.05629587918519974, "contribution_to_active": 0.4188868999481201, "mix_mean": 0.44002771377563477, "mix_low": 0.0024566650390625, "mix_high": 0.000225067138671875, "read_cosine": 0.26430368423461914, "beta_mean": 0.00024602573830634356, "nonfinite": 0.0, "weighted_memory_to_active": 0.025689110159873962, "mix_std": 0.2555719017982483, "beta_std": 0.0006248637801036239, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.02310444787144661, "evidence_weights": [-0.16813336312770844, -0.2245730608701706, 0.5510972738265991], "agreement_weight": 0.19219575822353363, "orthogonal_weight": -0.9700425863265991}]}, {"layer": 8, "memory_to_active": 0.18689976632595062, "contribution_to_active": 0.5543739199638367, "mix_mean": 0.4682850241661072, "mix_low": 0.017561595886945724, "mix_high": 0.001508871791884303, "read_cosine": 0.4279230237007141, "beta_mean": 0.015611352398991585, "nonfinite": 0.0, "weighted_memory_to_active": 0.08693893998861313, "mix_std": 0.2345898300409317, "beta_std": 0.02272069826722145, "beta_low": 0.7577006220817566, "beta_high": 0.0, "orthogonal_to_active": 0.07532892376184464, "output_influence": 0.5033713579177856, "residual_influence": 0.17443391680717468, "state_active": 0.04664658010005951, "state_memory": 0.005948594305664301, "state_peak": 45.16084671020508, "growth_active": 1.125281810760498, "growth_memory": 1.119044303894043, "heads": [{"head": 0, "memory_to_active": 0.061159923672676086, "contribution_to_active": 0.5702352523803711, "mix_mean": 0.6069797277450562, "mix_low": 0.0005130767822265625, "mix_high": 0.001468658447265625, "read_cosine": 0.4640251696109772, "beta_mean": 0.003674235427752137, "nonfinite": 0.0, "weighted_memory_to_active": 0.04335498437285423, "mix_std": 0.24676935374736786, "beta_std": 0.014125396497547626, "beta_low": 0.9215087890625, "beta_high": 0.0, "orthogonal_to_active": 0.027602683752775192, "evidence_weights": [-0.1680733561515808, -0.3891647756099701, 0.6397780179977417], "agreement_weight": 0.2348434180021286, "orthogonal_weight": -0.8055619597434998}, {"head": 1, "memory_to_active": 0.09278909862041473, "contribution_to_active": 0.542923092842102, "mix_mean": 0.510428786277771, "mix_low": 2.09808349609375e-05, "mix_high": 0.00091552734375, "read_cosine": 0.15354770421981812, "beta_mean": 0.0015536914579570293, "nonfinite": 0.0, "weighted_memory_to_active": 0.04310221970081329, "mix_std": 0.2359209656715393, "beta_std": 0.009587191045284271, "beta_low": 0.9849853515625, "beta_high": 0.0, "orthogonal_to_active": 0.031200852245092392, "evidence_weights": [-0.19430261850357056, -0.2751047909259796, 0.6957787275314331], "agreement_weight": 0.08099957555532455, "orthogonal_weight": -0.756098210811615}, {"head": 2, "memory_to_active": 0.016745194792747498, "contribution_to_active": 0.6785492897033691, "mix_mean": 0.6740743517875671, "mix_low": 0.0006589889526367188, "mix_high": 0.0030803680419921875, "read_cosine": 0.4382173717021942, "beta_mean": 0.0001550717861391604, "nonfinite": 0.0, "weighted_memory_to_active": 0.013258863240480423, "mix_std": 0.22357288002967834, "beta_std": 0.0031503408681601286, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.007892332039773464, "evidence_weights": [-0.22692656517028809, -0.34662488102912903, 1.019789457321167], "agreement_weight": 0.2743282914161682, "orthogonal_weight": -0.7020069360733032}, {"head": 3, "memory_to_active": 0.09811870753765106, "contribution_to_active": 0.6417552828788757, "mix_mean": 0.5767554044723511, "mix_low": 0.000835418701171875, "mix_high": 0.0034837722778320312, "read_cosine": 0.798117458820343, "beta_mean": 0.007881415076553822, "nonfinite": 0.0, "weighted_memory_to_active": 0.07441411912441254, "mix_std": 0.25653916597366333, "beta_std": 0.00750833610072732, "beta_low": 0.7369384765625, "beta_high": 0.0, "orthogonal_to_active": 0.023518936708569527, "evidence_weights": [-0.18681053817272186, -0.3023369610309601, 0.5620839595794678], "agreement_weight": 0.2268662303686142, "orthogonal_weight": -0.6312484741210938}, {"head": 4, "memory_to_active": 0.44778984785079956, "contribution_to_active": 0.24163097143173218, "mix_mean": 0.1818545013666153, "mix_low": 0.026251792907714844, "mix_high": 0.0, "read_cosine": 0.24651630222797394, "beta_mean": 0.006191120482981205, "nonfinite": 0.0, "weighted_memory_to_active": 0.14451847970485687, "mix_std": 0.17688269913196564, "beta_std": 0.01717916876077652, "beta_low": 0.8406982421875, "beta_high": 0.0, "orthogonal_to_active": 0.032276004552841187, "evidence_weights": [-0.059220410883426666, -0.14849026501178741, 0.3710908591747284], "agreement_weight": 0.0781543105840683, "orthogonal_weight": -0.2390752136707306}, {"head": 5, "memory_to_active": 0.4047958254814148, "contribution_to_active": 0.6511493921279907, "mix_mean": 0.25961726903915405, "mix_low": 0.07708930969238281, "mix_high": 0.0001049041748046875, "read_cosine": 0.46711409091949463, "beta_mean": 0.07421258091926575, "nonfinite": 0.0, "weighted_memory_to_active": 0.20298492908477783, "mix_std": 0.2678539752960205, "beta_std": 0.08477376401424408, "beta_low": 0.0621337890625, "beta_high": 0.0, "orthogonal_to_active": 0.3294827342033386, "evidence_weights": [-0.12683744728565216, -0.16497540473937988, -0.11507553607225418], "agreement_weight": 0.0005206788191571832, "orthogonal_weight": 1.8759136199951172}]}, {"layer": 9, "memory_to_active": 0.9559454321861267, "contribution_to_active": 0.6887547373771667, "mix_mean": 0.36066651344299316, "mix_low": 0.09955994784832001, "mix_high": 0.006237348075956106, "read_cosine": 0.3453090786933899, "beta_mean": 0.04745972529053688, "nonfinite": 0.0, "weighted_memory_to_active": 0.3880353569984436, "mix_std": 0.22768141329288483, "beta_std": 0.036972030997276306, "beta_low": 0.5167440176010132, "beta_high": 0.0, "orthogonal_to_active": 0.04152965545654297, "output_influence": 0.5370756983757019, "residual_influence": 0.1556355208158493, "state_active": 0.041599322110414505, "state_memory": 0.017082033678889275, "state_peak": 55.88330841064453, "growth_active": 1.1671992540359497, "growth_memory": 1.3187758922576904, "heads": [{"head": 0, "memory_to_active": 0.0014036486390978098, "contribution_to_active": 0.5725391507148743, "mix_mean": 0.6489242315292358, "mix_low": 0.001224517822265625, "mix_high": 0.02571868896484375, "read_cosine": 0.29385173320770264, "beta_mean": 4.5045230763207655e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0008865603595040739, "mix_std": 0.28635847568511963, "beta_std": 1.867158425739035e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009693179163150489, "evidence_weights": [-0.30533307790756226, -0.3116827607154846, 0.5922996997833252], "agreement_weight": 0.1157546117901802, "orthogonal_weight": -1.137342929840088}, {"head": 1, "memory_to_active": 0.020296484231948853, "contribution_to_active": 0.6907332539558411, "mix_mean": 0.5694023966789246, "mix_low": 0.007739067077636719, "mix_high": 0.0027570724487304688, "read_cosine": 0.4054129123687744, "beta_mean": 5.790407521999441e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014897176995873451, "mix_std": 0.28344443440437317, "beta_std": 0.00013901984493713826, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010343669913709164, "evidence_weights": [-0.2401261180639267, -0.2948441803455353, 1.0974231958389282], "agreement_weight": 0.4622199535369873, "orthogonal_weight": -0.8513255715370178}, {"head": 2, "memory_to_active": 0.17497095465660095, "contribution_to_active": 0.6446449160575867, "mix_mean": 0.5930122137069702, "mix_low": 0.0017099380493164062, "mix_high": 0.007657051086425781, "read_cosine": 0.3291645050048828, "beta_mean": 0.000725305057130754, "nonfinite": 0.0, "weighted_memory_to_active": 0.11814460158348083, "mix_std": 0.2705156207084656, "beta_std": 0.004502556286752224, "beta_low": 0.995849609375, "beta_high": 0.0, "orthogonal_to_active": 0.07025396823883057, "evidence_weights": [-0.20237520337104797, -0.27713507413864136, 0.747194766998291], "agreement_weight": 0.31937867403030396, "orthogonal_weight": -0.6810398697853088}, {"head": 3, "memory_to_active": 1.025343656539917, "contribution_to_active": 0.19122473895549774, "mix_mean": 0.08629784733057022, "mix_low": 0.2309732437133789, "mix_high": 0.00022983551025390625, "read_cosine": 0.39381811022758484, "beta_mean": 0.04503113031387329, "nonfinite": 0.0, "weighted_memory_to_active": 0.15975545346736908, "mix_std": 0.1450890302658081, "beta_std": 0.03793223202228546, "beta_low": 0.03741455078125, "beta_high": 0.0, "orthogonal_to_active": 0.03468907251954079, "evidence_weights": [0.03795202076435089, -0.034237004816532135, 0.14407014846801758], "agreement_weight": 0.09505437314510345, "orthogonal_weight": -0.22909916937351227}, {"head": 4, "memory_to_active": 2.883100748062134, "contribution_to_active": 1.4519673585891724, "mix_mean": 0.1494906097650528, "mix_low": 0.13225269317626953, "mix_high": 0.0008907318115234375, "read_cosine": 0.1754571795463562, "beta_mean": 0.16045114398002625, "nonfinite": 0.0, "weighted_memory_to_active": 1.4270681142807007, "mix_std": 0.20918875932693481, "beta_std": 0.09191371500492096, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.030823295935988426, "evidence_weights": [-0.08141809701919556, -0.15725232660770416, -0.01513293944299221], "agreement_weight": -0.06068505346775055, "orthogonal_weight": -0.022311242297291756}, {"head": 5, "memory_to_active": 1.6305567026138306, "contribution_to_active": 0.5814187526702881, "mix_mean": 0.11687177419662476, "mix_low": 0.22346019744873047, "mix_high": 0.00017070770263671875, "read_cosine": 0.47415006160736084, "beta_mean": 0.07848834991455078, "nonfinite": 0.0, "weighted_memory_to_active": 0.6074599623680115, "mix_std": 0.17149212956428528, "beta_std": 0.08732599020004272, "beta_low": 0.06719970703125, "beta_high": 0.0, "orthogonal_to_active": 0.10209859907627106, "evidence_weights": [0.09785418957471848, -0.12348619103431702, 0.14229750633239746], "agreement_weight": 0.013996658846735954, "orthogonal_weight": 0.22779926657676697}]}, {"layer": 10, "memory_to_active": 0.2351228892803192, "contribution_to_active": 0.7370569705963135, "mix_mean": 0.521416425704956, "mix_low": 0.021575771272182465, "mix_high": 0.005105654709041119, "read_cosine": 0.3755643367767334, "beta_mean": 0.04592391848564148, "nonfinite": 0.0, "weighted_memory_to_active": 0.1085948646068573, "mix_std": 0.2692902088165283, "beta_std": 0.04765017330646515, "beta_low": 0.66192626953125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.26334279775619507, "output_influence": 0.4503587484359741, "residual_influence": 0.15946917235851288, "state_active": 0.05557100474834442, "state_memory": 0.016863323748111725, "state_peak": 118.92684936523438, "growth_active": 1.0203526020050049, "growth_memory": 0.977380096912384, "heads": [{"head": 0, "memory_to_active": 0.9041423797607422, "contribution_to_active": 2.1374411582946777, "mix_mean": 0.5762463808059692, "mix_low": 0.0928802490234375, "mix_high": 0.008845329284667969, "read_cosine": 0.10056544840335846, "beta_mean": 0.21103841066360474, "nonfinite": 0.0, "weighted_memory_to_active": 0.534236490726471, "mix_std": 0.39423927664756775, "beta_std": 0.2222355604171753, "beta_low": 0.1610107421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.4967246055603027, "evidence_weights": [-0.0965949073433876, -0.38606950640678406, -1.7076854705810547], "agreement_weight": 0.028826257213950157, "orthogonal_weight": 2.8274807929992676}, {"head": 1, "memory_to_active": 0.010430514812469482, "contribution_to_active": 0.6055549383163452, "mix_mean": 0.6273007988929749, "mix_low": 0.0015163421630859375, "mix_high": 0.015372276306152344, "read_cosine": 0.28954261541366577, "beta_mean": 7.001133053563535e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005919421557337046, "mix_std": 0.2656252682209015, "beta_std": 0.00010046920215245336, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009017246775329113, "evidence_weights": [-0.25321727991104126, -0.19747747480869293, 0.5575076937675476], "agreement_weight": 0.558914065361023, "orthogonal_weight": -1.6369260549545288}, {"head": 2, "memory_to_active": 0.010709940455853939, "contribution_to_active": 0.6195234656333923, "mix_mean": 0.6532038450241089, "mix_low": 0.00020694732666015625, "mix_high": 0.004925727844238281, "read_cosine": 0.02403990738093853, "beta_mean": 2.1290708900778554e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004562057089060545, "mix_std": 0.2505624294281006, "beta_std": 8.094594522845e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020400655921548605, "evidence_weights": [-0.26022738218307495, -0.2814798653125763, 0.6139329671859741], "agreement_weight": -0.19720862805843353, "orthogonal_weight": -0.42616188526153564}, {"head": 3, "memory_to_active": 0.3379295766353607, "contribution_to_active": 0.09483641386032104, "mix_mean": 0.28458431363105774, "mix_low": 0.03271961212158203, "mix_high": 8.869171142578125e-05, "read_cosine": 0.9233143329620361, "beta_mean": 0.057506948709487915, "nonfinite": 0.0, "weighted_memory_to_active": 0.035596590489149094, "mix_std": 0.22963272035121918, "beta_std": 0.05525001883506775, "beta_low": 0.002197265625, "beta_high": 0.0, "orthogonal_to_active": 0.037277817726135254, "evidence_weights": [-0.10316390544176102, -0.14834702014923096, 0.16264088451862335], "agreement_weight": 0.013671426102519035, "orthogonal_weight": 1.4066041707992554}, {"head": 4, "memory_to_active": 0.01931089349091053, "contribution_to_active": 0.5553058981895447, "mix_mean": 0.5864611268043518, "mix_low": 0.0002689361572265625, "mix_high": 0.0013990402221679688, "read_cosine": 0.06234656274318695, "beta_mean": 5.508910180651583e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011279141530394554, "mix_std": 0.2488153576850891, "beta_std": 9.774777572602034e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01500441413372755, "evidence_weights": [-0.18644794821739197, -0.26244908571243286, 0.991330087184906], "agreement_weight": 0.15808099508285522, "orthogonal_weight": -1.4055962562561035}, {"head": 5, "memory_to_active": 0.1282140165567398, "contribution_to_active": 0.4096800684928894, "mix_mean": 0.4007016122341156, "mix_low": 0.0018625259399414062, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8535770177841187, "beta_mean": 0.006851729471236467, "nonfinite": 0.0, "weighted_memory_to_active": 0.05997545272111893, "mix_std": 0.22686631977558136, "beta_std": 0.008136291988193989, "beta_low": 0.808349609375, "beta_high": 0.0, "orthogonal_to_active": 0.01999245025217533, "evidence_weights": [-0.14150525629520416, -0.16241832077503204, 0.2688341736793518], "agreement_weight": 0.09151788055896759, "orthogonal_weight": -0.6610316038131714}]}, {"layer": 11, "memory_to_active": 0.08393991738557816, "contribution_to_active": 0.6160733699798584, "mix_mean": 0.5134522318840027, "mix_low": 0.11226590722799301, "mix_high": 0.04591258615255356, "read_cosine": 0.3516443967819214, "beta_mean": 0.026562876999378204, "nonfinite": 0.0, "weighted_memory_to_active": 0.04346384108066559, "mix_std": 0.3260467052459717, "beta_std": 0.03913640230894089, "beta_low": 0.6907552480697632, "beta_high": 0.0, "orthogonal_to_active": 0.04419315606355667, "output_influence": 0.5534666776657104, "residual_influence": 0.14350491762161255, "state_active": 0.20559191703796387, "state_memory": 0.0039064353331923485, "state_peak": 28.27766227722168, "growth_active": 1.0887823104858398, "growth_memory": 1.5562374591827393, "heads": [{"head": 0, "memory_to_active": 0.006505859084427357, "contribution_to_active": 0.5994271636009216, "mix_mean": 0.6416624188423157, "mix_low": 0.0009660720825195312, "mix_high": 0.008978843688964844, "read_cosine": 0.28757381439208984, "beta_mean": 8.361694926861674e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004617253318428993, "mix_std": 0.2604063153266907, "beta_std": 0.0037694962229579687, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.0035985896829515696, "evidence_weights": [-0.17165164649486542, -0.36390456557273865, 0.7360163927078247], "agreement_weight": 0.34949034452438354, "orthogonal_weight": -0.824985146522522}, {"head": 1, "memory_to_active": 0.16862215101718903, "contribution_to_active": 0.397857666015625, "mix_mean": 0.4412546157836914, "mix_low": 0.0072021484375, "mix_high": 0.0006580352783203125, "read_cosine": 0.48262232542037964, "beta_mean": 0.00840095803141594, "nonfinite": 0.0, "weighted_memory_to_active": 0.0764278918504715, "mix_std": 0.26995334029197693, "beta_std": 0.021127598360180855, "beta_low": 0.83331298828125, "beta_high": 0.0, "orthogonal_to_active": 0.03895176947116852, "evidence_weights": [-0.15576790273189545, -0.24099873006343842, 0.5965570211410522], "agreement_weight": 0.0723993331193924, "orthogonal_weight": -0.6003392338752747}, {"head": 2, "memory_to_active": 0.006728014908730984, "contribution_to_active": 0.984575092792511, "mix_mean": 0.4352905750274658, "mix_low": 0.3076333999633789, "mix_high": 0.1730976104736328, "read_cosine": 0.16001951694488525, "beta_mean": 0.07349243760108948, "nonfinite": 0.0, "weighted_memory_to_active": 0.004658843856304884, "mix_std": 0.4499211311340332, "beta_std": 0.09845426678657532, "beta_low": 0.484619140625, "beta_high": 0.0, "orthogonal_to_active": 0.04032432660460472, "evidence_weights": [0.03626644238829613, -0.39357614517211914, -1.3086466789245605], "agreement_weight": 0.011850394308567047, "orthogonal_weight": 8.98548698425293}, {"head": 3, "memory_to_active": 0.007537797559052706, "contribution_to_active": 0.8733243942260742, "mix_mean": 0.4080026149749756, "mix_low": 0.3481311798095703, "mix_high": 0.0853586196899414, "read_cosine": 0.5249242782592773, "beta_mean": 0.0545404776930809, "nonfinite": 0.0, "weighted_memory_to_active": 0.004897273611277342, "mix_std": 0.44060471653938293, "beta_std": 0.08275939524173737, "beta_low": 0.51873779296875, "beta_high": 0.0, "orthogonal_to_active": 0.03074183128774166, "evidence_weights": [0.19401121139526367, -0.30151525139808655, -0.16410517692565918], "agreement_weight": 0.0504758395254612, "orthogonal_weight": 9.285462379455566}, {"head": 4, "memory_to_active": 0.2317499965429306, "contribution_to_active": 0.4388434886932373, "mix_mean": 0.5973650217056274, "mix_low": 0.006732940673828125, "mix_high": 0.0019426345825195312, "read_cosine": 0.07026581466197968, "beta_mean": 0.01557387039065361, "nonfinite": 0.0, "weighted_memory_to_active": 0.1289750039577484, "mix_std": 0.27329909801483154, "beta_std": 0.018033811822533607, "beta_low": 0.5113525390625, "beta_high": 0.0, "orthogonal_to_active": 0.12529128789901733, "evidence_weights": [-0.24490797519683838, -0.2422984391450882, 1.9445641040802002], "agreement_weight": -0.2048133909702301, "orthogonal_weight": -0.9860374331474304}, {"head": 5, "memory_to_active": 0.08249565958976746, "contribution_to_active": 0.40241262316703796, "mix_mean": 0.5571379661560059, "mix_low": 0.0029296875, "mix_high": 0.00543975830078125, "read_cosine": 0.5844604969024658, "beta_mean": 0.007285900413990021, "nonfinite": 0.0, "weighted_memory_to_active": 0.04120676591992378, "mix_std": 0.26209554076194763, "beta_std": 0.010673852637410164, "beta_low": 0.79681396484375, "beta_high": 0.0, "orthogonal_to_active": 0.026251129806041718, "evidence_weights": [-0.2107236385345459, -0.3130319118499756, 0.6527560353279114], "agreement_weight": 0.0833100751042366, "orthogonal_weight": -0.8359541893005371}]}, {"layer": 12, "memory_to_active": 0.21606159210205078, "contribution_to_active": 0.5816960334777832, "mix_mean": 0.5369188189506531, "mix_low": 0.00895420741289854, "mix_high": 0.005314191337674856, "read_cosine": 0.23633474111557007, "beta_mean": 0.014760207384824753, "nonfinite": 0.0, "weighted_memory_to_active": 0.11299439519643784, "mix_std": 0.2630808353424072, "beta_std": 0.014445784501731396, "beta_low": 0.8292236924171448, "beta_high": 0.0, "orthogonal_to_active": 0.07264947891235352, "output_influence": 0.4950115978717804, "residual_influence": 0.13968831300735474, "state_active": 0.03435308486223221, "state_memory": 0.008772818371653557, "state_peak": 64.79767608642578, "growth_active": 1.1519052982330322, "growth_memory": 1.2151730060577393, "heads": [{"head": 0, "memory_to_active": 0.037081487476825714, "contribution_to_active": 0.5943896770477295, "mix_mean": 0.5605320930480957, "mix_low": 0.0004940032958984375, "mix_high": 0.0039234161376953125, "read_cosine": 0.2894439995288849, "beta_mean": 0.0006270198500715196, "nonfinite": 0.0, "weighted_memory_to_active": 0.019646387547254562, "mix_std": 0.26646149158477783, "beta_std": 0.0012188524706289172, "beta_low": 0.9974365234375, "beta_high": 0.0, "orthogonal_to_active": 0.017886599525809288, "evidence_weights": [-0.20784856379032135, -0.27531522512435913, 0.054859861731529236], "agreement_weight": 0.10087356716394424, "orthogonal_weight": -0.981045126914978}, {"head": 1, "memory_to_active": 0.08078967034816742, "contribution_to_active": 0.5437374711036682, "mix_mean": 0.5508006811141968, "mix_low": 0.00023937225341796875, "mix_high": 0.001552581787109375, "read_cosine": 0.4174772799015045, "beta_mean": 0.0006818261463195086, "nonfinite": 0.0, "weighted_memory_to_active": 0.04882674664258957, "mix_std": 0.26404643058776855, "beta_std": 0.007260266691446304, "beta_low": 0.99603271484375, "beta_high": 0.0, "orthogonal_to_active": 0.04085734114050865, "evidence_weights": [-0.20406176149845123, -0.27227428555488586, 1.0501890182495117], "agreement_weight": 0.22151152789592743, "orthogonal_weight": -0.9294381141662598}, {"head": 2, "memory_to_active": 0.6291981935501099, "contribution_to_active": 0.5590299367904663, "mix_mean": 0.33463120460510254, "mix_low": 0.027973175048828125, "mix_high": 0.0005464553833007812, "read_cosine": 0.4731234312057495, "beta_mean": 0.08614760637283325, "nonfinite": 0.0, "weighted_memory_to_active": 0.27607035636901855, "mix_std": 0.25262805819511414, "beta_std": 0.06272465735673904, "beta_low": 0.00128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.2985626459121704, "evidence_weights": [-0.21910572052001953, -0.05685894563794136, 0.005969629157334566], "agreement_weight": 0.026076702401041985, "orthogonal_weight": 1.3375866413116455}, {"head": 3, "memory_to_active": 0.03604502975940704, "contribution_to_active": 0.7397769689559937, "mix_mean": 0.7126299142837524, "mix_low": 0.000125885009765625, "mix_high": 0.016556739807128906, "read_cosine": 0.14839939773082733, "beta_mean": 0.0003078462032135576, "nonfinite": 0.0, "weighted_memory_to_active": 0.024230103939771652, "mix_std": 0.23098742961883545, "beta_std": 0.005281957797706127, "beta_low": 0.99749755859375, "beta_high": 0.0, "orthogonal_to_active": 0.023098327219486237, "evidence_weights": [-0.29006698727607727, -0.33757272362709045, 1.6417595148086548], "agreement_weight": 0.5107224583625793, "orthogonal_weight": -0.9759891629219055}, {"head": 4, "memory_to_active": 0.46108943223953247, "contribution_to_active": 0.5557525157928467, "mix_mean": 0.4370673894882202, "mix_low": 0.018285751342773438, "mix_high": 0.00446319580078125, "read_cosine": -0.15072964131832123, "beta_mean": 0.0005656173452734947, "nonfinite": 0.0, "weighted_memory_to_active": 0.27405527234077454, "mix_std": 0.28926390409469604, "beta_std": 0.007898899726569653, "beta_low": 0.98553466796875, "beta_high": 0.0, "orthogonal_to_active": 0.0286173727363348, "evidence_weights": [-0.15234480798244476, -0.21893668174743652, -0.0933358445763588], "agreement_weight": 0.008095541968941689, "orthogonal_weight": -0.10866302996873856}, {"head": 5, "memory_to_active": 0.052165623754262924, "contribution_to_active": 0.497489333152771, "mix_mean": 0.6258516311645508, "mix_low": 0.0066070556640625, "mix_high": 0.0048427581787109375, "read_cosine": 0.240293949842453, "beta_mean": 0.00023132513160817325, "nonfinite": 0.0, "weighted_memory_to_active": 0.035137444734573364, "mix_std": 0.2750976085662842, "beta_std": 0.002290078904479742, "beta_low": 0.99755859375, "beta_high": 0.0, "orthogonal_to_active": 0.02687455341219902, "evidence_weights": [-0.2318563312292099, -0.31713640689849854, 1.3060492277145386], "agreement_weight": 0.39236655831336975, "orthogonal_weight": -0.822289228439331}]}, {"layer": 13, "memory_to_active": 0.1232859343290329, "contribution_to_active": 0.5340321063995361, "mix_mean": 0.5396722555160522, "mix_low": 0.005832195747643709, "mix_high": 0.007835070602595806, "read_cosine": 0.40353119373321533, "beta_mean": 0.014804217964410782, "nonfinite": 0.0, "weighted_memory_to_active": 0.056485094130039215, "mix_std": 0.26259076595306396, "beta_std": 0.017085297033190727, "beta_low": 0.807708740234375, "beta_high": 0.0, "orthogonal_to_active": 0.05722040683031082, "output_influence": 0.5224761962890625, "residual_influence": 0.1780661791563034, "state_active": 0.13595280051231384, "state_memory": 0.04448673501610756, "state_peak": 97.2060317993164, "growth_active": 0.9330442547798157, "growth_memory": 1.0358301401138306, "heads": [{"head": 0, "memory_to_active": 0.0192730613052845, "contribution_to_active": 0.5911198258399963, "mix_mean": 0.6091893315315247, "mix_low": 0.0005769729614257812, "mix_high": 0.00428009033203125, "read_cosine": 0.45669472217559814, "beta_mean": 6.873023085063323e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012528505176305771, "mix_std": 0.26866793632507324, "beta_std": 0.00014045585703570396, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012056449428200722, "evidence_weights": [-0.19762738049030304, -0.2684405744075775, 0.7786816954612732], "agreement_weight": 0.15853561460971832, "orthogonal_weight": -1.0908846855163574}, {"head": 1, "memory_to_active": 0.015025395900011063, "contribution_to_active": 0.5944377183914185, "mix_mean": 0.5650709271430969, "mix_low": 0.0017871856689453125, "mix_high": 0.0038022994995117188, "read_cosine": 0.19970710575580597, "beta_mean": 5.930510815232992e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0084628751501441, "mix_std": 0.2743324041366577, "beta_std": 0.00010744195606093854, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011089184321463108, "evidence_weights": [-0.18296173214912415, -0.2645348608493805, 0.6195313930511475], "agreement_weight": 0.44819509983062744, "orthogonal_weight": -1.361477017402649}, {"head": 2, "memory_to_active": 0.08083758503198624, "contribution_to_active": 0.4744952917098999, "mix_mean": 0.39614468812942505, "mix_low": 0.0012063980102539062, "mix_high": 0.000202178955078125, "read_cosine": 0.6205558776855469, "beta_mean": 0.011327371001243591, "nonfinite": 0.0, "weighted_memory_to_active": 0.038661036640405655, "mix_std": 0.2363169640302658, "beta_std": 0.03970576450228691, "beta_low": 0.85186767578125, "beta_high": 0.0, "orthogonal_to_active": 0.021670322865247726, "evidence_weights": [-0.18564951419830322, -0.20829220116138458, 0.41939035058021545], "agreement_weight": 0.00728181004524231, "orthogonal_weight": -0.8420844078063965}, {"head": 3, "memory_to_active": 0.18301226198673248, "contribution_to_active": 0.5717210173606873, "mix_mean": 0.6473768949508667, "mix_low": 0.02370166778564453, "mix_high": 0.027330398559570312, "read_cosine": -0.04433487355709076, "beta_mean": 0.0008493732893839478, "nonfinite": 0.0, "weighted_memory_to_active": 0.13584324717521667, "mix_std": 0.28886494040489197, "beta_std": 0.007310794200748205, "beta_low": 0.9942626953125, "beta_high": 0.0, "orthogonal_to_active": 0.12140510976314545, "evidence_weights": [-0.31694820523262024, -0.2452869862318039, 1.6427830457687378], "agreement_weight": 5.807581901550293, "orthogonal_weight": -0.9702984094619751}, {"head": 4, "memory_to_active": 0.39215362071990967, "contribution_to_active": 0.26488441228866577, "mix_mean": 0.36446520686149597, "mix_low": 0.0048675537109375, "mix_high": 7.343292236328125e-05, "read_cosine": 0.8892378211021423, "beta_mean": 0.07629396766424179, "nonfinite": 0.0, "weighted_memory_to_active": 0.10864666104316711, "mix_std": 0.23558309674263, "beta_std": 0.052759893238544464, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.1475178301334381, "evidence_weights": [-0.08552809059619904, -0.2306986302137375, 0.466862291097641], "agreement_weight": 0.010401688516139984, "orthogonal_weight": 2.2296762466430664}, {"head": 5, "memory_to_active": 0.04941365867853165, "contribution_to_active": 0.7075346112251282, "mix_mean": 0.6557862758636475, "mix_low": 0.0028533935546875, "mix_high": 0.011322021484375, "read_cosine": 0.2993263602256775, "beta_mean": 0.00022655967040918767, "nonfinite": 0.0, "weighted_memory_to_active": 0.03476822003722191, "mix_std": 0.27177926898002625, "beta_std": 0.0024874310474842787, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.02958349511027336, "evidence_weights": [-0.22247807681560516, -0.30217453837394714, 0.868298351764679], "agreement_weight": 0.1693975180387497, "orthogonal_weight": -0.9244341850280762}]}, {"layer": 14, "memory_to_active": 0.22512710094451904, "contribution_to_active": 0.4221322238445282, "mix_mean": 0.4292188286781311, "mix_low": 0.026081880554556847, "mix_high": 0.0057619414292275906, "read_cosine": 0.6160515546798706, "beta_mean": 0.06362144649028778, "nonfinite": 0.0, "weighted_memory_to_active": 0.09768062829971313, "mix_std": 0.2458554357290268, "beta_std": 0.05983484908938408, "beta_low": 0.4879252314567566, "beta_high": 0.0, "orthogonal_to_active": 0.0413023978471756, "output_influence": 0.3524426221847534, "residual_influence": 0.13629311323165894, "state_active": 0.05664718896150589, "state_memory": 0.015025205910205841, "state_peak": 39.69554138183594, "growth_active": 1.1002920866012573, "growth_memory": 1.2601269483566284, "heads": [{"head": 0, "memory_to_active": 0.34394216537475586, "contribution_to_active": 0.20088964700698853, "mix_mean": 0.20030315220355988, "mix_low": 0.019286155700683594, "mix_high": 0.0, "read_cosine": 0.8040364980697632, "beta_mean": 0.06515990197658539, "nonfinite": 0.0, "weighted_memory_to_active": 0.08153047412633896, "mix_std": 0.189592644572258, "beta_std": 0.05681248754262924, "beta_low": 0.00885009765625, "beta_high": 0.0, "orthogonal_to_active": 0.08091309666633606, "evidence_weights": [-0.11081703007221222, -0.04387487471103668, 0.4272153675556183], "agreement_weight": 0.025893844664096832, "orthogonal_weight": 1.6924595832824707}, {"head": 1, "memory_to_active": 0.45575249195098877, "contribution_to_active": 0.3103817105293274, "mix_mean": 0.22082723677158356, "mix_low": 0.12140655517578125, "mix_high": 1.33514404296875e-05, "read_cosine": 0.8670708537101746, "beta_mean": 0.2634127736091614, "nonfinite": 0.0, "weighted_memory_to_active": 0.21165721118450165, "mix_std": 0.2451592981815338, "beta_std": 0.2508254945278168, "beta_low": 0.08685302734375, "beta_high": 0.0, "orthogonal_to_active": 0.012953899800777435, "evidence_weights": [-0.004110867623239756, -0.19500383734703064, -0.09035705029964447], "agreement_weight": 0.058388639241456985, "orthogonal_weight": 0.1775248944759369}, {"head": 2, "memory_to_active": 0.12299039214849472, "contribution_to_active": 0.5786435604095459, "mix_mean": 0.5886765718460083, "mix_low": 0.0031442642211914062, "mix_high": 0.0034093856811523438, "read_cosine": 0.42459702491760254, "beta_mean": 0.0020143105648458004, "nonfinite": 0.0, "weighted_memory_to_active": 0.0799344927072525, "mix_std": 0.26759636402130127, "beta_std": 0.002574231242761016, "beta_low": 0.98297119140625, "beta_high": 0.0, "orthogonal_to_active": 0.06216978654265404, "evidence_weights": [-0.13965743780136108, -0.31608423590660095, 0.9068419337272644], "agreement_weight": 0.0828726664185524, "orthogonal_weight": -0.8897092342376709}, {"head": 3, "memory_to_active": 0.047963835299015045, "contribution_to_active": 0.6064305305480957, "mix_mean": 0.6699657440185547, "mix_low": 0.0043125152587890625, "mix_high": 0.021384239196777344, "read_cosine": -0.03054201602935791, "beta_mean": 0.006427160929888487, "nonfinite": 0.0, "weighted_memory_to_active": 0.030336644500494003, "mix_std": 0.25613605976104736, "beta_std": 0.008258569054305553, "beta_low": 0.80877685546875, "beta_high": 0.0, "orthogonal_to_active": 0.023043280467391014, "evidence_weights": [-0.3485966920852661, -0.31233081221580505, 0.28463420271873474], "agreement_weight": 0.07190082967281342, "orthogonal_weight": -0.8403334617614746}, {"head": 4, "memory_to_active": 0.159327432513237, "contribution_to_active": 0.4997616708278656, "mix_mean": 0.5173870921134949, "mix_low": 0.0040493011474609375, "mix_high": 0.008493423461914062, "read_cosine": 0.8833810091018677, "beta_mean": 0.04311404749751091, "nonfinite": 0.0, "weighted_memory_to_active": 0.0951160117983818, "mix_std": 0.26175904273986816, "beta_std": 0.03174177184700966, "beta_low": 0.0662841796875, "beta_high": 0.0, "orthogonal_to_active": 0.026043348014354706, "evidence_weights": [-0.21878887712955475, -0.24566718935966492, 0.4319499433040619], "agreement_weight": 0.1364676058292389, "orthogonal_weight": -0.6118042469024658}, {"head": 5, "memory_to_active": 0.22078625857830048, "contribution_to_active": 0.3366861343383789, "mix_mean": 0.3781529664993286, "mix_low": 0.004292488098144531, "mix_high": 0.0012712478637695312, "read_cosine": 0.747765839099884, "beta_mean": 0.0016004438512027264, "nonfinite": 0.0, "weighted_memory_to_active": 0.0875089168548584, "mix_std": 0.25488919019699097, "beta_std": 0.008796527981758118, "beta_low": 0.97381591796875, "beta_high": 0.0, "orthogonal_to_active": 0.04269098490476608, "evidence_weights": [-0.17715078592300415, -0.17466869950294495, 0.3127719759941101], "agreement_weight": 0.0738646611571312, "orthogonal_weight": -0.7491010427474976}]}, {"layer": 15, "memory_to_active": 0.056434810161590576, "contribution_to_active": 0.49105048179626465, "mix_mean": 0.41124391555786133, "mix_low": 0.008747260086238384, "mix_high": 0.002704461570829153, "read_cosine": 0.6584845185279846, "beta_mean": 0.03868703544139862, "nonfinite": 0.0, "weighted_memory_to_active": 0.028198834508657455, "mix_std": 0.2851104736328125, "beta_std": 0.044584229588508606, "beta_low": 0.7094625234603882, "beta_high": 0.0, "orthogonal_to_active": 0.0074439640156924725, "output_influence": 0.2995671331882477, "residual_influence": 0.17221403121948242, "state_active": 0.14725351333618164, "state_memory": 0.010790731757879257, "state_peak": 31.79283905029297, "growth_active": 1.2252106666564941, "growth_memory": 1.3627088069915771, "heads": [{"head": 0, "memory_to_active": 0.09664314985275269, "contribution_to_active": 0.49356499314308167, "mix_mean": 0.3788791298866272, "mix_low": 0.0072154998779296875, "mix_high": 0.005194664001464844, "read_cosine": 0.6372265219688416, "beta_mean": 0.03856760635972023, "nonfinite": 0.0, "weighted_memory_to_active": 0.04765838757157326, "mix_std": 0.2828402519226074, "beta_std": 0.029585937038064003, "beta_low": 0.098876953125, "beta_high": 0.0, "orthogonal_to_active": 0.024475522339344025, "evidence_weights": [-0.18810896575450897, -0.18805061280727386, 0.44987645745277405], "agreement_weight": 0.09434312582015991, "orthogonal_weight": -0.7633312940597534}, {"head": 1, "memory_to_active": 0.12444035708904266, "contribution_to_active": 0.42144477367401123, "mix_mean": 0.4376538395881653, "mix_low": 0.001384735107421875, "mix_high": 0.005954742431640625, "read_cosine": 0.9230268001556396, "beta_mean": 0.0006409646011888981, "nonfinite": 0.0, "weighted_memory_to_active": 0.05373886227607727, "mix_std": 0.2868056893348694, "beta_std": 0.0024767594877630472, "beta_low": 0.9947509765625, "beta_high": 0.0, "orthogonal_to_active": 0.0012320374371483922, "evidence_weights": [-0.14840281009674072, -0.157160684466362, 0.17769554257392883], "agreement_weight": 0.12562216818332672, "orthogonal_weight": 0.047322243452072144}, {"head": 2, "memory_to_active": 0.0031231280881911516, "contribution_to_active": 0.48759517073631287, "mix_mean": 0.3313101530075073, "mix_low": 0.01517486572265625, "mix_high": 0.00013637542724609375, "read_cosine": 0.7508465647697449, "beta_mean": 4.490421270020306e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0016200101235881448, "mix_std": 0.2701300382614136, "beta_std": 8.041185355978087e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004993170732632279, "evidence_weights": [-0.2078496515750885, -0.2667705714702606, 0.1283303052186966], "agreement_weight": 0.009989080019295216, "orthogonal_weight": 0.7618649005889893}, {"head": 3, "memory_to_active": 0.00128119345754385, "contribution_to_active": 0.38335147500038147, "mix_mean": 0.38906365633010864, "mix_low": 0.0065593719482421875, "mix_high": 0.0004367828369140625, "read_cosine": 0.6054421067237854, "beta_mean": 6.610048330912832e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00042579954606480896, "mix_std": 0.2887780964374542, "beta_std": 1.3748773199040443e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003187296097166836, "evidence_weights": [-0.2649502158164978, -0.24237178266048431, 0.18879778683185577], "agreement_weight": 0.05115840211510658, "orthogonal_weight": -0.7526747584342957}, {"head": 4, "memory_to_active": 0.08576364815235138, "contribution_to_active": 0.6328399181365967, "mix_mean": 0.36918261647224426, "mix_low": 0.012022972106933594, "mix_high": 0.0011644363403320312, "read_cosine": 0.7339690923690796, "beta_mean": 0.1908394992351532, "nonfinite": 0.0, "weighted_memory_to_active": 0.04832170903682709, "mix_std": 0.30763930082321167, "beta_std": 0.21957822144031525, "beta_low": 0.19183349609375, "beta_high": 0.0, "orthogonal_to_active": 0.007427413947880268, "evidence_weights": [-0.20983362197875977, -0.2567373812198639, -0.005476347170770168], "agreement_weight": 0.012682893313467503, "orthogonal_weight": 0.34935086965560913}, {"head": 5, "memory_to_active": 0.02735736593604088, "contribution_to_active": 0.5275065302848816, "mix_mean": 0.5613740086555481, "mix_low": 0.010126113891601562, "mix_high": 0.0033397674560546875, "read_cosine": 0.30039599537849426, "beta_mean": 0.002022635657340288, "nonfinite": 0.0, "weighted_memory_to_active": 0.017428232356905937, "mix_std": 0.27446943521499634, "beta_std": 0.01577029377222061, "beta_low": 0.9713134765625, "beta_high": 0.0, "orthogonal_to_active": 0.010710760951042175, "evidence_weights": [-0.28716224431991577, -0.30511772632598877, 0.7055234313011169], "agreement_weight": 0.3014785647392273, "orthogonal_weight": 0.8086443543434143}]}]}, {"step": 56000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7727690799983975, "memory_to_active": 0.7992474436759949, "contribution_to_active": 0.5596858859062195, "mix_mean": 0.4132816791534424, "mix_low": 0.13111847639083862, "mix_high": 0.01059479545801878, "read_cosine": 0.35065531730651855, "beta_mean": 0.07186070084571838, "nonfinite": 0.0, "weighted_memory_to_active": 0.20660901069641113, "mix_std": 0.2230815589427948, "beta_std": 0.06324678659439087, "beta_low": 0.5641117095947266, "beta_high": 3.4332275390625e-05, "orthogonal_to_active": 0.06344673037528992, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0902974605560303, "normal_token_accuracy": 0.44938820600509644, "state_active": 0.062112677842378616, "state_memory": 0.018029747530817986, "state_peak": 120.08096313476562, "growth_active": 1.1248219013214111, "growth_memory": 1.3161323070526123, "output_influence": 0.4228356182575226, "residual_influence": 0.1437620222568512, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.266460657119751, "no_memory_loss_delta": 0.4812543988227844, "scrambled_loss_delta": 0.15604710578918457, "memory_only_loss_delta": 2.8426103591918945, "no_evidence_loss_delta": 0.25232377648353577, "no_agreement_loss_delta": 0.005780678708106279, "no_orthogonal_loss_delta": -0.011559569276869297}, {"index": 94, "tokens": 256, "normal_loss": 3.195829391479492, "no_memory_loss_delta": 0.6644846796989441, "scrambled_loss_delta": 0.3767204284667969, "memory_only_loss_delta": 3.3360226154327393, "no_evidence_loss_delta": 0.2904757857322693, "no_agreement_loss_delta": 0.038817860186100006, "no_orthogonal_loss_delta": 0.04434651508927345}, {"index": 188, "tokens": 256, "normal_loss": 3.3468410968780518, "no_memory_loss_delta": 0.7339558601379395, "scrambled_loss_delta": 0.17758959531784058, "memory_only_loss_delta": 3.171076536178589, "no_evidence_loss_delta": 0.3881326913833618, "no_agreement_loss_delta": 0.046639226377010345, "no_orthogonal_loss_delta": 0.04382422938942909}, {"index": 283, "tokens": 131, "normal_loss": 2.0384738445281982, "no_memory_loss_delta": 0.3836718797683716, "scrambled_loss_delta": 0.17317141592502594, "memory_only_loss_delta": 2.7128427028656006, "no_evidence_loss_delta": 0.12171211838722229, "no_agreement_loss_delta": 0.010107191279530525, "no_orthogonal_loss_delta": -0.013192729093134403}], "agreement_weight_abs_mean": 0.21693551540374756, "agreement_weight_abs_max": 5.878840446472168, "agreement_weight_mean": 0.1892613172531128, "orthogonal_weight_abs_mean": 0.871099054813385, "orthogonal_weight_abs_max": 9.415803909301758, "orthogonal_weight_mean": -0.10862550139427185, "no_memory_loss": 3.681468963623047, "no_memory_loss_delta": 0.5911712646484375, "no_memory_token_accuracy_gain": 0.09232480823993683, "no_memory_token_win_fraction": 0.6629588603973389, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 44.63215991362159, "scrambled_loss": 3.3178138732910156, "scrambled_loss_delta": 0.22751601040363312, "scrambled_token_accuracy_gain": 0.03448275849223137, "scrambled_token_win_fraction": 0.6184649467468262, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.34903310458921, "memory_only_loss": 6.148036479949951, "memory_only_loss_delta": 3.057739496231079, "memory_only_token_accuracy_gain": 0.27252504229545593, "memory_only_token_win_fraction": 0.8998887538909912, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.30061950827495, "no_evidence_loss": 3.373126268386841, "no_evidence_loss_delta": 0.2828286290168762, "no_evidence_token_accuracy_gain": 0.051167964935302734, "no_evidence_token_win_fraction": 0.6151279211044312, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 24.635106964811214, "no_agreement_loss": 3.1177515983581543, "no_agreement_loss_delta": 0.027453739196062088, "no_agreement_token_accuracy_gain": 0.006674082484096289, "no_agreement_token_win_fraction": 0.5061178803443909, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.7080310440578135, "no_orthogonal_loss": 3.1101911067962646, "no_orthogonal_loss_delta": 0.019893450662493706, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.57953280210495, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 1.9696881609800903, "short_no_memory_loss_delta": 0.5324627161026001, "long_context_memory_gain": 0.0587085485458374, "normal_context_loss_gain": 0.14758840203285217, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.416208505630493, "contribution_to_active": 1.140869379043579, "mix_mean": 0.10588210821151733, "mix_low": 0.5259763598442078, "mix_high": 0.0051210722886025906, "read_cosine": 0.41781818866729736, "beta_mean": 0.21705031394958496, "nonfinite": 0.0, "weighted_memory_to_active": 1.1764379739761353, "mix_std": 0.0918382927775383, "beta_std": 0.1502791941165924, "beta_low": 0.0903828963637352, "beta_high": 0.0, "orthogonal_to_active": 0.04351426661014557, "output_influence": 0.2133510857820511, "residual_influence": 0.2486945390701294, "state_active": 0.07170350104570389, "state_memory": 0.045023247599601746, "state_peak": 50.99031448364258, "growth_active": 1.3301126956939697, "growth_memory": 1.2769206762313843, "heads": [{"head": 0, "memory_to_active": 2.6196846961975098, "contribution_to_active": 0.4601927399635315, "mix_mean": 0.020712226629257202, "mix_low": 0.6338233947753906, "mix_high": 0.0, "read_cosine": 0.22189676761627197, "beta_mean": 0.19635212421417236, "nonfinite": 0.0, "weighted_memory_to_active": 0.595050573348999, "mix_std": 0.06350497156381607, "beta_std": 0.10576462745666504, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06209861859679222, "evidence_weights": [-0.005817008670419455, -0.049567170441150665, -0.05466363951563835], "agreement_weight": -0.20192000269889832, "orthogonal_weight": -0.12161589413881302}, {"head": 1, "memory_to_active": 6.181489944458008, "contribution_to_active": 2.7044990062713623, "mix_mean": 0.01931794360280037, "mix_low": 0.6079921722412109, "mix_high": 0.0, "read_cosine": 0.43489718437194824, "beta_mean": 0.44620633125305176, "nonfinite": 0.0, "weighted_memory_to_active": 2.9676146507263184, "mix_std": 0.054266538470983505, "beta_std": 0.21496275067329407, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.018854185938835144, "evidence_weights": [0.02344278246164322, -0.10266778618097305, 0.6094908118247986], "agreement_weight": 0.02574620582163334, "orthogonal_weight": 0.013057197444140911}, {"head": 2, "memory_to_active": 2.545006513595581, "contribution_to_active": 0.5423171520233154, "mix_mean": 0.016370637342333794, "mix_low": 0.6366796493530273, "mix_high": 0.0, "read_cosine": 0.28440314531326294, "beta_mean": 0.2608828544616699, "nonfinite": 0.0, "weighted_memory_to_active": 0.6329405307769775, "mix_std": 0.03595750033855438, "beta_std": 0.20541781187057495, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05094146728515625, "evidence_weights": [0.05948047712445259, 0.048803068697452545, 0.18715986609458923], "agreement_weight": 0.011508697643876076, "orthogonal_weight": -0.20547358691692352}, {"head": 3, "memory_to_active": 7.2806315422058105, "contribution_to_active": 2.608468770980835, "mix_mean": 0.03103727102279663, "mix_low": 0.4949483871459961, "mix_high": 0.0, "read_cosine": 0.3240085244178772, "beta_mean": 0.3169151842594147, "nonfinite": 0.0, "weighted_memory_to_active": 2.7599306106567383, "mix_std": 0.07244830578565598, "beta_std": 0.25391197204589844, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10988449305295944, "evidence_weights": [-0.04148166999220848, -0.04037296026945114, -0.16231489181518555], "agreement_weight": -0.010098530910909176, "orthogonal_weight": -0.04785407707095146}, {"head": 4, "memory_to_active": 1.771112322807312, "contribution_to_active": 0.06706713885068893, "mix_mean": 0.010494169779121876, "mix_low": 0.7675495147705078, "mix_high": 0.0, "read_cosine": 0.30492863059043884, "beta_mean": 0.06223750486969948, "nonfinite": 0.0, "weighted_memory_to_active": 0.049904290586709976, "mix_std": 0.03593830019235611, "beta_std": 0.10381276905536652, "beta_low": 0.2393798828125, "beta_high": 0.0, "orthogonal_to_active": 0.003391278674826026, "evidence_weights": [0.13810081779956818, 0.10110713541507721, -0.15590444207191467], "agreement_weight": -0.14242948591709137, "orthogonal_weight": 0.061044707894325256}, {"head": 5, "memory_to_active": 0.09932488203048706, "contribution_to_active": 0.46267175674438477, "mix_mean": 0.5373603701591492, "mix_low": 0.014864921569824219, "mix_high": 0.03072643280029297, "read_cosine": 0.9367750287055969, "beta_mean": 0.01970783621072769, "nonfinite": 0.0, "weighted_memory_to_active": 0.05318684130907059, "mix_std": 0.28891411423683167, "beta_std": 0.017805276438593864, "beta_low": 0.30291748046875, "beta_high": 0.0, "orthogonal_to_active": 0.015915540978312492, "evidence_weights": [-0.17931896448135376, -0.21800647675991058, 0.38438376784324646], "agreement_weight": 0.28993913531303406, "orthogonal_weight": -0.6774365305900574}]}, {"layer": 1, "memory_to_active": 2.2763912677764893, "contribution_to_active": 0.4902990758419037, "mix_mean": 0.21179528534412384, "mix_low": 0.36216259002685547, "mix_high": 0.0010717710247263312, "read_cosine": 0.29301539063453674, "beta_mean": 0.17390009760856628, "nonfinite": 0.0, "weighted_memory_to_active": 0.3242751359939575, "mix_std": 0.11614647507667542, "beta_std": 0.12846508622169495, "beta_low": 0.3469645380973816, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.04241400584578514, "output_influence": 0.24016553163528442, "residual_influence": 0.0636562705039978, "state_active": 0.030597183853387833, "state_memory": 0.035682983696460724, "state_peak": 37.77349090576172, "growth_active": 1.2410682439804077, "growth_memory": 1.4808202981948853, "heads": [{"head": 0, "memory_to_active": 0.034941695630550385, "contribution_to_active": 0.49739694595336914, "mix_mean": 0.5227800607681274, "mix_low": 0.006535530090332031, "mix_high": 0.0019207000732421875, "read_cosine": 0.014180544763803482, "beta_mean": 0.0002518545079510659, "nonfinite": 0.0, "weighted_memory_to_active": 0.020369764417409897, "mix_std": 0.2827437222003937, "beta_std": 0.00037389775388874114, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.019714174792170525, "evidence_weights": [-0.2098768949508667, -0.27118656039237976, 0.3572373390197754], "agreement_weight": 0.18850363790988922, "orthogonal_weight": -1.0382497310638428}, {"head": 1, "memory_to_active": 3.0036940574645996, "contribution_to_active": 0.28518685698509216, "mix_mean": 0.023683682084083557, "mix_low": 0.46498680114746094, "mix_high": 0.0, "read_cosine": 0.4787833094596863, "beta_mean": 0.2667940855026245, "nonfinite": 0.0, "weighted_memory_to_active": 0.315364807844162, "mix_std": 0.045477308332920074, "beta_std": 0.1826745867729187, "beta_low": 0.0028076171875, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.02046124078333378, "evidence_weights": [0.019720766693353653, -0.05368538945913315, 0.06005211919546127], "agreement_weight": -0.003223710460588336, "orthogonal_weight": 0.17977489531040192}, {"head": 2, "memory_to_active": 0.055594928562641144, "contribution_to_active": 0.5437421202659607, "mix_mean": 0.660537600517273, "mix_low": 0.0005340576171875, "mix_high": 0.004509925842285156, "read_cosine": 0.4627557694911957, "beta_mean": 0.0024048422928899527, "nonfinite": 0.0, "weighted_memory_to_active": 0.03773323819041252, "mix_std": 0.23513460159301758, "beta_std": 0.004842166788876057, "beta_low": 0.968017578125, "beta_high": 0.0, "orthogonal_to_active": 0.02522328868508339, "evidence_weights": [-0.2805544137954712, -0.3697378635406494, 0.6463998556137085], "agreement_weight": 0.036151766777038574, "orthogonal_weight": -0.7877861857414246}, {"head": 3, "memory_to_active": 2.908985137939453, "contribution_to_active": 0.15567588806152344, "mix_mean": 0.02334974892437458, "mix_low": 0.5576868057250977, "mix_high": 0.0, "read_cosine": 0.1900387406349182, "beta_mean": 0.3337187170982361, "nonfinite": 0.0, "weighted_memory_to_active": 0.20866172015666962, "mix_std": 0.060763318091630936, "beta_std": 0.2609036862850189, "beta_low": 0.050048828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08024973422288895, "evidence_weights": [0.03584204614162445, -0.020807772874832153, 0.01822490803897381], "agreement_weight": -0.027199620380997658, "orthogonal_weight": -0.47260770201683044}, {"head": 4, "memory_to_active": 3.7943620681762695, "contribution_to_active": 1.3860105276107788, "mix_mean": 0.0337047204375267, "mix_low": 0.3112907409667969, "mix_high": 0.0, "read_cosine": 0.3287177085876465, "beta_mean": 0.17171254754066467, "nonfinite": 0.0, "weighted_memory_to_active": 1.3134121894836426, "mix_std": 0.05753708630800247, "beta_std": 0.12620888650417328, "beta_low": 0.03216552734375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.09255355596542358, "evidence_weights": [-0.03215517848730087, -0.005341398064047098, 0.08746527880430222], "agreement_weight": -0.012819676660001278, "orthogonal_weight": 0.09812930971384048}, {"head": 5, "memory_to_active": 3.860769510269165, "contribution_to_active": 0.0737820416688919, "mix_mean": 0.006715826690196991, "mix_low": 0.8319416046142578, "mix_high": 0.0, "read_cosine": 0.28361624479293823, "beta_mean": 0.26851847767829895, "nonfinite": 0.0, "weighted_memory_to_active": 0.05010895058512688, "mix_std": 0.015222754329442978, "beta_std": 0.19578734040260315, "beta_low": 0.02874755859375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.016282035037875175, "evidence_weights": [0.16122566163539886, 0.08311653882265091, 0.13363812863826752], "agreement_weight": -0.12850573658943176, "orthogonal_weight": 0.3320378065109253}]}, {"layer": 2, "memory_to_active": 0.9943758845329285, "contribution_to_active": 0.35227805376052856, "mix_mean": 0.39822572469711304, "mix_low": 0.24608659744262695, "mix_high": 0.004551728721708059, "read_cosine": 0.30798277258872986, "beta_mean": 0.0828554555773735, "nonfinite": 0.0, "weighted_memory_to_active": 0.07605563849210739, "mix_std": 0.18971151113510132, "beta_std": 0.06723964214324951, "beta_low": 0.5806884765625, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.019567307084798813, "output_influence": 0.38924384117126465, "residual_influence": 0.10429570078849792, "state_active": 0.03687833994626999, "state_memory": 0.015325449407100677, "state_peak": 24.075424194335938, "growth_active": 1.040502905845642, "growth_memory": 1.5964990854263306, "heads": [{"head": 0, "memory_to_active": 0.014367595314979553, "contribution_to_active": 0.3555545210838318, "mix_mean": 0.6213892698287964, "mix_low": 0.000606536865234375, "mix_high": 0.008797645568847656, "read_cosine": 0.12981769442558289, "beta_mean": 8.383807289646938e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009430852718651295, "mix_std": 0.2716241478919983, "beta_std": 0.0001669092453084886, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00958399847149849, "evidence_weights": [-0.36872053146362305, -0.21648548543453217, 0.1349261850118637], "agreement_weight": 0.13201521337032318, "orthogonal_weight": -1.0504788160324097}, {"head": 1, "memory_to_active": 2.709562301635742, "contribution_to_active": 0.045026995241642, "mix_mean": 0.005333909299224615, "mix_low": 0.8904190063476562, "mix_high": 9.5367431640625e-07, "read_cosine": 0.11723005771636963, "beta_mean": 0.21774393320083618, "nonfinite": 0.0, "weighted_memory_to_active": 0.03401849418878555, "mix_std": 0.01305483654141426, "beta_std": 0.14059984683990479, "beta_low": 0.00201416015625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.011586915701627731, "evidence_weights": [0.13745298981666565, 0.1171632781624794, 0.5546201467514038], "agreement_weight": 0.016153961420059204, "orthogonal_weight": 0.3610508143901825}, {"head": 2, "memory_to_active": 0.009069702588021755, "contribution_to_active": 0.4005056619644165, "mix_mean": 0.5347030162811279, "mix_low": 0.00141143798828125, "mix_high": 0.0031442642211914062, "read_cosine": 0.39641493558883667, "beta_mean": 2.6886158593697473e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003726774360984564, "mix_std": 0.27715179324150085, "beta_std": 5.306171806296334e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00022155589249450713, "evidence_weights": [-0.2502205967903137, -0.29037803411483765, 0.5640619993209839], "agreement_weight": -0.010100874118506908, "orthogonal_weight": -0.07379551231861115}, {"head": 3, "memory_to_active": 0.12174268066883087, "contribution_to_active": 0.5417371988296509, "mix_mean": 0.5636872053146362, "mix_low": 0.00029277801513671875, "mix_high": 0.006018638610839844, "read_cosine": 0.6256543397903442, "beta_mean": 0.022997630760073662, "nonfinite": 0.0, "weighted_memory_to_active": 0.07425247877836227, "mix_std": 0.2594777047634125, "beta_std": 0.03039475344121456, "beta_low": 0.451416015625, "beta_high": 0.0, "orthogonal_to_active": 0.049467746168375015, "evidence_weights": [-0.20155009627342224, -0.31810325384140015, 0.7982355952262878], "agreement_weight": 0.057443805038928986, "orthogonal_weight": -0.8987390398979187}, {"head": 4, "memory_to_active": 3.097945213317871, "contribution_to_active": 0.24898311495780945, "mix_mean": 0.020521150901913643, "mix_low": 0.5829181671142578, "mix_high": 0.0, "read_cosine": 0.2036074697971344, "beta_mean": 0.2562326490879059, "nonfinite": 0.0, "weighted_memory_to_active": 0.3274479806423187, "mix_std": 0.051703859120607376, "beta_std": 0.23215946555137634, "beta_low": 0.03070068359375, "beta_high": 0.00048828125, "orthogonal_to_active": 0.04541740566492081, "evidence_weights": [0.021504901349544525, 0.0038989600725471973, 0.04504445567727089], "agreement_weight": 0.016048653051257133, "orthogonal_weight": -0.15484455227851868}, {"head": 5, "memory_to_active": 0.0135671840980649, "contribution_to_active": 0.5218607187271118, "mix_mean": 0.6437197327613831, "mix_low": 0.0008716583251953125, "mix_high": 0.009348869323730469, "read_cosine": 0.3751720190048218, "beta_mean": 4.7764566261321306e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007457243278622627, "mix_std": 0.2652566134929657, "beta_std": 6.384619337040931e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011262199841439724, "evidence_weights": [-0.2810574173927307, -0.2922598123550415, 1.029166579246521], "agreement_weight": 0.1725800633430481, "orthogonal_weight": -0.18855653703212738}]}, {"layer": 3, "memory_to_active": 0.45261311531066895, "contribution_to_active": 0.4670761823654175, "mix_mean": 0.4947037398815155, "mix_low": 0.05336109921336174, "mix_high": 0.010202567093074322, "read_cosine": 0.20867206156253815, "beta_mean": 0.028129613026976585, "nonfinite": 0.0, "weighted_memory_to_active": 0.1310693323612213, "mix_std": 0.262512743473053, "beta_std": 0.025259288027882576, "beta_low": 0.7678121328353882, "beta_high": 0.0, "orthogonal_to_active": 0.052903879433870316, "output_influence": 0.3999180495738983, "residual_influence": 0.09128221869468689, "state_active": 0.02485131099820137, "state_memory": 0.007134842220693827, "state_peak": 21.479461669921875, "growth_active": 1.079027533531189, "growth_memory": 1.6226446628570557, "heads": [{"head": 0, "memory_to_active": 0.3849739134311676, "contribution_to_active": 0.580714225769043, "mix_mean": 0.586745023727417, "mix_low": 0.0026483535766601562, "mix_high": 0.023584365844726562, "read_cosine": 0.26339825987815857, "beta_mean": 0.009188400581479073, "nonfinite": 0.0, "weighted_memory_to_active": 0.2357589453458786, "mix_std": 0.30729448795318604, "beta_std": 0.010671623051166534, "beta_low": 0.70062255859375, "beta_high": 0.0, "orthogonal_to_active": 0.16614878177642822, "evidence_weights": [-0.21523375809192657, -0.24931249022483826, 0.9483254551887512], "agreement_weight": 0.10710039734840393, "orthogonal_weight": -0.7528067827224731}, {"head": 1, "memory_to_active": 0.016056207939982414, "contribution_to_active": 0.4473036527633667, "mix_mean": 0.5633186101913452, "mix_low": 0.0041484832763671875, "mix_high": 0.008409500122070312, "read_cosine": 0.16224530339241028, "beta_mean": 6.90323649905622e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00915138702839613, "mix_std": 0.2978043556213379, "beta_std": 0.00017512895283289254, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007477549836039543, "evidence_weights": [-0.2666967809200287, -0.2162102311849594, 0.6562474966049194], "agreement_weight": 0.5055537223815918, "orthogonal_weight": -0.8911451697349548}, {"head": 2, "memory_to_active": 2.0423526763916016, "contribution_to_active": 0.30161434412002563, "mix_mean": 0.0559963658452034, "mix_low": 0.29882144927978516, "mix_high": 0.0, "read_cosine": 0.21928463876247406, "beta_mean": 0.15464815497398376, "nonfinite": 0.0, "weighted_memory_to_active": 0.3441978991031647, "mix_std": 0.1090887039899826, "beta_std": 0.13179664313793182, "beta_low": 0.01708984375, "beta_high": 0.0, "orthogonal_to_active": 0.005593743175268173, "evidence_weights": [-0.09695135802030563, -0.0031381200533360243, -0.009492344222962856], "agreement_weight": -0.01603325642645359, "orthogonal_weight": -0.019996337592601776}, {"head": 3, "memory_to_active": 0.024225834757089615, "contribution_to_active": 0.5108180046081543, "mix_mean": 0.5949780941009521, "mix_low": 0.0054798126220703125, "mix_high": 0.010828971862792969, "read_cosine": 0.3036806583404541, "beta_mean": 0.00010095405741594732, "nonfinite": 0.0, "weighted_memory_to_active": 0.01663491129875183, "mix_std": 0.2917231619358063, "beta_std": 0.00023136133677326143, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00456753745675087, "evidence_weights": [-0.2525201737880707, -0.2314341962337494, 2.1820003986358643], "agreement_weight": 2.053889513015747, "orthogonal_weight": -0.36298418045043945}, {"head": 4, "memory_to_active": 0.012411734089255333, "contribution_to_active": 0.3423730134963989, "mix_mean": 0.4702284336090088, "mix_low": 0.005892753601074219, "mix_high": 0.0026416778564453125, "read_cosine": 0.1650703400373459, "beta_mean": 6.745742575731128e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00789601355791092, "mix_std": 0.2931672930717468, "beta_std": 9.10641101654619e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007424383424222469, "evidence_weights": [-0.2113879770040512, -0.20770862698554993, 0.12672413885593414], "agreement_weight": -0.051194850355386734, "orthogonal_weight": -1.027700424194336}, {"head": 5, "memory_to_active": 0.2356584668159485, "contribution_to_active": 0.6196338534355164, "mix_mean": 0.6969556212425232, "mix_low": 0.0031757354736328125, "mix_high": 0.015750885009765625, "read_cosine": 0.13835322856903076, "beta_mean": 0.004703661426901817, "nonfinite": 0.0, "weighted_memory_to_active": 0.17277675867080688, "mix_std": 0.27599844336509705, "beta_std": 0.008589889854192734, "beta_low": 0.88916015625, "beta_high": 0.0, "orthogonal_to_active": 0.1262112706899643, "evidence_weights": [-0.2561931908130646, -0.32859888672828674, 0.48371174931526184], "agreement_weight": 0.17673878371715546, "orthogonal_weight": -0.7722965478897095}]}, {"layer": 4, "memory_to_active": 1.0409965515136719, "contribution_to_active": 0.46234607696533203, "mix_mean": 0.36716797947883606, "mix_low": 0.25963354110717773, "mix_high": 0.011173407547175884, "read_cosine": 0.32054704427719116, "beta_mean": 0.11627194285392761, "nonfinite": 0.0, "weighted_memory_to_active": 0.13408249616622925, "mix_std": 0.16040942072868347, "beta_std": 0.10605165362358093, "beta_low": 0.384124755859375, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.0460747629404068, "output_influence": 0.4707881808280945, "residual_influence": 0.13805869221687317, "state_active": 0.03985045477747917, "state_memory": 0.017728116363286972, "state_peak": 35.707733154296875, "growth_active": 1.2058086395263672, "growth_memory": 1.530204176902771, "heads": [{"head": 0, "memory_to_active": 0.18211729824543, "contribution_to_active": 0.7697737216949463, "mix_mean": 0.672497570514679, "mix_low": 0.015172958374023438, "mix_high": 0.02641010284423828, "read_cosine": 0.4866943955421448, "beta_mean": 0.2712348699569702, "nonfinite": 0.0, "weighted_memory_to_active": 0.15416894853115082, "mix_std": 0.3518553674221039, "beta_std": 0.26659318804740906, "beta_low": 0.2359619140625, "beta_high": 0.0, "orthogonal_to_active": 0.07318904250860214, "evidence_weights": [-0.26536592841148376, -0.3684857189655304, 0.004840496927499771], "agreement_weight": 0.0068405927158892155, "orthogonal_weight": 0.6121168732643127}, {"head": 1, "memory_to_active": 0.05416781082749367, "contribution_to_active": 0.6560572981834412, "mix_mean": 0.6971522569656372, "mix_low": 9.918212890625e-05, "mix_high": 0.019046783447265625, "read_cosine": 0.6224382519721985, "beta_mean": 0.0008584348252043128, "nonfinite": 0.0, "weighted_memory_to_active": 0.04005797952413559, "mix_std": 0.2401653677225113, "beta_std": 0.00583264883607626, "beta_low": 0.98309326171875, "beta_high": 0.0, "orthogonal_to_active": 0.021726509556174278, "evidence_weights": [-0.24275068938732147, -0.36304545402526855, 0.6377437114715576], "agreement_weight": 0.2934190034866333, "orthogonal_weight": -0.7651728987693787}, {"head": 2, "memory_to_active": 1.8409086465835571, "contribution_to_active": 0.31341588497161865, "mix_mean": 0.03470392897725105, "mix_low": 0.4488372802734375, "mix_high": 2.288818359375e-05, "read_cosine": 0.21867859363555908, "beta_mean": 0.07500049471855164, "nonfinite": 0.0, "weighted_memory_to_active": 0.25151780247688293, "mix_std": 0.0685553178191185, "beta_std": 0.07707232236862183, "beta_low": 0.0640869140625, "beta_high": 0.0, "orthogonal_to_active": 0.06735074520111084, "evidence_weights": [0.04784941300749779, -0.015758473426103592, 0.21071520447731018], "agreement_weight": 0.13612616062164307, "orthogonal_weight": -0.3568105399608612}, {"head": 3, "memory_to_active": 1.8401663303375244, "contribution_to_active": 0.14089787006378174, "mix_mean": 0.03263790160417557, "mix_low": 0.4313850402832031, "mix_high": 0.0, "read_cosine": 0.1955839842557907, "beta_mean": 0.1424497663974762, "nonfinite": 0.0, "weighted_memory_to_active": 0.20805014669895172, "mix_std": 0.06066285818815231, "beta_std": 0.12463901937007904, "beta_low": 0.0216064453125, "beta_high": 0.0, "orthogonal_to_active": 0.00031790288630872965, "evidence_weights": [-0.021361205726861954, 0.049647554755210876, -0.2684343159198761], "agreement_weight": 0.08341490477323532, "orthogonal_weight": -0.002476800698786974}, {"head": 4, "memory_to_active": 0.00650056404992938, "contribution_to_active": 0.6522424221038818, "mix_mean": 0.7524990439414978, "mix_low": 6.771087646484375e-05, "mix_high": 0.021554946899414062, "read_cosine": 0.17367148399353027, "beta_mean": 1.717376653687097e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004609274677932262, "mix_std": 0.21161192655563354, "beta_std": 2.975275128846988e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025704491417855024, "evidence_weights": [-0.3343985974788666, -0.3256033658981323, 0.46002233028411865], "agreement_weight": 0.34090369939804077, "orthogonal_weight": -0.5954663753509521}, {"head": 5, "memory_to_active": 2.322117567062378, "contribution_to_active": 0.24168917536735535, "mix_mean": 0.013517062179744244, "mix_low": 0.6622390747070312, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2262154519557953, "beta_mean": 0.20807090401649475, "nonfinite": 0.0, "weighted_memory_to_active": 0.1460908204317093, "mix_std": 0.029605623334646225, "beta_std": 0.16214299201965332, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.11129389703273773, "evidence_weights": [0.05500740557909012, -0.022960979491472244, -0.2655627727508545], "agreement_weight": -0.08097660541534424, "orthogonal_weight": 0.9887905120849609}]}, {"layer": 5, "memory_to_active": 0.4069632291793823, "contribution_to_active": 0.5281565189361572, "mix_mean": 0.5062456130981445, "mix_low": 0.05995575711131096, "mix_high": 0.020250480622053146, "read_cosine": 0.06905973702669144, "beta_mean": 0.056045420467853546, "nonfinite": 0.0, "weighted_memory_to_active": 0.14528104662895203, "mix_std": 0.25655996799468994, "beta_std": 0.07201707363128662, "beta_low": 0.6577555537223816, "beta_high": 0.0, "orthogonal_to_active": 0.05869559943675995, "output_influence": 0.5419283509254456, "residual_influence": 0.1562354564666748, "state_active": 0.04275638237595558, "state_memory": 0.010643837973475456, "state_peak": 60.846370697021484, "growth_active": 1.1144237518310547, "growth_memory": 1.2267425060272217, "heads": [{"head": 0, "memory_to_active": 0.2868596911430359, "contribution_to_active": 0.5419443249702454, "mix_mean": 0.4355536699295044, "mix_low": 0.014863014221191406, "mix_high": 0.0016756057739257812, "read_cosine": -0.09571288526058197, "beta_mean": 0.023704415187239647, "nonfinite": 0.0, "weighted_memory_to_active": 0.100990429520607, "mix_std": 0.28242871165275574, "beta_std": 0.03960520774126053, "beta_low": 0.47381591796875, "beta_high": 0.0, "orthogonal_to_active": 0.04160916432738304, "evidence_weights": [-0.15280623733997345, -0.17275579273700714, 0.058760058134794235], "agreement_weight": -0.02661421149969101, "orthogonal_weight": -0.4175712466239929}, {"head": 1, "memory_to_active": 0.008799819275736809, "contribution_to_active": 0.5452935099601746, "mix_mean": 0.7010437250137329, "mix_low": 0.0013761520385742188, "mix_high": 0.04210853576660156, "read_cosine": 0.20382453501224518, "beta_mean": 2.1337704311008565e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005268923006951809, "mix_std": 0.27287057042121887, "beta_std": 4.2463751015020534e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004906981252133846, "evidence_weights": [-0.2514886260032654, -0.28951495885849, 0.9079663157463074], "agreement_weight": 0.6757647395133972, "orthogonal_weight": -0.9812185764312744}, {"head": 2, "memory_to_active": 0.0159512460231781, "contribution_to_active": 0.575387716293335, "mix_mean": 0.6576715111732483, "mix_low": 0.0026187896728515625, "mix_high": 0.022515296936035156, "read_cosine": -0.5544519424438477, "beta_mean": 0.0001027843973133713, "nonfinite": 0.0, "weighted_memory_to_active": 0.009550658985972404, "mix_std": 0.2677066922187805, "beta_std": 0.00018665973038878292, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005248416680842638, "evidence_weights": [-0.2642970085144043, -0.3354710340499878, 0.22125205397605896], "agreement_weight": -0.06315045058727264, "orthogonal_weight": -0.6774901151657104}, {"head": 3, "memory_to_active": 1.582373857498169, "contribution_to_active": 0.2360699474811554, "mix_mean": 0.1137191653251648, "mix_low": 0.13225173950195312, "mix_high": 1.9073486328125e-06, "read_cosine": 0.15018972754478455, "beta_mean": 0.11288981139659882, "nonfinite": 0.0, "weighted_memory_to_active": 0.4005666971206665, "mix_std": 0.150151789188385, "beta_std": 0.16326794028282166, "beta_low": 0.1805419921875, "beta_high": 0.0, "orthogonal_to_active": 0.1541348546743393, "evidence_weights": [-0.0748102217912674, -0.10213904827833176, 0.04739425703883171], "agreement_weight": 0.0523810088634491, "orthogonal_weight": -0.4937981367111206}, {"head": 4, "memory_to_active": 0.451984703540802, "contribution_to_active": 0.5861021280288696, "mix_mean": 0.3225184679031372, "mix_low": 0.20699596405029297, "mix_high": 0.006381034851074219, "read_cosine": 0.5290791392326355, "beta_mean": 0.1992916464805603, "nonfinite": 0.0, "weighted_memory_to_active": 0.2883388102054596, "mix_std": 0.3419826924800873, "beta_std": 0.2277933657169342, "beta_low": 0.29461669921875, "beta_high": 0.0, "orthogonal_to_active": 0.10610955953598022, "evidence_weights": [-0.11618738621473312, -0.26100239157676697, -0.05027326941490173], "agreement_weight": 0.03653731569647789, "orthogonal_weight": 0.5326362252235413}, {"head": 5, "memory_to_active": 0.09580986201763153, "contribution_to_active": 0.6841411590576172, "mix_mean": 0.8069669008255005, "mix_low": 0.001628875732421875, "mix_high": 0.04882049560546875, "read_cosine": 0.18142977356910706, "beta_mean": 0.0002624857297632843, "nonfinite": 0.0, "weighted_memory_to_active": 0.06697075814008713, "mix_std": 0.22421923279762268, "beta_std": 0.0012067978968843818, "beta_low": 0.99755859375, "beta_high": 0.0, "orthogonal_to_active": 0.04016459733247757, "evidence_weights": [-0.3129924535751343, -0.3351338803768158, 1.920165777206421], "agreement_weight": 0.4685925841331482, "orthogonal_weight": -0.6407520771026611}]}, {"layer": 6, "memory_to_active": 0.2922978103160858, "contribution_to_active": 0.6184192895889282, "mix_mean": 0.5848311185836792, "mix_low": 0.05136474221944809, "mix_high": 0.03372812271118164, "read_cosine": 0.31710803508758545, "beta_mean": 0.06465455889701843, "nonfinite": 0.0, "weighted_memory_to_active": 0.1472296267747879, "mix_std": 0.23708558082580566, "beta_std": 0.07328134775161743, "beta_low": 0.5515035390853882, "beta_high": 0.0, "orthogonal_to_active": 0.04600841924548149, "output_influence": 0.5448416471481323, "residual_influence": 0.1818755865097046, "state_active": 0.04558003321290016, "state_memory": 0.007807031273841858, "state_peak": 94.33271026611328, "growth_active": 1.1618605852127075, "growth_memory": 1.1704514026641846, "heads": [{"head": 0, "memory_to_active": 0.12288042902946472, "contribution_to_active": 0.733575701713562, "mix_mean": 0.814966082572937, "mix_low": 0.00019550323486328125, "mix_high": 0.051975250244140625, "read_cosine": 0.23920938372612, "beta_mean": 0.015917090699076653, "nonfinite": 0.0, "weighted_memory_to_active": 0.10243583470582962, "mix_std": 0.20257686078548431, "beta_std": 0.033929530531167984, "beta_low": 0.65814208984375, "beta_high": 0.0, "orthogonal_to_active": 0.07610820233821869, "evidence_weights": [-0.31398847699165344, -0.3486938774585724, 1.204390287399292], "agreement_weight": 0.5566285848617554, "orthogonal_weight": -0.8012194633483887}, {"head": 1, "memory_to_active": 0.006050060503184795, "contribution_to_active": 0.5192270874977112, "mix_mean": 0.5780597925186157, "mix_low": 0.001117706298828125, "mix_high": 0.013985633850097656, "read_cosine": 0.5225961804389954, "beta_mean": 1.323449941992294e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0027574729174375534, "mix_std": 0.28425705432891846, "beta_std": 2.2674052161164582e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015201749047264457, "evidence_weights": [-0.19360440969467163, -0.2958691418170929, 1.1206105947494507], "agreement_weight": 0.07633281499147415, "orthogonal_weight": -0.7272814512252808}, {"head": 2, "memory_to_active": 0.8057209253311157, "contribution_to_active": 0.08568162471055984, "mix_mean": 0.04023190587759018, "mix_low": 0.29703235626220703, "mix_high": 0.0, "read_cosine": 0.25194722414016724, "beta_mean": 0.025155313313007355, "nonfinite": 0.0, "weighted_memory_to_active": 0.0587344728410244, "mix_std": 0.06275768578052521, "beta_std": 0.023070357739925385, "beta_low": 0.1748046875, "beta_high": 0.0, "orthogonal_to_active": 0.006383517757058144, "evidence_weights": [0.07518018782138824, -0.04253355413675308, 0.3092258870601654], "agreement_weight": 0.144000843167305, "orthogonal_weight": -0.12792463600635529}, {"head": 3, "memory_to_active": 0.14064204692840576, "contribution_to_active": 0.6766209602355957, "mix_mean": 0.6489052772521973, "mix_low": 0.008922576904296875, "mix_high": 0.02102947235107422, "read_cosine": 0.4755876064300537, "beta_mean": 0.2183290719985962, "nonfinite": 0.0, "weighted_memory_to_active": 0.10132133960723877, "mix_std": 0.32821694016456604, "beta_std": 0.24504344165325165, "beta_low": 0.19000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.0546858087182045, "evidence_weights": [-0.17006517946720123, -0.32386115193367004, 0.18309223651885986], "agreement_weight": 0.0024593172129243612, "orthogonal_weight": 0.6260964274406433}, {"head": 4, "memory_to_active": 0.6487723588943481, "contribution_to_active": 1.0651426315307617, "mix_mean": 0.7479187250137329, "mix_low": 0.0007648468017578125, "mix_high": 0.1085519790649414, "read_cosine": 0.13506671786308289, "beta_mean": 0.1283663511276245, "nonfinite": 0.0, "weighted_memory_to_active": 0.5970669388771057, "mix_std": 0.2991854250431061, "beta_std": 0.1373397409915924, "beta_low": 0.2861328125, "beta_high": 0.0, "orthogonal_to_active": 0.11693570017814636, "evidence_weights": [-0.21200783550739288, -0.563120424747467, -0.3863583505153656], "agreement_weight": -0.016908355057239532, "orthogonal_weight": 0.21822939813137054}, {"head": 5, "memory_to_active": 0.02972090244293213, "contribution_to_active": 0.6302679181098938, "mix_mean": 0.6789048910140991, "mix_low": 0.00015544891357421875, "mix_high": 0.0068264007568359375, "read_cosine": 0.27824100852012634, "beta_mean": 0.00014629456563852727, "nonfinite": 0.0, "weighted_memory_to_active": 0.021061701700091362, "mix_std": 0.2455194890499115, "beta_std": 0.00028236611979082227, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.020417118445038795, "evidence_weights": [-0.23921678960323334, -0.31464114785194397, 0.3888273537158966], "agreement_weight": 0.19190441071987152, "orthogonal_weight": -1.0372695922851562}]}, {"layer": 7, "memory_to_active": 1.6888412237167358, "contribution_to_active": 0.25418412685394287, "mix_mean": 0.15708062052726746, "mix_low": 0.2531069219112396, "mix_high": 0.0009826024761423469, "read_cosine": 0.23758840560913086, "beta_mean": 0.13509385287761688, "nonfinite": 0.0, "weighted_memory_to_active": 0.19640454649925232, "mix_std": 0.1443731188774109, "beta_std": 0.10100872069597244, "beta_low": 0.2032369077205658, "beta_high": 0.0003153483266942203, "orthogonal_to_active": 0.07768088579177856, "output_influence": 0.2711600661277771, "residual_influence": 0.06303101032972336, "state_active": 0.013395961374044418, "state_memory": 0.025073261931538582, "state_peak": 35.933406829833984, "growth_active": 1.1336829662322998, "growth_memory": 1.2904367446899414, "heads": [{"head": 0, "memory_to_active": 2.4464564323425293, "contribution_to_active": 0.1839037835597992, "mix_mean": 0.03636737912893295, "mix_low": 0.42276954650878906, "mix_high": 0.0, "read_cosine": 0.3465493619441986, "beta_mean": 0.15823081135749817, "nonfinite": 0.0, "weighted_memory_to_active": 0.21625980734825134, "mix_std": 0.07604116201400757, "beta_std": 0.1245415136218071, "beta_low": 0.0084228515625, "beta_high": 0.000244140625, "orthogonal_to_active": 0.00012188882101327181, "evidence_weights": [0.039068520069122314, -0.07315456122159958, 0.030892642214894295], "agreement_weight": -0.031001456081867218, "orthogonal_weight": -0.000710782827809453}, {"head": 1, "memory_to_active": 1.0175186395645142, "contribution_to_active": 0.4049009084701538, "mix_mean": 0.21617145836353302, "mix_low": 0.11965656280517578, "mix_high": 0.005373954772949219, "read_cosine": 0.36913827061653137, "beta_mean": 0.10211805999279022, "nonfinite": 0.0, "weighted_memory_to_active": 0.34775030612945557, "mix_std": 0.24072298407554626, "beta_std": 0.06923826038837433, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04401494562625885, "evidence_weights": [-0.21532556414604187, -0.10143161565065384, -0.2667521834373474], "agreement_weight": 0.05271422490477562, "orthogonal_weight": 0.15110360085964203}, {"head": 2, "memory_to_active": 0.6698483824729919, "contribution_to_active": 0.3417442739009857, "mix_mean": 0.19333039224147797, "mix_low": 0.03299140930175781, "mix_high": 0.00029754638671875, "read_cosine": 0.09910383820533752, "beta_mean": 0.048350267112255096, "nonfinite": 0.0, "weighted_memory_to_active": 0.21332798898220062, "mix_std": 0.200059175491333, "beta_std": 0.044247280806303024, "beta_low": 0.091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.11133391410112381, "evidence_weights": [-0.0788533166050911, -0.16209805011749268, 0.7736377120018005], "agreement_weight": 0.02086557261645794, "orthogonal_weight": -0.5370501279830933}, {"head": 3, "memory_to_active": 4.405111312866211, "contribution_to_active": 0.09441454708576202, "mix_mean": 0.031865473836660385, "mix_low": 0.3538055419921875, "mix_high": 0.0, "read_cosine": 0.09416242688894272, "beta_mean": 0.4281594753265381, "nonfinite": 0.0, "weighted_memory_to_active": 0.3200502395629883, "mix_std": 0.04891010373830795, "beta_std": 0.2884487807750702, "beta_low": 0.0050048828125, "beta_high": 0.00164794921875, "orthogonal_to_active": 0.2762160003185272, "evidence_weights": [0.07625534385442734, -0.024234475567936897, 0.7089566588401794], "agreement_weight": 0.05191347375512123, "orthogonal_weight": -0.9457265734672546}, {"head": 4, "memory_to_active": 1.544306993484497, "contribution_to_active": 0.08448224514722824, "mix_mean": 0.020369764417409897, "mix_low": 0.5867395401000977, "mix_high": 0.0, "read_cosine": 0.25021201372146606, "beta_mean": 0.07345199584960938, "nonfinite": 0.0, "weighted_memory_to_active": 0.05806745961308479, "mix_std": 0.045199401676654816, "beta_std": 0.07896461337804794, "beta_low": 0.114990234375, "beta_high": 0.0, "orthogonal_to_active": 0.013457782566547394, "evidence_weights": [0.14340218901634216, 0.0002693870628718287, 0.17710840702056885], "agreement_weight": 0.039828743785619736, "orthogonal_weight": -0.2744847238063812}, {"head": 5, "memory_to_active": 0.04980557784438133, "contribution_to_active": 0.41565895080566406, "mix_mean": 0.44437915086746216, "mix_low": 0.0026788711547851562, "mix_high": 0.00022411346435546875, "read_cosine": 0.26636454463005066, "beta_mean": 0.00025250163162127137, "nonfinite": 0.0, "weighted_memory_to_active": 0.022971536964178085, "mix_std": 0.2553059458732605, "beta_std": 0.0006118430756032467, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.020940763875842094, "evidence_weights": [-0.16925844550132751, -0.22550426423549652, 0.5536323189735413], "agreement_weight": 0.18927522003650665, "orthogonal_weight": -0.9853453040122986}]}, {"layer": 8, "memory_to_active": 0.1923903524875641, "contribution_to_active": 0.5508339405059814, "mix_mean": 0.4702379107475281, "mix_low": 0.016341209411621094, "mix_high": 0.001508871791884303, "read_cosine": 0.4251195788383484, "beta_mean": 0.01567792519927025, "nonfinite": 0.0, "weighted_memory_to_active": 0.08655959367752075, "mix_std": 0.23576988279819489, "beta_std": 0.02239932119846344, "beta_low": 0.7530111074447632, "beta_high": 0.0, "orthogonal_to_active": 0.07385226339101791, "output_influence": 0.500303328037262, "residual_influence": 0.17293314635753632, "state_active": 0.04623875766992569, "state_memory": 0.006100798025727272, "state_peak": 53.76333236694336, "growth_active": 1.1009680032730103, "growth_memory": 1.1295017004013062, "heads": [{"head": 0, "memory_to_active": 0.06802055984735489, "contribution_to_active": 0.5692347288131714, "mix_mean": 0.6106162667274475, "mix_low": 0.0004596710205078125, "mix_high": 0.001735687255859375, "read_cosine": 0.45708921551704407, "beta_mean": 0.004130475223064423, "nonfinite": 0.0, "weighted_memory_to_active": 0.048779405653476715, "mix_std": 0.24731822311878204, "beta_std": 0.01530939619988203, "beta_low": 0.91412353515625, "beta_high": 0.0, "orthogonal_to_active": 0.030627375468611717, "evidence_weights": [-0.1678154319524765, -0.387919157743454, 0.6298438310623169], "agreement_weight": 0.23086874186992645, "orthogonal_weight": -0.7952427864074707}, {"head": 1, "memory_to_active": 0.09101720154285431, "contribution_to_active": 0.5432062745094299, "mix_mean": 0.5100547075271606, "mix_low": 5.626678466796875e-05, "mix_high": 0.0011882781982421875, "read_cosine": 0.10830982029438019, "beta_mean": 0.001513421768322587, "nonfinite": 0.0, "weighted_memory_to_active": 0.04277261719107628, "mix_std": 0.2372240126132965, "beta_std": 0.008687054738402367, "beta_low": 0.98358154296875, "beta_high": 0.0, "orthogonal_to_active": 0.030780760571360588, "evidence_weights": [-0.1933857649564743, -0.274804025888443, 0.6824344992637634], "agreement_weight": 0.0734105333685875, "orthogonal_weight": -0.7508429288864136}, {"head": 2, "memory_to_active": 0.013033661991357803, "contribution_to_active": 0.6765101552009583, "mix_mean": 0.674767792224884, "mix_low": 0.000583648681640625, "mix_high": 0.0028123855590820312, "read_cosine": 0.4365867078304291, "beta_mean": 0.00013057295291218907, "nonfinite": 0.0, "weighted_memory_to_active": 0.010199830867350101, "mix_std": 0.2223070114850998, "beta_std": 0.0035010776482522488, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006159791722893715, "evidence_weights": [-0.22575025260448456, -0.3459465801715851, 1.0033328533172607], "agreement_weight": 0.2737570106983185, "orthogonal_weight": -0.7116360664367676}, {"head": 3, "memory_to_active": 0.10297355055809021, "contribution_to_active": 0.6351618766784668, "mix_mean": 0.5759027004241943, "mix_low": 0.0008287429809570312, "mix_high": 0.00316619873046875, "read_cosine": 0.7987554669380188, "beta_mean": 0.0081492830067873, "nonfinite": 0.0, "weighted_memory_to_active": 0.07727041840553284, "mix_std": 0.25701114535331726, "beta_std": 0.007693060673773289, "beta_low": 0.71356201171875, "beta_high": 0.0, "orthogonal_to_active": 0.02530401572585106, "evidence_weights": [-0.18475960195064545, -0.29974308609962463, 0.5592784285545349], "agreement_weight": 0.22977416217327118, "orthogonal_weight": -0.6313719153404236}, {"head": 4, "memory_to_active": 0.4923437833786011, "contribution_to_active": 0.23442254960536957, "mix_mean": 0.18804171681404114, "mix_low": 0.02332305908203125, "mix_high": 0.0, "read_cosine": 0.2643505334854126, "beta_mean": 0.0058354465290904045, "nonfinite": 0.0, "weighted_memory_to_active": 0.1424138993024826, "mix_std": 0.179677814245224, "beta_std": 0.01693294756114483, "beta_low": 0.848388671875, "beta_high": 0.0, "orthogonal_to_active": 0.03256695717573166, "evidence_weights": [-0.06455089151859283, -0.1511877179145813, 0.4028129577636719], "agreement_weight": 0.07099787145853043, "orthogonal_weight": -0.24495284259319305}, {"head": 5, "memory_to_active": 0.38695329427719116, "contribution_to_active": 0.6464680433273315, "mix_mean": 0.26204413175582886, "mix_low": 0.07279586791992188, "mix_high": 0.0001506805419921875, "read_cosine": 0.4856254458427429, "beta_mean": 0.0743083581328392, "nonfinite": 0.0, "weighted_memory_to_active": 0.19792138040065765, "mix_std": 0.2710810601711273, "beta_std": 0.08227239549160004, "beta_low": 0.0584716796875, "beta_high": 0.0, "orthogonal_to_active": 0.3176746368408203, "evidence_weights": [-0.1311609297990799, -0.16253620386123657, -0.1187109500169754], "agreement_weight": -0.022647691890597343, "orthogonal_weight": 1.8813328742980957}]}, {"layer": 9, "memory_to_active": 1.0287985801696777, "contribution_to_active": 0.7113606333732605, "mix_mean": 0.3605968952178955, "mix_low": 0.09227101504802704, "mix_high": 0.005840301513671875, "read_cosine": 0.35311201214790344, "beta_mean": 0.0469207763671875, "nonfinite": 0.0, "weighted_memory_to_active": 0.41579556465148926, "mix_std": 0.22750495374202728, "beta_std": 0.03732556849718094, "beta_low": 0.5214335322380066, "beta_high": 0.0, "orthogonal_to_active": 0.040704913437366486, "output_influence": 0.5374624133110046, "residual_influence": 0.15290889143943787, "state_active": 0.04069569706916809, "state_memory": 0.016880661249160767, "state_peak": 55.4892692565918, "growth_active": 1.1571177244186401, "growth_memory": 1.329457402229309, "heads": [{"head": 0, "memory_to_active": 0.0007633811328560114, "contribution_to_active": 0.5730299353599548, "mix_mean": 0.6477452516555786, "mix_low": 0.0010347366333007812, "mix_high": 0.0248870849609375, "read_cosine": 0.3094356060028076, "beta_mean": 2.758868049568264e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00047327543143182993, "mix_std": 0.28525686264038086, "beta_std": 1.096415326173883e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005228985683061182, "evidence_weights": [-0.30624106526374817, -0.312767893075943, 0.5671079158782959], "agreement_weight": 0.10655734688043594, "orthogonal_weight": -1.1575318574905396}, {"head": 1, "memory_to_active": 0.03177952393889427, "contribution_to_active": 0.6881581544876099, "mix_mean": 0.5664165019989014, "mix_low": 0.007241249084472656, "mix_high": 0.0026979446411132812, "read_cosine": 0.4027687907218933, "beta_mean": 8.394937322009355e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.023757068440318108, "mix_std": 0.28226926922798157, "beta_std": 0.00018609788094181567, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.016407109797000885, "evidence_weights": [-0.23988276720046997, -0.2935960292816162, 1.0851235389709473], "agreement_weight": 0.4723525941371918, "orthogonal_weight": -0.8550654053688049}, {"head": 2, "memory_to_active": 0.18498118221759796, "contribution_to_active": 0.6443559527397156, "mix_mean": 0.5928173065185547, "mix_low": 0.0017919540405273438, "mix_high": 0.00600433349609375, "read_cosine": 0.3532724380493164, "beta_mean": 0.0007164081325754523, "nonfinite": 0.0, "weighted_memory_to_active": 0.12605221569538116, "mix_std": 0.26893994212150574, "beta_std": 0.003894887398928404, "beta_low": 0.99566650390625, "beta_high": 0.0, "orthogonal_to_active": 0.07408905029296875, "evidence_weights": [-0.2018311619758606, -0.2763686776161194, 0.7497296333312988], "agreement_weight": 0.3255225121974945, "orthogonal_weight": -0.6883236765861511}, {"head": 3, "memory_to_active": 1.0968317985534668, "contribution_to_active": 0.20260819792747498, "mix_mean": 0.09599252045154572, "mix_low": 0.19910144805908203, "mix_high": 0.00018215179443359375, "read_cosine": 0.3818221390247345, "beta_mean": 0.051362935453653336, "nonfinite": 0.0, "weighted_memory_to_active": 0.1725241243839264, "mix_std": 0.15181757509708405, "beta_std": 0.042394720017910004, "beta_low": 0.02008056640625, "beta_high": 0.0, "orthogonal_to_active": 0.04085799306631088, "evidence_weights": [0.02503618784248829, -0.04781029373407364, 0.1578817218542099], "agreement_weight": 0.09408368915319443, "orthogonal_weight": -0.25292444229125977}, {"head": 4, "memory_to_active": 3.276921272277832, "contribution_to_active": 1.6343683004379272, "mix_mean": 0.14091669023036957, "mix_low": 0.13229942321777344, "mix_high": 0.0010786056518554688, "read_cosine": 0.16925914585590363, "beta_mean": 0.16336853802204132, "nonfinite": 0.0, "weighted_memory_to_active": 1.607685923576355, "mix_std": 0.20424839854240417, "beta_std": 0.0935082659125328, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.023798497393727303, "evidence_weights": [-0.08001447468996048, -0.1553392857313156, -0.007348316255956888], "agreement_weight": -0.06160389631986618, "orthogonal_weight": -0.015392432920634747}, {"head": 5, "memory_to_active": 1.5815138816833496, "contribution_to_active": 0.5256434082984924, "mix_mean": 0.1196930855512619, "mix_low": 0.2121572494506836, "mix_high": 0.00019168853759765625, "read_cosine": 0.502113938331604, "beta_mean": 0.06599005311727524, "nonfinite": 0.0, "weighted_memory_to_active": 0.5642805099487305, "mix_std": 0.17249761521816254, "beta_std": 0.08395847678184509, "beta_low": 0.11285400390625, "beta_high": 0.0, "orthogonal_to_active": 0.08855393528938293, "evidence_weights": [0.09712828695774078, -0.12549203634262085, 0.1424480527639389], "agreement_weight": 0.01522343885153532, "orthogonal_weight": 0.2210172861814499}]}, {"layer": 10, "memory_to_active": 0.244042307138443, "contribution_to_active": 0.7525947093963623, "mix_mean": 0.5132765769958496, "mix_low": 0.022323133423924446, "mix_high": 0.003992875572293997, "read_cosine": 0.37390968203544617, "beta_mean": 0.048195548355579376, "nonfinite": 0.0, "weighted_memory_to_active": 0.11341358721256256, "mix_std": 0.2670353055000305, "beta_std": 0.047638505697250366, "beta_low": 0.6559346914291382, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.28095799684524536, "output_influence": 0.44280388951301575, "residual_influence": 0.1544070839881897, "state_active": 0.05316290631890297, "state_memory": 0.015738045796751976, "state_peak": 120.08096313476562, "growth_active": 1.0243487358093262, "growth_memory": 0.9801895022392273, "heads": [{"head": 0, "memory_to_active": 0.9351809620857239, "contribution_to_active": 2.2563858032226562, "mix_mean": 0.5592511892318726, "mix_low": 0.09417533874511719, "mix_high": 0.004652976989746094, "read_cosine": 0.08521237969398499, "beta_mean": 0.21363294124603271, "nonfinite": 0.0, "weighted_memory_to_active": 0.558265209197998, "mix_std": 0.3895113468170166, "beta_std": 0.21950402855873108, "beta_low": 0.14111328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.600334644317627, "evidence_weights": [-0.09440005570650101, -0.38391414284706116, -1.7396599054336548], "agreement_weight": 0.01878019981086254, "orthogonal_weight": 2.8859481811523438}, {"head": 1, "memory_to_active": 0.008378961123526096, "contribution_to_active": 0.5979818105697632, "mix_mean": 0.6226215362548828, "mix_low": 0.0011529922485351562, "mix_high": 0.01366424560546875, "read_cosine": 0.29519349336624146, "beta_mean": 5.950300328549929e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004683381877839565, "mix_std": 0.26581478118896484, "beta_std": 8.422833343502134e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007180483546108007, "evidence_weights": [-0.25229761004447937, -0.19675731658935547, 0.5599895119667053], "agreement_weight": 0.5500959753990173, "orthogonal_weight": -1.655815601348877}, {"head": 2, "memory_to_active": 0.017328111454844475, "contribution_to_active": 0.6099992990493774, "mix_mean": 0.6520019173622131, "mix_low": 0.0003681182861328125, "mix_high": 0.003966331481933594, "read_cosine": 0.033313725143671036, "beta_mean": 3.4717420930974185e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007883618585765362, "mix_std": 0.25000208616256714, "beta_std": 0.0001245984749402851, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003508030204102397, "evidence_weights": [-0.2610834240913391, -0.2810238003730774, 0.6063426733016968], "agreement_weight": -0.20920348167419434, "orthogonal_weight": -0.4275641441345215}, {"head": 3, "memory_to_active": 0.35232794284820557, "contribution_to_active": 0.09191818535327911, "mix_mean": 0.2643234431743622, "mix_low": 0.036025047302246094, "mix_high": 8.20159912109375e-05, "read_cosine": 0.9165291786193848, "beta_mean": 0.06830602139234543, "nonfinite": 0.0, "weighted_memory_to_active": 0.03711896762251854, "mix_std": 0.22299176454544067, "beta_std": 0.058736398816108704, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.03899383172392845, "evidence_weights": [-0.10097572207450867, -0.14671443402767181, 0.1586429923772812], "agreement_weight": 0.013693825341761112, "orthogonal_weight": 1.4043021202087402}, {"head": 4, "memory_to_active": 0.020392099395394325, "contribution_to_active": 0.5525249242782593, "mix_mean": 0.5835458040237427, "mix_low": 0.00022602081298828125, "mix_high": 0.001590728759765625, "read_cosine": 0.06183850020170212, "beta_mean": 6.011468940414488e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011712216772139072, "mix_std": 0.24657057225704193, "beta_std": 0.00010913914593402296, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015808582305908203, "evidence_weights": [-0.18526612222194672, -0.26082539558410645, 0.9851300120353699], "agreement_weight": 0.15199051797389984, "orthogonal_weight": -1.4221115112304688}, {"head": 5, "memory_to_active": 0.13064569234848022, "contribution_to_active": 0.4067579507827759, "mix_mean": 0.39791518449783325, "mix_low": 0.00199127197265625, "mix_high": 9.5367431640625e-07, "read_cosine": 0.8513708114624023, "beta_mean": 0.0070799970999360085, "nonfinite": 0.0, "weighted_memory_to_active": 0.06081806868314743, "mix_std": 0.2273213267326355, "beta_std": 0.007272630929946899, "beta_low": 0.79345703125, "beta_high": 0.0, "orthogonal_to_active": 0.019922157749533653, "evidence_weights": [-0.14012515544891357, -0.16146129369735718, 0.26862475275993347], "agreement_weight": 0.09127100557088852, "orthogonal_weight": -0.6560367345809937}]}, {"layer": 11, "memory_to_active": 0.09575393050909042, "contribution_to_active": 0.6152464151382446, "mix_mean": 0.5180824398994446, "mix_low": 0.1039077490568161, "mix_high": 0.046508632600307465, "read_cosine": 0.35704419016838074, "beta_mean": 0.027614042162895203, "nonfinite": 0.0, "weighted_memory_to_active": 0.0503743477165699, "mix_std": 0.3243866264820099, "beta_std": 0.041088562458753586, "beta_low": 0.6756795644760132, "beta_high": 0.0, "orthogonal_to_active": 0.05062027648091316, "output_influence": 0.5443211197853088, "residual_influence": 0.1426788717508316, "state_active": 0.18979191780090332, "state_memory": 0.004278203006833792, "state_peak": 31.181255340576172, "growth_active": 1.1109634637832642, "growth_memory": 1.577913522720337, "heads": [{"head": 0, "memory_to_active": 0.009768216870725155, "contribution_to_active": 0.6006274819374084, "mix_mean": 0.6484873294830322, "mix_low": 0.0008687973022460938, "mix_high": 0.009352684020996094, "read_cosine": 0.255051851272583, "beta_mean": 8.092619100352749e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0072956206277012825, "mix_std": 0.25773847103118896, "beta_std": 0.0034608510322868824, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.005856619682163, "evidence_weights": [-0.1734044849872589, -0.3659152686595917, 0.7537767887115479], "agreement_weight": 0.3533444404602051, "orthogonal_weight": -0.8314141631126404}, {"head": 1, "memory_to_active": 0.17557711899280548, "contribution_to_active": 0.4055013358592987, "mix_mean": 0.4464670419692993, "mix_low": 0.00707244873046875, "mix_high": 0.000308990478515625, "read_cosine": 0.46373867988586426, "beta_mean": 0.009244010783731937, "nonfinite": 0.0, "weighted_memory_to_active": 0.08344889432191849, "mix_std": 0.2671031951904297, "beta_std": 0.025173112750053406, "beta_low": 0.84271240234375, "beta_high": 0.0, "orthogonal_to_active": 0.043041978031396866, "evidence_weights": [-0.15624359250068665, -0.24581076204776764, 0.5960105657577515], "agreement_weight": 0.07076019793748856, "orthogonal_weight": -0.6121073365211487}, {"head": 2, "memory_to_active": 0.007345259189605713, "contribution_to_active": 0.9817041754722595, "mix_mean": 0.4413285553455353, "mix_low": 0.28670787811279297, "mix_high": 0.1733999252319336, "read_cosine": 0.17275899648666382, "beta_mean": 0.07198513299226761, "nonfinite": 0.0, "weighted_memory_to_active": 0.005011591129004955, "mix_std": 0.4486294388771057, "beta_std": 0.09695009887218475, "beta_low": 0.48223876953125, "beta_high": 0.0, "orthogonal_to_active": 0.04416089877486229, "evidence_weights": [0.043782006949186325, -0.3877117335796356, -1.3220276832580566], "agreement_weight": 0.008138821460306644, "orthogonal_weight": 9.121280670166016}, {"head": 3, "memory_to_active": 0.00865904800593853, "contribution_to_active": 0.8585404753684998, "mix_mean": 0.4152136445045471, "mix_low": 0.32058238983154297, "mix_high": 0.0881357192993164, "read_cosine": 0.5730772018432617, "beta_mean": 0.05677337944507599, "nonfinite": 0.0, "weighted_memory_to_active": 0.005377164110541344, "mix_std": 0.43968337774276733, "beta_std": 0.08558947592973709, "beta_low": 0.519287109375, "beta_high": 0.0, "orthogonal_to_active": 0.035349052399396896, "evidence_weights": [0.1974782943725586, -0.29951074719429016, -0.15846598148345947], "agreement_weight": 0.047619860619306564, "orthogonal_weight": 9.415803909301758}, {"head": 4, "memory_to_active": 0.26444631814956665, "contribution_to_active": 0.44086432456970215, "mix_mean": 0.5868886709213257, "mix_low": 0.0064601898193359375, "mix_high": 0.0016841888427734375, "read_cosine": 0.033701345324516296, "beta_mean": 0.017910070717334747, "nonfinite": 0.0, "weighted_memory_to_active": 0.14481741189956665, "mix_std": 0.2734875977039337, "beta_std": 0.019611088559031487, "beta_low": 0.44818115234375, "beta_high": 0.0, "orthogonal_to_active": 0.14120551943778992, "evidence_weights": [-0.24187494814395905, -0.23995022475719452, 1.9292205572128296], "agreement_weight": -0.21272939443588257, "orthogonal_weight": -0.9876542091369629}, {"head": 5, "memory_to_active": 0.10872761905193329, "contribution_to_active": 0.40424084663391113, "mix_mean": 0.5701093673706055, "mix_low": 0.0017547607421875, "mix_high": 0.0061702728271484375, "read_cosine": 0.6439369320869446, "beta_mean": 0.00969071313738823, "nonfinite": 0.0, "weighted_memory_to_active": 0.05629539489746094, "mix_std": 0.2596776485443115, "beta_std": 0.015746746212244034, "beta_low": 0.761962890625, "beta_high": 0.0, "orthogonal_to_active": 0.03410758078098297, "evidence_weights": [-0.21199408173561096, -0.31524837017059326, 0.6533841490745544], "agreement_weight": 0.08475618064403534, "orthogonal_weight": -0.8444241881370544}]}, {"layer": 12, "memory_to_active": 0.23428723216056824, "contribution_to_active": 0.5779402256011963, "mix_mean": 0.5360517501831055, "mix_low": 0.009140650741755962, "mix_high": 0.005851904861629009, "read_cosine": 0.22648176550865173, "beta_mean": 0.01614866778254509, "nonfinite": 0.0, "weighted_memory_to_active": 0.12484723329544067, "mix_std": 0.2631934881210327, "beta_std": 0.015475375577807426, "beta_low": 0.8259684443473816, "beta_high": 0.0, "orthogonal_to_active": 0.07742661982774734, "output_influence": 0.49332672357559204, "residual_influence": 0.14097493886947632, "state_active": 0.03525590896606445, "state_memory": 0.009129918180406094, "state_peak": 64.80660247802734, "growth_active": 1.149298906326294, "growth_memory": 1.2289570569992065, "heads": [{"head": 0, "memory_to_active": 0.05967875197529793, "contribution_to_active": 0.59279465675354, "mix_mean": 0.5679829120635986, "mix_low": 0.0006132125854492188, "mix_high": 0.004948616027832031, "read_cosine": 0.28155434131622314, "beta_mean": 0.0009430685895495117, "nonfinite": 0.0, "weighted_memory_to_active": 0.031974054872989655, "mix_std": 0.2672366201877594, "beta_std": 0.0017905558925122023, "beta_low": 0.9923095703125, "beta_high": 0.0, "orthogonal_to_active": 0.029169347137212753, "evidence_weights": [-0.20878896117210388, -0.2752448320388794, 0.04492330551147461], "agreement_weight": 0.0929713174700737, "orthogonal_weight": -0.978566586971283}, {"head": 1, "memory_to_active": 0.11115077883005142, "contribution_to_active": 0.5488037467002869, "mix_mean": 0.5553691387176514, "mix_low": 0.00029850006103515625, "mix_high": 0.00189971923828125, "read_cosine": 0.4078148305416107, "beta_mean": 0.0008222791366279125, "nonfinite": 0.0, "weighted_memory_to_active": 0.06435446441173553, "mix_std": 0.2648681402206421, "beta_std": 0.00998217985033989, "beta_low": 0.9952392578125, "beta_high": 0.0, "orthogonal_to_active": 0.05361221730709076, "evidence_weights": [-0.2030084729194641, -0.2716277539730072, 1.0390706062316895], "agreement_weight": 0.2315916121006012, "orthogonal_weight": -0.9281872510910034}, {"head": 2, "memory_to_active": 0.6253355741500854, "contribution_to_active": 0.5257465839385986, "mix_mean": 0.3181193172931671, "mix_low": 0.02875804901123047, "mix_high": 0.0005102157592773438, "read_cosine": 0.5146836042404175, "beta_mean": 0.09384408593177795, "nonfinite": 0.0, "weighted_memory_to_active": 0.2704334855079651, "mix_std": 0.24947641789913177, "beta_std": 0.06472903490066528, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.27823397517204285, "evidence_weights": [-0.2158202826976776, -0.055150896310806274, 0.009648102335631847], "agreement_weight": 0.023300541564822197, "orthogonal_weight": 1.3495721817016602}, {"head": 3, "memory_to_active": 0.03500363230705261, "contribution_to_active": 0.7345877885818481, "mix_mean": 0.7099233269691467, "mix_low": 0.000110626220703125, "mix_high": 0.017164230346679688, "read_cosine": 0.09595386683940887, "beta_mean": 0.0002653792325872928, "nonfinite": 0.0, "weighted_memory_to_active": 0.027502285316586494, "mix_std": 0.2327396720647812, "beta_std": 0.005168047733604908, "beta_low": 0.9976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.026448192074894905, "evidence_weights": [-0.28882575035095215, -0.3359345495700836, 1.687778115272522], "agreement_weight": 0.5145044326782227, "orthogonal_weight": -0.9784379601478577}, {"head": 4, "memory_to_active": 0.4832005798816681, "contribution_to_active": 0.5756028890609741, "mix_mean": 0.4353053569793701, "mix_low": 0.018072128295898438, "mix_high": 0.0042476654052734375, "read_cosine": -0.13881200551986694, "beta_mean": 0.0005594709073193371, "nonfinite": 0.0, "weighted_memory_to_active": 0.2935325503349304, "mix_std": 0.28870272636413574, "beta_std": 0.0070817433297634125, "beta_low": 0.9830322265625, "beta_high": 0.0, "orthogonal_to_active": 0.030261583626270294, "evidence_weights": [-0.15221628546714783, -0.21816150844097137, -0.09519227594137192], "agreement_weight": 0.014598716981709003, "orthogonal_weight": -0.10714693367481232}, {"head": 5, "memory_to_active": 0.09135399013757706, "contribution_to_active": 0.4901055097579956, "mix_mean": 0.6296104192733765, "mix_low": 0.006991386413574219, "mix_high": 0.006340980529785156, "read_cosine": 0.19769589602947235, "beta_mean": 0.0004577182116918266, "nonfinite": 0.0, "weighted_memory_to_active": 0.06128653138875961, "mix_std": 0.2761373519897461, "beta_std": 0.004100687801837921, "beta_low": 0.98699951171875, "beta_high": 0.0, "orthogonal_to_active": 0.046834371984004974, "evidence_weights": [-0.23266653716564178, -0.31706666946411133, 1.307311773300171], "agreement_weight": 0.40169477462768555, "orthogonal_weight": -0.8210644125938416}]}, {"layer": 13, "memory_to_active": 0.12496841698884964, "contribution_to_active": 0.534287691116333, "mix_mean": 0.5381043553352356, "mix_low": 0.005992412567138672, "mix_high": 0.009467284195125103, "read_cosine": 0.4070500135421753, "beta_mean": 0.01594511792063713, "nonfinite": 0.0, "weighted_memory_to_active": 0.054852165281772614, "mix_std": 0.2617570757865906, "beta_std": 0.01806425303220749, "beta_low": 0.8082683086395264, "beta_high": 0.0, "orthogonal_to_active": 0.05600658431649208, "output_influence": 0.5228809118270874, "residual_influence": 0.17869463562965393, "state_active": 0.12471485882997513, "state_memory": 0.04492984712123871, "state_peak": 101.26313018798828, "growth_active": 0.916299045085907, "growth_memory": 1.0143367052078247, "heads": [{"head": 0, "memory_to_active": 0.020355969667434692, "contribution_to_active": 0.5859735012054443, "mix_mean": 0.6075421571731567, "mix_low": 0.0006380081176757812, "mix_high": 0.004269599914550781, "read_cosine": 0.432162344455719, "beta_mean": 7.414105493808165e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01289873942732811, "mix_std": 0.2710244655609131, "beta_std": 0.00016517135372851044, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012617483735084534, "evidence_weights": [-0.19650189578533173, -0.2676510214805603, 0.7779678106307983], "agreement_weight": 0.1606135368347168, "orthogonal_weight": -1.0859261751174927}, {"head": 1, "memory_to_active": 0.016401182860136032, "contribution_to_active": 0.5978313684463501, "mix_mean": 0.5710086822509766, "mix_low": 0.0018739700317382812, "mix_high": 0.004441261291503906, "read_cosine": 0.21745029091835022, "beta_mean": 6.32752271485515e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009032187052071095, "mix_std": 0.2746926546096802, "beta_std": 0.0001228995097335428, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011813322082161903, "evidence_weights": [-0.1848025768995285, -0.266431987285614, 0.6082770824432373], "agreement_weight": 0.450105220079422, "orthogonal_weight": -1.3609176874160767}, {"head": 2, "memory_to_active": 0.07370724529027939, "contribution_to_active": 0.47052013874053955, "mix_mean": 0.3878755569458008, "mix_low": 0.0014467239379882812, "mix_high": 0.00012111663818359375, "read_cosine": 0.545566737651825, "beta_mean": 0.011529587209224701, "nonfinite": 0.0, "weighted_memory_to_active": 0.03315366432070732, "mix_std": 0.2325214147567749, "beta_std": 0.042060356587171555, "beta_low": 0.856689453125, "beta_high": 0.0, "orthogonal_to_active": 0.021563174203038216, "evidence_weights": [-0.18288742005825043, -0.20644912123680115, 0.4067523181438446], "agreement_weight": 0.006311982870101929, "orthogonal_weight": -0.843778669834137}, {"head": 3, "memory_to_active": 0.1798601895570755, "contribution_to_active": 0.5908360481262207, "mix_mean": 0.6607581973075867, "mix_low": 0.023285865783691406, "mix_high": 0.035141944885253906, "read_cosine": 0.0019697551615536213, "beta_mean": 0.0009395579691044986, "nonfinite": 0.0, "weighted_memory_to_active": 0.1368332803249359, "mix_std": 0.2897602319717407, "beta_std": 0.009466283023357391, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.12036047875881195, "evidence_weights": [-0.31924864649772644, -0.24815727770328522, 1.6539992094039917], "agreement_weight": 5.878840446472168, "orthogonal_weight": -0.9679113030433655}, {"head": 4, "memory_to_active": 0.4130994975566864, "contribution_to_active": 0.25480565428733826, "mix_mean": 0.347801148891449, "mix_low": 0.005850791931152344, "mix_high": 1.239776611328125e-05, "read_cosine": 0.8918342590332031, "beta_mean": 0.08286484330892563, "nonfinite": 0.0, "weighted_memory_to_active": 0.10582855343818665, "mix_std": 0.22840850055217743, "beta_std": 0.05493469908833504, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.1438034176826477, "evidence_weights": [-0.0820118635892868, -0.2290918529033661, 0.46359699964523315], "agreement_weight": 0.008976250886917114, "orthogonal_weight": 2.2297632694244385}, {"head": 5, "memory_to_active": 0.04638640210032463, "contribution_to_active": 0.7057591080665588, "mix_mean": 0.6536402106285095, "mix_low": 0.0028591156005859375, "mix_high": 0.0128173828125, "read_cosine": 0.3533167839050293, "beta_mean": 0.0001992939505726099, "nonfinite": 0.0, "weighted_memory_to_active": 0.03136654198169708, "mix_std": 0.2741350829601288, "beta_std": 0.0016361052403226495, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.02588161826133728, "evidence_weights": [-0.2224993109703064, -0.3009726107120514, 0.8819002509117126], "agreement_weight": 0.18179182708263397, "orthogonal_weight": -0.9187788963317871}]}, {"layer": 14, "memory_to_active": 0.2383842021226883, "contribution_to_active": 0.41135770082473755, "mix_mean": 0.42452365159988403, "mix_low": 0.027912937104701996, "mix_high": 0.006008943077176809, "read_cosine": 0.6137636303901672, "beta_mean": 0.06567272543907166, "nonfinite": 0.0, "weighted_memory_to_active": 0.09932965040206909, "mix_std": 0.2438984215259552, "beta_std": 0.061258818954229355, "beta_low": 0.4927368462085724, "beta_high": 0.0, "orthogonal_to_active": 0.040707774460315704, "output_influence": 0.3484201431274414, "residual_influence": 0.13709180057048798, "state_active": 0.05664702132344246, "state_memory": 0.015972649678587914, "state_peak": 41.252281188964844, "growth_active": 1.0964155197143555, "growth_memory": 1.2794891595840454, "heads": [{"head": 0, "memory_to_active": 0.37670770287513733, "contribution_to_active": 0.18045714497566223, "mix_mean": 0.179390549659729, "mix_low": 0.02263641357421875, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8200157880783081, "beta_mean": 0.07316144555807114, "nonfinite": 0.0, "weighted_memory_to_active": 0.08030644059181213, "mix_std": 0.17777256667613983, "beta_std": 0.060796692967414856, "beta_low": 0.0029296875, "beta_high": 0.0, "orthogonal_to_active": 0.07945582270622253, "evidence_weights": [-0.10914354026317596, -0.042359013110399246, 0.42575088143348694], "agreement_weight": 0.025855207815766335, "orthogonal_weight": 1.712020993232727}, {"head": 1, "memory_to_active": 0.5098751783370972, "contribution_to_active": 0.2639982998371124, "mix_mean": 0.21283206343650818, "mix_low": 0.1280975341796875, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8753048181533813, "beta_mean": 0.2722288966178894, "nonfinite": 0.0, "weighted_memory_to_active": 0.22649002075195312, "mix_std": 0.24207966029644012, "beta_std": 0.256669282913208, "beta_low": 0.085205078125, "beta_high": 0.0, "orthogonal_to_active": 0.014392508193850517, "evidence_weights": [-0.008586728014051914, -0.2006671279668808, -0.09701432287693024], "agreement_weight": 0.05913969874382019, "orthogonal_weight": 0.17913255095481873}, {"head": 2, "memory_to_active": 0.12312173843383789, "contribution_to_active": 0.5872544646263123, "mix_mean": 0.5900483131408691, "mix_low": 0.0029211044311523438, "mix_high": 0.0035009384155273438, "read_cosine": 0.41681602597236633, "beta_mean": 0.0018729700241237879, "nonfinite": 0.0, "weighted_memory_to_active": 0.0803249254822731, "mix_std": 0.2661220133304596, "beta_std": 0.002882329747080803, "beta_low": 0.9842529296875, "beta_high": 0.0, "orthogonal_to_active": 0.061439838260412216, "evidence_weights": [-0.1389887034893036, -0.31603363156318665, 0.9161127209663391], "agreement_weight": 0.08207739144563675, "orthogonal_weight": -0.8758928179740906}, {"head": 3, "memory_to_active": 0.044778499752283096, "contribution_to_active": 0.5964324474334717, "mix_mean": 0.6701818704605103, "mix_low": 0.005283355712890625, "mix_high": 0.02196216583251953, "read_cosine": -0.06685376912355423, "beta_mean": 0.005905637517571449, "nonfinite": 0.0, "weighted_memory_to_active": 0.028016913682222366, "mix_std": 0.25943002104759216, "beta_std": 0.008368903771042824, "beta_low": 0.830322265625, "beta_high": 0.0, "orthogonal_to_active": 0.02114049717783928, "evidence_weights": [-0.3465205729007721, -0.31073662638664246, 0.275823712348938], "agreement_weight": 0.06532270461320877, "orthogonal_weight": -0.8326629996299744}, {"head": 4, "memory_to_active": 0.15831740200519562, "contribution_to_active": 0.5044062733650208, "mix_mean": 0.51542729139328, "mix_low": 0.004069328308105469, "mix_high": 0.009107589721679688, "read_cosine": 0.8834105730056763, "beta_mean": 0.039010874927043915, "nonfinite": 0.0, "weighted_memory_to_active": 0.09378142654895782, "mix_std": 0.26147523522377014, "beta_std": 0.028540480881929398, "beta_low": 0.0855712890625, "beta_high": 0.0, "orthogonal_to_active": 0.025693759322166443, "evidence_weights": [-0.2163597196340561, -0.24251259863376617, 0.4315900504589081], "agreement_weight": 0.13584169745445251, "orthogonal_weight": -0.6155344247817993}, {"head": 5, "memory_to_active": 0.21750469505786896, "contribution_to_active": 0.3355974853038788, "mix_mean": 0.37926146388053894, "mix_low": 0.004469871520996094, "mix_high": 0.0014677047729492188, "read_cosine": 0.7538882493972778, "beta_mean": 0.0018565311329439282, "nonfinite": 0.0, "weighted_memory_to_active": 0.0870581716299057, "mix_std": 0.2565109133720398, "beta_std": 0.010295226238667965, "beta_low": 0.9681396484375, "beta_high": 0.0, "orthogonal_to_active": 0.04212421551346779, "evidence_weights": [-0.17579694092273712, -0.17474070191383362, 0.3153562843799591], "agreement_weight": 0.0725787952542305, "orthogonal_weight": -0.755827009677887}]}, {"layer": 15, "memory_to_active": 0.06064598634839058, "contribution_to_active": 0.48772382736206055, "mix_mean": 0.4257013499736786, "mix_low": 0.008358797058463097, "mix_high": 0.0032561621628701687, "read_cosine": 0.682212769985199, "beta_mean": 0.039595186710357666, "nonfinite": 0.0, "weighted_memory_to_active": 0.029736516997218132, "mix_std": 0.28712189197540283, "beta_std": 0.045096129179000854, "beta_low": 0.7102864980697632, "beta_high": 0.0, "orthogonal_to_active": 0.008012151345610619, "output_influence": 0.3044546842575073, "residual_influence": 0.17337369918823242, "state_active": 0.14168256521224976, "state_memory": 0.011027061380445957, "state_peak": 34.375247955322266, "growth_active": 1.1352527141571045, "growth_memory": 1.3235514163970947, "heads": [{"head": 0, "memory_to_active": 0.10480271279811859, "contribution_to_active": 0.49523496627807617, "mix_mean": 0.39550769329071045, "mix_low": 0.008175849914550781, "mix_high": 0.006012916564941406, "read_cosine": 0.6297109127044678, "beta_mean": 0.03494495153427124, "nonfinite": 0.0, "weighted_memory_to_active": 0.04958486184477806, "mix_std": 0.28556063771247864, "beta_std": 0.02603691816329956, "beta_low": 0.09979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.02662026695907116, "evidence_weights": [-0.1898464858531952, -0.19107942283153534, 0.4504224359989166], "agreement_weight": 0.0920298770070076, "orthogonal_weight": -0.7718523144721985}, {"head": 1, "memory_to_active": 0.13817574083805084, "contribution_to_active": 0.4216667413711548, "mix_mean": 0.4405994415283203, "mix_low": 0.0016412734985351562, "mix_high": 0.006979942321777344, "read_cosine": 0.9250359535217285, "beta_mean": 0.0006824518204666674, "nonfinite": 0.0, "weighted_memory_to_active": 0.05967162176966667, "mix_std": 0.289598286151886, "beta_std": 0.002560650696977973, "beta_low": 0.9935302734375, "beta_high": 0.0, "orthogonal_to_active": 0.0018056086264550686, "evidence_weights": [-0.14826610684394836, -0.1567348837852478, 0.17506860196590424], "agreement_weight": 0.12437919527292252, "orthogonal_weight": 0.0630001351237297}, {"head": 2, "memory_to_active": 0.0034656997304409742, "contribution_to_active": 0.4946519136428833, "mix_mean": 0.34558790922164917, "mix_low": 0.012117385864257812, "mix_high": 0.00017261505126953125, "read_cosine": 0.7868996858596802, "beta_mean": 4.475010427995585e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0017514307983219624, "mix_std": 0.27320969104766846, "beta_std": 9.343092824565247e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00047908315900713205, "evidence_weights": [-0.20406801998615265, -0.26133012771606445, 0.1340269297361374], "agreement_weight": 0.011370696127414703, "orthogonal_weight": 0.7871353626251221}, {"head": 3, "memory_to_active": 0.0011837136698886752, "contribution_to_active": 0.3736724257469177, "mix_mean": 0.4002813696861267, "mix_low": 0.00516510009765625, "mix_high": 0.0005626678466796875, "read_cosine": 0.6632286310195923, "beta_mean": 5.563438207900617e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00038740909076295793, "mix_std": 0.2892420291900635, "beta_std": 1.120361230277922e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00029858763446100056, "evidence_weights": [-0.26419392228126526, -0.2406565099954605, 0.19799235463142395], "agreement_weight": 0.05767679214477539, "orthogonal_weight": -0.7317458391189575}, {"head": 4, "memory_to_active": 0.08818809688091278, "contribution_to_active": 0.6177421808242798, "mix_mean": 0.3839590549468994, "mix_low": 0.011693954467773438, "mix_high": 0.0015697479248046875, "read_cosine": 0.7310611605644226, "beta_mean": 0.1997666358947754, "nonfinite": 0.0, "weighted_memory_to_active": 0.04844814911484718, "mix_std": 0.31146588921546936, "beta_std": 0.22676126658916473, "beta_low": 0.19842529296875, "beta_high": 0.0, "orthogonal_to_active": 0.007694142870604992, "evidence_weights": [-0.20867134630680084, -0.25450924038887024, 0.001146499183960259], "agreement_weight": 0.012000939808785915, "orthogonal_weight": 0.35554325580596924}, {"head": 5, "memory_to_active": 0.02805991843342781, "contribution_to_active": 0.5233746767044067, "mix_mean": 0.5882725119590759, "mix_low": 0.011359214782714844, "mix_high": 0.004239082336425781, "read_cosine": 0.35734009742736816, "beta_mean": 0.0021267831325531006, "nonfinite": 0.0, "weighted_memory_to_active": 0.018575629219412804, "mix_std": 0.27365466952323914, "beta_std": 0.015113281086087227, "beta_low": 0.969970703125, "beta_high": 0.0, "orthogonal_to_active": 0.01117522083222866, "evidence_weights": [-0.2870948612689972, -0.30401086807250977, 0.7091365456581116], "agreement_weight": 0.30159854888916016, "orthogonal_weight": 0.81881183385849}]}]}, {"step": 57000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8707336379993649, "memory_to_active": 0.8060699701309204, "contribution_to_active": 0.5555684566497803, "mix_mean": 0.4150841236114502, "mix_low": 0.12970687448978424, "mix_high": 0.010724704712629318, "read_cosine": 0.3471609354019165, "beta_mean": 0.07229653745889664, "nonfinite": 0.0, "weighted_memory_to_active": 0.20270419120788574, "mix_std": 0.22273549437522888, "beta_std": 0.0632350966334343, "beta_low": 0.565201461315155, "beta_high": 3.24249267578125e-05, "orthogonal_to_active": 0.06106334924697876, "samples": 4, "scored_tokens": 899, "normal_loss": 3.082227945327759, "normal_token_accuracy": 0.4549499452114105, "state_active": 0.0644415020942688, "state_memory": 0.018107306212186813, "state_peak": 125.60242462158203, "growth_active": 1.134570598602295, "growth_memory": 1.3165781497955322, "output_influence": 0.4268934726715088, "residual_influence": 0.14603662490844727, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2109689712524414, "no_memory_loss_delta": 0.5618947744369507, "scrambled_loss_delta": 0.18191830813884735, "memory_only_loss_delta": 4.492746353149414, "no_evidence_loss_delta": 0.347489595413208, "no_agreement_loss_delta": 0.033070869743824005, "no_orthogonal_loss_delta": 0.007877150550484657}, {"index": 94, "tokens": 256, "normal_loss": 3.235175132751465, "no_memory_loss_delta": 0.7320588827133179, "scrambled_loss_delta": 0.35860055685043335, "memory_only_loss_delta": 3.590386390686035, "no_evidence_loss_delta": 0.36610177159309387, "no_agreement_loss_delta": 0.03932574391365051, "no_orthogonal_loss_delta": 0.026736833155155182}, {"index": 188, "tokens": 256, "normal_loss": 3.3962671756744385, "no_memory_loss_delta": 0.7930331230163574, "scrambled_loss_delta": 0.15178588032722473, "memory_only_loss_delta": 3.9947361946105957, "no_evidence_loss_delta": 0.4013269543647766, "no_agreement_loss_delta": 0.05448219180107117, "no_orthogonal_loss_delta": 0.035054177045822144}, {"index": 283, "tokens": 131, "normal_loss": 1.9180561304092407, "no_memory_loss_delta": 0.41132020950317383, "scrambled_loss_delta": 0.1626218557357788, "memory_only_loss_delta": 3.098876953125, "no_evidence_loss_delta": 0.17204931378364563, "no_agreement_loss_delta": 0.013352934271097183, "no_orthogonal_loss_delta": -0.015141059644520283}], "agreement_weight_abs_mean": 0.21945244073867798, "agreement_weight_abs_max": 5.955837726593018, "agreement_weight_mean": 0.19161121547222137, "orthogonal_weight_abs_mean": 0.8778822422027588, "orthogonal_weight_abs_max": 9.537224769592285, "orthogonal_weight_mean": -0.10720659047365189, "no_memory_loss": 3.7364563941955566, "no_memory_loss_delta": 0.654228687286377, "no_memory_token_accuracy_gain": 0.1001112312078476, "no_memory_token_win_fraction": 0.6729699373245239, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 48.01571305922604, "scrambled_loss": 3.3030660152435303, "scrambled_loss_delta": 0.22083812952041626, "scrambled_token_accuracy_gain": 0.022246941924095154, "scrambled_token_win_fraction": 0.6407119035720825, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.815353494239794, "memory_only_loss": 6.973092555999756, "memory_only_loss_delta": 3.890864610671997, "memory_only_token_accuracy_gain": 0.29032257199287415, "memory_only_token_win_fraction": 0.9076752066612244, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 97.95723235866657, "no_evidence_loss": 3.42478346824646, "no_evidence_loss_delta": 0.3425556719303131, "no_evidence_token_accuracy_gain": 0.04338153451681137, "no_evidence_token_win_fraction": 0.6340377926826477, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 29.0046406201671, "no_agreement_loss": 3.1203033924102783, "no_agreement_loss_delta": 0.038075871765613556, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.5305895209312439, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.736009906853451, "no_orthogonal_loss": 3.0998597145080566, "no_orthogonal_loss_delta": 0.01763244718313217, "no_orthogonal_token_accuracy_gain": -0.0055617354810237885, "no_orthogonal_token_win_fraction": 0.5773081183433533, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.7477905236951023, "short_no_memory_loss_delta": 0.5837377905845642, "long_context_memory_gain": 0.07049089670181274, "normal_context_loss_gain": 0.13316795229911804, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.418194055557251, "contribution_to_active": 1.130176067352295, "mix_mean": 0.10592252016067505, "mix_low": 0.527230441570282, "mix_high": 0.0056055388413369656, "read_cosine": 0.42731767892837524, "beta_mean": 0.21453112363815308, "nonfinite": 0.0, "weighted_memory_to_active": 1.1692569255828857, "mix_std": 0.09185866266489029, "beta_std": 0.14864836633205414, "beta_low": 0.0874939039349556, "beta_high": 0.0, "orthogonal_to_active": 0.04570890590548515, "output_influence": 0.21622826159000397, "residual_influence": 0.25025877356529236, "state_active": 0.0671883225440979, "state_memory": 0.045409053564071655, "state_peak": 50.02984619140625, "growth_active": 1.2526791095733643, "growth_memory": 1.2988940477371216, "heads": [{"head": 0, "memory_to_active": 2.630917549133301, "contribution_to_active": 0.48986363410949707, "mix_mean": 0.020084545016288757, "mix_low": 0.6382694244384766, "mix_high": 9.5367431640625e-07, "read_cosine": 0.218715637922287, "beta_mean": 0.18557190895080566, "nonfinite": 0.0, "weighted_memory_to_active": 0.6158550977706909, "mix_std": 0.06433284282684326, "beta_std": 0.10923575609922409, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06565890461206436, "evidence_weights": [-0.0027288601268082857, -0.04638582468032837, -0.047059495002031326], "agreement_weight": -0.20583869516849518, "orthogonal_weight": -0.11966084688901901}, {"head": 1, "memory_to_active": 5.901293754577637, "contribution_to_active": 2.5971288681030273, "mix_mean": 0.01923506334424019, "mix_low": 0.6257266998291016, "mix_high": 0.0, "read_cosine": 0.4450380504131317, "beta_mean": 0.4527505934238434, "nonfinite": 0.0, "weighted_memory_to_active": 2.8796041011810303, "mix_std": 0.05706892907619476, "beta_std": 0.24041631817817688, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013677576556801796, "evidence_weights": [0.023860584944486618, -0.10122807323932648, 0.6264911890029907], "agreement_weight": 0.028023913502693176, "orthogonal_weight": 0.010356627404689789}, {"head": 2, "memory_to_active": 2.822815418243408, "contribution_to_active": 0.5585254430770874, "mix_mean": 0.014866761863231659, "mix_low": 0.6535625457763672, "mix_high": 0.0, "read_cosine": 0.34004345536231995, "beta_mean": 0.25114643573760986, "nonfinite": 0.0, "weighted_memory_to_active": 0.6501045227050781, "mix_std": 0.03366437181830406, "beta_std": 0.18053749203681946, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.055976856499910355, "evidence_weights": [0.06174219772219658, 0.053777795284986496, 0.19844412803649902], "agreement_weight": 0.006942488718777895, "orthogonal_weight": -0.208360955119133}, {"head": 3, "memory_to_active": 7.241513729095459, "contribution_to_active": 2.6002752780914307, "mix_mean": 0.03198479861021042, "mix_low": 0.48362159729003906, "mix_high": 0.0, "read_cosine": 0.31570056080818176, "beta_mean": 0.3098631203174591, "nonfinite": 0.0, "weighted_memory_to_active": 2.760244607925415, "mix_std": 0.0721292495727539, "beta_std": 0.24022981524467468, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11929263174533844, "evidence_weights": [-0.04119637608528137, -0.04007286578416824, -0.1557801216840744], "agreement_weight": -0.008547293022274971, "orthogonal_weight": -0.05190030857920647}, {"head": 4, "memory_to_active": 1.804316759109497, "contribution_to_active": 0.07167980819940567, "mix_mean": 0.011482680216431618, "mix_low": 0.7474489212036133, "mix_high": 0.0, "read_cosine": 0.30176258087158203, "beta_mean": 0.06955894827842712, "nonfinite": 0.0, "weighted_memory_to_active": 0.05223463103175163, "mix_std": 0.035735517740249634, "beta_std": 0.10426502674818039, "beta_low": 0.20587158203125, "beta_high": 0.0, "orthogonal_to_active": 0.003361908020451665, "evidence_weights": [0.1337071657180786, 0.09836605936288834, -0.15844324231147766], "agreement_weight": -0.1392776072025299, "orthogonal_weight": 0.057362522929906845}, {"head": 5, "memory_to_active": 0.1083056703209877, "contribution_to_active": 0.46358346939086914, "mix_mean": 0.5378812551498413, "mix_low": 0.014753341674804688, "mix_high": 0.03363227844238281, "read_cosine": 0.9426454901695251, "beta_mean": 0.018295688554644585, "nonfinite": 0.0, "weighted_memory_to_active": 0.05749890208244324, "mix_std": 0.2882210314273834, "beta_std": 0.0172057393938303, "beta_low": 0.319091796875, "beta_high": 0.0, "orthogonal_to_active": 0.016285549849271774, "evidence_weights": [-0.1777699738740921, -0.21751365065574646, 0.3834074139595032], "agreement_weight": 0.2894779443740845, "orthogonal_weight": -0.6800805926322937}]}, {"layer": 1, "memory_to_active": 2.3786461353302, "contribution_to_active": 0.48273974657058716, "mix_mean": 0.21285180747509003, "mix_low": 0.3587406873703003, "mix_high": 0.0010876655578613281, "read_cosine": 0.29757189750671387, "beta_mean": 0.17890805006027222, "nonfinite": 0.0, "weighted_memory_to_active": 0.3193712532520294, "mix_std": 0.11651863157749176, "beta_std": 0.12883546948432922, "beta_low": 0.3459879755973816, "beta_high": 0.00011189778160769492, "orthogonal_to_active": 0.043343208730220795, "output_influence": 0.24676060676574707, "residual_influence": 0.06546101719141006, "state_active": 0.029019905254244804, "state_memory": 0.03538380563259125, "state_peak": 37.837425231933594, "growth_active": 1.1946033239364624, "growth_memory": 1.456182599067688, "heads": [{"head": 0, "memory_to_active": 0.030518677085638046, "contribution_to_active": 0.4867894947528839, "mix_mean": 0.5163182020187378, "mix_low": 0.006999015808105469, "mix_high": 0.001922607421875, "read_cosine": -0.004066460765898228, "beta_mean": 0.00021587146329693496, "nonfinite": 0.0, "weighted_memory_to_active": 0.01811540126800537, "mix_std": 0.28274524211883545, "beta_std": 0.0003139999171253294, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.018072765320539474, "evidence_weights": [-0.20858116447925568, -0.2707762122154236, 0.3695911169052124], "agreement_weight": 0.197829931974411, "orthogonal_weight": -1.0546884536743164}, {"head": 1, "memory_to_active": 2.8571300506591797, "contribution_to_active": 0.29303812980651855, "mix_mean": 0.025097370147705078, "mix_low": 0.4534759521484375, "mix_high": 1.9073486328125e-06, "read_cosine": 0.5220744609832764, "beta_mean": 0.27272820472717285, "nonfinite": 0.0, "weighted_memory_to_active": 0.3299601674079895, "mix_std": 0.0492498055100441, "beta_std": 0.18549659848213196, "beta_low": 0.00347900390625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.019256962463259697, "evidence_weights": [0.01879057101905346, -0.05748657509684563, 0.05296821519732475], "agreement_weight": -0.0020077431108802557, "orthogonal_weight": 0.1754014492034912}, {"head": 2, "memory_to_active": 0.06387240439653397, "contribution_to_active": 0.5579390525817871, "mix_mean": 0.6698603630065918, "mix_low": 0.000423431396484375, "mix_high": 0.0045986175537109375, "read_cosine": 0.48478227853775024, "beta_mean": 0.002666021464392543, "nonfinite": 0.0, "weighted_memory_to_active": 0.045023057609796524, "mix_std": 0.22960279881954193, "beta_std": 0.00541313923895359, "beta_low": 0.9637451171875, "beta_high": 0.0, "orthogonal_to_active": 0.030584964901208878, "evidence_weights": [-0.2815665900707245, -0.37259167432785034, 0.6489309072494507], "agreement_weight": 0.038818467408418655, "orthogonal_weight": -0.7895641326904297}, {"head": 3, "memory_to_active": 2.9488685131073, "contribution_to_active": 0.161718487739563, "mix_mean": 0.022410260513424873, "mix_low": 0.5719280242919922, "mix_high": 1.9073486328125e-06, "read_cosine": 0.18543440103530884, "beta_mean": 0.3191762864589691, "nonfinite": 0.0, "weighted_memory_to_active": 0.22043603658676147, "mix_std": 0.06048090010881424, "beta_std": 0.2545010447502136, "beta_low": 0.05596923828125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.09006547927856445, "evidence_weights": [0.03929156810045242, -0.01728815585374832, 0.01157290954142809], "agreement_weight": -0.02807352878153324, "orthogonal_weight": -0.47800877690315247}, {"head": 4, "memory_to_active": 3.80869722366333, "contribution_to_active": 1.314359426498413, "mix_mean": 0.03664562851190567, "mix_low": 0.28830718994140625, "mix_high": 9.5367431640625e-07, "read_cosine": 0.32100099325180054, "beta_mean": 0.18536628782749176, "nonfinite": 0.0, "weighted_memory_to_active": 1.2448010444641113, "mix_std": 0.06187645345926285, "beta_std": 0.12667399644851685, "beta_low": 0.0235595703125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0815306305885315, "evidence_weights": [-0.03868277370929718, -0.010603658854961395, 0.09783326089382172], "agreement_weight": -0.015118884854018688, "orthogonal_weight": 0.09253965318202972}, {"head": 5, "memory_to_active": 4.562788963317871, "contribution_to_active": 0.08259393274784088, "mix_mean": 0.006778940558433533, "mix_low": 0.8313102722167969, "mix_high": 0.0, "read_cosine": 0.2762055993080139, "beta_mean": 0.29329556226730347, "nonfinite": 0.0, "weighted_memory_to_active": 0.05789187178015709, "mix_std": 0.015156581997871399, "beta_std": 0.20061396062374115, "beta_low": 0.0291748046875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.020548442378640175, "evidence_weights": [0.15941938757896423, 0.08200932294130325, 0.10981577634811401], "agreement_weight": -0.12095363438129425, "orthogonal_weight": 0.3353597819805145}]}, {"layer": 2, "memory_to_active": 1.0197083950042725, "contribution_to_active": 0.34395813941955566, "mix_mean": 0.4000931978225708, "mix_low": 0.24476274847984314, "mix_high": 0.004516442772001028, "read_cosine": 0.2975522577762604, "beta_mean": 0.08747197687625885, "nonfinite": 0.0, "weighted_memory_to_active": 0.07409189641475677, "mix_std": 0.1897849589586258, "beta_std": 0.06888121366500854, "beta_low": 0.581207275390625, "beta_high": 8.138021075865254e-05, "orthogonal_to_active": 0.01930297166109085, "output_influence": 0.3841888904571533, "residual_influence": 0.10518637299537659, "state_active": 0.039393529295921326, "state_memory": 0.015288137830793858, "state_peak": 27.27021026611328, "growth_active": 1.0554683208465576, "growth_memory": 1.5590412616729736, "heads": [{"head": 0, "memory_to_active": 0.011808172799646854, "contribution_to_active": 0.3295811116695404, "mix_mean": 0.6303774118423462, "mix_low": 0.00046062469482421875, "mix_high": 0.009219169616699219, "read_cosine": 0.14061900973320007, "beta_mean": 8.964835433289409e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007861031219363213, "mix_std": 0.2694191038608551, "beta_std": 0.00017435448535252362, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008125925436615944, "evidence_weights": [-0.3715224862098694, -0.21871666610240936, 0.15553238987922668], "agreement_weight": 0.1390596181154251, "orthogonal_weight": -1.0806002616882324}, {"head": 1, "memory_to_active": 3.0481114387512207, "contribution_to_active": 0.049161702394485474, "mix_mean": 0.005093575455248356, "mix_low": 0.8899145126342773, "mix_high": 0.0, "read_cosine": 0.1184605285525322, "beta_mean": 0.23599153757095337, "nonfinite": 0.0, "weighted_memory_to_active": 0.03864988312125206, "mix_std": 0.011459414847195148, "beta_std": 0.15419551730155945, "beta_low": 0.00152587890625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.012686808593571186, "evidence_weights": [0.14038559794425964, 0.11835599690675735, 0.5670416355133057], "agreement_weight": 0.032341521233320236, "orthogonal_weight": 0.3638440668582916}, {"head": 2, "memory_to_active": 0.00574234826490283, "contribution_to_active": 0.4092445969581604, "mix_mean": 0.5368025302886963, "mix_low": 0.001056671142578125, "mix_high": 0.0033845901489257812, "read_cosine": 0.3446096181869507, "beta_mean": 1.6775535186752677e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002302789129316807, "mix_std": 0.2769501805305481, "beta_std": 2.8500237021944486e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00014454309712164104, "evidence_weights": [-0.250187486410141, -0.2917504906654358, 0.5746839046478271], "agreement_weight": 0.0007289608474820852, "orthogonal_weight": -0.07474315911531448}, {"head": 3, "memory_to_active": 0.11680164188146591, "contribution_to_active": 0.5408186316490173, "mix_mean": 0.5657106637954712, "mix_low": 0.00024318695068359375, "mix_high": 0.0062198638916015625, "read_cosine": 0.6051437854766846, "beta_mean": 0.0221085324883461, "nonfinite": 0.0, "weighted_memory_to_active": 0.0743287056684494, "mix_std": 0.2609803080558777, "beta_std": 0.029100503772497177, "beta_low": 0.4554443359375, "beta_high": 0.0, "orthogonal_to_active": 0.05093429982662201, "evidence_weights": [-0.2031049132347107, -0.319682240486145, 0.7952836751937866], "agreement_weight": 0.05509868636727333, "orthogonal_weight": -0.8944164514541626}, {"head": 4, "memory_to_active": 2.926177740097046, "contribution_to_active": 0.22489146888256073, "mix_mean": 0.02132568135857582, "mix_low": 0.5758323669433594, "mix_high": 0.0, "read_cosine": 0.21369147300720215, "beta_mean": 0.2665904462337494, "nonfinite": 0.0, "weighted_memory_to_active": 0.3162826597690582, "mix_std": 0.053527750074863434, "beta_std": 0.22973698377609253, "beta_low": 0.0302734375, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.043069493025541306, "evidence_weights": [0.020804909989237785, 0.002741304924711585, 0.038438860327005386], "agreement_weight": 0.017108911648392677, "orthogonal_weight": -0.1638946235179901}, {"head": 5, "memory_to_active": 0.009608613327145576, "contribution_to_active": 0.5100511908531189, "mix_mean": 0.6412492990493774, "mix_low": 0.0010690689086914062, "mix_high": 0.008275032043457031, "read_cosine": 0.3627890348434448, "beta_mean": 3.48667599610053e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005126266740262508, "mix_std": 0.2663729190826416, "beta_std": 5.1422353863017634e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008567540207877755, "evidence_weights": [-0.27800652384757996, -0.2888147830963135, 1.020782709121704], "agreement_weight": 0.17507518827915192, "orthogonal_weight": -0.20529833436012268}]}, {"layer": 3, "memory_to_active": 0.4471065402030945, "contribution_to_active": 0.46794670820236206, "mix_mean": 0.4950537085533142, "mix_low": 0.058837734162807465, "mix_high": 0.009687742218375206, "read_cosine": 0.21855859458446503, "beta_mean": 0.026056572794914246, "nonfinite": 0.0, "weighted_memory_to_active": 0.12396515905857086, "mix_std": 0.25911402702331543, "beta_std": 0.02432716265320778, "beta_low": 0.7686361074447632, "beta_high": 0.0, "orthogonal_to_active": 0.05106643959879875, "output_influence": 0.39740318059921265, "residual_influence": 0.09366324543952942, "state_active": 0.02601378783583641, "state_memory": 0.007113337516784668, "state_peak": 23.257286071777344, "growth_active": 1.0826148986816406, "growth_memory": 1.6363779306411743, "heads": [{"head": 0, "memory_to_active": 0.3652307391166687, "contribution_to_active": 0.5773484110832214, "mix_mean": 0.5877129435539246, "mix_low": 0.00225830078125, "mix_high": 0.021648406982421875, "read_cosine": 0.2896711826324463, "beta_mean": 0.009499434381723404, "nonfinite": 0.0, "weighted_memory_to_active": 0.22821687161922455, "mix_std": 0.3044072389602661, "beta_std": 0.011301051825284958, "beta_low": 0.7066650390625, "beta_high": 0.0, "orthogonal_to_active": 0.15856771171092987, "evidence_weights": [-0.2157420516014099, -0.24934594333171844, 0.9560423493385315], "agreement_weight": 0.11077610403299332, "orthogonal_weight": -0.755603551864624}, {"head": 1, "memory_to_active": 0.015182605013251305, "contribution_to_active": 0.4519639015197754, "mix_mean": 0.5620700120925903, "mix_low": 0.005671501159667969, "mix_high": 0.007683753967285156, "read_cosine": 0.18613117933273315, "beta_mean": 7.778704457450658e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008935044519603252, "mix_std": 0.29831165075302124, "beta_std": 0.00019231587066315114, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007421533577144146, "evidence_weights": [-0.26734012365341187, -0.2163730412721634, 0.6328243613243103], "agreement_weight": 0.5094764828681946, "orthogonal_weight": -0.901817798614502}, {"head": 2, "memory_to_active": 2.0470707416534424, "contribution_to_active": 0.26766765117645264, "mix_mean": 0.04958716779947281, "mix_low": 0.33030033111572266, "mix_high": 0.0, "read_cosine": 0.21843908727169037, "beta_mean": 0.14149276912212372, "nonfinite": 0.0, "weighted_memory_to_active": 0.31573933362960815, "mix_std": 0.09910587966442108, "beta_std": 0.12441389262676239, "beta_low": 0.02423095703125, "beta_high": 0.0, "orthogonal_to_active": 0.004524024203419685, "evidence_weights": [-0.09235098958015442, 0.0019902533385902643, -0.019109396263957024], "agreement_weight": -0.008298512548208237, "orthogonal_weight": -0.01840844564139843}, {"head": 3, "memory_to_active": 0.02029549330472946, "contribution_to_active": 0.5049512386322021, "mix_mean": 0.5947855710983276, "mix_low": 0.005654335021972656, "mix_high": 0.0091400146484375, "read_cosine": 0.29903677105903625, "beta_mean": 8.773419540375471e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013627396896481514, "mix_std": 0.2902264893054962, "beta_std": 0.00019699755648616701, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003638586262241006, "evidence_weights": [-0.2529566287994385, -0.22962525486946106, 2.2015089988708496], "agreement_weight": 2.0760998725891113, "orthogonal_weight": -0.35295045375823975}, {"head": 4, "memory_to_active": 0.011377232149243355, "contribution_to_active": 0.34071797132492065, "mix_mean": 0.46483999490737915, "mix_low": 0.0068645477294921875, "mix_high": 0.0013484954833984375, "read_cosine": 0.182034432888031, "beta_mean": 6.454217509599403e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006997199729084969, "mix_std": 0.29043054580688477, "beta_std": 8.88671274879016e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006614940706640482, "evidence_weights": [-0.21020622551441193, -0.2053869068622589, 0.10325247049331665], "agreement_weight": -0.05920163169503212, "orthogonal_weight": -1.0143941640853882}, {"head": 5, "memory_to_active": 0.22348228096961975, "contribution_to_active": 0.6650310158729553, "mix_mean": 0.7113264203071594, "mix_low": 0.002277374267578125, "mix_high": 0.01830577850341797, "read_cosine": 0.1360388845205307, "beta_mean": 0.005117164459079504, "nonfinite": 0.0, "weighted_memory_to_active": 0.17027507722377777, "mix_std": 0.27220219373703003, "beta_std": 0.009769857861101627, "beta_low": 0.88092041015625, "beta_high": 0.0, "orthogonal_to_active": 0.12563183903694153, "evidence_weights": [-0.2607157528400421, -0.3339645564556122, 0.4986308813095093], "agreement_weight": 0.1837947964668274, "orthogonal_weight": -0.7723058462142944}]}, {"layer": 4, "memory_to_active": 1.081431269645691, "contribution_to_active": 0.4784053862094879, "mix_mean": 0.36420154571533203, "mix_low": 0.2656814455986023, "mix_high": 0.0113805141299963, "read_cosine": 0.3051946759223938, "beta_mean": 0.12335868924856186, "nonfinite": 0.0, "weighted_memory_to_active": 0.14637497067451477, "mix_std": 0.16017654538154602, "beta_std": 0.11140292882919312, "beta_low": 0.3813069760799408, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05420432239770889, "output_influence": 0.4745638370513916, "residual_influence": 0.13936185836791992, "state_active": 0.03660783916711807, "state_memory": 0.018322553485631943, "state_peak": 37.43538284301758, "growth_active": 1.127021074295044, "growth_memory": 1.5181318521499634, "heads": [{"head": 0, "memory_to_active": 0.19721171259880066, "contribution_to_active": 0.7756510376930237, "mix_mean": 0.6822152733802795, "mix_low": 0.014435768127441406, "mix_high": 0.030729293823242188, "read_cosine": 0.4736451804637909, "beta_mean": 0.270374596118927, "nonfinite": 0.0, "weighted_memory_to_active": 0.16582950949668884, "mix_std": 0.34933534264564514, "beta_std": 0.26637351512908936, "beta_low": 0.24188232421875, "beta_high": 0.0, "orthogonal_to_active": 0.08417627960443497, "evidence_weights": [-0.26342645287513733, -0.367178350687027, -0.008877809159457684], "agreement_weight": -0.004396858159452677, "orthogonal_weight": 0.6350807547569275}, {"head": 1, "memory_to_active": 0.05301295593380928, "contribution_to_active": 0.6534298062324524, "mix_mean": 0.6892613768577576, "mix_low": 0.00014209747314453125, "mix_high": 0.018228530883789062, "read_cosine": 0.6075677871704102, "beta_mean": 0.0014089527539908886, "nonfinite": 0.0, "weighted_memory_to_active": 0.03854592144489288, "mix_std": 0.24199149012565613, "beta_std": 0.010957484133541584, "beta_low": 0.98004150390625, "beta_high": 0.0, "orthogonal_to_active": 0.02131616696715355, "evidence_weights": [-0.23985528945922852, -0.36094725131988525, 0.6405401825904846], "agreement_weight": 0.29952993988990784, "orthogonal_weight": -0.7707338333129883}, {"head": 2, "memory_to_active": 1.8851418495178223, "contribution_to_active": 0.3292613923549652, "mix_mean": 0.03270302340388298, "mix_low": 0.47496509552001953, "mix_high": 3.0517578125e-05, "read_cosine": 0.2069290578365326, "beta_mean": 0.07774020731449127, "nonfinite": 0.0, "weighted_memory_to_active": 0.28841522336006165, "mix_std": 0.06872394680976868, "beta_std": 0.08115261048078537, "beta_low": 0.05010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.07495936751365662, "evidence_weights": [0.0509926863014698, -0.013221696019172668, 0.22127094864845276], "agreement_weight": 0.13347004354000092, "orthogonal_weight": -0.3622591197490692}, {"head": 3, "memory_to_active": 1.9405597448349, "contribution_to_active": 0.14150846004486084, "mix_mean": 0.0302914809435606, "mix_low": 0.43748950958251953, "mix_high": 0.0, "read_cosine": 0.19646745920181274, "beta_mean": 0.1573960930109024, "nonfinite": 0.0, "weighted_memory_to_active": 0.20096448063850403, "mix_std": 0.05416807532310486, "beta_std": 0.13653171062469482, "beta_low": 0.01580810546875, "beta_high": 0.0, "orthogonal_to_active": 0.0012701357482001185, "evidence_weights": [-0.02111710235476494, 0.050686996430158615, -0.2748168408870697], "agreement_weight": 0.07230528444051743, "orthogonal_weight": -0.00967489369213581}, {"head": 4, "memory_to_active": 0.011756548658013344, "contribution_to_active": 0.6485255360603333, "mix_mean": 0.7374580502510071, "mix_low": 9.1552734375e-05, "mix_high": 0.01928997039794922, "read_cosine": 0.11536865681409836, "beta_mean": 2.180000774387736e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008139719255268574, "mix_std": 0.2178962081670761, "beta_std": 6.228058919077739e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004632771480828524, "evidence_weights": [-0.33079075813293457, -0.3209152817726135, 0.4385470151901245], "agreement_weight": 0.32924067974090576, "orthogonal_weight": -0.6040835380554199}, {"head": 5, "memory_to_active": 2.4009041786193848, "contribution_to_active": 0.32205599546432495, "mix_mean": 0.013279858976602554, "mix_low": 0.6669645309448242, "mix_high": 4.76837158203125e-06, "read_cosine": 0.23118969798088074, "beta_mean": 0.2332104742527008, "nonfinite": 0.0, "weighted_memory_to_active": 0.17635497450828552, "mix_std": 0.028944212943315506, "beta_std": 0.17333999276161194, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1388712078332901, "evidence_weights": [0.05440423637628555, -0.02265075407922268, -0.2748035788536072], "agreement_weight": -0.08726359903812408, "orthogonal_weight": 0.9975425601005554}]}, {"layer": 5, "memory_to_active": 0.42702770233154297, "contribution_to_active": 0.5299100875854492, "mix_mean": 0.5042240619659424, "mix_low": 0.059907279908657074, "mix_high": 0.019950708374381065, "read_cosine": 0.058783598244190216, "beta_mean": 0.054073136299848557, "nonfinite": 0.0, "weighted_memory_to_active": 0.13891065120697021, "mix_std": 0.25508251786231995, "beta_std": 0.06965592503547668, "beta_low": 0.6666768789291382, "beta_high": 0.0, "orthogonal_to_active": 0.055923476815223694, "output_influence": 0.5543282628059387, "residual_influence": 0.16222718358039856, "state_active": 0.04367274418473244, "state_memory": 0.010978997685015202, "state_peak": 61.2505989074707, "growth_active": 1.1278808116912842, "growth_memory": 1.2517023086547852, "heads": [{"head": 0, "memory_to_active": 0.24743300676345825, "contribution_to_active": 0.5325073003768921, "mix_mean": 0.43952953815460205, "mix_low": 0.014822006225585938, "mix_high": 0.0016260147094726562, "read_cosine": -0.10795256495475769, "beta_mean": 0.02290668897330761, "nonfinite": 0.0, "weighted_memory_to_active": 0.08039962500333786, "mix_std": 0.2821882367134094, "beta_std": 0.042162396013736725, "beta_low": 0.5045166015625, "beta_high": 0.0, "orthogonal_to_active": 0.0326395183801651, "evidence_weights": [-0.15462814271450043, -0.17455057799816132, 0.06468476355075836], "agreement_weight": -0.027060294523835182, "orthogonal_weight": -0.4191623330116272}, {"head": 1, "memory_to_active": 0.009596330113708973, "contribution_to_active": 0.5467000007629395, "mix_mean": 0.6944253444671631, "mix_low": 0.0017366409301757812, "mix_high": 0.036510467529296875, "read_cosine": 0.1888790726661682, "beta_mean": 2.4008611944736913e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005563085433095694, "mix_std": 0.27483606338500977, "beta_std": 4.591570905176923e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0052945297211408615, "evidence_weights": [-0.24922534823417664, -0.28776583075523376, 0.9060047268867493], "agreement_weight": 0.686147928237915, "orthogonal_weight": -0.9930222034454346}, {"head": 2, "memory_to_active": 0.012522093951702118, "contribution_to_active": 0.5944113731384277, "mix_mean": 0.6583241820335388, "mix_low": 0.0024576187133789062, "mix_high": 0.023223876953125, "read_cosine": -0.5672031044960022, "beta_mean": 8.252117550000548e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007909730076789856, "mix_std": 0.2658137083053589, "beta_std": 0.00014794310845900327, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004614950157701969, "evidence_weights": [-0.2655835449695587, -0.3352710008621216, 0.1991407722234726], "agreement_weight": -0.07393813133239746, "orthogonal_weight": -0.7393166422843933}, {"head": 3, "memory_to_active": 1.7370555400848389, "contribution_to_active": 0.2402161955833435, "mix_mean": 0.10957394540309906, "mix_low": 0.13562488555908203, "mix_high": 9.5367431640625e-07, "read_cosine": 0.14782901108264923, "beta_mean": 0.10575002431869507, "nonfinite": 0.0, "weighted_memory_to_active": 0.3746705949306488, "mix_std": 0.14530740678310394, "beta_std": 0.15502247214317322, "beta_low": 0.19891357421875, "beta_high": 0.0, "orthogonal_to_active": 0.1494717299938202, "evidence_weights": [-0.0714493989944458, -0.09837403148412704, 0.046318504959344864], "agreement_weight": 0.04231223464012146, "orthogonal_weight": -0.4969176650047302}, {"head": 4, "memory_to_active": 0.4619494080543518, "contribution_to_active": 0.5796359181404114, "mix_mean": 0.3195411264896393, "mix_low": 0.2030477523803711, "mix_high": 0.00649261474609375, "read_cosine": 0.5430270433425903, "beta_mean": 0.19541142880916595, "nonfinite": 0.0, "weighted_memory_to_active": 0.2984945774078369, "mix_std": 0.33824092149734497, "beta_std": 0.21934683620929718, "beta_low": 0.300048828125, "beta_high": 0.0, "orthogonal_to_active": 0.10447156429290771, "evidence_weights": [-0.11831187456846237, -0.26490336656570435, -0.05487440153956413], "agreement_weight": 0.03820327669382095, "orthogonal_weight": 0.5332388281822205}, {"head": 5, "memory_to_active": 0.0936097726225853, "contribution_to_active": 0.6859896183013916, "mix_mean": 0.8039500713348389, "mix_low": 0.0017547607421875, "mix_high": 0.051850318908691406, "read_cosine": 0.14812208712100983, "beta_mean": 0.0002641399041749537, "nonfinite": 0.0, "weighted_memory_to_active": 0.06642631441354752, "mix_std": 0.22410869598388672, "beta_std": 0.0012099964078515768, "beta_low": 0.99658203125, "beta_high": 0.0, "orthogonal_to_active": 0.03904855251312256, "evidence_weights": [-0.31249383091926575, -0.3346059322357178, 1.9104217290878296], "agreement_weight": 0.4720596969127655, "orthogonal_weight": -0.6295064091682434}]}, {"layer": 6, "memory_to_active": 0.29277920722961426, "contribution_to_active": 0.6210770606994629, "mix_mean": 0.5873937010765076, "mix_low": 0.04927396774291992, "mix_high": 0.03181489557027817, "read_cosine": 0.3186717629432678, "beta_mean": 0.06361802667379379, "nonfinite": 0.0, "weighted_memory_to_active": 0.1516851782798767, "mix_std": 0.2377282828092575, "beta_std": 0.07408124208450317, "beta_low": 0.567047119140625, "beta_high": 0.0, "orthogonal_to_active": 0.048030830919742584, "output_influence": 0.5527399182319641, "residual_influence": 0.18729537725448608, "state_active": 0.0473674051463604, "state_memory": 0.0077413590624928474, "state_peak": 94.88126373291016, "growth_active": 1.203023076057434, "growth_memory": 1.1617563962936401, "heads": [{"head": 0, "memory_to_active": 0.11981038004159927, "contribution_to_active": 0.7525351643562317, "mix_mean": 0.8241732716560364, "mix_low": 0.0001621246337890625, "mix_high": 0.05463409423828125, "read_cosine": 0.2455180138349533, "beta_mean": 0.014035314321517944, "nonfinite": 0.0, "weighted_memory_to_active": 0.10098851472139359, "mix_std": 0.19765907526016235, "beta_std": 0.03225046023726463, "beta_low": 0.70819091796875, "beta_high": 0.0, "orthogonal_to_active": 0.0739533007144928, "evidence_weights": [-0.3139974772930145, -0.35060346126556396, 1.2325971126556396], "agreement_weight": 0.5896977186203003, "orthogonal_weight": -0.79338538646698}, {"head": 1, "memory_to_active": 0.005108384415507317, "contribution_to_active": 0.4991586208343506, "mix_mean": 0.5806593298912048, "mix_low": 0.0012340545654296875, "mix_high": 0.015400886535644531, "read_cosine": 0.5082629919052124, "beta_mean": 1.10805358417565e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022246569860726595, "mix_std": 0.287849098443985, "beta_std": 2.282732748426497e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012469373177736998, "evidence_weights": [-0.19397886097431183, -0.2966975271701813, 1.1231586933135986], "agreement_weight": 0.08622083812952042, "orthogonal_weight": -0.7348130345344543}, {"head": 2, "memory_to_active": 0.790013313293457, "contribution_to_active": 0.09626967459917068, "mix_mean": 0.042578961700201035, "mix_low": 0.2856616973876953, "mix_high": 0.0, "read_cosine": 0.26274335384368896, "beta_mean": 0.025731896981596947, "nonfinite": 0.0, "weighted_memory_to_active": 0.06709527969360352, "mix_std": 0.06733119487762451, "beta_std": 0.02768293209373951, "beta_low": 0.2060546875, "beta_high": 0.0, "orthogonal_to_active": 0.007321689277887344, "evidence_weights": [0.07551514357328415, -0.04249481111764908, 0.30887719988822937], "agreement_weight": 0.14262300729751587, "orthogonal_weight": -0.13065357506275177}, {"head": 3, "memory_to_active": 0.13505271077156067, "contribution_to_active": 0.6654141545295715, "mix_mean": 0.6521339416503906, "mix_low": 0.007401466369628906, "mix_high": 0.021492958068847656, "read_cosine": 0.4725414514541626, "beta_mean": 0.21533988416194916, "nonfinite": 0.0, "weighted_memory_to_active": 0.09456588327884674, "mix_std": 0.32605576515197754, "beta_std": 0.24618083238601685, "beta_low": 0.1905517578125, "beta_high": 0.0, "orthogonal_to_active": 0.05205371230840683, "evidence_weights": [-0.17252184450626373, -0.3250977098941803, 0.17947040498256683], "agreement_weight": -0.009183122776448727, "orthogonal_weight": 0.6305133104324341}, {"head": 4, "memory_to_active": 0.6715531945228577, "contribution_to_active": 1.0915164947509766, "mix_mean": 0.7480535507202148, "mix_low": 0.0009288787841796875, "mix_high": 0.09204673767089844, "read_cosine": 0.1254868507385254, "beta_mean": 0.12639784812927246, "nonfinite": 0.0, "weighted_memory_to_active": 0.6200662851333618, "mix_std": 0.29767894744873047, "beta_std": 0.13799132406711578, "beta_low": 0.2974853515625, "beta_high": 0.0, "orthogonal_to_active": 0.1284996122121811, "evidence_weights": [-0.20908521115779877, -0.5599708557128906, -0.4224720001220703], "agreement_weight": -0.021291138604283333, "orthogonal_weight": 0.22839468717575073}, {"head": 5, "memory_to_active": 0.03513731062412262, "contribution_to_active": 0.6215678453445435, "mix_mean": 0.6767629981040955, "mix_low": 0.000255584716796875, "mix_high": 0.0073146820068359375, "read_cosine": 0.2974778413772583, "beta_mean": 0.00019210945174563676, "nonfinite": 0.0, "weighted_memory_to_active": 0.02517041377723217, "mix_std": 0.24979554116725922, "beta_std": 0.00035909487633034587, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.025109726935625076, "evidence_weights": [-0.23883390426635742, -0.31492170691490173, 0.39283496141433716], "agreement_weight": 0.1949254721403122, "orthogonal_weight": -1.0736243724822998}]}, {"layer": 7, "memory_to_active": 1.7543635368347168, "contribution_to_active": 0.2627516984939575, "mix_mean": 0.16528239846229553, "mix_low": 0.23786449432373047, "mix_high": 0.00113566720392555, "read_cosine": 0.23097342252731323, "beta_mean": 0.14107045531272888, "nonfinite": 0.0, "weighted_memory_to_active": 0.21108388900756836, "mix_std": 0.1481201946735382, "beta_std": 0.10511211305856705, "beta_low": 0.1971435546875, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.08676714450120926, "output_influence": 0.27298077940940857, "residual_influence": 0.06361686438322067, "state_active": 0.013941630721092224, "state_memory": 0.02517838031053543, "state_peak": 29.622329711914062, "growth_active": 1.1714991331100464, "growth_memory": 1.309265375137329, "heads": [{"head": 0, "memory_to_active": 2.3903300762176514, "contribution_to_active": 0.18918362259864807, "mix_mean": 0.03826455399394035, "mix_low": 0.40373897552490234, "mix_high": 0.0, "read_cosine": 0.3615802824497223, "beta_mean": 0.16813570261001587, "nonfinite": 0.0, "weighted_memory_to_active": 0.23278245329856873, "mix_std": 0.07757214456796646, "beta_std": 0.12945525348186493, "beta_low": 0.00250244140625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.00030992593383416533, "evidence_weights": [0.03646920621395111, -0.07695996761322021, 0.022015085443854332], "agreement_weight": -0.028995750471949577, "orthogonal_weight": 0.0018684956012293696}, {"head": 1, "memory_to_active": 0.9846977591514587, "contribution_to_active": 0.3896651268005371, "mix_mean": 0.2233046293258667, "mix_low": 0.1143026351928711, "mix_high": 0.006072998046875, "read_cosine": 0.4032348096370697, "beta_mean": 0.10043878108263016, "nonfinite": 0.0, "weighted_memory_to_active": 0.3338298499584198, "mix_std": 0.24505089223384857, "beta_std": 0.06733877211809158, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04047992825508118, "evidence_weights": [-0.21678361296653748, -0.10334388166666031, -0.277817964553833], "agreement_weight": 0.04866352677345276, "orthogonal_weight": 0.1460353583097458}, {"head": 2, "memory_to_active": 0.7360774278640747, "contribution_to_active": 0.3584865927696228, "mix_mean": 0.20542781054973602, "mix_low": 0.02960968017578125, "mix_high": 0.0003032684326171875, "read_cosine": 0.09348984062671661, "beta_mean": 0.05207211896777153, "nonfinite": 0.0, "weighted_memory_to_active": 0.24557149410247803, "mix_std": 0.20599579811096191, "beta_std": 0.047918401658535004, "beta_low": 0.063720703125, "beta_high": 0.0, "orthogonal_to_active": 0.1312454789876938, "evidence_weights": [-0.0826338604092598, -0.1661689728498459, 0.8072161078453064], "agreement_weight": 0.023179607465863228, "orthogonal_weight": -0.5489175915718079}, {"head": 3, "memory_to_active": 4.660556793212891, "contribution_to_active": 0.11443813145160675, "mix_mean": 0.036641381680965424, "mix_low": 0.3204374313354492, "mix_high": 0.0, "read_cosine": 0.0919647216796875, "beta_mean": 0.439797580242157, "nonfinite": 0.0, "weighted_memory_to_active": 0.3600449562072754, "mix_std": 0.057233475148677826, "beta_std": 0.2913495898246765, "beta_low": 0.00244140625, "beta_high": 0.00146484375, "orthogonal_to_active": 0.3051278591156006, "evidence_weights": [0.06931382417678833, -0.032970644533634186, 0.716157853603363], "agreement_weight": 0.053420476615428925, "orthogonal_weight": -0.9278228878974915}, {"head": 4, "memory_to_active": 1.69181227684021, "contribution_to_active": 0.08033046871423721, "mix_mean": 0.022142138332128525, "mix_low": 0.5566377639770508, "mix_high": 0.0, "read_cosine": 0.21735779941082, "beta_mean": 0.0856858342885971, "nonfinite": 0.0, "weighted_memory_to_active": 0.0636817067861557, "mix_std": 0.0455356240272522, "beta_std": 0.09378162026405334, "beta_low": 0.1153564453125, "beta_high": 0.0, "orthogonal_to_active": 0.014580786228179932, "evidence_weights": [0.13963527977466583, -0.005369539838284254, 0.19370326399803162], "agreement_weight": 0.042417775839567184, "orthogonal_weight": -0.2676125764846802}, {"head": 5, "memory_to_active": 0.06270675361156464, "contribution_to_active": 0.4444059431552887, "mix_mean": 0.4659138321876526, "mix_low": 0.002460479736328125, "mix_high": 0.00043773651123046875, "read_cosine": 0.21821311116218567, "beta_mean": 0.00029268526122905314, "nonfinite": 0.0, "weighted_memory_to_active": 0.03059283085167408, "mix_std": 0.25733327865600586, "beta_std": 0.0008289829129353166, "beta_low": 0.99884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.028858814388513565, "evidence_weights": [-0.1720142364501953, -0.22818464040756226, 0.5630549192428589], "agreement_weight": 0.1977871209383011, "orthogonal_weight": -1.0013984441757202}]}, {"layer": 8, "memory_to_active": 0.1752454936504364, "contribution_to_active": 0.5521447658538818, "mix_mean": 0.4749484062194824, "mix_low": 0.013654869049787521, "mix_high": 0.0015662511577829719, "read_cosine": 0.4236625134944916, "beta_mean": 0.01512183342128992, "nonfinite": 0.0, "weighted_memory_to_active": 0.08149220794439316, "mix_std": 0.23562857508659363, "beta_std": 0.02136268839240074, "beta_low": 0.7587993144989014, "beta_high": 0.0, "orthogonal_to_active": 0.07210914045572281, "output_influence": 0.509732723236084, "residual_influence": 0.1754634976387024, "state_active": 0.0471801832318306, "state_memory": 0.005847663152962923, "state_peak": 47.804378509521484, "growth_active": 1.1543470621109009, "growth_memory": 1.1312477588653564, "heads": [{"head": 0, "memory_to_active": 0.05381958186626434, "contribution_to_active": 0.5738206505775452, "mix_mean": 0.6099917888641357, "mix_low": 0.00043487548828125, "mix_high": 0.0018062591552734375, "read_cosine": 0.460550457239151, "beta_mean": 0.0031117754988372326, "nonfinite": 0.0, "weighted_memory_to_active": 0.03841289132833481, "mix_std": 0.2464606761932373, "beta_std": 0.01194459106773138, "beta_low": 0.93359375, "beta_high": 0.0, "orthogonal_to_active": 0.023639611899852753, "evidence_weights": [-0.16738319396972656, -0.3883509337902069, 0.6254806518554688], "agreement_weight": 0.237162783741951, "orthogonal_weight": -0.782170295715332}, {"head": 1, "memory_to_active": 0.08921226114034653, "contribution_to_active": 0.5416088104248047, "mix_mean": 0.5159919261932373, "mix_low": 4.1961669921875e-05, "mix_high": 0.0010833740234375, "read_cosine": 0.13637308776378632, "beta_mean": 0.0012958127772435546, "nonfinite": 0.0, "weighted_memory_to_active": 0.040114667266607285, "mix_std": 0.2358279526233673, "beta_std": 0.00855933129787445, "beta_low": 0.98681640625, "beta_high": 0.0, "orthogonal_to_active": 0.02890857495367527, "evidence_weights": [-0.19464509189128876, -0.2763667404651642, 0.6998264789581299], "agreement_weight": 0.080315962433815, "orthogonal_weight": -0.7504017353057861}, {"head": 2, "memory_to_active": 0.01019463874399662, "contribution_to_active": 0.6698775291442871, "mix_mean": 0.672691822052002, "mix_low": 0.000560760498046875, "mix_high": 0.0032281875610351562, "read_cosine": 0.4152965545654297, "beta_mean": 0.00010589313023956493, "nonfinite": 0.0, "weighted_memory_to_active": 0.007934707216918468, "mix_std": 0.2233642041683197, "beta_std": 0.003023214405402541, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004856312181800604, "evidence_weights": [-0.22437579929828644, -0.34453773498535156, 1.008205533027649], "agreement_weight": 0.2753491997718811, "orthogonal_weight": -0.7064200639724731}, {"head": 3, "memory_to_active": 0.10671471059322357, "contribution_to_active": 0.6404635906219482, "mix_mean": 0.5796282291412354, "mix_low": 0.0008106231689453125, "mix_high": 0.0032091140747070312, "read_cosine": 0.8015867471694946, "beta_mean": 0.008726153522729874, "nonfinite": 0.0, "weighted_memory_to_active": 0.08228777348995209, "mix_std": 0.2567366063594818, "beta_std": 0.008455559611320496, "beta_low": 0.6832275390625, "beta_high": 0.0, "orthogonal_to_active": 0.02633148804306984, "evidence_weights": [-0.1841718703508377, -0.29874664545059204, 0.5543676614761353], "agreement_weight": 0.22712194919586182, "orthogonal_weight": -0.6184759736061096}, {"head": 4, "memory_to_active": 0.39466798305511475, "contribution_to_active": 0.24375130236148834, "mix_mean": 0.20967458188533783, "mix_low": 0.016036033630371094, "mix_high": 0.0, "read_cosine": 0.2264138013124466, "beta_mean": 0.004652821458876133, "nonfinite": 0.0, "weighted_memory_to_active": 0.11751510202884674, "mix_std": 0.18878740072250366, "beta_std": 0.015810128301382065, "beta_low": 0.8890380859375, "beta_high": 0.0, "orthogonal_to_active": 0.027883518487215042, "evidence_weights": [-0.0750708356499672, -0.15836049616336823, 0.41750094294548035], "agreement_weight": 0.063802570104599, "orthogonal_weight": -0.2527834475040436}, {"head": 5, "memory_to_active": 0.39686375856399536, "contribution_to_active": 0.6433462500572205, "mix_mean": 0.26171207427978516, "mix_low": 0.06404495239257812, "mix_high": 7.05718994140625e-05, "read_cosine": 0.5017543435096741, "beta_mean": 0.07283853739500046, "nonfinite": 0.0, "weighted_memory_to_active": 0.20268812775611877, "mix_std": 0.26259472966194153, "beta_std": 0.0803832933306694, "beta_low": 0.0601806640625, "beta_high": 0.0, "orthogonal_to_active": 0.32103532552719116, "evidence_weights": [-0.13132590055465698, -0.15760871767997742, -0.12525416910648346], "agreement_weight": -0.010059275664389133, "orthogonal_weight": 1.8874691724777222}]}, {"layer": 9, "memory_to_active": 0.9668474197387695, "contribution_to_active": 0.6908304691314697, "mix_mean": 0.3610767126083374, "mix_low": 0.08870093524456024, "mix_high": 0.005800406448543072, "read_cosine": 0.34608396887779236, "beta_mean": 0.04877298325300217, "nonfinite": 0.0, "weighted_memory_to_active": 0.38719892501831055, "mix_std": 0.22734346985816956, "beta_std": 0.0402367040514946, "beta_low": 0.5180765986442566, "beta_high": 0.0, "orthogonal_to_active": 0.0388629287481308, "output_influence": 0.5410070419311523, "residual_influence": 0.15586350858211517, "state_active": 0.042425524443387985, "state_memory": 0.016888514161109924, "state_peak": 50.64774703979492, "growth_active": 1.201438307762146, "growth_memory": 1.3277837038040161, "heads": [{"head": 0, "memory_to_active": 0.0005027109873481095, "contribution_to_active": 0.5674260854721069, "mix_mean": 0.6450453400611877, "mix_low": 0.0011472702026367188, "mix_high": 0.024805068969726562, "read_cosine": 0.3345519006252289, "beta_mean": 1.6854007753863698e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0003101603651884943, "mix_std": 0.2868279814720154, "beta_std": 5.000290911993943e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00034637912176549435, "evidence_weights": [-0.30453014373779297, -0.3115616738796234, 0.5465934872627258], "agreement_weight": 0.12000394612550735, "orthogonal_weight": -1.182463526725769}, {"head": 1, "memory_to_active": 0.02597169764339924, "contribution_to_active": 0.6941995620727539, "mix_mean": 0.5681731700897217, "mix_low": 0.007370948791503906, "mix_high": 0.0025920867919921875, "read_cosine": 0.4011404812335968, "beta_mean": 6.761484837625176e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.019631490111351013, "mix_std": 0.2819398045539856, "beta_std": 0.00015200054622255266, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013478262349963188, "evidence_weights": [-0.24016869068145752, -0.2934996485710144, 1.0743387937545776], "agreement_weight": 0.4671146869659424, "orthogonal_weight": -0.8489180207252502}, {"head": 2, "memory_to_active": 0.17581260204315186, "contribution_to_active": 0.6512727737426758, "mix_mean": 0.5907557010650635, "mix_low": 0.0021066665649414062, "mix_high": 0.006198883056640625, "read_cosine": 0.35100674629211426, "beta_mean": 0.0006533113773912191, "nonfinite": 0.0, "weighted_memory_to_active": 0.11921575665473938, "mix_std": 0.2680841088294983, "beta_std": 0.004556721076369286, "beta_low": 0.99578857421875, "beta_high": 0.0, "orthogonal_to_active": 0.06989217549562454, "evidence_weights": [-0.20172876119613647, -0.27655401825904846, 0.7630491256713867], "agreement_weight": 0.3262483775615692, "orthogonal_weight": -0.6863741278648376}, {"head": 3, "memory_to_active": 1.1397368907928467, "contribution_to_active": 0.19213980436325073, "mix_mean": 0.09399853646755219, "mix_low": 0.2017965316772461, "mix_high": 0.00010013580322265625, "read_cosine": 0.3815683424472809, "beta_mean": 0.05956501141190529, "nonfinite": 0.0, "weighted_memory_to_active": 0.17934146523475647, "mix_std": 0.14985951781272888, "beta_std": 0.05222095176577568, "beta_low": 0.0142822265625, "beta_high": 0.0, "orthogonal_to_active": 0.04594140127301216, "evidence_weights": [0.02508038468658924, -0.04808138310909271, 0.17991970479488373], "agreement_weight": 0.09410735219717026, "orthogonal_weight": -0.2801823318004608}, {"head": 4, "memory_to_active": 2.968496084213257, "contribution_to_active": 1.5629687309265137, "mix_mean": 0.14473207294940948, "mix_low": 0.119354248046875, "mix_high": 0.0008487701416015625, "read_cosine": 0.10702499002218246, "beta_mean": 0.15337300300598145, "nonfinite": 0.0, "weighted_memory_to_active": 1.4933269023895264, "mix_std": 0.20229016244411469, "beta_std": 0.08807520568370819, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.024248681962490082, "evidence_weights": [-0.08057033270597458, -0.15524005889892578, -0.01119218673557043], "agreement_weight": -0.061952266842126846, "orthogonal_weight": -0.016751427203416824}, {"head": 5, "memory_to_active": 1.4905647039413452, "contribution_to_active": 0.47697561979293823, "mix_mean": 0.1237553134560585, "mix_low": 0.20042991638183594, "mix_high": 0.0002574920654296875, "read_cosine": 0.5012112855911255, "beta_mean": 0.07897723466157913, "nonfinite": 0.0, "weighted_memory_to_active": 0.5113674402236938, "mix_std": 0.17505910992622375, "beta_std": 0.09641033411026001, "beta_low": 0.098388671875, "beta_high": 0.0, "orthogonal_to_active": 0.07927066087722778, "evidence_weights": [0.09158851206302643, -0.13193461298942566, 0.1412253975868225], "agreement_weight": 0.011898357421159744, "orthogonal_weight": 0.21476416289806366}]}, {"layer": 10, "memory_to_active": 0.21412217617034912, "contribution_to_active": 0.7005873918533325, "mix_mean": 0.5160391330718994, "mix_low": 0.022991977632045746, "mix_high": 0.004183133598417044, "read_cosine": 0.3774802088737488, "beta_mean": 0.04590523988008499, "nonfinite": 0.0, "weighted_memory_to_active": 0.09597666561603546, "mix_std": 0.2680756449699402, "beta_std": 0.04645247384905815, "beta_low": 0.659454345703125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.2381243109703064, "output_influence": 0.4485240876674652, "residual_influence": 0.15563303232192993, "state_active": 0.05725441128015518, "state_memory": 0.016416754573583603, "state_peak": 125.60242462158203, "growth_active": 1.0314010381698608, "growth_memory": 1.004928469657898, "heads": [{"head": 0, "memory_to_active": 0.7844187021255493, "contribution_to_active": 1.9275007247924805, "mix_mean": 0.5544840097427368, "mix_low": 0.10546112060546875, "mix_high": 0.005570411682128906, "read_cosine": 0.10930894315242767, "beta_mean": 0.20336076617240906, "nonfinite": 0.0, "weighted_memory_to_active": 0.46137306094169617, "mix_std": 0.3912038207054138, "beta_std": 0.21305876970291138, "beta_low": 0.1319580078125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.3506669998168945, "evidence_weights": [-0.09546919167041779, -0.38643908500671387, -1.7500418424606323], "agreement_weight": 0.02568690851330757, "orthogonal_weight": 2.9557526111602783}, {"head": 1, "memory_to_active": 0.008814234286546707, "contribution_to_active": 0.6121629476547241, "mix_mean": 0.6241873502731323, "mix_low": 0.0016870498657226562, "mix_high": 0.013876914978027344, "read_cosine": 0.31432247161865234, "beta_mean": 5.768460687249899e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005040857940912247, "mix_std": 0.26652681827545166, "beta_std": 7.441190246026963e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007760194130241871, "evidence_weights": [-0.2526091933250427, -0.19707238674163818, 0.5611205697059631], "agreement_weight": 0.5645806193351746, "orthogonal_weight": -1.6633752584457397}, {"head": 2, "memory_to_active": 0.014826727099716663, "contribution_to_active": 0.6162967681884766, "mix_mean": 0.6529360413551331, "mix_low": 0.00020694732666015625, "mix_high": 0.004307746887207031, "read_cosine": 0.02117365226149559, "beta_mean": 2.9648883355548605e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006423256825655699, "mix_std": 0.2492169886827469, "beta_std": 0.00011616687697824091, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028485520742833614, "evidence_weights": [-0.261325478553772, -0.2803533971309662, 0.6408422589302063], "agreement_weight": -0.20632261037826538, "orthogonal_weight": -0.4245089590549469}, {"head": 3, "memory_to_active": 0.33743852376937866, "contribution_to_active": 0.09742726385593414, "mix_mean": 0.2883363962173462, "mix_low": 0.02777099609375, "mix_high": 0.0001430511474609375, "read_cosine": 0.9267164468765259, "beta_mean": 0.06541765481233597, "nonfinite": 0.0, "weighted_memory_to_active": 0.03911352902650833, "mix_std": 0.22627398371696472, "beta_std": 0.05793588608503342, "beta_low": 0.00201416015625, "beta_high": 0.0, "orthogonal_to_active": 0.03841537609696388, "evidence_weights": [-0.10225581377744675, -0.14850236475467682, 0.1612066626548767], "agreement_weight": 0.015267549082636833, "orthogonal_weight": 1.4097347259521484}, {"head": 4, "memory_to_active": 0.012656472623348236, "contribution_to_active": 0.5542488098144531, "mix_mean": 0.5788484811782837, "mix_low": 0.00024700164794921875, "mix_high": 0.0011949539184570312, "read_cosine": 0.039652757346630096, "beta_mean": 4.0583145164418966e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007297227159142494, "mix_std": 0.24806711077690125, "beta_std": 6.730686436640099e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009936010465025902, "evidence_weights": [-0.18466277420520782, -0.2599943280220032, 0.9882193207740784], "agreement_weight": 0.1464489847421646, "orthogonal_weight": -1.43491792678833}, {"head": 5, "memory_to_active": 0.12657827138900757, "contribution_to_active": 0.3958878219127655, "mix_mean": 0.39744240045547485, "mix_low": 0.0025787353515625, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8537070751190186, "beta_mean": 0.006525062490254641, "nonfinite": 0.0, "weighted_memory_to_active": 0.05661206319928169, "mix_std": 0.22716520726680756, "beta_std": 0.007462278939783573, "beta_low": 0.82275390625, "beta_high": 0.0, "orthogonal_to_active": 0.01911872625350952, "evidence_weights": [-0.13991308212280273, -0.16189271211624146, 0.26754507422447205], "agreement_weight": 0.09121246635913849, "orthogonal_weight": -0.6546463966369629}]}, {"layer": 11, "memory_to_active": 0.08612876385450363, "contribution_to_active": 0.6147337555885315, "mix_mean": 0.5218963623046875, "mix_low": 0.10229191184043884, "mix_high": 0.05019395053386688, "read_cosine": 0.33962076902389526, "beta_mean": 0.026453938335180283, "nonfinite": 0.0, "weighted_memory_to_active": 0.04555415362119675, "mix_std": 0.325216680765152, "beta_std": 0.03843245655298233, "beta_low": 0.6700134873390198, "beta_high": 0.0, "orthogonal_to_active": 0.04842023178935051, "output_influence": 0.5472981333732605, "residual_influence": 0.1458835005760193, "state_active": 0.20303449034690857, "state_memory": 0.00413540517911315, "state_peak": 31.95236587524414, "growth_active": 1.168922781944275, "growth_memory": 1.6004128456115723, "heads": [{"head": 0, "memory_to_active": 0.007307426072657108, "contribution_to_active": 0.6025018692016602, "mix_mean": 0.6484032869338989, "mix_low": 0.0009107589721679688, "mix_high": 0.008846282958984375, "read_cosine": 0.24612963199615479, "beta_mean": 7.544413529103622e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005321796517819166, "mix_std": 0.25810298323631287, "beta_std": 0.0032914024777710438, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.004338241647928953, "evidence_weights": [-0.17308738827705383, -0.3648253381252289, 0.757813572883606], "agreement_weight": 0.3554701507091522, "orthogonal_weight": -0.8437728881835938}, {"head": 1, "memory_to_active": 0.17569604516029358, "contribution_to_active": 0.41776058077812195, "mix_mean": 0.45573750138282776, "mix_low": 0.006320953369140625, "mix_high": 0.0005292892456054688, "read_cosine": 0.4645429253578186, "beta_mean": 0.008751709014177322, "nonfinite": 0.0, "weighted_memory_to_active": 0.08056288957595825, "mix_std": 0.26917386054992676, "beta_std": 0.02425868809223175, "beta_low": 0.8480224609375, "beta_high": 0.0, "orthogonal_to_active": 0.04290147125720978, "evidence_weights": [-0.15661722421646118, -0.24986933171749115, 0.6016696691513062], "agreement_weight": 0.07291149348020554, "orthogonal_weight": -0.6195240020751953}, {"head": 2, "memory_to_active": 0.007622931152582169, "contribution_to_active": 0.985206663608551, "mix_mean": 0.44597703218460083, "mix_low": 0.2838420867919922, "mix_high": 0.17890071868896484, "read_cosine": 0.18453586101531982, "beta_mean": 0.06905947625637054, "nonfinite": 0.0, "weighted_memory_to_active": 0.0055341883562505245, "mix_std": 0.4492587447166443, "beta_std": 0.08868298679590225, "beta_low": 0.46173095703125, "beta_high": 0.0, "orthogonal_to_active": 0.049277693033218384, "evidence_weights": [0.04331527277827263, -0.38624006509780884, -1.3314522504806519], "agreement_weight": -0.0018082446185871959, "orthogonal_weight": 9.247729301452637}, {"head": 3, "memory_to_active": 0.008303452283143997, "contribution_to_active": 0.8522853851318359, "mix_mean": 0.42510974407196045, "mix_low": 0.3135051727294922, "mix_high": 0.10604095458984375, "read_cosine": 0.5423774123191833, "beta_mean": 0.054611045867204666, "nonfinite": 0.0, "weighted_memory_to_active": 0.00515206903219223, "mix_std": 0.44255053997039795, "beta_std": 0.08113227784633636, "beta_low": 0.49859619140625, "beta_high": 0.0, "orthogonal_to_active": 0.036117203533649445, "evidence_weights": [0.198919415473938, -0.2980291545391083, -0.151846781373024], "agreement_weight": 0.05181356891989708, "orthogonal_weight": 9.537224769592285}, {"head": 4, "memory_to_active": 0.22936683893203735, "contribution_to_active": 0.42631927132606506, "mix_mean": 0.5955092310905457, "mix_low": 0.006533622741699219, "mix_high": 0.00165557861328125, "read_cosine": 0.054659802466630936, "beta_mean": 0.016662882640957832, "nonfinite": 0.0, "weighted_memory_to_active": 0.13042771816253662, "mix_std": 0.2716585695743561, "beta_std": 0.01853729784488678, "beta_low": 0.4677734375, "beta_high": 0.0, "orthogonal_to_active": 0.12757140398025513, "evidence_weights": [-0.24197253584861755, -0.23931269347667694, 1.9162160158157349], "agreement_weight": -0.21678951382637024, "orthogonal_weight": -0.9899912476539612}, {"head": 5, "memory_to_active": 0.08847584575414658, "contribution_to_active": 0.4043288230895996, "mix_mean": 0.5606411695480347, "mix_low": 0.0026388168334960938, "mix_high": 0.005190849304199219, "read_cosine": 0.5454789400100708, "beta_mean": 0.009563074447214603, "nonfinite": 0.0, "weighted_memory_to_active": 0.046326227486133575, "mix_std": 0.260555237531662, "beta_std": 0.014692072756588459, "beta_low": 0.744384765625, "beta_high": 0.0, "orthogonal_to_active": 0.030315380543470383, "evidence_weights": [-0.2095772624015808, -0.3133065104484558, 0.6527180075645447], "agreement_weight": 0.08441467583179474, "orthogonal_weight": -0.8527051210403442}]}, {"layer": 12, "memory_to_active": 0.23584729433059692, "contribution_to_active": 0.5799568891525269, "mix_mean": 0.5417464971542358, "mix_low": 0.00885502528399229, "mix_high": 0.006405354011803865, "read_cosine": 0.22969819605350494, "beta_mean": 0.015265392139554024, "nonfinite": 0.0, "weighted_memory_to_active": 0.12417811900377274, "mix_std": 0.25977665185928345, "beta_std": 0.015648186206817627, "beta_low": 0.825927734375, "beta_high": 0.0, "orthogonal_to_active": 0.07364549487829208, "output_influence": 0.49916794896125793, "residual_influence": 0.1395803689956665, "state_active": 0.03283878415822983, "state_memory": 0.008725440129637718, "state_peak": 60.06021499633789, "growth_active": 1.0987485647201538, "growth_memory": 1.2215344905853271, "heads": [{"head": 0, "memory_to_active": 0.04592517763376236, "contribution_to_active": 0.6020039319992065, "mix_mean": 0.5769943594932556, "mix_low": 0.00045490264892578125, "mix_high": 0.004981040954589844, "read_cosine": 0.2858482599258423, "beta_mean": 0.0007263815496116877, "nonfinite": 0.0, "weighted_memory_to_active": 0.02508673071861267, "mix_std": 0.2646523118019104, "beta_std": 0.001403982751071453, "beta_low": 0.99676513671875, "beta_high": 0.0, "orthogonal_to_active": 0.022962912917137146, "evidence_weights": [-0.20879238843917847, -0.2754690945148468, 0.04134324938058853], "agreement_weight": 0.09842640906572342, "orthogonal_weight": -0.9804369211196899}, {"head": 1, "memory_to_active": 0.09736688435077667, "contribution_to_active": 0.5401676297187805, "mix_mean": 0.5570341348648071, "mix_low": 0.00042629241943359375, "mix_high": 0.0019054412841796875, "read_cosine": 0.42551591992378235, "beta_mean": 0.000737025577109307, "nonfinite": 0.0, "weighted_memory_to_active": 0.059207431972026825, "mix_std": 0.26497361063957214, "beta_std": 0.009485691785812378, "beta_low": 0.99615478515625, "beta_high": 0.0, "orthogonal_to_active": 0.04850362613797188, "evidence_weights": [-0.20358902215957642, -0.27210238575935364, 1.0371984243392944], "agreement_weight": 0.2291051149368286, "orthogonal_weight": -0.9139888286590576}, {"head": 2, "memory_to_active": 0.6145722270011902, "contribution_to_active": 0.5092556476593018, "mix_mean": 0.32064083218574524, "mix_low": 0.02705669403076172, "mix_high": 2.09808349609375e-05, "read_cosine": 0.5131255388259888, "beta_mean": 0.08852657675743103, "nonfinite": 0.0, "weighted_memory_to_active": 0.24542362987995148, "mix_std": 0.23406928777694702, "beta_std": 0.06223567575216293, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.26799190044403076, "evidence_weights": [-0.21542909741401672, -0.05737490579485893, 0.008676491677761078], "agreement_weight": 0.03153716400265694, "orthogonal_weight": 1.352115511894226}, {"head": 3, "memory_to_active": 0.03001425415277481, "contribution_to_active": 0.7452584505081177, "mix_mean": 0.7191408276557922, "mix_low": 0.0001544952392578125, "mix_high": 0.020704269409179688, "read_cosine": 0.0954488217830658, "beta_mean": 0.00023653273819945753, "nonfinite": 0.0, "weighted_memory_to_active": 0.02224574238061905, "mix_std": 0.22850802540779114, "beta_std": 0.005686210468411446, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.021289292722940445, "evidence_weights": [-0.28976351022720337, -0.33660003542900085, 1.6878275871276855], "agreement_weight": 0.51016765832901, "orthogonal_weight": -0.9780975580215454}, {"head": 4, "memory_to_active": 0.5352064371109009, "contribution_to_active": 0.5881867408752441, "mix_mean": 0.43966802954673767, "mix_low": 0.01807117462158203, "mix_high": 0.0048465728759765625, "read_cosine": -0.14741498231887817, "beta_mean": 0.0007575313211418688, "nonfinite": 0.0, "weighted_memory_to_active": 0.3306070566177368, "mix_std": 0.2905939221382141, "beta_std": 0.008562414906919003, "beta_low": 0.9786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.033127255737781525, "evidence_weights": [-0.15197648108005524, -0.21712858974933624, -0.0940806120634079], "agreement_weight": 0.012314443476498127, "orthogonal_weight": -0.1042926162481308}, {"head": 5, "memory_to_active": 0.09199871867895126, "contribution_to_active": 0.4948689341545105, "mix_mean": 0.6370006799697876, "mix_low": 0.006966590881347656, "mix_high": 0.00597381591796875, "read_cosine": 0.2056656777858734, "beta_mean": 0.000608309346716851, "nonfinite": 0.0, "weighted_memory_to_active": 0.06249813362956047, "mix_std": 0.27586257457733154, "beta_std": 0.006515136919915676, "beta_low": 0.98468017578125, "beta_high": 0.0, "orthogonal_to_active": 0.04799800366163254, "evidence_weights": [-0.23406195640563965, -0.3205818235874176, 1.2850539684295654], "agreement_weight": 0.40739694237709045, "orthogonal_weight": -0.8207592368125916}]}, {"layer": 13, "memory_to_active": 0.11534546315670013, "contribution_to_active": 0.5315475463867188, "mix_mean": 0.5368081331253052, "mix_low": 0.00584729528054595, "mix_high": 0.009445826523005962, "read_cosine": 0.41725635528564453, "beta_mean": 0.0149651188403368, "nonfinite": 0.0, "weighted_memory_to_active": 0.05117131024599075, "mix_std": 0.25859975814819336, "beta_std": 0.01436131726950407, "beta_low": 0.8128560781478882, "beta_high": 0.0, "orthogonal_to_active": 0.05156683921813965, "output_influence": 0.5231180787086487, "residual_influence": 0.1809546798467636, "state_active": 0.1297723948955536, "state_memory": 0.04589548707008362, "state_peak": 94.73707580566406, "growth_active": 0.9606012105941772, "growth_memory": 1.0477672815322876, "heads": [{"head": 0, "memory_to_active": 0.013624561950564384, "contribution_to_active": 0.5821618437767029, "mix_mean": 0.607898473739624, "mix_low": 0.000728607177734375, "mix_high": 0.0038366317749023438, "read_cosine": 0.44949793815612793, "beta_mean": 4.7967652790248394e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00857479590922594, "mix_std": 0.2681209444999695, "beta_std": 9.836759272729978e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008228720165789127, "evidence_weights": [-0.19653832912445068, -0.2680255174636841, 0.78605717420578], "agreement_weight": 0.16398169100284576, "orthogonal_weight": -1.0786960124969482}, {"head": 1, "memory_to_active": 0.010176526382565498, "contribution_to_active": 0.5918487906455994, "mix_mean": 0.5656443238258362, "mix_low": 0.0020036697387695312, "mix_high": 0.004368782043457031, "read_cosine": 0.20174957811832428, "beta_mean": 3.911916428478435e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0055617643520236015, "mix_std": 0.2747318744659424, "beta_std": 6.902067252667621e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0072553385980427265, "evidence_weights": [-0.18183736503124237, -0.26423317193984985, 0.5924878716468811], "agreement_weight": 0.4541906416416168, "orthogonal_weight": -1.3518041372299194}, {"head": 2, "memory_to_active": 0.06063437461853027, "contribution_to_active": 0.4618903398513794, "mix_mean": 0.38459575176239014, "mix_low": 0.0013151168823242188, "mix_high": 0.000171661376953125, "read_cosine": 0.6209849715232849, "beta_mean": 0.00550881028175354, "nonfinite": 0.0, "weighted_memory_to_active": 0.027976127341389656, "mix_std": 0.2318790704011917, "beta_std": 0.02052517980337143, "beta_low": 0.88311767578125, "beta_high": 0.0, "orthogonal_to_active": 0.014816618524491787, "evidence_weights": [-0.18319079279899597, -0.20659340918064117, 0.4013654291629791], "agreement_weight": 0.005754829850047827, "orthogonal_weight": -0.8380420804023743}, {"head": 3, "memory_to_active": 0.1469760239124298, "contribution_to_active": 0.5890683531761169, "mix_mean": 0.6541850566864014, "mix_low": 0.023576736450195312, "mix_high": 0.03676414489746094, "read_cosine": 0.025446778163313866, "beta_mean": 0.0008749106782488525, "nonfinite": 0.0, "weighted_memory_to_active": 0.11388986557722092, "mix_std": 0.28983041644096375, "beta_std": 0.0072099193930625916, "beta_low": 0.99481201171875, "beta_high": 0.0, "orthogonal_to_active": 0.0977977067232132, "evidence_weights": [-0.3176070749759674, -0.24583859741687775, 1.6269863843917847], "agreement_weight": 5.955837726593018, "orthogonal_weight": -0.9654316902160645}, {"head": 4, "memory_to_active": 0.3974820375442505, "contribution_to_active": 0.2587757110595703, "mix_mean": 0.3535236120223999, "mix_low": 0.00499725341796875, "mix_high": 0.0, "read_cosine": 0.8870118856430054, "beta_mean": 0.0830419510602951, "nonfinite": 0.0, "weighted_memory_to_active": 0.1091448962688446, "mix_std": 0.21490779519081116, "beta_std": 0.05522976815700531, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.14615017175674438, "evidence_weights": [-0.08047900348901749, -0.23117730021476746, 0.4670039415359497], "agreement_weight": 0.008457176387310028, "orthogonal_weight": 2.2315361499786377}, {"head": 5, "memory_to_active": 0.06317917257547379, "contribution_to_active": 0.7055400609970093, "mix_mean": 0.6550015211105347, "mix_low": 0.0024623870849609375, "mix_high": 0.011533737182617188, "read_cosine": 0.3188469409942627, "beta_mean": 0.00027794906054623425, "nonfinite": 0.0, "weighted_memory_to_active": 0.04188040643930435, "mix_std": 0.2721284031867981, "beta_std": 0.0030356450006365776, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.03515248745679855, "evidence_weights": [-0.2224046289920807, -0.30060794949531555, 0.8807705640792847], "agreement_weight": 0.19034919142723083, "orthogonal_weight": -0.9202160239219666}]}, {"layer": 14, "memory_to_active": 0.22732405364513397, "contribution_to_active": 0.4190737009048462, "mix_mean": 0.43732208013534546, "mix_low": 0.022640546783804893, "mix_high": 0.0063037872314453125, "read_cosine": 0.6062105894088745, "beta_mean": 0.06394532322883606, "nonfinite": 0.0, "weighted_memory_to_active": 0.09481093287467957, "mix_std": 0.2459850311279297, "beta_std": 0.060458093881607056, "beta_low": 0.4890543818473816, "beta_high": 0.0, "orthogonal_to_active": 0.04234230890870094, "output_influence": 0.3589603304862976, "residual_influence": 0.14159588515758514, "state_active": 0.05772656574845314, "state_memory": 0.015402667224407196, "state_peak": 39.939231872558594, "growth_active": 1.1056989431381226, "growth_memory": 1.2977652549743652, "heads": [{"head": 0, "memory_to_active": 0.33834952116012573, "contribution_to_active": 0.1804836094379425, "mix_mean": 0.2022610604763031, "mix_low": 0.010411262512207031, "mix_high": 0.0, "read_cosine": 0.8013931512832642, "beta_mean": 0.06755577772855759, "nonfinite": 0.0, "weighted_memory_to_active": 0.07171478867530823, "mix_std": 0.17262355983257294, "beta_std": 0.05794069543480873, "beta_low": 0.00799560546875, "beta_high": 0.0, "orthogonal_to_active": 0.07936087250709534, "evidence_weights": [-0.11110317707061768, -0.04638829454779625, 0.42244336009025574], "agreement_weight": 0.02421882562339306, "orthogonal_weight": 1.7261221408843994}, {"head": 1, "memory_to_active": 0.4856237769126892, "contribution_to_active": 0.28068849444389343, "mix_mean": 0.23481124639511108, "mix_low": 0.11003494262695312, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8479030132293701, "beta_mean": 0.2649964988231659, "nonfinite": 0.0, "weighted_memory_to_active": 0.20820532739162445, "mix_std": 0.2574891448020935, "beta_std": 0.2568438649177551, "beta_low": 0.09942626953125, "beta_high": 0.0, "orthogonal_to_active": 0.016096003353595734, "evidence_weights": [-0.006901753135025501, -0.1995040476322174, -0.09317272901535034], "agreement_weight": 0.05862404406070709, "orthogonal_weight": 0.1855861395597458}, {"head": 2, "memory_to_active": 0.13105665147304535, "contribution_to_active": 0.591558575630188, "mix_mean": 0.6016461849212646, "mix_low": 0.0026378631591796875, "mix_high": 0.0045166015625, "read_cosine": 0.40790772438049316, "beta_mean": 0.0021979045122861862, "nonfinite": 0.0, "weighted_memory_to_active": 0.08730073273181915, "mix_std": 0.2659894824028015, "beta_std": 0.003106956370174885, "beta_low": 0.97625732421875, "beta_high": 0.0, "orthogonal_to_active": 0.06756768375635147, "evidence_weights": [-0.13962619006633759, -0.31740549206733704, 0.9286831021308899], "agreement_weight": 0.0865834504365921, "orthogonal_weight": -0.8805298805236816}, {"head": 3, "memory_to_active": 0.051835015416145325, "contribution_to_active": 0.6034340262413025, "mix_mean": 0.6718957424163818, "mix_low": 0.004889488220214844, "mix_high": 0.02178192138671875, "read_cosine": -0.062180034816265106, "beta_mean": 0.0066290199756622314, "nonfinite": 0.0, "weighted_memory_to_active": 0.03350425884127617, "mix_std": 0.25795695185661316, "beta_std": 0.008887830190360546, "beta_low": 0.80889892578125, "beta_high": 0.0, "orthogonal_to_active": 0.0256632249802351, "evidence_weights": [-0.34574657678604126, -0.3102343678474426, 0.2726810574531555], "agreement_weight": 0.07011444866657257, "orthogonal_weight": -0.8431373834609985}, {"head": 4, "memory_to_active": 0.14396706223487854, "contribution_to_active": 0.5168708562850952, "mix_mean": 0.5215494632720947, "mix_low": 0.0038251876831054688, "mix_high": 0.009804725646972656, "read_cosine": 0.8850579261779785, "beta_mean": 0.040705081075429916, "nonfinite": 0.0, "weighted_memory_to_active": 0.08563422411680222, "mix_std": 0.2629706859588623, "beta_std": 0.028197860345244408, "beta_low": 0.06842041015625, "beta_high": 0.0, "orthogonal_to_active": 0.023528311401605606, "evidence_weights": [-0.215292289853096, -0.24274234473705292, 0.43416592478752136], "agreement_weight": 0.13283301889896393, "orthogonal_weight": -0.6183608770370483}, {"head": 5, "memory_to_active": 0.2131122648715973, "contribution_to_active": 0.3414066433906555, "mix_mean": 0.39176860451698303, "mix_low": 0.004044532775878906, "mix_high": 0.00171661376953125, "read_cosine": 0.757181704044342, "beta_mean": 0.0015876624966040254, "nonfinite": 0.0, "weighted_memory_to_active": 0.08250623941421509, "mix_std": 0.2588804066181183, "beta_std": 0.007771342061460018, "beta_low": 0.97332763671875, "beta_high": 0.0, "orthogonal_to_active": 0.04183776304125786, "evidence_weights": [-0.1758459061384201, -0.17616091668605804, 0.3235008120536804], "agreement_weight": 0.0714353397488594, "orthogonal_weight": -0.758635938167572}]}, {"layer": 15, "memory_to_active": 0.05700204148888588, "contribution_to_active": 0.4832560122013092, "mix_mean": 0.4164851903915405, "mix_low": 0.008028666488826275, "mix_high": 0.0025173823814839125, "read_cosine": 0.6599379181861877, "beta_mean": 0.037226684391498566, "nonfinite": 0.0, "weighted_memory_to_active": 0.028144629672169685, "mix_std": 0.28475818037986755, "beta_std": 0.0438651517033577, "beta_low": 0.7135416865348816, "beta_high": 0.0, "orthogonal_to_active": 0.00759506830945611, "output_influence": 0.30329373478889465, "residual_influence": 0.1745409071445465, "state_active": 0.15762653946876526, "state_memory": 0.010989353992044926, "state_peak": 34.35466003417969, "growth_active": 1.2171825170516968, "growth_memory": 1.2424589395523071, "heads": [{"head": 0, "memory_to_active": 0.10013947635889053, "contribution_to_active": 0.48204270005226135, "mix_mean": 0.38348981738090515, "mix_low": 0.0074367523193359375, "mix_high": 0.0042266845703125, "read_cosine": 0.6391388177871704, "beta_mean": 0.034709054976701736, "nonfinite": 0.0, "weighted_memory_to_active": 0.0490022674202919, "mix_std": 0.2812737822532654, "beta_std": 0.027616621926426888, "beta_low": 0.1195068359375, "beta_high": 0.0, "orthogonal_to_active": 0.02342783659696579, "evidence_weights": [-0.18721222877502441, -0.18786688148975372, 0.45746639370918274], "agreement_weight": 0.09375393390655518, "orthogonal_weight": -0.771691083908081}, {"head": 1, "memory_to_active": 0.12034404277801514, "contribution_to_active": 0.4166618585586548, "mix_mean": 0.43128398060798645, "mix_low": 0.001537322998046875, "mix_high": 0.00542449951171875, "read_cosine": 0.926321804523468, "beta_mean": 0.0006512799300253391, "nonfinite": 0.0, "weighted_memory_to_active": 0.05136431008577347, "mix_std": 0.28567075729370117, "beta_std": 0.0026710163801908493, "beta_low": 0.9935302734375, "beta_high": 0.0, "orthogonal_to_active": 0.001409489894285798, "evidence_weights": [-0.14775320887565613, -0.15635418891906738, 0.1741357296705246], "agreement_weight": 0.1256001740694046, "orthogonal_weight": 0.059346508234739304}, {"head": 2, "memory_to_active": 0.008380573242902756, "contribution_to_active": 0.49788546562194824, "mix_mean": 0.33804914355278015, "mix_low": 0.014431953430175781, "mix_high": 4.863739013671875e-05, "read_cosine": 0.7572891712188721, "beta_mean": 0.00012070184311596677, "nonfinite": 0.0, "weighted_memory_to_active": 0.004468888975679874, "mix_std": 0.2699694037437439, "beta_std": 0.0002827700227499008, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0013594039482995868, "evidence_weights": [-0.2051459401845932, -0.26107102632522583, 0.11853604018688202], "agreement_weight": 0.009597070515155792, "orthogonal_weight": 0.7974427938461304}, {"head": 3, "memory_to_active": 0.0022177642676979303, "contribution_to_active": 0.3703462779521942, "mix_mean": 0.3907823860645294, "mix_low": 0.005658149719238281, "mix_high": 0.00034427642822265625, "read_cosine": 0.6027937531471252, "beta_mean": 1.1862559404107742e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007211567717604339, "mix_std": 0.2869507968425751, "beta_std": 2.535078237997368e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00054860336240381, "evidence_weights": [-0.2634844183921814, -0.2389666736125946, 0.18667906522750854], "agreement_weight": 0.047276776283979416, "orthogonal_weight": -0.7321579456329346}, {"head": 4, "memory_to_active": 0.0833660289645195, "contribution_to_active": 0.6175979971885681, "mix_mean": 0.3855767250061035, "mix_low": 0.008707046508789062, "mix_high": 0.0015192031860351562, "read_cosine": 0.7355941534042358, "beta_mean": 0.18532651662826538, "nonfinite": 0.0, "weighted_memory_to_active": 0.046029120683670044, "mix_std": 0.31008028984069824, "beta_std": 0.21277925372123718, "beta_low": 0.198974609375, "beta_high": 0.0, "orthogonal_to_active": 0.007523948326706886, "evidence_weights": [-0.20936356484889984, -0.25541913509368896, 0.0012832693755626678], "agreement_weight": 0.01303506176918745, "orthogonal_weight": 0.35895130038261414}, {"head": 5, "memory_to_active": 0.027564343065023422, "contribution_to_active": 0.5150017738342285, "mix_mean": 0.5697289705276489, "mix_low": 0.010400772094726562, "mix_high": 0.0035409927368164062, "read_cosine": 0.2984898090362549, "beta_mean": 0.002540689194574952, "nonfinite": 0.0, "weighted_memory_to_active": 0.017282024025917053, "mix_std": 0.27460405230522156, "beta_std": 0.019815895706415176, "beta_low": 0.96929931640625, "beta_high": 0.0, "orthogonal_to_active": 0.011301126331090927, "evidence_weights": [-0.2854548394680023, -0.30357635021209717, 0.6953586339950562], "agreement_weight": 0.30667543411254883, "orthogonal_weight": 0.8211170434951782}]}]}, {"step": 58000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8484671650003293, "memory_to_active": 0.7898080348968506, "contribution_to_active": 0.5575307607650757, "mix_mean": 0.41485098004341125, "mix_low": 0.131654292345047, "mix_high": 0.010289361700415611, "read_cosine": 0.34741002321243286, "beta_mean": 0.06982935965061188, "nonfinite": 0.0, "weighted_memory_to_active": 0.20008762180805206, "mix_std": 0.22321023046970367, "beta_std": 0.06117360293865204, "beta_low": 0.5689405202865601, "beta_high": 2.733866494963877e-05, "orthogonal_to_active": 0.0605282299220562, "samples": 4, "scored_tokens": 899, "normal_loss": 3.044692039489746, "normal_token_accuracy": 0.45606228709220886, "state_active": 0.06334053725004196, "state_memory": 0.017729375511407852, "state_peak": 125.62014770507812, "growth_active": 1.1296840906143188, "growth_memory": 1.3078513145446777, "output_influence": 0.4283958673477173, "residual_influence": 0.14668883383274078, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.188807964324951, "no_memory_loss_delta": 0.6294443607330322, "scrambled_loss_delta": 0.154983252286911, "memory_only_loss_delta": 3.4119317531585693, "no_evidence_loss_delta": 0.3665464520454407, "no_agreement_loss_delta": 0.022070521488785744, "no_orthogonal_loss_delta": 0.0017113583162426949}, {"index": 94, "tokens": 256, "normal_loss": 3.1842589378356934, "no_memory_loss_delta": 0.6877363920211792, "scrambled_loss_delta": 0.4018583297729492, "memory_only_loss_delta": 3.808997631072998, "no_evidence_loss_delta": 0.31374675035476685, "no_agreement_loss_delta": 0.045510586351156235, "no_orthogonal_loss_delta": 0.05662913993000984}, {"index": 188, "tokens": 256, "normal_loss": 3.332688808441162, "no_memory_loss_delta": 0.8151141405105591, "scrambled_loss_delta": 0.16223812103271484, "memory_only_loss_delta": 3.347485065460205, "no_evidence_loss_delta": 0.40512916445732117, "no_agreement_loss_delta": 0.0680604800581932, "no_orthogonal_loss_delta": 0.02884567342698574}, {"index": 283, "tokens": 131, "normal_loss": 1.9275175333023071, "no_memory_loss_delta": 0.5196778774261475, "scrambled_loss_delta": 0.3253769874572754, "memory_only_loss_delta": 2.8524281978607178, "no_evidence_loss_delta": 0.28934475779533386, "no_agreement_loss_delta": 0.0422905795276165, "no_orthogonal_loss_delta": -0.0024265660904347897}], "agreement_weight_abs_mean": 0.22159038484096527, "agreement_weight_abs_max": 6.017067909240723, "agreement_weight_mean": 0.1936783343553543, "orthogonal_weight_abs_mean": 0.8828940391540527, "orthogonal_weight_abs_max": 9.660650253295898, "orthogonal_weight_mean": -0.10441365838050842, "no_memory_loss": 3.727612018585205, "no_memory_loss_delta": 0.6829202771186829, "no_memory_token_accuracy_gain": 0.08453837782144547, "no_memory_token_win_fraction": 0.6907675266265869, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 49.48603115269015, "scrambled_loss": 3.2968711853027344, "scrambled_loss_delta": 0.2521788775920868, "scrambled_token_accuracy_gain": 0.02447163499891758, "scrambled_token_win_fraction": 0.6262513995170593, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 22.289428116417824, "memory_only_loss": 6.469811916351318, "memory_only_loss_delta": 3.425118923187256, "memory_only_token_accuracy_gain": 0.2925472855567932, "memory_only_token_win_fraction": 0.9054505228996277, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.74545896449762, "no_evidence_loss": 3.3959407806396484, "no_evidence_loss_delta": 0.3512483835220337, "no_evidence_token_accuracy_gain": 0.051167964935302734, "no_evidence_token_win_fraction": 0.6418242454528809, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 29.619108239449552, "no_agreement_loss": 3.08948016166687, "no_agreement_loss_delta": 0.04478789120912552, "no_agreement_token_accuracy_gain": 0.007786429487168789, "no_agreement_token_win_fraction": 0.486095666885376, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 4.379972119019899, "no_orthogonal_loss": 3.0691657066345215, "no_orthogonal_loss_delta": 0.02447361685335636, "no_orthogonal_token_accuracy_gain": -0.004449388012290001, "no_orthogonal_token_win_fraction": 0.5639599561691284, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.4176566128456463, "short_no_memory_loss_delta": 0.6214308142662048, "long_context_memory_gain": 0.06148946285247803, "normal_context_loss_gain": 0.1468161791563034, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.4651288986206055, "contribution_to_active": 1.1348445415496826, "mix_mean": 0.10532202571630478, "mix_low": 0.5373551249504089, "mix_high": 0.004949252121150494, "read_cosine": 0.4303884506225586, "beta_mean": 0.21830148994922638, "nonfinite": 0.0, "weighted_memory_to_active": 1.1706278324127197, "mix_std": 0.09007526934146881, "beta_std": 0.1434294879436493, "beta_low": 0.0817057341337204, "beta_high": 0.0, "orthogonal_to_active": 0.04200782626867294, "output_influence": 0.22606725990772247, "residual_influence": 0.2599230110645294, "state_active": 0.06761281937360764, "state_memory": 0.045736439526081085, "state_peak": 47.09318923950195, "growth_active": 1.2683255672454834, "growth_memory": 1.2851961851119995, "heads": [{"head": 0, "memory_to_active": 2.6810741424560547, "contribution_to_active": 0.5003684163093567, "mix_mean": 0.019086698070168495, "mix_low": 0.6375217437744141, "mix_high": 0.0, "read_cosine": 0.23091182112693787, "beta_mean": 0.1852591633796692, "nonfinite": 0.0, "weighted_memory_to_active": 0.626367449760437, "mix_std": 0.0598406046628952, "beta_std": 0.09611740708351135, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06509236246347427, "evidence_weights": [-0.0020769203547388315, -0.045288268476724625, -0.04870286211371422], "agreement_weight": -0.21141870319843292, "orthogonal_weight": -0.11857748031616211}, {"head": 1, "memory_to_active": 6.3890275955200195, "contribution_to_active": 2.7020492553710938, "mix_mean": 0.018695877864956856, "mix_low": 0.6406049728393555, "mix_high": 0.0, "read_cosine": 0.43799445033073425, "beta_mean": 0.46261370182037354, "nonfinite": 0.0, "weighted_memory_to_active": 2.9760167598724365, "mix_std": 0.05517970025539398, "beta_std": 0.22397348284721375, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.008731625974178314, "evidence_weights": [0.02229922078549862, -0.10355855524539948, 0.6286450624465942], "agreement_weight": 0.02581760101020336, "orthogonal_weight": 0.006477944552898407}, {"head": 2, "memory_to_active": 2.5389039516448975, "contribution_to_active": 0.5059260725975037, "mix_mean": 0.013461788184940815, "mix_low": 0.6785335540771484, "mix_high": 0.0, "read_cosine": 0.3062509298324585, "beta_mean": 0.25385117530822754, "nonfinite": 0.0, "weighted_memory_to_active": 0.5956162214279175, "mix_std": 0.03156282752752304, "beta_std": 0.17566636204719543, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.047981783747673035, "evidence_weights": [0.06752829253673553, 0.059123143553733826, 0.1964224874973297], "agreement_weight": 0.011502858251333237, "orthogonal_weight": -0.20285864174365997}, {"head": 3, "memory_to_active": 7.222509860992432, "contribution_to_active": 2.562854051589966, "mix_mean": 0.03058013692498207, "mix_low": 0.4959144592285156, "mix_high": 0.0, "read_cosine": 0.3428727984428406, "beta_mean": 0.33032795786857605, "nonfinite": 0.0, "weighted_memory_to_active": 2.7163381576538086, "mix_std": 0.07106442004442215, "beta_std": 0.25742459297180176, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11069361865520477, "evidence_weights": [-0.03974141180515289, -0.03907327726483345, -0.13832154870033264], "agreement_weight": -0.005414260551333427, "orthogonal_weight": -0.049262769520282745}, {"head": 4, "memory_to_active": 1.8446593284606934, "contribution_to_active": 0.06398682296276093, "mix_mean": 0.01132158376276493, "mix_low": 0.7569742202758789, "mix_high": 0.0, "read_cosine": 0.31799355149269104, "beta_mean": 0.05656298249959946, "nonfinite": 0.0, "weighted_memory_to_active": 0.04804745689034462, "mix_std": 0.03622078895568848, "beta_std": 0.08976151049137115, "beta_low": 0.2098388671875, "beta_high": 0.0, "orthogonal_to_active": 0.003253143047913909, "evidence_weights": [0.13296853005886078, 0.09776025265455246, -0.14689548313617706], "agreement_weight": -0.14674478769302368, "orthogonal_weight": 0.06154894828796387}, {"head": 5, "memory_to_active": 0.11459800601005554, "contribution_to_active": 0.4738824665546417, "mix_mean": 0.5387860536575317, "mix_low": 0.014581680297851562, "mix_high": 0.029695510864257812, "read_cosine": 0.9463071823120117, "beta_mean": 0.02119390480220318, "nonfinite": 0.0, "weighted_memory_to_active": 0.06138060241937637, "mix_std": 0.2865832448005676, "beta_std": 0.017633523792028427, "beta_low": 0.2803955078125, "beta_high": 0.0, "orthogonal_to_active": 0.01629442721605301, "evidence_weights": [-0.17813272774219513, -0.21925076842308044, 0.3847596347332001], "agreement_weight": 0.2928428649902344, "orthogonal_weight": -0.6787949800491333}]}, {"layer": 1, "memory_to_active": 2.34781551361084, "contribution_to_active": 0.46478140354156494, "mix_mean": 0.21004647016525269, "mix_low": 0.36752212047576904, "mix_high": 0.00106048583984375, "read_cosine": 0.2878042757511139, "beta_mean": 0.17717556655406952, "nonfinite": 0.0, "weighted_memory_to_active": 0.30402082204818726, "mix_std": 0.11669859290122986, "beta_std": 0.12826111912727356, "beta_low": 0.3523865044116974, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.040222033858299255, "output_influence": 0.24267227947711945, "residual_influence": 0.06501709669828415, "state_active": 0.028783366084098816, "state_memory": 0.03572992980480194, "state_peak": 37.47284698486328, "growth_active": 1.1831358671188354, "growth_memory": 1.4742454290390015, "heads": [{"head": 0, "memory_to_active": 0.028224287554621696, "contribution_to_active": 0.48131316900253296, "mix_mean": 0.519442617893219, "mix_low": 0.006028175354003906, "mix_high": 0.0022382736206054688, "read_cosine": -0.0032999739050865173, "beta_mean": 0.00020114172366447747, "nonfinite": 0.0, "weighted_memory_to_active": 0.01587098464369774, "mix_std": 0.28302183747291565, "beta_std": 0.0002914620563387871, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015377897769212723, "evidence_weights": [-0.20838265120983124, -0.2713523805141449, 0.37406298518180847], "agreement_weight": 0.20906269550323486, "orthogonal_weight": -1.0285142660140991}, {"head": 1, "memory_to_active": 3.098165988922119, "contribution_to_active": 0.26372969150543213, "mix_mean": 0.02403333969414234, "mix_low": 0.4695873260498047, "mix_high": 1.9073486328125e-06, "read_cosine": 0.5055508613586426, "beta_mean": 0.2750920057296753, "nonfinite": 0.0, "weighted_memory_to_active": 0.287371963262558, "mix_std": 0.04942867532372475, "beta_std": 0.17989666759967804, "beta_low": 0.00225830078125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.019682422280311584, "evidence_weights": [0.022094298154115677, -0.05569613724946976, 0.05399315059185028], "agreement_weight": -0.004971046466380358, "orthogonal_weight": 0.17596745491027832}, {"head": 2, "memory_to_active": 0.050291385501623154, "contribution_to_active": 0.551361620426178, "mix_mean": 0.6548933386802673, "mix_low": 0.00036525726318359375, "mix_high": 0.004122734069824219, "read_cosine": 0.4296150207519531, "beta_mean": 0.0019060819176957011, "nonfinite": 0.0, "weighted_memory_to_active": 0.034558750689029694, "mix_std": 0.2343471497297287, "beta_std": 0.005449797958135605, "beta_low": 0.9766845703125, "beta_high": 0.0, "orthogonal_to_active": 0.02409348450601101, "evidence_weights": [-0.2769078314304352, -0.36868369579315186, 0.6490572690963745], "agreement_weight": 0.04319116473197937, "orthogonal_weight": -0.8148126006126404}, {"head": 3, "memory_to_active": 2.9202117919921875, "contribution_to_active": 0.16605782508850098, "mix_mean": 0.022928472608327866, "mix_low": 0.5648717880249023, "mix_high": 0.0, "read_cosine": 0.19046765565872192, "beta_mean": 0.32867515087127686, "nonfinite": 0.0, "weighted_memory_to_active": 0.23366005718708038, "mix_std": 0.0598750039935112, "beta_std": 0.25715819001197815, "beta_low": 0.05670166015625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.09220713376998901, "evidence_weights": [0.040583983063697815, -0.015805207192897797, 0.00129512173589319], "agreement_weight": -0.0312345027923584, "orthogonal_weight": -0.4868824779987335}, {"head": 4, "memory_to_active": 3.8109662532806396, "contribution_to_active": 1.25514554977417, "mix_mean": 0.03233702480792999, "mix_low": 0.3278789520263672, "mix_high": 0.0, "read_cosine": 0.34371790289878845, "beta_mean": 0.1552049219608307, "nonfinite": 0.0, "weighted_memory_to_active": 1.2015855312347412, "mix_std": 0.05463417246937752, "beta_std": 0.1158323585987091, "beta_low": 0.0411376953125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.07275578379631042, "evidence_weights": [-0.028178371489048004, 0.0014045737916603684, 0.0940997302532196], "agreement_weight": -0.01595131866633892, "orthogonal_weight": 0.08546886593103409}, {"head": 5, "memory_to_active": 4.179032802581787, "contribution_to_active": 0.07108046859502792, "mix_mean": 0.00664399191737175, "mix_low": 0.8364009857177734, "mix_high": 0.0, "read_cosine": 0.260774165391922, "beta_mean": 0.3019740879535675, "nonfinite": 0.0, "weighted_memory_to_active": 0.051077499985694885, "mix_std": 0.018884701654314995, "beta_std": 0.2109382152557373, "beta_low": 0.03753662109375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.01721547171473503, "evidence_weights": [0.16112114489078522, 0.0848744809627533, 0.10933335870504379], "agreement_weight": -0.11555588245391846, "orthogonal_weight": 0.3402576148509979}]}, {"layer": 2, "memory_to_active": 0.9915792942047119, "contribution_to_active": 0.35150444507598877, "mix_mean": 0.40331077575683594, "mix_low": 0.2507379949092865, "mix_high": 0.004691283218562603, "read_cosine": 0.30813053250312805, "beta_mean": 0.08566366881132126, "nonfinite": 0.0, "weighted_memory_to_active": 0.08112819492816925, "mix_std": 0.18867364525794983, "beta_std": 0.07097968459129333, "beta_low": 0.5799561142921448, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.019928112626075745, "output_influence": 0.38988935947418213, "residual_influence": 0.10457128286361694, "state_active": 0.03994560241699219, "state_memory": 0.01535973884165287, "state_peak": 28.958040237426758, "growth_active": 1.0682554244995117, "growth_memory": 1.5792537927627563, "heads": [{"head": 0, "memory_to_active": 0.008818534202873707, "contribution_to_active": 0.3288526237010956, "mix_mean": 0.6274583339691162, "mix_low": 0.0006704330444335938, "mix_high": 0.009336471557617188, "read_cosine": 0.13897594809532166, "beta_mean": 7.2598057158757e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0057694693095982075, "mix_std": 0.27068546414375305, "beta_std": 0.00012264620454516262, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0060675255954265594, "evidence_weights": [-0.3696369230747223, -0.21696168184280396, 0.16585437953472137], "agreement_weight": 0.13321544229984283, "orthogonal_weight": -1.0923945903778076}, {"head": 1, "memory_to_active": 2.887258529663086, "contribution_to_active": 0.03898951783776283, "mix_mean": 0.0049714515917003155, "mix_low": 0.8909225463867188, "mix_high": 0.0, "read_cosine": 0.15743796527385712, "beta_mean": 0.23274420201778412, "nonfinite": 0.0, "weighted_memory_to_active": 0.03212842345237732, "mix_std": 0.009839901700615883, "beta_std": 0.15863610804080963, "beta_low": 0.002685546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.00968190934509039, "evidence_weights": [0.1419929713010788, 0.11777368187904358, 0.5722246170043945], "agreement_weight": 0.028135335072875023, "orthogonal_weight": 0.34377431869506836}, {"head": 2, "memory_to_active": 0.009390519931912422, "contribution_to_active": 0.42288219928741455, "mix_mean": 0.5466157793998718, "mix_low": 0.0011663436889648438, "mix_high": 0.0034685134887695312, "read_cosine": 0.37414222955703735, "beta_mean": 2.9315058782231063e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038043963722884655, "mix_std": 0.2764091491699219, "beta_std": 5.4309519327944145e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00019362739112693816, "evidence_weights": [-0.25170621275901794, -0.29410693049430847, 0.5729936361312866], "agreement_weight": 0.00793376099318266, "orthogonal_weight": -0.06039586663246155}, {"head": 3, "memory_to_active": 0.12748950719833374, "contribution_to_active": 0.5435517430305481, "mix_mean": 0.5721251368522644, "mix_low": 0.00029468536376953125, "mix_high": 0.0068416595458984375, "read_cosine": 0.6261298060417175, "beta_mean": 0.023531515151262283, "nonfinite": 0.0, "weighted_memory_to_active": 0.07961588352918625, "mix_std": 0.2609955072402954, "beta_std": 0.03113408386707306, "beta_low": 0.4400634765625, "beta_high": 0.0, "orthogonal_to_active": 0.05340691655874252, "evidence_weights": [-0.20391520857810974, -0.3207981586456299, 0.7917506694793701], "agreement_weight": 0.047679122537374496, "orthogonal_weight": -0.9007072448730469}, {"head": 4, "memory_to_active": 2.905506134033203, "contribution_to_active": 0.25124692916870117, "mix_mean": 0.019087743014097214, "mix_low": 0.6103944778442383, "mix_high": 0.0, "read_cosine": 0.2177448570728302, "beta_mean": 0.2575691342353821, "nonfinite": 0.0, "weighted_memory_to_active": 0.35921984910964966, "mix_std": 0.04942461475729942, "beta_std": 0.23587629199028015, "beta_low": 0.0369873046875, "beta_high": 0.00048828125, "orthogonal_to_active": 0.04911154881119728, "evidence_weights": [0.02663992904126644, 0.00709693506360054, 0.026959817856550217], "agreement_weight": 0.012418247759342194, "orthogonal_weight": -0.16710835695266724}, {"head": 5, "memory_to_active": 0.011012351140379906, "contribution_to_active": 0.5235036015510559, "mix_mean": 0.6496062278747559, "mix_low": 0.0009794235229492188, "mix_high": 0.008501052856445312, "read_cosine": 0.33435237407684326, "beta_mean": 3.5201006539864466e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006231073755770922, "mix_std": 0.264687180519104, "beta_std": 5.464929563459009e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011071523185819387, "evidence_weights": [-0.28050827980041504, -0.29209762811660767, 1.0137628316879272], "agreement_weight": 0.18632863461971283, "orthogonal_weight": -0.21893328428268433}]}, {"layer": 3, "memory_to_active": 0.43619978427886963, "contribution_to_active": 0.47115886211395264, "mix_mean": 0.5018577575683594, "mix_low": 0.0612543448805809, "mix_high": 0.010371049866080284, "read_cosine": 0.19827057421207428, "beta_mean": 0.028673488646745682, "nonfinite": 0.0, "weighted_memory_to_active": 0.12970302999019623, "mix_std": 0.26083624362945557, "beta_std": 0.026847075670957565, "beta_low": 0.7557576894760132, "beta_high": 0.0, "orthogonal_to_active": 0.052581124007701874, "output_influence": 0.40693455934524536, "residual_influence": 0.09561628848314285, "state_active": 0.02599499374628067, "state_memory": 0.007249830290675163, "state_peak": 23.531835556030273, "growth_active": 1.0880309343338013, "growth_memory": 1.6651312112808228, "heads": [{"head": 0, "memory_to_active": 0.3848760724067688, "contribution_to_active": 0.5796113610267639, "mix_mean": 0.598501443862915, "mix_low": 0.001712799072265625, "mix_high": 0.02373790740966797, "read_cosine": 0.2718674838542938, "beta_mean": 0.011021147482097149, "nonfinite": 0.0, "weighted_memory_to_active": 0.23940600454807281, "mix_std": 0.3066481649875641, "beta_std": 0.014201108366250992, "beta_low": 0.64691162109375, "beta_high": 0.0, "orthogonal_to_active": 0.16914303600788116, "evidence_weights": [-0.21449849009513855, -0.24837301671504974, 0.954293966293335], "agreement_weight": 0.09802179038524628, "orthogonal_weight": -0.7538998126983643}, {"head": 1, "memory_to_active": 0.018147755414247513, "contribution_to_active": 0.41928523778915405, "mix_mean": 0.562551736831665, "mix_low": 0.0053157806396484375, "mix_high": 0.008135795593261719, "read_cosine": 0.18097220361232758, "beta_mean": 8.57266568345949e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010677472688257694, "mix_std": 0.2992776930332184, "beta_std": 0.00020273025438655168, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00888894870877266, "evidence_weights": [-0.2640175521373749, -0.21357077360153198, 0.6905235648155212], "agreement_weight": 0.518419623374939, "orthogonal_weight": -0.9194190502166748}, {"head": 2, "memory_to_active": 1.952021598815918, "contribution_to_active": 0.2884265184402466, "mix_mean": 0.05029894411563873, "mix_low": 0.3477144241333008, "mix_high": 0.0, "read_cosine": 0.21759670972824097, "beta_mean": 0.15530714392662048, "nonfinite": 0.0, "weighted_memory_to_active": 0.3403843641281128, "mix_std": 0.10399473458528519, "beta_std": 0.1344740390777588, "beta_low": 0.01751708984375, "beta_high": 0.0, "orthogonal_to_active": 0.0019935257732868195, "evidence_weights": [-0.09026098251342773, 0.005376928485929966, -0.025351380929350853], "agreement_weight": -0.014848820865154266, "orthogonal_weight": -0.007611079141497612}, {"head": 3, "memory_to_active": 0.01851130835711956, "contribution_to_active": 0.5140210390090942, "mix_mean": 0.6017818450927734, "mix_low": 0.005450248718261719, "mix_high": 0.010365486145019531, "read_cosine": 0.2993762493133545, "beta_mean": 7.945892866700888e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012356032617390156, "mix_std": 0.29167091846466064, "beta_std": 0.00018432775686960667, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.003277531126514077, "evidence_weights": [-0.2552618086338043, -0.23006166517734528, 2.2075791358947754], "agreement_weight": 2.0920441150665283, "orthogonal_weight": -0.3459523916244507}, {"head": 4, "memory_to_active": 0.013214709237217903, "contribution_to_active": 0.3657686114311218, "mix_mean": 0.48823708295822144, "mix_low": 0.005175590515136719, "mix_high": 0.001483917236328125, "read_cosine": 0.1622961461544037, "beta_mean": 7.768235809635371e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008398063480854034, "mix_std": 0.29088151454925537, "beta_std": 0.00011279014142928645, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00801936350762844, "evidence_weights": [-0.21853023767471313, -0.21280309557914734, 0.13099375367164612], "agreement_weight": -0.05029028281569481, "orthogonal_weight": -1.0340449810028076}, {"head": 5, "memory_to_active": 0.2304273396730423, "contribution_to_active": 0.6598401665687561, "mix_mean": 0.7097755670547485, "mix_low": 0.0021572113037109375, "mix_high": 0.018503189086914062, "read_cosine": 0.057514630258083344, "beta_mean": 0.005469788797199726, "nonfinite": 0.0, "weighted_memory_to_active": 0.16699624061584473, "mix_std": 0.27254432439804077, "beta_std": 0.011907448060810566, "beta_low": 0.87017822265625, "beta_high": 0.0, "orthogonal_to_active": 0.12416432797908783, "evidence_weights": [-0.2598233222961426, -0.3338072896003723, 0.4746275544166565], "agreement_weight": 0.19211693108081818, "orthogonal_weight": -0.7670406699180603}]}, {"layer": 4, "memory_to_active": 0.9997373819351196, "contribution_to_active": 0.4716627597808838, "mix_mean": 0.3682037889957428, "mix_low": 0.2578577995300293, "mix_high": 0.010793527588248253, "read_cosine": 0.3036091923713684, "beta_mean": 0.11107227951288223, "nonfinite": 0.0, "weighted_memory_to_active": 0.12482903897762299, "mix_std": 0.15960556268692017, "beta_std": 0.1016615778207779, "beta_low": 0.3931274712085724, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04860988259315491, "output_influence": 0.48584258556365967, "residual_influence": 0.1403457522392273, "state_active": 0.037935152649879456, "state_memory": 0.01731003448367119, "state_peak": 34.13692855834961, "growth_active": 1.2173207998275757, "growth_memory": 1.5279399156570435, "heads": [{"head": 0, "memory_to_active": 0.1771019548177719, "contribution_to_active": 0.7872402667999268, "mix_mean": 0.6838842630386353, "mix_low": 0.012879371643066406, "mix_high": 0.028489112854003906, "read_cosine": 0.443685919046402, "beta_mean": 0.26622796058654785, "nonfinite": 0.0, "weighted_memory_to_active": 0.15026047825813293, "mix_std": 0.349254310131073, "beta_std": 0.2612641155719757, "beta_low": 0.2452392578125, "beta_high": 0.0, "orthogonal_to_active": 0.08099114149808884, "evidence_weights": [-0.26370665431022644, -0.3676210939884186, -0.037965960800647736], "agreement_weight": -0.002052189316600561, "orthogonal_weight": 0.6478867530822754}, {"head": 1, "memory_to_active": 0.0475032776594162, "contribution_to_active": 0.6547942757606506, "mix_mean": 0.6954153776168823, "mix_low": 0.00010585784912109375, "mix_high": 0.017108917236328125, "read_cosine": 0.6050606966018677, "beta_mean": 0.0009222936350852251, "nonfinite": 0.0, "weighted_memory_to_active": 0.03460511192679405, "mix_std": 0.23872514069080353, "beta_std": 0.006417709868401289, "beta_low": 0.98211669921875, "beta_high": 0.0, "orthogonal_to_active": 0.018912388011813164, "evidence_weights": [-0.24050772190093994, -0.36187317967414856, 0.6325331330299377], "agreement_weight": 0.30710867047309875, "orthogonal_weight": -0.7641128301620483}, {"head": 2, "memory_to_active": 1.5429264307022095, "contribution_to_active": 0.31445714831352234, "mix_mean": 0.034385278820991516, "mix_low": 0.44581127166748047, "mix_high": 1.049041748046875e-05, "read_cosine": 0.22333696484565735, "beta_mean": 0.05488833039999008, "nonfinite": 0.0, "weighted_memory_to_active": 0.23222950100898743, "mix_std": 0.06961869448423386, "beta_std": 0.05718815699219704, "beta_low": 0.10943603515625, "beta_high": 0.0, "orthogonal_to_active": 0.06308121979236603, "evidence_weights": [0.04669616371393204, -0.01761227659881115, 0.2312909960746765], "agreement_weight": 0.12463811784982681, "orthogonal_weight": -0.3637060523033142}, {"head": 3, "memory_to_active": 1.9468834400177002, "contribution_to_active": 0.1411287784576416, "mix_mean": 0.03296519070863724, "mix_low": 0.41903114318847656, "mix_high": 0.0, "read_cosine": 0.16611437499523163, "beta_mean": 0.1251445710659027, "nonfinite": 0.0, "weighted_memory_to_active": 0.18620459735393524, "mix_std": 0.05967988073825836, "beta_std": 0.1163952648639679, "beta_low": 0.02197265625, "beta_high": 0.0, "orthogonal_to_active": 0.0017904473934322596, "evidence_weights": [-0.01987631805241108, 0.052127987146377563, -0.2658947706222534], "agreement_weight": 0.07735270261764526, "orthogonal_weight": -0.013387833721935749}, {"head": 4, "memory_to_active": 0.005995679646730423, "contribution_to_active": 0.6486870646476746, "mix_mean": 0.7489951252937317, "mix_low": 6.866455078125e-05, "mix_high": 0.019147872924804688, "read_cosine": 0.1522737592458725, "beta_mean": 1.5348950910265557e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004303107038140297, "mix_std": 0.2105943262577057, "beta_std": 3.219988866476342e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002597304992377758, "evidence_weights": [-0.334364116191864, -0.324086993932724, 0.3809244632720947], "agreement_weight": 0.3250540792942047, "orthogonal_weight": -0.6434586644172668}, {"head": 5, "memory_to_active": 2.2780134677886963, "contribution_to_active": 0.28366872668266296, "mix_mean": 0.013577413745224476, "mix_low": 0.66925048828125, "mix_high": 4.76837158203125e-06, "read_cosine": 0.23118352890014648, "beta_mean": 0.2192351073026657, "nonfinite": 0.0, "weighted_memory_to_active": 0.14137142896652222, "mix_std": 0.029760969802737236, "beta_std": 0.16867196559906006, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12428675591945648, "evidence_weights": [0.05493622273206711, -0.021934431046247482, -0.26025277376174927], "agreement_weight": -0.08722545951604843, "orthogonal_weight": 1.0055712461471558}]}, {"layer": 5, "memory_to_active": 0.4003235697746277, "contribution_to_active": 0.5317385196685791, "mix_mean": 0.5050303936004639, "mix_low": 0.05978266894817352, "mix_high": 0.018812023103237152, "read_cosine": 0.06194184720516205, "beta_mean": 0.051550861448049545, "nonfinite": 0.0, "weighted_memory_to_active": 0.13578572869300842, "mix_std": 0.2556137442588806, "beta_std": 0.06652341783046722, "beta_low": 0.6759338974952698, "beta_high": 0.0, "orthogonal_to_active": 0.05574675649404526, "output_influence": 0.5512268543243408, "residual_influence": 0.157539963722229, "state_active": 0.040405817329883575, "state_memory": 0.010680010542273521, "state_peak": 67.44564056396484, "growth_active": 1.104683756828308, "growth_memory": 1.2381035089492798, "heads": [{"head": 0, "memory_to_active": 0.26160556077957153, "contribution_to_active": 0.5587968826293945, "mix_mean": 0.4469284117221832, "mix_low": 0.013422012329101562, "mix_high": 0.00229644775390625, "read_cosine": -0.11229868978261948, "beta_mean": 0.01762409880757332, "nonfinite": 0.0, "weighted_memory_to_active": 0.0871543288230896, "mix_std": 0.28008460998535156, "beta_std": 0.03165316581726074, "beta_low": 0.550537109375, "beta_high": 0.0, "orthogonal_to_active": 0.03656216710805893, "evidence_weights": [-0.15914639830589294, -0.17914825677871704, 0.07899080961942673], "agreement_weight": -0.024237463250756264, "orthogonal_weight": -0.43112924695014954}, {"head": 1, "memory_to_active": 0.00915069505572319, "contribution_to_active": 0.547671377658844, "mix_mean": 0.6991596221923828, "mix_low": 0.0010423660278320312, "mix_high": 0.03589153289794922, "read_cosine": 0.19378791749477386, "beta_mean": 2.0910558305331506e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00546760531142354, "mix_std": 0.2704998254776001, "beta_std": 4.5397602661978453e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005205526016652584, "evidence_weights": [-0.25029370188713074, -0.28845277428627014, 0.9002212882041931], "agreement_weight": 0.6878853440284729, "orthogonal_weight": -1.0019677877426147}, {"head": 2, "memory_to_active": 0.008511046878993511, "contribution_to_active": 0.5946921706199646, "mix_mean": 0.6535499095916748, "mix_low": 0.0027618408203125, "mix_high": 0.022291183471679688, "read_cosine": -0.5543427467346191, "beta_mean": 5.250384128885344e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005313584115356207, "mix_std": 0.267957478761673, "beta_std": 8.881877874955535e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003168084193021059, "evidence_weights": [-0.262488454580307, -0.3311401307582855, 0.20613990724086761], "agreement_weight": -0.08411651849746704, "orthogonal_weight": -0.7486812472343445}, {"head": 3, "memory_to_active": 1.5863003730773926, "contribution_to_active": 0.23852400481700897, "mix_mean": 0.11823288351297379, "mix_low": 0.12714290618896484, "mix_high": 2.86102294921875e-06, "read_cosine": 0.14800772070884705, "beta_mean": 0.10867252945899963, "nonfinite": 0.0, "weighted_memory_to_active": 0.36578401923179626, "mix_std": 0.15427827835083008, "beta_std": 0.1562940776348114, "beta_low": 0.1990966796875, "beta_high": 0.0, "orthogonal_to_active": 0.15129511058330536, "evidence_weights": [-0.07467752695083618, -0.10218846052885056, 0.060598623007535934], "agreement_weight": 0.053065087646245956, "orthogonal_weight": -0.4945768117904663}, {"head": 4, "memory_to_active": 0.42569980025291443, "contribution_to_active": 0.5686154365539551, "mix_mean": 0.3101828098297119, "mix_low": 0.21249675750732422, "mix_high": 0.005221366882324219, "read_cosine": 0.5496447086334229, "beta_mean": 0.18262900412082672, "nonfinite": 0.0, "weighted_memory_to_active": 0.2755700349807739, "mix_std": 0.33376774191856384, "beta_std": 0.209541454911232, "beta_low": 0.3104248046875, "beta_high": 0.0, "orthogonal_to_active": 0.09353836625814438, "evidence_weights": [-0.11316698789596558, -0.25984349846839905, -0.04610573500394821], "agreement_weight": 0.04232665151357651, "orthogonal_weight": 0.5416191220283508}, {"head": 5, "memory_to_active": 0.11067396402359009, "contribution_to_active": 0.6821310520172119, "mix_mean": 0.8021287322044373, "mix_low": 0.0018301010131835938, "mix_high": 0.047168731689453125, "read_cosine": 0.14685216546058655, "beta_mean": 0.00030612252885475755, "nonfinite": 0.0, "weighted_memory_to_active": 0.07542475312948227, "mix_std": 0.22709453105926514, "beta_std": 0.0015175858279690146, "beta_low": 0.99554443359375, "beta_high": 0.0, "orthogonal_to_active": 0.04471128433942795, "evidence_weights": [-0.3106963634490967, -0.33329030871391296, 1.8800410032272339], "agreement_weight": 0.47887659072875977, "orthogonal_weight": -0.6291206479072571}]}, {"layer": 6, "memory_to_active": 0.2820315361022949, "contribution_to_active": 0.620314359664917, "mix_mean": 0.5866676568984985, "mix_low": 0.047731559723615646, "mix_high": 0.03445736691355705, "read_cosine": 0.32464200258255005, "beta_mean": 0.06278619915246964, "nonfinite": 0.0, "weighted_memory_to_active": 0.14527833461761475, "mix_std": 0.23844951391220093, "beta_std": 0.07367616891860962, "beta_low": 0.5720621943473816, "beta_high": 0.0, "orthogonal_to_active": 0.048144787549972534, "output_influence": 0.5596113204956055, "residual_influence": 0.1939486563205719, "state_active": 0.04431638866662979, "state_memory": 0.007091065868735313, "state_peak": 93.56803131103516, "growth_active": 1.2031317949295044, "growth_memory": 1.1853078603744507, "heads": [{"head": 0, "memory_to_active": 0.11024122685194016, "contribution_to_active": 0.748162031173706, "mix_mean": 0.8161816596984863, "mix_low": 0.0002307891845703125, "mix_high": 0.0554962158203125, "read_cosine": 0.2801792323589325, "beta_mean": 0.01692085713148117, "nonfinite": 0.0, "weighted_memory_to_active": 0.09319023042917252, "mix_std": 0.2019025981426239, "beta_std": 0.037097953259944916, "beta_low": 0.66180419921875, "beta_high": 0.0, "orthogonal_to_active": 0.06688624620437622, "evidence_weights": [-0.3132922947406769, -0.3502563536167145, 1.243282675743103], "agreement_weight": 0.6223498582839966, "orthogonal_weight": -0.7860568165779114}, {"head": 1, "memory_to_active": 0.004714177921414375, "contribution_to_active": 0.5384853482246399, "mix_mean": 0.5823856592178345, "mix_low": 0.0012025833129882812, "mix_high": 0.013797760009765625, "read_cosine": 0.5196979641914368, "beta_mean": 1.0607975127641112e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021975506097078323, "mix_std": 0.2849677801132202, "beta_std": 1.8431306671118364e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011529948096722364, "evidence_weights": [-0.19522827863693237, -0.2980597913265228, 1.1165876388549805], "agreement_weight": 0.07442178577184677, "orthogonal_weight": -0.6933827996253967}, {"head": 2, "memory_to_active": 0.7784321308135986, "contribution_to_active": 0.09845482558012009, "mix_mean": 0.043033283203840256, "mix_low": 0.2773609161376953, "mix_high": 3.814697265625e-06, "read_cosine": 0.24834948778152466, "beta_mean": 0.023549087345600128, "nonfinite": 0.0, "weighted_memory_to_active": 0.07252636551856995, "mix_std": 0.06963609158992767, "beta_std": 0.024998672306537628, "beta_low": 0.24884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.008306512609124184, "evidence_weights": [0.07324547320604324, -0.04598261043429375, 0.32719460129737854], "agreement_weight": 0.1452340930700302, "orthogonal_weight": -0.1365365982055664}, {"head": 3, "memory_to_active": 0.1338159143924713, "contribution_to_active": 0.6673919558525085, "mix_mean": 0.6529325246810913, "mix_low": 0.006663322448730469, "mix_high": 0.020913124084472656, "read_cosine": 0.4696587324142456, "beta_mean": 0.2101460099220276, "nonfinite": 0.0, "weighted_memory_to_active": 0.09337358176708221, "mix_std": 0.32625287771224976, "beta_std": 0.24126240611076355, "beta_low": 0.21722412109375, "beta_high": 0.0, "orthogonal_to_active": 0.051265742629766464, "evidence_weights": [-0.17085197567939758, -0.32328447699546814, 0.17488797008991241], "agreement_weight": -0.010835865512490273, "orthogonal_weight": 0.6308913826942444}, {"head": 4, "memory_to_active": 0.6256027221679688, "contribution_to_active": 1.0494625568389893, "mix_mean": 0.7505979537963867, "mix_low": 0.0007925033569335938, "mix_high": 0.10995674133300781, "read_cosine": 0.16279800236225128, "beta_mean": 0.125893235206604, "nonfinite": 0.0, "weighted_memory_to_active": 0.5822311639785767, "mix_std": 0.29989930987358093, "beta_std": 0.13826388120651245, "beta_low": 0.3046875, "beta_high": 0.0, "orthogonal_to_active": 0.13317008316516876, "evidence_weights": [-0.20766328275203705, -0.5601069331169128, -0.44399645924568176], "agreement_weight": -0.01381414569914341, "orthogonal_weight": 0.2487747222185135}, {"head": 5, "memory_to_active": 0.03938300162553787, "contribution_to_active": 0.6199291944503784, "mix_mean": 0.674874484539032, "mix_low": 0.0001392364501953125, "mix_high": 0.0065765380859375, "read_cosine": 0.2671685814857483, "beta_mean": 0.00019739323761314154, "nonfinite": 0.0, "weighted_memory_to_active": 0.028150998055934906, "mix_std": 0.2480384111404419, "beta_std": 0.00041565607534721494, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.028087152168154716, "evidence_weights": [-0.23881405591964722, -0.3155512809753418, 0.3964488208293915], "agreement_weight": 0.19688118994235992, "orthogonal_weight": -1.0681328773498535}]}, {"layer": 7, "memory_to_active": 1.6853854656219482, "contribution_to_active": 0.26600146293640137, "mix_mean": 0.16483092308044434, "mix_low": 0.24269963800907135, "mix_high": 0.0010563533287495375, "read_cosine": 0.23390212655067444, "beta_mean": 0.1324172019958496, "nonfinite": 0.0, "weighted_memory_to_active": 0.20061644911766052, "mix_std": 0.14906588196754456, "beta_std": 0.09923040866851807, "beta_low": 0.209381103515625, "beta_high": 0.00020345053053461015, "orthogonal_to_active": 0.0795881450176239, "output_influence": 0.27440476417541504, "residual_influence": 0.06474655121564865, "state_active": 0.013451991602778435, "state_memory": 0.025591548532247543, "state_peak": 29.285184860229492, "growth_active": 1.1502082347869873, "growth_memory": 1.2985687255859375, "heads": [{"head": 0, "memory_to_active": 2.456331253051758, "contribution_to_active": 0.19534292817115784, "mix_mean": 0.03867143765091896, "mix_low": 0.41068363189697266, "mix_high": 0.0, "read_cosine": 0.32800811529159546, "beta_mean": 0.16102705895900726, "nonfinite": 0.0, "weighted_memory_to_active": 0.23001530766487122, "mix_std": 0.07980644702911377, "beta_std": 0.1233377605676651, "beta_low": 0.00445556640625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.00022048503160476685, "evidence_weights": [0.04263019934296608, -0.07356098294258118, 0.03419957309961319], "agreement_weight": -0.03211105614900589, "orthogonal_weight": -0.0012214435264468193}, {"head": 1, "memory_to_active": 0.9858953952789307, "contribution_to_active": 0.3967704772949219, "mix_mean": 0.2241545021533966, "mix_low": 0.12192440032958984, "mix_high": 0.005566596984863281, "read_cosine": 0.3806198239326477, "beta_mean": 0.0910169780254364, "nonfinite": 0.0, "weighted_memory_to_active": 0.33437374234199524, "mix_std": 0.24869698286056519, "beta_std": 0.06346417218446732, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.04586801305413246, "evidence_weights": [-0.2179160863161087, -0.10398411005735397, -0.2683233618736267], "agreement_weight": 0.050414811819791794, "orthogonal_weight": 0.1637721061706543}, {"head": 2, "memory_to_active": 0.6286708116531372, "contribution_to_active": 0.3524836599826813, "mix_mean": 0.2044927477836609, "mix_low": 0.02895069122314453, "mix_high": 0.00034332275390625, "read_cosine": 0.0898558497428894, "beta_mean": 0.039725594222545624, "nonfinite": 0.0, "weighted_memory_to_active": 0.2010648548603058, "mix_std": 0.205735981464386, "beta_std": 0.03868919238448143, "beta_low": 0.1168212890625, "beta_high": 0.0, "orthogonal_to_active": 0.10903657972812653, "evidence_weights": [-0.08206163346767426, -0.16662593185901642, 0.8179495930671692], "agreement_weight": 0.020290523767471313, "orthogonal_weight": -0.555923581123352}, {"head": 3, "memory_to_active": 4.439894676208496, "contribution_to_active": 0.10912420600652695, "mix_mean": 0.03558552637696266, "mix_low": 0.32793712615966797, "mix_high": 0.0, "read_cosine": 0.09259511530399323, "beta_mean": 0.4270676076412201, "nonfinite": 0.0, "weighted_memory_to_active": 0.34180325269699097, "mix_std": 0.055005401372909546, "beta_std": 0.28671568632125854, "beta_low": 0.00384521484375, "beta_high": 0.0010986328125, "orthogonal_to_active": 0.28710654377937317, "evidence_weights": [0.07178441435098648, -0.032651763409376144, 0.7149827480316162], "agreement_weight": 0.04871171712875366, "orthogonal_weight": -0.9324163794517517}, {"head": 4, "memory_to_active": 1.5543662309646606, "contribution_to_active": 0.10410115122795105, "mix_mean": 0.022075802087783813, "mix_low": 0.5646371841430664, "mix_high": 0.0, "read_cosine": 0.24796532094478607, "beta_mean": 0.07542622089385986, "nonfinite": 0.0, "weighted_memory_to_active": 0.07452274858951569, "mix_std": 0.04808875918388367, "beta_std": 0.08258156478404999, "beta_low": 0.13140869140625, "beta_high": 0.0, "orthogonal_to_active": 0.01517309620976448, "evidence_weights": [0.1412942111492157, -0.0053542545065283775, 0.18268060684204102], "agreement_weight": 0.041579339653253555, "orthogonal_weight": -0.25982895493507385}, {"head": 5, "memory_to_active": 0.04715295135974884, "contribution_to_active": 0.4381863474845886, "mix_mean": 0.4640054404735565, "mix_low": 0.0020647048950195312, "mix_high": 0.00042819976806640625, "read_cosine": 0.2643685042858124, "beta_mean": 0.00023974294890649617, "nonfinite": 0.0, "weighted_memory_to_active": 0.02191860042512417, "mix_std": 0.2570616602897644, "beta_std": 0.0005940269329585135, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.02012406475841999, "evidence_weights": [-0.1745872050523758, -0.23019230365753174, 0.545596182346344], "agreement_weight": 0.199250265955925, "orthogonal_weight": -0.9990191459655762}]}, {"layer": 8, "memory_to_active": 0.15591192245483398, "contribution_to_active": 0.5557005405426025, "mix_mean": 0.47863802313804626, "mix_low": 0.013784250244498253, "mix_high": 0.0017116866074502468, "read_cosine": 0.42246997356414795, "beta_mean": 0.013415493071079254, "nonfinite": 0.0, "weighted_memory_to_active": 0.07504633814096451, "mix_std": 0.23836380243301392, "beta_std": 0.018812689930200577, "beta_low": 0.7726033926010132, "beta_high": 0.0, "orthogonal_to_active": 0.06807354092597961, "output_influence": 0.5143731832504272, "residual_influence": 0.177435040473938, "state_active": 0.0449155829846859, "state_memory": 0.005400252062827349, "state_peak": 46.2903938293457, "growth_active": 1.1228241920471191, "growth_memory": 1.150370717048645, "heads": [{"head": 0, "memory_to_active": 0.05882413685321808, "contribution_to_active": 0.571265697479248, "mix_mean": 0.6059839129447937, "mix_low": 0.0004892349243164062, "mix_high": 0.001613616943359375, "read_cosine": 0.4696452021598816, "beta_mean": 0.0029604295268654823, "nonfinite": 0.0, "weighted_memory_to_active": 0.04232531040906906, "mix_std": 0.2496437132358551, "beta_std": 0.01084564346820116, "beta_low": 0.931884765625, "beta_high": 0.0, "orthogonal_to_active": 0.02613838016986847, "evidence_weights": [-0.1664019078016281, -0.38691630959510803, 0.622818112373352], "agreement_weight": 0.24467577040195465, "orthogonal_weight": -0.7913342118263245}, {"head": 1, "memory_to_active": 0.06421114504337311, "contribution_to_active": 0.5532753467559814, "mix_mean": 0.5233774185180664, "mix_low": 2.765655517578125e-05, "mix_high": 0.0011301040649414062, "read_cosine": 0.16197998821735382, "beta_mean": 0.0008438728982582688, "nonfinite": 0.0, "weighted_memory_to_active": 0.027917541563510895, "mix_std": 0.23603668808937073, "beta_std": 0.0046472931280732155, "beta_low": 0.99169921875, "beta_high": 0.0, "orthogonal_to_active": 0.019859887659549713, "evidence_weights": [-0.19723919034004211, -0.2782575786113739, 0.6910067200660706], "agreement_weight": 0.06841360032558441, "orthogonal_weight": -0.7564060091972351}, {"head": 2, "memory_to_active": 0.013691416010260582, "contribution_to_active": 0.6761854887008667, "mix_mean": 0.6765327453613281, "mix_low": 0.0005235671997070312, "mix_high": 0.0038518905639648438, "read_cosine": 0.40120500326156616, "beta_mean": 0.0001256318500963971, "nonfinite": 0.0, "weighted_memory_to_active": 0.010718483477830887, "mix_std": 0.22416599094867706, "beta_std": 0.0027125850319862366, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006446128711104393, "evidence_weights": [-0.22500963509082794, -0.34488698840141296, 1.0065613985061646], "agreement_weight": 0.2762663662433624, "orthogonal_weight": -0.6894571185112}, {"head": 3, "memory_to_active": 0.0991143211722374, "contribution_to_active": 0.639103889465332, "mix_mean": 0.5797181129455566, "mix_low": 0.0009298324584960938, "mix_high": 0.0034856796264648438, "read_cosine": 0.8108624815940857, "beta_mean": 0.008338112384080887, "nonfinite": 0.0, "weighted_memory_to_active": 0.07514745742082596, "mix_std": 0.2564011812210083, "beta_std": 0.007997668348252773, "beta_low": 0.70330810546875, "beta_high": 0.0, "orthogonal_to_active": 0.023396780714392662, "evidence_weights": [-0.18490533530712128, -0.29957807064056396, 0.5501856803894043], "agreement_weight": 0.23000241816043854, "orthogonal_weight": -0.621817946434021}, {"head": 4, "memory_to_active": 0.3301629424095154, "contribution_to_active": 0.2589632570743561, "mix_mean": 0.22783434391021729, "mix_low": 0.012904167175292969, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21684476733207703, "beta_mean": 0.0032943610567599535, "nonfinite": 0.0, "weighted_memory_to_active": 0.10432916134595871, "mix_std": 0.1960887759923935, "beta_std": 0.014686550945043564, "beta_low": 0.93255615234375, "beta_high": 0.0, "orthogonal_to_active": 0.025473814457654953, "evidence_weights": [-0.08425457775592804, -0.16565531492233276, 0.45288756489753723], "agreement_weight": 0.07213762402534485, "orthogonal_weight": -0.2589888274669647}, {"head": 5, "memory_to_active": 0.3694675862789154, "contribution_to_active": 0.6354090571403503, "mix_mean": 0.25838151574134827, "mix_low": 0.06783103942871094, "mix_high": 0.00018787384033203125, "read_cosine": 0.47428256273269653, "beta_mean": 0.06493055075407028, "nonfinite": 0.0, "weighted_memory_to_active": 0.18984007835388184, "mix_std": 0.2678464651107788, "beta_std": 0.07198639959096909, "beta_low": 0.07623291015625, "beta_high": 0.0, "orthogonal_to_active": 0.3071262538433075, "evidence_weights": [-0.13499070703983307, -0.1566094607114792, -0.1245473101735115], "agreement_weight": -0.005915926303714514, "orthogonal_weight": 1.8870935440063477}]}, {"layer": 9, "memory_to_active": 0.9305659532546997, "contribution_to_active": 0.7008136510848999, "mix_mean": 0.3632389008998871, "mix_low": 0.08614222705364227, "mix_high": 0.0061581931076943874, "read_cosine": 0.3342103958129883, "beta_mean": 0.040804311633110046, "nonfinite": 0.0, "weighted_memory_to_active": 0.39451098442077637, "mix_std": 0.22981223464012146, "beta_std": 0.033605970442295074, "beta_low": 0.53375244140625, "beta_high": 0.0, "orthogonal_to_active": 0.03849033638834953, "output_influence": 0.5320655703544617, "residual_influence": 0.15448163449764252, "state_active": 0.0443282425403595, "state_memory": 0.016295958310365677, "state_peak": 53.14569091796875, "growth_active": 1.1939619779586792, "growth_memory": 1.319928526878357, "heads": [{"head": 0, "memory_to_active": 0.0006700837984681129, "contribution_to_active": 0.5717759132385254, "mix_mean": 0.6446256637573242, "mix_low": 0.001399993896484375, "mix_high": 0.02494525909423828, "read_cosine": 0.33746030926704407, "beta_mean": 1.9347855868545594e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0004239559348206967, "mix_std": 0.2887495756149292, "beta_std": 5.490502189786639e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00047502643428742886, "evidence_weights": [-0.30449268221855164, -0.31227755546569824, 0.5429880619049072], "agreement_weight": 0.12183666974306107, "orthogonal_weight": -1.1854828596115112}, {"head": 1, "memory_to_active": 0.018199291080236435, "contribution_to_active": 0.6930575966835022, "mix_mean": 0.5626221895217896, "mix_low": 0.006926536560058594, "mix_high": 0.0030317306518554688, "read_cosine": 0.3717460036277771, "beta_mean": 4.742130477097817e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013504339382052422, "mix_std": 0.2820044755935669, "beta_std": 0.0001108531141653657, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009523190557956696, "evidence_weights": [-0.2417948842048645, -0.2965984642505646, 1.068925142288208], "agreement_weight": 0.4652756452560425, "orthogonal_weight": -0.8392514586448669}, {"head": 2, "memory_to_active": 0.1700502634048462, "contribution_to_active": 0.6493431925773621, "mix_mean": 0.593562126159668, "mix_low": 0.0017461776733398438, "mix_high": 0.007573127746582031, "read_cosine": 0.33206304907798767, "beta_mean": 0.0007381924078799784, "nonfinite": 0.0, "weighted_memory_to_active": 0.11399859189987183, "mix_std": 0.26808637380599976, "beta_std": 0.0047637890093028545, "beta_low": 0.99505615234375, "beta_high": 0.0, "orthogonal_to_active": 0.06786851584911346, "evidence_weights": [-0.20342965424060822, -0.2782573997974396, 0.7748313546180725], "agreement_weight": 0.32843291759490967, "orthogonal_weight": -0.6841399073600769}, {"head": 3, "memory_to_active": 1.0397069454193115, "contribution_to_active": 0.2059776782989502, "mix_mean": 0.09839114546775818, "mix_low": 0.18914031982421875, "mix_high": 7.534027099609375e-05, "read_cosine": 0.37308281660079956, "beta_mean": 0.05287719517946243, "nonfinite": 0.0, "weighted_memory_to_active": 0.18305285274982452, "mix_std": 0.15067991614341736, "beta_std": 0.045315951108932495, "beta_low": 0.0235595703125, "beta_high": 0.0, "orthogonal_to_active": 0.04926756024360657, "evidence_weights": [0.019006280228495598, -0.05373344570398331, 0.19987066090106964], "agreement_weight": 0.09607190638780594, "orthogonal_weight": -0.29917383193969727}, {"head": 4, "memory_to_active": 2.9024457931518555, "contribution_to_active": 1.5751935243606567, "mix_mean": 0.1469964236021042, "mix_low": 0.129119873046875, "mix_high": 0.0009737014770507812, "read_cosine": 0.09034769237041473, "beta_mean": 0.1416739672422409, "nonfinite": 0.0, "weighted_memory_to_active": 1.4960873126983643, "mix_std": 0.2054852694272995, "beta_std": 0.08507028967142105, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.021452536806464195, "evidence_weights": [-0.08168777078390121, -0.15644307434558868, -0.022741684690117836], "agreement_weight": -0.06460636109113693, "orthogonal_weight": -0.01483587734401226}, {"head": 5, "memory_to_active": 1.452323079109192, "contribution_to_active": 0.5095339417457581, "mix_mean": 0.1332358568906784, "mix_low": 0.1885204315185547, "mix_high": 0.00034999847412109375, "read_cosine": 0.5005624890327454, "beta_mean": 0.049487143754959106, "nonfinite": 0.0, "weighted_memory_to_active": 0.5599986910820007, "mix_std": 0.18386782705783844, "beta_std": 0.06636941432952881, "beta_low": 0.18389892578125, "beta_high": 0.0, "orthogonal_to_active": 0.08235517144203186, "evidence_weights": [0.0858946144580841, -0.137933149933815, 0.14450307190418243], "agreement_weight": 0.01370462030172348, "orthogonal_weight": 0.20844900608062744}]}, {"layer": 10, "memory_to_active": 0.2248145341873169, "contribution_to_active": 0.7251147627830505, "mix_mean": 0.5184199810028076, "mix_low": 0.020910264924168587, "mix_high": 0.0048732757568359375, "read_cosine": 0.3759984076023102, "beta_mean": 0.04291432723402977, "nonfinite": 0.0, "weighted_memory_to_active": 0.100313700735569, "mix_std": 0.2686474919319153, "beta_std": 0.04547818750143051, "beta_low": 0.6634725332260132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.25474414229393005, "output_influence": 0.45380789041519165, "residual_influence": 0.1604723334312439, "state_active": 0.05239266902208328, "state_memory": 0.015539870597422123, "state_peak": 125.62014770507812, "growth_active": 1.0360018014907837, "growth_memory": 0.9974712133407593, "heads": [{"head": 0, "memory_to_active": 0.8274674415588379, "contribution_to_active": 2.0424418449401855, "mix_mean": 0.5740320682525635, "mix_low": 0.08404731750488281, "mix_high": 0.009806632995605469, "read_cosine": 0.10024304687976837, "beta_mean": 0.18910691142082214, "nonfinite": 0.0, "weighted_memory_to_active": 0.4814884066581726, "mix_std": 0.39111509919166565, "beta_std": 0.20586873590946198, "beta_low": 0.143310546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.4378911256790161, "evidence_weights": [-0.09863673895597458, -0.38973236083984375, -1.7591341733932495], "agreement_weight": 0.02534715086221695, "orthogonal_weight": 3.0126309394836426}, {"head": 1, "memory_to_active": 0.011002110317349434, "contribution_to_active": 0.6073406934738159, "mix_mean": 0.6238793134689331, "mix_low": 0.0017175674438476562, "mix_high": 0.013648033142089844, "read_cosine": 0.2853543162345886, "beta_mean": 7.05387574271299e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006136632524430752, "mix_std": 0.26525142788887024, "beta_std": 0.00010629682219587266, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009566381573677063, "evidence_weights": [-0.2514208257198334, -0.19559569656848907, 0.558881402015686], "agreement_weight": 0.5738697648048401, "orthogonal_weight": -1.668747901916504}, {"head": 2, "memory_to_active": 0.013332825154066086, "contribution_to_active": 0.626882791519165, "mix_mean": 0.6562732458114624, "mix_low": 0.00026226043701171875, "mix_high": 0.0043582916259765625, "read_cosine": 0.02457161620259285, "beta_mean": 2.55136164923897e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005984268616884947, "mix_std": 0.2493017017841339, "beta_std": 9.820645936997607e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026890323497354984, "evidence_weights": [-0.26245805621147156, -0.2805783450603485, 0.6317851543426514], "agreement_weight": -0.2059430181980133, "orthogonal_weight": -0.43017715215682983}, {"head": 3, "memory_to_active": 0.3606017827987671, "contribution_to_active": 0.10506495088338852, "mix_mean": 0.271145761013031, "mix_low": 0.03658294677734375, "mix_high": 0.00022792816162109375, "read_cosine": 0.9121842384338379, "beta_mean": 0.061998359858989716, "nonfinite": 0.0, "weighted_memory_to_active": 0.04142170399427414, "mix_std": 0.23119983077049255, "beta_std": 0.05840267986059189, "beta_low": 0.00323486328125, "beta_high": 0.0, "orthogonal_to_active": 0.044916316866874695, "evidence_weights": [-0.09959817677736282, -0.14548353850841522, 0.16172969341278076], "agreement_weight": 0.01684490218758583, "orthogonal_weight": 1.4096554517745972}, {"head": 4, "memory_to_active": 0.01859303191304207, "contribution_to_active": 0.5525513887405396, "mix_mean": 0.5841024518013, "mix_low": 0.000194549560546875, "mix_high": 0.0011949539184570312, "read_cosine": 0.08582404255867004, "beta_mean": 5.4712247219868004e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010767437517642975, "mix_std": 0.2466745674610138, "beta_std": 0.00010559325892245397, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014755230396986008, "evidence_weights": [-0.18430347740650177, -0.25867581367492676, 0.9871315360069275], "agreement_weight": 0.14765286445617676, "orthogonal_weight": -1.4455454349517822}, {"head": 5, "memory_to_active": 0.11789005994796753, "contribution_to_active": 0.4164069890975952, "mix_mean": 0.4010868966579437, "mix_low": 0.0026569366455078125, "mix_high": 3.814697265625e-06, "read_cosine": 0.8478131294250488, "beta_mean": 0.006229911465197802, "nonfinite": 0.0, "weighted_memory_to_active": 0.056083738803863525, "mix_std": 0.22834227979183197, "beta_std": 0.008287585340440273, "beta_low": 0.83428955078125, "beta_high": 0.0, "orthogonal_to_active": 0.01864665374159813, "evidence_weights": [-0.14090821146965027, -0.16229218244552612, 0.26870742440223694], "agreement_weight": 0.09199962764978409, "orthogonal_weight": -0.6560330986976624}]}, {"layer": 11, "memory_to_active": 0.08688998222351074, "contribution_to_active": 0.6168278455734253, "mix_mean": 0.5154278874397278, "mix_low": 0.11118316650390625, "mix_high": 0.042199455201625824, "read_cosine": 0.3520200848579407, "beta_mean": 0.02424412965774536, "nonfinite": 0.0, "weighted_memory_to_active": 0.04636042192578316, "mix_std": 0.3243348300457001, "beta_std": 0.03492743521928787, "beta_low": 0.6776123046875, "beta_high": 0.0, "orthogonal_to_active": 0.04845767468214035, "output_influence": 0.5540012717247009, "residual_influence": 0.14587974548339844, "state_active": 0.202241912484169, "state_memory": 0.003827689215540886, "state_peak": 35.3094367980957, "growth_active": 1.1092077493667603, "growth_memory": 1.5230144262313843, "heads": [{"head": 0, "memory_to_active": 0.010314011946320534, "contribution_to_active": 0.5955430269241333, "mix_mean": 0.6426722407341003, "mix_low": 0.0008840560913085938, "mix_high": 0.0074748992919921875, "read_cosine": 0.25279170274734497, "beta_mean": 9.493535617366433e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0076166060753166676, "mix_std": 0.25947195291519165, "beta_std": 0.004036412574350834, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.006155610550194979, "evidence_weights": [-0.17195801436901093, -0.3634340465068817, 0.7617321610450745], "agreement_weight": 0.35889577865600586, "orthogonal_weight": -0.8386337757110596}, {"head": 1, "memory_to_active": 0.15701453387737274, "contribution_to_active": 0.4120064377784729, "mix_mean": 0.44665849208831787, "mix_low": 0.006137847900390625, "mix_high": 0.00052642822265625, "read_cosine": 0.466116726398468, "beta_mean": 0.006219481583684683, "nonfinite": 0.0, "weighted_memory_to_active": 0.073264941573143, "mix_std": 0.26704317331314087, "beta_std": 0.016489356756210327, "beta_low": 0.876953125, "beta_high": 0.0, "orthogonal_to_active": 0.038675855845212936, "evidence_weights": [-0.1520691066980362, -0.24909278750419617, 0.6019083857536316], "agreement_weight": 0.06993568688631058, "orthogonal_weight": -0.6287490129470825}, {"head": 2, "memory_to_active": 0.00656565185636282, "contribution_to_active": 0.9863698482513428, "mix_mean": 0.4358886778354645, "mix_low": 0.30733680725097656, "mix_high": 0.15927982330322266, "read_cosine": 0.2090976983308792, "beta_mean": 0.06433147937059402, "nonfinite": 0.0, "weighted_memory_to_active": 0.004826331976801157, "mix_std": 0.44889914989471436, "beta_std": 0.08304324746131897, "beta_low": 0.45745849609375, "beta_high": 0.0, "orthogonal_to_active": 0.043010927736759186, "evidence_weights": [0.04170763120055199, -0.3857838213443756, -1.3301855325698853], "agreement_weight": 0.007291911635547876, "orthogonal_weight": 9.374192237854004}, {"head": 3, "memory_to_active": 0.008769198320806026, "contribution_to_active": 0.8502501845359802, "mix_mean": 0.4083704650402069, "mix_low": 0.3451995849609375, "mix_high": 0.07845497131347656, "read_cosine": 0.5667914748191833, "beta_mean": 0.049198467284440994, "nonfinite": 0.0, "weighted_memory_to_active": 0.005537243559956551, "mix_std": 0.43848806619644165, "beta_std": 0.07371953874826431, "beta_low": 0.50067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.03518705070018768, "evidence_weights": [0.2035888433456421, -0.2939670979976654, -0.14840857684612274], "agreement_weight": 0.05266723781824112, "orthogonal_weight": 9.660650253295898}, {"head": 4, "memory_to_active": 0.24200454354286194, "contribution_to_active": 0.445748507976532, "mix_mean": 0.5900161862373352, "mix_low": 0.005648612976074219, "mix_high": 0.0017137527465820312, "read_cosine": 0.03668564558029175, "beta_mean": 0.01631191000342369, "nonfinite": 0.0, "weighted_memory_to_active": 0.13636060059070587, "mix_std": 0.27127718925476074, "beta_std": 0.018314087763428688, "beta_low": 0.477294921875, "beta_high": 0.0, "orthogonal_to_active": 0.13387474417686462, "evidence_weights": [-0.2410770058631897, -0.23960734903812408, 1.915095567703247], "agreement_weight": -0.21249093115329742, "orthogonal_weight": -0.995894193649292}, {"head": 5, "memory_to_active": 0.09667190909385681, "contribution_to_active": 0.4110490083694458, "mix_mean": 0.5689610838890076, "mix_low": 0.00189208984375, "mix_high": 0.0057468414306640625, "read_cosine": 0.5806373953819275, "beta_mean": 0.009308497421443462, "nonfinite": 0.0, "weighted_memory_to_active": 0.050556812435388565, "mix_std": 0.2608293294906616, "beta_std": 0.01396197360008955, "beta_low": 0.753662109375, "beta_high": 0.0, "orthogonal_to_active": 0.0338418185710907, "evidence_weights": [-0.2107611447572708, -0.31499016284942627, 0.6537122130393982], "agreement_weight": 0.08801373094320297, "orthogonal_weight": -0.8575368523597717}]}, {"layer": 12, "memory_to_active": 0.22340872883796692, "contribution_to_active": 0.5832207202911377, "mix_mean": 0.5378055572509766, "mix_low": 0.00906244944781065, "mix_high": 0.0062716808170080185, "read_cosine": 0.23038259148597717, "beta_mean": 0.015486681833863258, "nonfinite": 0.0, "weighted_memory_to_active": 0.11443252861499786, "mix_std": 0.26163434982299805, "beta_std": 0.015628833323717117, "beta_low": 0.8275655508041382, "beta_high": 0.0, "orthogonal_to_active": 0.07024569809436798, "output_influence": 0.5001683235168457, "residual_influence": 0.14297722280025482, "state_active": 0.03580359369516373, "state_memory": 0.008601146750152111, "state_peak": 62.70146942138672, "growth_active": 1.133479118347168, "growth_memory": 1.1528555154800415, "heads": [{"head": 0, "memory_to_active": 0.035859111696481705, "contribution_to_active": 0.6201033592224121, "mix_mean": 0.5770045518875122, "mix_low": 0.000514984130859375, "mix_high": 0.0057697296142578125, "read_cosine": 0.2961687743663788, "beta_mean": 0.0006307955482043326, "nonfinite": 0.0, "weighted_memory_to_active": 0.019787907600402832, "mix_std": 0.26625263690948486, "beta_std": 0.0013235234655439854, "beta_low": 0.9962158203125, "beta_high": 0.0, "orthogonal_to_active": 0.01823873445391655, "evidence_weights": [-0.20886914432048798, -0.2752115726470947, 0.026941528543829918], "agreement_weight": 0.10165827721357346, "orthogonal_weight": -0.9922385811805725}, {"head": 1, "memory_to_active": 0.09647966921329498, "contribution_to_active": 0.5481694936752319, "mix_mean": 0.5548701286315918, "mix_low": 0.00036716461181640625, "mix_high": 0.0016889572143554688, "read_cosine": 0.38453978300094604, "beta_mean": 0.0006349316681735218, "nonfinite": 0.0, "weighted_memory_to_active": 0.05942848324775696, "mix_std": 0.2660108208656311, "beta_std": 0.008615187369287014, "beta_low": 0.99658203125, "beta_high": 0.0, "orthogonal_to_active": 0.04988928139209747, "evidence_weights": [-0.20369397103786469, -0.27192968130111694, 1.038643717765808], "agreement_weight": 0.238877534866333, "orthogonal_weight": -0.9202601313591003}, {"head": 2, "memory_to_active": 0.6119035482406616, "contribution_to_active": 0.4959563612937927, "mix_mean": 0.3031166195869446, "mix_low": 0.02928638458251953, "mix_high": 0.0001697540283203125, "read_cosine": 0.49857455492019653, "beta_mean": 0.09026220440864563, "nonfinite": 0.0, "weighted_memory_to_active": 0.24037300050258636, "mix_std": 0.23815660178661346, "beta_std": 0.06644891947507858, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.2632615864276886, "evidence_weights": [-0.21501870453357697, -0.05759654939174652, 0.009364217519760132], "agreement_weight": 0.023778576403856277, "orthogonal_weight": 1.3617565631866455}, {"head": 3, "memory_to_active": 0.0452592633664608, "contribution_to_active": 0.744661271572113, "mix_mean": 0.709762692451477, "mix_low": 0.0001621246337890625, "mix_high": 0.018866539001464844, "read_cosine": 0.09878826141357422, "beta_mean": 0.00034677176154218614, "nonfinite": 0.0, "weighted_memory_to_active": 0.029424477368593216, "mix_std": 0.23249629139900208, "beta_std": 0.006571182049810886, "beta_low": 0.99725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.02782946452498436, "evidence_weights": [-0.2873614430427551, -0.3344409763813019, 1.6700234413146973], "agreement_weight": 0.5178803205490112, "orthogonal_weight": -0.970790684223175}, {"head": 4, "memory_to_active": 0.4875525236129761, "contribution_to_active": 0.5875924229621887, "mix_mean": 0.4495472311973572, "mix_low": 0.01739025115966797, "mix_high": 0.0051021575927734375, "read_cosine": -0.13625869154930115, "beta_mean": 0.0007189048919826746, "nonfinite": 0.0, "weighted_memory_to_active": 0.2957192063331604, "mix_std": 0.2911622226238251, "beta_std": 0.008012179285287857, "beta_low": 0.97998046875, "beta_high": 0.0, "orthogonal_to_active": 0.030606336891651154, "evidence_weights": [-0.15353992581367493, -0.21799206733703613, -0.09900557994842529], "agreement_weight": 0.006711430381983519, "orthogonal_weight": -0.10744519531726837}, {"head": 5, "memory_to_active": 0.06339816004037857, "contribution_to_active": 0.5028412938117981, "mix_mean": 0.6325318813323975, "mix_low": 0.006653785705566406, "mix_high": 0.0060329437255859375, "read_cosine": 0.24048291146755219, "beta_mean": 0.0003264758561272174, "nonfinite": 0.0, "weighted_memory_to_active": 0.041862063109874725, "mix_std": 0.2757275104522705, "beta_std": 0.0028020073659718037, "beta_low": 0.99462890625, "beta_high": 0.0, "orthogonal_to_active": 0.03164875507354736, "evidence_weights": [-0.23101866245269775, -0.31817227602005005, 1.276066541671753], "agreement_weight": 0.4055326581001282, "orthogonal_weight": -0.8167582154273987}]}, {"layer": 13, "memory_to_active": 0.11873509734869003, "contribution_to_active": 0.5327514410018921, "mix_mean": 0.5353879928588867, "mix_low": 0.0053385100327432156, "mix_high": 0.008487066254019737, "read_cosine": 0.4101351499557495, "beta_mean": 0.014112591743469238, "nonfinite": 0.0, "weighted_memory_to_active": 0.0513463132083416, "mix_std": 0.25812697410583496, "beta_std": 0.01566212996840477, "beta_low": 0.8093770742416382, "beta_high": 0.0, "orthogonal_to_active": 0.051223255693912506, "output_influence": 0.5102014541625977, "residual_influence": 0.17373529076576233, "state_active": 0.12683524191379547, "state_memory": 0.043028440326452255, "state_peak": 98.13121795654297, "growth_active": 0.9438730478286743, "growth_memory": 0.9823572635650635, "heads": [{"head": 0, "memory_to_active": 0.016552714630961418, "contribution_to_active": 0.5870139598846436, "mix_mean": 0.6134871244430542, "mix_low": 0.0007257461547851562, "mix_high": 0.0041217803955078125, "read_cosine": 0.4183967113494873, "beta_mean": 5.5053827964002267e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010516714304685593, "mix_std": 0.26840078830718994, "beta_std": 0.00010850878607016057, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01032158825546503, "evidence_weights": [-0.1967778205871582, -0.2682652473449707, 0.7845954298973083], "agreement_weight": 0.16114969551563263, "orthogonal_weight": -1.0858113765716553}, {"head": 1, "memory_to_active": 0.008896974846720695, "contribution_to_active": 0.5923420786857605, "mix_mean": 0.56391441822052, "mix_low": 0.0019483566284179688, "mix_high": 0.004492759704589844, "read_cosine": 0.2291509509086609, "beta_mean": 3.5790682886727154e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0048474776558578014, "mix_std": 0.2773926556110382, "beta_std": 6.148177635623142e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00627306941896677, "evidence_weights": [-0.18099819123744965, -0.26244819164276123, 0.5915082693099976], "agreement_weight": 0.4659714698791504, "orthogonal_weight": -1.3489493131637573}, {"head": 2, "memory_to_active": 0.06043504923582077, "contribution_to_active": 0.47709760069847107, "mix_mean": 0.39066895842552185, "mix_low": 0.0010356903076171875, "mix_high": 0.0001430511474609375, "read_cosine": 0.6071017384529114, "beta_mean": 0.00775696337223053, "nonfinite": 0.0, "weighted_memory_to_active": 0.02722315862774849, "mix_std": 0.23217809200286865, "beta_std": 0.028974004089832306, "beta_low": 0.86376953125, "beta_high": 0.0, "orthogonal_to_active": 0.015221389941871166, "evidence_weights": [-0.18352150917053223, -0.20788249373435974, 0.407998263835907], "agreement_weight": 0.007461373694241047, "orthogonal_weight": -0.8354094624519348}, {"head": 3, "memory_to_active": 0.16409987211227417, "contribution_to_active": 0.5747131705284119, "mix_mean": 0.6604098081588745, "mix_low": 0.021228790283203125, "mix_high": 0.028230667114257812, "read_cosine": -0.004649532958865166, "beta_mean": 0.0009713222971186042, "nonfinite": 0.0, "weighted_memory_to_active": 0.12652942538261414, "mix_std": 0.28432023525238037, "beta_std": 0.007466914132237434, "beta_low": 0.99310302734375, "beta_high": 0.0, "orthogonal_to_active": 0.10833359509706497, "evidence_weights": [-0.315513551235199, -0.24464388191699982, 1.6002198457717896], "agreement_weight": 6.017067909240723, "orthogonal_weight": -0.9764295220375061}, {"head": 4, "memory_to_active": 0.40781617164611816, "contribution_to_active": 0.24495352804660797, "mix_mean": 0.3221065402030945, "mix_low": 0.0047359466552734375, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8922159075737, "beta_mean": 0.07562323659658432, "nonfinite": 0.0, "weighted_memory_to_active": 0.10171933472156525, "mix_std": 0.2151884287595749, "beta_std": 0.05425494909286499, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.13575394451618195, "evidence_weights": [-0.07748246937990189, -0.2281622588634491, 0.46037736535072327], "agreement_weight": 0.011877772398293018, "orthogonal_weight": 2.230847120285034}, {"head": 5, "memory_to_active": 0.05460977554321289, "contribution_to_active": 0.7203882932662964, "mix_mean": 0.6617408990859985, "mix_low": 0.0023565292358398438, "mix_high": 0.013929367065429688, "read_cosine": 0.3185949921607971, "beta_mean": 0.00023318456078413874, "nonfinite": 0.0, "weighted_memory_to_active": 0.03724175691604614, "mix_std": 0.2712816298007965, "beta_std": 0.0031069209799170494, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.031435929238796234, "evidence_weights": [-0.22268037497997284, -0.30094993114471436, 0.8718133568763733], "agreement_weight": 0.1814860850572586, "orthogonal_weight": -0.9236865639686584}]}, {"layer": 14, "memory_to_active": 0.2345125377178192, "contribution_to_active": 0.4116598069667816, "mix_mean": 0.4283924102783203, "mix_low": 0.0265299491584301, "mix_high": 0.006101449951529503, "read_cosine": 0.6218520998954773, "beta_mean": 0.060612648725509644, "nonfinite": 0.0, "weighted_memory_to_active": 0.10058566927909851, "mix_std": 0.24694375693798065, "beta_std": 0.05911571532487869, "beta_low": 0.4936421811580658, "beta_high": 0.0, "orthogonal_to_active": 0.04244499281048775, "output_influence": 0.34745657444000244, "residual_influence": 0.13843148946762085, "state_active": 0.05516503006219864, "state_memory": 0.015535516664385796, "state_peak": 42.19459915161133, "growth_active": 1.074083685874939, "growth_memory": 1.2482383251190186, "heads": [{"head": 0, "memory_to_active": 0.3679281771183014, "contribution_to_active": 0.1873486191034317, "mix_mean": 0.19334125518798828, "mix_low": 0.01848316192626953, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8140691518783569, "beta_mean": 0.07093285024166107, "nonfinite": 0.0, "weighted_memory_to_active": 0.08152835816144943, "mix_std": 0.18331213295459747, "beta_std": 0.06225945055484772, "beta_low": 0.0067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.08334873616695404, "evidence_weights": [-0.10771170258522034, -0.04445489123463631, 0.4204898774623871], "agreement_weight": 0.02619224414229393, "orthogonal_weight": 1.7385497093200684}, {"head": 1, "memory_to_active": 0.4869706928730011, "contribution_to_active": 0.28941261768341064, "mix_mean": 0.22542591392993927, "mix_low": 0.1248178482055664, "mix_high": 1.621246337890625e-05, "read_cosine": 0.8730272650718689, "beta_mean": 0.24206528067588806, "nonfinite": 0.0, "weighted_memory_to_active": 0.2263280302286148, "mix_std": 0.2527157664299011, "beta_std": 0.24222905933856964, "beta_low": 0.10369873046875, "beta_high": 0.0, "orthogonal_to_active": 0.01595120318233967, "evidence_weights": [-0.0034128627739846706, -0.1973503679037094, -0.09133759886026382], "agreement_weight": 0.05916908010840416, "orthogonal_weight": 0.19912908971309662}, {"head": 2, "memory_to_active": 0.13269928097724915, "contribution_to_active": 0.5784815549850464, "mix_mean": 0.5878201723098755, "mix_low": 0.0028858184814453125, "mix_high": 0.0041599273681640625, "read_cosine": 0.41633960604667664, "beta_mean": 0.002085365355014801, "nonfinite": 0.0, "weighted_memory_to_active": 0.08796288073062897, "mix_std": 0.26825734972953796, "beta_std": 0.003191921394318342, "beta_low": 0.97894287109375, "beta_high": 0.0, "orthogonal_to_active": 0.06755054742097855, "evidence_weights": [-0.1368873119354248, -0.3153831660747528, 0.9323921799659729], "agreement_weight": 0.0904955044388771, "orthogonal_weight": -0.8794221878051758}, {"head": 3, "memory_to_active": 0.038771018385887146, "contribution_to_active": 0.5776183009147644, "mix_mean": 0.6645798683166504, "mix_low": 0.0048542022705078125, "mix_high": 0.021165847778320312, "read_cosine": -0.0202455073595047, "beta_mean": 0.005473504774272442, "nonfinite": 0.0, "weighted_memory_to_active": 0.024715252220630646, "mix_std": 0.25810447335243225, "beta_std": 0.008031276986002922, "beta_low": 0.85443115234375, "beta_high": 0.0, "orthogonal_to_active": 0.018965207040309906, "evidence_weights": [-0.34484031796455383, -0.30935198068618774, 0.28323134779930115], "agreement_weight": 0.06950853019952774, "orthogonal_weight": -0.8392584323883057}, {"head": 4, "memory_to_active": 0.16757841408252716, "contribution_to_active": 0.5042564272880554, "mix_mean": 0.5199962258338928, "mix_low": 0.0037364959716796875, "mix_high": 0.009649276733398438, "read_cosine": 0.8983501195907593, "beta_mean": 0.04100223630666733, "nonfinite": 0.0, "weighted_memory_to_active": 0.10034792870283127, "mix_std": 0.2627551555633545, "beta_std": 0.02914832904934883, "beta_low": 0.0653076171875, "beta_high": 0.0, "orthogonal_to_active": 0.02670312114059925, "evidence_weights": [-0.215126171708107, -0.2420763224363327, 0.4330188035964966], "agreement_weight": 0.13357940316200256, "orthogonal_weight": -0.6164751648902893}, {"head": 5, "memory_to_active": 0.21312765777111053, "contribution_to_active": 0.3328413665294647, "mix_mean": 0.37919098138809204, "mix_low": 0.00440216064453125, "mix_high": 0.0016107559204101562, "read_cosine": 0.7495717406272888, "beta_mean": 0.002116644522175193, "nonfinite": 0.0, "weighted_memory_to_active": 0.08263149112462997, "mix_std": 0.25651758909225464, "beta_std": 0.009834246709942818, "beta_low": 0.9527587890625, "beta_high": 0.0, "orthogonal_to_active": 0.04215113818645477, "evidence_weights": [-0.174361452460289, -0.17547273635864258, 0.3254537284374237], "agreement_weight": 0.07123833894729614, "orthogonal_weight": -0.7594243884086609}]}, {"layer": 15, "memory_to_active": 0.05388832837343216, "contribution_to_active": 0.48239797353744507, "mix_mean": 0.4150352478027344, "mix_low": 0.008576711639761925, "mix_high": 0.0026356379967182875, "read_cosine": 0.662803053855896, "beta_mean": 0.03803884983062744, "nonfinite": 0.0, "weighted_memory_to_active": 0.02681635692715645, "mix_std": 0.2844817042350769, "beta_std": 0.04493772238492966, "beta_low": 0.7047119736671448, "beta_high": 0.0, "orthogonal_to_active": 0.007943347096443176, "output_influence": 0.30561041831970215, "residual_influence": 0.1718999147415161, "state_active": 0.15332016348838806, "state_memory": 0.010692520067095757, "state_peak": 33.92912292480469, "growth_active": 1.1784220933914185, "growth_memory": 1.297639012336731, "heads": [{"head": 0, "memory_to_active": 0.09990963339805603, "contribution_to_active": 0.4858723282814026, "mix_mean": 0.3784165382385254, "mix_low": 0.010995864868164062, "mix_high": 0.005375862121582031, "read_cosine": 0.5822452306747437, "beta_mean": 0.03775593265891075, "nonfinite": 0.0, "weighted_memory_to_active": 0.049113642424345016, "mix_std": 0.2826453447341919, "beta_std": 0.026221776381134987, "beta_low": 0.067626953125, "beta_high": 0.0, "orthogonal_to_active": 0.02702554315328598, "evidence_weights": [-0.18741492927074432, -0.18832732737064362, 0.461937814950943], "agreement_weight": 0.09376074373722076, "orthogonal_weight": -0.7708067297935486}, {"head": 1, "memory_to_active": 0.11014869809150696, "contribution_to_active": 0.4223862290382385, "mix_mean": 0.4366588592529297, "mix_low": 0.0015277862548828125, "mix_high": 0.005690574645996094, "read_cosine": 0.9246295690536499, "beta_mean": 0.00044297651038505137, "nonfinite": 0.0, "weighted_memory_to_active": 0.04722553864121437, "mix_std": 0.2881307601928711, "beta_std": 0.0017248070798814297, "beta_low": 0.9964599609375, "beta_high": 0.0, "orthogonal_to_active": 0.0014340236084535718, "evidence_weights": [-0.1490115225315094, -0.15712198615074158, 0.17347821593284607], "agreement_weight": 0.1248074546456337, "orthogonal_weight": 0.0634675920009613}, {"head": 2, "memory_to_active": 0.005441063083708286, "contribution_to_active": 0.4771574139595032, "mix_mean": 0.3420524001121521, "mix_low": 0.013239860534667969, "mix_high": 5.14984130859375e-05, "read_cosine": 0.7880534529685974, "beta_mean": 7.554428884759545e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002857953542843461, "mix_std": 0.27029526233673096, "beta_std": 0.00015987057122401893, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008629314834252, "evidence_weights": [-0.2074299305677414, -0.2640833854675293, 0.11751510202884674], "agreement_weight": 0.010301033966243267, "orthogonal_weight": 0.7987159490585327}, {"head": 3, "memory_to_active": 0.0012455463875085115, "contribution_to_active": 0.3741408884525299, "mix_mean": 0.3964422643184662, "mix_low": 0.004500389099121094, "mix_high": 0.0004291534423828125, "read_cosine": 0.6156970858573914, "beta_mean": 6.260402187763248e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0003961186157539487, "mix_std": 0.2880171537399292, "beta_std": 1.202333987748716e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00029587262542918324, "evidence_weights": [-0.26528000831604004, -0.24021807312965393, 0.19360136985778809], "agreement_weight": 0.05010048672556877, "orthogonal_weight": -0.7155243158340454}, {"head": 4, "memory_to_active": 0.07829630374908447, "contribution_to_active": 0.6291908621788025, "mix_mean": 0.37216562032699585, "mix_low": 0.010964393615722656, "mix_high": 0.0009126663208007812, "read_cosine": 0.7442350387573242, "beta_mean": 0.1870010793209076, "nonfinite": 0.0, "weighted_memory_to_active": 0.04304388538002968, "mix_std": 0.306263267993927, "beta_std": 0.21954190731048584, "beta_low": 0.1968994140625, "beta_high": 0.0, "orthogonal_to_active": 0.006443657446652651, "evidence_weights": [-0.20679105818271637, -0.2529178559780121, -0.010774418711662292], "agreement_weight": 0.01333112083375454, "orthogonal_weight": 0.34841105341911316}, {"head": 5, "memory_to_active": 0.028288699686527252, "contribution_to_active": 0.5056400299072266, "mix_mean": 0.5644756555557251, "mix_low": 0.010231971740722656, "mix_high": 0.0033540725708007812, "read_cosine": 0.3219575881958008, "beta_mean": 0.0029513020999729633, "nonfinite": 0.0, "weighted_memory_to_active": 0.018261000514030457, "mix_std": 0.2715383470058441, "beta_std": 0.02196592651307583, "beta_low": 0.96728515625, "beta_high": 0.0, "orthogonal_to_active": 0.011598056182265282, "evidence_weights": [-0.28216299414634705, -0.3004186153411865, 0.6855509877204895], "agreement_weight": 0.30745282769203186, "orthogonal_weight": 0.8214974999427795}]}]}, {"step": 59000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8418924980032898, "memory_to_active": 0.8034167289733887, "contribution_to_active": 0.5621161460876465, "mix_mean": 0.4158247113227844, "mix_low": 0.13039763271808624, "mix_high": 0.010849843733012676, "read_cosine": 0.3459205627441406, "beta_mean": 0.06917613744735718, "nonfinite": 0.0, "weighted_memory_to_active": 0.20396573841571808, "mix_std": 0.22309048473834991, "beta_std": 0.062041766941547394, "beta_low": 0.5709667801856995, "beta_high": 3.306071084807627e-05, "orthogonal_to_active": 0.06064971536397934, "samples": 4, "scored_tokens": 899, "normal_loss": 3.076840400695801, "normal_token_accuracy": 0.44382646679878235, "state_active": 0.06432440876960754, "state_memory": 0.017605241388082504, "state_peak": 125.90663146972656, "growth_active": 1.1345264911651611, "growth_memory": 1.312481164932251, "output_influence": 0.43132346868515015, "residual_influence": 0.1484985649585724, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.26182222366333, "no_memory_loss_delta": 0.6185740232467651, "scrambled_loss_delta": 0.15300114452838898, "memory_only_loss_delta": 3.327554225921631, "no_evidence_loss_delta": 0.34777504205703735, "no_agreement_loss_delta": 0.02769860439002514, "no_orthogonal_loss_delta": -0.0009547779336571693}, {"index": 94, "tokens": 256, "normal_loss": 3.143792152404785, "no_memory_loss_delta": 0.6768685579299927, "scrambled_loss_delta": 0.45658859610557556, "memory_only_loss_delta": 4.059585094451904, "no_evidence_loss_delta": 0.29141712188720703, "no_agreement_loss_delta": 0.05227004736661911, "no_orthogonal_loss_delta": 0.051144566386938095}, {"index": 188, "tokens": 256, "normal_loss": 3.399179697036743, "no_memory_loss_delta": 0.7697336077690125, "scrambled_loss_delta": 0.13149049878120422, "memory_only_loss_delta": 2.977747678756714, "no_evidence_loss_delta": 0.3852730691432953, "no_agreement_loss_delta": 0.05897487327456474, "no_orthogonal_loss_delta": 0.05432884022593498}, {"index": 283, "tokens": 131, "normal_loss": 1.9545986652374268, "no_memory_loss_delta": 0.44818681478500366, "scrambled_loss_delta": 0.29419976472854614, "memory_only_loss_delta": 2.836470603942871, "no_evidence_loss_delta": 0.12227656692266464, "no_agreement_loss_delta": 0.0044403993524611, "no_orthogonal_loss_delta": -0.018577823415398598}], "agreement_weight_abs_mean": 0.22370195388793945, "agreement_weight_abs_max": 6.0794596672058105, "agreement_weight_mean": 0.19570298492908478, "orthogonal_weight_abs_mean": 0.8886618614196777, "orthogonal_weight_abs_max": 9.7765531539917, "orthogonal_weight_mean": -0.10064113140106201, "no_memory_loss": 3.7302303314208984, "no_memory_loss_delta": 0.6533899903297424, "no_memory_token_accuracy_gain": 0.08342602849006653, "no_memory_token_win_fraction": 0.6896551847457886, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 47.972095707662746, "scrambled_loss": 3.3307409286499023, "scrambled_loss_delta": 0.2539006769657135, "scrambled_token_accuracy_gain": 0.030033370479941368, "scrambled_token_win_fraction": 0.6195772886276245, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 22.42311500638218, "memory_only_loss": 6.441678047180176, "memory_only_loss_delta": 3.364837408065796, "memory_only_token_accuracy_gain": 0.2858732044696808, "memory_only_token_win_fraction": 0.9032257795333862, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.54323639200774, "no_evidence_loss": 3.3863861560821533, "no_evidence_loss_delta": 0.3095454275608063, "no_evidence_token_accuracy_gain": 0.0500556156039238, "no_evidence_token_win_fraction": 0.6162402629852295, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 26.621956321400692, "no_agreement_loss": 3.117053508758545, "no_agreement_loss_delta": 0.0402127206325531, "no_agreement_token_accuracy_gain": 0.011123470962047577, "no_agreement_token_win_fraction": 0.489432692527771, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.94149188485714, "no_orthogonal_loss": 3.103896379470825, "no_orthogonal_loss_delta": 0.02705569937825203, "no_orthogonal_token_accuracy_gain": 0.008898776024580002, "no_orthogonal_token_win_fraction": 0.5850945711135864, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 2.669297258177072, "short_no_memory_loss_delta": 0.5594522356987, "long_context_memory_gain": 0.09393775463104248, "normal_context_loss_gain": 0.1494128406047821, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.5117623805999756, "contribution_to_active": 1.1594879627227783, "mix_mean": 0.10651382803916931, "mix_low": 0.5381447672843933, "mix_high": 0.0050638518296182156, "read_cosine": 0.4225810170173645, "beta_mean": 0.20960354804992676, "nonfinite": 0.0, "weighted_memory_to_active": 1.1920980215072632, "mix_std": 0.09000237286090851, "beta_std": 0.14849361777305603, "beta_low": 0.098052978515625, "beta_high": 0.0, "orthogonal_to_active": 0.03979908674955368, "output_influence": 0.2265998274087906, "residual_influence": 0.2603030204772949, "state_active": 0.0657893642783165, "state_memory": 0.0444520078599453, "state_peak": 45.11723709106445, "growth_active": 1.233715534210205, "growth_memory": 1.278998851776123, "heads": [{"head": 0, "memory_to_active": 2.6729462146759033, "contribution_to_active": 0.5275601744651794, "mix_mean": 0.01945745386183262, "mix_low": 0.6436920166015625, "mix_high": 1.9073486328125e-05, "read_cosine": 0.21396364271640778, "beta_mean": 0.19944852590560913, "nonfinite": 0.0, "weighted_memory_to_active": 0.6489369869232178, "mix_std": 0.06192724406719208, "beta_std": 0.10111270099878311, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07029317319393158, "evidence_weights": [-0.003776309546083212, -0.046559419482946396, -0.05120399594306946], "agreement_weight": -0.21331235766410828, "orthogonal_weight": -0.12163975834846497}, {"head": 1, "memory_to_active": 6.666723251342773, "contribution_to_active": 2.883559465408325, "mix_mean": 0.019118182361125946, "mix_low": 0.640228271484375, "mix_high": 0.0, "read_cosine": 0.4357427656650543, "beta_mean": 0.42781978845596313, "nonfinite": 0.0, "weighted_memory_to_active": 3.177236318588257, "mix_std": 0.05643053725361824, "beta_std": 0.22934289276599884, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0059000845067203045, "evidence_weights": [0.023901527747511864, -0.10347849875688553, 0.637773334980011], "agreement_weight": 0.0289472509175539, "orthogonal_weight": 0.0041012875735759735}, {"head": 2, "memory_to_active": 2.535372495651245, "contribution_to_active": 0.4465794563293457, "mix_mean": 0.013552539050579071, "mix_low": 0.6702413558959961, "mix_high": 0.0, "read_cosine": 0.3210829794406891, "beta_mean": 0.2481948286294937, "nonfinite": 0.0, "weighted_memory_to_active": 0.5167685747146606, "mix_std": 0.03001897782087326, "beta_std": 0.19373348355293274, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04380834847688675, "evidence_weights": [0.07076703757047653, 0.0625213161110878, 0.18172860145568848], "agreement_weight": 0.006981002166867256, "orthogonal_weight": -0.19749774038791656}, {"head": 3, "memory_to_active": 7.2587890625, "contribution_to_active": 2.560861110687256, "mix_mean": 0.029984181746840477, "mix_low": 0.4927215576171875, "mix_high": 0.0, "read_cosine": 0.32538238167762756, "beta_mean": 0.3075155019760132, "nonfinite": 0.0, "weighted_memory_to_active": 2.7018871307373047, "mix_std": 0.06954273581504822, "beta_std": 0.27196410298347473, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.09930934011936188, "evidence_weights": [-0.03869299218058586, -0.03644805774092674, -0.15149250626564026], "agreement_weight": -0.0068303728476166725, "orthogonal_weight": -0.044036876410245895}, {"head": 4, "memory_to_active": 1.8339600563049316, "contribution_to_active": 0.06665761768817902, "mix_mean": 0.010504483245313168, "mix_low": 0.7669391632080078, "mix_high": 0.0, "read_cosine": 0.3081684112548828, "beta_mean": 0.05329499393701553, "nonfinite": 0.0, "weighted_memory_to_active": 0.05193514749407768, "mix_std": 0.03423840552568436, "beta_std": 0.07363787293434143, "beta_low": 0.25531005859375, "beta_high": 0.0, "orthogonal_to_active": 0.0024220820050686598, "evidence_weights": [0.13839364051818848, 0.1034916564822197, -0.1355404257774353], "agreement_weight": -0.14176861941814423, "orthogonal_weight": 0.042498454451560974}, {"head": 5, "memory_to_active": 0.10278280079364777, "contribution_to_active": 0.47170987725257874, "mix_mean": 0.546466052532196, "mix_low": 0.015046119689941406, "mix_high": 0.030364036560058594, "read_cosine": 0.9311457276344299, "beta_mean": 0.02134777419269085, "nonfinite": 0.0, "weighted_memory_to_active": 0.055823914706707, "mix_std": 0.2878563106060028, "beta_std": 0.021170584484934807, "beta_low": 0.33294677734375, "beta_high": 0.0, "orthogonal_to_active": 0.01706148125231266, "evidence_weights": [-0.1776571124792099, -0.22038257122039795, 0.3880230784416199], "agreement_weight": 0.2880476117134094, "orthogonal_weight": -0.6791316866874695}]}, {"layer": 1, "memory_to_active": 2.3467419147491455, "contribution_to_active": 0.47477662563323975, "mix_mean": 0.21142825484275818, "mix_low": 0.35884588956832886, "mix_high": 0.0010895729064941406, "read_cosine": 0.3016071617603302, "beta_mean": 0.1779540628194809, "nonfinite": 0.0, "weighted_memory_to_active": 0.31540632247924805, "mix_std": 0.1154797151684761, "beta_std": 0.1316423863172531, "beta_low": 0.3488667905330658, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.04072652757167816, "output_influence": 0.24749118089675903, "residual_influence": 0.0669902116060257, "state_active": 0.03011738508939743, "state_memory": 0.03535943478345871, "state_peak": 40.349571228027344, "growth_active": 1.2364954948425293, "growth_memory": 1.4608495235443115, "heads": [{"head": 0, "memory_to_active": 0.02416696399450302, "contribution_to_active": 0.49296462535858154, "mix_mean": 0.5171666741371155, "mix_low": 0.005347251892089844, "mix_high": 0.001949310302734375, "read_cosine": -0.016053928062319756, "beta_mean": 0.00017202654271386564, "nonfinite": 0.0, "weighted_memory_to_active": 0.014228553511202335, "mix_std": 0.2795695960521698, "beta_std": 0.00024921519798226655, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013762040063738823, "evidence_weights": [-0.20635440945625305, -0.2702561020851135, 0.3462451994419098], "agreement_weight": 0.19071516394615173, "orthogonal_weight": -1.0240319967269897}, {"head": 1, "memory_to_active": 2.8382205963134766, "contribution_to_active": 0.26852497458457947, "mix_mean": 0.02331315539777279, "mix_low": 0.4639301300048828, "mix_high": 0.0, "read_cosine": 0.5278937220573425, "beta_mean": 0.24054135382175446, "nonfinite": 0.0, "weighted_memory_to_active": 0.3090595006942749, "mix_std": 0.04438471049070358, "beta_std": 0.1812797337770462, "beta_low": 0.00323486328125, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.01977074146270752, "evidence_weights": [0.01809055358171463, -0.06118994578719139, 0.07431744784116745], "agreement_weight": -0.00017891579773277044, "orthogonal_weight": 0.1853952258825302}, {"head": 2, "memory_to_active": 0.045924291014671326, "contribution_to_active": 0.5475144386291504, "mix_mean": 0.6644204258918762, "mix_low": 0.0004138946533203125, "mix_high": 0.0045871734619140625, "read_cosine": 0.5094432830810547, "beta_mean": 0.001904582604765892, "nonfinite": 0.0, "weighted_memory_to_active": 0.03061548061668873, "mix_std": 0.23240014910697937, "beta_std": 0.00489682424813509, "beta_low": 0.97418212890625, "beta_high": 0.0, "orthogonal_to_active": 0.020305462181568146, "evidence_weights": [-0.278461754322052, -0.3703992962837219, 0.655665934085846], "agreement_weight": 0.04228932037949562, "orthogonal_weight": -0.8069443702697754}, {"head": 3, "memory_to_active": 2.9274468421936035, "contribution_to_active": 0.1822448968887329, "mix_mean": 0.024023378267884254, "mix_low": 0.5337028503417969, "mix_high": 0.0, "read_cosine": 0.18397045135498047, "beta_mean": 0.3380551040172577, "nonfinite": 0.0, "weighted_memory_to_active": 0.24902427196502686, "mix_std": 0.06203630566596985, "beta_std": 0.2561822533607483, "beta_low": 0.0557861328125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.09586071968078613, "evidence_weights": [0.03581762686371803, -0.020327795296907425, 0.009693700820207596], "agreement_weight": -0.02085540071129799, "orthogonal_weight": -0.47799962759017944}, {"head": 4, "memory_to_active": 3.7875256538391113, "contribution_to_active": 1.2759151458740234, "mix_mean": 0.03278061002492905, "mix_low": 0.3194255828857422, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3441500663757324, "beta_mean": 0.17566552758216858, "nonfinite": 0.0, "weighted_memory_to_active": 1.2275118827819824, "mix_std": 0.0549030601978302, "beta_std": 0.12954603135585785, "beta_low": 0.02313232421875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.07407993823289871, "evidence_weights": [-0.032196708023548126, -0.004064882639795542, 0.1229119598865509], "agreement_weight": -0.013246329501271248, "orthogonal_weight": 0.08646214008331299}, {"head": 5, "memory_to_active": 4.457167148590088, "contribution_to_active": 0.08149579167366028, "mix_mean": 0.006865264847874641, "mix_low": 0.8302555084228516, "mix_high": 0.0, "read_cosine": 0.26023924350738525, "beta_mean": 0.31138572096824646, "nonfinite": 0.0, "weighted_memory_to_active": 0.061998285353183746, "mix_std": 0.019584469497203827, "beta_std": 0.21770025789737701, "beta_low": 0.036865234375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.020580243319272995, "evidence_weights": [0.15258237719535828, 0.07724740356206894, 0.09876780211925507], "agreement_weight": -0.11135158687829971, "orthogonal_weight": 0.35741227865219116}]}, {"layer": 2, "memory_to_active": 0.989730179309845, "contribution_to_active": 0.3566097021102905, "mix_mean": 0.4027906656265259, "mix_low": 0.24603304266929626, "mix_high": 0.004810810089111328, "read_cosine": 0.28155291080474854, "beta_mean": 0.08098283410072327, "nonfinite": 0.0, "weighted_memory_to_active": 0.08368615806102753, "mix_std": 0.18904323875904083, "beta_std": 0.0680898055434227, "beta_low": 0.5871785879135132, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.0211666040122509, "output_influence": 0.38874655961990356, "residual_influence": 0.10687143355607986, "state_active": 0.03930018097162247, "state_memory": 0.01479118037968874, "state_peak": 24.49624252319336, "growth_active": 1.0605660676956177, "growth_memory": 1.5872395038604736, "heads": [{"head": 0, "memory_to_active": 0.010089187882840633, "contribution_to_active": 0.3433472812175751, "mix_mean": 0.6343305110931396, "mix_low": 0.0005331039428710938, "mix_high": 0.009512901306152344, "read_cosine": 0.12911207973957062, "beta_mean": 6.891149678267539e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006812326144427061, "mix_std": 0.2697007358074188, "beta_std": 0.0001373792183585465, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007230838760733604, "evidence_weights": [-0.3728010356426239, -0.2192009836435318, 0.15974466502666473], "agreement_weight": 0.13207168877124786, "orthogonal_weight": -1.1005830764770508}, {"head": 1, "memory_to_active": 3.0866763591766357, "contribution_to_active": 0.04242812469601631, "mix_mean": 0.004813238047063351, "mix_low": 0.8986129760742188, "mix_high": 0.0, "read_cosine": 0.134272962808609, "beta_mean": 0.23321813344955444, "nonfinite": 0.0, "weighted_memory_to_active": 0.03387461602687836, "mix_std": 0.010322955437004566, "beta_std": 0.1489267349243164, "beta_low": 0.00164794921875, "beta_high": 0.0, "orthogonal_to_active": 0.010655118152499199, "evidence_weights": [0.14394144713878632, 0.11942894011735916, 0.6159200072288513], "agreement_weight": 0.030159583315253258, "orthogonal_weight": 0.3470717668533325}, {"head": 2, "memory_to_active": 0.005849868059158325, "contribution_to_active": 0.41560280323028564, "mix_mean": 0.5443457365036011, "mix_low": 0.0012187957763671875, "mix_high": 0.0035200119018554688, "read_cosine": 0.358906626701355, "beta_mean": 1.8239645214634947e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0025293915532529354, "mix_std": 0.27697595953941345, "beta_std": 3.299966192571446e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00013779420987702906, "evidence_weights": [-0.25205689668655396, -0.2934740483760834, 0.5808186531066895], "agreement_weight": -0.006384173408150673, "orthogonal_weight": -0.0671546682715416}, {"head": 3, "memory_to_active": 0.11902914941310883, "contribution_to_active": 0.5304556488990784, "mix_mean": 0.56355881690979, "mix_low": 0.0003414154052734375, "mix_high": 0.006316184997558594, "read_cosine": 0.5239476561546326, "beta_mean": 0.02224084362387657, "nonfinite": 0.0, "weighted_memory_to_active": 0.07431527227163315, "mix_std": 0.2592741847038269, "beta_std": 0.030539100989699364, "beta_low": 0.4737548828125, "beta_high": 0.0, "orthogonal_to_active": 0.05474013462662697, "evidence_weights": [-0.20380063354969025, -0.3217886984348297, 0.7862125039100647], "agreement_weight": 0.05141223594546318, "orthogonal_weight": -0.894492506980896}, {"head": 4, "memory_to_active": 2.7059805393218994, "contribution_to_active": 0.2808282971382141, "mix_mean": 0.02116669900715351, "mix_low": 0.5746593475341797, "mix_high": 0.0, "read_cosine": 0.2134869545698166, "beta_mean": 0.23031672835350037, "nonfinite": 0.0, "weighted_memory_to_active": 0.37864381074905396, "mix_std": 0.05309270694851875, "beta_std": 0.22885528206825256, "beta_low": 0.04766845703125, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.05304918438196182, "evidence_weights": [0.02075793966650963, 0.0015014569507911801, 0.02408572845160961], "agreement_weight": 0.013699355535209179, "orthogonal_weight": -0.1622895747423172}, {"head": 5, "memory_to_active": 0.010755632072687149, "contribution_to_active": 0.5269960165023804, "mix_mean": 0.6485291123390198, "mix_low": 0.0008325576782226562, "mix_high": 0.009515762329101562, "read_cosine": 0.32959121465682983, "beta_mean": 3.413841113797389e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005941509734839201, "mix_std": 0.2648928165435791, "beta_std": 4.732724482892081e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011865406995639205, "evidence_weights": [-0.281594842672348, -0.2920985817909241, 0.9821281433105469], "agreement_weight": 0.19984516501426697, "orthogonal_weight": -0.23622040450572968}]}, {"layer": 3, "memory_to_active": 0.45385438203811646, "contribution_to_active": 0.47695258259773254, "mix_mean": 0.504041850566864, "mix_low": 0.05828046798706055, "mix_high": 0.011812051758170128, "read_cosine": 0.19110962748527527, "beta_mean": 0.026141222566366196, "nonfinite": 0.0, "weighted_memory_to_active": 0.12901894748210907, "mix_std": 0.26093024015426636, "beta_std": 0.025600004941225052, "beta_low": 0.7643128037452698, "beta_high": 0.0, "orthogonal_to_active": 0.05371660739183426, "output_influence": 0.4219789505004883, "residual_influence": 0.09693342447280884, "state_active": 0.02570284903049469, "state_memory": 0.006893699988722801, "state_peak": 23.124168395996094, "growth_active": 1.1020525693893433, "growth_memory": 1.6119587421417236, "heads": [{"head": 0, "memory_to_active": 0.35832011699676514, "contribution_to_active": 0.596381664276123, "mix_mean": 0.5994410514831543, "mix_low": 0.0016422271728515625, "mix_high": 0.02695178985595703, "read_cosine": 0.26622945070266724, "beta_mean": 0.010096068494021893, "nonfinite": 0.0, "weighted_memory_to_active": 0.23237890005111694, "mix_std": 0.3063453137874603, "beta_std": 0.012786672450602055, "beta_low": 0.6790771484375, "beta_high": 0.0, "orthogonal_to_active": 0.16249442100524902, "evidence_weights": [-0.21513371169567108, -0.24928103387355804, 0.9805105328559875], "agreement_weight": 0.1196996197104454, "orthogonal_weight": -0.7504432797431946}, {"head": 1, "memory_to_active": 0.01582668162882328, "contribution_to_active": 0.43474775552749634, "mix_mean": 0.5626832842826843, "mix_low": 0.0049591064453125, "mix_high": 0.009659767150878906, "read_cosine": 0.18017618358135223, "beta_mean": 7.500039646402001e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009371250867843628, "mix_std": 0.29852399230003357, "beta_std": 0.0001815900905057788, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008032144978642464, "evidence_weights": [-0.26385697722435, -0.2140050232410431, 0.6988190412521362], "agreement_weight": 0.5267858505249023, "orthogonal_weight": -0.9419728517532349}, {"head": 2, "memory_to_active": 2.066192388534546, "contribution_to_active": 0.2888728976249695, "mix_mean": 0.052883245050907135, "mix_low": 0.33029937744140625, "mix_high": 0.0, "read_cosine": 0.20220759510993958, "beta_mean": 0.14111565053462982, "nonfinite": 0.0, "weighted_memory_to_active": 0.32632583379745483, "mix_std": 0.10674473643302917, "beta_std": 0.12627841532230377, "beta_low": 0.02252197265625, "beta_high": 0.0, "orthogonal_to_active": 0.000503909308463335, "evidence_weights": [-0.09156880527734756, 0.0023218346759676933, -0.015074322000145912], "agreement_weight": -0.005834146402776241, "orthogonal_weight": -0.0019513156730681658}, {"head": 3, "memory_to_active": 0.016665054485201836, "contribution_to_active": 0.5246379971504211, "mix_mean": 0.6033299565315247, "mix_low": 0.005473136901855469, "mix_high": 0.0113983154296875, "read_cosine": 0.29698115587234497, "beta_mean": 7.473488221876323e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011582804843783379, "mix_std": 0.2918340265750885, "beta_std": 0.00018455524696037173, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0031234659254550934, "evidence_weights": [-0.25627532601356506, -0.22845295071601868, 2.2273881435394287], "agreement_weight": 2.1041200160980225, "orthogonal_weight": -0.3535010516643524}, {"head": 4, "memory_to_active": 0.01399756595492363, "contribution_to_active": 0.3630617558956146, "mix_mean": 0.4866105914115906, "mix_low": 0.005299568176269531, "mix_high": 0.0021581649780273438, "read_cosine": 0.15827646851539612, "beta_mean": 7.831848779460415e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008789732120931149, "mix_std": 0.29024970531463623, "beta_std": 0.00012174120638519526, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008441039361059666, "evidence_weights": [-0.2202989161014557, -0.21288444101810455, 0.13227930665016174], "agreement_weight": -0.04060369357466698, "orthogonal_weight": -1.033994197845459}, {"head": 5, "memory_to_active": 0.25212422013282776, "contribution_to_active": 0.6540133953094482, "mix_mean": 0.7193026542663574, "mix_low": 0.0020093917846679688, "mix_high": 0.020704269409179688, "read_cosine": 0.04278690367937088, "beta_mean": 0.005407554097473621, "nonfinite": 0.0, "weighted_memory_to_active": 0.1856650412082672, "mix_std": 0.27188366651535034, "beta_std": 0.014047049917280674, "beta_low": 0.88433837890625, "beta_high": 0.0, "orthogonal_to_active": 0.1397046595811844, "evidence_weights": [-0.2641027569770813, -0.33817148208618164, 0.47392335534095764], "agreement_weight": 0.19598104059696198, "orthogonal_weight": -0.7774670124053955}]}, {"layer": 4, "memory_to_active": 1.033355474472046, "contribution_to_active": 0.47423186898231506, "mix_mean": 0.3694361448287964, "mix_low": 0.26252493262290955, "mix_high": 0.012680689804255962, "read_cosine": 0.30855557322502136, "beta_mean": 0.11325669288635254, "nonfinite": 0.0, "weighted_memory_to_active": 0.1299709528684616, "mix_std": 0.15828010439872742, "beta_std": 0.09949147701263428, "beta_low": 0.38787841796875, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04864034056663513, "output_influence": 0.4824880063533783, "residual_influence": 0.14364519715309143, "state_active": 0.040413934737443924, "state_memory": 0.017300661653280258, "state_peak": 39.730587005615234, "growth_active": 1.2646877765655518, "growth_memory": 1.5352498292922974, "heads": [{"head": 0, "memory_to_active": 0.17599278688430786, "contribution_to_active": 0.7896004319190979, "mix_mean": 0.6951842308044434, "mix_low": 0.012598991394042969, "mix_high": 0.03851604461669922, "read_cosine": 0.4877985715866089, "beta_mean": 0.25384289026260376, "nonfinite": 0.0, "weighted_memory_to_active": 0.1523771733045578, "mix_std": 0.34539976716041565, "beta_std": 0.2497759759426117, "beta_low": 0.2310791015625, "beta_high": 0.0, "orthogonal_to_active": 0.0778493732213974, "evidence_weights": [-0.2645547091960907, -0.3693743348121643, -0.050503283739089966], "agreement_weight": -0.004154992289841175, "orthogonal_weight": 0.6620497107505798}, {"head": 1, "memory_to_active": 0.04620795324444771, "contribution_to_active": 0.6583836674690247, "mix_mean": 0.6967192888259888, "mix_low": 0.0001430511474609375, "mix_high": 0.01801586151123047, "read_cosine": 0.6084430813789368, "beta_mean": 0.0007771718665026128, "nonfinite": 0.0, "weighted_memory_to_active": 0.03407026082277298, "mix_std": 0.2400851994752884, "beta_std": 0.005821508821099997, "beta_low": 0.98370361328125, "beta_high": 0.0, "orthogonal_to_active": 0.01827489212155342, "evidence_weights": [-0.24169403314590454, -0.3636196255683899, 0.6343247890472412], "agreement_weight": 0.31480586528778076, "orthogonal_weight": -0.7611094117164612}, {"head": 2, "memory_to_active": 1.6120163202285767, "contribution_to_active": 0.33457714319229126, "mix_mean": 0.036067478358745575, "mix_low": 0.43853759765625, "mix_high": 1.049041748046875e-05, "read_cosine": 0.22517535090446472, "beta_mean": 0.0593058317899704, "nonfinite": 0.0, "weighted_memory_to_active": 0.24124960601329803, "mix_std": 0.07095473259687424, "beta_std": 0.05768585205078125, "beta_low": 0.08831787109375, "beta_high": 0.0, "orthogonal_to_active": 0.06825514137744904, "evidence_weights": [0.046867094933986664, -0.020239483565092087, 0.2562616169452667], "agreement_weight": 0.12130605429410934, "orthogonal_weight": -0.38394153118133545}, {"head": 3, "memory_to_active": 2.0143697261810303, "contribution_to_active": 0.1392330378293991, "mix_mean": 0.02930864877998829, "mix_low": 0.43419742584228516, "mix_high": 0.0, "read_cosine": 0.172904372215271, "beta_mean": 0.13520407676696777, "nonfinite": 0.0, "weighted_memory_to_active": 0.21132539212703705, "mix_std": 0.0510859489440918, "beta_std": 0.12289026379585266, "beta_low": 0.024169921875, "beta_high": 0.0, "orthogonal_to_active": 0.003581751836463809, "evidence_weights": [-0.01644911617040634, 0.05370252579450607, -0.2638925015926361], "agreement_weight": 0.0767180547118187, "orthogonal_weight": -0.027489660307765007}, {"head": 4, "memory_to_active": 0.007012652233242989, "contribution_to_active": 0.644065260887146, "mix_mean": 0.7469507455825806, "mix_low": 6.29425048828125e-05, "mix_high": 0.019535064697265625, "read_cosine": 0.144939124584198, "beta_mean": 1.6253172361757606e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0050764987245202065, "mix_std": 0.21256235241889954, "beta_std": 3.910905070370063e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003188287140801549, "evidence_weights": [-0.33217984437942505, -0.32194066047668457, 0.36258456110954285], "agreement_weight": 0.33437156677246094, "orthogonal_weight": -0.6670727133750916}, {"head": 5, "memory_to_active": 2.3445332050323486, "contribution_to_active": 0.2795317769050598, "mix_mean": 0.012386417016386986, "mix_low": 0.6896095275878906, "mix_high": 6.67572021484375e-06, "read_cosine": 0.21207302808761597, "beta_mean": 0.2303939163684845, "nonfinite": 0.0, "weighted_memory_to_active": 0.13572673499584198, "mix_std": 0.029592636972665787, "beta_std": 0.16073614358901978, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12069259583950043, "evidence_weights": [0.06136404350399971, -0.015369228087365627, -0.26155367493629456], "agreement_weight": -0.09100993722677231, "orthogonal_weight": 1.010473370552063}]}, {"layer": 5, "memory_to_active": 0.3953922390937805, "contribution_to_active": 0.5357855558395386, "mix_mean": 0.509127140045166, "mix_low": 0.05439901724457741, "mix_high": 0.01918935962021351, "read_cosine": 0.07333658635616302, "beta_mean": 0.05370211601257324, "nonfinite": 0.0, "weighted_memory_to_active": 0.1379433274269104, "mix_std": 0.25964078307151794, "beta_std": 0.06870527565479279, "beta_low": 0.6595560908317566, "beta_high": 0.0, "orthogonal_to_active": 0.05736880004405975, "output_influence": 0.5542936325073242, "residual_influence": 0.16685639321804047, "state_active": 0.04132530093193054, "state_memory": 0.010274816304445267, "state_peak": 57.75885009765625, "growth_active": 1.1199678182601929, "growth_memory": 1.215366244316101, "heads": [{"head": 0, "memory_to_active": 0.25013551115989685, "contribution_to_active": 0.5779633522033691, "mix_mean": 0.44637781381607056, "mix_low": 0.013002395629882812, "mix_high": 0.0019350051879882812, "read_cosine": -0.12288030982017517, "beta_mean": 0.02156691625714302, "nonfinite": 0.0, "weighted_memory_to_active": 0.08094257116317749, "mix_std": 0.28190451860427856, "beta_std": 0.03579133749008179, "beta_low": 0.4920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.033919524401426315, "evidence_weights": [-0.1610720455646515, -0.18119212985038757, 0.07705026865005493], "agreement_weight": -0.027757616713643074, "orthogonal_weight": -0.4378286302089691}, {"head": 1, "memory_to_active": 0.007328350562602282, "contribution_to_active": 0.5460201501846313, "mix_mean": 0.6933978796005249, "mix_low": 0.0015420913696289062, "mix_high": 0.03677558898925781, "read_cosine": 0.20439331233501434, "beta_mean": 1.708521813270636e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004379363730549812, "mix_std": 0.2723188102245331, "beta_std": 3.3007141610141844e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004130688030272722, "evidence_weights": [-0.2481190711259842, -0.28691163659095764, 0.924412190914154], "agreement_weight": 0.6811949014663696, "orthogonal_weight": -0.9918352365493774}, {"head": 2, "memory_to_active": 0.013555431738495827, "contribution_to_active": 0.5698742270469666, "mix_mean": 0.6565203070640564, "mix_low": 0.0025806427001953125, "mix_high": 0.02095508575439453, "read_cosine": -0.4908892512321472, "beta_mean": 8.982769213616848e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007958747446537018, "mix_std": 0.26629409193992615, "beta_std": 0.00021179098985157907, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0056190285831689835, "evidence_weights": [-0.2633521556854248, -0.3310093879699707, 0.17420926690101624], "agreement_weight": -0.10602206736803055, "orthogonal_weight": -0.7981374859809875}, {"head": 3, "memory_to_active": 1.5784330368041992, "contribution_to_active": 0.24470917880535126, "mix_mean": 0.1231321394443512, "mix_low": 0.11820316314697266, "mix_high": 0.0, "read_cosine": 0.15492108464241028, "beta_mean": 0.1107521802186966, "nonfinite": 0.0, "weighted_memory_to_active": 0.3886703550815582, "mix_std": 0.1571798175573349, "beta_std": 0.15634886920452118, "beta_low": 0.17694091796875, "beta_high": 0.0, "orthogonal_to_active": 0.1617351472377777, "evidence_weights": [-0.08046190440654755, -0.10503755509853363, 0.055215198546648026], "agreement_weight": 0.044808559119701385, "orthogonal_weight": -0.5005397200584412}, {"head": 4, "memory_to_active": 0.4652087688446045, "contribution_to_active": 0.5927368402481079, "mix_mean": 0.3395053744316101, "mix_low": 0.1892385482788086, "mix_high": 0.009394645690917969, "read_cosine": 0.5380695462226868, "beta_mean": 0.18961738049983978, "nonfinite": 0.0, "weighted_memory_to_active": 0.30769139528274536, "mix_std": 0.3505701720714569, "beta_std": 0.2191551923751831, "beta_low": 0.288818359375, "beta_high": 0.0, "orthogonal_to_active": 0.1158699095249176, "evidence_weights": [-0.11912252008914948, -0.2627902925014496, -0.04206742346286774], "agreement_weight": 0.03261609002947807, "orthogonal_weight": 0.5507450103759766}, {"head": 5, "memory_to_active": 0.05769241973757744, "contribution_to_active": 0.6834093332290649, "mix_mean": 0.7958292961120605, "mix_low": 0.001827239990234375, "mix_high": 0.04607582092285156, "read_cosine": 0.15640512108802795, "beta_mean": 0.0001693091180641204, "nonfinite": 0.0, "weighted_memory_to_active": 0.03801754489541054, "mix_std": 0.22957737743854523, "beta_std": 0.0006914627156220376, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.022938493639230728, "evidence_weights": [-0.30804669857025146, -0.3299092948436737, 1.879780888557434], "agreement_weight": 0.48150399327278137, "orthogonal_weight": -0.6338452100753784}]}, {"layer": 6, "memory_to_active": 0.2875828742980957, "contribution_to_active": 0.6246762871742249, "mix_mean": 0.5918751955032349, "mix_low": 0.04444297403097153, "mix_high": 0.03728993982076645, "read_cosine": 0.32924413681030273, "beta_mean": 0.06195620074868202, "nonfinite": 0.0, "weighted_memory_to_active": 0.14593663811683655, "mix_std": 0.23718883097171783, "beta_std": 0.07374529540538788, "beta_low": 0.5780029296875, "beta_high": 0.0, "orthogonal_to_active": 0.05030759423971176, "output_influence": 0.5569509267807007, "residual_influence": 0.18460167944431305, "state_active": 0.04521478712558746, "state_memory": 0.007397472392767668, "state_peak": 95.83951568603516, "growth_active": 1.2011864185333252, "growth_memory": 1.1803029775619507, "heads": [{"head": 0, "memory_to_active": 0.12589624524116516, "contribution_to_active": 0.7588794231414795, "mix_mean": 0.8153801560401917, "mix_low": 0.00018310546875, "mix_high": 0.054329872131347656, "read_cosine": 0.28712597489356995, "beta_mean": 0.01240004226565361, "nonfinite": 0.0, "weighted_memory_to_active": 0.10583575069904327, "mix_std": 0.2003929764032364, "beta_std": 0.02891378104686737, "beta_low": 0.745361328125, "beta_high": 0.0, "orthogonal_to_active": 0.07606152445077896, "evidence_weights": [-0.3116106688976288, -0.34969085454940796, 1.244022250175476], "agreement_weight": 0.6358072757720947, "orthogonal_weight": -0.7796729803085327}, {"head": 1, "memory_to_active": 0.005641912575811148, "contribution_to_active": 0.5152562856674194, "mix_mean": 0.5829018354415894, "mix_low": 0.001346588134765625, "mix_high": 0.014441490173339844, "read_cosine": 0.5381888747215271, "beta_mean": 1.2940431588503998e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002582813147455454, "mix_std": 0.2858220040798187, "beta_std": 2.433227564324625e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013289118651300669, "evidence_weights": [-0.19428548216819763, -0.29777902364730835, 1.10064697265625], "agreement_weight": 0.09183857589960098, "orthogonal_weight": -0.7042224407196045}, {"head": 2, "memory_to_active": 0.7956162095069885, "contribution_to_active": 0.09416874498128891, "mix_mean": 0.04524160921573639, "mix_low": 0.2591400146484375, "mix_high": 3.814697265625e-06, "read_cosine": 0.25325071811676025, "beta_mean": 0.02879352495074272, "nonfinite": 0.0, "weighted_memory_to_active": 0.06637950241565704, "mix_std": 0.07154237478971481, "beta_std": 0.03341091424226761, "beta_low": 0.19482421875, "beta_high": 0.0, "orthogonal_to_active": 0.008008640259504318, "evidence_weights": [0.07186120003461838, -0.04874127730727196, 0.3340015113353729], "agreement_weight": 0.13886220753192902, "orthogonal_weight": -0.14355292916297913}, {"head": 3, "memory_to_active": 0.13488392531871796, "contribution_to_active": 0.6892726421356201, "mix_mean": 0.6723140478134155, "mix_low": 0.00482940673828125, "mix_high": 0.0269317626953125, "read_cosine": 0.4666958153247833, "beta_mean": 0.2074258029460907, "nonfinite": 0.0, "weighted_memory_to_active": 0.09881309419870377, "mix_std": 0.3233495354652405, "beta_std": 0.24316030740737915, "beta_low": 0.2218017578125, "beta_high": 0.0, "orthogonal_to_active": 0.05481158196926117, "evidence_weights": [-0.1747305691242218, -0.3270050883293152, 0.1809779852628708], "agreement_weight": -0.005720183253288269, "orthogonal_weight": 0.6355783343315125}, {"head": 4, "memory_to_active": 0.6358430981636047, "contribution_to_active": 1.068840503692627, "mix_mean": 0.7570449113845825, "mix_low": 0.0008096694946289062, "mix_high": 0.11970329284667969, "read_cosine": 0.15418441593647003, "beta_mean": 0.12295476347208023, "nonfinite": 0.0, "weighted_memory_to_active": 0.5820524096488953, "mix_std": 0.29375922679901123, "beta_std": 0.13665473461151123, "beta_low": 0.3060302734375, "beta_high": 0.0, "orthogonal_to_active": 0.1419377624988556, "evidence_weights": [-0.20544889569282532, -0.556927502155304, -0.46782127022743225], "agreement_weight": -0.018433935940265656, "orthogonal_weight": 0.26617440581321716}, {"head": 5, "memory_to_active": 0.02761572040617466, "contribution_to_active": 0.6216400265693665, "mix_mean": 0.6783686876296997, "mix_low": 0.0003490447998046875, "mix_high": 0.008329391479492188, "read_cosine": 0.2760189175605774, "beta_mean": 0.0001501106162322685, "nonfinite": 0.0, "weighted_memory_to_active": 0.01995621807873249, "mix_std": 0.24826686084270477, "beta_std": 0.0003076578723266721, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.019697116687893867, "evidence_weights": [-0.2395632117986679, -0.31750181317329407, 0.4011732339859009], "agreement_weight": 0.19978104531764984, "orthogonal_weight": -1.0573477745056152}]}, {"layer": 7, "memory_to_active": 1.7822092771530151, "contribution_to_active": 0.2655807137489319, "mix_mean": 0.1638590395450592, "mix_low": 0.24098984897136688, "mix_high": 0.0009954770794138312, "read_cosine": 0.24532084167003632, "beta_mean": 0.13317084312438965, "nonfinite": 0.0, "weighted_memory_to_active": 0.20492051541805267, "mix_std": 0.14606547355651855, "beta_std": 0.10175018757581711, "beta_low": 0.2085774838924408, "beta_high": 0.000244140625, "orthogonal_to_active": 0.08452969789505005, "output_influence": 0.2740679681301117, "residual_influence": 0.06522567570209503, "state_active": 0.013470880687236786, "state_memory": 0.024978894740343094, "state_peak": 30.685935974121094, "growth_active": 1.1448389291763306, "growth_memory": 1.3040008544921875, "heads": [{"head": 0, "memory_to_active": 2.598557472229004, "contribution_to_active": 0.19086503982543945, "mix_mean": 0.03693820536136627, "mix_low": 0.4146718978881836, "mix_high": 0.0, "read_cosine": 0.3390222191810608, "beta_mean": 0.17064039409160614, "nonfinite": 0.0, "weighted_memory_to_active": 0.21859396994113922, "mix_std": 0.07717500627040863, "beta_std": 0.1298258900642395, "beta_low": 0.00445556640625, "beta_high": 0.00048828125, "orthogonal_to_active": 0.0017492776969447732, "evidence_weights": [0.042343005537986755, -0.07570286095142365, 0.013507498428225517], "agreement_weight": -0.03489729017019272, "orthogonal_weight": 0.009663338772952557}, {"head": 1, "memory_to_active": 1.016510248184204, "contribution_to_active": 0.4088749289512634, "mix_mean": 0.22547686100006104, "mix_low": 0.11338043212890625, "mix_high": 0.005396842956542969, "read_cosine": 0.3861904740333557, "beta_mean": 0.09965008497238159, "nonfinite": 0.0, "weighted_memory_to_active": 0.3495999276638031, "mix_std": 0.24447274208068848, "beta_std": 0.07002116739749908, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04704941436648369, "evidence_weights": [-0.21719563007354736, -0.1026478260755539, -0.2780611217021942], "agreement_weight": 0.056579165160655975, "orthogonal_weight": 0.1603846698999405}, {"head": 2, "memory_to_active": 0.6338027119636536, "contribution_to_active": 0.3444557189941406, "mix_mean": 0.20198264718055725, "mix_low": 0.02857494354248047, "mix_high": 0.00025177001953125, "read_cosine": 0.1067061573266983, "beta_mean": 0.04357214272022247, "nonfinite": 0.0, "weighted_memory_to_active": 0.2091151922941208, "mix_std": 0.2030588984489441, "beta_std": 0.043027084320783615, "beta_low": 0.11895751953125, "beta_high": 0.0, "orthogonal_to_active": 0.11263787746429443, "evidence_weights": [-0.08216611295938492, -0.16627149283885956, 0.8499130010604858], "agreement_weight": 0.01772734709084034, "orthogonal_weight": -0.5546200275421143}, {"head": 3, "memory_to_active": 4.722646713256836, "contribution_to_active": 0.11957234144210815, "mix_mean": 0.0343421995639801, "mix_low": 0.33329105377197266, "mix_high": 0.0, "read_cosine": 0.09939080476760864, "beta_mean": 0.406795859336853, "nonfinite": 0.0, "weighted_memory_to_active": 0.361836314201355, "mix_std": 0.05180215835571289, "beta_std": 0.28109821677207947, "beta_low": 0.00506591796875, "beta_high": 0.0009765625, "orthogonal_to_active": 0.3105665147304535, "evidence_weights": [0.07151487469673157, -0.034620251506567, 0.7225592732429504], "agreement_weight": 0.05350756645202637, "orthogonal_weight": -0.9418401718139648}, {"head": 4, "memory_to_active": 1.6766295433044434, "contribution_to_active": 0.09753072261810303, "mix_mean": 0.021517321467399597, "mix_low": 0.5531682968139648, "mix_high": 0.0, "read_cosine": 0.26782387495040894, "beta_mean": 0.07814483344554901, "nonfinite": 0.0, "weighted_memory_to_active": 0.06897598505020142, "mix_std": 0.044980499893426895, "beta_std": 0.08601285517215729, "beta_low": 0.12335205078125, "beta_high": 0.0, "orthogonal_to_active": 0.015295978635549545, "evidence_weights": [0.13895010948181152, -0.0052402629517018795, 0.18112720549106598], "agreement_weight": 0.03470583260059357, "orthogonal_weight": -0.26770785450935364}, {"head": 5, "memory_to_active": 0.04510914534330368, "contribution_to_active": 0.4321853518486023, "mix_mean": 0.4628969132900238, "mix_low": 0.0028524398803710938, "mix_high": 0.000324249267578125, "read_cosine": 0.27279144525527954, "beta_mean": 0.00022170942975208163, "nonfinite": 0.0, "weighted_memory_to_active": 0.02140161395072937, "mix_std": 0.25490355491638184, "beta_std": 0.0005159318097867072, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.019879061728715897, "evidence_weights": [-0.17415952682495117, -0.23033073544502258, 0.5400012731552124], "agreement_weight": 0.19415371119976044, "orthogonal_weight": -1.002280831336975}]}, {"layer": 8, "memory_to_active": 0.16077566146850586, "contribution_to_active": 0.5475133657455444, "mix_mean": 0.4755517244338989, "mix_low": 0.013745944015681744, "mix_high": 0.0015354156494140625, "read_cosine": 0.41330569982528687, "beta_mean": 0.013508382253348827, "nonfinite": 0.0, "weighted_memory_to_active": 0.07653281092643738, "mix_std": 0.23668088018894196, "beta_std": 0.020894542336463928, "beta_low": 0.7845052480697632, "beta_high": 0.0, "orthogonal_to_active": 0.06751043349504471, "output_influence": 0.5078008770942688, "residual_influence": 0.17730394005775452, "state_active": 0.04431387037038803, "state_memory": 0.005475793033838272, "state_peak": 54.358097076416016, "growth_active": 1.0898406505584717, "growth_memory": 1.100515604019165, "heads": [{"head": 0, "memory_to_active": 0.060440562665462494, "contribution_to_active": 0.5737462639808655, "mix_mean": 0.6000521779060364, "mix_low": 0.0004930496215820312, "mix_high": 0.0015039443969726562, "read_cosine": 0.4501490890979767, "beta_mean": 0.0035345708020031452, "nonfinite": 0.0, "weighted_memory_to_active": 0.043050505220890045, "mix_std": 0.24791589379310608, "beta_std": 0.014001828618347645, "beta_low": 0.92462158203125, "beta_high": 0.0, "orthogonal_to_active": 0.0269294586032629, "evidence_weights": [-0.16577911376953125, -0.38648635149002075, 0.6106337904930115], "agreement_weight": 0.2477540671825409, "orthogonal_weight": -0.7922258973121643}, {"head": 1, "memory_to_active": 0.07881499826908112, "contribution_to_active": 0.5461038947105408, "mix_mean": 0.5152004957199097, "mix_low": 5.054473876953125e-05, "mix_high": 0.0009679794311523438, "read_cosine": 0.15150682628154755, "beta_mean": 0.0013319933786988258, "nonfinite": 0.0, "weighted_memory_to_active": 0.036116279661655426, "mix_std": 0.2335202991962433, "beta_std": 0.008953388780355453, "beta_low": 0.9864501953125, "beta_high": 0.0, "orthogonal_to_active": 0.025730079039931297, "evidence_weights": [-0.1949668824672699, -0.2760152518749237, 0.6824997067451477], "agreement_weight": 0.06531417369842529, "orthogonal_weight": -0.7481502294540405}, {"head": 2, "memory_to_active": 0.017185868695378304, "contribution_to_active": 0.6794205904006958, "mix_mean": 0.678727388381958, "mix_low": 0.00043201446533203125, "mix_high": 0.0035085678100585938, "read_cosine": 0.39684927463531494, "beta_mean": 0.00015431299107149243, "nonfinite": 0.0, "weighted_memory_to_active": 0.0133211649954319, "mix_std": 0.22436806559562683, "beta_std": 0.003517357399687171, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.008138647302985191, "evidence_weights": [-0.2270539402961731, -0.3476176857948303, 1.0017750263214111], "agreement_weight": 0.28302353620529175, "orthogonal_weight": -0.7030378580093384}, {"head": 3, "memory_to_active": 0.0947284922003746, "contribution_to_active": 0.6200117468833923, "mix_mean": 0.5783178806304932, "mix_low": 0.00063323974609375, "mix_high": 0.0030355453491210938, "read_cosine": 0.79986172914505, "beta_mean": 0.007330090273171663, "nonfinite": 0.0, "weighted_memory_to_active": 0.07070112973451614, "mix_std": 0.2561209499835968, "beta_std": 0.007538998499512672, "beta_low": 0.763427734375, "beta_high": 0.0, "orthogonal_to_active": 0.02239314280450344, "evidence_weights": [-0.18445414304733276, -0.30011874437332153, 0.5515267848968506], "agreement_weight": 0.23431047797203064, "orthogonal_weight": -0.6219079494476318}, {"head": 4, "memory_to_active": 0.34340324997901917, "contribution_to_active": 0.25979357957839966, "mix_mean": 0.23529747128486633, "mix_low": 0.0105133056640625, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19432464241981506, "beta_mean": 0.0027805562131106853, "nonfinite": 0.0, "weighted_memory_to_active": 0.11438696831464767, "mix_std": 0.20008566975593567, "beta_std": 0.01469387486577034, "beta_low": 0.9478759765625, "beta_high": 0.0, "orthogonal_to_active": 0.028876392170786858, "evidence_weights": [-0.08797150105237961, -0.16801925003528595, 0.48206618428230286], "agreement_weight": 0.0721912533044815, "orthogonal_weight": -0.2653990387916565}, {"head": 5, "memory_to_active": 0.37008076906204224, "contribution_to_active": 0.606003999710083, "mix_mean": 0.24571493268013, "mix_low": 0.07035350799560547, "mix_high": 0.00019550323486328125, "read_cosine": 0.48714229464530945, "beta_mean": 0.06591877341270447, "nonfinite": 0.0, "weighted_memory_to_active": 0.18162080645561218, "mix_std": 0.2580742835998535, "beta_std": 0.07666180282831192, "beta_low": 0.08477783203125, "beta_high": 0.0, "orthogonal_to_active": 0.292994886636734, "evidence_weights": [-0.1390998512506485, -0.15731100738048553, -0.11832933127880096], "agreement_weight": -0.015179697424173355, "orthogonal_weight": 1.8947222232818604}]}, {"layer": 9, "memory_to_active": 0.9451950192451477, "contribution_to_active": 0.7177739143371582, "mix_mean": 0.3645825982093811, "mix_low": 0.08166615664958954, "mix_high": 0.005453586578369141, "read_cosine": 0.33725765347480774, "beta_mean": 0.038600411266088486, "nonfinite": 0.0, "weighted_memory_to_active": 0.40711888670921326, "mix_std": 0.23161932826042175, "beta_std": 0.03315737098455429, "beta_low": 0.5405782461166382, "beta_high": 0.0, "orthogonal_to_active": 0.03739186376333237, "output_influence": 0.5413621664047241, "residual_influence": 0.15712177753448486, "state_active": 0.041985027492046356, "state_memory": 0.01593533158302307, "state_peak": 56.92860794067383, "growth_active": 1.1696782112121582, "growth_memory": 1.3198655843734741, "heads": [{"head": 0, "memory_to_active": 0.0005828590365126729, "contribution_to_active": 0.5703155994415283, "mix_mean": 0.6429068446159363, "mix_low": 0.0013113021850585938, "mix_high": 0.02273845672607422, "read_cosine": 0.3283408582210541, "beta_mean": 1.6728681657696143e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00037803404848091304, "mix_std": 0.2877081036567688, "beta_std": 4.187210834061261e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00040627503767609596, "evidence_weights": [-0.3054024875164032, -0.31233254075050354, 0.5209344029426575], "agreement_weight": 0.11849712580442429, "orthogonal_weight": -1.1418381929397583}, {"head": 1, "memory_to_active": 0.01928088255226612, "contribution_to_active": 0.6891038417816162, "mix_mean": 0.5645083785057068, "mix_low": 0.007300376892089844, "mix_high": 0.0018758773803710938, "read_cosine": 0.40071240067481995, "beta_mean": 4.948355126543902e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014274593442678452, "mix_std": 0.28159067034721375, "beta_std": 0.00010518763883737847, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009987367317080498, "evidence_weights": [-0.24042637646198273, -0.2966240644454956, 1.0640372037887573], "agreement_weight": 0.472705602645874, "orthogonal_weight": -0.849948525428772}, {"head": 2, "memory_to_active": 0.17034262418746948, "contribution_to_active": 0.6486648321151733, "mix_mean": 0.5925269722938538, "mix_low": 0.001705169677734375, "mix_high": 0.006691932678222656, "read_cosine": 0.3248864412307739, "beta_mean": 0.0007060914067551494, "nonfinite": 0.0, "weighted_memory_to_active": 0.11180618405342102, "mix_std": 0.2684227228164673, "beta_std": 0.004534248262643814, "beta_low": 0.995361328125, "beta_high": 0.0, "orthogonal_to_active": 0.06711407005786896, "evidence_weights": [-0.20317085087299347, -0.2775680124759674, 0.7746114134788513], "agreement_weight": 0.3330659568309784, "orthogonal_weight": -0.6842940449714661}, {"head": 3, "memory_to_active": 0.9690101146697998, "contribution_to_active": 0.21198788285255432, "mix_mean": 0.10848642885684967, "mix_low": 0.17055892944335938, "mix_high": 8.20159912109375e-05, "read_cosine": 0.3803255558013916, "beta_mean": 0.03975357860326767, "nonfinite": 0.0, "weighted_memory_to_active": 0.1742272675037384, "mix_std": 0.16298820078372955, "beta_std": 0.03672999143600464, "beta_low": 0.0631103515625, "beta_high": 0.0, "orthogonal_to_active": 0.05264569818973541, "evidence_weights": [0.011987010948359966, -0.06073983758687973, 0.22225970029830933], "agreement_weight": 0.09745676070451736, "orthogonal_weight": -0.32770848274230957}, {"head": 4, "memory_to_active": 3.095452070236206, "contribution_to_active": 1.7444621324539185, "mix_mean": 0.14686667919158936, "mix_low": 0.12297344207763672, "mix_high": 0.0010156631469726562, "read_cosine": 0.09442145377397537, "beta_mean": 0.13888433575630188, "nonfinite": 0.0, "weighted_memory_to_active": 1.6763184070587158, "mix_std": 0.20711253583431244, "beta_std": 0.08736897259950638, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.02396845817565918, "evidence_weights": [-0.08270196616649628, -0.15602728724479675, -0.02528681606054306], "agreement_weight": -0.0655677318572998, "orthogonal_weight": -0.014844236895442009}, {"head": 5, "memory_to_active": 1.416501760482788, "contribution_to_active": 0.4421088695526123, "mix_mean": 0.13220012187957764, "mix_low": 0.18614768981933594, "mix_high": 0.00031757354736328125, "read_cosine": 0.4948592185974121, "beta_mean": 0.05220729112625122, "nonfinite": 0.0, "weighted_memory_to_active": 0.46570879220962524, "mix_std": 0.18189360201358795, "beta_std": 0.07020162045955658, "beta_low": 0.18499755859375, "beta_high": 0.0, "orthogonal_to_active": 0.07022932916879654, "evidence_weights": [0.08936739712953568, -0.13657839596271515, 0.14066313207149506], "agreement_weight": 0.012016271241009235, "orthogonal_weight": 0.2018773853778839}]}, {"layer": 10, "memory_to_active": 0.22171320021152496, "contribution_to_active": 0.7090625166893005, "mix_mean": 0.5168477296829224, "mix_low": 0.023465953767299652, "mix_high": 0.004905065521597862, "read_cosine": 0.38069573044776917, "beta_mean": 0.04425996541976929, "nonfinite": 0.0, "weighted_memory_to_active": 0.09654442965984344, "mix_std": 0.26938390731811523, "beta_std": 0.04650092124938965, "beta_low": 0.6653646230697632, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.2450963258743286, "output_influence": 0.45505985617637634, "residual_influence": 0.16035938262939453, "state_active": 0.05279605835676193, "state_memory": 0.01493002474308014, "state_peak": 125.90663146972656, "growth_active": 1.0452144145965576, "growth_memory": 0.9930155873298645, "heads": [{"head": 0, "memory_to_active": 0.8117578625679016, "contribution_to_active": 1.9591012001037598, "mix_mean": 0.5701382160186768, "mix_low": 0.10137462615966797, "mix_high": 0.008851051330566406, "read_cosine": 0.09083453565835953, "beta_mean": 0.19565565884113312, "nonfinite": 0.0, "weighted_memory_to_active": 0.4534061551094055, "mix_std": 0.3943459391593933, "beta_std": 0.21275177597999573, "beta_low": 0.15362548828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.3842015266418457, "evidence_weights": [-0.09617958217859268, -0.38575080037117004, -1.787574291229248], "agreement_weight": 0.032387975603342056, "orthogonal_weight": 3.0856614112854004}, {"head": 1, "memory_to_active": 0.009192489087581635, "contribution_to_active": 0.6047146320343018, "mix_mean": 0.6255506873130798, "mix_low": 0.0019388198852539062, "mix_high": 0.014360427856445312, "read_cosine": 0.30860215425491333, "beta_mean": 6.389828195096925e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005094851832836866, "mix_std": 0.26669931411743164, "beta_std": 9.183473594021052e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007799371145665646, "evidence_weights": [-0.2521730661392212, -0.19635488092899323, 0.5651284456253052], "agreement_weight": 0.5769241452217102, "orthogonal_weight": -1.6640757322311401}, {"head": 2, "memory_to_active": 0.02374115027487278, "contribution_to_active": 0.6277084350585938, "mix_mean": 0.6549147367477417, "mix_low": 0.000347137451171875, "mix_high": 0.0046825408935546875, "read_cosine": 0.026868462562561035, "beta_mean": 4.02962468797341e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01051507517695427, "mix_std": 0.25112995505332947, "beta_std": 0.0001821867044782266, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004484562203288078, "evidence_weights": [-0.2631053924560547, -0.2812533676624298, 0.6189348101615906], "agreement_weight": -0.20350846648216248, "orthogonal_weight": -0.40825262665748596}, {"head": 3, "memory_to_active": 0.3397151827812195, "contribution_to_active": 0.1001703217625618, "mix_mean": 0.2756884694099426, "mix_low": 0.03472423553466797, "mix_high": 0.0002880096435546875, "read_cosine": 0.9105029106140137, "beta_mean": 0.06349707394838333, "nonfinite": 0.0, "weighted_memory_to_active": 0.040512435138225555, "mix_std": 0.2317262887954712, "beta_std": 0.05888195335865021, "beta_low": 0.00482177734375, "beta_high": 0.0, "orthogonal_to_active": 0.041015930473804474, "evidence_weights": [-0.10122164338827133, -0.14683394134044647, 0.16393078863620758], "agreement_weight": 0.01721474900841713, "orthogonal_weight": 1.4113187789916992}, {"head": 4, "memory_to_active": 0.016445312649011612, "contribution_to_active": 0.5559800267219543, "mix_mean": 0.5830190181732178, "mix_low": 0.00032901763916015625, "mix_high": 0.0012454986572265625, "read_cosine": 0.0952659323811531, "beta_mean": 4.8698613682063296e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009606139734387398, "mix_std": 0.2468884289264679, "beta_std": 8.71586162247695e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013111211359500885, "evidence_weights": [-0.1852085292339325, -0.2593587040901184, 0.9758694767951965], "agreement_weight": 0.15537990629673004, "orthogonal_weight": -1.4424582719802856}, {"head": 5, "memory_to_active": 0.12942714989185333, "contribution_to_active": 0.40670034289360046, "mix_mean": 0.3917749226093292, "mix_low": 0.0020818710327148438, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8521001935005188, "beta_mean": 0.006254150532186031, "nonfinite": 0.0, "weighted_memory_to_active": 0.06013190373778343, "mix_std": 0.22551345825195312, "beta_std": 0.007010618224740028, "beta_low": 0.83380126953125, "beta_high": 0.0, "orthogonal_to_active": 0.0199652798473835, "evidence_weights": [-0.1409737467765808, -0.16301017999649048, 0.2684880793094635], "agreement_weight": 0.0927392840385437, "orthogonal_weight": -0.6571577191352844}]}, {"layer": 11, "memory_to_active": 0.09108802676200867, "contribution_to_active": 0.6165485978126526, "mix_mean": 0.5136044025421143, "mix_low": 0.11425463855266571, "mix_high": 0.044096313416957855, "read_cosine": 0.3459286391735077, "beta_mean": 0.025724273175001144, "nonfinite": 0.0, "weighted_memory_to_active": 0.04728255048394203, "mix_std": 0.3248402178287506, "beta_std": 0.03817012161016464, "beta_low": 0.6592203974723816, "beta_high": 0.0, "orthogonal_to_active": 0.049016401171684265, "output_influence": 0.5503832101821899, "residual_influence": 0.1451675295829773, "state_active": 0.21609872579574585, "state_memory": 0.004222681745886803, "state_peak": 31.514118194580078, "growth_active": 1.1484568119049072, "growth_memory": 1.6172443628311157, "heads": [{"head": 0, "memory_to_active": 0.008661407046020031, "contribution_to_active": 0.6071161031723022, "mix_mean": 0.649677038192749, "mix_low": 0.000957489013671875, "mix_high": 0.008882522583007812, "read_cosine": 0.2757861614227295, "beta_mean": 9.154233703156933e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006163980811834335, "mix_std": 0.2583372890949249, "beta_std": 0.0038464863318949938, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.004984633531421423, "evidence_weights": [-0.17374074459075928, -0.3651169240474701, 0.7684774994850159], "agreement_weight": 0.369660884141922, "orthogonal_weight": -0.8470311164855957}, {"head": 1, "memory_to_active": 0.15688082575798035, "contribution_to_active": 0.40004655718803406, "mix_mean": 0.45204606652259827, "mix_low": 0.006108283996582031, "mix_high": 0.00046634674072265625, "read_cosine": 0.49781566858291626, "beta_mean": 0.008827630430459976, "nonfinite": 0.0, "weighted_memory_to_active": 0.06970895826816559, "mix_std": 0.26776620745658875, "beta_std": 0.023058094084262848, "beta_low": 0.82647705078125, "beta_high": 0.0, "orthogonal_to_active": 0.037228088825941086, "evidence_weights": [-0.15381388366222382, -0.253469854593277, 0.5918760895729065], "agreement_weight": 0.07164014130830765, "orthogonal_weight": -0.6334481835365295}, {"head": 2, "memory_to_active": 0.006137963850051165, "contribution_to_active": 0.9832794070243835, "mix_mean": 0.4310082793235779, "mix_low": 0.3164339065551758, "mix_high": 0.16421222686767578, "read_cosine": 0.186870276927948, "beta_mean": 0.06384990364313126, "nonfinite": 0.0, "weighted_memory_to_active": 0.004212961532175541, "mix_std": 0.44953441619873047, "beta_std": 0.08649005740880966, "beta_low": 0.4869384765625, "beta_high": 0.0, "orthogonal_to_active": 0.03827953338623047, "evidence_weights": [0.043681759387254715, -0.3843551278114319, -1.3258496522903442], "agreement_weight": 0.0035857809707522392, "orthogonal_weight": 9.497034072875977}, {"head": 3, "memory_to_active": 0.00835922360420227, "contribution_to_active": 0.8663123846054077, "mix_mean": 0.4046962559223175, "mix_low": 0.35245513916015625, "mix_high": 0.08403587341308594, "read_cosine": 0.5814111828804016, "beta_mean": 0.05039471387863159, "nonfinite": 0.0, "weighted_memory_to_active": 0.005264781415462494, "mix_std": 0.4390975832939148, "beta_std": 0.07676011323928833, "beta_low": 0.51861572265625, "beta_high": 0.0, "orthogonal_to_active": 0.033461522310972214, "evidence_weights": [0.20405705273151398, -0.29446205496788025, -0.15821050107479095], "agreement_weight": 0.05178028717637062, "orthogonal_weight": 9.7765531539917}, {"head": 4, "memory_to_active": 0.2736414968967438, "contribution_to_active": 0.44320034980773926, "mix_mean": 0.5884690880775452, "mix_low": 0.007298469543457031, "mix_high": 0.0016880035400390625, "read_cosine": 0.004601703956723213, "beta_mean": 0.020700888708233833, "nonfinite": 0.0, "weighted_memory_to_active": 0.1490734964609146, "mix_std": 0.2723190486431122, "beta_std": 0.02410309948027134, "beta_low": 0.42205810546875, "beta_high": 0.0, "orthogonal_to_active": 0.1460382044315338, "evidence_weights": [-0.2401551902294159, -0.240815207362175, 1.9011695384979248], "agreement_weight": -0.2113320380449295, "orthogonal_weight": -0.9975013136863708}, {"head": 5, "memory_to_active": 0.09284721314907074, "contribution_to_active": 0.39933663606643677, "mix_mean": 0.5557293891906738, "mix_low": 0.0022745132446289062, "mix_high": 0.0052928924560546875, "read_cosine": 0.5290865898132324, "beta_mean": 0.010480953380465508, "nonfinite": 0.0, "weighted_memory_to_active": 0.049271099269390106, "mix_std": 0.261986643075943, "beta_std": 0.014762859791517258, "beta_low": 0.7015380859375, "beta_high": 0.0, "orthogonal_to_active": 0.03410641849040985, "evidence_weights": [-0.2082388997077942, -0.31380605697631836, 0.6530190706253052], "agreement_weight": 0.0848567932844162, "orthogonal_weight": -0.8464388251304626}]}, {"layer": 12, "memory_to_active": 0.23857709765434265, "contribution_to_active": 0.589888334274292, "mix_mean": 0.5398011207580566, "mix_low": 0.009236335754394531, "mix_high": 0.0063068075105547905, "read_cosine": 0.22412854433059692, "beta_mean": 0.015533264726400375, "nonfinite": 0.0, "weighted_memory_to_active": 0.1272398829460144, "mix_std": 0.2618750333786011, "beta_std": 0.015841426327824593, "beta_low": 0.827423095703125, "beta_high": 0.0, "orthogonal_to_active": 0.0742994025349617, "output_influence": 0.5024796724319458, "residual_influence": 0.14548535645008087, "state_active": 0.036540597677230835, "state_memory": 0.008625750429928303, "state_peak": 66.1968994140625, "growth_active": 1.189961552619934, "growth_memory": 1.201456904411316, "heads": [{"head": 0, "memory_to_active": 0.04748252034187317, "contribution_to_active": 0.6029977202415466, "mix_mean": 0.5740430355072021, "mix_low": 0.000370025634765625, "mix_high": 0.0052318572998046875, "read_cosine": 0.3082374632358551, "beta_mean": 0.0007563825929537416, "nonfinite": 0.0, "weighted_memory_to_active": 0.025625893846154213, "mix_std": 0.2672717571258545, "beta_std": 0.0014076933730393648, "beta_low": 0.99578857421875, "beta_high": 0.0, "orthogonal_to_active": 0.023824745789170265, "evidence_weights": [-0.2101600170135498, -0.276276171207428, 0.011651030741631985], "agreement_weight": 0.10232390463352203, "orthogonal_weight": -1.003656029701233}, {"head": 1, "memory_to_active": 0.0983695313334465, "contribution_to_active": 0.5548000335693359, "mix_mean": 0.5550435781478882, "mix_low": 0.00038623809814453125, "mix_high": 0.0017232894897460938, "read_cosine": 0.3849793076515198, "beta_mean": 0.0007995790801942348, "nonfinite": 0.0, "weighted_memory_to_active": 0.061999790370464325, "mix_std": 0.2662065029144287, "beta_std": 0.009460547007620335, "beta_low": 0.99365234375, "beta_high": 0.0, "orthogonal_to_active": 0.05211493372917175, "evidence_weights": [-0.2028341442346573, -0.271119624376297, 1.0501325130462646], "agreement_weight": 0.2455282360315323, "orthogonal_weight": -0.9221475124359131}, {"head": 2, "memory_to_active": 0.6134723424911499, "contribution_to_active": 0.5196627378463745, "mix_mean": 0.3208509683609009, "mix_low": 0.030313491821289062, "mix_high": 0.000141143798828125, "read_cosine": 0.49599868059158325, "beta_mean": 0.09039545804262161, "nonfinite": 0.0, "weighted_memory_to_active": 0.25098586082458496, "mix_std": 0.24154292047023773, "beta_std": 0.06668997555971146, "beta_low": 0.0013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.27436330914497375, "evidence_weights": [-0.21574002504348755, -0.05928724631667137, -0.00012718804646283388], "agreement_weight": 0.023186370730400085, "orthogonal_weight": 1.3688325881958008}, {"head": 3, "memory_to_active": 0.03362790495157242, "contribution_to_active": 0.7512302994728088, "mix_mean": 0.7155435085296631, "mix_low": 0.0001239776611328125, "mix_high": 0.018947601318359375, "read_cosine": 0.09843143820762634, "beta_mean": 0.0002877764927688986, "nonfinite": 0.0, "weighted_memory_to_active": 0.023990049958229065, "mix_std": 0.22938305139541626, "beta_std": 0.005951650906354189, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.022998351603746414, "evidence_weights": [-0.2915763556957245, -0.3374015986919403, 1.6718950271606445], "agreement_weight": 0.5317668318748474, "orthogonal_weight": -0.9793213605880737}, {"head": 4, "memory_to_active": 0.5662428140640259, "contribution_to_active": 0.6062901020050049, "mix_mean": 0.4390476644039154, "mix_low": 0.017690658569335938, "mix_high": 0.004962921142578125, "read_cosine": -0.13551440834999084, "beta_mean": 0.0006379294209182262, "nonfinite": 0.0, "weighted_memory_to_active": 0.35281091928482056, "mix_std": 0.2906983494758606, "beta_std": 0.00740515161305666, "beta_low": 0.98150634765625, "beta_high": 0.0, "orthogonal_to_active": 0.03576473519206047, "evidence_weights": [-0.15279556810855865, -0.21656163036823273, -0.10204868018627167], "agreement_weight": 0.017028238624334335, "orthogonal_weight": -0.10557565093040466}, {"head": 5, "memory_to_active": 0.07226744294166565, "contribution_to_active": 0.5043492317199707, "mix_mean": 0.6342775821685791, "mix_low": 0.006533622741699219, "mix_high": 0.0068340301513671875, "read_cosine": 0.19263872504234314, "beta_mean": 0.0003224538522772491, "nonfinite": 0.0, "weighted_memory_to_active": 0.04802672564983368, "mix_std": 0.2761475145816803, "beta_std": 0.004133541136980057, "beta_low": 0.9945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.0367303229868412, "evidence_weights": [-0.23128646612167358, -0.3190345764160156, 1.2715153694152832], "agreement_weight": 0.41202402114868164, "orthogonal_weight": -0.8159968256950378}]}, {"layer": 13, "memory_to_active": 0.11548194289207458, "contribution_to_active": 0.5407161116600037, "mix_mean": 0.5398810505867004, "mix_low": 0.005725066177546978, "mix_high": 0.009685834869742393, "read_cosine": 0.41326025128364563, "beta_mean": 0.015028512105345726, "nonfinite": 0.0, "weighted_memory_to_active": 0.048629678785800934, "mix_std": 0.2602003514766693, "beta_std": 0.01739318296313286, "beta_low": 0.809814453125, "beta_high": 0.0, "orthogonal_to_active": 0.05026683211326599, "output_influence": 0.5289509892463684, "residual_influence": 0.18382686376571655, "state_active": 0.12767013907432556, "state_memory": 0.04491619020700455, "state_peak": 95.08926391601562, "growth_active": 0.8812577128410339, "growth_memory": 0.9885932803153992, "heads": [{"head": 0, "memory_to_active": 0.011249733157455921, "contribution_to_active": 0.5925383567810059, "mix_mean": 0.6105024814605713, "mix_low": 0.0007505416870117188, "mix_high": 0.004923820495605469, "read_cosine": 0.45285072922706604, "beta_mean": 3.7902012991253287e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007240622770041227, "mix_std": 0.2701307535171509, "beta_std": 7.366883073700592e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006976896896958351, "evidence_weights": [-0.19709499180316925, -0.26863792538642883, 0.7808555960655212], "agreement_weight": 0.16773764789104462, "orthogonal_weight": -1.0813156366348267}, {"head": 1, "memory_to_active": 0.01478019542992115, "contribution_to_active": 0.5978018045425415, "mix_mean": 0.5669893026351929, "mix_low": 0.0019044876098632812, "mix_high": 0.005333900451660156, "read_cosine": 0.2261887490749359, "beta_mean": 6.182893412187696e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008281644433736801, "mix_std": 0.2777359187602997, "beta_std": 0.00011705189535859972, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01079371478408575, "evidence_weights": [-0.18247276544570923, -0.2639006972312927, 0.5753370523452759], "agreement_weight": 0.4667506515979767, "orthogonal_weight": -1.362410545349121}, {"head": 2, "memory_to_active": 0.07024022191762924, "contribution_to_active": 0.47813594341278076, "mix_mean": 0.3912108540534973, "mix_low": 0.0013151168823242188, "mix_high": 0.00013256072998046875, "read_cosine": 0.6031222343444824, "beta_mean": 0.009661873802542686, "nonfinite": 0.0, "weighted_memory_to_active": 0.034720949828624725, "mix_std": 0.23288780450820923, "beta_std": 0.036402247846126556, "beta_low": 0.86505126953125, "beta_high": 0.0, "orthogonal_to_active": 0.020896852016448975, "evidence_weights": [-0.18407957255840302, -0.20876574516296387, 0.40381529927253723], "agreement_weight": 0.007144400384277105, "orthogonal_weight": -0.8355892300605774}, {"head": 3, "memory_to_active": 0.13354669511318207, "contribution_to_active": 0.6119599938392639, "mix_mean": 0.6686149835586548, "mix_low": 0.021712303161621094, "mix_high": 0.03504180908203125, "read_cosine": 0.01762303337454796, "beta_mean": 0.0008776723989285529, "nonfinite": 0.0, "weighted_memory_to_active": 0.09823645651340485, "mix_std": 0.28516361117362976, "beta_std": 0.008435137569904327, "beta_low": 0.99407958984375, "beta_high": 0.0, "orthogonal_to_active": 0.08955326676368713, "evidence_weights": [-0.31777486205101013, -0.2466312199831009, 1.586761713027954], "agreement_weight": 6.0794596672058105, "orthogonal_weight": -0.9807994365692139}, {"head": 4, "memory_to_active": 0.4035680592060089, "contribution_to_active": 0.24951310455799103, "mix_mean": 0.3430045545101166, "mix_low": 0.006342887878417969, "mix_high": 3.24249267578125e-05, "read_cosine": 0.8969306945800781, "beta_mean": 0.07928255945444107, "nonfinite": 0.0, "weighted_memory_to_active": 0.103188157081604, "mix_std": 0.22269847989082336, "beta_std": 0.056653160601854324, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.13890086114406586, "evidence_weights": [-0.08080548048019409, -0.2314976006746292, 0.46084359288215637], "agreement_weight": 0.007003218401223421, "orthogonal_weight": 2.238389015197754}, {"head": 5, "memory_to_active": 0.059506792575120926, "contribution_to_active": 0.7143473625183105, "mix_mean": 0.6589639782905579, "mix_low": 0.0023250579833984375, "mix_high": 0.012650489807128906, "read_cosine": 0.2828459143638611, "beta_mean": 0.00024923388264141977, "nonfinite": 0.0, "weighted_memory_to_active": 0.04011023789644241, "mix_std": 0.272585391998291, "beta_std": 0.002677825978025794, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.03447941690683365, "evidence_weights": [-0.22133950889110565, -0.299093633890152, 0.8815075755119324], "agreement_weight": 0.18969184160232544, "orthogonal_weight": -0.9261752963066101}]}, {"layer": 14, "memory_to_active": 0.22749876976013184, "contribution_to_active": 0.4203311800956726, "mix_mean": 0.4256840944290161, "mix_low": 0.02627452276647091, "mix_high": 0.006060600746423006, "read_cosine": 0.6008301973342896, "beta_mean": 0.06173944100737572, "nonfinite": 0.0, "weighted_memory_to_active": 0.09536466002464294, "mix_std": 0.2453162968158722, "beta_std": 0.05956293269991875, "beta_low": 0.5020142197608948, "beta_high": 0.0, "orthogonal_to_active": 0.043045055121183395, "output_influence": 0.35946324467658997, "residual_influence": 0.14213478565216064, "state_active": 0.05458498001098633, "state_memory": 0.015375983901321888, "state_peak": 38.73290252685547, "growth_active": 1.091392993927002, "growth_memory": 1.2892793416976929, "heads": [{"head": 0, "memory_to_active": 0.36186477541923523, "contribution_to_active": 0.20369145274162292, "mix_mean": 0.1980305314064026, "mix_low": 0.01982402801513672, "mix_high": 0.0, "read_cosine": 0.8050844073295593, "beta_mean": 0.07171304523944855, "nonfinite": 0.0, "weighted_memory_to_active": 0.08240289986133575, "mix_std": 0.18433918058872223, "beta_std": 0.061750076711177826, "beta_low": 0.006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.08841085433959961, "evidence_weights": [-0.10655462741851807, -0.04573535546660423, 0.4143841862678528], "agreement_weight": 0.022812729701399803, "orthogonal_weight": 1.7522797584533691}, {"head": 1, "memory_to_active": 0.4591120183467865, "contribution_to_active": 0.29502081871032715, "mix_mean": 0.21484015882015228, "mix_low": 0.12121772766113281, "mix_high": 9.5367431640625e-07, "read_cosine": 0.8482292294502258, "beta_mean": 0.2522062361240387, "nonfinite": 0.0, "weighted_memory_to_active": 0.20063291490077972, "mix_std": 0.2446320652961731, "beta_std": 0.24757257103919983, "beta_low": 0.09783935546875, "beta_high": 0.0, "orthogonal_to_active": 0.015142370946705341, "evidence_weights": [-0.0027297521010041237, -0.1973380595445633, -0.0893133133649826], "agreement_weight": 0.05760583281517029, "orthogonal_weight": 0.19836115837097168}, {"head": 2, "memory_to_active": 0.13842296600341797, "contribution_to_active": 0.5775357484817505, "mix_mean": 0.5822189450263977, "mix_low": 0.0029621124267578125, "mix_high": 0.003920555114746094, "read_cosine": 0.399791419506073, "beta_mean": 0.0022694903891533613, "nonfinite": 0.0, "weighted_memory_to_active": 0.08913508802652359, "mix_std": 0.26660534739494324, "beta_std": 0.00283531635068357, "beta_low": 0.97747802734375, "beta_high": 0.0, "orthogonal_to_active": 0.06964768469333649, "evidence_weights": [-0.1350836157798767, -0.31520411372184753, 0.9288082122802734], "agreement_weight": 0.0917326807975769, "orthogonal_weight": -0.8803980350494385}, {"head": 3, "memory_to_active": 0.04270555078983307, "contribution_to_active": 0.6030092239379883, "mix_mean": 0.6662777662277222, "mix_low": 0.0046520233154296875, "mix_high": 0.022482872009277344, "read_cosine": -0.09172449260950089, "beta_mean": 0.005515932105481625, "nonfinite": 0.0, "weighted_memory_to_active": 0.026576492935419083, "mix_std": 0.25855278968811035, "beta_std": 0.008099968545138836, "beta_low": 0.84906005859375, "beta_high": 0.0, "orthogonal_to_active": 0.01998727396130562, "evidence_weights": [-0.34384581446647644, -0.3092888295650482, 0.2907702922821045], "agreement_weight": 0.06524074077606201, "orthogonal_weight": -0.8195403814315796}, {"head": 4, "memory_to_active": 0.14949196577072144, "contribution_to_active": 0.5055199861526489, "mix_mean": 0.5131122469902039, "mix_low": 0.004458427429199219, "mix_high": 0.008368492126464844, "read_cosine": 0.897451639175415, "beta_mean": 0.037125520408153534, "nonfinite": 0.0, "weighted_memory_to_active": 0.08831784129142761, "mix_std": 0.2617684006690979, "beta_std": 0.028493404388427734, "beta_low": 0.10943603515625, "beta_high": 0.0, "orthogonal_to_active": 0.023033786565065384, "evidence_weights": [-0.2153797596693039, -0.24182161688804626, 0.43324533104896545], "agreement_weight": 0.13355115056037903, "orthogonal_weight": -0.600563645362854}, {"head": 5, "memory_to_active": 0.21339523792266846, "contribution_to_active": 0.3372098207473755, "mix_mean": 0.3796248137950897, "mix_low": 0.004532814025878906, "mix_high": 0.001590728759765625, "read_cosine": 0.7461485266685486, "beta_mean": 0.0016064104856923223, "nonfinite": 0.0, "weighted_memory_to_active": 0.0851227268576622, "mix_std": 0.2560000419616699, "beta_std": 0.00862623006105423, "beta_low": 0.97216796875, "beta_high": 0.0, "orthogonal_to_active": 0.042048368602991104, "evidence_weights": [-0.17388054728507996, -0.17539207637310028, 0.3310224115848541], "agreement_weight": 0.07475779950618744, "orthogonal_weight": -0.7523458003997803}]}, {"layer": 15, "memory_to_active": 0.05370892584323883, "contribution_to_active": 0.4839231073856354, "mix_mean": 0.4181704521179199, "mix_low": 0.008332411758601665, "mix_high": 0.0026221275329589844, "read_cosine": 0.6660144329071045, "beta_mean": 0.03565637394785881, "nonfinite": 0.0, "weighted_memory_to_active": 0.02575812116265297, "mix_std": 0.28290098905563354, "beta_std": 0.0436297282576561, "beta_low": 0.7141215801239014, "beta_high": 0.0, "orthogonal_to_active": 0.007513859309256077, "output_influence": 0.3030581474304199, "residual_influence": 0.17315034568309784, "state_active": 0.1538664847612381, "state_memory": 0.010753917507827282, "state_peak": 28.297147750854492, "growth_active": 1.173112392425537, "growth_memory": 1.3157628774642944, "heads": [{"head": 0, "memory_to_active": 0.09362119436264038, "contribution_to_active": 0.4696056842803955, "mix_mean": 0.38509929180145264, "mix_low": 0.008160591125488281, "mix_high": 0.006014823913574219, "read_cosine": 0.6472841501235962, "beta_mean": 0.03599728271365166, "nonfinite": 0.0, "weighted_memory_to_active": 0.04202429950237274, "mix_std": 0.28236860036849976, "beta_std": 0.027819350361824036, "beta_low": 0.1070556640625, "beta_high": 0.0, "orthogonal_to_active": 0.023047029972076416, "evidence_weights": [-0.18693774938583374, -0.18793435394763947, 0.4645988643169403], "agreement_weight": 0.09471505135297775, "orthogonal_weight": -0.7749304175376892}, {"head": 1, "memory_to_active": 0.11415663361549377, "contribution_to_active": 0.4204150438308716, "mix_mean": 0.4301968514919281, "mix_low": 0.0018596649169921875, "mix_high": 0.0049991607666015625, "read_cosine": 0.9281008243560791, "beta_mean": 0.000601044506765902, "nonfinite": 0.0, "weighted_memory_to_active": 0.04856210574507713, "mix_std": 0.28651222586631775, "beta_std": 0.002532350830733776, "beta_low": 0.99468994140625, "beta_high": 0.0, "orthogonal_to_active": 0.001706301816739142, "evidence_weights": [-0.14871957898139954, -0.15634267032146454, 0.17216862738132477], "agreement_weight": 0.12450238317251205, "orthogonal_weight": 0.07452570647001266}, {"head": 2, "memory_to_active": 0.005612293258309364, "contribution_to_active": 0.49665141105651855, "mix_mean": 0.34921276569366455, "mix_low": 0.012345314025878906, "mix_high": 9.34600830078125e-05, "read_cosine": 0.7522339820861816, "beta_mean": 7.540663500549272e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029136703815311193, "mix_std": 0.2687012851238251, "beta_std": 0.00017092988127842546, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0009044712060131133, "evidence_weights": [-0.2076311558485031, -0.2636427581310272, 0.11846770346164703], "agreement_weight": 0.009117789566516876, "orthogonal_weight": 0.8199421763420105}, {"head": 3, "memory_to_active": 0.002102596452459693, "contribution_to_active": 0.37193578481674194, "mix_mean": 0.39675799012184143, "mix_low": 0.006211280822753906, "mix_high": 0.000324249267578125, "read_cosine": 0.5981805324554443, "beta_mean": 1.1609856301220134e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0006828616023994982, "mix_std": 0.28506651520729065, "beta_std": 2.63777983491309e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00043378621921874583, "evidence_weights": [-0.2691478431224823, -0.24186007678508759, 0.22333484888076782], "agreement_weight": 0.05905633047223091, "orthogonal_weight": -0.6756663918495178}, {"head": 4, "memory_to_active": 0.07678008824586868, "contribution_to_active": 0.6197776794433594, "mix_mean": 0.3711380362510681, "mix_low": 0.011129379272460938, "mix_high": 0.0009555816650390625, "read_cosine": 0.7407559156417847, "beta_mean": 0.17437244951725006, "nonfinite": 0.0, "weighted_memory_to_active": 0.04108981788158417, "mix_std": 0.30422940850257874, "beta_std": 0.20989888906478882, "beta_low": 0.21331787109375, "beta_high": 0.0, "orthogonal_to_active": 0.0069044893607497215, "evidence_weights": [-0.20421569049358368, -0.2508087456226349, -0.014945726841688156], "agreement_weight": 0.013246121816337109, "orthogonal_weight": 0.35920587182044983}, {"head": 5, "memory_to_active": 0.029980743303894997, "contribution_to_active": 0.5251531600952148, "mix_mean": 0.5766174793243408, "mix_low": 0.010288238525390625, "mix_high": 0.003345489501953125, "read_cosine": 0.32953134179115295, "beta_mean": 0.002880438230931759, "nonfinite": 0.0, "weighted_memory_to_active": 0.019275963306427002, "mix_std": 0.2705278992652893, "beta_std": 0.02133042737841606, "beta_low": 0.9697265625, "beta_high": 0.0, "orthogonal_to_active": 0.012087075971066952, "evidence_weights": [-0.2832885980606079, -0.3024274408817291, 0.6731186509132385], "agreement_weight": 0.30190587043762207, "orthogonal_weight": 0.8282853364944458}]}]}, {"step": 60000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8390100359974895, "memory_to_active": 0.8014918565750122, "contribution_to_active": 0.5682060718536377, "mix_mean": 0.4175218939781189, "mix_low": 0.12901735305786133, "mix_high": 0.010449926368892193, "read_cosine": 0.3481397330760956, "beta_mean": 0.07169722765684128, "nonfinite": 0.0, "weighted_memory_to_active": 0.21190951764583588, "mix_std": 0.22367623448371887, "beta_std": 0.06262291967868805, "beta_low": 0.5708440542221069, "beta_high": 5.5948898079805076e-05, "orthogonal_to_active": 0.06170202046632767, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0879294872283936, "normal_token_accuracy": 0.442714124917984, "state_active": 0.06408269703388214, "state_memory": 0.017580430954694748, "state_peak": 126.44132232666016, "growth_active": 1.1183110475540161, "growth_memory": 1.3194094896316528, "output_influence": 0.4327232837677002, "residual_influence": 0.14981240034103394, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.273749351501465, "no_memory_loss_delta": 0.5647556185722351, "scrambled_loss_delta": 0.11962021887302399, "memory_only_loss_delta": 3.328556776046753, "no_evidence_loss_delta": 0.3012688457965851, "no_agreement_loss_delta": 0.009694082662463188, "no_orthogonal_loss_delta": 0.0011029457673430443}, {"index": 94, "tokens": 256, "normal_loss": 3.1579971313476562, "no_memory_loss_delta": 0.6569095253944397, "scrambled_loss_delta": 0.3852595090866089, "memory_only_loss_delta": 3.304877758026123, "no_evidence_loss_delta": 0.26398980617523193, "no_agreement_loss_delta": 0.037650689482688904, "no_orthogonal_loss_delta": 0.0464504174888134}, {"index": 188, "tokens": 256, "normal_loss": 3.3816306591033936, "no_memory_loss_delta": 0.8131327629089355, "scrambled_loss_delta": 0.132596954703331, "memory_only_loss_delta": 3.0304017066955566, "no_evidence_loss_delta": 0.43087366223335266, "no_agreement_loss_delta": 0.052669961005449295, "no_orthogonal_loss_delta": 0.04570696875452995}, {"index": 283, "tokens": 131, "normal_loss": 2.0139238834381104, "no_memory_loss_delta": 0.3604676425457001, "scrambled_loss_delta": 0.21229210495948792, "memory_only_loss_delta": 2.690265417098999, "no_evidence_loss_delta": 0.11174467206001282, "no_agreement_loss_delta": -0.024543438106775284, "no_orthogonal_loss_delta": 0.021446838974952698}], "agreement_weight_abs_mean": 0.22726356983184814, "agreement_weight_abs_max": 6.15099573135376, "agreement_weight_mean": 0.19868822395801544, "orthogonal_weight_abs_mean": 0.8939946293830872, "orthogonal_weight_abs_max": 9.892640113830566, "orthogonal_weight_mean": -0.09836433082818985, "no_memory_loss": 3.719886302947998, "no_memory_loss_delta": 0.6319571733474731, "no_memory_token_accuracy_gain": 0.08342602849006653, "no_memory_token_win_fraction": 0.6718575954437256, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 46.844955408249156, "scrambled_loss": 3.3003926277160645, "scrambled_loss_delta": 0.21246303617954254, "scrambled_token_accuracy_gain": 0.01890989951789379, "scrambled_token_win_fraction": 0.6240267157554626, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.14097956478015, "memory_only_loss": 6.231829643249512, "memory_only_loss_delta": 3.1439006328582764, "memory_only_token_accuracy_gain": 0.2836484909057617, "memory_only_token_win_fraction": 0.9010010957717896, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 95.68857035569202, "no_evidence_loss": 3.3878724575042725, "no_evidence_loss_delta": 0.29994261264801025, "no_evidence_token_accuracy_gain": 0.03337041288614273, "no_evidence_token_win_fraction": 0.6195772886276245, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 25.91392645023968, "no_agreement_loss": 3.112833261489868, "no_agreement_loss_delta": 0.02490387111902237, "no_agreement_token_accuracy_gain": -0.0055617354810237885, "no_agreement_token_win_fraction": 0.47942158579826355, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.4596328014769986, "no_orthogonal_loss": 3.1176111698150635, "no_orthogonal_loss_delta": 0.029682070016860962, "no_orthogonal_token_accuracy_gain": -0.0055617354810237885, "no_orthogonal_token_win_fraction": 0.5951056480407715, "no_orthogonal_probe_win_fraction": 1.0, "no_orthogonal_ppl_reduction_pct": 2.9245883668113115, "short_no_memory_loss_delta": 0.5877103209495544, "long_context_memory_gain": 0.0442468523979187, "normal_context_loss_gain": 0.13430467247962952, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.4819130897521973, "contribution_to_active": 1.2533683776855469, "mix_mean": 0.10673568397760391, "mix_low": 0.517916202545166, "mix_high": 0.005073547828942537, "read_cosine": 0.42056509852409363, "beta_mean": 0.2103457748889923, "nonfinite": 0.0, "weighted_memory_to_active": 1.293318271636963, "mix_std": 0.09325374662876129, "beta_std": 0.14261598885059357, "beta_low": 0.11419677734375, "beta_high": 0.0, "orthogonal_to_active": 0.049279388040304184, "output_influence": 0.23408859968185425, "residual_influence": 0.2641046345233917, "state_active": 0.06871502846479416, "state_memory": 0.04370472580194473, "state_peak": 42.629207611083984, "growth_active": 1.3318864107131958, "growth_memory": 1.2773405313491821, "heads": [{"head": 0, "memory_to_active": 2.732682466506958, "contribution_to_active": 0.5491896867752075, "mix_mean": 0.019892994314432144, "mix_low": 0.625849723815918, "mix_high": 2.193450927734375e-05, "read_cosine": 0.2134483903646469, "beta_mean": 0.19041535258293152, "nonfinite": 0.0, "weighted_memory_to_active": 0.6806708574295044, "mix_std": 0.06261593103408813, "beta_std": 0.09543383866548538, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07324163615703583, "evidence_weights": [-0.003930348437279463, -0.04595380276441574, -0.04233725741505623], "agreement_weight": -0.2106906771659851, "orthogonal_weight": -0.12051897495985031}, {"head": 1, "memory_to_active": 6.478096008300781, "contribution_to_active": 3.0385336875915527, "mix_mean": 0.020618055015802383, "mix_low": 0.6131210327148438, "mix_high": 0.0, "read_cosine": 0.4323810935020447, "beta_mean": 0.44917166233062744, "nonfinite": 0.0, "weighted_memory_to_active": 3.3274731636047363, "mix_std": 0.05964135378599167, "beta_std": 0.2351963222026825, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.019610635936260223, "evidence_weights": [0.01873410865664482, -0.10861872136592865, 0.6482162475585938], "agreement_weight": 0.029330750927329063, "orthogonal_weight": 0.01275904756039381}, {"head": 2, "memory_to_active": 2.7058074474334717, "contribution_to_active": 0.6321227550506592, "mix_mean": 0.015694191679358482, "mix_low": 0.6486082077026367, "mix_high": 0.0, "read_cosine": 0.32025346159935, "beta_mean": 0.24057698249816895, "nonfinite": 0.0, "weighted_memory_to_active": 0.7379201054573059, "mix_std": 0.03656169772148132, "beta_std": 0.1808287799358368, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.06539654731750488, "evidence_weights": [0.062410615384578705, 0.05347803607583046, 0.18840816617012024], "agreement_weight": 0.011100013740360737, "orthogonal_weight": -0.20816460251808167}, {"head": 3, "memory_to_active": 7.24306058883667, "contribution_to_active": 2.745131254196167, "mix_mean": 0.03234000504016876, "mix_low": 0.4620180130004883, "mix_high": 0.0, "read_cosine": 0.32904136180877686, "beta_mean": 0.3154340386390686, "nonfinite": 0.0, "weighted_memory_to_active": 2.9086947441101074, "mix_std": 0.07431869953870773, "beta_std": 0.24556513130664825, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11980240046977997, "evidence_weights": [-0.04140923544764519, -0.03950037062168121, -0.13395808637142181], "agreement_weight": -0.005231223534792662, "orthogonal_weight": -0.049918364733457565}, {"head": 4, "memory_to_active": 1.6338506937026978, "contribution_to_active": 0.07569023966789246, "mix_mean": 0.0120112095028162, "mix_low": 0.7439231872558594, "mix_high": 0.0, "read_cosine": 0.29393112659454346, "beta_mean": 0.046539902687072754, "nonfinite": 0.0, "weighted_memory_to_active": 0.05250959098339081, "mix_std": 0.03929241746664047, "beta_std": 0.07680859416723251, "beta_low": 0.30615234375, "beta_high": 0.0, "orthogonal_to_active": 0.002018467988818884, "evidence_weights": [0.12988021969795227, 0.0948437973856926, -0.13563396036624908], "agreement_weight": -0.14951105415821075, "orthogonal_weight": 0.031873732805252075}, {"head": 5, "memory_to_active": 0.09798097610473633, "contribution_to_active": 0.47954222559928894, "mix_mean": 0.5398576259613037, "mix_low": 0.01397705078125, "mix_high": 0.030419349670410156, "read_cosine": 0.9343351125717163, "beta_mean": 0.019936691969633102, "nonfinite": 0.0, "weighted_memory_to_active": 0.05264043062925339, "mix_std": 0.2870923578739166, "beta_std": 0.02186325006186962, "beta_low": 0.37890625, "beta_high": 0.0, "orthogonal_to_active": 0.015606638044118881, "evidence_weights": [-0.17570851743221283, -0.21817661821842194, 0.38796839118003845], "agreement_weight": 0.29372167587280273, "orthogonal_weight": -0.6775128841400146}]}, {"layer": 1, "memory_to_active": 2.3366479873657227, "contribution_to_active": 0.49686330556869507, "mix_mean": 0.21314328908920288, "mix_low": 0.36718130111694336, "mix_high": 0.0013774236431345344, "read_cosine": 0.3149780035018921, "beta_mean": 0.18062302470207214, "nonfinite": 0.0, "weighted_memory_to_active": 0.3368690013885498, "mix_std": 0.11638708412647247, "beta_std": 0.13059470057487488, "beta_low": 0.3490194082260132, "beta_high": 9.1552734375e-05, "orthogonal_to_active": 0.04333408549427986, "output_influence": 0.2510839104652405, "residual_influence": 0.06830842792987823, "state_active": 0.029826639220118523, "state_memory": 0.03476688638329506, "state_peak": 41.0487174987793, "growth_active": 1.1921296119689941, "growth_memory": 1.435753583908081, "heads": [{"head": 0, "memory_to_active": 0.03369602933526039, "contribution_to_active": 0.4870382249355316, "mix_mean": 0.5201448202133179, "mix_low": 0.0057582855224609375, "mix_high": 0.0021982192993164062, "read_cosine": 0.048026613891124725, "beta_mean": 0.00023022937239147723, "nonfinite": 0.0, "weighted_memory_to_active": 0.020251192152500153, "mix_std": 0.2833107113838196, "beta_std": 0.0003701964160427451, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01970335841178894, "evidence_weights": [-0.20626680552959442, -0.2710583806037903, 0.31508946418762207], "agreement_weight": 0.19736891984939575, "orthogonal_weight": -1.029159426689148}, {"head": 1, "memory_to_active": 2.861802577972412, "contribution_to_active": 0.29532817006111145, "mix_mean": 0.022515008226037025, "mix_low": 0.48270225524902344, "mix_high": 0.0, "read_cosine": 0.5209556818008423, "beta_mean": 0.25551676750183105, "nonfinite": 0.0, "weighted_memory_to_active": 0.3471481502056122, "mix_std": 0.04415871948003769, "beta_std": 0.18413594365119934, "beta_low": 0.003662109375, "beta_high": 0.000244140625, "orthogonal_to_active": 0.020157771185040474, "evidence_weights": [0.022915178909897804, -0.05863330513238907, 0.07105113565921783], "agreement_weight": 0.003403583075851202, "orthogonal_weight": 0.19021806120872498}, {"head": 2, "memory_to_active": 0.05040658265352249, "contribution_to_active": 0.555787205696106, "mix_mean": 0.6731798648834229, "mix_low": 0.0004520416259765625, "mix_high": 0.006066322326660156, "read_cosine": 0.509458065032959, "beta_mean": 0.002012417418882251, "nonfinite": 0.0, "weighted_memory_to_active": 0.03390635550022125, "mix_std": 0.2344941794872284, "beta_std": 0.0036551151424646378, "beta_low": 0.97247314453125, "beta_high": 0.0, "orthogonal_to_active": 0.02231721580028534, "evidence_weights": [-0.2804834842681885, -0.37413638830184937, 0.6638674139976501], "agreement_weight": 0.041443269699811935, "orthogonal_weight": -0.8095616698265076}, {"head": 3, "memory_to_active": 2.995486259460449, "contribution_to_active": 0.1864100694656372, "mix_mean": 0.023165006190538406, "mix_low": 0.5513648986816406, "mix_high": 0.0, "read_cosine": 0.1992974728345871, "beta_mean": 0.3505588471889496, "nonfinite": 0.0, "weighted_memory_to_active": 0.24167516827583313, "mix_std": 0.06011159345507622, "beta_std": 0.2636204659938812, "beta_low": 0.04937744140625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.09372824430465698, "evidence_weights": [0.0392613410949707, -0.015777919441461563, 0.008828778751194477], "agreement_weight": -0.026644261553883553, "orthogonal_weight": -0.4839935600757599}, {"head": 4, "memory_to_active": 3.8379735946655273, "contribution_to_active": 1.363761305809021, "mix_mean": 0.03311601281166077, "mix_low": 0.3268918991088867, "mix_high": 0.0, "read_cosine": 0.34160754084587097, "beta_mean": 0.16097871959209442, "nonfinite": 0.0, "weighted_memory_to_active": 1.3132753372192383, "mix_std": 0.05837274715304375, "beta_std": 0.11424646526575089, "beta_low": 0.03179931640625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08060205727815628, "evidence_weights": [-0.02910754084587097, -0.004149680957198143, 0.11453994363546371], "agreement_weight": -0.011736230924725533, "orthogonal_weight": 0.08903097361326218}, {"head": 5, "memory_to_active": 4.240522384643555, "contribution_to_active": 0.09285462647676468, "mix_mean": 0.006738928146660328, "mix_low": 0.8359184265136719, "mix_high": 0.0, "read_cosine": 0.27052247524261475, "beta_mean": 0.31444114446640015, "nonfinite": 0.0, "weighted_memory_to_active": 0.06495767086744308, "mix_std": 0.017874538898468018, "beta_std": 0.21753999590873718, "beta_low": 0.03680419921875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.023495864123106003, "evidence_weights": [0.15705379843711853, 0.08105891942977905, 0.09867927432060242], "agreement_weight": -0.11952632665634155, "orthogonal_weight": 0.3624098300933838}]}, {"layer": 2, "memory_to_active": 1.0273288488388062, "contribution_to_active": 0.3651374578475952, "mix_mean": 0.4047432541847229, "mix_low": 0.24539312720298767, "mix_high": 0.004599412437528372, "read_cosine": 0.30478933453559875, "beta_mean": 0.09208263456821442, "nonfinite": 0.0, "weighted_memory_to_active": 0.08665887266397476, "mix_std": 0.1889602243900299, "beta_std": 0.07214042544364929, "beta_low": 0.5858663320541382, "beta_high": 0.0002848307485692203, "orthogonal_to_active": 0.020759202539920807, "output_influence": 0.3974556028842926, "residual_influence": 0.10666351020336151, "state_active": 0.037883102893829346, "state_memory": 0.015114402398467064, "state_peak": 26.65241241455078, "growth_active": 1.0164462327957153, "growth_memory": 1.5566470623016357, "heads": [{"head": 0, "memory_to_active": 0.013535914942622185, "contribution_to_active": 0.347022145986557, "mix_mean": 0.6217197179794312, "mix_low": 0.00060272216796875, "mix_high": 0.008090972900390625, "read_cosine": 0.12818631529808044, "beta_mean": 9.997015877161175e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008912119083106518, "mix_std": 0.2706212103366852, "beta_std": 0.00017742699128575623, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009534068405628204, "evidence_weights": [-0.3673674464225769, -0.2134544998407364, 0.17448973655700684], "agreement_weight": 0.15078097581863403, "orthogonal_weight": -1.1064852476119995}, {"head": 1, "memory_to_active": 3.2110071182250977, "contribution_to_active": 0.045417267829179764, "mix_mean": 0.005196649115532637, "mix_low": 0.8880910873413086, "mix_high": 0.0, "read_cosine": 0.14144150912761688, "beta_mean": 0.2715615928173065, "nonfinite": 0.0, "weighted_memory_to_active": 0.03801467642188072, "mix_std": 0.011305097490549088, "beta_std": 0.16286420822143555, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.010969512164592743, "evidence_weights": [0.14016540348529816, 0.11629916727542877, 0.598222017288208], "agreement_weight": 0.02750805951654911, "orthogonal_weight": 0.3288816213607788}, {"head": 2, "memory_to_active": 0.007997793145477772, "contribution_to_active": 0.4271533489227295, "mix_mean": 0.5488823652267456, "mix_low": 0.001247406005859375, "mix_high": 0.0036687850952148438, "read_cosine": 0.36320483684539795, "beta_mean": 2.4846161977620795e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003249481786042452, "mix_std": 0.2765192985534668, "beta_std": 4.202793206786737e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001427021052222699, "evidence_weights": [-0.254645973443985, -0.2946930229663849, 0.5895942449569702], "agreement_weight": -0.03165411204099655, "orthogonal_weight": -0.05234886705875397}, {"head": 3, "memory_to_active": 0.11599443852901459, "contribution_to_active": 0.5489096641540527, "mix_mean": 0.5765098929405212, "mix_low": 0.000225067138671875, "mix_high": 0.006241798400878906, "read_cosine": 0.6615560054779053, "beta_mean": 0.02022799290716648, "nonfinite": 0.0, "weighted_memory_to_active": 0.07290574163198471, "mix_std": 0.2570449709892273, "beta_std": 0.026779253035783768, "beta_low": 0.4847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.0465017631649971, "evidence_weights": [-0.2031305879354477, -0.32149145007133484, 0.7802423238754272], "agreement_weight": 0.053173936903476715, "orthogonal_weight": -0.893884003162384}, {"head": 4, "memory_to_active": 2.804476499557495, "contribution_to_active": 0.29041385650634766, "mix_mean": 0.02082628570497036, "mix_low": 0.5815210342407227, "mix_high": 9.5367431640625e-07, "read_cosine": 0.20616415143013, "beta_mean": 0.26055142283439636, "nonfinite": 0.0, "weighted_memory_to_active": 0.3907244801521301, "mix_std": 0.05520550161600113, "beta_std": 0.2429305911064148, "beta_low": 0.029541015625, "beta_high": 0.001708984375, "orthogonal_to_active": 0.05594796687364578, "evidence_weights": [0.024732504040002823, 0.006048389244824648, 0.02037181332707405], "agreement_weight": 0.022346826270222664, "orthogonal_weight": -0.16383953392505646}, {"head": 5, "memory_to_active": 0.01096070371568203, "contribution_to_active": 0.5319084525108337, "mix_mean": 0.6553246974945068, "mix_low": 0.00067138671875, "mix_high": 0.009593963623046875, "read_cosine": 0.32818320393562317, "beta_mean": 2.995561771967914e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006146697793155909, "mix_std": 0.2630651891231537, "beta_std": 4.905250170850195e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014592092484235764, "evidence_weights": [-0.2831793427467346, -0.29336637258529663, 0.9628486037254333], "agreement_weight": 0.21494898200035095, "orthogonal_weight": -0.27215516567230225}]}, {"layer": 3, "memory_to_active": 0.4592422842979431, "contribution_to_active": 0.4708244502544403, "mix_mean": 0.5038182735443115, "mix_low": 0.05658912658691406, "mix_high": 0.01201772689819336, "read_cosine": 0.20177119970321655, "beta_mean": 0.026693329215049744, "nonfinite": 0.0, "weighted_memory_to_active": 0.1305733621120453, "mix_std": 0.2615441381931305, "beta_std": 0.025745699182152748, "beta_low": 0.7644857168197632, "beta_high": 0.0, "orthogonal_to_active": 0.05325627699494362, "output_influence": 0.4078683853149414, "residual_influence": 0.09523583948612213, "state_active": 0.023370947688817978, "state_memory": 0.007028316147625446, "state_peak": 22.446428298950195, "growth_active": 1.0381375551223755, "growth_memory": 1.6207497119903564, "heads": [{"head": 0, "memory_to_active": 0.35834503173828125, "contribution_to_active": 0.5979946255683899, "mix_mean": 0.5957788228988647, "mix_low": 0.0026197433471679688, "mix_high": 0.0248565673828125, "read_cosine": 0.23946084082126617, "beta_mean": 0.010150664485991001, "nonfinite": 0.0, "weighted_memory_to_active": 0.2269563227891922, "mix_std": 0.30765199661254883, "beta_std": 0.014606593176722527, "beta_low": 0.68609619140625, "beta_high": 0.0, "orthogonal_to_active": 0.16075478494167328, "evidence_weights": [-0.21671752631664276, -0.2509509325027466, 0.9828349947929382], "agreement_weight": 0.11674164235591888, "orthogonal_weight": -0.7557418942451477}, {"head": 1, "memory_to_active": 0.009800046682357788, "contribution_to_active": 0.431560218334198, "mix_mean": 0.5626089572906494, "mix_low": 0.004834175109863281, "mix_high": 0.009976387023925781, "read_cosine": 0.1824025958776474, "beta_mean": 5.104445881443098e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005785043351352215, "mix_std": 0.2996307909488678, "beta_std": 0.00012792700727004558, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004766412544995546, "evidence_weights": [-0.2656748294830322, -0.2145870327949524, 0.6894857287406921], "agreement_weight": 0.5312734246253967, "orthogonal_weight": -0.900834321975708}, {"head": 2, "memory_to_active": 2.099123477935791, "contribution_to_active": 0.29901769757270813, "mix_mean": 0.05307338014245033, "mix_low": 0.31967926025390625, "mix_high": 0.0, "read_cosine": 0.2066909372806549, "beta_mean": 0.14476187527179718, "nonfinite": 0.0, "weighted_memory_to_active": 0.3368878960609436, "mix_std": 0.10552256554365158, "beta_std": 0.12785685062408447, "beta_low": 0.0181884765625, "beta_high": 0.0, "orthogonal_to_active": 0.0008245323551818728, "evidence_weights": [-0.0932103767991066, 0.0011298483004793525, -0.032365232706069946], "agreement_weight": -0.015403051860630512, "orthogonal_weight": -0.003071834100410342}, {"head": 3, "memory_to_active": 0.015682319179177284, "contribution_to_active": 0.5003451108932495, "mix_mean": 0.5891497135162354, "mix_low": 0.005825996398925781, "mix_high": 0.011487007141113281, "read_cosine": 0.28252726793289185, "beta_mean": 7.696318789385259e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010241981595754623, "mix_std": 0.29479730129241943, "beta_std": 0.0001885788224171847, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028160347137600183, "evidence_weights": [-0.25391677021980286, -0.22473157942295074, 2.2313499450683594], "agreement_weight": 2.1286087036132812, "orthogonal_weight": -0.35767653584480286}, {"head": 4, "memory_to_active": 0.012177187949419022, "contribution_to_active": 0.3670068383216858, "mix_mean": 0.5018477439880371, "mix_low": 0.004364967346191406, "mix_high": 0.0028657913208007812, "read_cosine": 0.14691950380802155, "beta_mean": 7.20502357580699e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007689233869314194, "mix_std": 0.28984081745147705, "beta_std": 0.00011728908430086449, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007484066300094128, "evidence_weights": [-0.22629287838935852, -0.2181580513715744, 0.15563583374023438], "agreement_weight": -0.03816037252545357, "orthogonal_weight": -1.0460596084594727}, {"head": 5, "memory_to_active": 0.26032549142837524, "contribution_to_active": 0.6290222406387329, "mix_mean": 0.720450758934021, "mix_low": 0.0022106170654296875, "mix_high": 0.022920608520507812, "read_cosine": 0.15262597799301147, "beta_mean": 0.0050473762676119804, "nonfinite": 0.0, "weighted_memory_to_active": 0.19587962329387665, "mix_std": 0.2718213200569153, "beta_std": 0.011576944962143898, "beta_low": 0.88262939453125, "beta_high": 0.0, "orthogonal_to_active": 0.1428918093442917, "evidence_weights": [-0.2626115083694458, -0.3374590575695038, 0.5054593682289124], "agreement_weight": 0.20773546397686005, "orthogonal_weight": -0.7704194784164429}]}, {"layer": 4, "memory_to_active": 0.9974463582038879, "contribution_to_active": 0.4721108675003052, "mix_mean": 0.36594337224960327, "mix_low": 0.2565101087093353, "mix_high": 0.010581175796687603, "read_cosine": 0.3140307664871216, "beta_mean": 0.1125769317150116, "nonfinite": 0.0, "weighted_memory_to_active": 0.12702347338199615, "mix_std": 0.1608133316040039, "beta_std": 0.10124879330396652, "beta_low": 0.38568115234375, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.049335576593875885, "output_influence": 0.48188894987106323, "residual_influence": 0.14353159070014954, "state_active": 0.0369865857064724, "state_memory": 0.017407111823558807, "state_peak": 39.4848518371582, "growth_active": 1.1761846542358398, "growth_memory": 1.5363037586212158, "heads": [{"head": 0, "memory_to_active": 0.16510774195194244, "contribution_to_active": 0.7898677587509155, "mix_mean": 0.6793925762176514, "mix_low": 0.013957977294921875, "mix_high": 0.027756690979003906, "read_cosine": 0.4638183116912842, "beta_mean": 0.252006471157074, "nonfinite": 0.0, "weighted_memory_to_active": 0.13969023525714874, "mix_std": 0.34860333800315857, "beta_std": 0.2503267824649811, "beta_low": 0.23992919921875, "beta_high": 0.0, "orthogonal_to_active": 0.07796403020620346, "evidence_weights": [-0.2597253918647766, -0.3659053444862366, -0.0597049705684185], "agreement_weight": 0.00010758695134427398, "orthogonal_weight": 0.6764803528785706}, {"head": 1, "memory_to_active": 0.04728034883737564, "contribution_to_active": 0.6588793992996216, "mix_mean": 0.6950263977050781, "mix_low": 0.00013637542724609375, "mix_high": 0.016900062561035156, "read_cosine": 0.6075410842895508, "beta_mean": 0.0007726546027697623, "nonfinite": 0.0, "weighted_memory_to_active": 0.03485732898116112, "mix_std": 0.23975786566734314, "beta_std": 0.005147155374288559, "beta_low": 0.9833984375, "beta_high": 0.0, "orthogonal_to_active": 0.018721846863627434, "evidence_weights": [-0.2411804348230362, -0.3638710677623749, 0.6226528286933899], "agreement_weight": 0.3180224299430847, "orthogonal_weight": -0.7549982070922852}, {"head": 2, "memory_to_active": 1.6420514583587646, "contribution_to_active": 0.33574795722961426, "mix_mean": 0.03691752254962921, "mix_low": 0.4250631332397461, "mix_high": 1.9073486328125e-05, "read_cosine": 0.22289060056209564, "beta_mean": 0.06494592875242233, "nonfinite": 0.0, "weighted_memory_to_active": 0.26627933979034424, "mix_std": 0.07200586050748825, "beta_std": 0.06215459108352661, "beta_low": 0.06158447265625, "beta_high": 0.0, "orthogonal_to_active": 0.07501904666423798, "evidence_weights": [0.04419756308197975, -0.024455714970827103, 0.2554038465023041], "agreement_weight": 0.13588447868824005, "orthogonal_weight": -0.39774081110954285}, {"head": 3, "memory_to_active": 1.8137433528900146, "contribution_to_active": 0.13753172755241394, "mix_mean": 0.031824588775634766, "mix_low": 0.4294862747192383, "mix_high": 0.0, "read_cosine": 0.20695221424102783, "beta_mean": 0.1358884871006012, "nonfinite": 0.0, "weighted_memory_to_active": 0.18670547008514404, "mix_std": 0.05682831257581711, "beta_std": 0.12572874128818512, "beta_low": 0.0291748046875, "beta_high": 0.0, "orthogonal_to_active": 0.002570346463471651, "evidence_weights": [-0.017211658880114555, 0.05244964733719826, -0.27081188559532166], "agreement_weight": 0.07055018097162247, "orthogonal_weight": -0.01951472833752632}, {"head": 4, "memory_to_active": 0.00764232873916626, "contribution_to_active": 0.6450268030166626, "mix_mean": 0.7387636303901672, "mix_low": 0.00021266937255859375, "mix_high": 0.018808364868164062, "read_cosine": 0.15071958303451538, "beta_mean": 1.9060013073612936e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005383703857660294, "mix_std": 0.21687819063663483, "beta_std": 3.625127283157781e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0033758818171918392, "evidence_weights": [-0.33030933141708374, -0.3202442526817322, 0.35412904620170593], "agreement_weight": 0.3411368727684021, "orthogonal_weight": -0.6621766090393066}, {"head": 5, "memory_to_active": 2.3088526725769043, "contribution_to_active": 0.26561135053634644, "mix_mean": 0.013735434040427208, "mix_low": 0.6702041625976562, "mix_high": 2.86102294921875e-06, "read_cosine": 0.23226293921470642, "beta_mean": 0.22182895243167877, "nonfinite": 0.0, "weighted_memory_to_active": 0.1292247474193573, "mix_std": 0.030806392431259155, "beta_std": 0.16409923136234283, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.11836230754852295, "evidence_weights": [0.05531392991542816, -0.020473813638091087, -0.2755173146724701], "agreement_weight": -0.08037921786308289, "orthogonal_weight": 1.019676923751831}]}, {"layer": 5, "memory_to_active": 0.41815125942230225, "contribution_to_active": 0.5303007960319519, "mix_mean": 0.506951093673706, "mix_low": 0.059258460998535156, "mix_high": 0.019544601440429688, "read_cosine": 0.09681833535432816, "beta_mean": 0.05443649739027023, "nonfinite": 0.0, "weighted_memory_to_active": 0.13792017102241516, "mix_std": 0.255031555891037, "beta_std": 0.0693327784538269, "beta_low": 0.6659647822380066, "beta_high": 0.0, "orthogonal_to_active": 0.058162838220596313, "output_influence": 0.5543256998062134, "residual_influence": 0.16461075842380524, "state_active": 0.039984121918678284, "state_memory": 0.010347193107008934, "state_peak": 65.28881072998047, "growth_active": 1.0943013429641724, "growth_memory": 1.235012173652649, "heads": [{"head": 0, "memory_to_active": 0.23803383111953735, "contribution_to_active": 0.5645958781242371, "mix_mean": 0.4550268352031708, "mix_low": 0.011694908142089844, "mix_high": 0.0019626617431640625, "read_cosine": -0.06739388406276703, "beta_mean": 0.020459556952118874, "nonfinite": 0.0, "weighted_memory_to_active": 0.08709654957056046, "mix_std": 0.2819727063179016, "beta_std": 0.03798018395900726, "beta_low": 0.54864501953125, "beta_high": 0.0, "orthogonal_to_active": 0.03730582073330879, "evidence_weights": [-0.16208741068840027, -0.18158678710460663, 0.0726095587015152], "agreement_weight": -0.02416887879371643, "orthogonal_weight": -0.436988890171051}, {"head": 1, "memory_to_active": 0.008174411952495575, "contribution_to_active": 0.5298596620559692, "mix_mean": 0.6917131543159485, "mix_low": 0.0010852813720703125, "mix_high": 0.03998279571533203, "read_cosine": 0.20836307108402252, "beta_mean": 1.985577000596095e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004864051938056946, "mix_std": 0.2751203179359436, "beta_std": 3.857723277178593e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004579936154186726, "evidence_weights": [-0.2495286911725998, -0.2880316972732544, 0.9302725195884705], "agreement_weight": 0.6918657422065735, "orthogonal_weight": -0.9938610792160034}, {"head": 2, "memory_to_active": 0.013855678960680962, "contribution_to_active": 0.590498685836792, "mix_mean": 0.6601126194000244, "mix_low": 0.0029630661010742188, "mix_high": 0.021913528442382812, "read_cosine": -0.4278641939163208, "beta_mean": 9.744449926074594e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008210055530071259, "mix_std": 0.2658071219921112, "beta_std": 0.00029887049458920956, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0059265755116939545, "evidence_weights": [-0.26413777470588684, -0.33021560311317444, 0.1670621633529663], "agreement_weight": -0.10761252045631409, "orthogonal_weight": -0.82558274269104}, {"head": 3, "memory_to_active": 1.715822458267212, "contribution_to_active": 0.25063806772232056, "mix_mean": 0.11306779086589813, "mix_low": 0.13078975677490234, "mix_high": 9.5367431640625e-07, "read_cosine": 0.15683725476264954, "beta_mean": 0.11685687303543091, "nonfinite": 0.0, "weighted_memory_to_active": 0.3855980336666107, "mix_std": 0.14970535039901733, "beta_std": 0.16196724772453308, "beta_low": 0.162109375, "beta_high": 0.0, "orthogonal_to_active": 0.16229033470153809, "evidence_weights": [-0.07470262795686722, -0.09996452927589417, 0.05943388491868973], "agreement_weight": 0.04944579303264618, "orthogonal_weight": -0.49573102593421936}, {"head": 4, "memory_to_active": 0.44842270016670227, "contribution_to_active": 0.5667511820793152, "mix_mean": 0.31470394134521484, "mix_low": 0.2073841094970703, "mix_high": 0.004887580871582031, "read_cosine": 0.5336825847625732, "beta_mean": 0.18894629180431366, "nonfinite": 0.0, "weighted_memory_to_active": 0.2838122844696045, "mix_std": 0.3347584903240204, "beta_std": 0.21462444961071014, "beta_low": 0.287109375, "beta_high": 0.0, "orthogonal_to_active": 0.1044921725988388, "evidence_weights": [-0.11743185669183731, -0.2598126530647278, -0.04375399276614189], "agreement_weight": 0.02825290523469448, "orthogonal_weight": 0.554364800453186}, {"head": 5, "memory_to_active": 0.08459865301847458, "contribution_to_active": 0.6794610023498535, "mix_mean": 0.8070822954177856, "mix_low": 0.0016336441040039062, "mix_high": 0.04852008819580078, "read_cosine": 0.1772850900888443, "beta_mean": 0.00023897249775473028, "nonfinite": 0.0, "weighted_memory_to_active": 0.05794008821249008, "mix_std": 0.22282524406909943, "beta_std": 0.00108728080522269, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.03438220173120499, "evidence_weights": [-0.31141144037246704, -0.3324957489967346, 1.8878661394119263], "agreement_weight": 0.4887229800224304, "orthogonal_weight": -0.629422664642334}]}, {"layer": 6, "memory_to_active": 0.2839851975440979, "contribution_to_active": 0.6212493181228638, "mix_mean": 0.5899304151535034, "mix_low": 0.04540793225169182, "mix_high": 0.03269672393798828, "read_cosine": 0.31102609634399414, "beta_mean": 0.06678905338048935, "nonfinite": 0.0, "weighted_memory_to_active": 0.14244458079338074, "mix_std": 0.23887822031974792, "beta_std": 0.07676491141319275, "beta_low": 0.5677490234375, "beta_high": 0.0, "orthogonal_to_active": 0.049635015428066254, "output_influence": 0.557375431060791, "residual_influence": 0.19307011365890503, "state_active": 0.04542122781276703, "state_memory": 0.007237973622977734, "state_peak": 85.14524841308594, "growth_active": 1.2077518701553345, "growth_memory": 1.149194598197937, "heads": [{"head": 0, "memory_to_active": 0.11273831129074097, "contribution_to_active": 0.7465598583221436, "mix_mean": 0.8150440454483032, "mix_low": 0.00033664703369140625, "mix_high": 0.050993919372558594, "read_cosine": 0.26524096727371216, "beta_mean": 0.011626767925918102, "nonfinite": 0.0, "weighted_memory_to_active": 0.09441376477479935, "mix_std": 0.20537728071212769, "beta_std": 0.025931447744369507, "beta_low": 0.75262451171875, "beta_high": 0.0, "orthogonal_to_active": 0.06750389188528061, "evidence_weights": [-0.31333693861961365, -0.3517549932003021, 1.2384783029556274], "agreement_weight": 0.6563617587089539, "orthogonal_weight": -0.7713857293128967}, {"head": 1, "memory_to_active": 0.00488643255084753, "contribution_to_active": 0.5244020223617554, "mix_mean": 0.5931168794631958, "mix_low": 0.0011138916015625, "mix_high": 0.014939308166503906, "read_cosine": 0.46866732835769653, "beta_mean": 9.889072316582315e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022635729983448982, "mix_std": 0.28446370363235474, "beta_std": 1.6612004401395097e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012296270579099655, "evidence_weights": [-0.19665920734405518, -0.30065807700157166, 1.1127369403839111], "agreement_weight": 0.0995049774646759, "orthogonal_weight": -0.7198370695114136}, {"head": 2, "memory_to_active": 0.7977118492126465, "contribution_to_active": 0.10073082894086838, "mix_mean": 0.045893073081970215, "mix_low": 0.26326847076416016, "mix_high": 9.5367431640625e-07, "read_cosine": 0.26087507605552673, "beta_mean": 0.028970593586564064, "nonfinite": 0.0, "weighted_memory_to_active": 0.070426344871521, "mix_std": 0.07354788482189178, "beta_std": 0.0325431264936924, "beta_low": 0.168701171875, "beta_high": 0.0, "orthogonal_to_active": 0.00897978339344263, "evidence_weights": [0.07139380276203156, -0.05130875110626221, 0.34738248586654663], "agreement_weight": 0.15042804181575775, "orthogonal_weight": -0.15377451479434967}, {"head": 3, "memory_to_active": 0.14240816235542297, "contribution_to_active": 0.6675853729248047, "mix_mean": 0.6542868614196777, "mix_low": 0.006964683532714844, "mix_high": 0.0190582275390625, "read_cosine": 0.4734441637992859, "beta_mean": 0.22743633389472961, "nonfinite": 0.0, "weighted_memory_to_active": 0.09852325916290283, "mix_std": 0.32503455877304077, "beta_std": 0.2560604512691498, "beta_low": 0.18994140625, "beta_high": 0.0, "orthogonal_to_active": 0.055475614964962006, "evidence_weights": [-0.17234961688518524, -0.32403460144996643, 0.1727030873298645], "agreement_weight": -0.0050192647613584995, "orthogonal_weight": 0.642665684223175}, {"head": 4, "memory_to_active": 0.6186106204986572, "contribution_to_active": 1.069056510925293, "mix_mean": 0.7529884576797485, "mix_low": 0.0005178451538085938, "mix_high": 0.10478591918945312, "read_cosine": 0.12395637482404709, "beta_mean": 0.13254371285438538, "nonfinite": 0.0, "weighted_memory_to_active": 0.569398045539856, "mix_std": 0.296455979347229, "beta_std": 0.14573632180690765, "beta_low": 0.2952880859375, "beta_high": 0.0, "orthogonal_to_active": 0.1453394889831543, "evidence_weights": [-0.20340529084205627, -0.556280791759491, -0.48032906651496887], "agreement_weight": -0.022896092385053635, "orthogonal_weight": 0.274530291557312}, {"head": 5, "memory_to_active": 0.02755563333630562, "contribution_to_active": 0.6191613674163818, "mix_mean": 0.6782530546188354, "mix_low": 0.0002460479736328125, "mix_high": 0.006402015686035156, "read_cosine": 0.2739725112915039, "beta_mean": 0.00014700730389449745, "nonfinite": 0.0, "weighted_memory_to_active": 0.019642455503344536, "mix_std": 0.2483898401260376, "beta_std": 0.00030148582300171256, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.019281689077615738, "evidence_weights": [-0.23864510655403137, -0.3175073266029358, 0.39076748490333557], "agreement_weight": 0.18984068930149078, "orthogonal_weight": -1.0497719049453735}]}, {"layer": 7, "memory_to_active": 1.7490370273590088, "contribution_to_active": 0.26673537492752075, "mix_mean": 0.16602501273155212, "mix_low": 0.24228572845458984, "mix_high": 0.0009442965383641422, "read_cosine": 0.23931235074996948, "beta_mean": 0.1401742547750473, "nonfinite": 0.0, "weighted_memory_to_active": 0.21142546832561493, "mix_std": 0.14627590775489807, "beta_std": 0.10113541036844254, "beta_low": 0.2049967497587204, "beta_high": 0.0005086263408884406, "orthogonal_to_active": 0.0855150893330574, "output_influence": 0.2765493094921112, "residual_influence": 0.06515413522720337, "state_active": 0.01371606532484293, "state_memory": 0.02444937825202942, "state_peak": 31.00988006591797, "growth_active": 1.1565887928009033, "growth_memory": 1.290146827697754, "heads": [{"head": 0, "memory_to_active": 2.498084545135498, "contribution_to_active": 0.21958361566066742, "mix_mean": 0.037889689207077026, "mix_low": 0.4106006622314453, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3372541069984436, "beta_mean": 0.16457343101501465, "nonfinite": 0.0, "weighted_memory_to_active": 0.26613879203796387, "mix_std": 0.07830220460891724, "beta_std": 0.12054453045129776, "beta_low": 0.00244140625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0012224162928760052, "evidence_weights": [0.042680416256189346, -0.07861623167991638, 0.014382204040884972], "agreement_weight": -0.02517620474100113, "orthogonal_weight": -0.006334096658974886}, {"head": 1, "memory_to_active": 0.971028745174408, "contribution_to_active": 0.3691767156124115, "mix_mean": 0.21754464507102966, "mix_low": 0.12109088897705078, "mix_high": 0.0050830841064453125, "read_cosine": 0.410481333732605, "beta_mean": 0.10078127682209015, "nonfinite": 0.0, "weighted_memory_to_active": 0.31783759593963623, "mix_std": 0.2417375147342682, "beta_std": 0.06474106013774872, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04207741469144821, "evidence_weights": [-0.2173190712928772, -0.10376831889152527, -0.28029054403305054], "agreement_weight": 0.05114256218075752, "orthogonal_weight": 0.16068191826343536}, {"head": 2, "memory_to_active": 0.656846821308136, "contribution_to_active": 0.35831308364868164, "mix_mean": 0.20974917709827423, "mix_low": 0.026355743408203125, "mix_high": 0.00019168853759765625, "read_cosine": 0.10961706936359406, "beta_mean": 0.04453996568918228, "nonfinite": 0.0, "weighted_memory_to_active": 0.23321661353111267, "mix_std": 0.20641736686229706, "beta_std": 0.04466726630926132, "beta_low": 0.1085205078125, "beta_high": 0.0, "orthogonal_to_active": 0.12764908373355865, "evidence_weights": [-0.08453687280416489, -0.16942118108272552, 0.8530739545822144], "agreement_weight": 0.016225168481469154, "orthogonal_weight": -0.5624174475669861}, {"head": 3, "memory_to_active": 4.691302299499512, "contribution_to_active": 0.10663672536611557, "mix_mean": 0.03264719620347023, "mix_low": 0.34619712829589844, "mix_high": 0.0, "read_cosine": 0.09892750531435013, "beta_mean": 0.45220598578453064, "nonfinite": 0.0, "weighted_memory_to_active": 0.35291582345962524, "mix_std": 0.0493323989212513, "beta_std": 0.29269346594810486, "beta_low": 0.002197265625, "beta_high": 0.0029296875, "orthogonal_to_active": 0.30112937092781067, "evidence_weights": [0.07701893895864487, -0.029638759791851044, 0.7162957191467285], "agreement_weight": 0.052746642380952835, "orthogonal_weight": -0.9362624287605286}, {"head": 4, "memory_to_active": 1.6196746826171875, "contribution_to_active": 0.1013445258140564, "mix_mean": 0.022096948698163033, "mix_low": 0.5476398468017578, "mix_high": 0.0, "read_cosine": 0.20636683702468872, "beta_mean": 0.0786699652671814, "nonfinite": 0.0, "weighted_memory_to_active": 0.07055611908435822, "mix_std": 0.04581431299448013, "beta_std": 0.08354698866605759, "beta_low": 0.1171875, "beta_high": 0.0, "orthogonal_to_active": 0.014937622472643852, "evidence_weights": [0.13610006868839264, -0.009660974144935608, 0.18540628254413605], "agreement_weight": 0.04069822281599045, "orthogonal_weight": -0.27023330330848694}, {"head": 5, "memory_to_active": 0.057285375893116, "contribution_to_active": 0.4453575611114502, "mix_mean": 0.4762223958969116, "mix_low": 0.0018301010131835938, "mix_high": 0.00039005279541015625, "read_cosine": 0.2732272148132324, "beta_mean": 0.0002748777042143047, "nonfinite": 0.0, "weighted_memory_to_active": 0.027887854725122452, "mix_std": 0.256051629781723, "beta_std": 0.0006191622815094888, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.026074592024087906, "evidence_weights": [-0.17504571378231049, -0.2313212752342224, 0.5427800416946411], "agreement_weight": 0.21214431524276733, "orthogonal_weight": -1.0091674327850342}]}, {"layer": 8, "memory_to_active": 0.16256414353847504, "contribution_to_active": 0.5580896735191345, "mix_mean": 0.4847525656223297, "mix_low": 0.012824217788875103, "mix_high": 0.0017763774376362562, "read_cosine": 0.4204309582710266, "beta_mean": 0.01447916030883789, "nonfinite": 0.0, "weighted_memory_to_active": 0.08021218329668045, "mix_std": 0.24004197120666504, "beta_std": 0.02037588693201542, "beta_low": 0.768951416015625, "beta_high": 0.0, "orthogonal_to_active": 0.07080784440040588, "output_influence": 0.5117233991622925, "residual_influence": 0.18128877878189087, "state_active": 0.04135069251060486, "state_memory": 0.00563756562769413, "state_peak": 52.31513214111328, "growth_active": 1.0448569059371948, "growth_memory": 1.1427223682403564, "heads": [{"head": 0, "memory_to_active": 0.06770185381174088, "contribution_to_active": 0.5783759951591492, "mix_mean": 0.6115796566009521, "mix_low": 0.0005655288696289062, "mix_high": 0.0015077590942382812, "read_cosine": 0.47180163860321045, "beta_mean": 0.003614474320784211, "nonfinite": 0.0, "weighted_memory_to_active": 0.04856351017951965, "mix_std": 0.24867746233940125, "beta_std": 0.012711961753666401, "beta_low": 0.9197998046875, "beta_high": 0.0, "orthogonal_to_active": 0.029905198141932487, "evidence_weights": [-0.16440223157405853, -0.38570091128349304, 0.6032457947731018], "agreement_weight": 0.25091734528541565, "orthogonal_weight": -0.7906235456466675}, {"head": 1, "memory_to_active": 0.08393502980470657, "contribution_to_active": 0.5486209392547607, "mix_mean": 0.5209767818450928, "mix_low": 3.24249267578125e-05, "mix_high": 0.0009794235229492188, "read_cosine": 0.15084120631217957, "beta_mean": 0.001254972885362804, "nonfinite": 0.0, "weighted_memory_to_active": 0.03858554735779762, "mix_std": 0.2367021143436432, "beta_std": 0.008282074704766273, "beta_low": 0.98663330078125, "beta_high": 0.0, "orthogonal_to_active": 0.027808964252471924, "evidence_weights": [-0.19519765675067902, -0.2759382426738739, 0.6887349486351013], "agreement_weight": 0.07954125851392746, "orthogonal_weight": -0.7554358839988708}, {"head": 2, "memory_to_active": 0.015806106850504875, "contribution_to_active": 0.6817895770072937, "mix_mean": 0.6778217554092407, "mix_low": 0.0005083084106445312, "mix_high": 0.0039958953857421875, "read_cosine": 0.4143570065498352, "beta_mean": 0.00014540019037667662, "nonfinite": 0.0, "weighted_memory_to_active": 0.012597505003213882, "mix_std": 0.2254752218723297, "beta_std": 0.0032927552238106728, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.007578376214951277, "evidence_weights": [-0.22480237483978271, -0.3462904691696167, 1.0035799741744995], "agreement_weight": 0.2887245714664459, "orthogonal_weight": -0.6993082761764526}, {"head": 3, "memory_to_active": 0.10656518489122391, "contribution_to_active": 0.6424022316932678, "mix_mean": 0.5884761810302734, "mix_low": 0.0005779266357421875, "mix_high": 0.004090309143066406, "read_cosine": 0.8130691051483154, "beta_mean": 0.00834374688565731, "nonfinite": 0.0, "weighted_memory_to_active": 0.0815153643488884, "mix_std": 0.25668036937713623, "beta_std": 0.007943768985569477, "beta_low": 0.69757080078125, "beta_high": 0.0, "orthogonal_to_active": 0.02520485781133175, "evidence_weights": [-0.18297173082828522, -0.2995833456516266, 0.5516447424888611], "agreement_weight": 0.23881424963474274, "orthogonal_weight": -0.6186047196388245}, {"head": 4, "memory_to_active": 0.32163766026496887, "contribution_to_active": 0.2746380567550659, "mix_mean": 0.2573777735233307, "mix_low": 0.008180618286132812, "mix_high": 1.9073486328125e-06, "read_cosine": 0.18787898123264313, "beta_mean": 0.0022053951397538185, "nonfinite": 0.0, "weighted_memory_to_active": 0.11057890951633453, "mix_std": 0.21098920702934265, "beta_std": 0.012611325830221176, "beta_low": 0.961669921875, "beta_high": 0.0, "orthogonal_to_active": 0.027467021718621254, "evidence_weights": [-0.0985603928565979, -0.1754971742630005, 0.5106939077377319], "agreement_weight": 0.058851975947618484, "orthogonal_weight": -0.2587307393550873}, {"head": 5, "memory_to_active": 0.379738986492157, "contribution_to_active": 0.6227112412452698, "mix_mean": 0.25228312611579895, "mix_low": 0.06708049774169922, "mix_high": 8.296966552734375e-05, "read_cosine": 0.4846377968788147, "beta_mean": 0.07131096720695496, "nonfinite": 0.0, "weighted_memory_to_active": 0.18943224847316742, "mix_std": 0.2617274224758148, "beta_std": 0.07741343230009079, "beta_low": 0.048095703125, "beta_high": 0.0, "orthogonal_to_active": 0.3068826198577881, "evidence_weights": [-0.1394965797662735, -0.15477986633777618, -0.1174016073346138], "agreement_weight": -0.02036948874592781, "orthogonal_weight": 1.8956798315048218}]}, {"layer": 9, "memory_to_active": 0.9586010575294495, "contribution_to_active": 0.709587574005127, "mix_mean": 0.3743290305137634, "mix_low": 0.07727766036987305, "mix_high": 0.006458600517362356, "read_cosine": 0.34247422218322754, "beta_mean": 0.040967218577861786, "nonfinite": 0.0, "weighted_memory_to_active": 0.40155869722366333, "mix_std": 0.2348276823759079, "beta_std": 0.03438645601272583, "beta_low": 0.5436910390853882, "beta_high": 0.0, "orthogonal_to_active": 0.03832775354385376, "output_influence": 0.5503665804862976, "residual_influence": 0.1597188264131546, "state_active": 0.04200687259435654, "state_memory": 0.016099564731121063, "state_peak": 60.17352294921875, "growth_active": 1.1546380519866943, "growth_memory": 1.3079150915145874, "heads": [{"head": 0, "memory_to_active": 0.00035287777427583933, "contribution_to_active": 0.5707432627677917, "mix_mean": 0.6469374895095825, "mix_low": 0.0013208389282226562, "mix_high": 0.02542877197265625, "read_cosine": 0.33852097392082214, "beta_mean": 1.0920998647634406e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0002241337060695514, "mix_std": 0.28814902901649475, "beta_std": 2.6450645691511454e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00024189395480789244, "evidence_weights": [-0.3035634458065033, -0.31033894419670105, 0.49124571681022644], "agreement_weight": 0.11189115047454834, "orthogonal_weight": -1.1402019262313843}, {"head": 1, "memory_to_active": 0.01965692825615406, "contribution_to_active": 0.7034028768539429, "mix_mean": 0.5816890001296997, "mix_low": 0.0070819854736328125, "mix_high": 0.0030384063720703125, "read_cosine": 0.3997723460197449, "beta_mean": 5.1860955863958225e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.015085979364812374, "mix_std": 0.28026261925697327, "beta_std": 0.00010763997852336615, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01052732951939106, "evidence_weights": [-0.2436964511871338, -0.2986152470111847, 1.0452476739883423], "agreement_weight": 0.4804946482181549, "orthogonal_weight": -0.8475143909454346}, {"head": 2, "memory_to_active": 0.1859649419784546, "contribution_to_active": 0.6575446128845215, "mix_mean": 0.6082934141159058, "mix_low": 0.00140380859375, "mix_high": 0.008618354797363281, "read_cosine": 0.3391138017177582, "beta_mean": 0.0007918708142824471, "nonfinite": 0.0, "weighted_memory_to_active": 0.12699002027511597, "mix_std": 0.2689282298088074, "beta_std": 0.0046796901151537895, "beta_low": 0.99444580078125, "beta_high": 0.0, "orthogonal_to_active": 0.07527440041303635, "evidence_weights": [-0.20551332831382751, -0.2798216640949249, 0.7675238847732544], "agreement_weight": 0.3333405554294586, "orthogonal_weight": -0.6830762028694153}, {"head": 3, "memory_to_active": 1.0099002122879028, "contribution_to_active": 0.2261376827955246, "mix_mean": 0.11728651821613312, "mix_low": 0.1573162078857422, "mix_high": 3.24249267578125e-05, "read_cosine": 0.3685806691646576, "beta_mean": 0.04541386291384697, "nonfinite": 0.0, "weighted_memory_to_active": 0.20806661248207092, "mix_std": 0.17129045724868774, "beta_std": 0.0420445017516613, "beta_low": 0.065185546875, "beta_high": 0.0, "orthogonal_to_active": 0.0649421289563179, "evidence_weights": [0.004617295693606138, -0.0673859566450119, 0.24218514561653137], "agreement_weight": 0.09613915532827377, "orthogonal_weight": -0.35101431608200073}, {"head": 4, "memory_to_active": 3.202848434448242, "contribution_to_active": 1.6692922115325928, "mix_mean": 0.15105408430099487, "mix_low": 0.1226053237915039, "mix_high": 0.0013360977172851562, "read_cosine": 0.12702813744544983, "beta_mean": 0.14724409580230713, "nonfinite": 0.0, "weighted_memory_to_active": 1.5962951183319092, "mix_std": 0.21082046627998352, "beta_std": 0.08630609512329102, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016821302473545074, "evidence_weights": [-0.08298826217651367, -0.15613244473934174, -0.029981952160596848], "agreement_weight": -0.06348850578069687, "orthogonal_weight": -0.010916582308709621}, {"head": 5, "memory_to_active": 1.3328830003738403, "contribution_to_active": 0.4304048717021942, "mix_mean": 0.140713632106781, "mix_low": 0.17393779754638672, "mix_high": 0.00029754638671875, "read_cosine": 0.48182928562164307, "beta_mean": 0.05230053886771202, "nonfinite": 0.0, "weighted_memory_to_active": 0.46269047260284424, "mix_std": 0.18951523303985596, "beta_std": 0.07317814230918884, "beta_low": 0.2025146484375, "beta_high": 0.0, "orthogonal_to_active": 0.062159452587366104, "evidence_weights": [0.08495304733514786, -0.14257580041885376, 0.13899832963943481], "agreement_weight": 0.010311294347047806, "orthogonal_weight": 0.1854574978351593}]}, {"layer": 10, "memory_to_active": 0.20632606744766235, "contribution_to_active": 0.6981815099716187, "mix_mean": 0.5171737670898438, "mix_low": 0.02188190072774887, "mix_high": 0.004710833542048931, "read_cosine": 0.37753060460090637, "beta_mean": 0.047456204891204834, "nonfinite": 0.0, "weighted_memory_to_active": 0.09191535413265228, "mix_std": 0.2676030993461609, "beta_std": 0.04787512868642807, "beta_low": 0.66021728515625, "beta_high": 0.0, "orthogonal_to_active": 0.23856860399246216, "output_influence": 0.45462143421173096, "residual_influence": 0.16045883297920227, "state_active": 0.0547574982047081, "state_memory": 0.015095170587301254, "state_peak": 126.44132232666016, "growth_active": 1.052498459815979, "growth_memory": 1.0389126539230347, "heads": [{"head": 0, "memory_to_active": 0.726671040058136, "contribution_to_active": 1.894353985786438, "mix_mean": 0.5587589740753174, "mix_low": 0.09765625, "mix_high": 0.006194114685058594, "read_cosine": 0.10717549920082092, "beta_mean": 0.21013040840625763, "nonfinite": 0.0, "weighted_memory_to_active": 0.4302482008934021, "mix_std": 0.3892238438129425, "beta_std": 0.22038534283638, "beta_low": 0.154541015625, "beta_high": 0.0, "orthogonal_to_active": 1.3439446687698364, "evidence_weights": [-0.09336747229099274, -0.3836972117424011, -1.8195465803146362], "agreement_weight": 0.02527196705341339, "orthogonal_weight": 3.152912139892578}, {"head": 1, "memory_to_active": 0.008199899457395077, "contribution_to_active": 0.6218180656433105, "mix_mean": 0.6283756494522095, "mix_low": 0.0016012191772460938, "mix_high": 0.015680313110351562, "read_cosine": 0.3092855215072632, "beta_mean": 6.03222033532802e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004672138020396233, "mix_std": 0.26605916023254395, "beta_std": 0.0001049021229846403, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007181938737630844, "evidence_weights": [-0.2509479522705078, -0.19528813660144806, 0.5587493181228638], "agreement_weight": 0.585115373134613, "orthogonal_weight": -1.6661039590835571}, {"head": 2, "memory_to_active": 0.014362755231559277, "contribution_to_active": 0.6196672916412354, "mix_mean": 0.656309962272644, "mix_low": 0.00023937225341796875, "mix_high": 0.0050506591796875, "read_cosine": 0.01642557978630066, "beta_mean": 2.9906641429988667e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006509881000965834, "mix_std": 0.2510393261909485, "beta_std": 0.00012821036216337234, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002816127147525549, "evidence_weights": [-0.26518118381500244, -0.28113794326782227, 0.6108736991882324], "agreement_weight": -0.20292994379997253, "orthogonal_weight": -0.4153318405151367}, {"head": 3, "memory_to_active": 0.3394870162010193, "contribution_to_active": 0.09507788717746735, "mix_mean": 0.27508091926574707, "mix_low": 0.028891563415527344, "mix_high": 0.00017452239990234375, "read_cosine": 0.9206310510635376, "beta_mean": 0.0675087720155716, "nonfinite": 0.0, "weighted_memory_to_active": 0.03853461146354675, "mix_std": 0.227015420794487, "beta_std": 0.058398373425006866, "beta_low": 0.00079345703125, "beta_high": 0.0, "orthogonal_to_active": 0.038331858813762665, "evidence_weights": [-0.10199711471796036, -0.14900705218315125, 0.16120389103889465], "agreement_weight": 0.01766200177371502, "orthogonal_weight": 1.4116101264953613}, {"head": 4, "memory_to_active": 0.024854905903339386, "contribution_to_active": 0.5542653799057007, "mix_mean": 0.5869444012641907, "mix_low": 0.00024700164794921875, "mix_high": 0.0011653900146484375, "read_cosine": 0.059886589646339417, "beta_mean": 6.867892079753801e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014460875652730465, "mix_std": 0.24503055214881897, "beta_std": 0.0001135498023359105, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.019763130694627762, "evidence_weights": [-0.18398867547512054, -0.25821930170059204, 0.9824092388153076], "agreement_weight": 0.17049354314804077, "orthogonal_weight": -1.4466058015823364}, {"head": 5, "memory_to_active": 0.1243806779384613, "contribution_to_active": 0.4039064049720764, "mix_mean": 0.3975725769996643, "mix_low": 0.0026559829711914062, "mix_high": 0.0, "read_cosine": 0.8517792224884033, "beta_mean": 0.006939139682799578, "nonfinite": 0.0, "weighted_memory_to_active": 0.057066433131694794, "mix_std": 0.2272501289844513, "beta_std": 0.008120392449200153, "beta_low": 0.80596923828125, "beta_high": 0.0, "orthogonal_to_active": 0.0193739403039217, "evidence_weights": [-0.14039777219295502, -0.1625269651412964, 0.26771870255470276], "agreement_weight": 0.09245690703392029, "orthogonal_weight": -0.6638579964637756}]}, {"layer": 11, "memory_to_active": 0.08343320339918137, "contribution_to_active": 0.6199876070022583, "mix_mean": 0.5166037082672119, "mix_low": 0.11232630908489227, "mix_high": 0.04307715222239494, "read_cosine": 0.32481706142425537, "beta_mean": 0.027576828375458717, "nonfinite": 0.0, "weighted_memory_to_active": 0.04426008462905884, "mix_std": 0.3249785900115967, "beta_std": 0.039752714335918427, "beta_low": 0.6807556748390198, "beta_high": 0.0, "orthogonal_to_active": 0.04666057974100113, "output_influence": 0.5580407381057739, "residual_influence": 0.1495150923728943, "state_active": 0.22033821046352386, "state_memory": 0.0038826006930321455, "state_peak": 34.82868957519531, "growth_active": 1.0986747741699219, "growth_memory": 1.6025556325912476, "heads": [{"head": 0, "memory_to_active": 0.008149552159011364, "contribution_to_active": 0.6087504625320435, "mix_mean": 0.6493521928787231, "mix_low": 0.0008993148803710938, "mix_high": 0.008836746215820312, "read_cosine": 0.22912797331809998, "beta_mean": 9.491043601883575e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00609236117452383, "mix_std": 0.25841158628463745, "beta_std": 0.003946641925722361, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.004981929901987314, "evidence_weights": [-0.17363357543945312, -0.364639014005661, 0.7761551737785339], "agreement_weight": 0.38259896636009216, "orthogonal_weight": -0.8473876714706421}, {"head": 1, "memory_to_active": 0.14943382143974304, "contribution_to_active": 0.418315589427948, "mix_mean": 0.4576435387134552, "mix_low": 0.006150245666503906, "mix_high": 0.00042057037353515625, "read_cosine": 0.48447757959365845, "beta_mean": 0.0060757482424378395, "nonfinite": 0.0, "weighted_memory_to_active": 0.06985025107860565, "mix_std": 0.26991257071495056, "beta_std": 0.017127184197306633, "beta_low": 0.8785400390625, "beta_high": 0.0, "orthogonal_to_active": 0.038039132952690125, "evidence_weights": [-0.15503086149692535, -0.25854161381721497, 0.5951262712478638], "agreement_weight": 0.07125739753246307, "orthogonal_weight": -0.6426652073860168}, {"head": 2, "memory_to_active": 0.006142497528344393, "contribution_to_active": 0.9864359498023987, "mix_mean": 0.43414729833602905, "mix_low": 0.3127174377441406, "mix_high": 0.16336345672607422, "read_cosine": 0.16352519392967224, "beta_mean": 0.0759420245885849, "nonfinite": 0.0, "weighted_memory_to_active": 0.004339461214840412, "mix_std": 0.44942155480384827, "beta_std": 0.10099124163389206, "beta_low": 0.47503662109375, "beta_high": 0.0, "orthogonal_to_active": 0.040125541388988495, "evidence_weights": [0.0486166775226593, -0.3806718587875366, -1.3190019130706787], "agreement_weight": 0.0018979448359459639, "orthogonal_weight": 9.618053436279297}, {"head": 3, "memory_to_active": 0.007714867126196623, "contribution_to_active": 0.8639735579490662, "mix_mean": 0.40637487173080444, "mix_low": 0.34479236602783203, "mix_high": 0.07879638671875, "read_cosine": 0.47832345962524414, "beta_mean": 0.057930946350097656, "nonfinite": 0.0, "weighted_memory_to_active": 0.004868126008659601, "mix_std": 0.4380780756473541, "beta_std": 0.0855146273970604, "beta_low": 0.50830078125, "beta_high": 0.0, "orthogonal_to_active": 0.034757260233163834, "evidence_weights": [0.2067326158285141, -0.2921143174171448, -0.1533251404762268], "agreement_weight": 0.052216704934835434, "orthogonal_weight": 9.892640113830566}, {"head": 4, "memory_to_active": 0.2395372837781906, "contribution_to_active": 0.4332391917705536, "mix_mean": 0.5933904647827148, "mix_low": 0.00701141357421875, "mix_high": 0.0021085739135742188, "read_cosine": 0.03164621442556381, "beta_mean": 0.017036698758602142, "nonfinite": 0.0, "weighted_memory_to_active": 0.13302822411060333, "mix_std": 0.27319473028182983, "beta_std": 0.01823251135647297, "beta_low": 0.4453125, "beta_high": 0.0, "orthogonal_to_active": 0.1301642209291458, "evidence_weights": [-0.23909611999988556, -0.24069489538669586, 1.8947365283966064], "agreement_weight": -0.21101835370063782, "orthogonal_weight": -0.9942556619644165}, {"head": 5, "memory_to_active": 0.0896211490035057, "contribution_to_active": 0.4092106819152832, "mix_mean": 0.5587137341499329, "mix_low": 0.0023870468139648438, "mix_high": 0.004937171936035156, "read_cosine": 0.5618019104003906, "beta_mean": 0.008380657061934471, "nonfinite": 0.0, "weighted_memory_to_active": 0.04738205671310425, "mix_std": 0.26085302233695984, "beta_std": 0.01270407997071743, "beta_low": 0.77783203125, "beta_high": 0.0, "orthogonal_to_active": 0.031895384192466736, "evidence_weights": [-0.20763973891735077, -0.3142636716365814, 0.6523345708847046], "agreement_weight": 0.0845813974738121, "orthogonal_weight": -0.8521680235862732}]}, {"layer": 12, "memory_to_active": 0.24516458809375763, "contribution_to_active": 0.5849704742431641, "mix_mean": 0.5413652062416077, "mix_low": 0.008735975250601768, "mix_high": 0.006102879997342825, "read_cosine": 0.2224622517824173, "beta_mean": 0.017627041786909103, "nonfinite": 0.0, "weighted_memory_to_active": 0.12848424911499023, "mix_std": 0.261569619178772, "beta_std": 0.01777821034193039, "beta_low": 0.8259176015853882, "beta_high": 0.0, "orthogonal_to_active": 0.07615445554256439, "output_influence": 0.5026992559432983, "residual_influence": 0.14215180277824402, "state_active": 0.03492150455713272, "state_memory": 0.009168198332190514, "state_peak": 64.20974731445312, "growth_active": 1.1556185483932495, "growth_memory": 1.1992783546447754, "heads": [{"head": 0, "memory_to_active": 0.04844164848327637, "contribution_to_active": 0.611850917339325, "mix_mean": 0.5769694447517395, "mix_low": 0.00044155120849609375, "mix_high": 0.005280494689941406, "read_cosine": 0.27530521154403687, "beta_mean": 0.0008477214141748846, "nonfinite": 0.0, "weighted_memory_to_active": 0.026837972924113274, "mix_std": 0.266429603099823, "beta_std": 0.0017684678314253688, "beta_low": 0.99346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.024983184412121773, "evidence_weights": [-0.20958082377910614, -0.276170551776886, -0.0015722415409982204], "agreement_weight": 0.09976744651794434, "orthogonal_weight": -0.994425356388092}, {"head": 1, "memory_to_active": 0.11396666616201401, "contribution_to_active": 0.5494557023048401, "mix_mean": 0.5597783327102661, "mix_low": 0.0004444122314453125, "mix_high": 0.0021619796752929688, "read_cosine": 0.389962762594223, "beta_mean": 0.0008602984598837793, "nonfinite": 0.0, "weighted_memory_to_active": 0.07239019125699997, "mix_std": 0.2671245336532593, "beta_std": 0.012515954673290253, "beta_low": 0.99603271484375, "beta_high": 0.0, "orthogonal_to_active": 0.060812972486019135, "evidence_weights": [-0.20378948748111725, -0.27285024523735046, 1.0490784645080566], "agreement_weight": 0.24700571596622467, "orthogonal_weight": -0.9197537302970886}, {"head": 2, "memory_to_active": 0.6236362457275391, "contribution_to_active": 0.5017789602279663, "mix_mean": 0.3180636167526245, "mix_low": 0.027433395385742188, "mix_high": 0.00041103363037109375, "read_cosine": 0.5233240723609924, "beta_mean": 0.10230008512735367, "nonfinite": 0.0, "weighted_memory_to_active": 0.25012993812561035, "mix_std": 0.24118131399154663, "beta_std": 0.06989137828350067, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.2662471532821655, "evidence_weights": [-0.21342244744300842, -0.059012189507484436, -0.0053733582608401775], "agreement_weight": 0.028108881786465645, "orthogonal_weight": 1.3688710927963257}, {"head": 3, "memory_to_active": 0.04521831497550011, "contribution_to_active": 0.7510098218917847, "mix_mean": 0.7151989936828613, "mix_low": 0.00016689300537109375, "mix_high": 0.01735210418701172, "read_cosine": 0.1175750121474266, "beta_mean": 0.0003088169323746115, "nonfinite": 0.0, "weighted_memory_to_active": 0.02961331605911255, "mix_std": 0.22927400469779968, "beta_std": 0.006672759540379047, "beta_low": 0.9976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.028062691912055016, "evidence_weights": [-0.2899262309074402, -0.3362419903278351, 1.6852171421051025], "agreement_weight": 0.5324628353118896, "orthogonal_weight": -0.9740744829177856}, {"head": 4, "memory_to_active": 0.5495622754096985, "contribution_to_active": 0.5949959754943848, "mix_mean": 0.44123151898384094, "mix_low": 0.017286300659179688, "mix_high": 0.004853248596191406, "read_cosine": -0.15046070516109467, "beta_mean": 0.0009585799998603761, "nonfinite": 0.0, "weighted_memory_to_active": 0.33396637439727783, "mix_std": 0.29092469811439514, "beta_std": 0.009923698380589485, "beta_low": 0.97833251953125, "beta_high": 0.0, "orthogonal_to_active": 0.03251826763153076, "evidence_weights": [-0.15301446616649628, -0.21640633046627045, -0.10979419946670532], "agreement_weight": 0.013542192988097668, "orthogonal_weight": -0.10152197629213333}, {"head": 5, "memory_to_active": 0.09016222506761551, "contribution_to_active": 0.5007312893867493, "mix_mean": 0.6369493007659912, "mix_low": 0.0066432952880859375, "mix_high": 0.006558418273925781, "read_cosine": 0.17906710505485535, "beta_mean": 0.0004867335665039718, "nonfinite": 0.0, "weighted_memory_to_active": 0.057967618107795715, "mix_std": 0.27448347210884094, "beta_std": 0.005897007416933775, "beta_low": 0.98980712890625, "beta_high": 0.0, "orthogonal_to_active": 0.04430246353149414, "evidence_weights": [-0.23025770485401154, -0.3183567225933075, 1.2627676725387573], "agreement_weight": 0.40791672468185425, "orthogonal_weight": -0.8170608282089233}]}, {"layer": 13, "memory_to_active": 0.12152239680290222, "contribution_to_active": 0.5378195643424988, "mix_mean": 0.5391387939453125, "mix_low": 0.005765438079833984, "mix_high": 0.00889730453491211, "read_cosine": 0.40622392296791077, "beta_mean": 0.01531308051198721, "nonfinite": 0.0, "weighted_memory_to_active": 0.05346888303756714, "mix_std": 0.26015663146972656, "beta_std": 0.01832064986228943, "beta_low": 0.8085734248161316, "beta_high": 0.0, "orthogonal_to_active": 0.05529782548546791, "output_influence": 0.5223304033279419, "residual_influence": 0.18140637874603271, "state_active": 0.12794938683509827, "state_memory": 0.044534649699926376, "state_peak": 101.63789367675781, "growth_active": 0.912311315536499, "growth_memory": 1.0300747156143188, "heads": [{"head": 0, "memory_to_active": 0.01377315167337656, "contribution_to_active": 0.5921439528465271, "mix_mean": 0.6120921969413757, "mix_low": 0.0007543563842773438, "mix_high": 0.004544258117675781, "read_cosine": 0.46194082498550415, "beta_mean": 5.091990533401258e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00879790261387825, "mix_std": 0.26963913440704346, "beta_std": 0.00010321573790861294, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00855328794568777, "evidence_weights": [-0.19533991813659668, -0.26724231243133545, 0.7654354572296143], "agreement_weight": 0.17460951209068298, "orthogonal_weight": -1.0993967056274414}, {"head": 1, "memory_to_active": 0.01691988669335842, "contribution_to_active": 0.6018658876419067, "mix_mean": 0.5719086527824402, "mix_low": 0.0018320083618164062, "mix_high": 0.004845619201660156, "read_cosine": 0.2224501669406891, "beta_mean": 7.559833466075361e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009425354190170765, "mix_std": 0.2785775065422058, "beta_std": 0.00014187063788995147, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012171687558293343, "evidence_weights": [-0.1818186640739441, -0.26382625102996826, 0.5717982649803162], "agreement_weight": 0.4870462715625763, "orthogonal_weight": -1.345366358757019}, {"head": 2, "memory_to_active": 0.06893934309482574, "contribution_to_active": 0.48971349000930786, "mix_mean": 0.386401504278183, "mix_low": 0.0011396408081054688, "mix_high": 0.0001888275146484375, "read_cosine": 0.5939217805862427, "beta_mean": 0.012535803020000458, "nonfinite": 0.0, "weighted_memory_to_active": 0.032663263380527496, "mix_std": 0.23139406740665436, "beta_std": 0.047153323888778687, "beta_low": 0.85858154296875, "beta_high": 0.0, "orthogonal_to_active": 0.019191522151231766, "evidence_weights": [-0.18366827070713043, -0.20820565521717072, 0.4054553210735321], "agreement_weight": 0.007769413758069277, "orthogonal_weight": -0.8416690826416016}, {"head": 3, "memory_to_active": 0.1814553439617157, "contribution_to_active": 0.5782818794250488, "mix_mean": 0.6566095948219299, "mix_low": 0.024888992309570312, "mix_high": 0.03155040740966797, "read_cosine": -0.04962730035185814, "beta_mean": 0.000967198284342885, "nonfinite": 0.0, "weighted_memory_to_active": 0.1326592117547989, "mix_std": 0.29042431712150574, "beta_std": 0.008193224668502808, "beta_low": 0.99322509765625, "beta_high": 0.0, "orthogonal_to_active": 0.12052484601736069, "evidence_weights": [-0.31721070408821106, -0.2461383044719696, 1.6115162372589111], "agreement_weight": 6.15099573135376, "orthogonal_weight": -0.9808943271636963}, {"head": 4, "memory_to_active": 0.3905304968357086, "contribution_to_active": 0.24711117148399353, "mix_mean": 0.3463004231452942, "mix_low": 0.0039920806884765625, "mix_high": 2.002716064453125e-05, "read_cosine": 0.8898181319236755, "beta_mean": 0.07798260450363159, "nonfinite": 0.0, "weighted_memory_to_active": 0.0975801944732666, "mix_std": 0.22006084024906158, "beta_std": 0.05299363285303116, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.13819248974323273, "evidence_weights": [-0.07989280670881271, -0.2299189567565918, 0.4553252160549164], "agreement_weight": 0.00964322965592146, "orthogonal_weight": 2.245910882949829}, {"head": 5, "memory_to_active": 0.057516083121299744, "contribution_to_active": 0.7178010940551758, "mix_mean": 0.6615203619003296, "mix_low": 0.0019855499267578125, "mix_high": 0.012234687805175781, "read_cosine": 0.3188399076461792, "beta_mean": 0.0002663646009750664, "nonfinite": 0.0, "weighted_memory_to_active": 0.03968734294176102, "mix_std": 0.2708439230918884, "beta_std": 0.0013386242790147662, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.03315310552716255, "evidence_weights": [-0.22140301764011383, -0.29810047149658203, 0.8735749125480652], "agreement_weight": 0.19192232191562653, "orthogonal_weight": -0.9235799312591553}]}, {"layer": 14, "memory_to_active": 0.2367410957813263, "contribution_to_active": 0.41445815563201904, "mix_mean": 0.4262462854385376, "mix_low": 0.026343824341893196, "mix_high": 0.0062268576584756374, "read_cosine": 0.6100673079490662, "beta_mean": 0.06407807022333145, "nonfinite": 0.0, "weighted_memory_to_active": 0.09709871560335159, "mix_std": 0.2425416111946106, "beta_std": 0.060947563499212265, "beta_low": 0.4881388545036316, "beta_high": 0.0, "orthogonal_to_active": 0.04339645057916641, "output_influence": 0.3503079116344452, "residual_influence": 0.1426788717508316, "state_active": 0.05459749326109886, "state_memory": 0.015689712017774582, "state_peak": 40.43144226074219, "growth_active": 1.0914984941482544, "growth_memory": 1.2888433933258057, "heads": [{"head": 0, "memory_to_active": 0.37749016284942627, "contribution_to_active": 0.18201225996017456, "mix_mean": 0.1828586459159851, "mix_low": 0.01653289794921875, "mix_high": 0.0, "read_cosine": 0.817854642868042, "beta_mean": 0.07216903567314148, "nonfinite": 0.0, "weighted_memory_to_active": 0.07732434570789337, "mix_std": 0.1734694391489029, "beta_std": 0.06128527596592903, "beta_low": 0.004150390625, "beta_high": 0.0, "orthogonal_to_active": 0.08181476593017578, "evidence_weights": [-0.10407289117574692, -0.04622925445437431, 0.40969565510749817], "agreement_weight": 0.022153915837407112, "orthogonal_weight": 1.7589608430862427}, {"head": 1, "memory_to_active": 0.4817749857902527, "contribution_to_active": 0.2706482708454132, "mix_mean": 0.20863500237464905, "mix_low": 0.12597942352294922, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8747996091842651, "beta_mean": 0.26410937309265137, "nonfinite": 0.0, "weighted_memory_to_active": 0.20374609529972076, "mix_std": 0.2382347583770752, "beta_std": 0.25706565380096436, "beta_low": 0.09552001953125, "beta_high": 0.0, "orthogonal_to_active": 0.014441126957535744, "evidence_weights": [-0.0025195369962602854, -0.19668518006801605, -0.08535470068454742], "agreement_weight": 0.05907285585999489, "orthogonal_weight": 0.2029758244752884}, {"head": 2, "memory_to_active": 0.14146162569522858, "contribution_to_active": 0.5801955461502075, "mix_mean": 0.593485414981842, "mix_low": 0.0033550262451171875, "mix_high": 0.004685401916503906, "read_cosine": 0.39803045988082886, "beta_mean": 0.002494530286639929, "nonfinite": 0.0, "weighted_memory_to_active": 0.09249531477689743, "mix_std": 0.2686437964439392, "beta_std": 0.00345324887894094, "beta_low": 0.96405029296875, "beta_high": 0.0, "orthogonal_to_active": 0.07111985236406326, "evidence_weights": [-0.13668543100357056, -0.3167923390865326, 0.9270619750022888], "agreement_weight": 0.08542410284280777, "orthogonal_weight": -0.8742154836654663}, {"head": 3, "memory_to_active": 0.05670139193534851, "contribution_to_active": 0.6107643246650696, "mix_mean": 0.6711279153823853, "mix_low": 0.004769325256347656, "mix_high": 0.021867752075195312, "read_cosine": -0.06487791240215302, "beta_mean": 0.007167953997850418, "nonfinite": 0.0, "weighted_memory_to_active": 0.036328524351119995, "mix_std": 0.25901466608047485, "beta_std": 0.01034854631870985, "beta_low": 0.79132080078125, "beta_high": 0.0, "orthogonal_to_active": 0.02779281884431839, "evidence_weights": [-0.3447214961051941, -0.31080320477485657, 0.28915607929229736], "agreement_weight": 0.07117073982954025, "orthogonal_weight": -0.8419342637062073}, {"head": 4, "memory_to_active": 0.15481263399124146, "contribution_to_active": 0.5056294798851013, "mix_mean": 0.520944356918335, "mix_low": 0.0035600662231445312, "mix_high": 0.009265899658203125, "read_cosine": 0.8848260045051575, "beta_mean": 0.03726140409708023, "nonfinite": 0.0, "weighted_memory_to_active": 0.09197840094566345, "mix_std": 0.26126933097839355, "beta_std": 0.027972828596830368, "beta_low": 0.09552001953125, "beta_high": 0.0, "orthogonal_to_active": 0.02532244101166725, "evidence_weights": [-0.21567077934741974, -0.242435023188591, 0.4330775737762451], "agreement_weight": 0.13516180217266083, "orthogonal_weight": -0.6119647026062012}, {"head": 5, "memory_to_active": 0.2082056999206543, "contribution_to_active": 0.33749908208847046, "mix_mean": 0.380426287651062, "mix_low": 0.0038661956787109375, "mix_high": 0.0015392303466796875, "read_cosine": 0.7497709393501282, "beta_mean": 0.0012661509681493044, "nonfinite": 0.0, "weighted_memory_to_active": 0.08071960508823395, "mix_std": 0.25461772084236145, "beta_std": 0.005559774115681648, "beta_low": 0.978271484375, "beta_high": 0.0, "orthogonal_to_active": 0.039887696504592896, "evidence_weights": [-0.1749170571565628, -0.17702680826187134, 0.33582979440689087], "agreement_weight": 0.0713801458477974, "orthogonal_weight": -0.7537909746170044}]}, {"layer": 15, "memory_to_active": 0.05576463043689728, "contribution_to_active": 0.49161314964294434, "mix_mean": 0.42345064878463745, "mix_low": 0.008580207824707031, "mix_high": 0.0031139058992266655, "read_cosine": 0.6629379987716675, "beta_mean": 0.035936541855335236, "nonfinite": 0.0, "weighted_memory_to_active": 0.02732076123356819, "mix_std": 0.2859562635421753, "beta_std": 0.042951442301273346, "beta_low": 0.71929931640625, "beta_high": 0.0, "orthogonal_to_active": 0.008741381578147411, "output_influence": 0.3128468990325928, "residual_influence": 0.17910069227218628, "state_active": 0.1534978449344635, "state_memory": 0.011123431846499443, "state_peak": 30.456581115722656, "growth_active": 1.169453501701355, "growth_memory": 1.3991001844406128, "heads": [{"head": 0, "memory_to_active": 0.11262907832860947, "contribution_to_active": 0.4821789860725403, "mix_mean": 0.3830605745315552, "mix_low": 0.009496688842773438, "mix_high": 0.0056018829345703125, "read_cosine": 0.5712541341781616, "beta_mean": 0.042276978492736816, "nonfinite": 0.0, "weighted_memory_to_active": 0.051553092896938324, "mix_std": 0.28367188572883606, "beta_std": 0.0337972417473793, "beta_low": 0.1337890625, "beta_high": 0.0, "orthogonal_to_active": 0.030240509659051895, "evidence_weights": [-0.18798215687274933, -0.18919068574905396, 0.46475422382354736], "agreement_weight": 0.09171447157859802, "orthogonal_weight": -0.7918920516967773}, {"head": 1, "memory_to_active": 0.10986834019422531, "contribution_to_active": 0.43159812688827515, "mix_mean": 0.44107329845428467, "mix_low": 0.0014791488647460938, "mix_high": 0.0074634552001953125, "read_cosine": 0.9314255714416504, "beta_mean": 0.0005248527741059661, "nonfinite": 0.0, "weighted_memory_to_active": 0.04829665273427963, "mix_std": 0.288935124874115, "beta_std": 0.002010867465287447, "beta_low": 0.9964599609375, "beta_high": 0.0, "orthogonal_to_active": 0.0018688563723117113, "evidence_weights": [-0.1490989774465561, -0.15694750845432281, 0.17315241694450378], "agreement_weight": 0.1259724348783493, "orthogonal_weight": 0.08618233352899551}, {"head": 2, "memory_to_active": 0.007433487102389336, "contribution_to_active": 0.4958222508430481, "mix_mean": 0.3462352454662323, "mix_low": 0.013561248779296875, "mix_high": 9.059906005859375e-05, "read_cosine": 0.7825179100036621, "beta_mean": 0.00010211630433332175, "nonfinite": 0.0, "weighted_memory_to_active": 0.003939366899430752, "mix_std": 0.2712135314941406, "beta_std": 0.00019895508012268692, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011235728161409497, "evidence_weights": [-0.20723655819892883, -0.26302608847618103, 0.09144215285778046], "agreement_weight": 0.009912087582051754, "orthogonal_weight": 0.8251290321350098}, {"head": 3, "memory_to_active": 0.0015486598713323474, "contribution_to_active": 0.3731842041015625, "mix_mean": 0.404631108045578, "mix_low": 0.0061473846435546875, "mix_high": 0.00040531158447265625, "read_cosine": 0.6131039261817932, "beta_mean": 8.69024825078668e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005104886367917061, "mix_std": 0.2905662953853607, "beta_std": 1.9284456357127056e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00034370500361546874, "evidence_weights": [-0.26954880356788635, -0.2404806762933731, 0.24438084661960602], "agreement_weight": 0.04671230912208557, "orthogonal_weight": -0.6813446283340454}, {"head": 4, "memory_to_active": 0.07397894561290741, "contribution_to_active": 0.641110360622406, "mix_mean": 0.3856508433818817, "mix_low": 0.011006355285644531, "mix_high": 0.0017652511596679688, "read_cosine": 0.7492551207542419, "beta_mean": 0.17040708661079407, "nonfinite": 0.0, "weighted_memory_to_active": 0.04061311483383179, "mix_std": 0.309059739112854, "beta_std": 0.2039939910173416, "beta_low": 0.213134765625, "beta_high": 0.0, "orthogonal_to_active": 0.006664076820015907, "evidence_weights": [-0.20764170587062836, -0.2537880837917328, -0.002885812660679221], "agreement_weight": 0.013424542732536793, "orthogonal_weight": 0.3688659071922302}, {"head": 5, "memory_to_active": 0.029129255563020706, "contribution_to_active": 0.525784969329834, "mix_mean": 0.5800528526306152, "mix_low": 0.009790420532226562, "mix_high": 0.00335693359375, "read_cosine": 0.33007103204727173, "beta_mean": 0.0022995166946202517, "nonfinite": 0.0, "weighted_memory_to_active": 0.019011855125427246, "mix_std": 0.2722910940647125, "beta_std": 0.01768827997148037, "beta_low": 0.972412109375, "beta_high": 0.0, "orthogonal_to_active": 0.012207567691802979, "evidence_weights": [-0.2829138934612274, -0.3020275831222534, 0.6595847010612488], "agreement_weight": 0.3076499402523041, "orthogonal_weight": 0.8334355354309082}]}]}, {"step": 61000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 36.120996217000084, "memory_to_active": 0.7994270324707031, "contribution_to_active": 0.5591753721237183, "mix_mean": 0.4194980263710022, "mix_low": 0.13012823462486267, "mix_high": 0.010482599958777428, "read_cosine": 0.3440837860107422, "beta_mean": 0.07077822089195251, "nonfinite": 0.0, "weighted_memory_to_active": 0.20143279433250427, "mix_std": 0.22327300906181335, "beta_std": 0.06196335703134537, "beta_low": 0.5727157592773438, "beta_high": 2.606709676911123e-05, "orthogonal_to_active": 0.060307953506708145, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0711514949798584, "normal_token_accuracy": 0.44160178303718567, "state_active": 0.06245117634534836, "state_memory": 0.017374129965901375, "state_peak": 125.10999298095703, "growth_active": 1.1328539848327637, "growth_memory": 1.3154046535491943, "output_influence": 0.43276384472846985, "residual_influence": 0.15002553164958954, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2012293338775635, "no_memory_loss_delta": 0.6484248638153076, "scrambled_loss_delta": 0.1844082474708557, "memory_only_loss_delta": 3.4775567054748535, "no_evidence_loss_delta": 0.37529242038726807, "no_agreement_loss_delta": 0.019584836438298225, "no_orthogonal_loss_delta": 0.029252640902996063}, {"index": 94, "tokens": 256, "normal_loss": 3.170560598373413, "no_memory_loss_delta": 0.6333174705505371, "scrambled_loss_delta": 0.355471134185791, "memory_only_loss_delta": 3.694437265396118, "no_evidence_loss_delta": 0.28719189763069153, "no_agreement_loss_delta": 0.04186689853668213, "no_orthogonal_loss_delta": 0.06472606956958771}, {"index": 188, "tokens": 256, "normal_loss": 3.397753953933716, "no_memory_loss_delta": 0.8473770022392273, "scrambled_loss_delta": 0.1318197250366211, "memory_only_loss_delta": 2.923638343811035, "no_evidence_loss_delta": 0.39460456371307373, "no_agreement_loss_delta": 0.0503542535007, "no_orthogonal_loss_delta": 0.047688212245702744}, {"index": 283, "tokens": 131, "normal_loss": 1.9844411611557007, "no_memory_loss_delta": 0.3612048327922821, "scrambled_loss_delta": 0.13890261948108673, "memory_only_loss_delta": 2.9061968326568604, "no_evidence_loss_delta": 0.16219882667064667, "no_agreement_loss_delta": 0.0050259362906217575, "no_orthogonal_loss_delta": -0.012145481072366238}], "agreement_weight_abs_mean": 0.2298409342765808, "agreement_weight_abs_max": 6.208828449249268, "agreement_weight_mean": 0.19994838535785675, "orthogonal_weight_abs_mean": 0.898093044757843, "orthogonal_weight_abs_max": 10.009846687316895, "orthogonal_weight_mean": -0.0938488021492958, "no_memory_loss": 3.730074882507324, "no_memory_loss_delta": 0.6589236855506897, "no_memory_token_accuracy_gain": 0.0945494994521141, "no_memory_token_win_fraction": 0.6718575954437256, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 48.25920714674701, "scrambled_loss": 3.282665491104126, "scrambled_loss_delta": 0.21151414513587952, "scrambled_token_accuracy_gain": 0.027808675542473793, "scrambled_token_win_fraction": 0.621802031993866, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.06421675047605, "memory_only_loss": 6.369475841522217, "memory_only_loss_delta": 3.2983241081237793, "memory_only_token_accuracy_gain": 0.2858732044696808, "memory_only_token_win_fraction": 0.9187986850738525, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.30549685739139, "no_evidence_loss": 3.395803928375244, "no_evidence_loss_delta": 0.3246527314186096, "no_evidence_token_accuracy_gain": 0.04338153451681137, "no_evidence_token_win_fraction": 0.6151279211044312, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 27.722169173700717, "no_agreement_loss": 3.103721857070923, "no_agreement_loss_delta": 0.032570336014032364, "no_agreement_token_accuracy_gain": 0.0, "no_agreement_token_win_fraction": 0.5127919912338257, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.2045634614886582, "no_orthogonal_loss": 3.10972261428833, "no_orthogonal_loss_delta": 0.03857138380408287, "no_orthogonal_token_accuracy_gain": 0.006674082484096289, "no_orthogonal_token_win_fraction": 0.5984427332878113, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 3.7836980568137495, "short_no_memory_loss_delta": 0.5971162915229797, "long_context_memory_gain": 0.06180739402770996, "normal_context_loss_gain": 0.13499408960342407, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.510608196258545, "contribution_to_active": 1.157528042793274, "mix_mean": 0.10558785498142242, "mix_low": 0.5259675979614258, "mix_high": 0.005372524261474609, "read_cosine": 0.4155639410018921, "beta_mean": 0.2108924239873886, "nonfinite": 0.0, "weighted_memory_to_active": 1.1891329288482666, "mix_std": 0.08952146768569946, "beta_std": 0.14270427823066711, "beta_low": 0.1092427670955658, "beta_high": 0.0, "orthogonal_to_active": 0.04755071550607681, "output_influence": 0.21955382823944092, "residual_influence": 0.24884706735610962, "state_active": 0.06866177916526794, "state_memory": 0.04446273669600487, "state_peak": 47.90980911254883, "growth_active": 1.3158628940582275, "growth_memory": 1.2795262336730957, "heads": [{"head": 0, "memory_to_active": 2.787576913833618, "contribution_to_active": 0.4910666346549988, "mix_mean": 0.01848532259464264, "mix_low": 0.6382637023925781, "mix_high": 9.5367431640625e-07, "read_cosine": 0.22961682081222534, "beta_mean": 0.21121104061603546, "nonfinite": 0.0, "weighted_memory_to_active": 0.6043992042541504, "mix_std": 0.056681931018829346, "beta_std": 0.1041567474603653, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06646127998828888, "evidence_weights": [-0.0011901326943188906, -0.042581189423799515, -0.05484884977340698], "agreement_weight": -0.21606974303722382, "orthogonal_weight": -0.12342361360788345}, {"head": 1, "memory_to_active": 6.709134101867676, "contribution_to_active": 3.0033843517303467, "mix_mean": 0.021052883937954903, "mix_low": 0.6029796600341797, "mix_high": 0.0, "read_cosine": 0.436174601316452, "beta_mean": 0.43787580728530884, "nonfinite": 0.0, "weighted_memory_to_active": 3.267507314682007, "mix_std": 0.0582188181579113, "beta_std": 0.21072521805763245, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.03527211397886276, "evidence_weights": [0.01584339141845703, -0.11153612285852432, 0.6532799005508423], "agreement_weight": 0.029527906328439713, "orthogonal_weight": 0.022762252017855644}, {"head": 2, "memory_to_active": 2.533749580383301, "contribution_to_active": 0.5563755035400391, "mix_mean": 0.013181177899241447, "mix_low": 0.6931495666503906, "mix_high": 0.0, "read_cosine": 0.26475852727890015, "beta_mean": 0.25568175315856934, "nonfinite": 0.0, "weighted_memory_to_active": 0.6569969654083252, "mix_std": 0.032295290380716324, "beta_std": 0.2097994089126587, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05411803349852562, "evidence_weights": [0.07142534106969833, 0.06120475381612778, 0.2038186490535736], "agreement_weight": 0.014293207786977291, "orthogonal_weight": -0.2129492461681366}, {"head": 3, "memory_to_active": 7.260322570800781, "contribution_to_active": 2.3628933429718018, "mix_mean": 0.029978938400745392, "mix_low": 0.4594392776489258, "mix_high": 0.0, "read_cosine": 0.32499486207962036, "beta_mean": 0.29467928409576416, "nonfinite": 0.0, "weighted_memory_to_active": 2.5099563598632812, "mix_std": 0.06682349741458893, "beta_std": 0.23938535153865814, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11230595409870148, "evidence_weights": [-0.03939678519964218, -0.037599626928567886, -0.12797386944293976], "agreement_weight": -0.008942286483943462, "orthogonal_weight": -0.053081147372722626}, {"head": 4, "memory_to_active": 1.6746668815612793, "contribution_to_active": 0.05765178054571152, "mix_mean": 0.011900216341018677, "mix_low": 0.7478752136230469, "mix_high": 0.0, "read_cosine": 0.30223655700683594, "beta_mean": 0.04596979171037674, "nonfinite": 0.0, "weighted_memory_to_active": 0.043949320912361145, "mix_std": 0.036251332610845566, "beta_std": 0.07329216599464417, "beta_low": 0.28802490234375, "beta_high": 0.0, "orthogonal_to_active": 0.0013336916454136372, "evidence_weights": [0.13093316555023193, 0.09610594063997269, -0.13882505893707275], "agreement_weight": -0.17415928840637207, "orthogonal_weight": 0.028030820190906525}, {"head": 5, "memory_to_active": 0.0981963574886322, "contribution_to_active": 0.47379669547080994, "mix_mean": 0.5389285683631897, "mix_low": 0.014098167419433594, "mix_high": 0.03223419189453125, "read_cosine": 0.9356021881103516, "beta_mean": 0.019936878234148026, "nonfinite": 0.0, "weighted_memory_to_active": 0.05198787525296211, "mix_std": 0.2868579030036926, "beta_std": 0.018866758793592453, "beta_low": 0.367431640625, "beta_high": 0.0, "orthogonal_to_active": 0.015813205391168594, "evidence_weights": [-0.1738206297159195, -0.21802878379821777, 0.3869698643684387], "agreement_weight": 0.29212939739227295, "orthogonal_weight": -0.6813446283340454}]}, {"layer": 1, "memory_to_active": 2.3876523971557617, "contribution_to_active": 0.4781930446624756, "mix_mean": 0.2123354822397232, "mix_low": 0.36172056198120117, "mix_high": 0.0011766751995310187, "read_cosine": 0.3025655448436737, "beta_mean": 0.17724111676216125, "nonfinite": 0.0, "weighted_memory_to_active": 0.3154946565628052, "mix_std": 0.11535380780696869, "beta_std": 0.12827304005622864, "beta_low": 0.3524882197380066, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.04087259992957115, "output_influence": 0.24897897243499756, "residual_influence": 0.0670178085565567, "state_active": 0.028424430638551712, "state_memory": 0.034907374531030655, "state_peak": 41.283939361572266, "growth_active": 1.1557875871658325, "growth_memory": 1.4473798274993896, "heads": [{"head": 0, "memory_to_active": 0.03131581097841263, "contribution_to_active": 0.4649980366230011, "mix_mean": 0.518311619758606, "mix_low": 0.0052394866943359375, "mix_high": 0.0024824142456054688, "read_cosine": -0.007579687982797623, "beta_mean": 0.0002194548724219203, "nonfinite": 0.0, "weighted_memory_to_active": 0.01813000999391079, "mix_std": 0.2832731306552887, "beta_std": 0.00030922889709472656, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01799258217215538, "evidence_weights": [-0.2040044069290161, -0.2678949534893036, 0.2791731655597687], "agreement_weight": 0.18890056014060974, "orthogonal_weight": -1.044083595275879}, {"head": 1, "memory_to_active": 2.96417236328125, "contribution_to_active": 0.2758665680885315, "mix_mean": 0.02370673045516014, "mix_low": 0.45607948303222656, "mix_high": 2.86102294921875e-06, "read_cosine": 0.5205270648002625, "beta_mean": 0.2762991189956665, "nonfinite": 0.0, "weighted_memory_to_active": 0.3112819492816925, "mix_std": 0.04643061384558678, "beta_std": 0.18872426450252533, "beta_low": 0.0013427734375, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.02211954817175865, "evidence_weights": [0.018536120653152466, -0.06561628729104996, 0.062063317745923996], "agreement_weight": -0.000995382433757186, "orthogonal_weight": 0.1945563405752182}, {"head": 2, "memory_to_active": 0.04466065764427185, "contribution_to_active": 0.5503780245780945, "mix_mean": 0.6688291430473328, "mix_low": 0.00035858154296875, "mix_high": 0.004573822021484375, "read_cosine": 0.5152361989021301, "beta_mean": 0.0015368969179689884, "nonfinite": 0.0, "weighted_memory_to_active": 0.029584376141428947, "mix_std": 0.23022928833961487, "beta_std": 0.0033762024249881506, "beta_low": 0.9853515625, "beta_high": 0.0, "orthogonal_to_active": 0.01912629045546055, "evidence_weights": [-0.27890586853027344, -0.37222614884376526, 0.6717849969863892], "agreement_weight": 0.043302129954099655, "orthogonal_weight": -0.8104080557823181}, {"head": 3, "memory_to_active": 2.9439244270324707, "contribution_to_active": 0.1721528321504593, "mix_mean": 0.023715797811746597, "mix_low": 0.5335264205932617, "mix_high": 0.0, "read_cosine": 0.17878037691116333, "beta_mean": 0.3208698630332947, "nonfinite": 0.0, "weighted_memory_to_active": 0.21416129171848297, "mix_std": 0.058790743350982666, "beta_std": 0.2571694552898407, "beta_low": 0.059326171875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08658201992511749, "evidence_weights": [0.039653729647397995, -0.017643870785832405, 0.001412814250215888], "agreement_weight": -0.03181840479373932, "orthogonal_weight": -0.4896763563156128}, {"head": 4, "memory_to_active": 3.9484031200408936, "contribution_to_active": 1.3187737464904785, "mix_mean": 0.033047597855329514, "mix_low": 0.3271322250366211, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3354238271713257, "beta_mean": 0.1565575897693634, "nonfinite": 0.0, "weighted_memory_to_active": 1.261726975440979, "mix_std": 0.05581409111618996, "beta_std": 0.1046653613448143, "beta_low": 0.03564453125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.0777716413140297, "evidence_weights": [-0.025305218994617462, -0.0019507515244185925, 0.1332772821187973], "agreement_weight": -0.015569076873362064, "orthogonal_weight": 0.08533542603254318}, {"head": 5, "memory_to_active": 4.393438339233398, "contribution_to_active": 0.08698886632919312, "mix_mean": 0.0064019253477454185, "mix_low": 0.847987174987793, "mix_high": 0.0, "read_cosine": 0.27300551533699036, "beta_mean": 0.3079637587070465, "nonfinite": 0.0, "weighted_memory_to_active": 0.0580831803381443, "mix_std": 0.017584970220923424, "beta_std": 0.21539366245269775, "beta_low": 0.03326416015625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.02164352685213089, "evidence_weights": [0.16094395518302917, 0.08260258287191391, 0.11133050918579102], "agreement_weight": -0.12464490532875061, "orthogonal_weight": 0.3724910020828247}]}, {"layer": 2, "memory_to_active": 0.9671780467033386, "contribution_to_active": 0.34759968519210815, "mix_mean": 0.4060485363006592, "mix_low": 0.25286421179771423, "mix_high": 0.004558563232421875, "read_cosine": 0.2722463011741638, "beta_mean": 0.08564725518226624, "nonfinite": 0.0, "weighted_memory_to_active": 0.0663304328918457, "mix_std": 0.18828514218330383, "beta_std": 0.06879759579896927, "beta_low": 0.5852763652801514, "beta_high": 7.120767986634746e-05, "orthogonal_to_active": 0.017939530313014984, "output_influence": 0.3919100761413574, "residual_influence": 0.10806051641702652, "state_active": 0.038004860281944275, "state_memory": 0.014733933843672276, "state_peak": 29.17148780822754, "growth_active": 1.0238264799118042, "growth_memory": 1.5733624696731567, "heads": [{"head": 0, "memory_to_active": 0.01212406624108553, "contribution_to_active": 0.3541245460510254, "mix_mean": 0.6270487904548645, "mix_low": 0.0007047653198242188, "mix_high": 0.008736610412597656, "read_cosine": 0.1344432681798935, "beta_mean": 8.763941150391474e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00792896468192339, "mix_std": 0.27012789249420166, "beta_std": 0.00015281062223948538, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008406882174313068, "evidence_weights": [-0.3705313205718994, -0.21632948517799377, 0.19817735254764557], "agreement_weight": 0.1351311355829239, "orthogonal_weight": -1.101820707321167}, {"head": 1, "memory_to_active": 2.9391300678253174, "contribution_to_active": 0.03572604060173035, "mix_mean": 0.004791934043169022, "mix_low": 0.9037647247314453, "mix_high": 0.0, "read_cosine": 0.13046924769878387, "beta_mean": 0.24870330095291138, "nonfinite": 0.0, "weighted_memory_to_active": 0.028126899152994156, "mix_std": 0.011488636955618858, "beta_std": 0.15274183452129364, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.008462684229016304, "evidence_weights": [0.1452292799949646, 0.12049441784620285, 0.5852660536766052], "agreement_weight": 0.022604554891586304, "orthogonal_weight": 0.32691490650177}, {"head": 2, "memory_to_active": 0.020241443067789078, "contribution_to_active": 0.4304715394973755, "mix_mean": 0.5574920177459717, "mix_low": 0.0011997222900390625, "mix_high": 0.0035066604614257812, "read_cosine": 0.21282529830932617, "beta_mean": 3.178117913194001e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00965476967394352, "mix_std": 0.2746254801750183, "beta_std": 6.637684418819845e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003695715859066695, "evidence_weights": [-0.2560332417488098, -0.2963319420814514, 0.5844187140464783], "agreement_weight": -0.041237473487854004, "orthogonal_weight": -0.04879268258810043}, {"head": 3, "memory_to_active": 0.12116444110870361, "contribution_to_active": 0.5431495904922485, "mix_mean": 0.5741196870803833, "mix_low": 0.000316619873046875, "mix_high": 0.00643157958984375, "read_cosine": 0.5630983710289001, "beta_mean": 0.023670658469200134, "nonfinite": 0.0, "weighted_memory_to_active": 0.07609323412179947, "mix_std": 0.2586278021335602, "beta_std": 0.03276321664452553, "beta_low": 0.47247314453125, "beta_high": 0.0, "orthogonal_to_active": 0.05303100496530533, "evidence_weights": [-0.20366401970386505, -0.32225605845451355, 0.7776439189910889], "agreement_weight": 0.05446784570813179, "orthogonal_weight": -0.894484281539917}, {"head": 4, "memory_to_active": 2.7004799842834473, "contribution_to_active": 0.1959725320339203, "mix_mean": 0.019111044704914093, "mix_low": 0.6104030609130859, "mix_high": 0.0, "read_cosine": 0.2109270989894867, "beta_mean": 0.24135327339172363, "nonfinite": 0.0, "weighted_memory_to_active": 0.27036842703819275, "mix_std": 0.05112282559275627, "beta_std": 0.2270115464925766, "beta_low": 0.03814697265625, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.036023013293743134, "evidence_weights": [0.028290769085288048, 0.009720931760966778, 0.01897658221423626], "agreement_weight": 0.016502035781741142, "orthogonal_weight": -0.15690398216247559}, {"head": 5, "memory_to_active": 0.009928205981850624, "contribution_to_active": 0.5261538028717041, "mix_mean": 0.6537274718284607, "mix_low": 0.0007963180541992188, "mix_high": 0.008676528930664062, "read_cosine": 0.381714403629303, "beta_mean": 3.6825651477556676e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005810298025608063, "mix_std": 0.2637181282043457, "beta_std": 4.980526500730775e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013440280454233289, "evidence_weights": [-0.2822044789791107, -0.2920454740524292, 0.9518018960952759], "agreement_weight": 0.230682373046875, "orthogonal_weight": -0.2855517566204071}]}, {"layer": 3, "memory_to_active": 0.453237384557724, "contribution_to_active": 0.48161911964416504, "mix_mean": 0.512860894203186, "mix_low": 0.05569092556834221, "mix_high": 0.012565454468131065, "read_cosine": 0.20828217267990112, "beta_mean": 0.02604251354932785, "nonfinite": 0.0, "weighted_memory_to_active": 0.11946894228458405, "mix_std": 0.26065337657928467, "beta_std": 0.024721568450331688, "beta_low": 0.769927978515625, "beta_high": 0.0, "orthogonal_to_active": 0.04655306041240692, "output_influence": 0.4292978048324585, "residual_influence": 0.10248326510190964, "state_active": 0.026398956775665283, "state_memory": 0.0068478756584227085, "state_peak": 22.741424560546875, "growth_active": 1.1299328804016113, "growth_memory": 1.5943368673324585, "heads": [{"head": 0, "memory_to_active": 0.33063846826553345, "contribution_to_active": 0.5970228910446167, "mix_mean": 0.6046239733695984, "mix_low": 0.001850128173828125, "mix_high": 0.026144027709960938, "read_cosine": 0.2810388207435608, "beta_mean": 0.008886734023690224, "nonfinite": 0.0, "weighted_memory_to_active": 0.20696868002414703, "mix_std": 0.3054564595222473, "beta_std": 0.011581678874790668, "beta_low": 0.72271728515625, "beta_high": 0.0, "orthogonal_to_active": 0.14608389139175415, "evidence_weights": [-0.21817529201507568, -0.25224602222442627, 1.0068191289901733], "agreement_weight": 0.11816573888063431, "orthogonal_weight": -0.7588140368461609}, {"head": 1, "memory_to_active": 0.013934850692749023, "contribution_to_active": 0.4538031220436096, "mix_mean": 0.5854088664054871, "mix_low": 0.005074501037597656, "mix_high": 0.011934280395507812, "read_cosine": 0.17570778727531433, "beta_mean": 7.76524466346018e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008625540882349014, "mix_std": 0.29733824729919434, "beta_std": 0.00018707984418142587, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00707346573472023, "evidence_weights": [-0.27147912979125977, -0.22113604843616486, 0.7006824612617493], "agreement_weight": 0.5429570078849792, "orthogonal_weight": -0.8914517760276794}, {"head": 2, "memory_to_active": 2.132237434387207, "contribution_to_active": 0.2895035147666931, "mix_mean": 0.054732512682676315, "mix_low": 0.3150463104248047, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2151719331741333, "beta_mean": 0.14185860753059387, "nonfinite": 0.0, "weighted_memory_to_active": 0.32690733671188354, "mix_std": 0.10616591572761536, "beta_std": 0.12425742298364639, "beta_low": 0.019775390625, "beta_high": 0.0, "orthogonal_to_active": 0.0008762846700847149, "evidence_weights": [-0.0937831923365593, 0.0005453831981867552, -0.03680112212896347], "agreement_weight": -0.01290597952902317, "orthogonal_weight": -0.0033804485574364662}, {"head": 3, "memory_to_active": 0.01587517559528351, "contribution_to_active": 0.5222835540771484, "mix_mean": 0.6088799238204956, "mix_low": 0.005017280578613281, "mix_high": 0.012307167053222656, "read_cosine": 0.3048206567764282, "beta_mean": 8.17145046312362e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010902305133640766, "mix_std": 0.2928338348865509, "beta_std": 0.00018030061619356275, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00285450741648674, "evidence_weights": [-0.25699490308761597, -0.22638891637325287, 2.234936237335205], "agreement_weight": 2.144123077392578, "orthogonal_weight": -0.344546377658844}, {"head": 4, "memory_to_active": 0.010073879733681679, "contribution_to_active": 0.38615700602531433, "mix_mean": 0.5015907883644104, "mix_low": 0.004858970642089844, "mix_high": 0.0023298263549804688, "read_cosine": 0.1759534627199173, "beta_mean": 6.0974933148827404e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006368035450577736, "mix_std": 0.29163432121276855, "beta_std": 8.921141852624714e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0059648482128977776, "evidence_weights": [-0.22628198564052582, -0.2164742350578308, 0.17037437856197357], "agreement_weight": -0.040475815534591675, "orthogonal_weight": -1.0083262920379639}, {"head": 5, "memory_to_active": 0.2166644036769867, "contribution_to_active": 0.6409445405006409, "mix_mean": 0.7219292521476746, "mix_low": 0.0022983551025390625, "mix_high": 0.02267169952392578, "read_cosine": 0.09700034558773041, "beta_mean": 0.0052893925458192825, "nonfinite": 0.0, "weighted_memory_to_active": 0.15704168379306793, "mix_std": 0.27049148082733154, "beta_std": 0.012033718638122082, "beta_low": 0.8770751953125, "beta_high": 0.0, "orthogonal_to_active": 0.11646535992622375, "evidence_weights": [-0.26380306482315063, -0.3396807610988617, 0.5169833302497864], "agreement_weight": 0.20335547626018524, "orthogonal_weight": -0.7661479711532593}]}, {"layer": 4, "memory_to_active": 1.0532699823379517, "contribution_to_active": 0.4654693007469177, "mix_mean": 0.3683493137359619, "mix_low": 0.25350460410118103, "mix_high": 0.011610032059252262, "read_cosine": 0.3079383373260498, "beta_mean": 0.11535730212926865, "nonfinite": 0.0, "weighted_memory_to_active": 0.12649258971214294, "mix_std": 0.1617548018693924, "beta_std": 0.1032976284623146, "beta_low": 0.3848775625228882, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04741226136684418, "output_influence": 0.4827617108821869, "residual_influence": 0.14730967581272125, "state_active": 0.0387796051800251, "state_memory": 0.017652835696935654, "state_peak": 41.140567779541016, "growth_active": 1.23601496219635, "growth_memory": 1.5210931301116943, "heads": [{"head": 0, "memory_to_active": 0.1714944988489151, "contribution_to_active": 0.7898637056350708, "mix_mean": 0.6840905547142029, "mix_low": 0.013113021850585938, "mix_high": 0.03095531463623047, "read_cosine": 0.4659197926521301, "beta_mean": 0.2505086362361908, "nonfinite": 0.0, "weighted_memory_to_active": 0.14801353216171265, "mix_std": 0.34986212849617004, "beta_std": 0.25148090720176697, "beta_low": 0.2486572265625, "beta_high": 0.0, "orthogonal_to_active": 0.0802449956536293, "evidence_weights": [-0.26020556688308716, -0.3675323724746704, -0.07439727336168289], "agreement_weight": -0.004609106574207544, "orthogonal_weight": 0.6886586546897888}, {"head": 1, "memory_to_active": 0.04634778946638107, "contribution_to_active": 0.6513599753379822, "mix_mean": 0.6900993585586548, "mix_low": 0.0001087188720703125, "mix_high": 0.018248558044433594, "read_cosine": 0.6029168367385864, "beta_mean": 0.0008318328764289618, "nonfinite": 0.0, "weighted_memory_to_active": 0.03417251259088516, "mix_std": 0.24354590475559235, "beta_std": 0.005383788608014584, "beta_low": 0.98150634765625, "beta_high": 0.0, "orthogonal_to_active": 0.018197473138570786, "evidence_weights": [-0.23863184452056885, -0.3628678321838379, 0.6157582402229309], "agreement_weight": 0.3151116967201233, "orthogonal_weight": -0.7441309094429016}, {"head": 2, "memory_to_active": 1.7214055061340332, "contribution_to_active": 0.3317008316516876, "mix_mean": 0.03951902687549591, "mix_low": 0.4001884460449219, "mix_high": 1.9073486328125e-05, "read_cosine": 0.20116132497787476, "beta_mean": 0.07343041151762009, "nonfinite": 0.0, "weighted_memory_to_active": 0.25734829902648926, "mix_std": 0.07500490546226501, "beta_std": 0.07356581836938858, "beta_low": 0.05963134765625, "beta_high": 0.0, "orthogonal_to_active": 0.07712756097316742, "evidence_weights": [0.040426354855298996, -0.03170071542263031, 0.25032585859298706], "agreement_weight": 0.13091592490673065, "orthogonal_weight": -0.40230444073677063}, {"head": 3, "memory_to_active": 1.9739480018615723, "contribution_to_active": 0.14814059436321259, "mix_mean": 0.03234715014696121, "mix_low": 0.42674732208251953, "mix_high": 0.0, "read_cosine": 0.19701942801475525, "beta_mean": 0.1440485566854477, "nonfinite": 0.0, "weighted_memory_to_active": 0.18820610642433167, "mix_std": 0.0578586608171463, "beta_std": 0.12741096317768097, "beta_low": 0.0194091796875, "beta_high": 0.0, "orthogonal_to_active": 0.002223384566605091, "evidence_weights": [-0.020816922187805176, 0.049089185893535614, -0.25835561752319336], "agreement_weight": 0.07663440704345703, "orthogonal_weight": -0.015744468197226524}, {"head": 4, "memory_to_active": 0.007298873737454414, "contribution_to_active": 0.6451149582862854, "mix_mean": 0.7510985136032104, "mix_low": 0.000125885009765625, "mix_high": 0.02042865753173828, "read_cosine": 0.1496654748916626, "beta_mean": 1.9240735127823427e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005300603341311216, "mix_std": 0.2142047882080078, "beta_std": 3.7706900911871344e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003250026609748602, "evidence_weights": [-0.3338533937931061, -0.3236641585826874, 0.3723280429840088], "agreement_weight": 0.3515818417072296, "orthogonal_weight": -0.6548247933387756}, {"head": 5, "memory_to_active": 2.3991246223449707, "contribution_to_active": 0.22663560509681702, "mix_mean": 0.012941067107021809, "mix_low": 0.6807441711425781, "mix_high": 8.58306884765625e-06, "read_cosine": 0.2309471070766449, "beta_mean": 0.2233051210641861, "nonfinite": 0.0, "weighted_memory_to_active": 0.12591442465782166, "mix_std": 0.030052397400140762, "beta_std": 0.16190658509731293, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1034301221370697, "evidence_weights": [0.05933312326669693, -0.01690753549337387, -0.25370457768440247], "agreement_weight": -0.08317139744758606, "orthogonal_weight": 1.0279721021652222}]}, {"layer": 5, "memory_to_active": 0.39493876695632935, "contribution_to_active": 0.535426139831543, "mix_mean": 0.5078842639923096, "mix_low": 0.06057008355855942, "mix_high": 0.02089119143784046, "read_cosine": 0.06480620801448822, "beta_mean": 0.05325821787118912, "nonfinite": 0.0, "weighted_memory_to_active": 0.1435321569442749, "mix_std": 0.2563413977622986, "beta_std": 0.06969238817691803, "beta_low": 0.6645914912223816, "beta_high": 0.0, "orthogonal_to_active": 0.06059073284268379, "output_influence": 0.5525765419006348, "residual_influence": 0.16965168714523315, "state_active": 0.03948187828063965, "state_memory": 0.009904238395392895, "state_peak": 58.66402053833008, "growth_active": 1.0917330980300903, "growth_memory": 1.2283912897109985, "heads": [{"head": 0, "memory_to_active": 0.26799964904785156, "contribution_to_active": 0.5532335042953491, "mix_mean": 0.44882094860076904, "mix_low": 0.010955810546875, "mix_high": 0.0013036727905273438, "read_cosine": -0.12934955954551697, "beta_mean": 0.022630859166383743, "nonfinite": 0.0, "weighted_memory_to_active": 0.10143492370843887, "mix_std": 0.28062158823013306, "beta_std": 0.038222089409828186, "beta_low": 0.4976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.044089656323194504, "evidence_weights": [-0.1605464667081833, -0.1801878809928894, 0.08174840360879898], "agreement_weight": -0.020876947790384293, "orthogonal_weight": -0.45137912034988403}, {"head": 1, "memory_to_active": 0.007843962870538235, "contribution_to_active": 0.5375187397003174, "mix_mean": 0.6968852281570435, "mix_low": 0.0014810562133789062, "mix_high": 0.04306793212890625, "read_cosine": 0.20266741514205933, "beta_mean": 1.9001285181730054e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004836877807974815, "mix_std": 0.27269718050956726, "beta_std": 4.099921716260724e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004492675419896841, "evidence_weights": [-0.251156210899353, -0.29083529114723206, 0.9490013122558594], "agreement_weight": 0.697600781917572, "orthogonal_weight": -0.9831252694129944}, {"head": 2, "memory_to_active": 0.013798505067825317, "contribution_to_active": 0.5927679538726807, "mix_mean": 0.6578949689865112, "mix_low": 0.0025968551635742188, "mix_high": 0.022789955139160156, "read_cosine": -0.524226188659668, "beta_mean": 8.224420889746398e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00819519441574812, "mix_std": 0.2661117911338806, "beta_std": 0.00020366822718642652, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005977965425699949, "evidence_weights": [-0.26258715987205505, -0.32800623774528503, 0.15788763761520386], "agreement_weight": -0.1028190329670906, "orthogonal_weight": -0.8642210960388184}, {"head": 3, "memory_to_active": 1.5430549383163452, "contribution_to_active": 0.25546616315841675, "mix_mean": 0.11344486474990845, "mix_low": 0.1341228485107422, "mix_high": 0.0, "read_cosine": 0.15357311069965363, "beta_mean": 0.10976872593164444, "nonfinite": 0.0, "weighted_memory_to_active": 0.40237748622894287, "mix_std": 0.14935456216335297, "beta_std": 0.16188928484916687, "beta_low": 0.182373046875, "beta_high": 0.0, "orthogonal_to_active": 0.1615269035100937, "evidence_weights": [-0.07490082085132599, -0.10015199333429337, 0.05419354513287544], "agreement_weight": 0.05755472555756569, "orthogonal_weight": -0.5022676587104797}, {"head": 4, "memory_to_active": 0.46134811639785767, "contribution_to_active": 0.5880377292633057, "mix_mean": 0.3252754211425781, "mix_low": 0.21258831024169922, "mix_high": 0.006232261657714844, "read_cosine": 0.5030168294906616, "beta_mean": 0.18682357668876648, "nonfinite": 0.0, "weighted_memory_to_active": 0.2931477725505829, "mix_std": 0.3437403738498688, "beta_std": 0.2167418897151947, "beta_low": 0.3096923828125, "beta_high": 0.0, "orthogonal_to_active": 0.11753059178590775, "evidence_weights": [-0.11863375455141068, -0.26047617197036743, -0.04395424574613571], "agreement_weight": 0.033408310264348984, "orthogonal_weight": 0.567650318145752}, {"head": 5, "memory_to_active": 0.07558729499578476, "contribution_to_active": 0.6855325698852539, "mix_mean": 0.8049837350845337, "mix_low": 0.0016756057739257812, "mix_high": 0.05195331573486328, "read_cosine": 0.1831556260585785, "beta_mean": 0.00022488541435450315, "nonfinite": 0.0, "weighted_memory_to_active": 0.05120062083005905, "mix_std": 0.22552289068698883, "beta_std": 0.0010563242249190807, "beta_low": 0.997802734375, "beta_high": 0.0, "orthogonal_to_active": 0.029926598072052002, "evidence_weights": [-0.31206807494163513, -0.33420228958129883, 1.8820003271102905], "agreement_weight": 0.4946117699146271, "orthogonal_weight": -0.6212314367294312}]}, {"layer": 6, "memory_to_active": 0.27890196442604065, "contribution_to_active": 0.6322891712188721, "mix_mean": 0.590858519077301, "mix_low": 0.043146297335624695, "mix_high": 0.0335383415222168, "read_cosine": 0.3238743841648102, "beta_mean": 0.06457626819610596, "nonfinite": 0.0, "weighted_memory_to_active": 0.15178793668746948, "mix_std": 0.23833847045898438, "beta_std": 0.07459528744220734, "beta_low": 0.5859476923942566, "beta_high": 0.0, "orthogonal_to_active": 0.05377108231186867, "output_influence": 0.5514937043190002, "residual_influence": 0.18976661562919617, "state_active": 0.04653359204530716, "state_memory": 0.007508289068937302, "state_peak": 101.40127563476562, "growth_active": 1.2531168460845947, "growth_memory": 1.2013193368911743, "heads": [{"head": 0, "memory_to_active": 0.11068128049373627, "contribution_to_active": 0.765720009803772, "mix_mean": 0.8265126943588257, "mix_low": 0.00021266937255859375, "mix_high": 0.059821128845214844, "read_cosine": 0.35754358768463135, "beta_mean": 0.012000057846307755, "nonfinite": 0.0, "weighted_memory_to_active": 0.0962669774889946, "mix_std": 0.19824302196502686, "beta_std": 0.03178686648607254, "beta_low": 0.77294921875, "beta_high": 0.0, "orthogonal_to_active": 0.06537935882806778, "evidence_weights": [-0.31633472442626953, -0.35545432567596436, 1.2243696451187134], "agreement_weight": 0.681131899356842, "orthogonal_weight": -0.7599865198135376}, {"head": 1, "memory_to_active": 0.004827905911952257, "contribution_to_active": 0.5049644708633423, "mix_mean": 0.5953888893127441, "mix_low": 0.0012798309326171875, "mix_high": 0.01596546173095703, "read_cosine": 0.4658415913581848, "beta_mean": 1.1065056241932325e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021920769941061735, "mix_std": 0.2839808762073517, "beta_std": 1.802241604309529e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012090979143977165, "evidence_weights": [-0.1951298862695694, -0.3013494610786438, 1.1160739660263062], "agreement_weight": 0.10826412588357925, "orthogonal_weight": -0.7361007928848267}, {"head": 2, "memory_to_active": 0.7018133997917175, "contribution_to_active": 0.09882950037717819, "mix_mean": 0.04739169031381607, "mix_low": 0.24927520751953125, "mix_high": 0.0, "read_cosine": 0.2563309669494629, "beta_mean": 0.022258151322603226, "nonfinite": 0.0, "weighted_memory_to_active": 0.0615215040743351, "mix_std": 0.07407274842262268, "beta_std": 0.025761030614376068, "beta_low": 0.2811279296875, "beta_high": 0.0, "orthogonal_to_active": 0.007805889006704092, "evidence_weights": [0.070885568857193, -0.05235124006867409, 0.3473205864429474], "agreement_weight": 0.13994398713111877, "orthogonal_weight": -0.15160973370075226}, {"head": 3, "memory_to_active": 0.13983692228794098, "contribution_to_active": 0.6656761169433594, "mix_mean": 0.6550760269165039, "mix_low": 0.006823539733886719, "mix_high": 0.02041339874267578, "read_cosine": 0.49177098274230957, "beta_mean": 0.22106632590293884, "nonfinite": 0.0, "weighted_memory_to_active": 0.09702321887016296, "mix_std": 0.3250734508037567, "beta_std": 0.24648110568523407, "beta_low": 0.1756591796875, "beta_high": 0.0, "orthogonal_to_active": 0.053465764969587326, "evidence_weights": [-0.17150068283081055, -0.323307603597641, 0.17235703766345978], "agreement_weight": -0.014126325026154518, "orthogonal_weight": 0.6412405967712402}, {"head": 4, "memory_to_active": 0.685558557510376, "contribution_to_active": 1.1341829299926758, "mix_mean": 0.7397050261497498, "mix_low": 0.000988006591796875, "mix_high": 0.09739017486572266, "read_cosine": 0.1072874441742897, "beta_mean": 0.13198457658290863, "nonfinite": 0.0, "weighted_memory_to_active": 0.6316170692443848, "mix_std": 0.3017439544200897, "beta_std": 0.1432349979877472, "beta_low": 0.2860107421875, "beta_high": 0.0, "orthogonal_to_active": 0.17286226153373718, "evidence_weights": [-0.20044063031673431, -0.5537190437316895, -0.5096719861030579], "agreement_weight": -0.015156391076743603, "orthogonal_weight": 0.29866287112236023}, {"head": 5, "memory_to_active": 0.030693812295794487, "contribution_to_active": 0.6243615746498108, "mix_mean": 0.6810768246650696, "mix_low": 0.00029850006103515625, "mix_high": 0.007639884948730469, "read_cosine": 0.2644718289375305, "beta_mean": 0.00013743132876697928, "nonfinite": 0.0, "weighted_memory_to_active": 0.02210673689842224, "mix_std": 0.2469167411327362, "beta_std": 0.00028964877128601074, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.021904099732637405, "evidence_weights": [-0.23578965663909912, -0.31531959772109985, 0.3777437210083008], "agreement_weight": 0.19413481652736664, "orthogonal_weight": -1.0584298372268677}]}, {"layer": 7, "memory_to_active": 1.7717833518981934, "contribution_to_active": 0.2649003863334656, "mix_mean": 0.16574227809906006, "mix_low": 0.24687258899211884, "mix_high": 0.0009950002422556281, "read_cosine": 0.22709152102470398, "beta_mean": 0.133411705493927, "nonfinite": 0.0, "weighted_memory_to_active": 0.2175523191690445, "mix_std": 0.14474228024482727, "beta_std": 0.0982862114906311, "beta_low": 0.205596923828125, "beta_high": 0.00022379557776730508, "orthogonal_to_active": 0.09615390002727509, "output_influence": 0.2833740711212158, "residual_influence": 0.06766761094331741, "state_active": 0.013688258826732635, "state_memory": 0.02436715178191662, "state_peak": 29.304454803466797, "growth_active": 1.1462074518203735, "growth_memory": 1.3041390180587769, "heads": [{"head": 0, "memory_to_active": 2.527456760406494, "contribution_to_active": 0.18412336707115173, "mix_mean": 0.03480284661054611, "mix_low": 0.43847179412841797, "mix_high": 0.0, "read_cosine": 0.32247161865234375, "beta_mean": 0.15288971364498138, "nonfinite": 0.0, "weighted_memory_to_active": 0.21532952785491943, "mix_std": 0.07346843183040619, "beta_std": 0.11401157081127167, "beta_low": 0.00396728515625, "beta_high": 0.000244140625, "orthogonal_to_active": 0.0010694528464227915, "evidence_weights": [0.04665188863873482, -0.07615984976291656, 0.014285389333963394], "agreement_weight": -0.028129247948527336, "orthogonal_weight": -0.00647495873272419}, {"head": 1, "memory_to_active": 0.9697704911231995, "contribution_to_active": 0.37377411127090454, "mix_mean": 0.21007880568504333, "mix_low": 0.12105274200439453, "mix_high": 0.005184173583984375, "read_cosine": 0.39162111282348633, "beta_mean": 0.09189020097255707, "nonfinite": 0.0, "weighted_memory_to_active": 0.32309913635253906, "mix_std": 0.23626753687858582, "beta_std": 0.06361415982246399, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.04247850179672241, "evidence_weights": [-0.21401405334472656, -0.09968896955251694, -0.28138256072998047], "agreement_weight": 0.054105594754219055, "orthogonal_weight": 0.16046853363513947}, {"head": 2, "memory_to_active": 0.6870149374008179, "contribution_to_active": 0.35807734727859497, "mix_mean": 0.21014106273651123, "mix_low": 0.026244163513183594, "mix_high": 0.00040149688720703125, "read_cosine": 0.0891040787100792, "beta_mean": 0.0444544181227684, "nonfinite": 0.0, "weighted_memory_to_active": 0.23490113019943237, "mix_std": 0.20622894167900085, "beta_std": 0.04440096393227577, "beta_low": 0.107666015625, "beta_high": 0.0, "orthogonal_to_active": 0.13132011890411377, "evidence_weights": [-0.08696065843105316, -0.17252278327941895, 0.8381589651107788], "agreement_weight": 0.02300080843269825, "orthogonal_weight": -0.5756031274795532}, {"head": 3, "memory_to_active": 4.86362361907959, "contribution_to_active": 0.11278152465820312, "mix_mean": 0.03346293419599533, "mix_low": 0.3428764343261719, "mix_high": 0.0, "read_cosine": 0.10450787842273712, "beta_mean": 0.4333847165107727, "nonfinite": 0.0, "weighted_memory_to_active": 0.4053020775318146, "mix_std": 0.051568932831287384, "beta_std": 0.2851608991622925, "beta_low": 0.00396728515625, "beta_high": 0.0010986328125, "orthogonal_to_active": 0.3515070080757141, "evidence_weights": [0.07367222756147385, -0.034939032047986984, 0.7144920825958252], "agreement_weight": 0.054035354405641556, "orthogonal_weight": -0.933759331703186}, {"head": 4, "memory_to_active": 1.50957190990448, "contribution_to_active": 0.1099805235862732, "mix_mean": 0.02207000181078911, "mix_low": 0.5506677627563477, "mix_high": 0.0, "read_cosine": 0.2134641855955124, "beta_mean": 0.07752973586320877, "nonfinite": 0.0, "weighted_memory_to_active": 0.08923284709453583, "mix_std": 0.0454135425388813, "beta_std": 0.08158576488494873, "beta_low": 0.11944580078125, "beta_high": 0.0, "orthogonal_to_active": 0.01580520160496235, "evidence_weights": [0.1353142261505127, -0.01177226658910513, 0.18766142427921295], "agreement_weight": 0.045289307832717896, "orthogonal_weight": -0.27211543917655945}, {"head": 5, "memory_to_active": 0.07326249778270721, "contribution_to_active": 0.45066550374031067, "mix_mean": 0.48389798402786255, "mix_low": 0.001922607421875, "mix_high": 0.00038433074951171875, "read_cosine": 0.24138015508651733, "beta_mean": 0.0003214695898350328, "nonfinite": 0.0, "weighted_memory_to_active": 0.0374491885304451, "mix_std": 0.25550615787506104, "beta_std": 0.0009439163841307163, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.03474308177828789, "evidence_weights": [-0.1778094619512558, -0.23387059569358826, 0.5400394201278687], "agreement_weight": 0.21367472410202026, "orthogonal_weight": -1.000331163406372}]}, {"layer": 8, "memory_to_active": 0.1515732854604721, "contribution_to_active": 0.5629186034202576, "mix_mean": 0.48959681391716003, "mix_low": 0.010680994018912315, "mix_high": 0.0017069181194528937, "read_cosine": 0.42032527923583984, "beta_mean": 0.014205601066350937, "nonfinite": 0.0, "weighted_memory_to_active": 0.07778479903936386, "mix_std": 0.24109195172786713, "beta_std": 0.02029181458055973, "beta_low": 0.7744344472885132, "beta_high": 0.0, "orthogonal_to_active": 0.07112038135528564, "output_influence": 0.514567494392395, "residual_influence": 0.1825452744960785, "state_active": 0.0459318570792675, "state_memory": 0.005570738576352596, "state_peak": 52.1722297668457, "growth_active": 1.1464996337890625, "growth_memory": 1.1220507621765137, "heads": [{"head": 0, "memory_to_active": 0.057896360754966736, "contribution_to_active": 0.5730496048927307, "mix_mean": 0.6097475290298462, "mix_low": 0.0006160736083984375, "mix_high": 0.0017232894897460938, "read_cosine": 0.46275782585144043, "beta_mean": 0.0036929205525666475, "nonfinite": 0.0, "weighted_memory_to_active": 0.04140828922390938, "mix_std": 0.24897781014442444, "beta_std": 0.014327162876725197, "beta_low": 0.9232177734375, "beta_high": 0.0, "orthogonal_to_active": 0.025878600776195526, "evidence_weights": [-0.1650739312171936, -0.3869771361351013, 0.6012707352638245], "agreement_weight": 0.25004255771636963, "orthogonal_weight": -0.7965169548988342}, {"head": 1, "memory_to_active": 0.06958500295877457, "contribution_to_active": 0.5557100772857666, "mix_mean": 0.5253987908363342, "mix_low": 5.626678466796875e-05, "mix_high": 0.0010900497436523438, "read_cosine": 0.1633073389530182, "beta_mean": 0.0011199184227734804, "nonfinite": 0.0, "weighted_memory_to_active": 0.03121982142329216, "mix_std": 0.2335842102766037, "beta_std": 0.008210060186684132, "beta_low": 0.98895263671875, "beta_high": 0.0, "orthogonal_to_active": 0.022094711661338806, "evidence_weights": [-0.19589708745479584, -0.2768710255622864, 0.6942930817604065], "agreement_weight": 0.07101015001535416, "orthogonal_weight": -0.7475182414054871}, {"head": 2, "memory_to_active": 0.014594973996281624, "contribution_to_active": 0.6821317076683044, "mix_mean": 0.6761865615844727, "mix_low": 0.000537872314453125, "mix_high": 0.0037031173706054688, "read_cosine": 0.41869401931762695, "beta_mean": 0.00013769474753644317, "nonfinite": 0.0, "weighted_memory_to_active": 0.011477768421173096, "mix_std": 0.22496184706687927, "beta_std": 0.0034079134929925203, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00692359684035182, "evidence_weights": [-0.22430482506752014, -0.3456912338733673, 1.0152912139892578], "agreement_weight": 0.2916204035282135, "orthogonal_weight": -0.7014538049697876}, {"head": 3, "memory_to_active": 0.1058301031589508, "contribution_to_active": 0.6265406012535095, "mix_mean": 0.5824106931686401, "mix_low": 0.0006952285766601562, "mix_high": 0.0034885406494140625, "read_cosine": 0.7934949398040771, "beta_mean": 0.00820951908826828, "nonfinite": 0.0, "weighted_memory_to_active": 0.0798683762550354, "mix_std": 0.25606048107147217, "beta_std": 0.008381648920476437, "beta_low": 0.71221923828125, "beta_high": 0.0, "orthogonal_to_active": 0.02539214864373207, "evidence_weights": [-0.18188439309597015, -0.29909372329711914, 0.5496975183486938], "agreement_weight": 0.24091091752052307, "orthogonal_weight": -0.6179786324501038}, {"head": 4, "memory_to_active": 0.27960845828056335, "contribution_to_active": 0.2966340184211731, "mix_mean": 0.2744487524032593, "mix_low": 0.0064849853515625, "mix_high": 2.86102294921875e-06, "read_cosine": 0.18429896235466003, "beta_mean": 0.0016661930130794644, "nonfinite": 0.0, "weighted_memory_to_active": 0.10159371793270111, "mix_std": 0.21624793112277985, "beta_std": 0.011537902057170868, "beta_low": 0.9794921875, "beta_high": 0.0, "orthogonal_to_active": 0.02584984339773655, "evidence_weights": [-0.1073361411690712, -0.1814514547586441, 0.5211692452430725], "agreement_weight": 0.06327235698699951, "orthogonal_weight": -0.26438143849372864}, {"head": 5, "memory_to_active": 0.3819247782230377, "contribution_to_active": 0.6434455513954163, "mix_mean": 0.2693885564804077, "mix_low": 0.055695533752441406, "mix_high": 0.00023365020751953125, "read_cosine": 0.49939846992492676, "beta_mean": 0.07040736079216003, "nonfinite": 0.0, "weighted_memory_to_active": 0.20114079117774963, "mix_std": 0.2667194902896881, "beta_std": 0.07588620483875275, "beta_low": 0.04278564453125, "beta_high": 0.0, "orthogonal_to_active": 0.32058337330818176, "evidence_weights": [-0.14388705790042877, -0.15662480890750885, -0.10626222938299179], "agreement_weight": -0.009608210995793343, "orthogonal_weight": 1.90327787399292}]}, {"layer": 9, "memory_to_active": 0.9033560752868652, "contribution_to_active": 0.6922323703765869, "mix_mean": 0.36808109283447266, "mix_low": 0.07789485156536102, "mix_high": 0.005955219734460115, "read_cosine": 0.35590678453445435, "beta_mean": 0.04184086620807648, "nonfinite": 0.0, "weighted_memory_to_active": 0.3873547911643982, "mix_std": 0.2333553582429886, "beta_std": 0.03573060408234596, "beta_low": 0.5522868037223816, "beta_high": 0.0, "orthogonal_to_active": 0.03874410688877106, "output_influence": 0.540239691734314, "residual_influence": 0.15786807239055634, "state_active": 0.043371573090553284, "state_memory": 0.015807032585144043, "state_peak": 53.739540100097656, "growth_active": 1.1665652990341187, "growth_memory": 1.3403128385543823, "heads": [{"head": 0, "memory_to_active": 0.0004002609639428556, "contribution_to_active": 0.5668547749519348, "mix_mean": 0.6402751803398132, "mix_low": 0.0012331008911132812, "mix_high": 0.023509979248046875, "read_cosine": 0.3356459140777588, "beta_mean": 1.1905533483513864e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0002539636625442654, "mix_std": 0.289249449968338, "beta_std": 3.202153266101959e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0002694338036235422, "evidence_weights": [-0.3047412633895874, -0.3123278319835663, 0.4871847629547119], "agreement_weight": 0.11911525577306747, "orthogonal_weight": -1.1197764873504639}, {"head": 1, "memory_to_active": 0.02250717021524906, "contribution_to_active": 0.6979596614837646, "mix_mean": 0.5666592121124268, "mix_low": 0.00693511962890625, "mix_high": 0.002613067626953125, "read_cosine": 0.42960503697395325, "beta_mean": 5.555827374337241e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01719723641872406, "mix_std": 0.2816043794155121, "beta_std": 0.00012930278899148107, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011468345299363136, "evidence_weights": [-0.24221576750278473, -0.2973162531852722, 1.0401484966278076], "agreement_weight": 0.48599740862846375, "orthogonal_weight": -0.8346286416053772}, {"head": 2, "memory_to_active": 0.21220330893993378, "contribution_to_active": 0.6507501602172852, "mix_mean": 0.6013262271881104, "mix_low": 0.0019311904907226562, "mix_high": 0.008169174194335938, "read_cosine": 0.38858577609062195, "beta_mean": 0.0009283455437980592, "nonfinite": 0.0, "weighted_memory_to_active": 0.14860913157463074, "mix_std": 0.26909103989601135, "beta_std": 0.005268723703920841, "beta_low": 0.9921875, "beta_high": 0.0, "orthogonal_to_active": 0.0845872238278389, "evidence_weights": [-0.20323410630226135, -0.27748334407806396, 0.7626949548721313], "agreement_weight": 0.3320251703262329, "orthogonal_weight": -0.6846806406974792}, {"head": 3, "memory_to_active": 0.965943455696106, "contribution_to_active": 0.21892669796943665, "mix_mean": 0.1124717965722084, "mix_low": 0.15927600860595703, "mix_high": 7.724761962890625e-05, "read_cosine": 0.37976956367492676, "beta_mean": 0.04107598960399628, "nonfinite": 0.0, "weighted_memory_to_active": 0.1814412921667099, "mix_std": 0.16585855185985565, "beta_std": 0.0399818979203701, "beta_low": 0.0810546875, "beta_high": 0.0, "orthogonal_to_active": 0.05935174226760864, "evidence_weights": [0.004886643961071968, -0.06726766377687454, 0.26107728481292725], "agreement_weight": 0.09523458778858185, "orthogonal_weight": -0.3606177568435669}, {"head": 4, "memory_to_active": 2.8825840950012207, "contribution_to_active": 1.6142685413360596, "mix_mean": 0.1467554271221161, "mix_low": 0.12360382080078125, "mix_high": 0.0010099411010742188, "read_cosine": 0.10846913605928421, "beta_mean": 0.15380491316318512, "nonfinite": 0.0, "weighted_memory_to_active": 1.5404021739959717, "mix_std": 0.20598164200782776, "beta_std": 0.08645278215408325, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.020250344648957253, "evidence_weights": [-0.07983580976724625, -0.15314991772174835, -0.022506294772028923], "agreement_weight": -0.06533261388540268, "orthogonal_weight": -0.013714817352592945}, {"head": 5, "memory_to_active": 1.3364976644515991, "contribution_to_active": 0.40463387966156006, "mix_mean": 0.14099858701229095, "mix_low": 0.17438983917236328, "mix_high": 0.00035190582275390625, "read_cosine": 0.4933651387691498, "beta_mean": 0.05517918989062309, "nonfinite": 0.0, "weighted_memory_to_active": 0.4362245798110962, "mix_std": 0.18834693729877472, "beta_std": 0.08254771679639816, "beta_low": 0.240478515625, "beta_high": 0.0, "orthogonal_to_active": 0.05653754994273186, "evidence_weights": [0.0840814933180809, -0.14579644799232483, 0.13434775173664093], "agreement_weight": 0.01373809389770031, "orthogonal_weight": 0.17649227380752563}]}, {"layer": 10, "memory_to_active": 0.19095778465270996, "contribution_to_active": 0.6704015731811523, "mix_mean": 0.5245399475097656, "mix_low": 0.02337169647216797, "mix_high": 0.004906336776912212, "read_cosine": 0.3840968608856201, "beta_mean": 0.04824008047580719, "nonfinite": 0.0, "weighted_memory_to_active": 0.08394075930118561, "mix_std": 0.2689783573150635, "beta_std": 0.047705329954624176, "beta_low": 0.654876708984375, "beta_high": 0.0, "orthogonal_to_active": 0.2160765826702118, "output_influence": 0.45786184072494507, "residual_influence": 0.16129398345947266, "state_active": 0.05193154513835907, "state_memory": 0.014390050433576107, "state_peak": 125.10999298095703, "growth_active": 1.0240764617919922, "growth_memory": 1.023229718208313, "heads": [{"head": 0, "memory_to_active": 0.6004356145858765, "contribution_to_active": 1.708949327468872, "mix_mean": 0.5633173584938049, "mix_low": 0.10572624206542969, "mix_high": 0.006961822509765625, "read_cosine": 0.1253049373626709, "beta_mean": 0.21198618412017822, "nonfinite": 0.0, "weighted_memory_to_active": 0.3760991096496582, "mix_std": 0.3933906853199005, "beta_std": 0.2163063883781433, "beta_low": 0.1298828125, "beta_high": 0.0, "orthogonal_to_active": 1.1979953050613403, "evidence_weights": [-0.09286103397607803, -0.38326430320739746, -1.8599683046340942], "agreement_weight": 0.029262684285640717, "orthogonal_weight": 3.2242913246154785}, {"head": 1, "memory_to_active": 0.011415427550673485, "contribution_to_active": 0.6156902313232422, "mix_mean": 0.6275960803031921, "mix_low": 0.0016355514526367188, "mix_high": 0.015598297119140625, "read_cosine": 0.3104479908943176, "beta_mean": 7.646611629752442e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006421421188861132, "mix_std": 0.26499348878860474, "beta_std": 0.00011248898226767778, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009894650429487228, "evidence_weights": [-0.25040003657341003, -0.19485940039157867, 0.5614379048347473], "agreement_weight": 0.5952404737472534, "orthogonal_weight": -1.6662548780441284}, {"head": 2, "memory_to_active": 0.013322676531970501, "contribution_to_active": 0.6214320659637451, "mix_mean": 0.6642272472381592, "mix_low": 0.00013637542724609375, "mix_high": 0.005316734313964844, "read_cosine": 0.02090279571712017, "beta_mean": 2.8934642614331096e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006483440287411213, "mix_std": 0.2495526224374771, "beta_std": 0.00010428813402540982, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0027459636330604553, "evidence_weights": [-0.2677170932292938, -0.28198352456092834, 0.6334348320960999], "agreement_weight": -0.2138148844242096, "orthogonal_weight": -0.4129980802536011}, {"head": 3, "memory_to_active": 0.3733469545841217, "contribution_to_active": 0.11064477264881134, "mix_mean": 0.30253639817237854, "mix_low": 0.030114173889160156, "mix_high": 9.1552734375e-05, "read_cosine": 0.9037706851959229, "beta_mean": 0.07010385394096375, "nonfinite": 0.0, "weighted_memory_to_active": 0.04405993968248367, "mix_std": 0.23474815487861633, "beta_std": 0.06103269010782242, "beta_low": 0.001953125, "beta_high": 0.0, "orthogonal_to_active": 0.04958381876349449, "evidence_weights": [-0.10174595564603806, -0.14861786365509033, 0.16432735323905945], "agreement_weight": 0.014906590804457664, "orthogonal_weight": 1.4164127111434937}, {"head": 4, "memory_to_active": 0.01963798701763153, "contribution_to_active": 0.5623912215232849, "mix_mean": 0.5902471542358398, "mix_low": 0.000274658203125, "mix_high": 0.0014629364013671875, "read_cosine": 0.09518951177597046, "beta_mean": 5.76897946302779e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011745225638151169, "mix_std": 0.24518033862113953, "beta_std": 0.00013759381545241922, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.016222577542066574, "evidence_weights": [-0.1854877471923828, -0.25927814841270447, 0.9673590064048767], "agreement_weight": 0.16896648705005646, "orthogonal_weight": -1.4544187784194946}, {"head": 5, "memory_to_active": 0.12758798897266388, "contribution_to_active": 0.403301477432251, "mix_mean": 0.39931532740592957, "mix_low": 0.0023431777954101562, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8489653468132019, "beta_mean": 0.007187356241047382, "nonfinite": 0.0, "weighted_memory_to_active": 0.05883537605404854, "mix_std": 0.22600483894348145, "beta_std": 0.008538533002138138, "beta_low": 0.79742431640625, "beta_high": 0.0, "orthogonal_to_active": 0.02001713030040264, "evidence_weights": [-0.13997788727283478, -0.16229192912578583, 0.26727092266082764], "agreement_weight": 0.09193261712789536, "orthogonal_weight": -0.6609495878219604}]}, {"layer": 11, "memory_to_active": 0.08211826533079147, "contribution_to_active": 0.6281293630599976, "mix_mean": 0.5158599615097046, "mix_low": 0.12027311325073242, "mix_high": 0.03950802609324455, "read_cosine": 0.32180002331733704, "beta_mean": 0.02616281434893608, "nonfinite": 0.0, "weighted_memory_to_active": 0.04465038701891899, "mix_std": 0.3246115446090698, "beta_std": 0.0387292318046093, "beta_low": 0.6927592158317566, "beta_high": 0.0, "orthogonal_to_active": 0.04808497428894043, "output_influence": 0.5479501485824585, "residual_influence": 0.14686894416809082, "state_active": 0.1904240846633911, "state_memory": 0.0037602661177515984, "state_peak": 32.02699279785156, "growth_active": 1.1210423707962036, "growth_memory": 1.5777168273925781, "heads": [{"head": 0, "memory_to_active": 0.009883344173431396, "contribution_to_active": 0.6154024004936218, "mix_mean": 0.6526399254798889, "mix_low": 0.0010776519775390625, "mix_high": 0.007815361022949219, "read_cosine": 0.24999725818634033, "beta_mean": 9.298097575083375e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007201819214969873, "mix_std": 0.25715532898902893, "beta_std": 0.004204415716230869, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.005848334636539221, "evidence_weights": [-0.1738668829202652, -0.3647945523262024, 0.790992021560669], "agreement_weight": 0.3791212737560272, "orthogonal_weight": -0.8399749398231506}, {"head": 1, "memory_to_active": 0.14646831154823303, "contribution_to_active": 0.4245714545249939, "mix_mean": 0.4680725336074829, "mix_low": 0.005695343017578125, "mix_high": 0.0004482269287109375, "read_cosine": 0.4436104893684387, "beta_mean": 0.006655111908912659, "nonfinite": 0.0, "weighted_memory_to_active": 0.0715615302324295, "mix_std": 0.27053481340408325, "beta_std": 0.02000514604151249, "beta_low": 0.87896728515625, "beta_high": 0.0, "orthogonal_to_active": 0.039181582629680634, "evidence_weights": [-0.15594634413719177, -0.26348593831062317, 0.585613489151001], "agreement_weight": 0.07249366492033005, "orthogonal_weight": -0.6408393979072571}, {"head": 2, "memory_to_active": 0.0068448614329099655, "contribution_to_active": 0.9832279682159424, "mix_mean": 0.42138803005218506, "mix_low": 0.3373136520385742, "mix_high": 0.15025806427001953, "read_cosine": 0.17912404239177704, "beta_mean": 0.06838180869817734, "nonfinite": 0.0, "weighted_memory_to_active": 0.004752357956022024, "mix_std": 0.44878190755844116, "beta_std": 0.09099036455154419, "beta_low": 0.48150634765625, "beta_high": 0.0, "orthogonal_to_active": 0.04440657049417496, "evidence_weights": [0.049851156771183014, -0.37636297941207886, -1.3358322381973267], "agreement_weight": 0.00026693689869716763, "orthogonal_weight": 9.740411758422852}, {"head": 3, "memory_to_active": 0.008492979221045971, "contribution_to_active": 0.8709741830825806, "mix_mean": 0.3955524265766144, "mix_low": 0.36838245391845703, "mix_high": 0.07136058807373047, "read_cosine": 0.4772743880748749, "beta_mean": 0.056759949773550034, "nonfinite": 0.0, "weighted_memory_to_active": 0.005545728839933872, "mix_std": 0.43657827377319336, "beta_std": 0.08523259311914444, "beta_low": 0.51409912109375, "beta_high": 0.0, "orthogonal_to_active": 0.036337047815322876, "evidence_weights": [0.20587855577468872, -0.2908884882926941, -0.1398371458053589], "agreement_weight": 0.04968110844492912, "orthogonal_weight": 10.009846687316895}, {"head": 4, "memory_to_active": 0.23592939972877502, "contribution_to_active": 0.46135762333869934, "mix_mean": 0.5981667637825012, "mix_low": 0.007152557373046875, "mix_high": 0.001880645751953125, "read_cosine": 0.04169711098074913, "beta_mean": 0.017164811491966248, "nonfinite": 0.0, "weighted_memory_to_active": 0.13472190499305725, "mix_std": 0.27324411273002625, "beta_std": 0.02057364024221897, "beta_low": 0.4893798828125, "beta_high": 0.0, "orthogonal_to_active": 0.1325913518667221, "evidence_weights": [-0.23875966668128967, -0.2417965829372406, 1.898077368736267], "agreement_weight": -0.21037833392620087, "orthogonal_weight": -1.000401258468628}, {"head": 5, "memory_to_active": 0.08509070426225662, "contribution_to_active": 0.41324231028556824, "mix_mean": 0.5593400001525879, "mix_low": 0.0020170211791992188, "mix_high": 0.0052852630615234375, "read_cosine": 0.5390967130661011, "beta_mean": 0.007922216318547726, "nonfinite": 0.0, "weighted_memory_to_active": 0.044118981808423996, "mix_std": 0.26137465238571167, "beta_std": 0.011369225569069386, "beta_low": 0.7930908203125, "beta_high": 0.0, "orthogonal_to_active": 0.030144942924380302, "evidence_weights": [-0.2043650895357132, -0.31159353256225586, 0.6573849320411682], "agreement_weight": 0.08616048097610474, "orthogonal_weight": -0.857940673828125}]}, {"layer": 12, "memory_to_active": 0.24180373549461365, "contribution_to_active": 0.5901882648468018, "mix_mean": 0.5468961000442505, "mix_low": 0.008869966492056847, "mix_high": 0.006293932907283306, "read_cosine": 0.22319382429122925, "beta_mean": 0.01727285608649254, "nonfinite": 0.0, "weighted_memory_to_active": 0.125520259141922, "mix_std": 0.2615059018135071, "beta_std": 0.016918927431106567, "beta_low": 0.8267924189567566, "beta_high": 0.0, "orthogonal_to_active": 0.07656217366456985, "output_influence": 0.5105385184288025, "residual_influence": 0.1466626673936844, "state_active": 0.03537212684750557, "state_memory": 0.008914612233638763, "state_peak": 72.5448989868164, "growth_active": 1.160552978515625, "growth_memory": 1.2021563053131104, "heads": [{"head": 0, "memory_to_active": 0.04763691872358322, "contribution_to_active": 0.6229454278945923, "mix_mean": 0.5831019878387451, "mix_low": 0.00042057037353515625, "mix_high": 0.0053272247314453125, "read_cosine": 0.26512134075164795, "beta_mean": 0.0008231549290940166, "nonfinite": 0.0, "weighted_memory_to_active": 0.026457499712705612, "mix_std": 0.26491111516952515, "beta_std": 0.0016032446874305606, "beta_low": 0.9942626953125, "beta_high": 0.0, "orthogonal_to_active": 0.024767568334937096, "evidence_weights": [-0.21143826842308044, -0.2778817415237427, -0.000934260489884764], "agreement_weight": 0.10596735775470734, "orthogonal_weight": -0.9984272718429565}, {"head": 1, "memory_to_active": 0.10768561065196991, "contribution_to_active": 0.5523500442504883, "mix_mean": 0.5608053803443909, "mix_low": 0.00034332275390625, "mix_high": 0.0020589828491210938, "read_cosine": 0.3842153549194336, "beta_mean": 0.0007221479900181293, "nonfinite": 0.0, "weighted_memory_to_active": 0.0672178566455841, "mix_std": 0.2668408751487732, "beta_std": 0.010732580907642841, "beta_low": 0.99658203125, "beta_high": 0.0, "orthogonal_to_active": 0.05611699819564819, "evidence_weights": [-0.20349961519241333, -0.2726574242115021, 1.0500024557113647], "agreement_weight": 0.24616500735282898, "orthogonal_weight": -0.9181347489356995}, {"head": 2, "memory_to_active": 0.6326650977134705, "contribution_to_active": 0.5138484239578247, "mix_mean": 0.3264338970184326, "mix_low": 0.0285491943359375, "mix_high": 0.00030231475830078125, "read_cosine": 0.5531705617904663, "beta_mean": 0.1006409302353859, "nonfinite": 0.0, "weighted_memory_to_active": 0.2527891993522644, "mix_std": 0.24505038559436798, "beta_std": 0.06873893737792969, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.2764681875705719, "evidence_weights": [-0.20985908806324005, -0.05703302100300789, -0.017110245302319527], "agreement_weight": 0.02971057780086994, "orthogonal_weight": 1.3773372173309326}, {"head": 3, "memory_to_active": 0.04750934615731239, "contribution_to_active": 0.7523577809333801, "mix_mean": 0.7197228670120239, "mix_low": 0.00016689300537109375, "mix_high": 0.018888473510742188, "read_cosine": 0.12191303074359894, "beta_mean": 0.0003294203197583556, "nonfinite": 0.0, "weighted_memory_to_active": 0.03241114690899849, "mix_std": 0.2269270420074463, "beta_std": 0.006915707606822252, "beta_low": 0.99737548828125, "beta_high": 0.0, "orthogonal_to_active": 0.030522767454385757, "evidence_weights": [-0.2894017994403839, -0.33618807792663574, 1.6379133462905884], "agreement_weight": 0.5432432293891907, "orthogonal_weight": -0.9736173748970032}, {"head": 4, "memory_to_active": 0.5388422012329102, "contribution_to_active": 0.5944221019744873, "mix_mean": 0.4510962665081024, "mix_low": 0.017490386962890625, "mix_high": 0.00490570068359375, "read_cosine": -0.17542704939842224, "beta_mean": 0.0007732126978226006, "nonfinite": 0.0, "weighted_memory_to_active": 0.32395464181900024, "mix_std": 0.2925640344619751, "beta_std": 0.009228910319507122, "beta_low": 0.9794921875, "beta_high": 0.0, "orthogonal_to_active": 0.03292938321828842, "evidence_weights": [-0.1542225182056427, -0.21736687421798706, -0.11385910958051682], "agreement_weight": 0.00943775288760662, "orthogonal_weight": -0.10583878308534622}, {"head": 5, "memory_to_active": 0.0764831006526947, "contribution_to_active": 0.505205512046814, "mix_mean": 0.6402161121368408, "mix_low": 0.006249427795410156, "mix_high": 0.0062808990478515625, "read_cosine": 0.19016961753368378, "beta_mean": 0.00034827477065846324, "nonfinite": 0.0, "weighted_memory_to_active": 0.050291113555431366, "mix_std": 0.27274203300476074, "beta_std": 0.004294168204069138, "beta_low": 0.99261474609375, "beta_high": 0.0, "orthogonal_to_active": 0.03856813162565231, "evidence_weights": [-0.22955772280693054, -0.3176182508468628, 1.2651808261871338], "agreement_weight": 0.4139315187931061, "orthogonal_weight": -0.8173150420188904}]}, {"layer": 13, "memory_to_active": 0.11400388926267624, "contribution_to_active": 0.5327016115188599, "mix_mean": 0.5399589538574219, "mix_low": 0.005824248306453228, "mix_high": 0.009091059677302837, "read_cosine": 0.40604424476623535, "beta_mean": 0.014976424165070057, "nonfinite": 0.0, "weighted_memory_to_active": 0.04908175766468048, "mix_std": 0.25976240634918213, "beta_std": 0.015448902733623981, "beta_low": 0.8104553818702698, "beta_high": 0.0, "orthogonal_to_active": 0.051647309213876724, "output_influence": 0.5236616134643555, "residual_influence": 0.18269971013069153, "state_active": 0.12606924772262573, "state_memory": 0.04281961917877197, "state_peak": 96.38114166259766, "growth_active": 0.8996561765670776, "growth_memory": 1.015688180923462, "heads": [{"head": 0, "memory_to_active": 0.009265199303627014, "contribution_to_active": 0.5910805463790894, "mix_mean": 0.6065650582313538, "mix_low": 0.000812530517578125, "mix_high": 0.004261016845703125, "read_cosine": 0.45788276195526123, "beta_mean": 3.1459025194635615e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005948331672698259, "mix_std": 0.2678370475769043, "beta_std": 5.942538336967118e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005757972598075867, "evidence_weights": [-0.1948283612728119, -0.2670421600341797, 0.7748826146125793], "agreement_weight": 0.1840963065624237, "orthogonal_weight": -1.093530297279358}, {"head": 1, "memory_to_active": 0.011841453611850739, "contribution_to_active": 0.5919487476348877, "mix_mean": 0.5652061104774475, "mix_low": 0.001800537109375, "mix_high": 0.005138397216796875, "read_cosine": 0.23060044646263123, "beta_mean": 4.936008554068394e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006436214782297611, "mix_std": 0.2760264277458191, "beta_std": 9.471759403822944e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008397283032536507, "evidence_weights": [-0.17937065660953522, -0.2623614966869354, 0.5761846899986267], "agreement_weight": 0.4787401258945465, "orthogonal_weight": -1.3552099466323853}, {"head": 2, "memory_to_active": 0.06495977193117142, "contribution_to_active": 0.47692421078681946, "mix_mean": 0.3899654150009155, "mix_low": 0.0009584426879882812, "mix_high": 0.00018024444580078125, "read_cosine": 0.5625430345535278, "beta_mean": 0.007833684794604778, "nonfinite": 0.0, "weighted_memory_to_active": 0.029140722006559372, "mix_std": 0.23263448476791382, "beta_std": 0.028923368081450462, "beta_low": 0.86798095703125, "beta_high": 0.0, "orthogonal_to_active": 0.017316700890660286, "evidence_weights": [-0.1835048496723175, -0.2087595909833908, 0.40990763902664185], "agreement_weight": 0.005325080826878548, "orthogonal_weight": -0.8416543006896973}, {"head": 3, "memory_to_active": 0.149295836687088, "contribution_to_active": 0.560501217842102, "mix_mean": 0.65559983253479, "mix_low": 0.024298667907714844, "mix_high": 0.03287315368652344, "read_cosine": -0.03101988323032856, "beta_mean": 0.0007847095257602632, "nonfinite": 0.0, "weighted_memory_to_active": 0.11016934365034103, "mix_std": 0.28852179646492004, "beta_std": 0.007932553067803383, "beta_low": 0.994873046875, "beta_high": 0.0, "orthogonal_to_active": 0.09973421692848206, "evidence_weights": [-0.31704261898994446, -0.24565190076828003, 1.5855376720428467], "agreement_weight": 6.208828449249268, "orthogonal_weight": -0.9777094721794128}, {"head": 4, "memory_to_active": 0.39859703183174133, "contribution_to_active": 0.26931527256965637, "mix_mean": 0.36624595522880554, "mix_low": 0.004622459411621094, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8864378333091736, "beta_mean": 0.08091557025909424, "nonfinite": 0.0, "weighted_memory_to_active": 0.10791183263063431, "mix_std": 0.22059939801692963, "beta_std": 0.05397554486989975, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.14972054958343506, "evidence_weights": [-0.07952458411455154, -0.23156173527240753, 0.45147308707237244], "agreement_weight": 0.00836645346134901, "orthogonal_weight": 2.247739553451538}, {"head": 5, "memory_to_active": 0.05006401613354683, "contribution_to_active": 0.7064393162727356, "mix_mean": 0.6561710238456726, "mix_low": 0.002452850341796875, "mix_high": 0.012087821960449219, "read_cosine": 0.3298211693763733, "beta_mean": 0.00024375366047024727, "nonfinite": 0.0, "weighted_memory_to_active": 0.03488408029079437, "mix_std": 0.27295511960983276, "beta_std": 0.001707804505713284, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.028957147151231766, "evidence_weights": [-0.2203715294599533, -0.296650767326355, 0.8597010970115662], "agreement_weight": 0.192122682929039, "orthogonal_weight": -0.9199536442756653}]}, {"layer": 14, "memory_to_active": 0.2322453111410141, "contribution_to_active": 0.4182344675064087, "mix_mean": 0.42626693844795227, "mix_low": 0.027252197265625, "mix_high": 0.006614526268094778, "read_cosine": 0.6137174367904663, "beta_mean": 0.06245020031929016, "nonfinite": 0.0, "weighted_memory_to_active": 0.09680242836475372, "mix_std": 0.2432178258895874, "beta_std": 0.05930888280272484, "beta_low": 0.4901530146598816, "beta_high": 0.0, "orthogonal_to_active": 0.043217163532972336, "output_influence": 0.3624594211578369, "residual_influence": 0.1456323266029358, "state_active": 0.05636202171444893, "state_memory": 0.015361395664513111, "state_peak": 45.51377487182617, "growth_active": 1.0923113822937012, "growth_memory": 1.3055479526519775, "heads": [{"head": 0, "memory_to_active": 0.36678481101989746, "contribution_to_active": 0.1870179921388626, "mix_mean": 0.19215333461761475, "mix_low": 0.019334793090820312, "mix_high": 0.0, "read_cosine": 0.8117462992668152, "beta_mean": 0.07054341584444046, "nonfinite": 0.0, "weighted_memory_to_active": 0.0774143785238266, "mix_std": 0.1778046190738678, "beta_std": 0.05975537747144699, "beta_low": 0.00604248046875, "beta_high": 0.0, "orthogonal_to_active": 0.08594392240047455, "evidence_weights": [-0.1020335927605629, -0.04488092288374901, 0.40609318017959595], "agreement_weight": 0.021821722388267517, "orthogonal_weight": 1.77372145652771}, {"head": 1, "memory_to_active": 0.48666903376579285, "contribution_to_active": 0.2652363181114197, "mix_mean": 0.20241209864616394, "mix_low": 0.1278543472290039, "mix_high": 1.049041748046875e-05, "read_cosine": 0.8737340569496155, "beta_mean": 0.2578161656856537, "nonfinite": 0.0, "weighted_memory_to_active": 0.2100435495376587, "mix_std": 0.23374514281749725, "beta_std": 0.25116679072380066, "beta_low": 0.0888671875, "beta_high": 0.0, "orthogonal_to_active": 0.013925297185778618, "evidence_weights": [0.0013977977214381099, -0.1935265213251114, -0.09626452624797821], "agreement_weight": 0.05803170055150986, "orthogonal_weight": 0.20160260796546936}, {"head": 2, "memory_to_active": 0.1408304125070572, "contribution_to_active": 0.5791108012199402, "mix_mean": 0.588506281375885, "mix_low": 0.0027265548706054688, "mix_high": 0.004956245422363281, "read_cosine": 0.39970308542251587, "beta_mean": 0.0026102415286004543, "nonfinite": 0.0, "weighted_memory_to_active": 0.09277348965406418, "mix_std": 0.2681804895401001, "beta_std": 0.0036653061397373676, "beta_low": 0.95703125, "beta_high": 0.0, "orthogonal_to_active": 0.07202678918838501, "evidence_weights": [-0.13448964059352875, -0.31607988476753235, 0.9313915371894836], "agreement_weight": 0.08958546072244644, "orthogonal_weight": -0.8796183466911316}, {"head": 3, "memory_to_active": 0.0553390234708786, "contribution_to_active": 0.625064492225647, "mix_mean": 0.6738348603248596, "mix_low": 0.004909515380859375, "mix_high": 0.023512840270996094, "read_cosine": -0.06106700748205185, "beta_mean": 0.006204530596733093, "nonfinite": 0.0, "weighted_memory_to_active": 0.03530701994895935, "mix_std": 0.26073694229125977, "beta_std": 0.009215001948177814, "beta_low": 0.82501220703125, "beta_high": 0.0, "orthogonal_to_active": 0.026306383311748505, "evidence_weights": [-0.3444984257221222, -0.31146439909935, 0.2890975773334503], "agreement_weight": 0.06905671209096909, "orthogonal_weight": -0.8254392147064209}, {"head": 4, "memory_to_active": 0.14584499597549438, "contribution_to_active": 0.5109437704086304, "mix_mean": 0.5189341306686401, "mix_low": 0.004347801208496094, "mix_high": 0.0097198486328125, "read_cosine": 0.9028054475784302, "beta_mean": 0.03629830852150917, "nonfinite": 0.0, "weighted_memory_to_active": 0.08665364980697632, "mix_std": 0.2640562355518341, "beta_std": 0.025977127254009247, "beta_low": 0.08380126953125, "beta_high": 0.0, "orthogonal_to_active": 0.02281440980732441, "evidence_weights": [-0.21332652866840363, -0.2398151308298111, 0.4312461018562317], "agreement_weight": 0.13262046873569489, "orthogonal_weight": -0.6172297596931458}, {"head": 5, "memory_to_active": 0.19800353050231934, "contribution_to_active": 0.3420332670211792, "mix_mean": 0.381760835647583, "mix_low": 0.004340171813964844, "mix_high": 0.00148773193359375, "read_cosine": 0.755382776260376, "beta_mean": 0.001228545093908906, "nonfinite": 0.0, "weighted_memory_to_active": 0.07862246781587601, "mix_std": 0.25478339195251465, "beta_std": 0.006073693744838238, "beta_low": 0.98016357421875, "beta_high": 0.0, "orthogonal_to_active": 0.03828618675470352, "evidence_weights": [-0.1733018159866333, -0.17618349194526672, 0.34026268124580383], "agreement_weight": 0.07251429557800293, "orthogonal_weight": -0.7557107210159302}]}, {"layer": 15, "memory_to_active": 0.05720413476228714, "contribution_to_active": 0.4889758229255676, "mix_mean": 0.43110164999961853, "mix_low": 0.007547855842858553, "mix_high": 0.002937793731689453, "read_cosine": 0.6578879356384277, "beta_mean": 0.04087595269083977, "nonfinite": 0.0, "weighted_memory_to_active": 0.02799767255783081, "mix_std": 0.2848539352416992, "beta_std": 0.046911999583244324, "beta_low": 0.7037456035614014, "beta_high": 0.0, "orthogonal_to_active": 0.008630702272057533, "output_influence": 0.30699586868286133, "residual_influence": 0.1760333627462387, "state_active": 0.14978301525115967, "state_memory": 0.010977930389344692, "state_peak": 24.668041229248047, "growth_active": 1.1624767780303955, "growth_memory": 1.310222864151001, "heads": [{"head": 0, "memory_to_active": 0.1037098616361618, "contribution_to_active": 0.4841327369213104, "mix_mean": 0.3893163204193115, "mix_low": 0.00885772705078125, "mix_high": 0.004962921142578125, "read_cosine": 0.5900150537490845, "beta_mean": 0.043427035212516785, "nonfinite": 0.0, "weighted_memory_to_active": 0.048379819840192795, "mix_std": 0.286775141954422, "beta_std": 0.03298209607601166, "beta_low": 0.0743408203125, "beta_high": 0.0, "orthogonal_to_active": 0.028061633929610252, "evidence_weights": [-0.18751463294029236, -0.1889047920703888, 0.47460275888442993], "agreement_weight": 0.09341558068990707, "orthogonal_weight": -0.7974953651428223}, {"head": 1, "memory_to_active": 0.12497871369123459, "contribution_to_active": 0.42644912004470825, "mix_mean": 0.4405609965324402, "mix_low": 0.0015811920166015625, "mix_high": 0.0067844390869140625, "read_cosine": 0.9280216097831726, "beta_mean": 0.0006084693013690412, "nonfinite": 0.0, "weighted_memory_to_active": 0.054206836968660355, "mix_std": 0.2881888747215271, "beta_std": 0.0024644616059958935, "beta_low": 0.994873046875, "beta_high": 0.0, "orthogonal_to_active": 0.0022297841496765614, "evidence_weights": [-0.14907711744308472, -0.15687797963619232, 0.1712503433227539], "agreement_weight": 0.1263481229543686, "orthogonal_weight": 0.08699366450309753}, {"head": 2, "memory_to_active": 0.005295667797327042, "contribution_to_active": 0.485723078250885, "mix_mean": 0.35591819882392883, "mix_low": 0.011774063110351562, "mix_high": 6.67572021484375e-05, "read_cosine": 0.7765713930130005, "beta_mean": 6.941275933058932e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002669355133548379, "mix_std": 0.2707791328430176, "beta_std": 0.00014265837671700865, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008745499071665108, "evidence_weights": [-0.20622120797634125, -0.26280757784843445, 0.09415760636329651], "agreement_weight": 0.006737266201525927, "orthogonal_weight": 0.8400253653526306}, {"head": 3, "memory_to_active": 0.0013782975729554892, "contribution_to_active": 0.3770522177219391, "mix_mean": 0.4173198342323303, "mix_low": 0.0045642852783203125, "mix_high": 0.00038433074951171875, "read_cosine": 0.6028836965560913, "beta_mean": 7.712823389738332e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00042675575241446495, "mix_std": 0.28596368432044983, "beta_std": 1.6705533198546618e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0002887483569793403, "evidence_weights": [-0.27194470167160034, -0.24202734231948853, 0.2535306513309479], "agreement_weight": 0.05238614231348038, "orthogonal_weight": -0.6524530053138733}, {"head": 4, "memory_to_active": 0.07734240591526031, "contribution_to_active": 0.6193370819091797, "mix_mean": 0.3954806327819824, "mix_low": 0.009163856506347656, "mix_high": 0.0020799636840820312, "read_cosine": 0.7259500026702881, "beta_mean": 0.1979173719882965, "nonfinite": 0.0, "weighted_memory_to_active": 0.04205745458602905, "mix_std": 0.30878496170043945, "beta_std": 0.22375771403312683, "beta_low": 0.186767578125, "beta_high": 0.0, "orthogonal_to_active": 0.007476578000932932, "evidence_weights": [-0.2062472403049469, -0.2532033920288086, -0.0037039327435195446], "agreement_weight": 0.012167048640549183, "orthogonal_weight": 0.3692573010921478}, {"head": 5, "memory_to_active": 0.03051983192563057, "contribution_to_active": 0.5411607027053833, "mix_mean": 0.588013768196106, "mix_low": 0.00934600830078125, "mix_high": 0.0033483505249023438, "read_cosine": 0.32388564944267273, "beta_mean": 0.003225714899599552, "nonfinite": 0.0, "weighted_memory_to_active": 0.02024579420685768, "mix_std": 0.2686316967010498, "beta_std": 0.022108370438218117, "beta_low": 0.96649169921875, "beta_high": 0.0, "orthogonal_to_active": 0.012852920219302177, "evidence_weights": [-0.2816702723503113, -0.3019641041755676, 0.6728771328926086], "agreement_weight": 0.3111267685890198, "orthogonal_weight": 0.8215692043304443}]}]}, {"step": 62000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8068465790001937, "memory_to_active": 0.8009650111198425, "contribution_to_active": 0.5614340901374817, "mix_mean": 0.41990604996681213, "mix_low": 0.1296272873878479, "mix_high": 0.010589202865958214, "read_cosine": 0.3470126986503601, "beta_mean": 0.07056979835033417, "nonfinite": 0.0, "weighted_memory_to_active": 0.20099154114723206, "mix_std": 0.2230096161365509, "beta_std": 0.06132107973098755, "beta_low": 0.5772368311882019, "beta_high": 3.0517578125e-05, "orthogonal_to_active": 0.06262612342834473, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0886805057525635, "normal_token_accuracy": 0.4371523857116699, "state_active": 0.06548129767179489, "state_memory": 0.01752951741218567, "state_peak": 125.22693634033203, "growth_active": 1.128629207611084, "growth_memory": 1.3096227645874023, "output_influence": 0.4332836866378784, "residual_influence": 0.15066276490688324, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.294126510620117, "no_memory_loss_delta": 0.5802415013313293, "scrambled_loss_delta": 0.14299774169921875, "memory_only_loss_delta": 3.3676815032958984, "no_evidence_loss_delta": 0.3463321030139923, "no_agreement_loss_delta": 0.021115249022841454, "no_orthogonal_loss_delta": 0.002256734762340784}, {"index": 94, "tokens": 256, "normal_loss": 3.1549742221832275, "no_memory_loss_delta": 0.5879747867584229, "scrambled_loss_delta": 0.39979851245880127, "memory_only_loss_delta": 4.396677017211914, "no_evidence_loss_delta": 0.23270882666110992, "no_agreement_loss_delta": 0.028563648462295532, "no_orthogonal_loss_delta": 0.05669272318482399}, {"index": 188, "tokens": 256, "normal_loss": 3.347353219985962, "no_memory_loss_delta": 0.8252811431884766, "scrambled_loss_delta": 0.13706868886947632, "memory_only_loss_delta": 3.137713670730591, "no_evidence_loss_delta": 0.4427514672279358, "no_agreement_loss_delta": 0.053276851773262024, "no_orthogonal_loss_delta": 0.05716872960329056}, {"index": 283, "tokens": 131, "normal_loss": 2.052150011062622, "no_memory_loss_delta": 0.2530548870563507, "scrambled_loss_delta": 0.20763225853443146, "memory_only_loss_delta": 2.7637739181518555, "no_evidence_loss_delta": 0.037388984113931656, "no_agreement_loss_delta": -0.010308131575584412, "no_orthogonal_loss_delta": -0.02622907981276512}], "agreement_weight_abs_mean": 0.2319037765264511, "agreement_weight_abs_max": 6.273057460784912, "agreement_weight_mean": 0.201689675450325, "orthogonal_weight_abs_mean": 0.9028237462043762, "orthogonal_weight_abs_max": 10.115826606750488, "orthogonal_weight_mean": -0.09109761565923691, "no_memory_loss": 3.6932249069213867, "no_memory_loss_delta": 0.6045445203781128, "no_memory_token_accuracy_gain": 0.06562847644090652, "no_memory_token_win_fraction": 0.6573970913887024, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 45.367679093356074, "scrambled_loss": 3.312535285949707, "scrambled_loss_delta": 0.22385457158088684, "scrambled_token_accuracy_gain": 0.007786429487168789, "scrambled_token_win_fraction": 0.6129032373428345, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.05686140413328, "memory_only_loss": 6.595893859863281, "memory_only_loss_delta": 3.5072133541107178, "memory_only_token_accuracy_gain": 0.27808675169944763, "memory_only_token_win_fraction": 0.9043381810188293, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 97.00196572607757, "no_evidence_loss": 3.3850955963134766, "no_evidence_loss_delta": 0.29641470313072205, "no_evidence_token_accuracy_gain": 0.025583982467651367, "no_evidence_token_win_fraction": 0.6117908954620361, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 25.65209590058598, "no_agreement_loss": 3.1164965629577637, "no_agreement_loss_delta": 0.027815690264105797, "no_agreement_token_accuracy_gain": 0.0, "no_agreement_token_win_fraction": 0.486095666885376, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.7432396038611944, "no_orthogonal_loss": 3.117924451828003, "no_orthogonal_loss_delta": 0.02924387902021408, "no_orthogonal_token_accuracy_gain": -0.006674082484096289, "no_orthogonal_token_win_fraction": 0.5862069129943848, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.882041474286029, "short_no_memory_loss_delta": 0.5641559362411499, "long_context_memory_gain": 0.04038858413696289, "normal_context_loss_gain": 0.1264738142490387, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.483150005340576, "contribution_to_active": 1.1685636043548584, "mix_mean": 0.10619065165519714, "mix_low": 0.5360366702079773, "mix_high": 0.00543085765093565, "read_cosine": 0.40551576018333435, "beta_mean": 0.2113036811351776, "nonfinite": 0.0, "weighted_memory_to_active": 1.1967394351959229, "mix_std": 0.09022441506385803, "beta_std": 0.13979384303092957, "beta_low": 0.1239725798368454, "beta_high": 0.0, "orthogonal_to_active": 0.04536470025777817, "output_influence": 0.22107955813407898, "residual_influence": 0.2500464916229248, "state_active": 0.06863990426063538, "state_memory": 0.04472535848617554, "state_peak": 47.40635681152344, "growth_active": 1.275808334350586, "growth_memory": 1.2724720239639282, "heads": [{"head": 0, "memory_to_active": 2.631747245788574, "contribution_to_active": 0.5030995607376099, "mix_mean": 0.020006243139505386, "mix_low": 0.6385860443115234, "mix_high": 9.5367431640625e-07, "read_cosine": 0.199092835187912, "beta_mean": 0.1898345798254013, "nonfinite": 0.0, "weighted_memory_to_active": 0.628882646560669, "mix_std": 0.06262005120515823, "beta_std": 0.10517877340316772, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0697450190782547, "evidence_weights": [-0.0031563316006213427, -0.04361966997385025, -0.053273603320121765], "agreement_weight": -0.21593321859836578, "orthogonal_weight": -0.1256130188703537}, {"head": 1, "memory_to_active": 6.597177028656006, "contribution_to_active": 2.915055513381958, "mix_mean": 0.019166622310876846, "mix_low": 0.6268720626831055, "mix_high": 0.0, "read_cosine": 0.41643276810646057, "beta_mean": 0.4522393047809601, "nonfinite": 0.0, "weighted_memory_to_active": 3.173654079437256, "mix_std": 0.05568116158246994, "beta_std": 0.22731801867485046, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.031064342707395554, "evidence_weights": [0.019828826189041138, -0.10818853974342346, 0.6505784392356873], "agreement_weight": 0.026684779673814774, "orthogonal_weight": 0.020822161808609962}, {"head": 2, "memory_to_active": 2.6708290576934814, "contribution_to_active": 0.4954478144645691, "mix_mean": 0.013949595391750336, "mix_low": 0.6860446929931641, "mix_high": 0.0, "read_cosine": 0.2700411081314087, "beta_mean": 0.2575392723083496, "nonfinite": 0.0, "weighted_memory_to_active": 0.5778590440750122, "mix_std": 0.03185339272022247, "beta_std": 0.17851312458515167, "beta_low": 0.00482177734375, "beta_high": 0.0, "orthogonal_to_active": 0.04766540974378586, "evidence_weights": [0.069633848965168, 0.059417374432086945, 0.20981140434741974], "agreement_weight": 0.00951544288545847, "orthogonal_weight": -0.2033480852842331}, {"head": 3, "memory_to_active": 7.375619888305664, "contribution_to_active": 2.5733695030212402, "mix_mean": 0.029597051441669464, "mix_low": 0.49424266815185547, "mix_high": 0.0, "read_cosine": 0.3001672327518463, "beta_mean": 0.313835084438324, "nonfinite": 0.0, "weighted_memory_to_active": 2.70782208442688, "mix_std": 0.06879839301109314, "beta_std": 0.24785450100898743, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.10770972073078156, "evidence_weights": [-0.036536309868097305, -0.03428836539387703, -0.12698408961296082], "agreement_weight": -0.010595076717436314, "orthogonal_weight": -0.047025587409734726}, {"head": 4, "memory_to_active": 1.5229313373565674, "contribution_to_active": 0.052651047706604004, "mix_mean": 0.011592707596719265, "mix_low": 0.7554855346679688, "mix_high": 0.0, "read_cosine": 0.3063163161277771, "beta_mean": 0.036956869065761566, "nonfinite": 0.0, "weighted_memory_to_active": 0.039041608572006226, "mix_std": 0.03632577508687973, "beta_std": 0.0641590878367424, "beta_low": 0.34869384765625, "beta_high": 0.0, "orthogonal_to_active": 0.0006679068319499493, "evidence_weights": [0.1336199939250946, 0.09714783728122711, -0.12827014923095703], "agreement_weight": -0.18192416429519653, "orthogonal_weight": 0.015353815630078316}, {"head": 5, "memory_to_active": 0.10059257596731186, "contribution_to_active": 0.47175806760787964, "mix_mean": 0.5428316593170166, "mix_low": 0.014988899230957031, "mix_high": 0.032584190368652344, "read_cosine": 0.941044270992279, "beta_mean": 0.01741696707904339, "nonfinite": 0.0, "weighted_memory_to_active": 0.05317680165171623, "mix_std": 0.2860676348209381, "beta_std": 0.015739498659968376, "beta_low": 0.3902587890625, "beta_high": 0.0, "orthogonal_to_active": 0.015335761941969395, "evidence_weights": [-0.17348408699035645, -0.2177736461162567, 0.38800594210624695], "agreement_weight": 0.29044291377067566, "orthogonal_weight": -0.6883229613304138}]}, {"layer": 1, "memory_to_active": 2.391414165496826, "contribution_to_active": 0.46600341796875, "mix_mean": 0.21434465050697327, "mix_low": 0.36738842725753784, "mix_high": 0.0012294452171772718, "read_cosine": 0.3012228012084961, "beta_mean": 0.18604181706905365, "nonfinite": 0.0, "weighted_memory_to_active": 0.3017050623893738, "mix_std": 0.11516296863555908, "beta_std": 0.1339988112449646, "beta_low": 0.3519287109375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.04323996976017952, "output_influence": 0.25471338629722595, "residual_influence": 0.06922758370637894, "state_active": 0.027073903009295464, "state_memory": 0.035715941339731216, "state_peak": 42.16856384277344, "growth_active": 1.1377739906311035, "growth_memory": 1.454725980758667, "heads": [{"head": 0, "memory_to_active": 0.027086948975920677, "contribution_to_active": 0.4939085841178894, "mix_mean": 0.5285159945487976, "mix_low": 0.0049724578857421875, "mix_high": 0.0026102066040039062, "read_cosine": -0.011300770565867424, "beta_mean": 0.00019031846022699028, "nonfinite": 0.0, "weighted_memory_to_active": 0.015633895993232727, "mix_std": 0.2814081907272339, "beta_std": 0.0002618725993670523, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014940825290977955, "evidence_weights": [-0.20850682258605957, -0.27310529351234436, 0.2943556606769562], "agreement_weight": 0.19228105247020721, "orthogonal_weight": -1.01862370967865}, {"head": 1, "memory_to_active": 2.8344850540161133, "contribution_to_active": 0.24177956581115723, "mix_mean": 0.023130573332309723, "mix_low": 0.4743776321411133, "mix_high": 3.814697265625e-06, "read_cosine": 0.5225249528884888, "beta_mean": 0.2684972882270813, "nonfinite": 0.0, "weighted_memory_to_active": 0.26650938391685486, "mix_std": 0.04657948389649391, "beta_std": 0.19525116682052612, "beta_low": 0.00244140625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.01972588151693344, "evidence_weights": [0.02005636878311634, -0.066041499376297, 0.06351814419031143], "agreement_weight": -0.004804946016520262, "orthogonal_weight": 0.19715645909309387}, {"head": 2, "memory_to_active": 0.049026008695364, "contribution_to_active": 0.5634447932243347, "mix_mean": 0.6707487106323242, "mix_low": 0.0004138946533203125, "mix_high": 0.0047626495361328125, "read_cosine": 0.5187231302261353, "beta_mean": 0.0017336637247353792, "nonfinite": 0.0, "weighted_memory_to_active": 0.03295920416712761, "mix_std": 0.22953253984451294, "beta_std": 0.0041200146079063416, "beta_low": 0.9783935546875, "beta_high": 0.0, "orthogonal_to_active": 0.02144421637058258, "evidence_weights": [-0.2777709364891052, -0.3711073696613312, 0.667439341545105], "agreement_weight": 0.04094734042882919, "orthogonal_weight": -0.7989346385002136}, {"head": 3, "memory_to_active": 2.979971408843994, "contribution_to_active": 0.18167269229888916, "mix_mean": 0.023470256477594376, "mix_low": 0.5454492568969727, "mix_high": 0.0, "read_cosine": 0.18362818658351898, "beta_mean": 0.33187854290008545, "nonfinite": 0.0, "weighted_memory_to_active": 0.2567691504955292, "mix_std": 0.059790804982185364, "beta_std": 0.26256081461906433, "beta_low": 0.0650634765625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.10887458175420761, "evidence_weights": [0.039260439574718475, -0.017928756773471832, -0.024409012869000435], "agreement_weight": -0.03446386009454727, "orthogonal_weight": -0.4975363612174988}, {"head": 4, "memory_to_active": 3.894728183746338, "contribution_to_active": 1.205309510231018, "mix_mean": 0.03415602445602417, "mix_low": 0.3139228820800781, "mix_high": 0.0, "read_cosine": 0.33142906427383423, "beta_mean": 0.17336753010749817, "nonfinite": 0.0, "weighted_memory_to_active": 1.1607054471969604, "mix_std": 0.05639101192355156, "beta_std": 0.12373314797878265, "beta_low": 0.03997802734375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.06669969856739044, "evidence_weights": [-0.028910066932439804, -0.006083079148083925, 0.12182490527629852], "agreement_weight": -0.014378570020198822, "orthogonal_weight": 0.08275405317544937}, {"head": 5, "memory_to_active": 4.5631866455078125, "contribution_to_active": 0.10990528762340546, "mix_mean": 0.006046219728887081, "mix_low": 0.8651943206787109, "mix_high": 0.0, "read_cosine": 0.2623322606086731, "beta_mean": 0.3405834436416626, "nonfinite": 0.0, "weighted_memory_to_active": 0.07765308022499084, "mix_std": 0.01727576181292534, "beta_std": 0.21806572377681732, "beta_low": 0.02569580078125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.027754588052630424, "evidence_weights": [0.1662297397851944, 0.08773063868284225, 0.08619478344917297], "agreement_weight": -0.12674623727798462, "orthogonal_weight": 0.3713507652282715}]}, {"layer": 2, "memory_to_active": 0.9973422884941101, "contribution_to_active": 0.35018184781074524, "mix_mean": 0.4078187644481659, "mix_low": 0.2532787322998047, "mix_high": 0.004843711853027344, "read_cosine": 0.3104366660118103, "beta_mean": 0.08585056662559509, "nonfinite": 0.0, "weighted_memory_to_active": 0.07246122509241104, "mix_std": 0.1874336451292038, "beta_std": 0.0694686770439148, "beta_low": 0.5888163447380066, "beta_high": 0.00011189778160769492, "orthogonal_to_active": 0.017515674233436584, "output_influence": 0.39327096939086914, "residual_influence": 0.10741022229194641, "state_active": 0.03976809233427048, "state_memory": 0.014354519546031952, "state_peak": 31.86446762084961, "growth_active": 1.0550568103790283, "growth_memory": 1.5774309635162354, "heads": [{"head": 0, "memory_to_active": 0.008667581714689732, "contribution_to_active": 0.35228604078292847, "mix_mean": 0.6356388330459595, "mix_low": 0.0005083084106445312, "mix_high": 0.009225845336914062, "read_cosine": 0.14261099696159363, "beta_mean": 6.557501183124259e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005718502681702375, "mix_std": 0.26789116859436035, "beta_std": 0.00011044740676879883, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0059612831100821495, "evidence_weights": [-0.37103918194770813, -0.2163042277097702, 0.1982298493385315], "agreement_weight": 0.13934797048568726, "orthogonal_weight": -1.0869966745376587}, {"head": 1, "memory_to_active": 3.158456325531006, "contribution_to_active": 0.04016207158565521, "mix_mean": 0.004887146409600973, "mix_low": 0.8963050842285156, "mix_high": 0.0, "read_cosine": 0.13268502056598663, "beta_mean": 0.25305718183517456, "nonfinite": 0.0, "weighted_memory_to_active": 0.031990841031074524, "mix_std": 0.010972472839057446, "beta_std": 0.1484336107969284, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.00932365469634533, "evidence_weights": [0.14344832301139832, 0.1192496195435524, 0.5715340375900269], "agreement_weight": 0.0357234850525856, "orthogonal_weight": 0.3180694580078125}, {"head": 2, "memory_to_active": 0.009010330773890018, "contribution_to_active": 0.4120751917362213, "mix_mean": 0.5530072450637817, "mix_low": 0.0010929107666015625, "mix_high": 0.0040416717529296875, "read_cosine": 0.35311007499694824, "beta_mean": 2.3993034119484946e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035951577592641115, "mix_std": 0.27670782804489136, "beta_std": 4.637742677005008e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001780268648872152, "evidence_weights": [-0.2531098425388336, -0.29307007789611816, 0.5872600674629211], "agreement_weight": -0.040008820593357086, "orthogonal_weight": -0.05859605595469475}, {"head": 3, "memory_to_active": 0.11016713827848434, "contribution_to_active": 0.5379366874694824, "mix_mean": 0.5760083198547363, "mix_low": 0.0002231597900390625, "mix_high": 0.006039619445800781, "read_cosine": 0.6479740738868713, "beta_mean": 0.020108520984649658, "nonfinite": 0.0, "weighted_memory_to_active": 0.07003764063119888, "mix_std": 0.2573801875114441, "beta_std": 0.027447273954749107, "beta_low": 0.48486328125, "beta_high": 0.0, "orthogonal_to_active": 0.04589737206697464, "evidence_weights": [-0.20412321388721466, -0.3229087293148041, 0.7685887813568115], "agreement_weight": 0.04934908077120781, "orthogonal_weight": -0.8929721117019653}, {"head": 4, "memory_to_active": 2.6890745162963867, "contribution_to_active": 0.23070451617240906, "mix_mean": 0.018634207546710968, "mix_low": 0.6208391189575195, "mix_high": 0.0, "read_cosine": 0.21112051606178284, "beta_mean": 0.24181610345840454, "nonfinite": 0.0, "weighted_memory_to_active": 0.31866419315338135, "mix_std": 0.049968454986810684, "beta_std": 0.2407306283712387, "beta_low": 0.04730224609375, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.04254931956529617, "evidence_weights": [0.03101278282701969, 0.01164513360708952, 0.022558510303497314], "agreement_weight": 0.02826029434800148, "orthogonal_weight": -0.15592920780181885}, {"head": 5, "memory_to_active": 0.00867762416601181, "contribution_to_active": 0.5279265642166138, "mix_mean": 0.6587365865707397, "mix_low": 0.0007038116455078125, "mix_high": 0.009755134582519531, "read_cosine": 0.37511923909187317, "beta_mean": 3.1931402190821245e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00476099131628871, "mix_std": 0.2616816759109497, "beta_std": 4.372732655610889e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011843852698802948, "evidence_weights": [-0.28310883045196533, -0.29252371191978455, 0.9312155246734619], "agreement_weight": 0.2341664731502533, "orthogonal_weight": -0.3041192591190338}]}, {"layer": 3, "memory_to_active": 0.4481813311576843, "contribution_to_active": 0.47909149527549744, "mix_mean": 0.5083730220794678, "mix_low": 0.052254676818847656, "mix_high": 0.010855675674974918, "read_cosine": 0.20165559649467468, "beta_mean": 0.0250176340341568, "nonfinite": 0.0, "weighted_memory_to_active": 0.13249488174915314, "mix_std": 0.25963228940963745, "beta_std": 0.024862021207809448, "beta_low": 0.7630106806755066, "beta_high": 0.0, "orthogonal_to_active": 0.05590581148862839, "output_influence": 0.4228968918323517, "residual_influence": 0.10049491375684738, "state_active": 0.025079237297177315, "state_memory": 0.007047818973660469, "state_peak": 23.214168548583984, "growth_active": 1.1089773178100586, "growth_memory": 1.7080186605453491, "heads": [{"head": 0, "memory_to_active": 0.3797187805175781, "contribution_to_active": 0.5921812057495117, "mix_mean": 0.5989726781845093, "mix_low": 0.0020198822021484375, "mix_high": 0.023390769958496094, "read_cosine": 0.2650929391384125, "beta_mean": 0.009858248755335808, "nonfinite": 0.0, "weighted_memory_to_active": 0.25050079822540283, "mix_std": 0.3039494454860687, "beta_std": 0.011774330399930477, "beta_low": 0.68475341796875, "beta_high": 0.0, "orthogonal_to_active": 0.17866352200508118, "evidence_weights": [-0.2155410200357437, -0.24905970692634583, 1.0122921466827393], "agreement_weight": 0.10993396490812302, "orthogonal_weight": -0.7611377835273743}, {"head": 1, "memory_to_active": 0.016912654042243958, "contribution_to_active": 0.4544674754142761, "mix_mean": 0.5710369348526001, "mix_low": 0.004256248474121094, "mix_high": 0.009653091430664062, "read_cosine": 0.17124734818935394, "beta_mean": 8.439362864010036e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010287734679877758, "mix_std": 0.29694896936416626, "beta_std": 0.00019304532906971872, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008706210181117058, "evidence_weights": [-0.2690598964691162, -0.21852029860019684, 0.7126055359840393], "agreement_weight": 0.5497708320617676, "orthogonal_weight": -0.9233185052871704}, {"head": 2, "memory_to_active": 2.017857551574707, "contribution_to_active": 0.28724172711372375, "mix_mean": 0.05561837553977966, "mix_low": 0.2963523864746094, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19983690977096558, "beta_mean": 0.13422852754592896, "nonfinite": 0.0, "weighted_memory_to_active": 0.3282007873058319, "mix_std": 0.10801556706428528, "beta_std": 0.12269999831914902, "beta_low": 0.02276611328125, "beta_high": 0.0, "orthogonal_to_active": 0.0026325946673750877, "evidence_weights": [-0.09673510491847992, -0.0029941026587039232, -0.0354054793715477], "agreement_weight": -0.009886872954666615, "orthogonal_weight": -0.010153825394809246}, {"head": 3, "memory_to_active": 0.01854964718222618, "contribution_to_active": 0.5175129771232605, "mix_mean": 0.5954941511154175, "mix_low": 0.0050182342529296875, "mix_high": 0.010230064392089844, "read_cosine": 0.28192877769470215, "beta_mean": 9.221458458341658e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01249354425817728, "mix_std": 0.29098984599113464, "beta_std": 0.000196669134311378, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003291334491223097, "evidence_weights": [-0.2568959593772888, -0.22425536811351776, 2.253464937210083], "agreement_weight": 2.1718697547912598, "orthogonal_weight": -0.340970516204834}, {"head": 4, "memory_to_active": 0.010527110658586025, "contribution_to_active": 0.36550313234329224, "mix_mean": 0.5025293231010437, "mix_low": 0.003975868225097656, "mix_high": 0.0016040802001953125, "read_cosine": 0.15419144928455353, "beta_mean": 5.683684867108241e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006716454401612282, "mix_std": 0.2897731363773346, "beta_std": 8.922535926103592e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006157705094665289, "evidence_weights": [-0.23038968443870544, -0.2194925993680954, 0.15079845488071442], "agreement_weight": -0.04146749526262283, "orthogonal_weight": -0.9993115663528442}, {"head": 5, "memory_to_active": 0.2455221265554428, "contribution_to_active": 0.6576424837112427, "mix_mean": 0.7265865802764893, "mix_low": 0.0019054412841796875, "mix_high": 0.020255088806152344, "read_cosine": 0.13763608038425446, "beta_mean": 0.005785591900348663, "nonfinite": 0.0, "weighted_memory_to_active": 0.1867700070142746, "mix_std": 0.26811689138412476, "beta_std": 0.014218856580555439, "beta_low": 0.87054443359375, "beta_high": 0.0, "orthogonal_to_active": 0.13598349690437317, "evidence_weights": [-0.2656063437461853, -0.3429078459739685, 0.5278512239456177], "agreement_weight": 0.21424978971481323, "orthogonal_weight": -0.7655652165412903}]}, {"layer": 4, "memory_to_active": 1.0240294933319092, "contribution_to_active": 0.4687633216381073, "mix_mean": 0.3665020763874054, "mix_low": 0.25423526763916016, "mix_high": 0.011213939636945724, "read_cosine": 0.30514174699783325, "beta_mean": 0.10951109975576401, "nonfinite": 0.0, "weighted_memory_to_active": 0.12817980349063873, "mix_std": 0.1610393077135086, "beta_std": 0.09996486455202103, "beta_low": 0.3923848569393158, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.0512918122112751, "output_influence": 0.48112308979034424, "residual_influence": 0.14626814424991608, "state_active": 0.03827137500047684, "state_memory": 0.017022818326950073, "state_peak": 36.738182067871094, "growth_active": 1.217183232307434, "growth_memory": 1.5171091556549072, "heads": [{"head": 0, "memory_to_active": 0.1726790964603424, "contribution_to_active": 0.7912428379058838, "mix_mean": 0.6766919493675232, "mix_low": 0.01389312744140625, "mix_high": 0.02963733673095703, "read_cosine": 0.4737928807735443, "beta_mean": 0.25344181060791016, "nonfinite": 0.0, "weighted_memory_to_active": 0.1478574126958847, "mix_std": 0.3500242531299591, "beta_std": 0.2519591152667999, "beta_low": 0.24798583984375, "beta_high": 0.0, "orthogonal_to_active": 0.0814187228679657, "evidence_weights": [-0.2572200298309326, -0.3652031719684601, -0.08800742775201797], "agreement_weight": 0.0009602567297406495, "orthogonal_weight": 0.6987759470939636}, {"head": 1, "memory_to_active": 0.04812158644199371, "contribution_to_active": 0.6573885679244995, "mix_mean": 0.6944499015808105, "mix_low": 0.00010395050048828125, "mix_high": 0.01787853240966797, "read_cosine": 0.5987653732299805, "beta_mean": 0.0009931004606187344, "nonfinite": 0.0, "weighted_memory_to_active": 0.03586575388908386, "mix_std": 0.24014659225940704, "beta_std": 0.00655642244964838, "beta_low": 0.97833251953125, "beta_high": 0.0, "orthogonal_to_active": 0.018881700932979584, "evidence_weights": [-0.2397785633802414, -0.3651050627231598, 0.6162346005439758], "agreement_weight": 0.3233196437358856, "orthogonal_weight": -0.7397717833518982}, {"head": 2, "memory_to_active": 1.6207269430160522, "contribution_to_active": 0.3320763111114502, "mix_mean": 0.037674956023693085, "mix_low": 0.4165000915527344, "mix_high": 2.384185791015625e-05, "read_cosine": 0.2163255512714386, "beta_mean": 0.05849388241767883, "nonfinite": 0.0, "weighted_memory_to_active": 0.25326624512672424, "mix_std": 0.07406014204025269, "beta_std": 0.06225411593914032, "beta_low": 0.10418701171875, "beta_high": 0.0, "orthogonal_to_active": 0.07909482717514038, "evidence_weights": [0.04420934617519379, -0.030103420838713646, 0.24837884306907654], "agreement_weight": 0.13291002810001373, "orthogonal_weight": -0.4167955815792084}, {"head": 3, "memory_to_active": 1.9616057872772217, "contribution_to_active": 0.13939110934734344, "mix_mean": 0.031879547983407974, "mix_low": 0.4303302764892578, "mix_high": 0.0, "read_cosine": 0.19711512327194214, "beta_mean": 0.13454508781433105, "nonfinite": 0.0, "weighted_memory_to_active": 0.19544723629951477, "mix_std": 0.05688901245594025, "beta_std": 0.12356886267662048, "beta_low": 0.0238037109375, "beta_high": 0.0, "orthogonal_to_active": 0.0015100513119250536, "evidence_weights": [-0.01833181641995907, 0.051226064562797546, -0.2616336941719055], "agreement_weight": 0.08092797547578812, "orthogonal_weight": -0.011339397169649601}, {"head": 4, "memory_to_active": 0.01114887185394764, "contribution_to_active": 0.6361950039863586, "mix_mean": 0.7454157471656799, "mix_low": 4.76837158203125e-05, "mix_high": 0.019739151000976562, "read_cosine": 0.13050588965415955, "beta_mean": 2.355865581193939e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007966131903231144, "mix_std": 0.21567684412002563, "beta_std": 7.533830648753792e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00494943605735898, "evidence_weights": [-0.33280694484710693, -0.32220539450645447, 0.3521023392677307], "agreement_weight": 0.34041285514831543, "orthogonal_weight": -0.6581348776817322}, {"head": 5, "memory_to_active": 2.329894542694092, "contribution_to_active": 0.2562859356403351, "mix_mean": 0.0129003394395113, "mix_low": 0.6645364761352539, "mix_high": 4.76837158203125e-06, "read_cosine": 0.21434558928012848, "beta_mean": 0.20956911146640778, "nonfinite": 0.0, "weighted_memory_to_active": 0.1286759376525879, "mix_std": 0.029438970610499382, "beta_std": 0.15537533164024353, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12189611792564392, "evidence_weights": [0.05683983489871025, -0.020480532199144363, -0.24033716320991516], "agreement_weight": -0.08161966502666473, "orthogonal_weight": 1.0389467477798462}]}, {"layer": 5, "memory_to_active": 0.4026961028575897, "contribution_to_active": 0.5321497917175293, "mix_mean": 0.5116369128227234, "mix_low": 0.056183818727731705, "mix_high": 0.019406160339713097, "read_cosine": 0.0786534920334816, "beta_mean": 0.05331294983625412, "nonfinite": 0.0, "weighted_memory_to_active": 0.14384540915489197, "mix_std": 0.25563839077949524, "beta_std": 0.06865198910236359, "beta_low": 0.66412353515625, "beta_high": 0.0, "orthogonal_to_active": 0.061217375099658966, "output_influence": 0.5528993606567383, "residual_influence": 0.17060703039169312, "state_active": 0.04077165201306343, "state_memory": 0.010256946086883545, "state_peak": 57.72201156616211, "growth_active": 1.102613091468811, "growth_memory": 1.2390897274017334, "heads": [{"head": 0, "memory_to_active": 0.23075461387634277, "contribution_to_active": 0.5355781316757202, "mix_mean": 0.46444088220596313, "mix_low": 0.011022567749023438, "mix_high": 0.001560211181640625, "read_cosine": -0.09257731586694717, "beta_mean": 0.020790785551071167, "nonfinite": 0.0, "weighted_memory_to_active": 0.08239778876304626, "mix_std": 0.27946314215660095, "beta_std": 0.03474069759249687, "beta_low": 0.5030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.03639882430434227, "evidence_weights": [-0.16533643007278442, -0.18501131236553192, 0.08215084671974182], "agreement_weight": -0.023784561082720757, "orthogonal_weight": -0.45059892535209656}, {"head": 1, "memory_to_active": 0.005370315629988909, "contribution_to_active": 0.5309364795684814, "mix_mean": 0.6903851628303528, "mix_low": 0.0009870529174804688, "mix_high": 0.03580760955810547, "read_cosine": 0.22589650750160217, "beta_mean": 1.309434537688503e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0032439278438687325, "mix_std": 0.272407591342926, "beta_std": 2.7070893338532187e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029682721942663193, "evidence_weights": [-0.24690532684326172, -0.2872186005115509, 0.9369932413101196], "agreement_weight": 0.6898766756057739, "orthogonal_weight": -0.9735475182533264}, {"head": 2, "memory_to_active": 0.016149790957570076, "contribution_to_active": 0.60639888048172, "mix_mean": 0.6641358137130737, "mix_low": 0.0023183822631835938, "mix_high": 0.02261829376220703, "read_cosine": -0.49429750442504883, "beta_mean": 9.688855789136142e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010275736451148987, "mix_std": 0.2632768750190735, "beta_std": 0.0002522428403608501, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00813947431743145, "evidence_weights": [-0.26434487104415894, -0.329607754945755, 0.14603471755981445], "agreement_weight": -0.11748246848583221, "orthogonal_weight": -0.9177836179733276}, {"head": 3, "memory_to_active": 1.5883972644805908, "contribution_to_active": 0.24256253242492676, "mix_mean": 0.1207573413848877, "mix_low": 0.11811542510986328, "mix_high": 0.0, "read_cosine": 0.1513388454914093, "beta_mean": 0.10282711684703827, "nonfinite": 0.0, "weighted_memory_to_active": 0.3977941870689392, "mix_std": 0.1543114185333252, "beta_std": 0.1562103033065796, "beta_low": 0.20428466796875, "beta_high": 0.0, "orthogonal_to_active": 0.15990807116031647, "evidence_weights": [-0.07867627590894699, -0.10317367315292358, 0.059741146862506866], "agreement_weight": 0.05583040043711662, "orthogonal_weight": -0.5075533986091614}, {"head": 4, "memory_to_active": 0.48205921053886414, "contribution_to_active": 0.5850998163223267, "mix_mean": 0.32435259222984314, "mix_low": 0.2028522491455078, "mix_high": 0.006789207458496094, "read_cosine": 0.5195176601409912, "beta_mean": 0.19586746394634247, "nonfinite": 0.0, "weighted_memory_to_active": 0.30521535873413086, "mix_std": 0.3416160047054291, "beta_std": 0.21921846270561218, "beta_low": 0.28106689453125, "beta_high": 0.0, "orthogonal_to_active": 0.1217658668756485, "evidence_weights": [-0.12020555883646011, -0.26096606254577637, -0.045997485518455505], "agreement_weight": 0.036062806844711304, "orthogonal_weight": 0.5700147747993469}, {"head": 5, "memory_to_active": 0.09344534575939178, "contribution_to_active": 0.6923223733901978, "mix_mean": 0.8057493567466736, "mix_low": 0.0018072128295898438, "mix_high": 0.04966163635253906, "read_cosine": 0.1620427519083023, "beta_mean": 0.0002823412651196122, "nonfinite": 0.0, "weighted_memory_to_active": 0.06414546072483063, "mix_std": 0.22275538742542267, "beta_std": 0.0014631911180913448, "beta_low": 0.99639892578125, "beta_high": 0.0, "orthogonal_to_active": 0.03812373802065849, "evidence_weights": [-0.3118439316749573, -0.33418217301368713, 1.8837403059005737], "agreement_weight": 0.5052761435508728, "orthogonal_weight": -0.6286781430244446}]}, {"layer": 6, "memory_to_active": 0.27562981843948364, "contribution_to_active": 0.630755603313446, "mix_mean": 0.5925217866897583, "mix_low": 0.04301516339182854, "mix_high": 0.03468068689107895, "read_cosine": 0.3198179304599762, "beta_mean": 0.06214972585439682, "nonfinite": 0.0, "weighted_memory_to_active": 0.14540119469165802, "mix_std": 0.23597848415374756, "beta_std": 0.07345651835203171, "beta_low": 0.5946248769760132, "beta_high": 0.0, "orthogonal_to_active": 0.05605345219373703, "output_influence": 0.5670901536941528, "residual_influence": 0.1924775242805481, "state_active": 0.04412265121936798, "state_memory": 0.007126758340746164, "state_peak": 97.76566314697266, "growth_active": 1.173221468925476, "growth_memory": 1.1814987659454346, "heads": [{"head": 0, "memory_to_active": 0.12276804447174072, "contribution_to_active": 0.7627307772636414, "mix_mean": 0.8301481604576111, "mix_low": 0.0002384185791015625, "mix_high": 0.05885028839111328, "read_cosine": 0.3083951771259308, "beta_mean": 0.011236376129090786, "nonfinite": 0.0, "weighted_memory_to_active": 0.10814828425645828, "mix_std": 0.19495955109596252, "beta_std": 0.02886485680937767, "beta_low": 0.794189453125, "beta_high": 0.0, "orthogonal_to_active": 0.07465296983718872, "evidence_weights": [-0.3134118616580963, -0.3537912666797638, 1.2264533042907715], "agreement_weight": 0.7017127871513367, "orthogonal_weight": -0.7606043815612793}, {"head": 1, "memory_to_active": 0.009588832966983318, "contribution_to_active": 0.5402085185050964, "mix_mean": 0.5951426029205322, "mix_low": 0.0007314682006835938, "mix_high": 0.015287399291992188, "read_cosine": 0.48462459444999695, "beta_mean": 2.2890380932949483e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004591119010001421, "mix_std": 0.281327486038208, "beta_std": 4.6518733142875135e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002449383959174156, "evidence_weights": [-0.19521205127239227, -0.30171236395835876, 1.0994679927825928], "agreement_weight": 0.1177486926317215, "orthogonal_weight": -0.7087955474853516}, {"head": 2, "memory_to_active": 0.7284777760505676, "contribution_to_active": 0.08968623727560043, "mix_mean": 0.0470154732465744, "mix_low": 0.24711990356445312, "mix_high": 1.9073486328125e-06, "read_cosine": 0.26644814014434814, "beta_mean": 0.02408621460199356, "nonfinite": 0.0, "weighted_memory_to_active": 0.06053643673658371, "mix_std": 0.07087311893701553, "beta_std": 0.028205158188939095, "beta_low": 0.2628173828125, "beta_high": 0.0, "orthogonal_to_active": 0.008398346602916718, "evidence_weights": [0.0681551843881607, -0.05399410426616669, 0.3437182605266571], "agreement_weight": 0.13338737189769745, "orthogonal_weight": -0.16499078273773193}, {"head": 3, "memory_to_active": 0.13149923086166382, "contribution_to_active": 0.6636627912521362, "mix_mean": 0.6494921445846558, "mix_low": 0.008930206298828125, "mix_high": 0.021226882934570312, "read_cosine": 0.45084238052368164, "beta_mean": 0.2073085457086563, "nonfinite": 0.0, "weighted_memory_to_active": 0.0900094136595726, "mix_std": 0.3273145258426666, "beta_std": 0.2397851198911667, "beta_low": 0.22039794921875, "beta_high": 0.0, "orthogonal_to_active": 0.05162743479013443, "evidence_weights": [-0.17098984122276306, -0.3224182426929474, 0.1624157577753067], "agreement_weight": -0.010271419771015644, "orthogonal_weight": 0.6471627354621887}, {"head": 4, "memory_to_active": 0.6350181698799133, "contribution_to_active": 1.0999867916107178, "mix_mean": 0.7501327991485596, "mix_low": 0.0008783340454101562, "mix_high": 0.10569286346435547, "read_cosine": 0.13017332553863525, "beta_mean": 0.13011373579502106, "nonfinite": 0.0, "weighted_memory_to_active": 0.5900147557258606, "mix_std": 0.2976846992969513, "beta_std": 0.14354249835014343, "beta_low": 0.2904052734375, "beta_high": 0.0, "orthogonal_to_active": 0.1804288625717163, "evidence_weights": [-0.20081976056098938, -0.553302526473999, -0.5126539468765259], "agreement_weight": -0.017470555379986763, "orthogonal_weight": 0.32811933755874634}, {"head": 5, "memory_to_active": 0.026426872238516808, "contribution_to_active": 0.628258466720581, "mix_mean": 0.6831992864608765, "mix_low": 0.0001926422119140625, "mix_high": 0.0070247650146484375, "read_cosine": 0.27842390537261963, "beta_mean": 0.0001305788173340261, "nonfinite": 0.0, "weighted_memory_to_active": 0.01910717412829399, "mix_std": 0.2437114417552948, "beta_std": 0.00029493687907233834, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01876368559896946, "evidence_weights": [-0.23649878799915314, -0.31661146879196167, 0.38403668999671936], "agreement_weight": 0.19935542345046997, "orthogonal_weight": -1.0516782999038696}]}, {"layer": 7, "memory_to_active": 1.7923510074615479, "contribution_to_active": 0.2606373429298401, "mix_mean": 0.1684369593858719, "mix_low": 0.24108046293258667, "mix_high": 0.0011065801372751594, "read_cosine": 0.2465299814939499, "beta_mean": 0.14167332649230957, "nonfinite": 0.0, "weighted_memory_to_active": 0.19824020564556122, "mix_std": 0.14741356670856476, "beta_std": 0.09826617687940598, "beta_low": 0.2074585109949112, "beta_high": 0.000244140625, "orthogonal_to_active": 0.08416535705327988, "output_influence": 0.2745911180973053, "residual_influence": 0.06577953696250916, "state_active": 0.013911073096096516, "state_memory": 0.024775022640824318, "state_peak": 34.353126525878906, "growth_active": 1.129883050918579, "growth_memory": 1.2959071397781372, "heads": [{"head": 0, "memory_to_active": 2.7430152893066406, "contribution_to_active": 0.17338958382606506, "mix_mean": 0.038798533380031586, "mix_low": 0.4051475524902344, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3395548164844513, "beta_mean": 0.17605674266815186, "nonfinite": 0.0, "weighted_memory_to_active": 0.2053101658821106, "mix_std": 0.08007080107927322, "beta_std": 0.13124500215053558, "beta_low": 0.00396728515625, "beta_high": 0.000244140625, "orthogonal_to_active": 0.000484140298794955, "evidence_weights": [0.03976527974009514, -0.08414093405008316, 0.009177815169095993], "agreement_weight": -0.028918564319610596, "orthogonal_weight": -0.0029349441174417734}, {"head": 1, "memory_to_active": 0.9531354904174805, "contribution_to_active": 0.3872930407524109, "mix_mean": 0.22045570611953735, "mix_low": 0.11900806427001953, "mix_high": 0.0059051513671875, "read_cosine": 0.397640585899353, "beta_mean": 0.09529025852680206, "nonfinite": 0.0, "weighted_memory_to_active": 0.3238431513309479, "mix_std": 0.24453844130039215, "beta_std": 0.06287753582000732, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04536958783864975, "evidence_weights": [-0.2175493985414505, -0.10418450832366943, -0.26970651745796204], "agreement_weight": 0.048535775393247604, "orthogonal_weight": 0.16764502227306366}, {"head": 2, "memory_to_active": 0.6435712575912476, "contribution_to_active": 0.3576292395591736, "mix_mean": 0.21569785475730896, "mix_low": 0.023598670959472656, "mix_high": 0.00028514862060546875, "read_cosine": 0.10763321816921234, "beta_mean": 0.041435107588768005, "nonfinite": 0.0, "weighted_memory_to_active": 0.22736281156539917, "mix_std": 0.20783653855323792, "beta_std": 0.040849268436431885, "beta_low": 0.11724853515625, "beta_high": 0.0, "orthogonal_to_active": 0.12700672447681427, "evidence_weights": [-0.08836202323436737, -0.17394372820854187, 0.8406829237937927], "agreement_weight": 0.023572873324155807, "orthogonal_weight": -0.5751303434371948}, {"head": 3, "memory_to_active": 4.772130489349365, "contribution_to_active": 0.11061308532953262, "mix_mean": 0.03321167454123497, "mix_low": 0.34784507751464844, "mix_high": 0.0, "read_cosine": 0.11272044479846954, "beta_mean": 0.46717098355293274, "nonfinite": 0.0, "weighted_memory_to_active": 0.35652679204940796, "mix_std": 0.05071073770523071, "beta_std": 0.28039973974227905, "beta_low": 0.001953125, "beta_high": 0.001220703125, "orthogonal_to_active": 0.3002737760543823, "evidence_weights": [0.07490666955709457, -0.034400198608636856, 0.723400890827179], "agreement_weight": 0.052233804017305374, "orthogonal_weight": -0.9357462525367737}, {"head": 4, "memory_to_active": 1.6025140285491943, "contribution_to_active": 0.08980122953653336, "mix_mean": 0.021627934649586678, "mix_low": 0.5491123199462891, "mix_high": 0.0, "read_cosine": 0.23552870750427246, "beta_mean": 0.06988810002803802, "nonfinite": 0.0, "weighted_memory_to_active": 0.05669502541422844, "mix_std": 0.04488630220293999, "beta_std": 0.07381095737218857, "beta_low": 0.121826171875, "beta_high": 0.0, "orthogonal_to_active": 0.013644249178469181, "evidence_weights": [0.1337064504623413, -0.014426575042307377, 0.1790432631969452], "agreement_weight": 0.03977281600236893, "orthogonal_weight": -0.27384263277053833}, {"head": 5, "memory_to_active": 0.039740029722452164, "contribution_to_active": 0.445097953081131, "mix_mean": 0.48083001375198364, "mix_low": 0.0017709732055664062, "mix_high": 0.0004482269287109375, "read_cosine": 0.2861021161079407, "beta_mean": 0.00019870713003911078, "nonfinite": 0.0, "weighted_memory_to_active": 0.019703341647982597, "mix_std": 0.25643855333328247, "beta_std": 0.0004145167185924947, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.01821362040936947, "evidence_weights": [-0.17758548259735107, -0.23418749868869781, 0.5360532402992249], "agreement_weight": 0.21316252648830414, "orthogonal_weight": -0.991380512714386}]}, {"layer": 8, "memory_to_active": 0.14426732063293457, "contribution_to_active": 0.5610885620117188, "mix_mean": 0.4869193434715271, "mix_low": 0.012408415786921978, "mix_high": 0.0015052161179482937, "read_cosine": 0.41031497716903687, "beta_mean": 0.0139393862336874, "nonfinite": 0.0, "weighted_memory_to_active": 0.07376453280448914, "mix_std": 0.24080970883369446, "beta_std": 0.02066335827112198, "beta_low": 0.781524658203125, "beta_high": 0.0, "orthogonal_to_active": 0.06891857087612152, "output_influence": 0.5119691491127014, "residual_influence": 0.1840982884168625, "state_active": 0.04539386183023453, "state_memory": 0.005367082078009844, "state_peak": 46.374244689941406, "growth_active": 1.1080163717269897, "growth_memory": 1.0840489864349365, "heads": [{"head": 0, "memory_to_active": 0.05445569381117821, "contribution_to_active": 0.5735399723052979, "mix_mean": 0.6048793196678162, "mix_low": 0.00051116943359375, "mix_high": 0.0015506744384765625, "read_cosine": 0.4738481044769287, "beta_mean": 0.003518463112413883, "nonfinite": 0.0, "weighted_memory_to_active": 0.038767050951719284, "mix_std": 0.24829116463661194, "beta_std": 0.014342688024044037, "beta_low": 0.92596435546875, "beta_high": 0.0, "orthogonal_to_active": 0.023983964696526527, "evidence_weights": [-0.16285918653011322, -0.3852280080318451, 0.597547709941864], "agreement_weight": 0.2573891878128052, "orthogonal_weight": -0.7938090562820435}, {"head": 1, "memory_to_active": 0.07302865386009216, "contribution_to_active": 0.5413218140602112, "mix_mean": 0.5204038023948669, "mix_low": 6.866455078125e-05, "mix_high": 0.0010042190551757812, "read_cosine": 0.13349443674087524, "beta_mean": 0.0011078428942710161, "nonfinite": 0.0, "weighted_memory_to_active": 0.03376978635787964, "mix_std": 0.23453477025032043, "beta_std": 0.007990941405296326, "beta_low": 0.9888916015625, "beta_high": 0.0, "orthogonal_to_active": 0.024189935997128487, "evidence_weights": [-0.19507117569446564, -0.27657318115234375, 0.6803402900695801], "agreement_weight": 0.06674440205097198, "orthogonal_weight": -0.7516301870346069}, {"head": 2, "memory_to_active": 0.01101660542190075, "contribution_to_active": 0.6761743426322937, "mix_mean": 0.6774420738220215, "mix_low": 0.0006265640258789062, "mix_high": 0.0033283233642578125, "read_cosine": 0.390987753868103, "beta_mean": 0.00011293326679151505, "nonfinite": 0.0, "weighted_memory_to_active": 0.008521668612957, "mix_std": 0.2247961163520813, "beta_std": 0.003445249516516924, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005263759754598141, "evidence_weights": [-0.2237190306186676, -0.34499773383140564, 1.0136315822601318], "agreement_weight": 0.2940506637096405, "orthogonal_weight": -0.6990514397621155}, {"head": 3, "memory_to_active": 0.10033583641052246, "contribution_to_active": 0.6466399431228638, "mix_mean": 0.5789029598236084, "mix_low": 0.0007848739624023438, "mix_high": 0.0029287338256835938, "read_cosine": 0.7922670841217041, "beta_mean": 0.00795777328312397, "nonfinite": 0.0, "weighted_memory_to_active": 0.07701908051967621, "mix_std": 0.25552308559417725, "beta_std": 0.007853603921830654, "beta_low": 0.72698974609375, "beta_high": 0.0, "orthogonal_to_active": 0.02472965605556965, "evidence_weights": [-0.18200074136257172, -0.3005387783050537, 0.5464099049568176], "agreement_weight": 0.2436058521270752, "orthogonal_weight": -0.6175975203514099}, {"head": 4, "memory_to_active": 0.24211828410625458, "contribution_to_active": 0.29627829790115356, "mix_mean": 0.28353947401046753, "mix_low": 0.005888938903808594, "mix_high": 4.76837158203125e-06, "read_cosine": 0.1818501353263855, "beta_mean": 0.0011859002988785505, "nonfinite": 0.0, "weighted_memory_to_active": 0.09005333483219147, "mix_std": 0.21999859809875488, "beta_std": 0.01157381758093834, "beta_low": 0.98883056640625, "beta_high": 0.0, "orthogonal_to_active": 0.021771671250462532, "evidence_weights": [-0.11187764257192612, -0.18451903760433197, 0.5400565266609192], "agreement_weight": 0.051591020077466965, "orthogonal_weight": -0.24927030503749847}, {"head": 5, "memory_to_active": 0.38464877009391785, "contribution_to_active": 0.632577121257782, "mix_mean": 0.2563484311103821, "mix_low": 0.06657028198242188, "mix_high": 0.00021457672119140625, "read_cosine": 0.48944219946861267, "beta_mean": 0.06975339353084564, "nonfinite": 0.0, "weighted_memory_to_active": 0.19445621967315674, "mix_std": 0.26171454787254333, "beta_std": 0.07877384126186371, "beta_low": 0.05853271484375, "beta_high": 0.0, "orthogonal_to_active": 0.3135724365711212, "evidence_weights": [-0.14082814753055573, -0.15019895136356354, -0.11531702429056168], "agreement_weight": 0.0037936922162771225, "orthogonal_weight": 1.9084376096725464}]}, {"layer": 9, "memory_to_active": 0.9303774833679199, "contribution_to_active": 0.6914483308792114, "mix_mean": 0.3714825510978699, "mix_low": 0.07617966830730438, "mix_high": 0.006201903335750103, "read_cosine": 0.3544448912143707, "beta_mean": 0.04011007770895958, "nonfinite": 0.0, "weighted_memory_to_active": 0.384515643119812, "mix_std": 0.23607264459133148, "beta_std": 0.03400612995028496, "beta_low": 0.556854248046875, "beta_high": 0.0, "orthogonal_to_active": 0.036025047302246094, "output_influence": 0.5452572107315063, "residual_influence": 0.16016601026058197, "state_active": 0.04341568797826767, "state_memory": 0.016102870926260948, "state_peak": 59.458717346191406, "growth_active": 1.1797387599945068, "growth_memory": 1.3260197639465332, "heads": [{"head": 0, "memory_to_active": 0.0005094627267681062, "contribution_to_active": 0.5740413069725037, "mix_mean": 0.6460456848144531, "mix_low": 0.0011663436889648438, "mix_high": 0.025150299072265625, "read_cosine": 0.33727365732192993, "beta_mean": 1.4555803318216931e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0003207237459719181, "mix_std": 0.2885093092918396, "beta_std": 3.960819867643295e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00034260176471434534, "evidence_weights": [-0.3043467700481415, -0.3121592104434967, 0.44808682799339294], "agreement_weight": 0.11792813986539841, "orthogonal_weight": -1.1299538612365723}, {"head": 1, "memory_to_active": 0.025171145796775818, "contribution_to_active": 0.6998279094696045, "mix_mean": 0.5707231163978577, "mix_low": 0.007117271423339844, "mix_high": 0.0028820037841796875, "read_cosine": 0.4146309196949005, "beta_mean": 6.1018999986117706e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.018925361335277557, "mix_std": 0.28127381205558777, "beta_std": 0.0001375628780806437, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01304054819047451, "evidence_weights": [-0.2400996834039688, -0.2964133322238922, 1.0309739112854004], "agreement_weight": 0.48594167828559875, "orthogonal_weight": -0.845909059047699}, {"head": 2, "memory_to_active": 0.17028188705444336, "contribution_to_active": 0.6504688262939453, "mix_mean": 0.5943074226379395, "mix_low": 0.0026731491088867188, "mix_high": 0.007536888122558594, "read_cosine": 0.33088797330856323, "beta_mean": 0.0006609333213418722, "nonfinite": 0.0, "weighted_memory_to_active": 0.11412541568279266, "mix_std": 0.26837900280952454, "beta_std": 0.004639115184545517, "beta_low": 0.99542236328125, "beta_high": 0.0, "orthogonal_to_active": 0.06759680807590485, "evidence_weights": [-0.20447422564029694, -0.2780378460884094, 0.7746480107307434], "agreement_weight": 0.33308759331703186, "orthogonal_weight": -0.6803691983222961}, {"head": 3, "memory_to_active": 0.9203575253486633, "contribution_to_active": 0.2195936143398285, "mix_mean": 0.12151839584112167, "mix_low": 0.1538562774658203, "mix_high": 6.103515625e-05, "read_cosine": 0.38163530826568604, "beta_mean": 0.04356177896261215, "nonfinite": 0.0, "weighted_memory_to_active": 0.1921442300081253, "mix_std": 0.17537453770637512, "beta_std": 0.04174889624118805, "beta_low": 0.06854248046875, "beta_high": 0.0, "orthogonal_to_active": 0.06581375002861023, "evidence_weights": [0.0007890447741374373, -0.07192575186491013, 0.2898147702217102], "agreement_weight": 0.09510894119739532, "orthogonal_weight": -0.38439324498176575}, {"head": 4, "memory_to_active": 3.2582051753997803, "contribution_to_active": 1.6434494256973267, "mix_mean": 0.1442074179649353, "mix_low": 0.12921619415283203, "mix_high": 0.0010747909545898438, "read_cosine": 0.1606166660785675, "beta_mean": 0.15381021797657013, "nonfinite": 0.0, "weighted_memory_to_active": 1.604675531387329, "mix_std": 0.2049405872821808, "beta_std": 0.09505188465118408, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.020916493609547615, "evidence_weights": [-0.07950689643621445, -0.15292611718177795, -0.040345869958400726], "agreement_weight": -0.06284577399492264, "orthogonal_weight": -0.013574534095823765}, {"head": 5, "memory_to_active": 1.2077397108078003, "contribution_to_active": 0.36130833625793457, "mix_mean": 0.15209314227104187, "mix_low": 0.16304874420166016, "mix_high": 0.0005064010620117188, "read_cosine": 0.5016248226165771, "beta_mean": 0.04256506264209747, "nonfinite": 0.0, "weighted_memory_to_active": 0.3769025504589081, "mix_std": 0.1979585886001587, "beta_std": 0.06245534494519234, "beta_low": 0.27716064453125, "beta_high": 0.0, "orthogonal_to_active": 0.04844006896018982, "evidence_weights": [0.07950097322463989, -0.15203072130680084, 0.14046745002269745], "agreement_weight": 0.011503789573907852, "orthogonal_weight": 0.16818556189537048}]}, {"layer": 10, "memory_to_active": 0.21344837546348572, "contribution_to_active": 0.7179000377655029, "mix_mean": 0.5181143879890442, "mix_low": 0.020691078156232834, "mix_high": 0.004603386390954256, "read_cosine": 0.3803761303424835, "beta_mean": 0.04493621736764908, "nonfinite": 0.0, "weighted_memory_to_active": 0.09522473067045212, "mix_std": 0.26587969064712524, "beta_std": 0.04603720083832741, "beta_low": 0.6616312861442566, "beta_high": 0.0, "orthogonal_to_active": 0.258507639169693, "output_influence": 0.4528883695602417, "residual_influence": 0.15991154313087463, "state_active": 0.05349009484052658, "state_memory": 0.015118436887860298, "state_peak": 125.22693634033203, "growth_active": 1.0306601524353027, "growth_memory": 1.0052577257156372, "heads": [{"head": 0, "memory_to_active": 0.7601199150085449, "contribution_to_active": 2.032122850418091, "mix_mean": 0.5682471990585327, "mix_low": 0.08966445922851562, "mix_high": 0.006974220275878906, "read_cosine": 0.10689415037631989, "beta_mean": 0.19791601598262787, "nonfinite": 0.0, "weighted_memory_to_active": 0.44859015941619873, "mix_std": 0.3914758563041687, "beta_std": 0.20659415423870087, "beta_low": 0.141357421875, "beta_high": 0.0, "orthogonal_to_active": 1.4628397226333618, "evidence_weights": [-0.0916685238480568, -0.3817768096923828, -1.8545660972595215], "agreement_weight": 0.013949418440461159, "orthogonal_weight": 3.2866246700286865}, {"head": 1, "memory_to_active": 0.010712694376707077, "contribution_to_active": 0.6096460819244385, "mix_mean": 0.6254105567932129, "mix_low": 0.0016450881958007812, "mix_high": 0.014838218688964844, "read_cosine": 0.3001191318035126, "beta_mean": 7.940229988889769e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006018802523612976, "mix_std": 0.262713223695755, "beta_std": 0.00013656409282702953, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0093575743958354, "evidence_weights": [-0.2502784729003906, -0.19447603821754456, 0.5525479912757874], "agreement_weight": 0.6043598651885986, "orthogonal_weight": -1.6769641637802124}, {"head": 2, "memory_to_active": 0.006810937542468309, "contribution_to_active": 0.614501953125, "mix_mean": 0.6571491956710815, "mix_low": 0.00026416778564453125, "mix_high": 0.0046138763427734375, "read_cosine": 0.04974525794386864, "beta_mean": 1.6564081306569278e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00312149990350008, "mix_std": 0.24941107630729675, "beta_std": 5.192006938159466e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013620792888104916, "evidence_weights": [-0.2652585804462433, -0.28009843826293945, 0.6296173334121704], "agreement_weight": -0.21464884281158447, "orthogonal_weight": -0.42077142000198364}, {"head": 3, "memory_to_active": 0.3491804003715515, "contribution_to_active": 0.09783012419939041, "mix_mean": 0.26959365606307983, "mix_low": 0.030055999755859375, "mix_high": 5.14984130859375e-05, "read_cosine": 0.9130066633224487, "beta_mean": 0.06512149423360825, "nonfinite": 0.0, "weighted_memory_to_active": 0.04034983366727829, "mix_std": 0.2195538729429245, "beta_std": 0.061660315841436386, "beta_low": 0.00146484375, "beta_high": 0.0, "orthogonal_to_active": 0.04022997245192528, "evidence_weights": [-0.10077362507581711, -0.14763328433036804, 0.1664462387561798], "agreement_weight": 0.017757054418325424, "orthogonal_weight": 1.4190646409988403}, {"head": 4, "memory_to_active": 0.019772473722696304, "contribution_to_active": 0.5473668575286865, "mix_mean": 0.5816398859024048, "mix_low": 0.000308990478515625, "mix_high": 0.0011386871337890625, "read_cosine": 0.06405583769083023, "beta_mean": 5.7538905821274966e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011434257961809635, "mix_std": 0.24606963992118835, "beta_std": 0.00010364178160671145, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01597084291279316, "evidence_weights": [-0.18401795625686646, -0.2568618953227997, 0.9646970629692078], "agreement_weight": 0.16240206360816956, "orthogonal_weight": -1.472562551498413}, {"head": 5, "memory_to_active": 0.1340937614440918, "contribution_to_active": 0.40593230724334717, "mix_mean": 0.40664562582969666, "mix_low": 0.0022077560424804688, "mix_high": 3.814697265625e-06, "read_cosine": 0.8484356999397278, "beta_mean": 0.0064263069070875645, "nonfinite": 0.0, "weighted_memory_to_active": 0.06183377653360367, "mix_std": 0.22605440020561218, "beta_std": 0.007676593028008938, "beta_low": 0.82696533203125, "beta_high": 0.0, "orthogonal_to_active": 0.021285608410835266, "evidence_weights": [-0.14184121787548065, -0.16420531272888184, 0.26823535561561584], "agreement_weight": 0.09214064478874207, "orthogonal_weight": -0.6604320406913757}]}, {"layer": 11, "memory_to_active": 0.07845933735370636, "contribution_to_active": 0.6208289265632629, "mix_mean": 0.5200673937797546, "mix_low": 0.11524137854576111, "mix_high": 0.04252449795603752, "read_cosine": 0.33921122550964355, "beta_mean": 0.025357406586408615, "nonfinite": 0.0, "weighted_memory_to_active": 0.0426790751516819, "mix_std": 0.3246943950653076, "beta_std": 0.03740773722529411, "beta_low": 0.6964620351791382, "beta_high": 0.0, "orthogonal_to_active": 0.04584925249218941, "output_influence": 0.5567512512207031, "residual_influence": 0.14901793003082275, "state_active": 0.22046919167041779, "state_memory": 0.0035191099159419537, "state_peak": 35.8504638671875, "growth_active": 1.1208996772766113, "growth_memory": 1.5059099197387695, "heads": [{"head": 0, "memory_to_active": 0.007488278206437826, "contribution_to_active": 0.6008576154708862, "mix_mean": 0.6556061506271362, "mix_low": 0.0009756088256835938, "mix_high": 0.009411811828613281, "read_cosine": 0.2640564739704132, "beta_mean": 6.164333899505436e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005602552555501461, "mix_std": 0.2574874758720398, "beta_std": 0.0028984483797103167, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.004628301132470369, "evidence_weights": [-0.1747407168149948, -0.3645305037498474, 0.7848840951919556], "agreement_weight": 0.3808898329734802, "orthogonal_weight": -0.8462857007980347}, {"head": 1, "memory_to_active": 0.12794442474842072, "contribution_to_active": 0.422396183013916, "mix_mean": 0.45910853147506714, "mix_low": 0.005672454833984375, "mix_high": 0.0007610321044921875, "read_cosine": 0.4799134135246277, "beta_mean": 0.003968746401369572, "nonfinite": 0.0, "weighted_memory_to_active": 0.06044919788837433, "mix_std": 0.2701803147792816, "beta_std": 0.012584807351231575, "beta_low": 0.92462158203125, "beta_high": 0.0, "orthogonal_to_active": 0.033003609627485275, "evidence_weights": [-0.15184634923934937, -0.2634018361568451, 0.5823094844818115], "agreement_weight": 0.07182653993368149, "orthogonal_weight": -0.6420771479606628}, {"head": 2, "memory_to_active": 0.00585584482178092, "contribution_to_active": 0.9880688190460205, "mix_mean": 0.42898029088974, "mix_low": 0.32295894622802734, "mix_high": 0.15473175048828125, "read_cosine": 0.16646207869052887, "beta_mean": 0.06854939460754395, "nonfinite": 0.0, "weighted_memory_to_active": 0.004102808423340321, "mix_std": 0.449547678232193, "beta_std": 0.09397877752780914, "beta_low": 0.481689453125, "beta_high": 0.0, "orthogonal_to_active": 0.03885289654135704, "evidence_weights": [0.04968063160777092, -0.37220653891563416, -1.3410998582839966], "agreement_weight": 0.005425603128969669, "orthogonal_weight": 9.847458839416504}, {"head": 3, "memory_to_active": 0.008031422272324562, "contribution_to_active": 0.8551753759384155, "mix_mean": 0.4058670997619629, "mix_low": 0.3532123565673828, "mix_high": 0.08144855499267578, "read_cosine": 0.5376542210578918, "beta_mean": 0.05374744161963463, "nonfinite": 0.0, "weighted_memory_to_active": 0.0050336699932813644, "mix_std": 0.43947505950927734, "beta_std": 0.08291590213775635, "beta_low": 0.5169677734375, "beta_high": 0.0, "orthogonal_to_active": 0.03548505902290344, "evidence_weights": [0.2093428075313568, -0.28869712352752686, -0.13616827130317688], "agreement_weight": 0.05239272117614746, "orthogonal_weight": 10.115826606750488}, {"head": 4, "memory_to_active": 0.233608216047287, "contribution_to_active": 0.4455130696296692, "mix_mean": 0.6000545024871826, "mix_low": 0.007176399230957031, "mix_high": 0.0018978118896484375, "read_cosine": 0.03531385958194733, "beta_mean": 0.017536232247948647, "nonfinite": 0.0, "weighted_memory_to_active": 0.13498559594154358, "mix_std": 0.2720908522605896, "beta_std": 0.020791472867131233, "beta_low": 0.48529052734375, "beta_high": 0.0, "orthogonal_to_active": 0.13158275187015533, "evidence_weights": [-0.2374795377254486, -0.2412387728691101, 1.8996344804763794], "agreement_weight": -0.2130245417356491, "orthogonal_weight": -0.9903038740158081}, {"head": 5, "memory_to_active": 0.08782785385847092, "contribution_to_active": 0.4129626154899597, "mix_mean": 0.570787787437439, "mix_low": 0.0014524459838867188, "mix_high": 0.006896018981933594, "read_cosine": 0.5518671274185181, "beta_mean": 0.008280972018837929, "nonfinite": 0.0, "weighted_memory_to_active": 0.0459006205201149, "mix_std": 0.25938495993614197, "beta_std": 0.011277027428150177, "beta_low": 0.7705078125, "beta_high": 0.0, "orthogonal_to_active": 0.03154288977384567, "evidence_weights": [-0.20750287175178528, -0.31614264845848083, 0.6640897393226624], "agreement_weight": 0.0854618176817894, "orthogonal_weight": -0.8706636428833008}]}, {"layer": 12, "memory_to_active": 0.2350013256072998, "contribution_to_active": 0.5917033553123474, "mix_mean": 0.5487849712371826, "mix_low": 0.008001009933650494, "mix_high": 0.0061732931062579155, "read_cosine": 0.20470550656318665, "beta_mean": 0.015612676739692688, "nonfinite": 0.0, "weighted_memory_to_active": 0.12697511911392212, "mix_std": 0.2599109411239624, "beta_std": 0.016193825751543045, "beta_low": 0.8276774883270264, "beta_high": 0.0, "orthogonal_to_active": 0.07659260183572769, "output_influence": 0.507523775100708, "residual_influence": 0.14801760017871857, "state_active": 0.0352923609316349, "state_memory": 0.00856112316250801, "state_peak": 62.71555709838867, "growth_active": 1.1448615789413452, "growth_memory": 1.1892483234405518, "heads": [{"head": 0, "memory_to_active": 0.04171149805188179, "contribution_to_active": 0.6171686053276062, "mix_mean": 0.5797044634819031, "mix_low": 0.0005140304565429688, "mix_high": 0.005413055419921875, "read_cosine": 0.2861306667327881, "beta_mean": 0.0007180344546213746, "nonfinite": 0.0, "weighted_memory_to_active": 0.02279571443796158, "mix_std": 0.26520323753356934, "beta_std": 0.0014509237371385098, "beta_low": 0.99578857421875, "beta_high": 0.0, "orthogonal_to_active": 0.021029911935329437, "evidence_weights": [-0.2092752307653427, -0.2761750817298889, -0.006195324473083019], "agreement_weight": 0.1060100793838501, "orthogonal_weight": -0.9898666143417358}, {"head": 1, "memory_to_active": 0.12032680213451385, "contribution_to_active": 0.5527532696723938, "mix_mean": 0.5650655031204224, "mix_low": 0.00036144256591796875, "mix_high": 0.002223968505859375, "read_cosine": 0.3527229428291321, "beta_mean": 0.00072511937469244, "nonfinite": 0.0, "weighted_memory_to_active": 0.07620421797037125, "mix_std": 0.2652403712272644, "beta_std": 0.010255442000925541, "beta_low": 0.995849609375, "beta_high": 0.0, "orthogonal_to_active": 0.06509306281805038, "evidence_weights": [-0.20220907032489777, -0.27198851108551025, 1.0403028726577759], "agreement_weight": 0.2510091960430145, "orthogonal_weight": -0.9262928366661072}, {"head": 2, "memory_to_active": 0.6062211394309998, "contribution_to_active": 0.516272246837616, "mix_mean": 0.33001720905303955, "mix_low": 0.022874832153320312, "mix_high": 4.482269287109375e-05, "read_cosine": 0.5103862881660461, "beta_mean": 0.0908600389957428, "nonfinite": 0.0, "weighted_memory_to_active": 0.25795626640319824, "mix_std": 0.2369617074728012, "beta_std": 0.06506486982107162, "beta_low": 0.00091552734375, "beta_high": 0.0, "orthogonal_to_active": 0.27220427989959717, "evidence_weights": [-0.21030190587043762, -0.05934246629476547, -0.009566441178321838], "agreement_weight": 0.031951263546943665, "orthogonal_weight": 1.3815505504608154}, {"head": 3, "memory_to_active": 0.03287065029144287, "contribution_to_active": 0.7537619471549988, "mix_mean": 0.7196671366691589, "mix_low": 0.00015354156494140625, "mix_high": 0.01798725128173828, "read_cosine": 0.07144178450107574, "beta_mean": 0.0002678940072655678, "nonfinite": 0.0, "weighted_memory_to_active": 0.02478620782494545, "mix_std": 0.22675350308418274, "beta_std": 0.005718720145523548, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.0237878430634737, "evidence_weights": [-0.28972163796424866, -0.33702656626701355, 1.6633615493774414], "agreement_weight": 0.5587789416313171, "orthogonal_weight": -0.9745673537254333}, {"head": 4, "memory_to_active": 0.5194997787475586, "contribution_to_active": 0.6050525903701782, "mix_mean": 0.45406225323677063, "mix_low": 0.017484664916992188, "mix_high": 0.004909515380859375, "read_cosine": -0.1846373826265335, "beta_mean": 0.0006482265889644623, "nonfinite": 0.0, "weighted_memory_to_active": 0.3202838599681854, "mix_std": 0.29161494970321655, "beta_std": 0.009584668092429638, "beta_low": 0.986328125, "beta_high": 0.0, "orthogonal_to_active": 0.031929973512887955, "evidence_weights": [-0.15553465485572815, -0.2181936502456665, -0.11994985491037369], "agreement_weight": 0.013840614818036556, "orthogonal_weight": -0.10418201982975006}, {"head": 5, "memory_to_active": 0.0893779918551445, "contribution_to_active": 0.5052114725112915, "mix_mean": 0.6441932320594788, "mix_low": 0.006617546081542969, "mix_high": 0.006461143493652344, "read_cosine": 0.19218873977661133, "beta_mean": 0.0004567409341689199, "nonfinite": 0.0, "weighted_memory_to_active": 0.05982442572712898, "mix_std": 0.27369192242622375, "beta_std": 0.005088329315185547, "beta_low": 0.98944091796875, "beta_high": 0.0, "orthogonal_to_active": 0.04551052674651146, "evidence_weights": [-0.23030659556388855, -0.31835439801216125, 1.253549575805664], "agreement_weight": 0.4158822000026703, "orthogonal_weight": -0.8130814433097839}]}, {"layer": 13, "memory_to_active": 0.11675142496824265, "contribution_to_active": 0.536058783531189, "mix_mean": 0.5393895506858826, "mix_low": 0.006078879348933697, "mix_high": 0.010097980499267578, "read_cosine": 0.40646615624427795, "beta_mean": 0.01466208416968584, "nonfinite": 0.0, "weighted_memory_to_active": 0.05123220384120941, "mix_std": 0.25946125388145447, "beta_std": 0.014873357489705086, "beta_low": 0.8124593496322632, "beta_high": 0.0, "orthogonal_to_active": 0.05214851722121239, "output_influence": 0.5280082821846008, "residual_influence": 0.18587765097618103, "state_active": 0.13696332275867462, "state_memory": 0.04471161961555481, "state_peak": 97.53564453125, "growth_active": 0.9298092126846313, "growth_memory": 1.0130913257598877, "heads": [{"head": 0, "memory_to_active": 0.011960603296756744, "contribution_to_active": 0.5994680523872375, "mix_mean": 0.6122817397117615, "mix_low": 0.000782012939453125, "mix_high": 0.0041866302490234375, "read_cosine": 0.43143388628959656, "beta_mean": 4.221198469167575e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007694427389651537, "mix_std": 0.26735949516296387, "beta_std": 8.982398867374286e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007435612380504608, "evidence_weights": [-0.19491299986839294, -0.2681414484977722, 0.7694284915924072], "agreement_weight": 0.18300338089466095, "orthogonal_weight": -1.0763647556304932}, {"head": 1, "memory_to_active": 0.011134087108075619, "contribution_to_active": 0.596979022026062, "mix_mean": 0.5679288506507874, "mix_low": 0.0016994476318359375, "mix_high": 0.0046405792236328125, "read_cosine": 0.19148945808410645, "beta_mean": 4.280730354366824e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006160247139632702, "mix_std": 0.2767570912837982, "beta_std": 7.485052628908306e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008034148253500462, "evidence_weights": [-0.17900323867797852, -0.26252424716949463, 0.5852822661399841], "agreement_weight": 0.48972752690315247, "orthogonal_weight": -1.3533384799957275}, {"head": 2, "memory_to_active": 0.06621484458446503, "contribution_to_active": 0.48913297057151794, "mix_mean": 0.3986667990684509, "mix_low": 0.0013532638549804688, "mix_high": 0.00030422210693359375, "read_cosine": 0.5778195858001709, "beta_mean": 0.006198178511112928, "nonfinite": 0.0, "weighted_memory_to_active": 0.030590323731303215, "mix_std": 0.2355792224407196, "beta_std": 0.02334531955420971, "beta_low": 0.88226318359375, "beta_high": 0.0, "orthogonal_to_active": 0.018085427582263947, "evidence_weights": [-0.18512693047523499, -0.21135224401950836, 0.4052748382091522], "agreement_weight": 0.005350132007151842, "orthogonal_weight": -0.8456393480300903}, {"head": 3, "memory_to_active": 0.16280168294906616, "contribution_to_active": 0.5778555870056152, "mix_mean": 0.6631104946136475, "mix_low": 0.02572345733642578, "mix_high": 0.03859996795654297, "read_cosine": 0.00353215541690588, "beta_mean": 0.000924988416954875, "nonfinite": 0.0, "weighted_memory_to_active": 0.1232164055109024, "mix_std": 0.29050499200820923, "beta_std": 0.009154104627668858, "beta_low": 0.99273681640625, "beta_high": 0.0, "orthogonal_to_active": 0.10794298350811005, "evidence_weights": [-0.31537291407585144, -0.24472786486148834, 1.6002397537231445], "agreement_weight": 6.273057460784912, "orthogonal_weight": -0.9679648876190186}, {"head": 4, "memory_to_active": 0.3933016061782837, "contribution_to_active": 0.24454113841056824, "mix_mean": 0.34071841835975647, "mix_low": 0.004429817199707031, "mix_high": 3.814697265625e-06, "read_cosine": 0.8883687853813171, "beta_mean": 0.08048854768276215, "nonfinite": 0.0, "weighted_memory_to_active": 0.09974245727062225, "mix_std": 0.21238166093826294, "beta_std": 0.05526532977819443, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.13846947252750397, "evidence_weights": [-0.07956492900848389, -0.23125703632831573, 0.44814473390579224], "agreement_weight": 0.008244088850915432, "orthogonal_weight": 2.255002498626709}, {"head": 5, "memory_to_active": 0.055095646530389786, "contribution_to_active": 0.7083760499954224, "mix_mean": 0.6536310911178589, "mix_low": 0.0024852752685546875, "mix_high": 0.012852668762207031, "read_cosine": 0.34615299105644226, "beta_mean": 0.00027576868887990713, "nonfinite": 0.0, "weighted_memory_to_active": 0.03998932987451553, "mix_std": 0.2741849720478058, "beta_std": 0.0013107131235301495, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.032923467457294464, "evidence_weights": [-0.2194441854953766, -0.29577112197875977, 0.8487456440925598], "agreement_weight": 0.19397135078907013, "orthogonal_weight": -0.9226359724998474}]}, {"layer": 14, "memory_to_active": 0.22968995571136475, "contribution_to_active": 0.4182623326778412, "mix_mean": 0.42989063262939453, "mix_low": 0.02388986200094223, "mix_high": 0.006348133087158203, "read_cosine": 0.6144136190414429, "beta_mean": 0.06402140855789185, "nonfinite": 0.0, "weighted_memory_to_active": 0.09615443646907806, "mix_std": 0.24271762371063232, "beta_std": 0.06068102642893791, "beta_low": 0.4974263906478882, "beta_high": 0.0, "orthogonal_to_active": 0.04130920022726059, "output_influence": 0.3579646944999695, "residual_influence": 0.14428043365478516, "state_active": 0.055399954319000244, "state_memory": 0.01536865346133709, "state_peak": 41.8634147644043, "growth_active": 1.1061341762542725, "growth_memory": 1.2512109279632568, "heads": [{"head": 0, "memory_to_active": 0.3715420067310333, "contribution_to_active": 0.19151489436626434, "mix_mean": 0.1952652931213379, "mix_low": 0.011322975158691406, "mix_high": 0.0, "read_cosine": 0.8185064792633057, "beta_mean": 0.07648696005344391, "nonfinite": 0.0, "weighted_memory_to_active": 0.07962167263031006, "mix_std": 0.1693403124809265, "beta_std": 0.06467045843601227, "beta_low": 0.003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.08525993674993515, "evidence_weights": [-0.10367453098297119, -0.048044975847005844, 0.4083382487297058], "agreement_weight": 0.025955768302083015, "orthogonal_weight": 1.7838720083236694}, {"head": 1, "memory_to_active": 0.47816741466522217, "contribution_to_active": 0.2784575819969177, "mix_mean": 0.21780171990394592, "mix_low": 0.11720848083496094, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8749135732650757, "beta_mean": 0.2611774206161499, "nonfinite": 0.0, "weighted_memory_to_active": 0.2172875851392746, "mix_std": 0.24380026757717133, "beta_std": 0.2516103684902191, "beta_low": 0.10125732421875, "beta_high": 0.0, "orthogonal_to_active": 0.015839193016290665, "evidence_weights": [-0.002593686105683446, -0.19692447781562805, -0.09389113634824753], "agreement_weight": 0.05915768817067146, "orthogonal_weight": 0.2141677588224411}, {"head": 2, "memory_to_active": 0.11894500255584717, "contribution_to_active": 0.5803091526031494, "mix_mean": 0.5885829329490662, "mix_low": 0.0028095245361328125, "mix_high": 0.0046024322509765625, "read_cosine": 0.40722978115081787, "beta_mean": 0.0017849765717983246, "nonfinite": 0.0, "weighted_memory_to_active": 0.07746667414903641, "mix_std": 0.26724478602409363, "beta_std": 0.002406427636742592, "beta_low": 0.985595703125, "beta_high": 0.0, "orthogonal_to_active": 0.059492822736501694, "evidence_weights": [-0.13406126201152802, -0.3163043260574341, 0.929358720779419], "agreement_weight": 0.08545506000518799, "orthogonal_weight": -0.8718843460083008}, {"head": 3, "memory_to_active": 0.04634978622198105, "contribution_to_active": 0.6175302267074585, "mix_mean": 0.6728821992874146, "mix_low": 0.004837989807128906, "mix_high": 0.022139549255371094, "read_cosine": -0.05551758408546448, "beta_mean": 0.005922770127654076, "nonfinite": 0.0, "weighted_memory_to_active": 0.029750611633062363, "mix_std": 0.25794243812561035, "beta_std": 0.009096750058233738, "beta_low": 0.83270263671875, "beta_high": 0.0, "orthogonal_to_active": 0.02237650938332081, "evidence_weights": [-0.3453618586063385, -0.3125215172767639, 0.2952375113964081], "agreement_weight": 0.06857001036405563, "orthogonal_weight": -0.8308008313179016}, {"head": 4, "memory_to_active": 0.1584862768650055, "contribution_to_active": 0.49965059757232666, "mix_mean": 0.5188512802124023, "mix_low": 0.0036411285400390625, "mix_high": 0.009668350219726562, "read_cosine": 0.8986943960189819, "beta_mean": 0.03742256015539169, "nonfinite": 0.0, "weighted_memory_to_active": 0.09298066794872284, "mix_std": 0.2627863883972168, "beta_std": 0.02815329097211361, "beta_low": 0.0823974609375, "beta_high": 0.0, "orthogonal_to_active": 0.025092970579862595, "evidence_weights": [-0.2113892287015915, -0.23832976818084717, 0.42946717143058777], "agreement_weight": 0.13321711122989655, "orthogonal_weight": -0.6182161569595337}, {"head": 5, "memory_to_active": 0.20464910566806793, "contribution_to_active": 0.34211140871047974, "mix_mean": 0.385960191488266, "mix_low": 0.0035190582275390625, "mix_high": 0.0016756057739257812, "read_cosine": 0.7426550984382629, "beta_mean": 0.0013337794225662947, "nonfinite": 0.0, "weighted_memory_to_active": 0.07981937378644943, "mix_std": 0.2551915645599365, "beta_std": 0.008148860186338425, "beta_low": 0.97894287109375, "beta_high": 0.0, "orthogonal_to_active": 0.039793744683265686, "evidence_weights": [-0.1749599277973175, -0.17822889983654022, 0.3460848331451416], "agreement_weight": 0.0710810124874115, "orthogonal_weight": -0.7551976442337036}]}, {"layer": 15, "memory_to_active": 0.052651695907115936, "contribution_to_active": 0.48950818181037903, "mix_mean": 0.42802295088768005, "mix_low": 0.00807317066937685, "mix_high": 0.0032057762145996094, "read_cosine": 0.6732964515686035, "beta_mean": 0.035616688430309296, "nonfinite": 0.0, "weighted_memory_to_active": 0.02625180222094059, "mix_std": 0.28608438372612, "beta_std": 0.042811714112758636, "beta_low": 0.7154338359832764, "beta_high": 0.0, "orthogonal_to_active": 0.007913012057542801, "output_influence": 0.30451175570487976, "residual_influence": 0.17692333459854126, "state_active": 0.15963837504386902, "state_memory": 0.010698182508349419, "state_peak": 27.109769821166992, "growth_active": 1.2374308109283447, "growth_memory": 1.3329248428344727, "heads": [{"head": 0, "memory_to_active": 0.09212358295917511, "contribution_to_active": 0.47335654497146606, "mix_mean": 0.3889940679073334, "mix_low": 0.008641242980957031, "mix_high": 0.0056171417236328125, "read_cosine": 0.5780766010284424, "beta_mean": 0.03561388701200485, "nonfinite": 0.0, "weighted_memory_to_active": 0.04380070045590401, "mix_std": 0.2858448028564453, "beta_std": 0.026059091091156006, "beta_low": 0.10723876953125, "beta_high": 0.0, "orthogonal_to_active": 0.025428621098399162, "evidence_weights": [-0.1894828826189041, -0.1907428652048111, 0.47565895318984985], "agreement_weight": 0.08993251621723175, "orthogonal_weight": -0.7978980541229248}, {"head": 1, "memory_to_active": 0.113882876932621, "contribution_to_active": 0.4299667775630951, "mix_mean": 0.4421598017215729, "mix_low": 0.00130462646484375, "mix_high": 0.007350921630859375, "read_cosine": 0.9296231865882874, "beta_mean": 0.0003996515879407525, "nonfinite": 0.0, "weighted_memory_to_active": 0.05011291801929474, "mix_std": 0.2903757691383362, "beta_std": 0.00258141802623868, "beta_low": 0.99700927734375, "beta_high": 0.0, "orthogonal_to_active": 0.002078765071928501, "evidence_weights": [-0.15043829381465912, -0.15883944928646088, 0.17019495368003845], "agreement_weight": 0.12482938915491104, "orthogonal_weight": 0.09251634776592255}, {"head": 2, "memory_to_active": 0.006876919884234667, "contribution_to_active": 0.4865587651729584, "mix_mean": 0.3517126739025116, "mix_low": 0.012358665466308594, "mix_high": 6.771087646484375e-05, "read_cosine": 0.805573582649231, "beta_mean": 9.291234891861677e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0036553330719470978, "mix_std": 0.27145326137542725, "beta_std": 0.00019390361558180302, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0011705177603289485, "evidence_weights": [-0.20458635687828064, -0.26197507977485657, 0.10108036547899246], "agreement_weight": 0.0106129702180624, "orthogonal_weight": 0.8504356741905212}, {"head": 3, "memory_to_active": 0.0022780399303883314, "contribution_to_active": 0.3807578682899475, "mix_mean": 0.41149064898490906, "mix_low": 0.005923271179199219, "mix_high": 0.000640869140625, "read_cosine": 0.6385901570320129, "beta_mean": 1.2103564586141147e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007766421185806394, "mix_std": 0.2908054292201996, "beta_std": 2.6429161152918823e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005100874113850296, "evidence_weights": [-0.2712276577949524, -0.23898464441299438, 0.2420634925365448], "agreement_weight": 0.05415631830692291, "orthogonal_weight": -0.6668981909751892}, {"head": 4, "memory_to_active": 0.0718233734369278, "contribution_to_active": 0.6323968172073364, "mix_mean": 0.39017176628112793, "mix_low": 0.010442733764648438, "mix_high": 0.0017347335815429688, "read_cosine": 0.7745983004570007, "beta_mean": 0.17531831562519073, "nonfinite": 0.0, "weighted_memory_to_active": 0.040068477392196655, "mix_std": 0.3099331259727478, "beta_std": 0.20999693870544434, "beta_low": 0.2171630859375, "beta_high": 0.0, "orthogonal_to_active": 0.00634186202660203, "evidence_weights": [-0.20507597923278809, -0.251798152923584, -0.014858798123896122], "agreement_weight": 0.013155659660696983, "orthogonal_weight": 0.36938968300819397}, {"head": 5, "memory_to_active": 0.028925351798534393, "contribution_to_active": 0.5340120196342468, "mix_mean": 0.5836086273193359, "mix_low": 0.009768486022949219, "mix_high": 0.0038232803344726562, "read_cosine": 0.31331658363342285, "beta_mean": 0.0022632689215242863, "nonfinite": 0.0, "weighted_memory_to_active": 0.019096728414297104, "mix_std": 0.2680938243865967, "beta_std": 0.018012478947639465, "beta_low": 0.97125244140625, "beta_high": 0.0, "orthogonal_to_active": 0.01194821298122406, "evidence_weights": [-0.2828342616558075, -0.30339565873146057, 0.6611698865890503], "agreement_weight": 0.31058305501937866, "orthogonal_weight": 0.8141548037528992}]}]}, {"step": 63000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8162720250002167, "memory_to_active": 0.8017827272415161, "contribution_to_active": 0.5658330917358398, "mix_mean": 0.42093154788017273, "mix_low": 0.12692603468894958, "mix_high": 0.010769357904791832, "read_cosine": 0.3439909815788269, "beta_mean": 0.07104356586933136, "nonfinite": 0.0, "weighted_memory_to_active": 0.2003699243068695, "mix_std": 0.2237095683813095, "beta_std": 0.06262794882059097, "beta_low": 0.5789592266082764, "beta_high": 4.895528400084004e-05, "orthogonal_to_active": 0.06231962889432907, "samples": 4, "scored_tokens": 899, "normal_loss": 3.073230504989624, "normal_token_accuracy": 0.44160178303718567, "state_active": 0.06469301879405975, "state_memory": 0.017305713146924973, "state_peak": 124.82110595703125, "growth_active": 1.1336309909820557, "growth_memory": 1.3133323192596436, "output_influence": 0.4358764886856079, "residual_influence": 0.15254274010658264, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.23242449760437, "no_memory_loss_delta": 0.6371481418609619, "scrambled_loss_delta": 0.10839613527059555, "memory_only_loss_delta": 3.659726619720459, "no_evidence_loss_delta": 0.37983936071395874, "no_agreement_loss_delta": 0.021620096638798714, "no_orthogonal_loss_delta": -0.0014412207528948784}, {"index": 94, "tokens": 256, "normal_loss": 3.1817102432250977, "no_memory_loss_delta": 0.6584686636924744, "scrambled_loss_delta": 0.4642217755317688, "memory_only_loss_delta": 3.4942946434020996, "no_evidence_loss_delta": 0.3153468072414398, "no_agreement_loss_delta": 0.04759245738387108, "no_orthogonal_loss_delta": 0.0682188868522644}, {"index": 188, "tokens": 256, "normal_loss": 3.3850202560424805, "no_memory_loss_delta": 0.823218584060669, "scrambled_loss_delta": 0.13519440591335297, "memory_only_loss_delta": 3.038652181625366, "no_evidence_loss_delta": 0.45044124126434326, "no_agreement_loss_delta": 0.0540170781314373, "no_orthogonal_loss_delta": 0.032094553112983704}, {"index": 283, "tokens": 131, "normal_loss": 1.9408434629440308, "no_memory_loss_delta": 0.33076998591423035, "scrambled_loss_delta": 0.16903460025787354, "memory_only_loss_delta": 3.2725565433502197, "no_evidence_loss_delta": 0.12638218700885773, "no_agreement_loss_delta": 0.015449796803295612, "no_orthogonal_loss_delta": -0.016488324850797653}], "agreement_weight_abs_mean": 0.23388385772705078, "agreement_weight_abs_max": 6.334452152252197, "agreement_weight_mean": 0.2030797153711319, "orthogonal_weight_abs_mean": 0.9066630005836487, "orthogonal_weight_abs_max": 10.230239868164062, "orthogonal_weight_mean": -0.08767608553171158, "no_memory_loss": 3.7247910499572754, "no_memory_loss_delta": 0.651560366153717, "no_memory_token_accuracy_gain": 0.08231367915868759, "no_memory_token_win_fraction": 0.674082338809967, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 47.87681706066251, "scrambled_loss": 3.2994191646575928, "scrambled_loss_delta": 0.2261885553598404, "scrambled_token_accuracy_gain": 0.021134594455361366, "scrambled_token_win_fraction": 0.6095662117004395, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.243229817824254, "memory_only_loss": 6.452573299407959, "memory_only_loss_delta": 3.379343032836914, "memory_only_token_accuracy_gain": 0.27808675169944763, "memory_only_token_win_fraction": 0.9176863431930542, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.59301698565037, "no_evidence_loss": 3.4178764820098877, "no_evidence_loss_delta": 0.3446459472179413, "no_evidence_token_accuracy_gain": 0.04449388384819031, "no_evidence_token_win_fraction": 0.607341468334198, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 29.152885475160012, "no_agreement_loss": 3.110572576522827, "no_agreement_loss_delta": 0.03734227642416954, "no_agreement_token_accuracy_gain": 0.0022246940061450005, "no_agreement_token_win_fraction": 0.4927697479724884, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.6653651830203486, "no_orthogonal_loss": 3.098982572555542, "no_orthogonal_loss_delta": 0.025752300396561623, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5939933061599731, "no_orthogonal_probe_win_fraction": 0.5, "no_orthogonal_ppl_reduction_pct": 2.542353808326012, "short_no_memory_loss_delta": 0.6076561808586121, "long_context_memory_gain": 0.04390418529510498, "normal_context_loss_gain": 0.13509927690029144, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.5504653453826904, "contribution_to_active": 1.174232006072998, "mix_mean": 0.10693073272705078, "mix_low": 0.5232855677604675, "mix_high": 0.005470911972224712, "read_cosine": 0.4200323224067688, "beta_mean": 0.22065302729606628, "nonfinite": 0.0, "weighted_memory_to_active": 1.2000732421875, "mix_std": 0.09206317365169525, "beta_std": 0.1469094455242157, "beta_low": 0.13153076171875, "beta_high": 0.0, "orthogonal_to_active": 0.04274371638894081, "output_influence": 0.2250826358795166, "residual_influence": 0.2549439072608948, "state_active": 0.06558886170387268, "state_memory": 0.044455092400312424, "state_peak": 49.256874084472656, "growth_active": 1.2511976957321167, "growth_memory": 1.2704154253005981, "heads": [{"head": 0, "memory_to_active": 2.741682529449463, "contribution_to_active": 0.49610793590545654, "mix_mean": 0.02051255851984024, "mix_low": 0.6135883331298828, "mix_high": 0.0, "read_cosine": 0.23339948058128357, "beta_mean": 0.197422593832016, "nonfinite": 0.0, "weighted_memory_to_active": 0.625594437122345, "mix_std": 0.060484737157821655, "beta_std": 0.11805542558431625, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06717807799577713, "evidence_weights": [-0.0052111647091805935, -0.04552161321043968, -0.056386616080999374], "agreement_weight": -0.21781279146671295, "orthogonal_weight": -0.12740075588226318}, {"head": 1, "memory_to_active": 6.830683708190918, "contribution_to_active": 2.8282923698425293, "mix_mean": 0.01969672366976738, "mix_low": 0.6194028854370117, "mix_high": 0.0, "read_cosine": 0.4295716881752014, "beta_mean": 0.45563119649887085, "nonfinite": 0.0, "weighted_memory_to_active": 3.072068691253662, "mix_std": 0.05506739765405655, "beta_std": 0.2150004506111145, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.024615053087472916, "evidence_weights": [0.01950143277645111, -0.10577251017093658, 0.6523475646972656], "agreement_weight": 0.026217149570584297, "orthogonal_weight": 0.017198242247104645}, {"head": 2, "memory_to_active": 2.5998435020446777, "contribution_to_active": 0.5298177003860474, "mix_mean": 0.014291195198893547, "mix_low": 0.673583984375, "mix_high": 0.0, "read_cosine": 0.3112775981426239, "beta_mean": 0.29025545716285706, "nonfinite": 0.0, "weighted_memory_to_active": 0.6176934242248535, "mix_std": 0.03306475654244423, "beta_std": 0.21010617911815643, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.050657130777835846, "evidence_weights": [0.06824894994497299, 0.05619752034544945, 0.22715458273887634], "agreement_weight": 0.009787996299564838, "orthogonal_weight": -0.2003416121006012}, {"head": 3, "memory_to_active": 7.340033531188965, "contribution_to_active": 2.64725923538208, "mix_mean": 0.030992567539215088, "mix_low": 0.4892730712890625, "mix_high": 0.0, "read_cosine": 0.3107454478740692, "beta_mean": 0.31776341795921326, "nonfinite": 0.0, "weighted_memory_to_active": 2.7768313884735107, "mix_std": 0.07085821032524109, "beta_std": 0.25135278701782227, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.09566947817802429, "evidence_weights": [-0.037268638610839844, -0.03465846925973892, -0.12252644449472427], "agreement_weight": -0.009007593616843224, "orthogonal_weight": -0.04055732861161232}, {"head": 4, "memory_to_active": 1.6872656345367432, "contribution_to_active": 0.0692804604768753, "mix_mean": 0.013919282704591751, "mix_low": 0.7302789688110352, "mix_high": 0.0, "read_cosine": 0.304625004529953, "beta_mean": 0.04498211294412613, "nonfinite": 0.0, "weighted_memory_to_active": 0.052050430327653885, "mix_std": 0.04464544728398323, "beta_std": 0.06681657582521439, "beta_low": 0.34765625, "beta_high": 0.0, "orthogonal_to_active": 0.0006621988723054528, "evidence_weights": [0.12462964653968811, 0.08769674599170685, -0.12560443580150604], "agreement_weight": -0.17666718363761902, "orthogonal_weight": 0.011399392038583755}, {"head": 5, "memory_to_active": 0.10328221321105957, "contribution_to_active": 0.47463446855545044, "mix_mean": 0.5421720147132874, "mix_low": 0.013586044311523438, "mix_high": 0.032825469970703125, "read_cosine": 0.9305745363235474, "beta_mean": 0.01786341890692711, "nonfinite": 0.0, "weighted_memory_to_active": 0.05620042234659195, "mix_std": 0.28825843334198, "beta_std": 0.02012515440583229, "beta_low": 0.4415283203125, "beta_high": 0.0, "orthogonal_to_active": 0.017680341377854347, "evidence_weights": [-0.17431806027889252, -0.21860621869564056, 0.3863469660282135], "agreement_weight": 0.28918853402137756, "orthogonal_weight": -0.6962008476257324}]}, {"layer": 1, "memory_to_active": 2.4322006702423096, "contribution_to_active": 0.46527859568595886, "mix_mean": 0.21249383687973022, "mix_low": 0.36267727613449097, "mix_high": 0.0011633237591013312, "read_cosine": 0.29372185468673706, "beta_mean": 0.17844289541244507, "nonfinite": 0.0, "weighted_memory_to_active": 0.3079022169113159, "mix_std": 0.1151244193315506, "beta_std": 0.13564646244049072, "beta_low": 0.3535970449447632, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.04340810328722, "output_influence": 0.25450557470321655, "residual_influence": 0.06905189156532288, "state_active": 0.027205662801861763, "state_memory": 0.035442840307950974, "state_peak": 42.514892578125, "growth_active": 1.1239025592803955, "growth_memory": 1.448353886604309, "heads": [{"head": 0, "memory_to_active": 0.0375443771481514, "contribution_to_active": 0.4877532720565796, "mix_mean": 0.5220699906349182, "mix_low": 0.005911827087402344, "mix_high": 0.002544403076171875, "read_cosine": 0.023698555305600166, "beta_mean": 0.00022778325364924967, "nonfinite": 0.0, "weighted_memory_to_active": 0.0236035268753767, "mix_std": 0.2805419862270355, "beta_std": 0.0004436285817064345, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.022856639698147774, "evidence_weights": [-0.20607005059719086, -0.2704974412918091, 0.31580090522766113], "agreement_weight": 0.20543786883354187, "orthogonal_weight": -1.0136840343475342}, {"head": 1, "memory_to_active": 3.003908157348633, "contribution_to_active": 0.2754099667072296, "mix_mean": 0.023562125861644745, "mix_low": 0.4622049331665039, "mix_high": 0.0, "read_cosine": 0.5174293518066406, "beta_mean": 0.25674229860305786, "nonfinite": 0.0, "weighted_memory_to_active": 0.3122625946998596, "mix_std": 0.04591890424489975, "beta_std": 0.19432929158210754, "beta_low": 0.0025634765625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.020451465621590614, "evidence_weights": [0.021729959174990654, -0.06445059925317764, 0.06802408397197723], "agreement_weight": 9.71708504948765e-06, "orthogonal_weight": 0.19697017967700958}, {"head": 2, "memory_to_active": 0.04511694237589836, "contribution_to_active": 0.5591914653778076, "mix_mean": 0.6659634113311768, "mix_low": 0.0004291534423828125, "mix_high": 0.004435539245605469, "read_cosine": 0.44437462091445923, "beta_mean": 0.0014376051258295774, "nonfinite": 0.0, "weighted_memory_to_active": 0.030250731855630875, "mix_std": 0.23087693750858307, "beta_std": 0.0034719922114163637, "beta_low": 0.9859619140625, "beta_high": 0.0, "orthogonal_to_active": 0.020187970250844955, "evidence_weights": [-0.27884796261787415, -0.3731591999530792, 0.6754113435745239], "agreement_weight": 0.04253446310758591, "orthogonal_weight": -0.8043516874313354}, {"head": 3, "memory_to_active": 3.016317129135132, "contribution_to_active": 0.1806531697511673, "mix_mean": 0.025357112288475037, "mix_low": 0.5108604431152344, "mix_high": 0.0, "read_cosine": 0.19693848490715027, "beta_mean": 0.3276379406452179, "nonfinite": 0.0, "weighted_memory_to_active": 0.2689668834209442, "mix_std": 0.061384886503219604, "beta_std": 0.26632824540138245, "beta_low": 0.06182861328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1103530153632164, "evidence_weights": [0.03646623715758324, -0.021334005519747734, -0.035159848630428314], "agreement_weight": -0.028590068221092224, "orthogonal_weight": -0.49388444423675537}, {"head": 4, "memory_to_active": 3.9499619007110596, "contribution_to_active": 1.1840577125549316, "mix_mean": 0.03213680163025856, "mix_low": 0.3245248794555664, "mix_high": 0.0, "read_cosine": 0.32234567403793335, "beta_mean": 0.17651844024658203, "nonfinite": 0.0, "weighted_memory_to_active": 1.1418404579162598, "mix_std": 0.05281840264797211, "beta_std": 0.12655094265937805, "beta_low": 0.035400390625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.06055847927927971, "evidence_weights": [-0.02149869129061699, -0.0010093688033521175, 0.15270131826400757], "agreement_weight": -0.010260770097374916, "orthogonal_weight": 0.07859635353088379}, {"head": 5, "memory_to_active": 4.540353775024414, "contribution_to_active": 0.10460567474365234, "mix_mean": 0.005873477086424828, "mix_low": 0.8721323013305664, "mix_high": 0.0, "read_cosine": 0.2575444281101227, "beta_mean": 0.30809324979782104, "nonfinite": 0.0, "weighted_memory_to_active": 0.07048918306827545, "mix_std": 0.01920536532998085, "beta_std": 0.22275464236736298, "beta_low": 0.03582763671875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.026041055098176003, "evidence_weights": [0.16898514330387115, 0.09013307839632034, 0.08702085167169571], "agreement_weight": -0.11729304492473602, "orthogonal_weight": 0.3716753125190735}]}, {"layer": 2, "memory_to_active": 1.0030043125152588, "contribution_to_active": 0.3485758900642395, "mix_mean": 0.4064318835735321, "mix_low": 0.24864499270915985, "mix_high": 0.004229545593261719, "read_cosine": 0.2990885078907013, "beta_mean": 0.08817499876022339, "nonfinite": 0.0, "weighted_memory_to_active": 0.06899917125701904, "mix_std": 0.18692553043365479, "beta_std": 0.06990151107311249, "beta_low": 0.593536376953125, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.016473131254315376, "output_influence": 0.3939989507198334, "residual_influence": 0.10685623437166214, "state_active": 0.03964832052588463, "state_memory": 0.014033819548785686, "state_peak": 33.582115173339844, "growth_active": 1.0432711839675903, "growth_memory": 1.5435227155685425, "heads": [{"head": 0, "memory_to_active": 0.007411208003759384, "contribution_to_active": 0.3757384419441223, "mix_mean": 0.6430859565734863, "mix_low": 0.000553131103515625, "mix_high": 0.008955955505371094, "read_cosine": 0.14740589261054993, "beta_mean": 5.8565809013089165e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004932223819196224, "mix_std": 0.2651142179965973, "beta_std": 9.772494377102703e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005258017219603062, "evidence_weights": [-0.376238077878952, -0.22005756199359894, 0.19746160507202148], "agreement_weight": 0.12449992448091507, "orthogonal_weight": -1.1101694107055664}, {"head": 1, "memory_to_active": 3.070176124572754, "contribution_to_active": 0.04266032949090004, "mix_mean": 0.005299902521073818, "mix_low": 0.8846035003662109, "mix_high": 0.0, "read_cosine": 0.14290697872638702, "beta_mean": 0.24185530841350555, "nonfinite": 0.0, "weighted_memory_to_active": 0.03307411074638367, "mix_std": 0.011661756783723831, "beta_std": 0.1543252021074295, "beta_low": 0.00140380859375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.010012537240982056, "evidence_weights": [0.13902989029884338, 0.11549461632966995, 0.5902246832847595], "agreement_weight": 0.033921558409929276, "orthogonal_weight": 0.3222462832927704}, {"head": 2, "memory_to_active": 0.010675126686692238, "contribution_to_active": 0.42750322818756104, "mix_mean": 0.5507434010505676, "mix_low": 0.0012798309326171875, "mix_high": 0.0030469894409179688, "read_cosine": 0.3889097571372986, "beta_mean": 3.4983364457730204e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004474418703466654, "mix_std": 0.2740684747695923, "beta_std": 7.793859549565241e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00011301189078949392, "evidence_weights": [-0.2538849711418152, -0.29427415132522583, 0.5960087180137634], "agreement_weight": -0.04558992013335228, "orthogonal_weight": -0.030234411358833313}, {"head": 3, "memory_to_active": 0.10933192074298859, "contribution_to_active": 0.5308719873428345, "mix_mean": 0.5722158551216125, "mix_low": 0.000244140625, "mix_high": 0.005944252014160156, "read_cosine": 0.5691465735435486, "beta_mean": 0.018994158133864403, "nonfinite": 0.0, "weighted_memory_to_active": 0.067715123295784, "mix_std": 0.2576625645160675, "beta_std": 0.02754727564752102, "beta_low": 0.525390625, "beta_high": 0.0, "orthogonal_to_active": 0.04810711741447449, "evidence_weights": [-0.20474591851234436, -0.3238210678100586, 0.7653244733810425], "agreement_weight": 0.05233146995306015, "orthogonal_weight": -0.893692672252655}, {"head": 4, "memory_to_active": 2.8111114501953125, "contribution_to_active": 0.19514891505241394, "mix_mean": 0.0194768738001585, "mix_low": 0.6044082641601562, "mix_high": 0.0, "read_cosine": 0.20496945083141327, "beta_mean": 0.2680753469467163, "nonfinite": 0.0, "weighted_memory_to_active": 0.298710435628891, "mix_std": 0.05040769651532173, "beta_std": 0.23731796443462372, "beta_low": 0.034423828125, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.03402145579457283, "evidence_weights": [0.028105979785323143, 0.008259670808911324, 0.030197953805327415], "agreement_weight": 0.027000682428479195, "orthogonal_weight": -0.15351155400276184}, {"head": 5, "memory_to_active": 0.009319775737822056, "contribution_to_active": 0.519532322883606, "mix_mean": 0.647769033908844, "mix_low": 0.0007810592651367188, "mix_high": 0.007430076599121094, "read_cosine": 0.341192364692688, "beta_mean": 3.165808448102325e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005088697653263807, "mix_std": 0.26263850927352905, "beta_std": 4.2959742131643e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013266440946608782, "evidence_weights": [-0.2793406546115875, -0.2888047993183136, 0.9295858144760132], "agreement_weight": 0.25329816341400146, "orthogonal_weight": -0.308951199054718}]}, {"layer": 3, "memory_to_active": 0.4480486810207367, "contribution_to_active": 0.48342999815940857, "mix_mean": 0.5147569179534912, "mix_low": 0.05075661465525627, "mix_high": 0.011449973098933697, "read_cosine": 0.20251058042049408, "beta_mean": 0.02367609366774559, "nonfinite": 0.0, "weighted_memory_to_active": 0.13056083023548126, "mix_std": 0.25814297795295715, "beta_std": 0.024134747684001923, "beta_low": 0.7698466181755066, "beta_high": 0.0, "orthogonal_to_active": 0.053711384534835815, "output_influence": 0.42108404636383057, "residual_influence": 0.09759783744812012, "state_active": 0.02309693768620491, "state_memory": 0.006863623857498169, "state_peak": 23.85007667541504, "growth_active": 1.0374467372894287, "growth_memory": 1.6715571880340576, "heads": [{"head": 0, "memory_to_active": 0.35752272605895996, "contribution_to_active": 0.6066414713859558, "mix_mean": 0.6077232360839844, "mix_low": 0.0012636184692382812, "mix_high": 0.02406787872314453, "read_cosine": 0.24041491746902466, "beta_mean": 0.010127437300980091, "nonfinite": 0.0, "weighted_memory_to_active": 0.2255970686674118, "mix_std": 0.30333131551742554, "beta_std": 0.01268005557358265, "beta_low": 0.693115234375, "beta_high": 0.0, "orthogonal_to_active": 0.1630418747663498, "evidence_weights": [-0.21704789996147156, -0.25024738907814026, 1.0141628980636597], "agreement_weight": 0.11604002863168716, "orthogonal_weight": -0.7623618841171265}, {"head": 1, "memory_to_active": 0.008637184277176857, "contribution_to_active": 0.46224623918533325, "mix_mean": 0.5796257257461548, "mix_low": 0.0036182403564453125, "mix_high": 0.010099411010742188, "read_cosine": 0.17504921555519104, "beta_mean": 4.1215476812794805e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0053131780587136745, "mix_std": 0.2955206632614136, "beta_std": 9.321256948169321e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004485022276639938, "evidence_weights": [-0.2694864869117737, -0.21829406917095184, 0.6770487427711487], "agreement_weight": 0.5353537201881409, "orthogonal_weight": -0.9251154661178589}, {"head": 2, "memory_to_active": 2.036428928375244, "contribution_to_active": 0.30006474256515503, "mix_mean": 0.05737566575407982, "mix_low": 0.2894926071166992, "mix_high": 1.621246337890625e-05, "read_cosine": 0.20175521075725555, "beta_mean": 0.12729302048683167, "nonfinite": 0.0, "weighted_memory_to_active": 0.33597269654273987, "mix_std": 0.10935746133327484, "beta_std": 0.1201343983411789, "beta_low": 0.0208740234375, "beta_high": 0.0, "orthogonal_to_active": 0.0006986780790612102, "evidence_weights": [-0.10051116347312927, -0.007353570312261581, -0.028230061754584312], "agreement_weight": -0.018952513113617897, "orthogonal_weight": -0.002637699479237199}, {"head": 3, "memory_to_active": 0.01629815623164177, "contribution_to_active": 0.5177686810493469, "mix_mean": 0.5974310040473938, "mix_low": 0.004984855651855469, "mix_high": 0.009070396423339844, "read_cosine": 0.281206876039505, "beta_mean": 7.856517913751304e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010919769294559956, "mix_std": 0.28928452730178833, "beta_std": 0.0001805124047677964, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0027854079380631447, "evidence_weights": [-0.2589191198348999, -0.22489935159683228, 2.2632663249969482], "agreement_weight": 2.1814475059509277, "orthogonal_weight": -0.3309461176395416}, {"head": 4, "memory_to_active": 0.012231283821165562, "contribution_to_active": 0.365081250667572, "mix_mean": 0.5075852274894714, "mix_low": 0.0034914016723632812, "mix_high": 0.0020380020141601562, "read_cosine": 0.14527340233325958, "beta_mean": 6.80139783071354e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008022984489798546, "mix_std": 0.2898791432380676, "beta_std": 0.00012125206558266655, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007206922397017479, "evidence_weights": [-0.23051096498966217, -0.21812109649181366, 0.1324317902326584], "agreement_weight": -0.052299827337265015, "orthogonal_weight": -0.978714644908905}, {"head": 5, "memory_to_active": 0.25717368721961975, "contribution_to_active": 0.6487773060798645, "mix_mean": 0.7388002872467041, "mix_low": 0.0016889572143554688, "mix_high": 0.023407936096191406, "read_cosine": 0.17136383056640625, "beta_mean": 0.004448302090167999, "nonfinite": 0.0, "weighted_memory_to_active": 0.197539284825325, "mix_std": 0.26148471236228943, "beta_std": 0.011599058285355568, "beta_low": 0.90509033203125, "beta_high": 0.0, "orthogonal_to_active": 0.14405040442943573, "evidence_weights": [-0.26460668444633484, -0.3431563675403595, 0.5332134962081909], "agreement_weight": 0.21626615524291992, "orthogonal_weight": -0.7746396660804749}]}, {"layer": 4, "memory_to_active": 0.9823941588401794, "contribution_to_active": 0.47278833389282227, "mix_mean": 0.36731740832328796, "mix_low": 0.2518247067928314, "mix_high": 0.009780407883226871, "read_cosine": 0.3045027256011963, "beta_mean": 0.10842689871788025, "nonfinite": 0.0, "weighted_memory_to_active": 0.12466850876808167, "mix_std": 0.161018505692482, "beta_std": 0.10247990489006042, "beta_low": 0.394439697265625, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04976068437099457, "output_influence": 0.4917568564414978, "residual_influence": 0.14969198405742645, "state_active": 0.040245622396469116, "state_memory": 0.016622016206383705, "state_peak": 33.85520553588867, "growth_active": 1.2393367290496826, "growth_memory": 1.5258855819702148, "heads": [{"head": 0, "memory_to_active": 0.15672767162322998, "contribution_to_active": 0.7951122522354126, "mix_mean": 0.672960638999939, "mix_low": 0.013484001159667969, "mix_high": 0.026376724243164062, "read_cosine": 0.43371495604515076, "beta_mean": 0.26267164945602417, "nonfinite": 0.0, "weighted_memory_to_active": 0.13226047158241272, "mix_std": 0.35247981548309326, "beta_std": 0.2574233412742615, "beta_low": 0.2520751953125, "beta_high": 0.0, "orthogonal_to_active": 0.07987324893474579, "evidence_weights": [-0.25365468859672546, -0.3619481325149536, -0.11418051272630692], "agreement_weight": 0.0036609789822250605, "orthogonal_weight": 0.7086525559425354}, {"head": 1, "memory_to_active": 0.05266714096069336, "contribution_to_active": 0.6597365140914917, "mix_mean": 0.6960139870643616, "mix_low": 0.0001125335693359375, "mix_high": 0.01581573486328125, "read_cosine": 0.5788639783859253, "beta_mean": 0.001417156308889389, "nonfinite": 0.0, "weighted_memory_to_active": 0.03898916766047478, "mix_std": 0.2399469017982483, "beta_std": 0.010266759432852268, "beta_low": 0.9781494140625, "beta_high": 0.0, "orthogonal_to_active": 0.020354274660348892, "evidence_weights": [-0.2380814403295517, -0.3634970188140869, 0.6079862713813782], "agreement_weight": 0.33448469638824463, "orthogonal_weight": -0.7326748967170715}, {"head": 2, "memory_to_active": 1.5693854093551636, "contribution_to_active": 0.3390631675720215, "mix_mean": 0.03965803608298302, "mix_low": 0.40253734588623047, "mix_high": 1.9073486328125e-05, "read_cosine": 0.2385130673646927, "beta_mean": 0.0617031566798687, "nonfinite": 0.0, "weighted_memory_to_active": 0.2299501746892929, "mix_std": 0.07726599276065826, "beta_std": 0.0668434202671051, "beta_low": 0.1123046875, "beta_high": 0.0, "orthogonal_to_active": 0.07428714632987976, "evidence_weights": [0.041670773178339005, -0.03480762988328934, 0.2541905641555786], "agreement_weight": 0.1377793848514557, "orthogonal_weight": -0.420791357755661}, {"head": 3, "memory_to_active": 1.810829997062683, "contribution_to_active": 0.1416717767715454, "mix_mean": 0.03214051574468613, "mix_low": 0.4195432662963867, "mix_high": 0.0, "read_cosine": 0.20711274445056915, "beta_mean": 0.13734658062458038, "nonfinite": 0.0, "weighted_memory_to_active": 0.20495757460594177, "mix_std": 0.05648815631866455, "beta_std": 0.1249292865395546, "beta_low": 0.0240478515625, "beta_high": 0.0, "orthogonal_to_active": 0.0006392882205545902, "evidence_weights": [-0.02193089947104454, 0.04795757681131363, -0.24642395973205566], "agreement_weight": 0.07929366827011108, "orthogonal_weight": -0.0047020092606544495}, {"head": 4, "memory_to_active": 0.008713647723197937, "contribution_to_active": 0.6348915100097656, "mix_mean": 0.7503958940505981, "mix_low": 4.100799560546875e-05, "mix_high": 0.01646137237548828, "read_cosine": 0.14775404334068298, "beta_mean": 2.2014963178662583e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006214642897248268, "mix_std": 0.21033838391304016, "beta_std": 4.507166886469349e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003928079269826412, "evidence_weights": [-0.332180917263031, -0.32146430015563965, 0.3555142879486084], "agreement_weight": 0.334966778755188, "orthogonal_weight": -0.6688160300254822}, {"head": 5, "memory_to_active": 2.2960410118103027, "contribution_to_active": 0.26625481247901917, "mix_mean": 0.012735459953546524, "mix_low": 0.6752300262451172, "mix_high": 9.5367431640625e-06, "read_cosine": 0.22105740010738373, "beta_mean": 0.18740084767341614, "nonfinite": 0.0, "weighted_memory_to_active": 0.1356389969587326, "mix_std": 0.029591763392090797, "beta_std": 0.1553715616464615, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.11948207765817642, "evidence_weights": [0.059297576546669006, -0.01697978377342224, -0.27710890769958496], "agreement_weight": -0.08516938984394073, "orthogonal_weight": 1.0441312789916992}]}, {"layer": 5, "memory_to_active": 0.39363402128219604, "contribution_to_active": 0.5363034009933472, "mix_mean": 0.5096603035926819, "mix_low": 0.05762752145528793, "mix_high": 0.020353158935904503, "read_cosine": 0.07184602320194244, "beta_mean": 0.056405067443847656, "nonfinite": 0.0, "weighted_memory_to_active": 0.1285376250743866, "mix_std": 0.25302889943122864, "beta_std": 0.07159269601106644, "beta_low": 0.6716919541358948, "beta_high": 0.0, "orthogonal_to_active": 0.05406869202852249, "output_influence": 0.5676804780960083, "residual_influence": 0.17500829696655273, "state_active": 0.04120732471346855, "state_memory": 0.010284680873155594, "state_peak": 61.020416259765625, "growth_active": 1.1061146259307861, "growth_memory": 1.2577635049819946, "heads": [{"head": 0, "memory_to_active": 0.19124464690685272, "contribution_to_active": 0.5539745092391968, "mix_mean": 0.45067447423934937, "mix_low": 0.008959770202636719, "mix_high": 0.0012273788452148438, "read_cosine": -0.140414297580719, "beta_mean": 0.01928623765707016, "nonfinite": 0.0, "weighted_memory_to_active": 0.06871946156024933, "mix_std": 0.2772758901119232, "beta_std": 0.032853174954652786, "beta_low": 0.53619384765625, "beta_high": 0.0, "orthogonal_to_active": 0.029746901243925095, "evidence_weights": [-0.16263209283351898, -0.18265706300735474, 0.09663448482751846], "agreement_weight": -0.019972248002886772, "orthogonal_weight": -0.46136319637298584}, {"head": 1, "memory_to_active": 0.008063293993473053, "contribution_to_active": 0.5440560579299927, "mix_mean": 0.7024536728858948, "mix_low": 0.000949859619140625, "mix_high": 0.042473793029785156, "read_cosine": 0.2009657323360443, "beta_mean": 2.043561835307628e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004951581824570894, "mix_std": 0.26923608779907227, "beta_std": 4.398212695377879e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00453563965857029, "evidence_weights": [-0.2516503632068634, -0.29094386100769043, 0.9682375192642212], "agreement_weight": 0.6967535018920898, "orthogonal_weight": -0.9680868983268738}, {"head": 2, "memory_to_active": 0.01725476235151291, "contribution_to_active": 0.6098516583442688, "mix_mean": 0.6649922728538513, "mix_low": 0.00205230712890625, "mix_high": 0.023458480834960938, "read_cosine": -0.4845661520957947, "beta_mean": 0.00010456382005941123, "nonfinite": 0.0, "weighted_memory_to_active": 0.010864000767469406, "mix_std": 0.2621702551841736, "beta_std": 0.00027784419944509864, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.009006673470139503, "evidence_weights": [-0.26340365409851074, -0.3274405002593994, 0.14863784611225128], "agreement_weight": -0.12773729860782623, "orthogonal_weight": -0.9549117088317871}, {"head": 3, "memory_to_active": 1.6391465663909912, "contribution_to_active": 0.23750552535057068, "mix_mean": 0.1139865517616272, "mix_low": 0.12511348724365234, "mix_high": 0.0, "read_cosine": 0.1529838740825653, "beta_mean": 0.11348798871040344, "nonfinite": 0.0, "weighted_memory_to_active": 0.36129170656204224, "mix_std": 0.14747047424316406, "beta_std": 0.1655283272266388, "beta_low": 0.1925048828125, "beta_high": 0.0, "orthogonal_to_active": 0.147628054022789, "evidence_weights": [-0.0759214386343956, -0.09940145164728165, 0.050615567713975906], "agreement_weight": 0.045299284160137177, "orthogonal_weight": -0.5023314356803894}, {"head": 4, "memory_to_active": 0.43225112557411194, "contribution_to_active": 0.5839998126029968, "mix_mean": 0.31786400079727173, "mix_low": 0.2070770263671875, "mix_high": 0.00563812255859375, "read_cosine": 0.5377204418182373, "beta_mean": 0.20529747009277344, "nonfinite": 0.0, "weighted_memory_to_active": 0.27447742223739624, "mix_std": 0.3403525948524475, "beta_std": 0.22979360818862915, "beta_low": 0.303955078125, "beta_high": 0.0, "orthogonal_to_active": 0.1032392680644989, "evidence_weights": [-0.11818266659975052, -0.2578766942024231, -0.049574486911296844], "agreement_weight": 0.03150156885385513, "orthogonal_weight": 0.572332501411438}, {"head": 5, "memory_to_active": 0.0738435909152031, "contribution_to_active": 0.688432514667511, "mix_mean": 0.8079906105995178, "mix_low": 0.0016126632690429688, "mix_high": 0.04932117462158203, "read_cosine": 0.16438648104667664, "beta_mean": 0.00023369365953840315, "nonfinite": 0.0, "weighted_memory_to_active": 0.05092158913612366, "mix_std": 0.22166807949543, "beta_std": 0.0010591915342956781, "beta_low": 0.99755859375, "beta_high": 0.0, "orthogonal_to_active": 0.030255606397986412, "evidence_weights": [-0.30920639634132385, -0.33316880464553833, 1.8945428133010864], "agreement_weight": 0.5005428194999695, "orthogonal_weight": -0.6232545375823975}]}, {"layer": 6, "memory_to_active": 0.2743551433086395, "contribution_to_active": 0.6246801614761353, "mix_mean": 0.592350959777832, "mix_low": 0.04073333740234375, "mix_high": 0.0314229354262352, "read_cosine": 0.32069727778434753, "beta_mean": 0.06848625838756561, "nonfinite": 0.0, "weighted_memory_to_active": 0.14014390110969543, "mix_std": 0.23861563205718994, "beta_std": 0.0808487981557846, "beta_low": 0.5840861201286316, "beta_high": 0.0, "orthogonal_to_active": 0.056978125125169754, "output_influence": 0.5628941059112549, "residual_influence": 0.19792845845222473, "state_active": 0.046377506107091904, "state_memory": 0.007316756062209606, "state_peak": 95.56683349609375, "growth_active": 1.2251858711242676, "growth_memory": 1.1959073543548584, "heads": [{"head": 0, "memory_to_active": 0.11796052753925323, "contribution_to_active": 0.7610189318656921, "mix_mean": 0.8222858309745789, "mix_low": 0.000110626220703125, "mix_high": 0.05681419372558594, "read_cosine": 0.26469361782073975, "beta_mean": 0.014113374054431915, "nonfinite": 0.0, "weighted_memory_to_active": 0.10347799211740494, "mix_std": 0.20068570971488953, "beta_std": 0.0408799946308136, "beta_low": 0.78656005859375, "beta_high": 0.0, "orthogonal_to_active": 0.07291331887245178, "evidence_weights": [-0.3103734254837036, -0.35196635127067566, 1.2143422365188599], "agreement_weight": 0.7267709374427795, "orthogonal_weight": -0.7517629265785217}, {"head": 1, "memory_to_active": 0.006471461616456509, "contribution_to_active": 0.5182834267616272, "mix_mean": 0.5964590311050415, "mix_low": 0.0008335113525390625, "mix_high": 0.0157012939453125, "read_cosine": 0.4866602122783661, "beta_mean": 1.3846733963873703e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029037059284746647, "mix_std": 0.2820785641670227, "beta_std": 2.1688098058803007e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015163326170295477, "evidence_weights": [-0.19631339609622955, -0.3029553294181824, 1.1060352325439453], "agreement_weight": 0.11211079359054565, "orthogonal_weight": -0.686953067779541}, {"head": 2, "memory_to_active": 0.7488187551498413, "contribution_to_active": 0.099031001329422, "mix_mean": 0.050714097917079926, "mix_low": 0.2359638214111328, "mix_high": 0.0, "read_cosine": 0.2707306146621704, "beta_mean": 0.02711157314479351, "nonfinite": 0.0, "weighted_memory_to_active": 0.06766238808631897, "mix_std": 0.07664380967617035, "beta_std": 0.032681163400411606, "beta_low": 0.208984375, "beta_high": 0.0, "orthogonal_to_active": 0.010021189227700233, "evidence_weights": [0.06672301143407822, -0.056845296174287796, 0.3468070924282074], "agreement_weight": 0.14563627541065216, "orthogonal_weight": -0.17400804162025452}, {"head": 3, "memory_to_active": 0.1442706286907196, "contribution_to_active": 0.6718254089355469, "mix_mean": 0.655617356300354, "mix_low": 0.00678253173828125, "mix_high": 0.021648406982421875, "read_cosine": 0.4795711040496826, "beta_mean": 0.2337043732404709, "nonfinite": 0.0, "weighted_memory_to_active": 0.10117550939321518, "mix_std": 0.3253863751888275, "beta_std": 0.2604767382144928, "beta_low": 0.20123291015625, "beta_high": 0.0, "orthogonal_to_active": 0.05680170655250549, "evidence_weights": [-0.17204931378364563, -0.32315054535865784, 0.14978384971618652], "agreement_weight": -0.016530221328139305, "orthogonal_weight": 0.6498348116874695}, {"head": 4, "memory_to_active": 0.5950261950492859, "contribution_to_active": 1.0641834735870361, "mix_mean": 0.742973268032074, "mix_low": 0.0005674362182617188, "mix_high": 0.0866556167602539, "read_cosine": 0.1437329649925232, "beta_mean": 0.13581857085227966, "nonfinite": 0.0, "weighted_memory_to_active": 0.5416355729103088, "mix_std": 0.3023989200592041, "beta_std": 0.15072289109230042, "beta_low": 0.3077392578125, "beta_high": 0.0, "orthogonal_to_active": 0.17733532190322876, "evidence_weights": [-0.19638758897781372, -0.5486094951629639, -0.5325320363044739], "agreement_weight": -0.01666417345404625, "orthogonal_weight": 0.35288161039352417}, {"head": 5, "memory_to_active": 0.033583059906959534, "contribution_to_active": 0.63373863697052, "mix_mean": 0.6860560178756714, "mix_low": 0.00014209747314453125, "mix_high": 0.007718086242675781, "read_cosine": 0.2787952423095703, "beta_mean": 0.00015580607578158379, "nonfinite": 0.0, "weighted_memory_to_active": 0.024008188396692276, "mix_std": 0.24450036883354187, "beta_std": 0.00031031356775201857, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.023280862718820572, "evidence_weights": [-0.23682743310928345, -0.31743162870407104, 0.38514402508735657], "agreement_weight": 0.2107059806585312, "orthogonal_weight": -1.0417033433914185}]}, {"layer": 7, "memory_to_active": 1.7682470083236694, "contribution_to_active": 0.27723076939582825, "mix_mean": 0.17243927717208862, "mix_low": 0.22983218729496002, "mix_high": 0.0011739730834960938, "read_cosine": 0.23967128992080688, "beta_mean": 0.14021120965480804, "nonfinite": 0.0, "weighted_memory_to_active": 0.21615242958068848, "mix_std": 0.14922939240932465, "beta_std": 0.10230579227209091, "beta_low": 0.207489013671875, "beta_high": 0.0004781087627634406, "orthogonal_to_active": 0.09211887419223785, "output_influence": 0.28671717643737793, "residual_influence": 0.06909756362438202, "state_active": 0.01415538601577282, "state_memory": 0.024599017575383186, "state_peak": 31.229652404785156, "growth_active": 1.1627546548843384, "growth_memory": 1.2711399793624878, "heads": [{"head": 0, "memory_to_active": 2.5144717693328857, "contribution_to_active": 0.2108871042728424, "mix_mean": 0.03929607570171356, "mix_low": 0.4027719497680664, "mix_high": 9.5367431640625e-07, "read_cosine": 0.33536916971206665, "beta_mean": 0.17249172925949097, "nonfinite": 0.0, "weighted_memory_to_active": 0.24604883790016174, "mix_std": 0.07879827916622162, "beta_std": 0.1366603970527649, "beta_low": 0.004150390625, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.0003623563679866493, "evidence_weights": [0.04139580950140953, -0.08312416821718216, 0.012210965156555176], "agreement_weight": -0.029966847971081734, "orthogonal_weight": -0.0019764539320021868}, {"head": 1, "memory_to_active": 0.9711072444915771, "contribution_to_active": 0.3943798840045929, "mix_mean": 0.22396814823150635, "mix_low": 0.11398696899414062, "mix_high": 0.0063419342041015625, "read_cosine": 0.39259451627731323, "beta_mean": 0.09786013513803482, "nonfinite": 0.0, "weighted_memory_to_active": 0.33136624097824097, "mix_std": 0.24676959216594696, "beta_std": 0.06725738942623138, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04538227617740631, "evidence_weights": [-0.21689951419830322, -0.10405008494853973, -0.28957512974739075], "agreement_weight": 0.05423762649297714, "orthogonal_weight": 0.16474421322345734}, {"head": 2, "memory_to_active": 0.6063470840454102, "contribution_to_active": 0.37325042486190796, "mix_mean": 0.2202485203742981, "mix_low": 0.022772789001464844, "mix_high": 0.00035953521728515625, "read_cosine": 0.11420223116874695, "beta_mean": 0.04032401368021965, "nonfinite": 0.0, "weighted_memory_to_active": 0.22333277761936188, "mix_std": 0.21072332561016083, "beta_std": 0.03951406106352806, "beta_low": 0.109375, "beta_high": 0.0, "orthogonal_to_active": 0.12575948238372803, "evidence_weights": [-0.09102540463209152, -0.17659330368041992, 0.8244855999946594], "agreement_weight": 0.01898229494690895, "orthogonal_weight": -0.5800416469573975}, {"head": 3, "memory_to_active": 4.946448802947998, "contribution_to_active": 0.11802494525909424, "mix_mean": 0.03660135343670845, "mix_low": 0.3246927261352539, "mix_high": 0.0, "read_cosine": 0.10650649666786194, "beta_mean": 0.4624391794204712, "nonfinite": 0.0, "weighted_memory_to_active": 0.40413689613342285, "mix_std": 0.055638961493968964, "beta_std": 0.2951728105545044, "beta_low": 0.0018310546875, "beta_high": 0.00244140625, "orthogonal_to_active": 0.346426784992218, "evidence_weights": [0.07302091270685196, -0.03719939664006233, 0.7188131213188171], "agreement_weight": 0.05087752640247345, "orthogonal_weight": -0.9324309229850769}, {"head": 4, "memory_to_active": 1.5310778617858887, "contribution_to_active": 0.10597041249275208, "mix_mean": 0.024751242250204086, "mix_low": 0.5132970809936523, "mix_high": 0.0, "read_cosine": 0.22838324308395386, "beta_mean": 0.06796707212924957, "nonfinite": 0.0, "weighted_memory_to_active": 0.07171206176280975, "mix_std": 0.05054086819291115, "beta_std": 0.07476896792650223, "beta_low": 0.1297607421875, "beta_high": 0.0, "orthogonal_to_active": 0.01586354523897171, "evidence_weights": [0.13021527230739594, -0.02009284868836403, 0.1908448487520218], "agreement_weight": 0.035733964294195175, "orthogonal_weight": -0.27561140060424805}, {"head": 5, "memory_to_active": 0.04002850502729416, "contribution_to_active": 0.46087175607681274, "mix_mean": 0.4897702932357788, "mix_low": 0.0014715194702148438, "mix_high": 0.0003414154052734375, "read_cosine": 0.2609720230102539, "beta_mean": 0.00018511178495828062, "nonfinite": 0.0, "weighted_memory_to_active": 0.02031760662794113, "mix_std": 0.2529052793979645, "beta_std": 0.0004611482727341354, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.01891881413757801, "evidence_weights": [-0.18120206892490387, -0.23730482161045074, 0.5345689654350281], "agreement_weight": 0.22219975292682648, "orthogonal_weight": -1.0005029439926147}]}, {"layer": 8, "memory_to_active": 0.13868823647499084, "contribution_to_active": 0.5627164244651794, "mix_mean": 0.49168065190315247, "mix_low": 0.011478106491267681, "mix_high": 0.0018698375206440687, "read_cosine": 0.41358456015586853, "beta_mean": 0.013892917893826962, "nonfinite": 0.0, "weighted_memory_to_active": 0.07131306827068329, "mix_std": 0.2413352131843567, "beta_std": 0.0193539559841156, "beta_low": 0.780426025390625, "beta_high": 0.0, "orthogonal_to_active": 0.0644398182630539, "output_influence": 0.5155066251754761, "residual_influence": 0.18397928774356842, "state_active": 0.04849729314446449, "state_memory": 0.005240270867943764, "state_peak": 49.721275329589844, "growth_active": 1.1821714639663696, "growth_memory": 1.1027079820632935, "heads": [{"head": 0, "memory_to_active": 0.06201322376728058, "contribution_to_active": 0.5733753442764282, "mix_mean": 0.6061888933181763, "mix_low": 0.0006656646728515625, "mix_high": 0.0016984939575195312, "read_cosine": 0.4453560709953308, "beta_mean": 0.0032856883481144905, "nonfinite": 0.0, "weighted_memory_to_active": 0.04426416754722595, "mix_std": 0.2505921423435211, "beta_std": 0.012793583795428276, "beta_low": 0.93072509765625, "beta_high": 0.0, "orthogonal_to_active": 0.027970165014266968, "evidence_weights": [-0.16222326457500458, -0.38386186957359314, 0.5904656648635864], "agreement_weight": 0.2575834095478058, "orthogonal_weight": -0.7949407696723938}, {"head": 1, "memory_to_active": 0.07485302537679672, "contribution_to_active": 0.5595417618751526, "mix_mean": 0.5283178091049194, "mix_low": 7.343292236328125e-05, "mix_high": 0.0015306472778320312, "read_cosine": 0.10156922042369843, "beta_mean": 0.0009926084894686937, "nonfinite": 0.0, "weighted_memory_to_active": 0.03343099728226662, "mix_std": 0.23531049489974976, "beta_std": 0.006376066245138645, "beta_low": 0.98919677734375, "beta_high": 0.0, "orthogonal_to_active": 0.024079248309135437, "evidence_weights": [-0.19546233117580414, -0.2765156030654907, 0.6865139603614807], "agreement_weight": 0.07168973982334137, "orthogonal_weight": -0.7569553852081299}, {"head": 2, "memory_to_active": 0.013618354685604572, "contribution_to_active": 0.6847472190856934, "mix_mean": 0.6814767122268677, "mix_low": 0.0005254745483398438, "mix_high": 0.004050254821777344, "read_cosine": 0.4240358769893646, "beta_mean": 0.00012827820319216698, "nonfinite": 0.0, "weighted_memory_to_active": 0.010778005234897137, "mix_std": 0.22316840291023254, "beta_std": 0.0030138124711811543, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006489743944257498, "evidence_weights": [-0.22468747198581696, -0.3464248776435852, 1.0147372484207153], "agreement_weight": 0.28954169154167175, "orthogonal_weight": -0.7033138275146484}, {"head": 3, "memory_to_active": 0.09486960619688034, "contribution_to_active": 0.6405656337738037, "mix_mean": 0.5779851675033569, "mix_low": 0.0006628036499023438, "mix_high": 0.0037555694580078125, "read_cosine": 0.8155407309532166, "beta_mean": 0.00797745306044817, "nonfinite": 0.0, "weighted_memory_to_active": 0.07223265618085861, "mix_std": 0.25603875517845154, "beta_std": 0.007749306969344616, "beta_low": 0.7269287109375, "beta_high": 0.0, "orthogonal_to_active": 0.021935705095529556, "evidence_weights": [-0.17892161011695862, -0.29881614446640015, 0.5427582859992981], "agreement_weight": 0.24272289872169495, "orthogonal_weight": -0.6078681945800781}, {"head": 4, "memory_to_active": 0.2360285371541977, "contribution_to_active": 0.30728504061698914, "mix_mean": 0.2941349744796753, "mix_low": 0.005114555358886719, "mix_high": 9.5367431640625e-07, "read_cosine": 0.18255682289600372, "beta_mean": 0.000990026630461216, "nonfinite": 0.0, "weighted_memory_to_active": 0.08925872296094894, "mix_std": 0.22065266966819763, "beta_std": 0.011024716310203075, "beta_low": 0.99102783203125, "beta_high": 0.0, "orthogonal_to_active": 0.02120020240545273, "evidence_weights": [-0.11605378985404968, -0.18805429339408875, 0.5738134980201721], "agreement_weight": 0.05698704347014427, "orthogonal_weight": -0.24421551823616028}, {"head": 5, "memory_to_active": 0.35074660181999207, "contribution_to_active": 0.6107832789421082, "mix_mean": 0.2619801163673401, "mix_low": 0.06182670593261719, "mix_high": 0.00018310546875, "read_cosine": 0.5124483704566956, "beta_mean": 0.06998344510793686, "nonfinite": 0.0, "weighted_memory_to_active": 0.1779138296842575, "mix_std": 0.26224884390830994, "beta_std": 0.0751662403345108, "beta_low": 0.04473876953125, "beta_high": 0.0, "orthogonal_to_active": 0.28496378660202026, "evidence_weights": [-0.14498262107372284, -0.15132839977741241, -0.09217066317796707], "agreement_weight": -0.01132972352206707, "orthogonal_weight": 1.907970666885376}]}, {"layer": 9, "memory_to_active": 0.9242373704910278, "contribution_to_active": 0.7130573391914368, "mix_mean": 0.37644070386886597, "mix_low": 0.07346455752849579, "mix_high": 0.006489753723144531, "read_cosine": 0.353402316570282, "beta_mean": 0.04251379519701004, "nonfinite": 0.0, "weighted_memory_to_active": 0.39142316579818726, "mix_std": 0.23841311037540436, "beta_std": 0.0360826700925827, "beta_low": 0.56396484375, "beta_high": 0.0, "orthogonal_to_active": 0.03418761491775513, "output_influence": 0.5494924783706665, "residual_influence": 0.1605653315782547, "state_active": 0.041748158633708954, "state_memory": 0.015890976414084435, "state_peak": 55.94074249267578, "growth_active": 1.1808757781982422, "growth_memory": 1.3206729888916016, "heads": [{"head": 0, "memory_to_active": 0.0004568930889945477, "contribution_to_active": 0.573447585105896, "mix_mean": 0.647107720375061, "mix_low": 0.0014858245849609375, "mix_high": 0.026162147521972656, "read_cosine": 0.35006216168403625, "beta_mean": 1.3430446870188462e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00028972342261113226, "mix_std": 0.2875123620033264, "beta_std": 3.5555501654016552e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003139811451546848, "evidence_weights": [-0.30488142371177673, -0.31222131848335266, 0.44098812341690063], "agreement_weight": 0.11757653206586838, "orthogonal_weight": -1.1489214897155762}, {"head": 1, "memory_to_active": 0.01751645654439926, "contribution_to_active": 0.6987363696098328, "mix_mean": 0.5749599933624268, "mix_low": 0.007755279541015625, "mix_high": 0.0022954940795898438, "read_cosine": 0.42290061712265015, "beta_mean": 4.4806212827097625e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012935475446283817, "mix_std": 0.2799675166606903, "beta_std": 9.60116449277848e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00904691033065319, "evidence_weights": [-0.23913338780403137, -0.29546141624450684, 1.0159335136413574], "agreement_weight": 0.4914300739765167, "orthogonal_weight": -0.8473024964332581}, {"head": 2, "memory_to_active": 0.16325265169143677, "contribution_to_active": 0.6567923426628113, "mix_mean": 0.6014617681503296, "mix_low": 0.0018243789672851562, "mix_high": 0.008695602416992188, "read_cosine": 0.3758702278137207, "beta_mean": 0.0006327164592221379, "nonfinite": 0.0, "weighted_memory_to_active": 0.11127574741840363, "mix_std": 0.26811671257019043, "beta_std": 0.004218111280351877, "beta_low": 0.99615478515625, "beta_high": 0.0, "orthogonal_to_active": 0.06396722793579102, "evidence_weights": [-0.20411014556884766, -0.2776627242565155, 0.7724282145500183], "agreement_weight": 0.33801329135894775, "orthogonal_weight": -0.6773096323013306}, {"head": 3, "memory_to_active": 0.8900970220565796, "contribution_to_active": 0.24474921822547913, "mix_mean": 0.1299913227558136, "mix_low": 0.13937759399414062, "mix_high": 0.00011730194091796875, "read_cosine": 0.3855137526988983, "beta_mean": 0.040126699954271317, "nonfinite": 0.0, "weighted_memory_to_active": 0.1889544278383255, "mix_std": 0.1841113418340683, "beta_std": 0.03930597007274628, "beta_low": 0.10003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.07017847895622253, "evidence_weights": [-0.0052211894653737545, -0.07743135839700699, 0.305123895406723], "agreement_weight": 0.09841153770685196, "orthogonal_weight": -0.41466715931892395}, {"head": 4, "memory_to_active": 3.178191661834717, "contribution_to_active": 1.7205637693405151, "mix_mean": 0.1472245752811432, "mix_low": 0.13360881805419922, "mix_high": 0.0012407302856445312, "read_cosine": 0.11378385126590729, "beta_mean": 0.16338446736335754, "nonfinite": 0.0, "weighted_memory_to_active": 1.645483374595642, "mix_std": 0.20901407301425934, "beta_std": 0.0952494814991951, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.014830293133854866, "evidence_weights": [-0.08096106350421906, -0.15306194126605988, -0.04359186068177223], "agreement_weight": -0.0634646937251091, "orthogonal_weight": -0.009307687170803547}, {"head": 5, "memory_to_active": 1.2959094047546387, "contribution_to_active": 0.38405415415763855, "mix_mean": 0.15789887309074402, "mix_low": 0.15673542022705078, "mix_high": 0.00042724609375, "read_cosine": 0.47228315472602844, "beta_mean": 0.05089271813631058, "nonfinite": 0.0, "weighted_memory_to_active": 0.3896000385284424, "mix_std": 0.2017565667629242, "beta_std": 0.07762287557125092, "beta_low": 0.28759765625, "beta_high": 0.0, "orthogonal_to_active": 0.046788785606622696, "evidence_weights": [0.07722489535808563, -0.1553795039653778, 0.1386922001838684], "agreement_weight": 0.008741140365600586, "orthogonal_weight": 0.15425503253936768}]}, {"layer": 10, "memory_to_active": 0.21938134729862213, "contribution_to_active": 0.7321903109550476, "mix_mean": 0.5227622985839844, "mix_low": 0.023509979248046875, "mix_high": 0.005251884926110506, "read_cosine": 0.3787541389465332, "beta_mean": 0.04473165050148964, "nonfinite": 0.0, "weighted_memory_to_active": 0.09637127816677094, "mix_std": 0.2687951922416687, "beta_std": 0.04560428857803345, "beta_low": 0.6638895869255066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.2675972580909729, "output_influence": 0.45923495292663574, "residual_influence": 0.1626231074333191, "state_active": 0.055625393986701965, "state_memory": 0.014937974512577057, "state_peak": 124.82110595703125, "growth_active": 1.0743428468704224, "growth_memory": 1.0179288387298584, "heads": [{"head": 0, "memory_to_active": 0.78619384765625, "contribution_to_active": 2.080592155456543, "mix_mean": 0.5618140697479248, "mix_low": 0.10811614990234375, "mix_high": 0.010218620300292969, "read_cosine": 0.08901629596948624, "beta_mean": 0.19807681441307068, "nonfinite": 0.0, "weighted_memory_to_active": 0.4555833339691162, "mix_std": 0.3941860795021057, "beta_std": 0.20905783772468567, "beta_low": 0.1572265625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.5155973434448242, "evidence_weights": [-0.08982148766517639, -0.38121891021728516, -1.8599903583526611], "agreement_weight": 0.029109468683600426, "orthogonal_weight": 3.35408616065979}, {"head": 1, "memory_to_active": 0.00859419908374548, "contribution_to_active": 0.6172777414321899, "mix_mean": 0.6288776993751526, "mix_low": 0.0015354156494140625, "mix_high": 0.015203475952148438, "read_cosine": 0.31432175636291504, "beta_mean": 6.231461884453893e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005052132066339254, "mix_std": 0.2641603946685791, "beta_std": 0.00010420425678603351, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007840716280043125, "evidence_weights": [-0.25101739168167114, -0.19486676156520844, 0.5565451979637146], "agreement_weight": 0.6049624085426331, "orthogonal_weight": -1.691412091255188}, {"head": 2, "memory_to_active": 0.015973376110196114, "contribution_to_active": 0.618456244468689, "mix_mean": 0.6623168587684631, "mix_low": 0.00026035308837890625, "mix_high": 0.004329681396484375, "read_cosine": 0.01941196620464325, "beta_mean": 3.0126344427117147e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007374684792011976, "mix_std": 0.2491462230682373, "beta_std": 0.00012111154501326382, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0031568531412631273, "evidence_weights": [-0.26584139466285706, -0.27930110692977905, 0.6254129409790039], "agreement_weight": -0.21522659063339233, "orthogonal_weight": -0.41283464431762695}, {"head": 3, "memory_to_active": 0.36049142479896545, "contribution_to_active": 0.099652960896492, "mix_mean": 0.29051852226257324, "mix_low": 0.028548240661621094, "mix_high": 0.0002651214599609375, "read_cosine": 0.9238929152488708, "beta_mean": 0.06369829177856445, "nonfinite": 0.0, "weighted_memory_to_active": 0.03923202306032181, "mix_std": 0.23077592253684998, "beta_std": 0.056964874267578125, "beta_low": 0.0015869140625, "beta_high": 0.0, "orthogonal_to_active": 0.0413779579102993, "evidence_weights": [-0.09887294471263885, -0.14650237560272217, 0.1727444976568222], "agreement_weight": 0.020082656294107437, "orthogonal_weight": 1.425920844078064}, {"head": 4, "memory_to_active": 0.02211354300379753, "contribution_to_active": 0.5602129697799683, "mix_mean": 0.5885542631149292, "mix_low": 0.00026607513427734375, "mix_high": 0.0014934539794921875, "read_cosine": 0.07392792403697968, "beta_mean": 6.68829889036715e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013001920655369759, "mix_std": 0.24680322408676147, "beta_std": 0.0001194964861497283, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01826407015323639, "evidence_weights": [-0.18480287492275238, -0.2571553587913513, 0.9583233594894409], "agreement_weight": 0.16302767395973206, "orthogonal_weight": -1.4823776483535767}, {"head": 5, "memory_to_active": 0.12292160093784332, "contribution_to_active": 0.4169498682022095, "mix_mean": 0.4044925272464752, "mix_low": 0.0023336410522460938, "mix_high": 9.5367431640625e-07, "read_cosine": 0.8519538044929504, "beta_mean": 0.0064554521813988686, "nonfinite": 0.0, "weighted_memory_to_active": 0.057983554899692535, "mix_std": 0.22769910097122192, "beta_std": 0.007258190307766199, "beta_low": 0.82452392578125, "beta_high": 0.0, "orthogonal_to_active": 0.019346676766872406, "evidence_weights": [-0.1407770812511444, -0.1639166921377182, 0.2681030333042145], "agreement_weight": 0.09413689374923706, "orthogonal_weight": -0.6642792224884033}]}, {"layer": 11, "memory_to_active": 0.07903057336807251, "contribution_to_active": 0.6208038330078125, "mix_mean": 0.519780695438385, "mix_low": 0.10789570212364197, "mix_high": 0.048586368560791016, "read_cosine": 0.32664725184440613, "beta_mean": 0.02506934106349945, "nonfinite": 0.0, "weighted_memory_to_active": 0.04255174845457077, "mix_std": 0.32509705424308777, "beta_std": 0.03669581189751625, "beta_low": 0.683807373046875, "beta_high": 0.0, "orthogonal_to_active": 0.04589768499135971, "output_influence": 0.5475209951400757, "residual_influence": 0.15021845698356628, "state_active": 0.19927847385406494, "state_memory": 0.0034640012308955193, "state_peak": 33.164485931396484, "growth_active": 1.070732593536377, "growth_memory": 1.5204828977584839, "heads": [{"head": 0, "memory_to_active": 0.007276605814695358, "contribution_to_active": 0.604205310344696, "mix_mean": 0.6443103551864624, "mix_low": 0.0009136199951171875, "mix_high": 0.007648468017578125, "read_cosine": 0.23441699147224426, "beta_mean": 7.704160816501826e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005301019176840782, "mix_std": 0.2565959692001343, "beta_std": 0.0031498426105827093, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.004319236613810062, "evidence_weights": [-0.17226657271385193, -0.36244139075279236, 0.7925857901573181], "agreement_weight": 0.3838575780391693, "orthogonal_weight": -0.8431103825569153}, {"head": 1, "memory_to_active": 0.12633374333381653, "contribution_to_active": 0.42462438344955444, "mix_mean": 0.4579816460609436, "mix_low": 0.005695343017578125, "mix_high": 0.0007658004760742188, "read_cosine": 0.45533132553100586, "beta_mean": 0.004110478796064854, "nonfinite": 0.0, "weighted_memory_to_active": 0.06108522415161133, "mix_std": 0.269270658493042, "beta_std": 0.014650488272309303, "beta_low": 0.92828369140625, "beta_high": 0.0, "orthogonal_to_active": 0.03293098881840706, "evidence_weights": [-0.14912140369415283, -0.2645014822483063, 0.5811173915863037], "agreement_weight": 0.07375343143939972, "orthogonal_weight": -0.6399111151695251}, {"head": 2, "memory_to_active": 0.006444821134209633, "contribution_to_active": 0.9846963882446289, "mix_mean": 0.43908292055130005, "mix_low": 0.3029603958129883, "mix_high": 0.1772785186767578, "read_cosine": 0.16808374226093292, "beta_mean": 0.06338974833488464, "nonfinite": 0.0, "weighted_memory_to_active": 0.0044485353864729404, "mix_std": 0.45052528381347656, "beta_std": 0.08443057537078857, "beta_low": 0.48675537109375, "beta_high": 0.0, "orthogonal_to_active": 0.04293645918369293, "evidence_weights": [0.045252010226249695, -0.3693842887878418, -1.3398230075836182], "agreement_weight": 0.0010085809044539928, "orthogonal_weight": 9.95785140991211}, {"head": 3, "memory_to_active": 0.00765032134950161, "contribution_to_active": 0.8682982921600342, "mix_mean": 0.4158138334751129, "mix_low": 0.3297843933105469, "mix_high": 0.09759235382080078, "read_cosine": 0.4905708432197571, "beta_mean": 0.053876977413892746, "nonfinite": 0.0, "weighted_memory_to_active": 0.00495285727083683, "mix_std": 0.4415261149406433, "beta_std": 0.08092629909515381, "beta_low": 0.51739501953125, "beta_high": 0.0, "orthogonal_to_active": 0.033990390598773956, "evidence_weights": [0.20810125768184662, -0.2896474301815033, -0.1268089860677719], "agreement_weight": 0.05097955837845802, "orthogonal_weight": 10.230239868164062}, {"head": 4, "memory_to_active": 0.23176272213459015, "contribution_to_active": 0.4291984736919403, "mix_mean": 0.5933769941329956, "mix_low": 0.0067806243896484375, "mix_high": 0.0017566680908203125, "read_cosine": 0.05141879618167877, "beta_mean": 0.01919369399547577, "nonfinite": 0.0, "weighted_memory_to_active": 0.1300988644361496, "mix_std": 0.2731969654560089, "beta_std": 0.021893886849284172, "beta_low": 0.44439697265625, "beta_high": 0.0, "orthogonal_to_active": 0.12737053632736206, "evidence_weights": [-0.23407213389873505, -0.23777653276920319, 1.883472204208374], "agreement_weight": -0.21606449782848358, "orthogonal_weight": -0.9971564412117004}, {"head": 5, "memory_to_active": 0.0947151854634285, "contribution_to_active": 0.4138002395629883, "mix_mean": 0.568118155002594, "mix_low": 0.001239776611328125, "mix_high": 0.006476402282714844, "read_cosine": 0.5600616931915283, "beta_mean": 0.009768093004822731, "nonfinite": 0.0, "weighted_memory_to_active": 0.04942400008440018, "mix_std": 0.2594672739505768, "beta_std": 0.015123773366212845, "beta_low": 0.72650146484375, "beta_high": 0.0, "orthogonal_to_active": 0.033838480710983276, "evidence_weights": [-0.20464357733726501, -0.31417858600616455, 0.6547374129295349], "agreement_weight": 0.08417627960443497, "orthogonal_weight": -0.8602729439735413}]}, {"layer": 12, "memory_to_active": 0.2261127233505249, "contribution_to_active": 0.5903327465057373, "mix_mean": 0.5480409264564514, "mix_low": 0.008708953857421875, "mix_high": 0.006515662185847759, "read_cosine": 0.22743384540081024, "beta_mean": 0.01608848385512829, "nonfinite": 0.0, "weighted_memory_to_active": 0.11803975701332092, "mix_std": 0.2617720365524292, "beta_std": 0.015653396025300026, "beta_low": 0.8268636465072632, "beta_high": 0.0, "orthogonal_to_active": 0.07498365640640259, "output_influence": 0.5068607330322266, "residual_influence": 0.14769333600997925, "state_active": 0.03381229564547539, "state_memory": 0.008338222280144691, "state_peak": 66.62428283691406, "growth_active": 1.1685097217559814, "growth_memory": 1.177633285522461, "heads": [{"head": 0, "memory_to_active": 0.04517808556556702, "contribution_to_active": 0.6132047772407532, "mix_mean": 0.5832996368408203, "mix_low": 0.00042247772216796875, "mix_high": 0.005621910095214844, "read_cosine": 0.2680620551109314, "beta_mean": 0.0008889685850590467, "nonfinite": 0.0, "weighted_memory_to_active": 0.025053899735212326, "mix_std": 0.26403406262397766, "beta_std": 0.002028598915785551, "beta_low": 0.99139404296875, "beta_high": 0.0, "orthogonal_to_active": 0.02314653806388378, "evidence_weights": [-0.20982125401496887, -0.27709221839904785, -0.01149060484021902], "agreement_weight": 0.10631968826055527, "orthogonal_weight": -0.9842013120651245}, {"head": 1, "memory_to_active": 0.09806204587221146, "contribution_to_active": 0.5581209063529968, "mix_mean": 0.563521146774292, "mix_low": 0.00030422210693359375, "mix_high": 0.0022106170654296875, "read_cosine": 0.4348350465297699, "beta_mean": 0.0007695202948525548, "nonfinite": 0.0, "weighted_memory_to_active": 0.0606265515089035, "mix_std": 0.2656268775463104, "beta_std": 0.008806283585727215, "beta_low": 0.99554443359375, "beta_high": 0.0, "orthogonal_to_active": 0.04998394474387169, "evidence_weights": [-0.20248588919639587, -0.27256646752357483, 1.039061188697815], "agreement_weight": 0.2463545799255371, "orthogonal_weight": -0.9261525273323059}, {"head": 2, "memory_to_active": 0.6310691237449646, "contribution_to_active": 0.5383819341659546, "mix_mean": 0.3362230658531189, "mix_low": 0.028410911560058594, "mix_high": 0.000354766845703125, "read_cosine": 0.5419245958328247, "beta_mean": 0.09357821941375732, "nonfinite": 0.0, "weighted_memory_to_active": 0.2619670629501343, "mix_std": 0.24951747059822083, "beta_std": 0.06584323942661285, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.2865888774394989, "evidence_weights": [-0.21047505736351013, -0.05986259505152702, -0.006258600391447544], "agreement_weight": 0.024940025061368942, "orthogonal_weight": 1.390912413597107}, {"head": 3, "memory_to_active": 0.03506694734096527, "contribution_to_active": 0.7487320899963379, "mix_mean": 0.7164060473442078, "mix_low": 0.0001316070556640625, "mix_high": 0.019331932067871094, "read_cosine": 0.07219292968511581, "beta_mean": 0.0002632836694829166, "nonfinite": 0.0, "weighted_memory_to_active": 0.025153659284114838, "mix_std": 0.2268417477607727, "beta_std": 0.00538280326873064, "beta_low": 0.998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.023806173354387283, "evidence_weights": [-0.28670892119407654, -0.3344322144985199, 1.683427333831787], "agreement_weight": 0.5639536380767822, "orthogonal_weight": -0.9657016396522522}, {"head": 4, "memory_to_active": 0.4733404815196991, "contribution_to_active": 0.5848352909088135, "mix_mean": 0.4516109228134155, "mix_low": 0.016861915588378906, "mix_high": 0.004723548889160156, "read_cosine": -0.18599657714366913, "beta_mean": 0.0006578841130249202, "nonfinite": 0.0, "weighted_memory_to_active": 0.28655511140823364, "mix_std": 0.29010212421417236, "beta_std": 0.008023707196116447, "beta_low": 0.9833984375, "beta_high": 0.0, "orthogonal_to_active": 0.0293548796325922, "evidence_weights": [-0.15516448020935059, -0.21687336266040802, -0.12179426848888397], "agreement_weight": 0.01235161256045103, "orthogonal_weight": -0.10679029673337936}, {"head": 5, "memory_to_active": 0.07395965605974197, "contribution_to_active": 0.4987214207649231, "mix_mean": 0.637184739112854, "mix_low": 0.006122589111328125, "mix_high": 0.0068511962890625, "read_cosine": 0.23358504474163055, "beta_mean": 0.00037302233977243304, "nonfinite": 0.0, "weighted_memory_to_active": 0.04888223111629486, "mix_std": 0.27450990676879883, "beta_std": 0.003835738869383931, "beta_low": 0.99188232421875, "beta_high": 0.0, "orthogonal_to_active": 0.03702153265476227, "evidence_weights": [-0.22907401621341705, -0.31857720017433167, 1.2424265146255493], "agreement_weight": 0.41967445611953735, "orthogonal_weight": -0.8194631934165955}]}, {"layer": 13, "memory_to_active": 0.11290682107210159, "contribution_to_active": 0.5367761850357056, "mix_mean": 0.5421571731567383, "mix_low": 0.005450090393424034, "mix_high": 0.009028276428580284, "read_cosine": 0.38302022218704224, "beta_mean": 0.013982701115310192, "nonfinite": 0.0, "weighted_memory_to_active": 0.048725321888923645, "mix_std": 0.26013731956481934, "beta_std": 0.014537121169269085, "beta_low": 0.812225341796875, "beta_high": 0.0, "orthogonal_to_active": 0.05063913017511368, "output_influence": 0.5195513963699341, "residual_influence": 0.18377621471881866, "state_active": 0.13613738119602203, "state_memory": 0.04378689453005791, "state_peak": 99.30296325683594, "growth_active": 0.9330469369888306, "growth_memory": 1.0271034240722656, "heads": [{"head": 0, "memory_to_active": 0.012569544836878777, "contribution_to_active": 0.5962696075439453, "mix_mean": 0.6150461435317993, "mix_low": 0.0006561279296875, "mix_high": 0.005138397216796875, "read_cosine": 0.4216769337654114, "beta_mean": 4.615174839273095e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008152035996317863, "mix_std": 0.2688061594963074, "beta_std": 0.00010332372039556503, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007875111885368824, "evidence_weights": [-0.19429725408554077, -0.2672988176345825, 0.762313187122345], "agreement_weight": 0.1866871863603592, "orthogonal_weight": -1.0731289386749268}, {"head": 1, "memory_to_active": 0.01491670310497284, "contribution_to_active": 0.6023209095001221, "mix_mean": 0.5728629231452942, "mix_low": 0.0014715194702148438, "mix_high": 0.005015373229980469, "read_cosine": 0.21891391277313232, "beta_mean": 5.928391692577861e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008353065699338913, "mix_std": 0.2767731547355652, "beta_std": 0.00010997117351507768, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011025959625840187, "evidence_weights": [-0.18148215115070343, -0.26460108160972595, 0.5808361768722534], "agreement_weight": 0.4932268261909485, "orthogonal_weight": -1.3729960918426514}, {"head": 2, "memory_to_active": 0.05911106616258621, "contribution_to_active": 0.4823313355445862, "mix_mean": 0.3889632523059845, "mix_low": 0.001270294189453125, "mix_high": 0.000263214111328125, "read_cosine": 0.4820896089076996, "beta_mean": 0.005960503127425909, "nonfinite": 0.0, "weighted_memory_to_active": 0.025854021310806274, "mix_std": 0.2324698269367218, "beta_std": 0.02367856539785862, "beta_low": 0.88165283203125, "beta_high": 0.0, "orthogonal_to_active": 0.015571794472634792, "evidence_weights": [-0.18404200673103333, -0.21084889769554138, 0.4132843315601349], "agreement_weight": 0.005288589745759964, "orthogonal_weight": -0.8461067080497742}, {"head": 3, "memory_to_active": 0.15408135950565338, "contribution_to_active": 0.5830252170562744, "mix_mean": 0.6639832258224487, "mix_low": 0.022106170654296875, "mix_high": 0.030577659606933594, "read_cosine": -0.0386761911213398, "beta_mean": 0.0009389995248056948, "nonfinite": 0.0, "weighted_memory_to_active": 0.11596085131168365, "mix_std": 0.2837587296962738, "beta_std": 0.00915120542049408, "beta_low": 0.99176025390625, "beta_high": 0.0, "orthogonal_to_active": 0.10154152661561966, "evidence_weights": [-0.3153936266899109, -0.24378859996795654, 1.5575131177902222], "agreement_weight": 6.334452152252197, "orthogonal_weight": -0.9686516523361206}, {"head": 4, "memory_to_active": 0.38779452443122864, "contribution_to_active": 0.24767865240573883, "mix_mean": 0.3517443537712097, "mix_low": 0.0051937103271484375, "mix_high": 8.7738037109375e-05, "read_cosine": 0.8939230442047119, "beta_mean": 0.07667011767625809, "nonfinite": 0.0, "weighted_memory_to_active": 0.10075337439775467, "mix_std": 0.2281164824962616, "beta_std": 0.052755847573280334, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.14003503322601318, "evidence_weights": [-0.07732395827770233, -0.23071280121803284, 0.4458472430706024], "agreement_weight": 0.001826768391765654, "orthogonal_weight": 2.2601521015167236}, {"head": 5, "memory_to_active": 0.04896768182516098, "contribution_to_active": 0.709031343460083, "mix_mean": 0.6603430509567261, "mix_low": 0.002002716064453125, "mix_high": 0.013087272644042969, "read_cosine": 0.320193886756897, "beta_mean": 0.0002211482060374692, "nonfinite": 0.0, "weighted_memory_to_active": 0.03327857330441475, "mix_std": 0.2708994150161743, "beta_std": 0.0014238132862374187, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.027785340324044228, "evidence_weights": [-0.22102658450603485, -0.2971278429031372, 0.8686419129371643], "agreement_weight": 0.18791063129901886, "orthogonal_weight": -0.9196619391441345}]}, {"layer": 14, "memory_to_active": 0.22358395159244537, "contribution_to_active": 0.4222157299518585, "mix_mean": 0.4283447265625, "mix_low": 0.026481151580810547, "mix_high": 0.0063343048095703125, "read_cosine": 0.6155250072479248, "beta_mean": 0.05843481048941612, "nonfinite": 0.0, "weighted_memory_to_active": 0.09423384815454483, "mix_std": 0.24421045184135437, "beta_std": 0.057342324405908585, "beta_low": 0.5175069570541382, "beta_high": 0.0, "orthogonal_to_active": 0.04220125079154968, "output_influence": 0.36205071210861206, "residual_influence": 0.14849776029586792, "state_active": 0.05807693675160408, "state_memory": 0.015016978606581688, "state_peak": 42.77748107910156, "growth_active": 1.1278066635131836, "growth_memory": 1.2722362279891968, "heads": [{"head": 0, "memory_to_active": 0.3660656213760376, "contribution_to_active": 0.20001840591430664, "mix_mean": 0.20889955759048462, "mix_low": 0.015730857849121094, "mix_high": 3.4332275390625e-05, "read_cosine": 0.8161495327949524, "beta_mean": 0.071921706199646, "nonfinite": 0.0, "weighted_memory_to_active": 0.08457618206739426, "mix_std": 0.18638873100280762, "beta_std": 0.06156526878476143, "beta_low": 0.00897216796875, "beta_high": 0.0, "orthogonal_to_active": 0.09115815907716751, "evidence_weights": [-0.10120665282011032, -0.04759116470813751, 0.4034655690193176], "agreement_weight": 0.02342420443892479, "orthogonal_weight": 1.793203353881836}, {"head": 1, "memory_to_active": 0.45956695079803467, "contribution_to_active": 0.2894718050956726, "mix_mean": 0.20523111522197723, "mix_low": 0.12833595275878906, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8638284802436829, "beta_mean": 0.23896050453186035, "nonfinite": 0.0, "weighted_memory_to_active": 0.20742657780647278, "mix_std": 0.2370327115058899, "beta_std": 0.24213558435440063, "beta_low": 0.10687255859375, "beta_high": 0.0, "orthogonal_to_active": 0.01575975865125656, "evidence_weights": [-0.0007610185421071947, -0.19480352103710175, -0.09689591825008392], "agreement_weight": 0.05906587839126587, "orthogonal_weight": 0.2220447063446045}, {"head": 2, "memory_to_active": 0.13330745697021484, "contribution_to_active": 0.5807167887687683, "mix_mean": 0.5871182680130005, "mix_low": 0.0026836395263671875, "mix_high": 0.003909111022949219, "read_cosine": 0.4187396764755249, "beta_mean": 0.0018104680348187685, "nonfinite": 0.0, "weighted_memory_to_active": 0.08690845221281052, "mix_std": 0.26618725061416626, "beta_std": 0.002318739891052246, "beta_low": 0.9844970703125, "beta_high": 0.0, "orthogonal_to_active": 0.06683939695358276, "evidence_weights": [-0.1339477002620697, -0.3168625831604004, 0.9286133646965027], "agreement_weight": 0.08695321530103683, "orthogonal_weight": -0.8742772936820984}, {"head": 3, "memory_to_active": 0.036313071846961975, "contribution_to_active": 0.6107686758041382, "mix_mean": 0.6684229373931885, "mix_low": 0.0042400360107421875, "mix_high": 0.024068832397460938, "read_cosine": -0.05785210058093071, "beta_mean": 0.003924324177205563, "nonfinite": 0.0, "weighted_memory_to_active": 0.023446496576070786, "mix_std": 0.26002806425094604, "beta_std": 0.005692770704627037, "beta_low": 0.90972900390625, "beta_high": 0.0, "orthogonal_to_active": 0.017139188945293427, "evidence_weights": [-0.3445281684398651, -0.31229931116104126, 0.2930469512939453], "agreement_weight": 0.07270241528749466, "orthogonal_weight": -0.8325636982917786}, {"head": 4, "memory_to_active": 0.1385764330625534, "contribution_to_active": 0.512594997882843, "mix_mean": 0.5181172490119934, "mix_low": 0.004017829895019531, "mix_high": 0.008541107177734375, "read_cosine": 0.8994418382644653, "beta_mean": 0.0323735848069191, "nonfinite": 0.0, "weighted_memory_to_active": 0.08168651163578033, "mix_std": 0.2607472538948059, "beta_std": 0.023994924500584602, "beta_low": 0.12451171875, "beta_high": 0.0, "orthogonal_to_active": 0.021973872557282448, "evidence_weights": [-0.2122150957584381, -0.23942838609218597, 0.434253066778183], "agreement_weight": 0.1361657828092575, "orthogonal_weight": -0.6175684928894043}, {"head": 5, "memory_to_active": 0.20767414569854736, "contribution_to_active": 0.3397236764431, "mix_mean": 0.38227933645248413, "mix_low": 0.0038785934448242188, "mix_high": 0.0014505386352539062, "read_cosine": 0.7528423070907593, "beta_mean": 0.0016182548133656383, "nonfinite": 0.0, "weighted_memory_to_active": 0.08135883510112762, "mix_std": 0.2548786699771881, "beta_std": 0.00834664050489664, "beta_low": 0.970458984375, "beta_high": 0.0, "orthogonal_to_active": 0.04033709317445755, "evidence_weights": [-0.17530447244644165, -0.17848829925060272, 0.34665152430534363], "agreement_weight": 0.0706055760383606, "orthogonal_weight": -0.7546921968460083}]}, {"layer": 15, "memory_to_active": 0.05223418027162552, "contribution_to_active": 0.49271783232688904, "mix_mean": 0.4233163893222809, "mix_low": 0.008445581421256065, "mix_high": 0.003189404960721731, "read_cosine": 0.6534176468849182, "beta_mean": 0.037506964057683945, "nonfinite": 0.0, "weighted_memory_to_active": 0.02622305601835251, "mix_std": 0.28544411063194275, "beta_std": 0.04295824095606804, "beta_low": 0.7084452509880066, "beta_high": 0.0, "orthogonal_to_active": 0.007904952391982079, "output_influence": 0.31008607149124146, "residual_influence": 0.1831541359424591, "state_active": 0.16438671946525574, "state_memory": 0.010598242282867432, "state_peak": 33.7895393371582, "growth_active": 1.2113995552062988, "growth_memory": 1.3900067806243896, "heads": [{"head": 0, "memory_to_active": 0.0943908542394638, "contribution_to_active": 0.5070819854736328, "mix_mean": 0.39092159271240234, "mix_low": 0.008586883544921875, "mix_high": 0.005724906921386719, "read_cosine": 0.5949223041534424, "beta_mean": 0.040580347180366516, "nonfinite": 0.0, "weighted_memory_to_active": 0.047205571085214615, "mix_std": 0.28735247254371643, "beta_std": 0.028360608965158463, "beta_low": 0.076171875, "beta_high": 0.0, "orthogonal_to_active": 0.02687963843345642, "evidence_weights": [-0.18913377821445465, -0.19096286594867706, 0.4848247766494751], "agreement_weight": 0.09245899319648743, "orthogonal_weight": -0.8024668097496033}, {"head": 1, "memory_to_active": 0.10896238684654236, "contribution_to_active": 0.4279273748397827, "mix_mean": 0.43705564737319946, "mix_low": 0.0011587142944335938, "mix_high": 0.0084075927734375, "read_cosine": 0.9293780326843262, "beta_mean": 0.00046182237565517426, "nonfinite": 0.0, "weighted_memory_to_active": 0.046729832887649536, "mix_std": 0.29009419679641724, "beta_std": 0.00217990018427372, "beta_low": 0.99609375, "beta_high": 0.0, "orthogonal_to_active": 0.0021149576641619205, "evidence_weights": [-0.15005014836788177, -0.1585751622915268, 0.1671682745218277], "agreement_weight": 0.12303600460290909, "orthogonal_weight": 0.09689974784851074}, {"head": 2, "memory_to_active": 0.005648805294185877, "contribution_to_active": 0.49386894702911377, "mix_mean": 0.3453426957130432, "mix_low": 0.013213157653808594, "mix_high": 5.245208740234375e-05, "read_cosine": 0.7458586692810059, "beta_mean": 7.533017924288288e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002979762852191925, "mix_std": 0.27083930373191833, "beta_std": 0.0001512243616161868, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009293804178014398, "evidence_weights": [-0.2033509761095047, -0.26206403970718384, 0.11326862871646881], "agreement_weight": 0.010771437548100948, "orthogonal_weight": 0.86166912317276}, {"head": 3, "memory_to_active": 0.001565601211041212, "contribution_to_active": 0.38224825263023376, "mix_mean": 0.4030880928039551, "mix_low": 0.005429267883300781, "mix_high": 0.0004177093505859375, "read_cosine": 0.5903995037078857, "beta_mean": 8.356749276572373e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.000534956285264343, "mix_std": 0.28775110840797424, "beta_std": 2.100938218063675e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003314639034215361, "evidence_weights": [-0.273448646068573, -0.2380441427230835, 0.23722900450229645], "agreement_weight": 0.05869566649198532, "orthogonal_weight": -0.638377845287323}, {"head": 4, "memory_to_active": 0.07654184848070145, "contribution_to_active": 0.6304502487182617, "mix_mean": 0.38684606552124023, "mix_low": 0.012685775756835938, "mix_high": 0.0011415481567382812, "read_cosine": 0.7355189323425293, "beta_mean": 0.18198628723621368, "nonfinite": 0.0, "weighted_memory_to_active": 0.04289921000599861, "mix_std": 0.30818676948547363, "beta_std": 0.21189484000205994, "beta_low": 0.2073974609375, "beta_high": 0.0, "orthogonal_to_active": 0.0065327659249305725, "evidence_weights": [-0.2068735808134079, -0.2544131278991699, -0.01725410856306553], "agreement_weight": 0.013060457073152065, "orthogonal_weight": 0.35915929079055786}, {"head": 5, "memory_to_active": 0.02629556506872177, "contribution_to_active": 0.5147300958633423, "mix_mean": 0.5766441822052002, "mix_low": 0.009599685668945312, "mix_high": 0.0033922195434570312, "read_cosine": 0.3244282305240631, "beta_mean": 0.0019296283135190606, "nonfinite": 0.0, "weighted_memory_to_active": 0.01698899455368519, "mix_std": 0.26844072341918945, "beta_std": 0.015141846612095833, "beta_low": 0.97100830078125, "beta_high": 0.0, "orthogonal_to_active": 0.010641506873071194, "evidence_weights": [-0.283527135848999, -0.30404365062713623, 0.6590386033058167], "agreement_weight": 0.31166166067123413, "orthogonal_weight": 0.8063454627990723}]}]}, {"step": 64000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.852903396000329, "memory_to_active": 0.796611487865448, "contribution_to_active": 0.5665149688720703, "mix_mean": 0.4222530126571655, "mix_low": 0.1267881691455841, "mix_high": 0.010436734184622765, "read_cosine": 0.34477150440216064, "beta_mean": 0.0689348578453064, "nonfinite": 0.0, "weighted_memory_to_active": 0.2026495635509491, "mix_std": 0.22347676753997803, "beta_std": 0.06081679090857506, "beta_low": 0.5810298919677734, "beta_high": 4.895528400084004e-05, "orthogonal_to_active": 0.061402998864650726, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0824062824249268, "normal_token_accuracy": 0.4571746289730072, "state_active": 0.062398429960012436, "state_memory": 0.016915608197450638, "state_peak": 121.29936981201172, "growth_active": 1.1198174953460693, "growth_memory": 1.2979023456573486, "output_influence": 0.438077449798584, "residual_influence": 0.15355390310287476, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2172789573669434, "no_memory_loss_delta": 0.6820517778396606, "scrambled_loss_delta": 0.16102395951747894, "memory_only_loss_delta": 3.8014156818389893, "no_evidence_loss_delta": 0.43008673191070557, "no_agreement_loss_delta": 0.02734954096376896, "no_orthogonal_loss_delta": 0.007503387052565813}, {"index": 94, "tokens": 256, "normal_loss": 3.2173831462860107, "no_memory_loss_delta": 0.6855150461196899, "scrambled_loss_delta": 0.3690314292907715, "memory_only_loss_delta": 3.49221134185791, "no_evidence_loss_delta": 0.31988170742988586, "no_agreement_loss_delta": 0.03720644861459732, "no_orthogonal_loss_delta": 0.02958066761493683}, {"index": 188, "tokens": 256, "normal_loss": 3.3630881309509277, "no_memory_loss_delta": 0.833808183670044, "scrambled_loss_delta": 0.09918299317359924, "memory_only_loss_delta": 3.1999330520629883, "no_evidence_loss_delta": 0.4896043539047241, "no_agreement_loss_delta": 0.053029514849185944, "no_orthogonal_loss_delta": 0.025368567556142807}, {"index": 283, "tokens": 131, "normal_loss": 2.0065596103668213, "no_memory_loss_delta": 0.43541133403778076, "scrambled_loss_delta": 0.2440529614686966, "memory_only_loss_delta": 2.9442310333251953, "no_evidence_loss_delta": 0.14433129131793976, "no_agreement_loss_delta": 0.038814280182123184, "no_orthogonal_loss_delta": -0.04228563234210014}], "agreement_weight_abs_mean": 0.2345612645149231, "agreement_weight_abs_max": 6.395313262939453, "agreement_weight_mean": 0.2040548473596573, "orthogonal_weight_abs_mean": 0.9121546149253845, "orthogonal_weight_abs_max": 10.332039833068848, "orthogonal_weight_mean": -0.08493109792470932, "no_memory_loss": 3.772718667984009, "no_memory_loss_delta": 0.6903124451637268, "no_memory_token_accuracy_gain": 0.1034482792019844, "no_memory_token_win_fraction": 0.6818687319755554, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 49.85806214710903, "scrambled_loss": 3.297151565551758, "scrambled_loss_delta": 0.21474523842334747, "scrambled_token_accuracy_gain": 0.036707453429698944, "scrambled_token_win_fraction": 0.6129032373428345, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.32530578778497, "memory_only_loss": 6.499587059020996, "memory_only_loss_delta": 3.4171805381774902, "memory_only_token_accuracy_gain": 0.30812013149261475, "memory_only_token_win_fraction": 0.9121245741844177, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.71952034557229, "no_evidence_loss": 3.4564197063446045, "no_evidence_loss_delta": 0.37401342391967773, "no_evidence_token_accuracy_gain": 0.05784204602241516, "no_evidence_token_win_fraction": 0.6129032373428345, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 31.20323234558381, "no_agreement_loss": 3.1215460300445557, "no_agreement_loss_delta": 0.03913966566324234, "no_agreement_token_accuracy_gain": 0.01890989951789379, "no_agreement_token_win_fraction": 0.49165740609169006, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.838360502454278, "no_orthogonal_loss": 3.0940287113189697, "no_orthogonal_loss_delta": 0.011622306890785694, "no_orthogonal_token_accuracy_gain": 0.0033370412420481443, "no_orthogonal_token_win_fraction": 0.5684093236923218, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.155502877659356, "short_no_memory_loss_delta": 0.662765622138977, "long_context_memory_gain": 0.027546823024749756, "normal_context_loss_gain": 0.1301954835653305, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.542689085006714, "contribution_to_active": 1.175537109375, "mix_mean": 0.10638948529958725, "mix_low": 0.5272464752197266, "mix_high": 0.004648844711482525, "read_cosine": 0.41773200035095215, "beta_mean": 0.20876725018024445, "nonfinite": 0.0, "weighted_memory_to_active": 1.210286259651184, "mix_std": 0.09102194756269455, "beta_std": 0.13556823134422302, "beta_low": 0.1334330290555954, "beta_high": 0.0, "orthogonal_to_active": 0.040868181735277176, "output_influence": 0.22094054520130157, "residual_influence": 0.24904319643974304, "state_active": 0.06158529594540596, "state_memory": 0.04464864730834961, "state_peak": 47.58399963378906, "growth_active": 1.1749520301818848, "growth_memory": 1.2792670726776123, "heads": [{"head": 0, "memory_to_active": 2.750204086303711, "contribution_to_active": 0.5044318437576294, "mix_mean": 0.018856199458241463, "mix_low": 0.6431255340576172, "mix_high": 0.0, "read_cosine": 0.21650922298431396, "beta_mean": 0.18568849563598633, "nonfinite": 0.0, "weighted_memory_to_active": 0.6360236406326294, "mix_std": 0.058999691158533096, "beta_std": 0.10097283124923706, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07063579559326172, "evidence_weights": [-0.0028978714253753424, -0.042972903698682785, -0.060372307896614075], "agreement_weight": -0.21503275632858276, "orthogonal_weight": -0.12738381326198578}, {"head": 1, "memory_to_active": 6.713043212890625, "contribution_to_active": 2.8868494033813477, "mix_mean": 0.019065940752625465, "mix_low": 0.6409311294555664, "mix_high": 0.0, "read_cosine": 0.4456639289855957, "beta_mean": 0.4476549029350281, "nonfinite": 0.0, "weighted_memory_to_active": 3.169013023376465, "mix_std": 0.05593519285321236, "beta_std": 0.2143862545490265, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.001147266710177064, "evidence_weights": [0.01940007507801056, -0.10594125092029572, 0.665797233581543], "agreement_weight": 0.02653094194829464, "orthogonal_weight": -0.0007844622596167028}, {"head": 2, "memory_to_active": 2.59419584274292, "contribution_to_active": 0.5268107056617737, "mix_mean": 0.014649486169219017, "mix_low": 0.6598711013793945, "mix_high": 0.0, "read_cosine": 0.3019310235977173, "beta_mean": 0.25669175386428833, "nonfinite": 0.0, "weighted_memory_to_active": 0.6154809594154358, "mix_std": 0.033022958785295486, "beta_std": 0.18311847746372223, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.050854261964559555, "evidence_weights": [0.06575044244527817, 0.05326315388083458, 0.22426146268844604], "agreement_weight": 0.012417804449796677, "orthogonal_weight": -0.2058173269033432}, {"head": 3, "memory_to_active": 7.489529132843018, "contribution_to_active": 2.592830181121826, "mix_mean": 0.029788604006171227, "mix_low": 0.48967456817626953, "mix_high": 0.0, "read_cosine": 0.31476467847824097, "beta_mean": 0.3044653534889221, "nonfinite": 0.0, "weighted_memory_to_active": 2.7288832664489746, "mix_std": 0.06875550001859665, "beta_std": 0.2325991839170456, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10250717401504517, "evidence_weights": [-0.037831153720617294, -0.03437209129333496, -0.12238816171884537], "agreement_weight": -0.009868290275335312, "orthogonal_weight": -0.04437962546944618}, {"head": 4, "memory_to_active": 1.6027226448059082, "contribution_to_active": 0.07046598941087723, "mix_mean": 0.014311233535408974, "mix_low": 0.7152633666992188, "mix_high": 0.0, "read_cosine": 0.3030956983566284, "beta_mean": 0.039523184299468994, "nonfinite": 0.0, "weighted_memory_to_active": 0.05495483800768852, "mix_std": 0.04428926110267639, "beta_std": 0.063334159553051, "beta_low": 0.401611328125, "beta_high": 0.0, "orthogonal_to_active": 0.0012823769357055426, "evidence_weights": [0.11764056235551834, 0.0806017741560936, -0.12583209574222565], "agreement_weight": -0.17992745339870453, "orthogonal_weight": 0.02211427316069603}, {"head": 5, "memory_to_active": 0.10643722116947174, "contribution_to_active": 0.47183507680892944, "mix_mean": 0.5416654348373413, "mix_low": 0.014613151550292969, "mix_high": 0.02789306640625, "read_cosine": 0.9244273900985718, "beta_mean": 0.01857970654964447, "nonfinite": 0.0, "weighted_memory_to_active": 0.05736178532242775, "mix_std": 0.28512901067733765, "beta_std": 0.018998410552740097, "beta_low": 0.39898681640625, "beta_high": 0.0, "orthogonal_to_active": 0.01878221333026886, "evidence_weights": [-0.1729011982679367, -0.21787552535533905, 0.38531115651130676], "agreement_weight": 0.2886776924133301, "orthogonal_weight": -0.6874564290046692}]}, {"layer": 1, "memory_to_active": 2.467050313949585, "contribution_to_active": 0.5081530809402466, "mix_mean": 0.21416710317134857, "mix_low": 0.3561573326587677, "mix_high": 0.0011828740825876594, "read_cosine": 0.3059588372707367, "beta_mean": 0.18787351250648499, "nonfinite": 0.0, "weighted_memory_to_active": 0.35005223751068115, "mix_std": 0.11561118066310883, "beta_std": 0.13466651737689972, "beta_low": 0.351348876953125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.044708769768476486, "output_influence": 0.25686267018318176, "residual_influence": 0.07056593894958496, "state_active": 0.027867391705513, "state_memory": 0.03512950986623764, "state_peak": 40.079776763916016, "growth_active": 1.159791111946106, "growth_memory": 1.4321792125701904, "heads": [{"head": 0, "memory_to_active": 0.032003942877054214, "contribution_to_active": 0.4843103587627411, "mix_mean": 0.5303735733032227, "mix_low": 0.005169868469238281, "mix_high": 0.0027379989624023438, "read_cosine": 0.043895214796066284, "beta_mean": 0.00021298445062711835, "nonfinite": 0.0, "weighted_memory_to_active": 0.019898366183042526, "mix_std": 0.28244683146476746, "beta_std": 0.0004041894571855664, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01941766031086445, "evidence_weights": [-0.20771034061908722, -0.27240386605262756, 0.305324524641037], "agreement_weight": 0.21146538853645325, "orthogonal_weight": -1.0173156261444092}, {"head": 1, "memory_to_active": 3.1307926177978516, "contribution_to_active": 0.3055979609489441, "mix_mean": 0.02352154441177845, "mix_low": 0.45805835723876953, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5102333426475525, "beta_mean": 0.28783679008483887, "nonfinite": 0.0, "weighted_memory_to_active": 0.33188408613204956, "mix_std": 0.04607430100440979, "beta_std": 0.19431984424591064, "beta_low": 0.0023193359375, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.024571258574724197, "evidence_weights": [0.019877979531884193, -0.0682092010974884, 0.05544354394078255], "agreement_weight": -0.0028732451610267162, "orthogonal_weight": 0.20271135866641998}, {"head": 2, "memory_to_active": 0.04194127768278122, "contribution_to_active": 0.5403698682785034, "mix_mean": 0.6657083034515381, "mix_low": 0.00041103363037109375, "mix_high": 0.0043582916259765625, "read_cosine": 0.4932519197463989, "beta_mean": 0.0015538566512987018, "nonfinite": 0.0, "weighted_memory_to_active": 0.027822894975543022, "mix_std": 0.231018528342247, "beta_std": 0.0033038887195289135, "beta_low": 0.9801025390625, "beta_high": 0.0, "orthogonal_to_active": 0.01849934831261635, "evidence_weights": [-0.27633920311927795, -0.3714282810688019, 0.6735321879386902], "agreement_weight": 0.039506711065769196, "orthogonal_weight": -0.7973949313163757}, {"head": 3, "memory_to_active": 3.0745081901550293, "contribution_to_active": 0.20093347132205963, "mix_mean": 0.025143800303339958, "mix_low": 0.5180225372314453, "mix_high": 0.0, "read_cosine": 0.18081919848918915, "beta_mean": 0.3456304669380188, "nonfinite": 0.0, "weighted_memory_to_active": 0.2842198610305786, "mix_std": 0.06020992994308472, "beta_std": 0.2714923620223999, "beta_low": 0.0703125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.11022046208381653, "evidence_weights": [0.03603985160589218, -0.02074548602104187, -0.03716547414660454], "agreement_weight": -0.02959338016808033, "orthogonal_weight": -0.4865441918373108}, {"head": 4, "memory_to_active": 4.065779685974121, "contribution_to_active": 1.4149342775344849, "mix_mean": 0.03343930095434189, "mix_low": 0.3201770782470703, "mix_high": 0.0, "read_cosine": 0.3349628448486328, "beta_mean": 0.1719343066215515, "nonfinite": 0.0, "weighted_memory_to_active": 1.3651678562164307, "mix_std": 0.05584442615509033, "beta_std": 0.12267974019050598, "beta_low": 0.0245361328125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.06842038035392761, "evidence_weights": [-0.023918434977531433, -0.0023467293940484524, 0.13359540700912476], "agreement_weight": -0.016014693304896355, "orthogonal_weight": 0.07192854583263397}, {"head": 5, "memory_to_active": 4.457275867462158, "contribution_to_active": 0.10277222841978073, "mix_mean": 0.006816118955612183, "mix_low": 0.8351049423217773, "mix_high": 0.0, "read_cosine": 0.2725904583930969, "beta_mean": 0.320072740316391, "nonfinite": 0.0, "weighted_memory_to_active": 0.07132022827863693, "mix_std": 0.01807299070060253, "beta_std": 0.2157990038394928, "beta_low": 0.03082275390625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.02712351642549038, "evidence_weights": [0.16086554527282715, 0.08126813918352127, 0.10751596838235855], "agreement_weight": -0.11013154685497284, "orthogonal_weight": 0.3788972496986389}]}, {"layer": 2, "memory_to_active": 1.0190744400024414, "contribution_to_active": 0.34831830859184265, "mix_mean": 0.40450870990753174, "mix_low": 0.2447652816772461, "mix_high": 0.004044055938720703, "read_cosine": 0.30392593145370483, "beta_mean": 0.0797324851155281, "nonfinite": 0.0, "weighted_memory_to_active": 0.07015903294086456, "mix_std": 0.1884242445230484, "beta_std": 0.06732088327407837, "beta_low": 0.5992431640625, "beta_high": 0.00044759115553461015, "orthogonal_to_active": 0.01655617170035839, "output_influence": 0.39719972014427185, "residual_influence": 0.10804314911365509, "state_active": 0.03894547000527382, "state_memory": 0.014003976248204708, "state_peak": 31.954059600830078, "growth_active": 1.0087069272994995, "growth_memory": 1.566920518875122, "heads": [{"head": 0, "memory_to_active": 0.006792924366891384, "contribution_to_active": 0.3251838684082031, "mix_mean": 0.6289283037185669, "mix_low": 0.000827789306640625, "mix_high": 0.008116722106933594, "read_cosine": 0.14172136783599854, "beta_mean": 5.5924905609572306e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004348818678408861, "mix_std": 0.2694348096847534, "beta_std": 8.810566214378923e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0046320161782205105, "evidence_weights": [-0.3709145188331604, -0.21358701586723328, 0.1813395917415619], "agreement_weight": 0.12967365980148315, "orthogonal_weight": -1.1075280904769897}, {"head": 1, "memory_to_active": 3.0305488109588623, "contribution_to_active": 0.042409513145685196, "mix_mean": 0.005459135863929987, "mix_low": 0.8808984756469727, "mix_high": 0.0, "read_cosine": 0.13436613976955414, "beta_mean": 0.21592381596565247, "nonfinite": 0.0, "weighted_memory_to_active": 0.031533267349004745, "mix_std": 0.01233547180891037, "beta_std": 0.14718768000602722, "beta_low": 0.00286865234375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.009897314943373203, "evidence_weights": [0.1380334347486496, 0.1147317886352539, 0.5944131016731262], "agreement_weight": 0.022323567420244217, "orthogonal_weight": 0.3360830843448639}, {"head": 2, "memory_to_active": 0.01257551833987236, "contribution_to_active": 0.41640517115592957, "mix_mean": 0.5436215996742249, "mix_low": 0.0015621185302734375, "mix_high": 0.0030641555786132812, "read_cosine": 0.34722888469696045, "beta_mean": 3.413172817090526e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005318730138242245, "mix_std": 0.27679991722106934, "beta_std": 6.613205914618447e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 8.938277460401878e-05, "evidence_weights": [-0.2517581880092621, -0.2926883101463318, 0.5867488980293274], "agreement_weight": -0.032231807708740234, "orthogonal_weight": -0.020164573565125465}, {"head": 3, "memory_to_active": 0.10542884469032288, "contribution_to_active": 0.5545352101325989, "mix_mean": 0.5824043154716492, "mix_low": 0.00021266937255859375, "mix_high": 0.006184577941894531, "read_cosine": 0.6467925310134888, "beta_mean": 0.01721883751451969, "nonfinite": 0.0, "weighted_memory_to_active": 0.06852025538682938, "mix_std": 0.2547304332256317, "beta_std": 0.025679800659418106, "beta_low": 0.53753662109375, "beta_high": 0.0, "orthogonal_to_active": 0.04510078579187393, "evidence_weights": [-0.2043807953596115, -0.3253585994243622, 0.7707120776176453], "agreement_weight": 0.053419142961502075, "orthogonal_weight": -0.8937389254570007}, {"head": 4, "memory_to_active": 2.948169708251953, "contribution_to_active": 0.23384948074817657, "mix_mean": 0.02087423950433731, "mix_low": 0.5841178894042969, "mix_high": 0.0, "read_cosine": 0.19817498326301575, "beta_mean": 0.24512363970279694, "nonfinite": 0.0, "weighted_memory_to_active": 0.3052402138710022, "mix_std": 0.05302602797746658, "beta_std": 0.23084262013435364, "beta_low": 0.0550537109375, "beta_high": 0.00262451171875, "orthogonal_to_active": 0.03808176517486572, "evidence_weights": [0.024642126634716988, 0.004409921355545521, 0.022984012961387634], "agreement_weight": 0.026380503550171852, "orthogonal_weight": -0.14668723940849304}, {"head": 5, "memory_to_active": 0.01093045063316822, "contribution_to_active": 0.5175266265869141, "mix_mean": 0.6457645893096924, "mix_low": 0.000972747802734375, "mix_high": 0.0068988800048828125, "read_cosine": 0.3552716374397278, "beta_mean": 3.8548929296666756e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00599293690174818, "mix_std": 0.2642187774181366, "beta_std": 6.095996650401503e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015357647789642215, "evidence_weights": [-0.27862706780433655, -0.2884475290775299, 0.9107370376586914], "agreement_weight": 0.24379290640354156, "orthogonal_weight": -0.3032650947570801}]}, {"layer": 3, "memory_to_active": 0.41866180300712585, "contribution_to_active": 0.4862437844276428, "mix_mean": 0.5158716440200806, "mix_low": 0.054301902651786804, "mix_high": 0.01140070054680109, "read_cosine": 0.20235010981559753, "beta_mean": 0.023662293329834938, "nonfinite": 0.0, "weighted_memory_to_active": 0.12441793829202652, "mix_std": 0.2572881579399109, "beta_std": 0.022812413051724434, "beta_low": 0.7685242295265198, "beta_high": 0.0, "orthogonal_to_active": 0.051305413246154785, "output_influence": 0.43152663111686707, "residual_influence": 0.10217565298080444, "state_active": 0.023372720927000046, "state_memory": 0.006763090379536152, "state_peak": 23.922060012817383, "growth_active": 1.0627906322479248, "growth_memory": 1.6458690166473389, "heads": [{"head": 0, "memory_to_active": 0.35798370838165283, "contribution_to_active": 0.6002341508865356, "mix_mean": 0.5996124744415283, "mix_low": 0.0022096633911132812, "mix_high": 0.02272796630859375, "read_cosine": 0.249290332198143, "beta_mean": 0.010011096484959126, "nonfinite": 0.0, "weighted_memory_to_active": 0.22715187072753906, "mix_std": 0.3020915985107422, "beta_std": 0.012428437359631062, "beta_low": 0.70465087890625, "beta_high": 0.0, "orthogonal_to_active": 0.16413214802742004, "evidence_weights": [-0.21546471118927002, -0.24959085881710052, 1.0241469144821167], "agreement_weight": 0.11162000149488449, "orthogonal_weight": -0.7645169496536255}, {"head": 1, "memory_to_active": 0.010492773726582527, "contribution_to_active": 0.46074870228767395, "mix_mean": 0.5855494141578674, "mix_low": 0.0034198760986328125, "mix_high": 0.010329246520996094, "read_cosine": 0.18238413333892822, "beta_mean": 5.174821490072645e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006378372199833393, "mix_std": 0.2942185401916504, "beta_std": 0.00015513435937464237, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00524606928229332, "evidence_weights": [-0.26916763186454773, -0.21813014149665833, 0.681707501411438], "agreement_weight": 0.5378829836845398, "orthogonal_weight": -0.8989508152008057}, {"head": 2, "memory_to_active": 1.8893288373947144, "contribution_to_active": 0.2902525067329407, "mix_mean": 0.05422043800354004, "mix_low": 0.3101081848144531, "mix_high": 0.0, "read_cosine": 0.2034875452518463, "beta_mean": 0.12727409601211548, "nonfinite": 0.0, "weighted_memory_to_active": 0.31964752078056335, "mix_std": 0.10692569613456726, "beta_std": 0.11515822261571884, "beta_low": 0.01904296875, "beta_high": 0.0, "orthogonal_to_active": 0.0017578168772161007, "evidence_weights": [-0.10035539418458939, -0.0063880872912704945, -0.023095937445759773], "agreement_weight": -0.01157214306294918, "orthogonal_weight": -0.006882538553327322}, {"head": 3, "memory_to_active": 0.020634165033698082, "contribution_to_active": 0.5226978063583374, "mix_mean": 0.605217456817627, "mix_low": 0.004553794860839844, "mix_high": 0.00923919677734375, "read_cosine": 0.2813876271247864, "beta_mean": 9.221995424013585e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014277544803917408, "mix_std": 0.285929799079895, "beta_std": 0.00022020611504558474, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0036279039923101664, "evidence_weights": [-0.26026007533073425, -0.22551698982715607, 2.264646530151367], "agreement_weight": 2.1911275386810303, "orthogonal_weight": -0.33103182911872864}, {"head": 4, "memory_to_active": 0.014661569148302078, "contribution_to_active": 0.37562310695648193, "mix_mean": 0.5168705582618713, "mix_low": 0.0036325454711914062, "mix_high": 0.001987457275390625, "read_cosine": 0.15732347965240479, "beta_mean": 8.059509855229408e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00946501363068819, "mix_std": 0.2898162305355072, "beta_std": 0.0001096096311812289, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00892062857747078, "evidence_weights": [-0.2334500104188919, -0.22049486637115479, 0.13416431844234467], "agreement_weight": -0.04217955097556114, "orthogonal_weight": -1.023382544517517}, {"head": 5, "memory_to_active": 0.21886973083019257, "contribution_to_active": 0.6679065227508545, "mix_mean": 0.7337592244148254, "mix_low": 0.0018873214721679688, "mix_high": 0.024120330810546875, "read_cosine": 0.14022748172283173, "beta_mean": 0.004463977180421352, "nonfinite": 0.0, "weighted_memory_to_active": 0.16958726942539215, "mix_std": 0.2647469937801361, "beta_std": 0.008802862837910652, "beta_low": 0.88751220703125, "beta_high": 0.0, "orthogonal_to_active": 0.12414786964654922, "evidence_weights": [-0.2662266194820404, -0.3452015221118927, 0.541706383228302], "agreement_weight": 0.22219820320606232, "orthogonal_weight": -0.7735877633094788}]}, {"layer": 4, "memory_to_active": 0.9565222859382629, "contribution_to_active": 0.45869138836860657, "mix_mean": 0.37009286880493164, "mix_low": 0.25231966376304626, "mix_high": 0.011284828186035156, "read_cosine": 0.316611111164093, "beta_mean": 0.11040699481964111, "nonfinite": 0.0, "weighted_memory_to_active": 0.12566590309143066, "mix_std": 0.1592751145362854, "beta_std": 0.10095404833555222, "beta_low": 0.39080810546875, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04751119762659073, "output_influence": 0.4958660304546356, "residual_influence": 0.14923541247844696, "state_active": 0.03996560722589493, "state_memory": 0.01647992804646492, "state_peak": 42.841468811035156, "growth_active": 1.2437230348587036, "growth_memory": 1.5295262336730957, "heads": [{"head": 0, "memory_to_active": 0.1600239872932434, "contribution_to_active": 0.7975471615791321, "mix_mean": 0.6881127953529358, "mix_low": 0.011986732482910156, "mix_high": 0.030225753784179688, "read_cosine": 0.44391560554504395, "beta_mean": 0.2572682201862335, "nonfinite": 0.0, "weighted_memory_to_active": 0.136421337723732, "mix_std": 0.3473326861858368, "beta_std": 0.2577948570251465, "beta_low": 0.25665283203125, "beta_high": 0.0, "orthogonal_to_active": 0.08306935429573059, "evidence_weights": [-0.25555700063705444, -0.3641775846481323, -0.11749643087387085], "agreement_weight": 0.0009707914432510734, "orthogonal_weight": 0.7280420064926147}, {"head": 1, "memory_to_active": 0.04765966162085533, "contribution_to_active": 0.6611413955688477, "mix_mean": 0.7018216848373413, "mix_low": 0.00010204315185546875, "mix_high": 0.017838478088378906, "read_cosine": 0.6189550161361694, "beta_mean": 0.0007966061821207404, "nonfinite": 0.0, "weighted_memory_to_active": 0.035772521048784256, "mix_std": 0.23790842294692993, "beta_std": 0.006097942590713501, "beta_low": 0.9827880859375, "beta_high": 0.0, "orthogonal_to_active": 0.01815076172351837, "evidence_weights": [-0.24059437215328217, -0.3658634424209595, 0.5990481376647949], "agreement_weight": 0.3381721079349518, "orthogonal_weight": -0.7317259907722473}, {"head": 2, "memory_to_active": 1.6229541301727295, "contribution_to_active": 0.3457948863506317, "mix_mean": 0.04014028608798981, "mix_low": 0.4009246826171875, "mix_high": 4.00543212890625e-05, "read_cosine": 0.23831626772880554, "beta_mean": 0.06432510912418365, "nonfinite": 0.0, "weighted_memory_to_active": 0.3081386983394623, "mix_std": 0.07929007709026337, "beta_std": 0.06322579830884933, "beta_low": 0.075439453125, "beta_high": 0.0, "orthogonal_to_active": 0.09528549760580063, "evidence_weights": [0.04363643750548363, -0.03555858135223389, 0.27197909355163574], "agreement_weight": 0.13367538154125214, "orthogonal_weight": -0.4331744909286499}, {"head": 3, "memory_to_active": 1.7321585416793823, "contribution_to_active": 0.11698679625988007, "mix_mean": 0.030598629266023636, "mix_low": 0.425537109375, "mix_high": 0.0, "read_cosine": 0.22770962119102478, "beta_mean": 0.13273945450782776, "nonfinite": 0.0, "weighted_memory_to_active": 0.16344967484474182, "mix_std": 0.05296957865357399, "beta_std": 0.12053859978914261, "beta_low": 0.02996826171875, "beta_high": 0.0, "orthogonal_to_active": 0.0006989639950916171, "evidence_weights": [-0.019202912226319313, 0.049328651279211044, -0.2418716847896576], "agreement_weight": 0.07370804250240326, "orthogonal_weight": -0.006227751262485981}, {"head": 4, "memory_to_active": 0.005833802744746208, "contribution_to_active": 0.6490270495414734, "mix_mean": 0.747482419013977, "mix_low": 1.9073486328125e-05, "mix_high": 0.019601821899414062, "read_cosine": 0.15074652433395386, "beta_mean": 1.435274043615209e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004217257257550955, "mix_std": 0.21214599907398224, "beta_std": 3.8339017919497564e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002725126687437296, "evidence_weights": [-0.3324202001094818, -0.3211425244808197, 0.32194092869758606], "agreement_weight": 0.3200220763683319, "orthogonal_weight": -0.6868699193000793}, {"head": 5, "memory_to_active": 2.170503616333008, "contribution_to_active": 0.181650772690773, "mix_mean": 0.01240128930658102, "mix_low": 0.6753482818603516, "mix_high": 2.86102294921875e-06, "read_cosine": 0.2200234979391098, "beta_mean": 0.2072981894016266, "nonfinite": 0.0, "weighted_memory_to_active": 0.10599587112665176, "mix_std": 0.02600390650331974, "beta_std": 0.158028706908226, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08513747155666351, "evidence_weights": [0.058645520359277725, -0.01628517359495163, -0.28514355421066284], "agreement_weight": -0.08943106234073639, "orthogonal_weight": 1.0515708923339844}]}, {"layer": 5, "memory_to_active": 0.3567403554916382, "contribution_to_active": 0.5394811630249023, "mix_mean": 0.514432966709137, "mix_low": 0.055377960205078125, "mix_high": 0.0188090018928051, "read_cosine": 0.07072657346725464, "beta_mean": 0.05210116133093834, "nonfinite": 0.0, "weighted_memory_to_active": 0.1348823606967926, "mix_std": 0.2561424672603607, "beta_std": 0.06707479059696198, "beta_low": 0.6729940176010132, "beta_high": 0.0, "orthogonal_to_active": 0.05584660917520523, "output_influence": 0.5716352462768555, "residual_influence": 0.18083436787128448, "state_active": 0.04055485129356384, "state_memory": 0.010182006284594536, "state_peak": 64.89225769042969, "growth_active": 1.0991119146347046, "growth_memory": 1.2234488725662231, "heads": [{"head": 0, "memory_to_active": 0.21771466732025146, "contribution_to_active": 0.5787674188613892, "mix_mean": 0.47198790311813354, "mix_low": 0.010175704956054688, "mix_high": 0.0016155242919921875, "read_cosine": -0.13205774128437042, "beta_mean": 0.01856166124343872, "nonfinite": 0.0, "weighted_memory_to_active": 0.07343660295009613, "mix_std": 0.27998146414756775, "beta_std": 0.0316786952316761, "beta_low": 0.5357666015625, "beta_high": 0.0, "orthogonal_to_active": 0.03191736713051796, "evidence_weights": [-0.16618604958057404, -0.1857682466506958, 0.09634348005056381], "agreement_weight": -0.026013633236289024, "orthogonal_weight": -0.46185725927352905}, {"head": 1, "memory_to_active": 0.0063339099287986755, "contribution_to_active": 0.5438398718833923, "mix_mean": 0.7007158994674683, "mix_low": 0.0010175704956054688, "mix_high": 0.03886699676513672, "read_cosine": 0.176938995718956, "beta_mean": 1.4554973859048914e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038642236031591892, "mix_std": 0.2707839012145996, "beta_std": 3.0030843845452182e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035652201622724533, "evidence_weights": [-0.24950526654720306, -0.28961315751075745, 0.9707322120666504], "agreement_weight": 0.6949269771575928, "orthogonal_weight": -0.9661073684692383}, {"head": 2, "memory_to_active": 0.017134401947259903, "contribution_to_active": 0.6026344895362854, "mix_mean": 0.6590385437011719, "mix_low": 0.0025463104248046875, "mix_high": 0.021924972534179688, "read_cosine": -0.4929841458797455, "beta_mean": 8.942307613324374e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011002262122929096, "mix_std": 0.26510098576545715, "beta_std": 0.00028936739545315504, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.009586344473063946, "evidence_weights": [-0.2605374753475189, -0.32372966408729553, 0.1436309814453125], "agreement_weight": -0.1392177790403366, "orthogonal_weight": -0.9909545183181763}, {"head": 3, "memory_to_active": 1.3771874904632568, "contribution_to_active": 0.23793278634548187, "mix_mean": 0.12003563344478607, "mix_low": 0.12181663513183594, "mix_high": 0.0, "read_cosine": 0.1692640781402588, "beta_mean": 0.09916525334119797, "nonfinite": 0.0, "weighted_memory_to_active": 0.3827550411224365, "mix_std": 0.15355053544044495, "beta_std": 0.1501687616109848, "beta_low": 0.20941162109375, "beta_high": 0.0, "orthogonal_to_active": 0.1470433473587036, "evidence_weights": [-0.07659889757633209, -0.09985019266605377, 0.045599766075611115], "agreement_weight": 0.03773849457502365, "orthogonal_weight": -0.49905604124069214}, {"head": 4, "memory_to_active": 0.4388965964317322, "contribution_to_active": 0.5842542052268982, "mix_mean": 0.32983115315437317, "mix_low": 0.19503116607666016, "mix_high": 0.005305290222167969, "read_cosine": 0.5460408926010132, "beta_mean": 0.19457660615444183, "nonfinite": 0.0, "weighted_memory_to_active": 0.28006911277770996, "mix_std": 0.34345543384552, "beta_std": 0.21933189034461975, "beta_low": 0.29510498046875, "beta_high": 0.0, "orthogonal_to_active": 0.1088239997625351, "evidence_weights": [-0.11903852969408035, -0.2577590048313141, -0.05371904745697975], "agreement_weight": 0.02862701006233692, "orthogonal_weight": 0.5804251432418823}, {"head": 5, "memory_to_active": 0.08317483961582184, "contribution_to_active": 0.6894580721855164, "mix_mean": 0.8049885034561157, "mix_low": 0.0016803741455078125, "mix_high": 0.04514122009277344, "read_cosine": 0.15715737640857697, "beta_mean": 0.0001994621561607346, "nonfinite": 0.0, "weighted_memory_to_active": 0.05816682428121567, "mix_std": 0.22398242354393005, "beta_std": 0.0009499727166257799, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.034143369644880295, "evidence_weights": [-0.30548256635665894, -0.3296199440956116, 1.8797374963760376], "agreement_weight": 0.49412739276885986, "orthogonal_weight": -0.6234503388404846}]}, {"layer": 6, "memory_to_active": 0.26598870754241943, "contribution_to_active": 0.6261234879493713, "mix_mean": 0.5938266515731812, "mix_low": 0.04249875247478485, "mix_high": 0.03200690075755119, "read_cosine": 0.32263851165771484, "beta_mean": 0.06520585715770721, "nonfinite": 0.0, "weighted_memory_to_active": 0.13506481051445007, "mix_std": 0.23634886741638184, "beta_std": 0.07562170177698135, "beta_low": 0.5898946523666382, "beta_high": 0.0, "orthogonal_to_active": 0.055949706584215164, "output_influence": 0.5793211460113525, "residual_influence": 0.20181432366371155, "state_active": 0.04441998898983002, "state_memory": 0.006732423324137926, "state_peak": 87.90558624267578, "growth_active": 1.20182204246521, "growth_memory": 1.1862155199050903, "heads": [{"head": 0, "memory_to_active": 0.10826407372951508, "contribution_to_active": 0.7784132957458496, "mix_mean": 0.8181855082511902, "mix_low": 0.00013637542724609375, "mix_high": 0.05169200897216797, "read_cosine": 0.3067660927772522, "beta_mean": 0.013139759190380573, "nonfinite": 0.0, "weighted_memory_to_active": 0.09334224462509155, "mix_std": 0.20135337114334106, "beta_std": 0.03600797802209854, "beta_low": 0.799560546875, "beta_high": 0.0, "orthogonal_to_active": 0.06350700557231903, "evidence_weights": [-0.30717480182647705, -0.349791020154953, 1.210371971130371], "agreement_weight": 0.7539082765579224, "orthogonal_weight": -0.7495331764221191}, {"head": 1, "memory_to_active": 0.00525897741317749, "contribution_to_active": 0.5019617676734924, "mix_mean": 0.6010555624961853, "mix_low": 0.0008115768432617188, "mix_high": 0.01556396484375, "read_cosine": 0.49698635935783386, "beta_mean": 1.2013477316941135e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0024227499961853027, "mix_std": 0.2819441258907318, "beta_std": 2.5254574211430736e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012797742383554578, "evidence_weights": [-0.1972299963235855, -0.3047330975532532, 1.1040090322494507], "agreement_weight": 0.11802901327610016, "orthogonal_weight": -0.6867743730545044}, {"head": 2, "memory_to_active": 0.750702977180481, "contribution_to_active": 0.10354261845350266, "mix_mean": 0.047964151948690414, "mix_low": 0.24764251708984375, "mix_high": 3.814697265625e-06, "read_cosine": 0.2670404613018036, "beta_mean": 0.02451426163315773, "nonfinite": 0.0, "weighted_memory_to_active": 0.0724722295999527, "mix_std": 0.0726805329322815, "beta_std": 0.02737177163362503, "beta_low": 0.244873046875, "beta_high": 0.0, "orthogonal_to_active": 0.00980687327682972, "evidence_weights": [0.06828344613313675, -0.05562279000878334, 0.34274235367774963], "agreement_weight": 0.14360499382019043, "orthogonal_weight": -0.16998286545276642}, {"head": 3, "memory_to_active": 0.13811936974525452, "contribution_to_active": 0.6650664806365967, "mix_mean": 0.6578360795974731, "mix_low": 0.0054607391357421875, "mix_high": 0.019026756286621094, "read_cosine": 0.4847773313522339, "beta_mean": 0.227408304810524, "nonfinite": 0.0, "weighted_memory_to_active": 0.0975913479924202, "mix_std": 0.3231104016304016, "beta_std": 0.2532362639904022, "beta_low": 0.19537353515625, "beta_high": 0.0, "orthogonal_to_active": 0.054699137806892395, "evidence_weights": [-0.166959747672081, -0.31825944781303406, 0.14714688062667847], "agreement_weight": -0.0040381476283073425, "orthogonal_weight": 0.6485521793365479}, {"head": 4, "memory_to_active": 0.570927083492279, "contribution_to_active": 1.0814683437347412, "mix_mean": 0.7540410161018372, "mix_low": 0.000713348388671875, "mix_high": 0.09930896759033203, "read_cosine": 0.10740342736244202, "beta_mean": 0.12605226039886475, "nonfinite": 0.0, "weighted_memory_to_active": 0.5281405448913574, "mix_std": 0.2964085340499878, "beta_std": 0.13685660064220428, "beta_low": 0.299560546875, "beta_high": 0.0, "orthogonal_to_active": 0.19043593108654022, "evidence_weights": [-0.1933339238166809, -0.5470601320266724, -0.5471166968345642], "agreement_weight": -0.01784142293035984, "orthogonal_weight": 0.3832901418209076}, {"head": 5, "memory_to_active": 0.022659750655293465, "contribution_to_active": 0.6262884140014648, "mix_mean": 0.6838774681091309, "mix_low": 0.00022792816162109375, "mix_high": 0.006445884704589844, "read_cosine": 0.2728573977947235, "beta_mean": 0.00010854375432245433, "nonfinite": 0.0, "weighted_memory_to_active": 0.016419777646660805, "mix_std": 0.2425961047410965, "beta_std": 0.00023235141998156905, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01596951112151146, "evidence_weights": [-0.23634210228919983, -0.3169955909252167, 0.38232263922691345], "agreement_weight": 0.21019180119037628, "orthogonal_weight": -1.0392029285430908}]}, {"layer": 7, "memory_to_active": 1.7515954971313477, "contribution_to_active": 0.2651800215244293, "mix_mean": 0.17158542573451996, "mix_low": 0.23218664526939392, "mix_high": 0.0011450450401753187, "read_cosine": 0.23735158145427704, "beta_mean": 0.1300373077392578, "nonfinite": 0.0, "weighted_memory_to_active": 0.19949468970298767, "mix_std": 0.14708000421524048, "beta_std": 0.09988291561603546, "beta_low": 0.2142130583524704, "beta_high": 0.00019327799964230508, "orthogonal_to_active": 0.08601493388414383, "output_influence": 0.2816413640975952, "residual_influence": 0.0688837468624115, "state_active": 0.01404019258916378, "state_memory": 0.024529648944735527, "state_peak": 31.297809600830078, "growth_active": 1.1709201335906982, "growth_memory": 1.281072974205017, "heads": [{"head": 0, "memory_to_active": 2.6094696521759033, "contribution_to_active": 0.16764786839485168, "mix_mean": 0.03651850298047066, "mix_low": 0.42525482177734375, "mix_high": 9.5367431640625e-07, "read_cosine": 0.33737218379974365, "beta_mean": 0.16636745631694794, "nonfinite": 0.0, "weighted_memory_to_active": 0.19169119000434875, "mix_std": 0.07556179910898209, "beta_std": 0.13149899244308472, "beta_low": 0.01226806640625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.0006088977679610252, "evidence_weights": [0.04450458288192749, -0.08228692412376404, 0.009456582367420197], "agreement_weight": -0.032618045806884766, "orthogonal_weight": 0.0037443412002176046}, {"head": 1, "memory_to_active": 0.9699606895446777, "contribution_to_active": 0.4031106233596802, "mix_mean": 0.21986117959022522, "mix_low": 0.11115455627441406, "mix_high": 0.0060329437255859375, "read_cosine": 0.37939509749412537, "beta_mean": 0.09341427683830261, "nonfinite": 0.0, "weighted_memory_to_active": 0.3349430561065674, "mix_std": 0.24289648234844208, "beta_std": 0.06648324429988861, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.046136483550071716, "evidence_weights": [-0.21446961164474487, -0.10057441145181656, -0.2957107424736023], "agreement_weight": 0.05801117420196533, "orthogonal_weight": 0.16347604990005493}, {"head": 2, "memory_to_active": 0.60234135389328, "contribution_to_active": 0.36464008688926697, "mix_mean": 0.22276346385478973, "mix_low": 0.021904945373535156, "mix_high": 0.00047588348388671875, "read_cosine": 0.12188589572906494, "beta_mean": 0.03565680608153343, "nonfinite": 0.0, "weighted_memory_to_active": 0.21694456040859222, "mix_std": 0.21221056580543518, "beta_std": 0.03828895837068558, "beta_low": 0.15380859375, "beta_high": 0.0, "orthogonal_to_active": 0.12303674221038818, "evidence_weights": [-0.09087840467691422, -0.177198126912117, 0.8392045497894287], "agreement_weight": 0.020068110898137093, "orthogonal_weight": -0.5840260982513428}, {"head": 3, "memory_to_active": 4.829973220825195, "contribution_to_active": 0.10738488286733627, "mix_mean": 0.03555603697896004, "mix_low": 0.31922149658203125, "mix_high": 0.0, "read_cosine": 0.10093075037002563, "beta_mean": 0.40923964977264404, "nonfinite": 0.0, "weighted_memory_to_active": 0.369154691696167, "mix_std": 0.05178985744714737, "beta_std": 0.28065192699432373, "beta_low": 0.00482177734375, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.3133385181427002, "evidence_weights": [0.07320814579725266, -0.03775753080844879, 0.71474689245224], "agreement_weight": 0.04599139839410782, "orthogonal_weight": -0.936603307723999}, {"head": 4, "memory_to_active": 1.4606385231018066, "contribution_to_active": 0.09209984540939331, "mix_mean": 0.024101343005895615, "mix_low": 0.5137853622436523, "mix_high": 0.0, "read_cosine": 0.220645010471344, "beta_mean": 0.07537776231765747, "nonfinite": 0.0, "weighted_memory_to_active": 0.06518947333097458, "mix_std": 0.04762621968984604, "beta_std": 0.08199033886194229, "beta_low": 0.11444091796875, "beta_high": 0.0, "orthogonal_to_active": 0.01495752576738596, "evidence_weights": [0.1286560446023941, -0.02104344218969345, 0.1858585923910141], "agreement_weight": 0.03984551876783371, "orthogonal_weight": -0.2927233874797821}, {"head": 5, "memory_to_active": 0.037189166992902756, "contribution_to_active": 0.4561968147754669, "mix_mean": 0.49071207642555237, "mix_low": 0.0017986297607421875, "mix_high": 0.0003604888916015625, "read_cosine": 0.26388052105903625, "beta_mean": 0.00016783742466941476, "nonfinite": 0.0, "weighted_memory_to_active": 0.019045166671276093, "mix_std": 0.25239497423171997, "beta_std": 0.00038403310463763773, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.018011407926678658, "evidence_weights": [-0.18140153586864471, -0.2380470335483551, 0.5180485248565674], "agreement_weight": 0.22588464617729187, "orthogonal_weight": -1.0112338066101074}]}, {"layer": 8, "memory_to_active": 0.1509077101945877, "contribution_to_active": 0.5696593523025513, "mix_mean": 0.498351514339447, "mix_low": 0.011650880798697472, "mix_high": 0.0019202232360839844, "read_cosine": 0.40594765543937683, "beta_mean": 0.012895564548671246, "nonfinite": 0.0, "weighted_memory_to_active": 0.07802096009254456, "mix_std": 0.24281242489814758, "beta_std": 0.019764665514230728, "beta_low": 0.7906189560890198, "beta_high": 0.0, "orthogonal_to_active": 0.07087908685207367, "output_influence": 0.5161899328231812, "residual_influence": 0.18504489958286285, "state_active": 0.04316486418247223, "state_memory": 0.0051810964941978455, "state_peak": 54.734622955322266, "growth_active": 1.1029387712478638, "growth_memory": 1.0713664293289185, "heads": [{"head": 0, "memory_to_active": 0.062018804252147675, "contribution_to_active": 0.572108268737793, "mix_mean": 0.605229914188385, "mix_low": 0.0006351470947265625, "mix_high": 0.0014772415161132812, "read_cosine": 0.45768848061561584, "beta_mean": 0.003834207309409976, "nonfinite": 0.0, "weighted_memory_to_active": 0.04380384832620621, "mix_std": 0.24989637732505798, "beta_std": 0.015350323170423508, "beta_low": 0.92156982421875, "beta_high": 0.0, "orthogonal_to_active": 0.027674494311213493, "evidence_weights": [-0.16084390878677368, -0.3823617100715637, 0.5829825401306152], "agreement_weight": 0.2596684396266937, "orthogonal_weight": -0.801045835018158}, {"head": 1, "memory_to_active": 0.07902495563030243, "contribution_to_active": 0.5623202919960022, "mix_mean": 0.5313200950622559, "mix_low": 7.915496826171875e-05, "mix_high": 0.0014925003051757812, "read_cosine": 0.10186396539211273, "beta_mean": 0.0008860150701366365, "nonfinite": 0.0, "weighted_memory_to_active": 0.03470002859830856, "mix_std": 0.23483334481716156, "beta_std": 0.006424015387892723, "beta_low": 0.9912109375, "beta_high": 0.0, "orthogonal_to_active": 0.025128088891506195, "evidence_weights": [-0.19640693068504333, -0.2771788239479065, 0.6710972785949707], "agreement_weight": 0.0709349513053894, "orthogonal_weight": -0.7597649693489075}, {"head": 2, "memory_to_active": 0.012152258306741714, "contribution_to_active": 0.687046468257904, "mix_mean": 0.6862682700157166, "mix_low": 0.0003299713134765625, "mix_high": 0.0038461685180664062, "read_cosine": 0.39424288272857666, "beta_mean": 0.00012125806824769825, "nonfinite": 0.0, "weighted_memory_to_active": 0.009410370141267776, "mix_std": 0.22190174460411072, "beta_std": 0.0034556572791188955, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005729749798774719, "evidence_weights": [-0.2242860645055771, -0.346067875623703, 1.0085252523422241], "agreement_weight": 0.288431853055954, "orthogonal_weight": -0.6956707835197449}, {"head": 3, "memory_to_active": 0.09941929578781128, "contribution_to_active": 0.6318963766098022, "mix_mean": 0.59242844581604, "mix_low": 0.0004596710205078125, "mix_high": 0.0043430328369140625, "read_cosine": 0.8188136219978333, "beta_mean": 0.007036583963781595, "nonfinite": 0.0, "weighted_memory_to_active": 0.07361447066068649, "mix_std": 0.2537740468978882, "beta_std": 0.0068312110379338264, "beta_low": 0.78033447265625, "beta_high": 0.0, "orthogonal_to_active": 0.021973568946123123, "evidence_weights": [-0.18113753199577332, -0.3015678822994232, 0.543663740158081], "agreement_weight": 0.24312494695186615, "orthogonal_weight": -0.603233277797699}, {"head": 4, "memory_to_active": 0.27357733249664307, "contribution_to_active": 0.3211689293384552, "mix_mean": 0.31293660402297974, "mix_low": 0.0036172866821289062, "mix_high": 4.76837158203125e-06, "read_cosine": 0.18039590120315552, "beta_mean": 0.0014688924420624971, "nonfinite": 0.0, "weighted_memory_to_active": 0.11121094971895218, "mix_std": 0.22771838307380676, "beta_std": 0.013988536782562733, "beta_low": 0.9815673828125, "beta_high": 0.0, "orthogonal_to_active": 0.02478417195379734, "evidence_weights": [-0.12264125049114227, -0.1917654126882553, 0.6060346961021423], "agreement_weight": 0.04622046649456024, "orthogonal_weight": -0.23148368299007416}, {"head": 5, "memory_to_active": 0.3792535960674286, "contribution_to_active": 0.6434154510498047, "mix_mean": 0.2619258165359497, "mix_low": 0.06478404998779297, "mix_high": 0.00035762786865234375, "read_cosine": 0.48268115520477295, "beta_mean": 0.06402643024921417, "nonfinite": 0.0, "weighted_memory_to_active": 0.1953860968351364, "mix_std": 0.2687506079673767, "beta_std": 0.07253824174404144, "beta_low": 0.069091796875, "beta_high": 0.0, "orthogonal_to_active": 0.31998443603515625, "evidence_weights": [-0.1450078785419464, -0.14909835159778595, -0.095116026699543], "agreement_weight": -0.010660993866622448, "orthogonal_weight": 1.9140448570251465}]}, {"layer": 9, "memory_to_active": 0.9272721409797668, "contribution_to_active": 0.7069945335388184, "mix_mean": 0.3804051876068115, "mix_low": 0.07107909768819809, "mix_high": 0.006629626266658306, "read_cosine": 0.35979247093200684, "beta_mean": 0.04138266295194626, "nonfinite": 0.0, "weighted_memory_to_active": 0.3942824602127075, "mix_std": 0.23973557353019714, "beta_std": 0.03503532335162163, "beta_low": 0.5797017812728882, "beta_high": 0.0, "orthogonal_to_active": 0.03547695651650429, "output_influence": 0.5463676452636719, "residual_influence": 0.16156834363937378, "state_active": 0.0432293675839901, "state_memory": 0.01566523127257824, "state_peak": 56.83834457397461, "growth_active": 1.1953922510147095, "growth_memory": 1.3159366846084595, "heads": [{"head": 0, "memory_to_active": 0.0005786198889836669, "contribution_to_active": 0.5751678347587585, "mix_mean": 0.6524859070777893, "mix_low": 0.0014162063598632812, "mix_high": 0.027179718017578125, "read_cosine": 0.3445878326892853, "beta_mean": 1.5092667808858096e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00036808112054131925, "mix_std": 0.2849162220954895, "beta_std": 3.7346962926676497e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003989863907918334, "evidence_weights": [-0.30549153685569763, -0.31344738602638245, 0.42745935916900635], "agreement_weight": 0.11564521491527557, "orthogonal_weight": -1.145938754081726}, {"head": 1, "memory_to_active": 0.015059365890920162, "contribution_to_active": 0.7004652619361877, "mix_mean": 0.5789808630943298, "mix_low": 0.0072727203369140625, "mix_high": 0.0031185150146484375, "read_cosine": 0.41508010029792786, "beta_mean": 3.598822149797343e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011323039419949055, "mix_std": 0.27901625633239746, "beta_std": 7.767022907501087e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007755340076982975, "evidence_weights": [-0.23918284475803375, -0.2954747676849365, 0.9977468848228455], "agreement_weight": 0.5007554888725281, "orthogonal_weight": -0.8350780606269836}, {"head": 2, "memory_to_active": 0.17409175634384155, "contribution_to_active": 0.6549370288848877, "mix_mean": 0.600216269493103, "mix_low": 0.0021524429321289062, "mix_high": 0.007640838623046875, "read_cosine": 0.39261624217033386, "beta_mean": 0.0006656523328274488, "nonfinite": 0.0, "weighted_memory_to_active": 0.11951343715190887, "mix_std": 0.26738598942756653, "beta_std": 0.004500939976423979, "beta_low": 0.9957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.06930119544267654, "evidence_weights": [-0.20254036784172058, -0.2769252359867096, 0.7580409646034241], "agreement_weight": 0.33701291680336, "orthogonal_weight": -0.6838354468345642}, {"head": 3, "memory_to_active": 0.872245192527771, "contribution_to_active": 0.2362770438194275, "mix_mean": 0.13538944721221924, "mix_low": 0.13379573822021484, "mix_high": 0.00014209747314453125, "read_cosine": 0.38113486766815186, "beta_mean": 0.03695174306631088, "nonfinite": 0.0, "weighted_memory_to_active": 0.1883765608072281, "mix_std": 0.18949896097183228, "beta_std": 0.03682882711291313, "beta_low": 0.1292724609375, "beta_high": 0.0, "orthogonal_to_active": 0.07208144664764404, "evidence_weights": [-0.007542645558714867, -0.07998669147491455, 0.3362507224082947], "agreement_weight": 0.09311189502477646, "orthogonal_weight": -0.41957297921180725}, {"head": 4, "memory_to_active": 3.2246804237365723, "contribution_to_active": 1.6686869859695435, "mix_mean": 0.1476832777261734, "mix_low": 0.13232994079589844, "mix_high": 0.0010900497436523438, "read_cosine": 0.15249502658843994, "beta_mean": 0.16354648768901825, "nonfinite": 0.0, "weighted_memory_to_active": 1.6386864185333252, "mix_std": 0.2079254686832428, "beta_std": 0.09151018410921097, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016524771228432655, "evidence_weights": [-0.0800456628203392, -0.15198563039302826, -0.04159007966518402], "agreement_weight": -0.06352027505636215, "orthogonal_weight": -0.010502023622393608}, {"head": 5, "memory_to_active": 1.2769775390625, "contribution_to_active": 0.4064333140850067, "mix_mean": 0.16767524182796478, "mix_low": 0.1495075225830078, "mix_high": 0.000606536865234375, "read_cosine": 0.47284072637557983, "beta_mean": 0.04709458723664284, "nonfinite": 0.0, "weighted_memory_to_active": 0.4074271619319916, "mix_std": 0.20967034995555878, "beta_std": 0.07729057967662811, "beta_low": 0.35321044921875, "beta_high": 0.0, "orthogonal_to_active": 0.046800002455711365, "evidence_weights": [0.07271844893693924, -0.16046422719955444, 0.14165452122688293], "agreement_weight": 0.008735333569347858, "orthogonal_weight": 0.14660106599330902}]}, {"layer": 10, "memory_to_active": 0.19441241025924683, "contribution_to_active": 0.7243767976760864, "mix_mean": 0.5261362195014954, "mix_low": 0.020973842591047287, "mix_high": 0.00535011338070035, "read_cosine": 0.3872186839580536, "beta_mean": 0.041354335844516754, "nonfinite": 0.0, "weighted_memory_to_active": 0.09196458011865616, "mix_std": 0.2679486572742462, "beta_std": 0.04437077417969704, "beta_low": 0.6660360097885132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.2614232301712036, "output_influence": 0.4651683568954468, "residual_influence": 0.16525191068649292, "state_active": 0.05241699144244194, "state_memory": 0.013902107253670692, "state_peak": 121.29936981201172, "growth_active": 1.0579135417938232, "growth_memory": 1.0086134672164917, "heads": [{"head": 0, "memory_to_active": 0.6968907713890076, "contribution_to_active": 2.042579174041748, "mix_mean": 0.5756545066833496, "mix_low": 0.09294700622558594, "mix_high": 0.010755538940429688, "read_cosine": 0.10815245658159256, "beta_mean": 0.1842668205499649, "nonfinite": 0.0, "weighted_memory_to_active": 0.4407530426979065, "mix_std": 0.3959043323993683, "beta_std": 0.20148077607154846, "beta_low": 0.157958984375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.4889159202575684, "evidence_weights": [-0.08792661130428314, -0.3776099681854248, -1.8463724851608276], "agreement_weight": 0.03051620163023472, "orthogonal_weight": 3.409202814102173}, {"head": 1, "memory_to_active": 0.00839568953961134, "contribution_to_active": 0.6159638166427612, "mix_mean": 0.6310718059539795, "mix_low": 0.0014734268188476562, "mix_high": 0.015607833862304688, "read_cosine": 0.30814340710639954, "beta_mean": 5.889426392968744e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004825832787901163, "mix_std": 0.2646513283252716, "beta_std": 9.478320862399414e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007571881171315908, "evidence_weights": [-0.25061073899269104, -0.1946418434381485, 0.571414053440094], "agreement_weight": 0.6096451282501221, "orthogonal_weight": -1.6975334882736206}, {"head": 2, "memory_to_active": 0.01419462077319622, "contribution_to_active": 0.6177974939346313, "mix_mean": 0.6621857285499573, "mix_low": 0.00027751922607421875, "mix_high": 0.004450798034667969, "read_cosine": 0.03920169919729233, "beta_mean": 2.7160320314578712e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006359955295920372, "mix_std": 0.24852946400642395, "beta_std": 0.0001238855329575017, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0027328322175890207, "evidence_weights": [-0.2666952311992645, -0.2796488404273987, 0.6156066656112671], "agreement_weight": -0.21545201539993286, "orthogonal_weight": -0.4122074246406555}, {"head": 3, "memory_to_active": 0.31409311294555664, "contribution_to_active": 0.0940374955534935, "mix_mean": 0.28393810987472534, "mix_low": 0.028265953063964844, "mix_high": 0.00015354156494140625, "read_cosine": 0.9141968488693237, "beta_mean": 0.05737011134624481, "nonfinite": 0.0, "weighted_memory_to_active": 0.03544728830456734, "mix_std": 0.22846509516239166, "beta_std": 0.056577641516923904, "beta_low": 0.0113525390625, "beta_high": 0.0, "orthogonal_to_active": 0.037929534912109375, "evidence_weights": [-0.09940892457962036, -0.1474524736404419, 0.1741349846124649], "agreement_weight": 0.021104266867041588, "orthogonal_weight": 1.4299577474594116}, {"head": 4, "memory_to_active": 0.015340065583586693, "contribution_to_active": 0.5573004484176636, "mix_mean": 0.5930972099304199, "mix_low": 0.00016498565673828125, "mix_high": 0.0011301040649414062, "read_cosine": 0.10014663636684418, "beta_mean": 4.5311411668080837e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008962138555943966, "mix_std": 0.24368268251419067, "beta_std": 8.506949234288186e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012648411095142365, "evidence_weights": [-0.1856761872768402, -0.2571481466293335, 0.9678073525428772], "agreement_weight": 0.16718412935733795, "orthogonal_weight": -1.4917097091674805}, {"head": 5, "memory_to_active": 0.11756004393100739, "contribution_to_active": 0.41858208179473877, "mix_mean": 0.4108699560165405, "mix_low": 0.0027141571044921875, "mix_high": 2.86102294921875e-06, "read_cosine": 0.853471040725708, "beta_mean": 0.006357709877192974, "nonfinite": 0.0, "weighted_memory_to_active": 0.05543919652700424, "mix_std": 0.22645892202854156, "beta_std": 0.00786249153316021, "beta_low": 0.826904296875, "beta_high": 0.0, "orthogonal_to_active": 0.018740639090538025, "evidence_weights": [-0.1409417688846588, -0.16468040645122528, 0.26884353160858154], "agreement_weight": 0.09397823363542557, "orthogonal_weight": -0.6633173227310181}]}, {"layer": 11, "memory_to_active": 0.0817209854722023, "contribution_to_active": 0.6241723299026489, "mix_mean": 0.5228466987609863, "mix_low": 0.1087850034236908, "mix_high": 0.043761733919382095, "read_cosine": 0.31496089696884155, "beta_mean": 0.023595359176397324, "nonfinite": 0.0, "weighted_memory_to_active": 0.04316992312669754, "mix_std": 0.32495683431625366, "beta_std": 0.03559146076440811, "beta_low": 0.6915690898895264, "beta_high": 0.0, "orthogonal_to_active": 0.04638507589697838, "output_influence": 0.5511467456817627, "residual_influence": 0.15223033726215363, "state_active": 0.1914268583059311, "state_memory": 0.003504267893731594, "state_peak": 34.00653076171875, "growth_active": 1.0822088718414307, "growth_memory": 1.542155385017395, "heads": [{"head": 0, "memory_to_active": 0.006777845788747072, "contribution_to_active": 0.6029464602470398, "mix_mean": 0.6530585885047913, "mix_low": 0.0009250640869140625, "mix_high": 0.008365631103515625, "read_cosine": 0.2335168570280075, "beta_mean": 9.730669989949092e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004969634581357241, "mix_std": 0.2573752999305725, "beta_std": 0.003928566351532936, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0040749358013272285, "evidence_weights": [-0.17460501194000244, -0.36240315437316895, 0.7940895557403564], "agreement_weight": 0.3834216296672821, "orthogonal_weight": -0.851931095123291}, {"head": 1, "memory_to_active": 0.1337238997220993, "contribution_to_active": 0.42929574847221375, "mix_mean": 0.4685993194580078, "mix_low": 0.005672454833984375, "mix_high": 0.0007076263427734375, "read_cosine": 0.4802441895008087, "beta_mean": 0.004409180488437414, "nonfinite": 0.0, "weighted_memory_to_active": 0.06428807228803635, "mix_std": 0.27141958475112915, "beta_std": 0.01573929190635681, "beta_low": 0.92486572265625, "beta_high": 0.0, "orthogonal_to_active": 0.03474067896604538, "evidence_weights": [-0.14970453083515167, -0.26785415410995483, 0.5882003903388977], "agreement_weight": 0.0741335079073906, "orthogonal_weight": -0.6427289843559265}, {"head": 2, "memory_to_active": 0.005759534426033497, "contribution_to_active": 0.9848992228507996, "mix_mean": 0.43535250425338745, "mix_low": 0.30930614471435547, "mix_high": 0.1603412628173828, "read_cosine": 0.1710343062877655, "beta_mean": 0.06184585392475128, "nonfinite": 0.0, "weighted_memory_to_active": 0.004055723547935486, "mix_std": 0.44954824447631836, "beta_std": 0.0853472650051117, "beta_low": 0.48388671875, "beta_high": 0.0, "orthogonal_to_active": 0.03931102529168129, "evidence_weights": [0.0444074347615242, -0.3629254996776581, -1.3174244165420532], "agreement_weight": 0.00424637645483017, "orthogonal_weight": 10.053568840026855}, {"head": 3, "memory_to_active": 0.008215445093810558, "contribution_to_active": 0.8576271533966064, "mix_mean": 0.41240495443344116, "mix_low": 0.32836055755615234, "mix_high": 0.08495616912841797, "read_cosine": 0.4788140058517456, "beta_mean": 0.049799129366874695, "nonfinite": 0.0, "weighted_memory_to_active": 0.005184503737837076, "mix_std": 0.4398907423019409, "beta_std": 0.07669626176357269, "beta_low": 0.5181884765625, "beta_high": 0.0, "orthogonal_to_active": 0.037875302135944366, "evidence_weights": [0.20648720860481262, -0.28938376903533936, -0.12148734927177429], "agreement_weight": 0.051875270903110504, "orthogonal_weight": 10.332039833068848}, {"head": 4, "memory_to_active": 0.231988787651062, "contribution_to_active": 0.44890862703323364, "mix_mean": 0.6002934575080872, "mix_low": 0.007218360900878906, "mix_high": 0.0018014907836914062, "read_cosine": -0.013664931058883667, "beta_mean": 0.01563814841210842, "nonfinite": 0.0, "weighted_memory_to_active": 0.12555699050426483, "mix_std": 0.2718079090118408, "beta_std": 0.017190052196383476, "beta_low": 0.49676513671875, "beta_high": 0.0, "orthogonal_to_active": 0.12386072427034378, "evidence_weights": [-0.23439927399158478, -0.23746860027313232, 1.861149787902832], "agreement_weight": -0.21608944237232208, "orthogonal_weight": -1.000180721282959}, {"head": 5, "memory_to_active": 0.10386037081480026, "contribution_to_active": 0.42135682702064514, "mix_mean": 0.5673710703849792, "mix_low": 0.0012273788452148438, "mix_high": 0.006398200988769531, "read_cosine": 0.5398208498954773, "beta_mean": 0.00978253223001957, "nonfinite": 0.0, "weighted_memory_to_active": 0.05496462062001228, "mix_std": 0.2596990466117859, "beta_std": 0.014647331088781357, "beta_low": 0.726318359375, "beta_high": 0.0, "orthogonal_to_active": 0.03844776004552841, "evidence_weights": [-0.2040569931268692, -0.3133162260055542, 0.6503940224647522], "agreement_weight": 0.08298768103122711, "orthogonal_weight": -0.8660605549812317}]}, {"layer": 12, "memory_to_active": 0.22404727339744568, "contribution_to_active": 0.5875825881958008, "mix_mean": 0.5467426776885986, "mix_low": 0.008802573196589947, "mix_high": 0.006921609863638878, "read_cosine": 0.22375547885894775, "beta_mean": 0.013329561799764633, "nonfinite": 0.0, "weighted_memory_to_active": 0.11706376820802689, "mix_std": 0.25955501198768616, "beta_std": 0.014582605101168156, "beta_low": 0.8270976543426514, "beta_high": 0.0, "orthogonal_to_active": 0.0703381672501564, "output_influence": 0.5084134340286255, "residual_influence": 0.14787863194942474, "state_active": 0.03347795084118843, "state_memory": 0.00786823220551014, "state_peak": 64.95126342773438, "growth_active": 1.154719352722168, "growth_memory": 1.186720848083496, "heads": [{"head": 0, "memory_to_active": 0.04696910083293915, "contribution_to_active": 0.622037410736084, "mix_mean": 0.5861666202545166, "mix_low": 0.0006160736083984375, "mix_high": 0.007077217102050781, "read_cosine": 0.2789633572101593, "beta_mean": 0.0009230140130966902, "nonfinite": 0.0, "weighted_memory_to_active": 0.026186473667621613, "mix_std": 0.26517191529273987, "beta_std": 0.002155792433768511, "beta_low": 0.98876953125, "beta_high": 0.0, "orthogonal_to_active": 0.024503376334905624, "evidence_weights": [-0.2087145894765854, -0.27694615721702576, -0.02845112606883049], "agreement_weight": 0.10271826386451721, "orthogonal_weight": -0.9963859915733337}, {"head": 1, "memory_to_active": 0.10540752857923508, "contribution_to_active": 0.5485703945159912, "mix_mean": 0.5608522891998291, "mix_low": 0.0003147125244140625, "mix_high": 0.0020742416381835938, "read_cosine": 0.38768690824508667, "beta_mean": 0.0006425119354389608, "nonfinite": 0.0, "weighted_memory_to_active": 0.06625578552484512, "mix_std": 0.265325129032135, "beta_std": 0.00993703305721283, "beta_low": 0.9969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.05600227043032646, "evidence_weights": [-0.2004496306180954, -0.2708144783973694, 1.0406155586242676], "agreement_weight": 0.25599607825279236, "orthogonal_weight": -0.9255027770996094}, {"head": 2, "memory_to_active": 0.5860599279403687, "contribution_to_active": 0.48438629508018494, "mix_mean": 0.3088638484477997, "mix_low": 0.029569625854492188, "mix_high": 2.956390380859375e-05, "read_cosine": 0.5303621292114258, "beta_mean": 0.0771327018737793, "nonfinite": 0.0, "weighted_memory_to_active": 0.2302292287349701, "mix_std": 0.2353779673576355, "beta_std": 0.059083979576826096, "beta_low": 0.0042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.2555691599845886, "evidence_weights": [-0.20905816555023193, -0.059556614607572556, -0.008542394265532494], "agreement_weight": 0.025222525000572205, "orthogonal_weight": 1.4005967378616333}, {"head": 3, "memory_to_active": 0.03490494191646576, "contribution_to_active": 0.754469633102417, "mix_mean": 0.7229330539703369, "mix_low": 0.000110626220703125, "mix_high": 0.019537925720214844, "read_cosine": 0.10488317906856537, "beta_mean": 0.0002809874131344259, "nonfinite": 0.0, "weighted_memory_to_active": 0.025859283283352852, "mix_std": 0.22405090928077698, "beta_std": 0.005685837008059025, "beta_low": 0.997802734375, "beta_high": 0.0, "orthogonal_to_active": 0.024447374045848846, "evidence_weights": [-0.2865687906742096, -0.3351746201515198, 1.67145836353302], "agreement_weight": 0.5687448382377625, "orthogonal_weight": -0.9643599390983582}, {"head": 4, "memory_to_active": 0.5123928189277649, "contribution_to_active": 0.6090379953384399, "mix_mean": 0.46083077788352966, "mix_low": 0.01607227325439453, "mix_high": 0.005542755126953125, "read_cosine": -0.16765055060386658, "beta_mean": 0.0006809890037402511, "nonfinite": 0.0, "weighted_memory_to_active": 0.315099835395813, "mix_std": 0.2932780981063843, "beta_std": 0.00757208839058876, "beta_low": 0.98101806640625, "beta_high": 0.0, "orthogonal_to_active": 0.031963448971509933, "evidence_weights": [-0.15608429908752441, -0.216970294713974, -0.12510155141353607], "agreement_weight": 0.015670863911509514, "orthogonal_weight": -0.10531347244977951}, {"head": 5, "memory_to_active": 0.05854931101202965, "contribution_to_active": 0.5069936513900757, "mix_mean": 0.6408094167709351, "mix_low": 0.0061321258544921875, "mix_high": 0.007267951965332031, "read_cosine": 0.20828786492347717, "beta_mean": 0.00031716329976916313, "nonfinite": 0.0, "weighted_memory_to_active": 0.03875197842717171, "mix_std": 0.27412593364715576, "beta_std": 0.0030608908273279667, "beta_low": 0.9937744140625, "beta_high": 0.0, "orthogonal_to_active": 0.02954336442053318, "evidence_weights": [-0.2271454930305481, -0.31705519556999207, 1.2274671792984009], "agreement_weight": 0.4173939824104309, "orthogonal_weight": -0.8130092024803162}]}, {"layer": 13, "memory_to_active": 0.10872375965118408, "contribution_to_active": 0.5364008545875549, "mix_mean": 0.5396015048027039, "mix_low": 0.005881468765437603, "mix_high": 0.008573056198656559, "read_cosine": 0.3910413980484009, "beta_mean": 0.013099287636578083, "nonfinite": 0.0, "weighted_memory_to_active": 0.045728202909231186, "mix_std": 0.2593688368797302, "beta_std": 0.014991017058491707, "beta_low": 0.8117574453353882, "beta_high": 0.0, "orthogonal_to_active": 0.04801340773701668, "output_influence": 0.5188344717025757, "residual_influence": 0.18498443067073822, "state_active": 0.12836886942386627, "state_memory": 0.040674641728401184, "state_peak": 98.29444122314453, "growth_active": 0.9047249555587769, "growth_memory": 1.0042428970336914, "heads": [{"head": 0, "memory_to_active": 0.008494431152939796, "contribution_to_active": 0.5991222858428955, "mix_mean": 0.6161453723907471, "mix_low": 0.0006952285766601562, "mix_high": 0.00469207763671875, "read_cosine": 0.43711817264556885, "beta_mean": 3.132159326924011e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005465078167617321, "mix_std": 0.2675805687904358, "beta_std": 6.26803666818887e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005237319972366095, "evidence_weights": [-0.1935463845729828, -0.2667540907859802, 0.7549239993095398], "agreement_weight": 0.18411506712436676, "orthogonal_weight": -1.0718615055084229}, {"head": 1, "memory_to_active": 0.014566412195563316, "contribution_to_active": 0.6074815392494202, "mix_mean": 0.5721530914306641, "mix_low": 0.0018377304077148438, "mix_high": 0.004918098449707031, "read_cosine": 0.22127127647399902, "beta_mean": 6.202531949384138e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00839908979833126, "mix_std": 0.27718600630760193, "beta_std": 0.00013378950825426728, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011085230857133865, "evidence_weights": [-0.1798122078180313, -0.26326984167099, 0.5773118734359741], "agreement_weight": 0.5012231469154358, "orthogonal_weight": -1.3758171796798706}, {"head": 2, "memory_to_active": 0.06068895757198334, "contribution_to_active": 0.4872226417064667, "mix_mean": 0.39314642548561096, "mix_low": 0.0011205673217773438, "mix_high": 0.00021648406982421875, "read_cosine": 0.5152298212051392, "beta_mean": 0.007644826080650091, "nonfinite": 0.0, "weighted_memory_to_active": 0.02966928854584694, "mix_std": 0.2341008484363556, "beta_std": 0.028674818575382233, "beta_low": 0.876220703125, "beta_high": 0.0, "orthogonal_to_active": 0.017590157687664032, "evidence_weights": [-0.18225860595703125, -0.21004264056682587, 0.41621923446655273], "agreement_weight": 0.00750448415055871, "orthogonal_weight": -0.8458399176597595}, {"head": 3, "memory_to_active": 0.1459118276834488, "contribution_to_active": 0.5628532767295837, "mix_mean": 0.6626130938529968, "mix_low": 0.02305316925048828, "mix_high": 0.030162811279296875, "read_cosine": -0.0374944806098938, "beta_mean": 0.0008692995761521161, "nonfinite": 0.0, "weighted_memory_to_active": 0.10640870034694672, "mix_std": 0.2883688509464264, "beta_std": 0.008259118534624577, "beta_low": 0.99420166015625, "beta_high": 0.0, "orthogonal_to_active": 0.09436395764350891, "evidence_weights": [-0.3143787682056427, -0.24251289665699005, 1.5506234169006348], "agreement_weight": 6.395313262939453, "orthogonal_weight": -0.9699270129203796}, {"head": 4, "memory_to_active": 0.3760190010070801, "contribution_to_active": 0.2421720027923584, "mix_mean": 0.33352571725845337, "mix_low": 0.0062694549560546875, "mix_high": 9.5367431640625e-07, "read_cosine": 0.8844776153564453, "beta_mean": 0.06979870051145554, "nonfinite": 0.0, "weighted_memory_to_active": 0.09247158467769623, "mix_std": 0.21844743192195892, "beta_std": 0.05082665756344795, "beta_low": 0.000732421875, "beta_high": 0.0, "orthogonal_to_active": 0.13309818506240845, "evidence_weights": [-0.0754585936665535, -0.23050622642040253, 0.4419553577899933], "agreement_weight": 0.006140032317489386, "orthogonal_weight": 2.2702784538269043}, {"head": 5, "memory_to_active": 0.04666188359260559, "contribution_to_active": 0.7195532917976379, "mix_mean": 0.6600252389907837, "mix_low": 0.0023126602172851562, "mix_high": 0.011447906494140625, "read_cosine": 0.3256458640098572, "beta_mean": 0.0001895558089017868, "nonfinite": 0.0, "weighted_memory_to_active": 0.031955476850271225, "mix_std": 0.2705291211605072, "beta_std": 0.0019890358671545982, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.026705600321292877, "evidence_weights": [-0.2223079949617386, -0.29691189527511597, 0.8620073199272156], "agreement_weight": 0.19244593381881714, "orthogonal_weight": -0.9184533953666687}]}, {"layer": 14, "memory_to_active": 0.225813090801239, "contribution_to_active": 0.4169980585575104, "mix_mean": 0.42761147022247314, "mix_low": 0.02777942083775997, "mix_high": 0.0062228841707110405, "read_cosine": 0.6075687408447266, "beta_mean": 0.06228230148553848, "nonfinite": 0.0, "weighted_memory_to_active": 0.09502378106117249, "mix_std": 0.2437257021665573, "beta_std": 0.06011149287223816, "beta_low": 0.502105712890625, "beta_high": 0.0, "orthogonal_to_active": 0.042519018054008484, "output_influence": 0.3541598618030548, "residual_influence": 0.14546647667884827, "state_active": 0.053829602897167206, "state_memory": 0.014858564361929893, "state_peak": 47.33060836791992, "growth_active": 1.1107481718063354, "growth_memory": 1.2767983675003052, "heads": [{"head": 0, "memory_to_active": 0.37588030099868774, "contribution_to_active": 0.1934705376625061, "mix_mean": 0.19159053266048431, "mix_low": 0.02071666717529297, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8067395687103271, "beta_mean": 0.06753845512866974, "nonfinite": 0.0, "weighted_memory_to_active": 0.08065838366746902, "mix_std": 0.18364690244197845, "beta_std": 0.05975371599197388, "beta_low": 0.00799560546875, "beta_high": 0.0, "orthogonal_to_active": 0.09107761085033417, "evidence_weights": [-0.10078826546669006, -0.04741859436035156, 0.39982014894485474], "agreement_weight": 0.024356314912438393, "orthogonal_weight": 1.8081459999084473}, {"head": 1, "memory_to_active": 0.4630977213382721, "contribution_to_active": 0.29057517647743225, "mix_mean": 0.20323339104652405, "mix_low": 0.12971878051757812, "mix_high": 9.5367431640625e-06, "read_cosine": 0.8685693740844727, "beta_mean": 0.2638128995895386, "nonfinite": 0.0, "weighted_memory_to_active": 0.2123231142759323, "mix_std": 0.23660433292388916, "beta_std": 0.2566557228565216, "beta_low": 0.0938720703125, "beta_high": 0.0, "orthogonal_to_active": 0.015905030071735382, "evidence_weights": [0.0003988025418948382, -0.1952744573354721, -0.09096589684486389], "agreement_weight": 0.06032823398709297, "orthogonal_weight": 0.23213234543800354}, {"head": 2, "memory_to_active": 0.12914399802684784, "contribution_to_active": 0.5800802707672119, "mix_mean": 0.5955228209495544, "mix_low": 0.002956390380859375, "mix_high": 0.004672050476074219, "read_cosine": 0.39457377791404724, "beta_mean": 0.002102743834257126, "nonfinite": 0.0, "weighted_memory_to_active": 0.08466266840696335, "mix_std": 0.2665737569332123, "beta_std": 0.002611949574202299, "beta_low": 0.9796142578125, "beta_high": 0.0, "orthogonal_to_active": 0.06662645936012268, "evidence_weights": [-0.1364205926656723, -0.3192930221557617, 0.9381314516067505], "agreement_weight": 0.0858636423945427, "orthogonal_weight": -0.8843235969543457}, {"head": 3, "memory_to_active": 0.04162760078907013, "contribution_to_active": 0.5950976014137268, "mix_mean": 0.671717643737793, "mix_low": 0.004694938659667969, "mix_high": 0.023337364196777344, "read_cosine": -0.0754488855600357, "beta_mean": 0.005936519708484411, "nonfinite": 0.0, "weighted_memory_to_active": 0.025941455736756325, "mix_std": 0.2586093544960022, "beta_std": 0.008791692554950714, "beta_low": 0.83489990234375, "beta_high": 0.0, "orthogonal_to_active": 0.020262420177459717, "evidence_weights": [-0.3429492115974426, -0.3111775815486908, 0.2886699438095093], "agreement_weight": 0.06993317604064941, "orthogonal_weight": -0.8420494198799133}, {"head": 4, "memory_to_active": 0.1606307029724121, "contribution_to_active": 0.5017021894454956, "mix_mean": 0.5204738974571228, "mix_low": 0.004689216613769531, "mix_high": 0.008196830749511719, "read_cosine": 0.8938692808151245, "beta_mean": 0.03301670402288437, "nonfinite": 0.0, "weighted_memory_to_active": 0.09397928416728973, "mix_std": 0.26217758655548096, "beta_std": 0.02441522851586342, "beta_low": 0.11639404296875, "beta_high": 0.0, "orthogonal_to_active": 0.025073949247598648, "evidence_weights": [-0.2127385139465332, -0.23989905416965485, 0.43497687578201294], "agreement_weight": 0.13315734267234802, "orthogonal_weight": -0.6107813715934753}, {"head": 5, "memory_to_active": 0.18449810147285461, "contribution_to_active": 0.34106242656707764, "mix_mean": 0.3831304907798767, "mix_low": 0.0039005279541015625, "mix_high": 0.0011167526245117188, "read_cosine": 0.7571090459823608, "beta_mean": 0.0012864603195339441, "nonfinite": 0.0, "weighted_memory_to_active": 0.07257772237062454, "mix_std": 0.2547423243522644, "beta_std": 0.008440643548965454, "beta_low": 0.9798583984375, "beta_high": 0.0, "orthogonal_to_active": 0.03616861253976822, "evidence_weights": [-0.17440155148506165, -0.17899928987026215, 0.35212579369544983], "agreement_weight": 0.06923267990350723, "orthogonal_weight": -0.7565760612487793}]}, {"layer": 15, "memory_to_active": 0.05456371605396271, "contribution_to_active": 0.4903258681297302, "mix_mean": 0.4234783947467804, "mix_low": 0.00880448054522276, "mix_high": 0.0030862491112202406, "read_cosine": 0.6487643718719482, "beta_mean": 0.03723173961043358, "nonfinite": 0.0, "weighted_memory_to_active": 0.027116190642118454, "mix_std": 0.2863333225250244, "beta_std": 0.04471978917717934, "beta_low": 0.7071330547332764, "beta_high": 0.0, "orthogonal_to_active": 0.008652010932564735, "output_influence": 0.3139656186103821, "residual_influence": 0.183841735124588, "state_active": 0.16170889139175415, "state_memory": 0.010526367463171482, "state_peak": 24.686365127563477, "growth_active": 1.1866183280944824, "growth_memory": 1.21610426902771, "heads": [{"head": 0, "memory_to_active": 0.103605717420578, "contribution_to_active": 0.4871298670768738, "mix_mean": 0.3928667902946472, "mix_low": 0.008908271789550781, "mix_high": 0.0066242218017578125, "read_cosine": 0.5900842547416687, "beta_mean": 0.042014844715595245, "nonfinite": 0.0, "weighted_memory_to_active": 0.050540465861558914, "mix_std": 0.28745996952056885, "beta_std": 0.030843138694763184, "beta_low": 0.07421875, "beta_high": 0.0, "orthogonal_to_active": 0.02963736653327942, "evidence_weights": [-0.19004225730895996, -0.19367580115795135, 0.4964819550514221], "agreement_weight": 0.09160381555557251, "orthogonal_weight": -0.8104637265205383}, {"head": 1, "memory_to_active": 0.10903853178024292, "contribution_to_active": 0.43352383375167847, "mix_mean": 0.4441607892513275, "mix_low": 0.0009660720825195312, "mix_high": 0.006740570068359375, "read_cosine": 0.9302973747253418, "beta_mean": 0.00041330515523441136, "nonfinite": 0.0, "weighted_memory_to_active": 0.04754786193370819, "mix_std": 0.287090539932251, "beta_std": 0.0019841124303638935, "beta_low": 0.99725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.002264627255499363, "evidence_weights": [-0.15173500776290894, -0.1605163961648941, 0.16721023619174957], "agreement_weight": 0.12377190589904785, "orthogonal_weight": 0.10268028825521469}, {"head": 2, "memory_to_active": 0.004685459658503532, "contribution_to_active": 0.48032346367836, "mix_mean": 0.340549111366272, "mix_low": 0.01607227325439453, "mix_high": 5.435943603515625e-05, "read_cosine": 0.7655239701271057, "beta_mean": 6.287997530307621e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0023861676454544067, "mix_std": 0.2705349028110504, "beta_std": 0.00013900033081881702, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007734828977845609, "evidence_weights": [-0.20597776770591736, -0.26385483145713806, 0.1035069152712822], "agreement_weight": 0.00965663231909275, "orthogonal_weight": 0.864730954170227}, {"head": 3, "memory_to_active": 0.0016807328211143613, "contribution_to_active": 0.38249334692955017, "mix_mean": 0.4008665084838867, "mix_low": 0.005925178527832031, "mix_high": 0.0004558563232421875, "read_cosine": 0.5643104314804077, "beta_mean": 9.032167326950002e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.000557691790163517, "mix_std": 0.2912820875644684, "beta_std": 2.0500005120993592e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00037980196066200733, "evidence_weights": [-0.2734032869338989, -0.23743155598640442, 0.22284014523029327], "agreement_weight": 0.04998364672064781, "orthogonal_weight": -0.6513358354568481}, {"head": 4, "memory_to_active": 0.0776461809873581, "contribution_to_active": 0.6300767660140991, "mix_mean": 0.39027732610702515, "mix_low": 0.0105743408203125, "mix_high": 0.0014467239379882812, "read_cosine": 0.7367252707481384, "beta_mean": 0.1773371696472168, "nonfinite": 0.0, "weighted_memory_to_active": 0.04261184483766556, "mix_std": 0.3099084496498108, "beta_std": 0.2089865356683731, "beta_low": 0.20440673828125, "beta_high": 0.0, "orthogonal_to_active": 0.006903637200593948, "evidence_weights": [-0.20666202902793884, -0.2544326186180115, -0.02085285820066929], "agreement_weight": 0.013140412978827953, "orthogonal_weight": 0.3594737648963928}, {"head": 5, "memory_to_active": 0.030725684016942978, "contribution_to_active": 0.5284077525138855, "mix_mean": 0.5721498727798462, "mix_low": 0.010380744934082031, "mix_high": 0.0031957626342773438, "read_cosine": 0.30564504861831665, "beta_mean": 0.0035531949251890182, "nonfinite": 0.0, "weighted_memory_to_active": 0.01905311644077301, "mix_std": 0.27172398567199707, "beta_std": 0.02634543925523758, "beta_low": 0.9669189453125, "beta_high": 0.0, "orthogonal_to_active": 0.011953143402934074, "evidence_weights": [-0.28087496757507324, -0.30164507031440735, 0.6422451734542847], "agreement_weight": 0.31280288100242615, "orthogonal_weight": 0.8124370574951172}]}]}, {"step": 65000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8186489100007748, "memory_to_active": 0.8086044788360596, "contribution_to_active": 0.5773385763168335, "mix_mean": 0.4217938780784607, "mix_low": 0.12618564069271088, "mix_high": 0.01032737921923399, "read_cosine": 0.3438000977039337, "beta_mean": 0.06760810315608978, "nonfinite": 0.0, "weighted_memory_to_active": 0.21002665162086487, "mix_std": 0.2234303057193756, "beta_std": 0.0598822645843029, "beta_low": 0.5865377187728882, "beta_high": 3.941853719879873e-05, "orthogonal_to_active": 0.06469764560461044, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0656888484954834, "normal_token_accuracy": 0.43492770195007324, "state_active": 0.0631350576877594, "state_memory": 0.01727665774524212, "state_peak": 119.71126556396484, "growth_active": 1.1182605028152466, "growth_memory": 1.286747932434082, "output_influence": 0.4379276633262634, "residual_influence": 0.15315032005310059, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.187838554382324, "no_memory_loss_delta": 0.6049801707267761, "scrambled_loss_delta": 0.14803434908390045, "memory_only_loss_delta": 3.5708060264587402, "no_evidence_loss_delta": 0.3814513087272644, "no_agreement_loss_delta": 0.018354341387748718, "no_orthogonal_loss_delta": 0.013271139934659004}, {"index": 94, "tokens": 256, "normal_loss": 3.152519941329956, "no_memory_loss_delta": 0.6505454182624817, "scrambled_loss_delta": 0.35936614871025085, "memory_only_loss_delta": 3.310959815979004, "no_evidence_loss_delta": 0.29308992624282837, "no_agreement_loss_delta": 0.04409312829375267, "no_orthogonal_loss_delta": 0.049997322261333466}, {"index": 188, "tokens": 256, "normal_loss": 3.4155001640319824, "no_memory_loss_delta": 0.8544062376022339, "scrambled_loss_delta": 0.12740126252174377, "memory_only_loss_delta": 3.2111196517944336, "no_evidence_loss_delta": 0.45796871185302734, "no_agreement_loss_delta": 0.04567011818289757, "no_orthogonal_loss_delta": 0.04509139060974121}, {"index": 283, "tokens": 131, "normal_loss": 1.9736984968185425, "no_memory_loss_delta": 0.3578655421733856, "scrambled_loss_delta": 0.13190430402755737, "memory_only_loss_delta": 2.905008554458618, "no_evidence_loss_delta": 0.1313428282737732, "no_agreement_loss_delta": -0.021028418093919754, "no_orthogonal_loss_delta": -0.012014450505375862}], "agreement_weight_abs_mean": 0.2362155169248581, "agreement_weight_abs_max": 6.452627658843994, "agreement_weight_mean": 0.20586277544498444, "orthogonal_weight_abs_mean": 0.9163448810577393, "orthogonal_weight_abs_max": 10.435698509216309, "orthogonal_weight_mean": -0.08061951398849487, "no_memory_loss": 3.7186620235443115, "no_memory_loss_delta": 0.6529732346534729, "no_memory_token_accuracy_gain": 0.07341490685939789, "no_memory_token_win_fraction": 0.6640712022781372, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 47.95040826435585, "scrambled_loss": 3.265676259994507, "scrambled_loss_delta": 0.19998744130134583, "scrambled_token_accuracy_gain": 0.02002224698662758, "scrambled_token_win_fraction": 0.604004442691803, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.125896466464557, "memory_only_loss": 6.363058567047119, "memory_only_loss_delta": 3.2973692417144775, "memory_only_token_accuracy_gain": 0.27919912338256836, "memory_only_token_win_fraction": 0.9087875485420227, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.30196741563695, "no_evidence_loss": 3.407322406768799, "no_evidence_loss_delta": 0.3416334390640259, "no_evidence_token_accuracy_gain": 0.030033370479941368, "no_evidence_token_win_fraction": 0.621802031993866, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 28.939136165855516, "no_agreement_loss": 3.093412160873413, "no_agreement_loss_delta": 0.02772344835102558, "no_agreement_token_accuracy_gain": -0.01223581749945879, "no_agreement_token_win_fraction": 0.47385984659194946, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.7342680404515454, "no_orthogonal_loss": 3.094794750213623, "no_orthogonal_loss_delta": 0.029105927795171738, "no_orthogonal_token_accuracy_gain": -0.004449388012290001, "no_orthogonal_token_win_fraction": 0.57953280210495, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.8686430087876786, "short_no_memory_loss_delta": 0.5998607277870178, "long_context_memory_gain": 0.05311250686645508, "normal_context_loss_gain": 0.17077083885669708, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.652012825012207, "contribution_to_active": 1.28456711769104, "mix_mean": 0.10732588917016983, "mix_low": 0.5195261836051941, "mix_high": 0.004955291748046875, "read_cosine": 0.40041404962539673, "beta_mean": 0.21156403422355652, "nonfinite": 0.0, "weighted_memory_to_active": 1.3209341764450073, "mix_std": 0.09282181411981583, "beta_std": 0.13858696818351746, "beta_low": 0.15032958984375, "beta_high": 0.0, "orthogonal_to_active": 0.04483265429735184, "output_influence": 0.21358971297740936, "residual_influence": 0.24295024573802948, "state_active": 0.06460458785295486, "state_memory": 0.044815920293331146, "state_peak": 46.077579498291016, "growth_active": 1.2094123363494873, "growth_memory": 1.2863556146621704, "heads": [{"head": 0, "memory_to_active": 2.806112051010132, "contribution_to_active": 0.4785512685775757, "mix_mean": 0.021344251930713654, "mix_low": 0.6016731262207031, "mix_high": 0.0, "read_cosine": 0.2154836356639862, "beta_mean": 0.19917899370193481, "nonfinite": 0.0, "weighted_memory_to_active": 0.6018555760383606, "mix_std": 0.06314972788095474, "beta_std": 0.10643675923347473, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06492527574300766, "evidence_weights": [-0.008021146059036255, -0.04817293584346771, -0.06198493763804436], "agreement_weight": -0.21749378740787506, "orthogonal_weight": -0.12310884147882462}, {"head": 1, "memory_to_active": 6.969900608062744, "contribution_to_active": 3.1287474632263184, "mix_mean": 0.019718505442142487, "mix_low": 0.6234159469604492, "mix_high": 0.0, "read_cosine": 0.41344431042671204, "beta_mean": 0.46280670166015625, "nonfinite": 0.0, "weighted_memory_to_active": 3.4077792167663574, "mix_std": 0.057742197066545486, "beta_std": 0.2206403911113739, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.022362958639860153, "evidence_weights": [0.017705067992210388, -0.11024817079305649, 0.6841241717338562], "agreement_weight": 0.029202327132225037, "orthogonal_weight": 0.014074277132749557}, {"head": 2, "memory_to_active": 2.6287176609039307, "contribution_to_active": 0.5903283357620239, "mix_mean": 0.01546331774443388, "mix_low": 0.6444482803344727, "mix_high": 0.0, "read_cosine": 0.26807931065559387, "beta_mean": 0.23755505681037903, "nonfinite": 0.0, "weighted_memory_to_active": 0.6857975721359253, "mix_std": 0.03556659817695618, "beta_std": 0.17078329622745514, "beta_low": 0.00494384765625, "beta_high": 0.0, "orthogonal_to_active": 0.05925250053405762, "evidence_weights": [0.06070350110530853, 0.04743274301290512, 0.23702648282051086], "agreement_weight": 0.009305687621235847, "orthogonal_weight": -0.20010018348693848}, {"head": 3, "memory_to_active": 7.831664085388184, "contribution_to_active": 2.9947876930236816, "mix_mean": 0.030102450400590897, "mix_low": 0.4921121597290039, "mix_high": 0.0, "read_cosine": 0.27295735478401184, "beta_mean": 0.3123563528060913, "nonfinite": 0.0, "weighted_memory_to_active": 3.1284806728363037, "mix_std": 0.07139436900615692, "beta_std": 0.24473625421524048, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.10142800211906433, "evidence_weights": [-0.037217289209365845, -0.03294462710618973, -0.14297886192798615], "agreement_weight": -0.007826743647456169, "orthogonal_weight": -0.03781165927648544}, {"head": 4, "memory_to_active": 1.5646560192108154, "contribution_to_active": 0.05804489552974701, "mix_mean": 0.01344248652458191, "mix_low": 0.7411775588989258, "mix_high": 0.0, "read_cosine": 0.31171900033950806, "beta_mean": 0.037151336669921875, "nonfinite": 0.0, "weighted_memory_to_active": 0.042319536209106445, "mix_std": 0.043765246868133545, "beta_std": 0.0667317807674408, "beta_low": 0.4671630859375, "beta_high": 0.0, "orthogonal_to_active": 0.0008283538045361638, "evidence_weights": [0.12220898270606995, 0.0854407325387001, -0.151311457157135], "agreement_weight": -0.20017065107822418, "orthogonal_weight": 0.01819169893860817}, {"head": 5, "memory_to_active": 0.11102554947137833, "contribution_to_active": 0.4569423794746399, "mix_mean": 0.54388427734375, "mix_low": 0.014329910278320312, "mix_high": 0.02973175048828125, "read_cosine": 0.920800507068634, "beta_mean": 0.020335765555500984, "nonfinite": 0.0, "weighted_memory_to_active": 0.05937252193689346, "mix_std": 0.285312682390213, "beta_std": 0.02219327539205551, "beta_low": 0.4298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.02019881084561348, "evidence_weights": [-0.17277072370052338, -0.21706680953502655, 0.3820666968822479], "agreement_weight": 0.28730520606040955, "orthogonal_weight": -0.6902980208396912}]}, {"layer": 1, "memory_to_active": 2.4086599349975586, "contribution_to_active": 0.4714415371417999, "mix_mean": 0.21355590224266052, "mix_low": 0.35944145917892456, "mix_high": 0.0012617111206054688, "read_cosine": 0.28013116121292114, "beta_mean": 0.17263665795326233, "nonfinite": 0.0, "weighted_memory_to_active": 0.3120219111442566, "mix_std": 0.11568953841924667, "beta_std": 0.12706241011619568, "beta_low": 0.35382080078125, "beta_high": 0.00010172525799134746, "orthogonal_to_active": 0.03910525143146515, "output_influence": 0.2613697946071625, "residual_influence": 0.07113680243492126, "state_active": 0.027835767716169357, "state_memory": 0.03489511087536812, "state_peak": 39.36973190307617, "growth_active": 1.146357536315918, "growth_memory": 1.4730056524276733, "heads": [{"head": 0, "memory_to_active": 0.03625160828232765, "contribution_to_active": 0.4754413664340973, "mix_mean": 0.5245767831802368, "mix_low": 0.005306243896484375, "mix_high": 0.0026235580444335938, "read_cosine": -0.0006345920264720917, "beta_mean": 0.00019523467926774174, "nonfinite": 0.0, "weighted_memory_to_active": 0.02387813664972782, "mix_std": 0.28168386220932007, "beta_std": 0.0005238429293967783, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.023743322119116783, "evidence_weights": [-0.20627281069755554, -0.2705557942390442, 0.3015323579311371], "agreement_weight": 0.2025943398475647, "orthogonal_weight": -1.0301717519760132}, {"head": 1, "memory_to_active": 2.9404377937316895, "contribution_to_active": 0.2954710125923157, "mix_mean": 0.022870894521474838, "mix_low": 0.4648771286010742, "mix_high": 9.5367431640625e-07, "read_cosine": 0.513775646686554, "beta_mean": 0.2742551565170288, "nonfinite": 0.0, "weighted_memory_to_active": 0.32619455456733704, "mix_std": 0.04208526015281677, "beta_std": 0.18870384991168976, "beta_low": 0.0010986328125, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.022334348410367966, "evidence_weights": [0.023220382630825043, -0.06547310203313828, 0.056004445999860764], "agreement_weight": -0.0027409240137785673, "orthogonal_weight": 0.20051081478595734}, {"head": 2, "memory_to_active": 0.04690533131361008, "contribution_to_active": 0.5449120998382568, "mix_mean": 0.668093204498291, "mix_low": 0.00038909912109375, "mix_high": 0.0049457550048828125, "read_cosine": 0.3972615897655487, "beta_mean": 0.0016631721518933773, "nonfinite": 0.0, "weighted_memory_to_active": 0.03152807801961899, "mix_std": 0.2315666228532791, "beta_std": 0.004540321882814169, "beta_low": 0.977294921875, "beta_high": 0.0, "orthogonal_to_active": 0.021804053336381912, "evidence_weights": [-0.2761121988296509, -0.37084782123565674, 0.6725776791572571], "agreement_weight": 0.046012960374355316, "orthogonal_weight": -0.8006423115730286}, {"head": 3, "memory_to_active": 2.8827908039093018, "contribution_to_active": 0.15818671882152557, "mix_mean": 0.02554965391755104, "mix_low": 0.5238246917724609, "mix_high": 0.0, "read_cosine": 0.19188356399536133, "beta_mean": 0.31965190172195435, "nonfinite": 0.0, "weighted_memory_to_active": 0.20250555872917175, "mix_std": 0.06565794348716736, "beta_std": 0.2588915228843689, "beta_low": 0.0723876953125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.08388465642929077, "evidence_weights": [0.03835539519786835, -0.018170783296227455, -0.033724721521139145], "agreement_weight": -0.027771981433033943, "orthogonal_weight": -0.4859822392463684}, {"head": 4, "memory_to_active": 3.936638832092285, "contribution_to_active": 1.2532105445861816, "mix_mean": 0.033693376928567886, "mix_low": 0.3191871643066406, "mix_high": 0.0, "read_cosine": 0.32266488671302795, "beta_mean": 0.15544618666172028, "nonfinite": 0.0, "weighted_memory_to_active": 1.21847403049469, "mix_std": 0.05502775311470032, "beta_std": 0.1185392364859581, "beta_low": 0.0322265625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.05545523017644882, "evidence_weights": [-0.025597646832466125, -0.006601948291063309, 0.14490960538387299], "agreement_weight": -0.008024987764656544, "orthogonal_weight": 0.06683071702718735}, {"head": 5, "memory_to_active": 4.60893440246582, "contribution_to_active": 0.10142739862203598, "mix_mean": 0.006551466882228851, "mix_low": 0.8430643081665039, "mix_high": 0.0, "read_cosine": 0.2558358907699585, "beta_mean": 0.28460830450057983, "nonfinite": 0.0, "weighted_memory_to_active": 0.06955111026763916, "mix_std": 0.018115725368261337, "beta_std": 0.19117560982704163, "beta_low": 0.040283203125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.027409885078668594, "evidence_weights": [0.16147573292255402, 0.08053574711084366, 0.08108482509851456], "agreement_weight": -0.11429207026958466, "orthogonal_weight": 0.3824387788772583}]}, {"layer": 2, "memory_to_active": 0.9989942312240601, "contribution_to_active": 0.36774009466171265, "mix_mean": 0.4099982976913452, "mix_low": 0.24660174548625946, "mix_high": 0.004927953705191612, "read_cosine": 0.2927078604698181, "beta_mean": 0.07799209654331207, "nonfinite": 0.0, "weighted_memory_to_active": 0.07646745443344116, "mix_std": 0.18822205066680908, "beta_std": 0.06648708134889603, "beta_low": 0.5999044179916382, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.017343468964099884, "output_influence": 0.4038721024990082, "residual_influence": 0.1087210476398468, "state_active": 0.03956270590424538, "state_memory": 0.014133492484688759, "state_peak": 27.045286178588867, "growth_active": 1.0598722696304321, "growth_memory": 1.550599217414856, "heads": [{"head": 0, "memory_to_active": 0.008365768007934093, "contribution_to_active": 0.3760051131248474, "mix_mean": 0.6381658315658569, "mix_low": 0.0007028579711914062, "mix_high": 0.010152816772460938, "read_cosine": 0.14098656177520752, "beta_mean": 5.85577727179043e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005575297866016626, "mix_std": 0.2690202593803406, "beta_std": 0.00010070198914036155, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005881962366402149, "evidence_weights": [-0.37503954768180847, -0.21758459508419037, 0.18076179921627045], "agreement_weight": 0.12805496156215668, "orthogonal_weight": -1.0940616130828857}, {"head": 1, "memory_to_active": 3.0425257682800293, "contribution_to_active": 0.05324782431125641, "mix_mean": 0.005509491078555584, "mix_low": 0.8854217529296875, "mix_high": 9.5367431640625e-07, "read_cosine": 0.12039785087108612, "beta_mean": 0.22138957679271698, "nonfinite": 0.0, "weighted_memory_to_active": 0.042061273008584976, "mix_std": 0.013939894735813141, "beta_std": 0.15334975719451904, "beta_low": 0.0029296875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.012622395530343056, "evidence_weights": [0.13866718113422394, 0.11505832523107529, 0.5895056128501892], "agreement_weight": 0.03898077458143234, "orthogonal_weight": 0.3287300169467926}, {"head": 2, "memory_to_active": 0.01483946107327938, "contribution_to_active": 0.42931777238845825, "mix_mean": 0.5581455230712891, "mix_low": 0.0010385513305664062, "mix_high": 0.0036468505859375, "read_cosine": 0.2837910056114197, "beta_mean": 2.988510277646128e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0064783357083797455, "mix_std": 0.2746015787124634, "beta_std": 4.377834920887835e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 4.1197450627805665e-05, "evidence_weights": [-0.2553691565990448, -0.2961590886116028, 0.5725996494293213], "agreement_weight": -0.01860114373266697, "orthogonal_weight": -0.007814512588083744}, {"head": 3, "memory_to_active": 0.10262678563594818, "contribution_to_active": 0.5556080937385559, "mix_mean": 0.5849459171295166, "mix_low": 0.00020885467529296875, "mix_high": 0.0068607330322265625, "read_cosine": 0.6522021293640137, "beta_mean": 0.016699954867362976, "nonfinite": 0.0, "weighted_memory_to_active": 0.06388784199953079, "mix_std": 0.2543666958808899, "beta_std": 0.022363942116498947, "beta_low": 0.54107666015625, "beta_high": 0.0, "orthogonal_to_active": 0.04129472374916077, "evidence_weights": [-0.20372314751148224, -0.3253045082092285, 0.76588374376297], "agreement_weight": 0.05268924683332443, "orthogonal_weight": -0.888325035572052}, {"head": 4, "memory_to_active": 2.8144145011901855, "contribution_to_active": 0.2608363628387451, "mix_mean": 0.020625527948141098, "mix_low": 0.5913600921630859, "mix_high": 0.0, "read_cosine": 0.195660799741745, "beta_mean": 0.22973549365997314, "nonfinite": 0.0, "weighted_memory_to_active": 0.334375262260437, "mix_std": 0.05428756773471832, "beta_std": 0.22300221025943756, "beta_low": 0.055419921875, "beta_high": 0.00177001953125, "orthogonal_to_active": 0.042557086795568466, "evidence_weights": [0.023385418578982353, 0.003881314070895314, 0.019825978204607964], "agreement_weight": 0.016117433086037636, "orthogonal_weight": -0.1439702808856964}, {"head": 5, "memory_to_active": 0.011193517595529556, "contribution_to_active": 0.5314252972602844, "mix_mean": 0.6525973677635193, "mix_low": 0.0008783340454101562, "mix_high": 0.008906364440917969, "read_cosine": 0.36320871114730835, "beta_mean": 3.908158396370709e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006426716223359108, "mix_std": 0.2631162405014038, "beta_std": 6.204113742569461e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016634525964036584, "evidence_weights": [-0.28110557794570923, -0.29156550765037537, 0.9113858342170715], "agreement_weight": 0.25257253646850586, "orthogonal_weight": -0.31103068590164185}]}, {"layer": 3, "memory_to_active": 0.43781641125679016, "contribution_to_active": 0.48757320642471313, "mix_mean": 0.5166652202606201, "mix_low": 0.0513916052877903, "mix_high": 0.011494636535644531, "read_cosine": 0.19463874399662018, "beta_mean": 0.026421628892421722, "nonfinite": 0.0, "weighted_memory_to_active": 0.12649652361869812, "mix_std": 0.25958937406539917, "beta_std": 0.025124099105596542, "beta_low": 0.7701314687728882, "beta_high": 0.0, "orthogonal_to_active": 0.05234909802675247, "output_influence": 0.4253237247467041, "residual_influence": 0.10033658146858215, "state_active": 0.02570248395204544, "state_memory": 0.006896332837641239, "state_peak": 21.82978630065918, "growth_active": 1.1302738189697266, "growth_memory": 1.5629453659057617, "heads": [{"head": 0, "memory_to_active": 0.3688217103481293, "contribution_to_active": 0.6082841157913208, "mix_mean": 0.6069974899291992, "mix_low": 0.0016794204711914062, "mix_high": 0.026304244995117188, "read_cosine": 0.2344053089618683, "beta_mean": 0.009712792932987213, "nonfinite": 0.0, "weighted_memory_to_active": 0.23377937078475952, "mix_std": 0.3027115762233734, "beta_std": 0.012395907193422318, "beta_low": 0.71368408203125, "beta_high": 0.0, "orthogonal_to_active": 0.16953693330287933, "evidence_weights": [-0.217939093708992, -0.25169962644577026, 1.0459003448486328], "agreement_weight": 0.11276295781135559, "orthogonal_weight": -0.7674179673194885}, {"head": 1, "memory_to_active": 0.007604704704135656, "contribution_to_active": 0.4461861252784729, "mix_mean": 0.5786501169204712, "mix_low": 0.003978729248046875, "mix_high": 0.009795188903808594, "read_cosine": 0.18870294094085693, "beta_mean": 3.940573878935538e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004586480092257261, "mix_std": 0.29641470313072205, "beta_std": 0.0001182009800686501, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003795776516199112, "evidence_weights": [-0.2671716511249542, -0.21578769385814667, 0.6969631314277649], "agreement_weight": 0.550626814365387, "orthogonal_weight": -0.9048092365264893}, {"head": 2, "memory_to_active": 1.9938496351242065, "contribution_to_active": 0.2933230996131897, "mix_mean": 0.05729174241423607, "mix_low": 0.2925586700439453, "mix_high": 4.76837158203125e-06, "read_cosine": 0.21201191842556, "beta_mean": 0.14370067417621613, "nonfinite": 0.0, "weighted_memory_to_active": 0.3304576575756073, "mix_std": 0.10916568338871002, "beta_std": 0.12649986147880554, "beta_low": 0.01776123046875, "beta_high": 0.0, "orthogonal_to_active": 0.0015151796396821737, "evidence_weights": [-0.10123242437839508, -0.007526493165642023, -0.03506975248456001], "agreement_weight": -0.021473558619618416, "orthogonal_weight": -0.005720901768654585}, {"head": 3, "memory_to_active": 0.01738433726131916, "contribution_to_active": 0.5208883881568909, "mix_mean": 0.6082556247711182, "mix_low": 0.00467681884765625, "mix_high": 0.0106353759765625, "read_cosine": 0.2728261947631836, "beta_mean": 8.283673378173262e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011894814670085907, "mix_std": 0.2880343496799469, "beta_std": 0.00019300388521514833, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030193340498954058, "evidence_weights": [-0.2626059949398041, -0.22648370265960693, 2.2684268951416016], "agreement_weight": 2.1954751014709473, "orthogonal_weight": -0.3269626796245575}, {"head": 4, "memory_to_active": 0.014997901394963264, "contribution_to_active": 0.3980386257171631, "mix_mean": 0.5316658616065979, "mix_low": 0.0029134750366210938, "mix_high": 0.0035114288330078125, "read_cosine": 0.16916057467460632, "beta_mean": 7.424345676554367e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010406156070530415, "mix_std": 0.2904490530490875, "beta_std": 0.00010571967141004279, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00956583023071289, "evidence_weights": [-0.23864920437335968, -0.22421814501285553, 0.14160872995853424], "agreement_weight": -0.04417029768228531, "orthogonal_weight": -1.007289171218872}, {"head": 5, "memory_to_active": 0.22424009442329407, "contribution_to_active": 0.6587187051773071, "mix_mean": 0.7171303629875183, "mix_low": 0.0025424957275390625, "mix_high": 0.018716812133789062, "read_cosine": 0.09072545170783997, "beta_mean": 0.004919823259115219, "nonfinite": 0.0, "weighted_memory_to_active": 0.1678546965122223, "mix_std": 0.27076074481010437, "beta_std": 0.011431874707341194, "beta_low": 0.88934326171875, "beta_high": 0.0, "orthogonal_to_active": 0.1266615241765976, "evidence_weights": [-0.2593514621257782, -0.3385862112045288, 0.5488317012786865], "agreement_weight": 0.21746252477169037, "orthogonal_weight": -0.7821894884109497}]}, {"layer": 4, "memory_to_active": 0.9857833385467529, "contribution_to_active": 0.4668723940849304, "mix_mean": 0.3723219037055969, "mix_low": 0.24549691379070282, "mix_high": 0.012660503387451172, "read_cosine": 0.30412328243255615, "beta_mean": 0.1049053743481636, "nonfinite": 0.0, "weighted_memory_to_active": 0.12714049220085144, "mix_std": 0.16203972697257996, "beta_std": 0.09739798307418823, "beta_low": 0.4006551206111908, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.0512160062789917, "output_influence": 0.4913310408592224, "residual_influence": 0.1483975350856781, "state_active": 0.038922786712646484, "state_memory": 0.016461657360196114, "state_peak": 44.99081039428711, "growth_active": 1.2297905683517456, "growth_memory": 1.492644190788269, "heads": [{"head": 0, "memory_to_active": 0.16737422347068787, "contribution_to_active": 0.8018665313720703, "mix_mean": 0.6970151662826538, "mix_low": 0.010404586791992188, "mix_high": 0.03837394714355469, "read_cosine": 0.4510374665260315, "beta_mean": 0.23790203034877777, "nonfinite": 0.0, "weighted_memory_to_active": 0.14682549238204956, "mix_std": 0.346619576215744, "beta_std": 0.2379792034626007, "beta_low": 0.2548828125, "beta_high": 0.0, "orthogonal_to_active": 0.0862431600689888, "evidence_weights": [-0.2557566463947296, -0.36577972769737244, -0.13840511441230774], "agreement_weight": 0.0005097291432321072, "orthogonal_weight": 0.7427006363868713}, {"head": 1, "memory_to_active": 0.0445898100733757, "contribution_to_active": 0.6527982354164124, "mix_mean": 0.6962534189224243, "mix_low": 0.00010967254638671875, "mix_high": 0.01875591278076172, "read_cosine": 0.5687271356582642, "beta_mean": 0.0010141676757484674, "nonfinite": 0.0, "weighted_memory_to_active": 0.03264167159795761, "mix_std": 0.24265314638614655, "beta_std": 0.007580709643661976, "beta_low": 0.98040771484375, "beta_high": 0.0, "orthogonal_to_active": 0.017317064106464386, "evidence_weights": [-0.2397044450044632, -0.3653404712677002, 0.5970258712768555], "agreement_weight": 0.33497875928878784, "orthogonal_weight": -0.722868025302887}, {"head": 2, "memory_to_active": 1.5436331033706665, "contribution_to_active": 0.33388569951057434, "mix_mean": 0.04521762579679489, "mix_low": 0.3769683837890625, "mix_high": 2.002716064453125e-05, "read_cosine": 0.22259946167469025, "beta_mean": 0.05127592012286186, "nonfinite": 0.0, "weighted_memory_to_active": 0.25067421793937683, "mix_std": 0.08465058356523514, "beta_std": 0.05438459292054176, "beta_low": 0.13616943359375, "beta_high": 0.0, "orthogonal_to_active": 0.0857071653008461, "evidence_weights": [0.03653862699866295, -0.04242707043886185, 0.279318630695343], "agreement_weight": 0.13146936893463135, "orthogonal_weight": -0.44501593708992004}, {"head": 3, "memory_to_active": 1.7391551733016968, "contribution_to_active": 0.14205996692180634, "mix_mean": 0.03262421116232872, "mix_low": 0.41651344299316406, "mix_high": 0.0, "read_cosine": 0.23385874927043915, "beta_mean": 0.1247672438621521, "nonfinite": 0.0, "weighted_memory_to_active": 0.191318079829216, "mix_std": 0.05708658695220947, "beta_std": 0.1180415004491806, "beta_low": 0.032470703125, "beta_high": 0.0, "orthogonal_to_active": 0.0008276381995528936, "evidence_weights": [-0.018209150061011314, 0.05139986053109169, -0.2517911195755005], "agreement_weight": 0.07426689565181732, "orthogonal_weight": -0.006196832749992609}, {"head": 4, "memory_to_active": 0.0062912823632359505, "contribution_to_active": 0.6255250573158264, "mix_mean": 0.7496364116668701, "mix_low": 9.1552734375e-05, "mix_high": 0.018808364868164062, "read_cosine": 0.13805846869945526, "beta_mean": 1.5563231499982066e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004507085308432579, "mix_std": 0.2110293209552765, "beta_std": 3.9159396692411974e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002870267955586314, "evidence_weights": [-0.33342161774635315, -0.3217131197452545, 0.31570059061050415], "agreement_weight": 0.31367024779319763, "orthogonal_weight": -0.6719228625297546}, {"head": 5, "memory_to_active": 2.413656234741211, "contribution_to_active": 0.24509887397289276, "mix_mean": 0.013184481300413609, "mix_low": 0.6688938140869141, "mix_high": 4.76837158203125e-06, "read_cosine": 0.2104583978652954, "beta_mean": 0.21445730328559875, "nonfinite": 0.0, "weighted_memory_to_active": 0.13687638938426971, "mix_std": 0.030199065804481506, "beta_std": 0.1663627028465271, "beta_low": 0.0, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.11433075368404388, "evidence_weights": [0.05639451742172241, -0.017946457490324974, -0.27716705203056335], "agreement_weight": -0.0852503553032875, "orthogonal_weight": 1.0581029653549194}]}, {"layer": 5, "memory_to_active": 0.36793458461761475, "contribution_to_active": 0.5396625399589539, "mix_mean": 0.5135509967803955, "mix_low": 0.05657609552145004, "mix_high": 0.018363794311881065, "read_cosine": 0.08125001937150955, "beta_mean": 0.0492083765566349, "nonfinite": 0.0, "weighted_memory_to_active": 0.1348932385444641, "mix_std": 0.2541315257549286, "beta_std": 0.06525682657957077, "beta_low": 0.6817830801010132, "beta_high": 0.0, "orthogonal_to_active": 0.055823784321546555, "output_influence": 0.5717154741287231, "residual_influence": 0.1771363466978073, "state_active": 0.0421447679400444, "state_memory": 0.010130899026989937, "state_peak": 63.12229919433594, "growth_active": 1.1279391050338745, "growth_memory": 1.2169280052185059, "heads": [{"head": 0, "memory_to_active": 0.21050600707530975, "contribution_to_active": 0.5727576613426208, "mix_mean": 0.47162675857543945, "mix_low": 0.009761810302734375, "mix_high": 0.0017604827880859375, "read_cosine": -0.127640038728714, "beta_mean": 0.01660924404859543, "nonfinite": 0.0, "weighted_memory_to_active": 0.08175477385520935, "mix_std": 0.2777615189552307, "beta_std": 0.03003520518541336, "beta_low": 0.58001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.03640212491154671, "evidence_weights": [-0.1657751202583313, -0.18429052829742432, 0.08661895245313644], "agreement_weight": -0.01899680122733116, "orthogonal_weight": -0.46601057052612305}, {"head": 1, "memory_to_active": 0.005461106076836586, "contribution_to_active": 0.5585829615592957, "mix_mean": 0.7065189480781555, "mix_low": 0.001224517822265625, "mix_high": 0.03727245330810547, "read_cosine": 0.24544860422611237, "beta_mean": 1.1727495802915655e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0033923769369721413, "mix_std": 0.26820316910743713, "beta_std": 2.5487979655736126e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030727244447916746, "evidence_weights": [-0.2502008378505707, -0.2898657023906708, 0.9591440558433533], "agreement_weight": 0.6892784237861633, "orthogonal_weight": -0.9562008380889893}, {"head": 2, "memory_to_active": 0.016141068190336227, "contribution_to_active": 0.593601405620575, "mix_mean": 0.6528970003128052, "mix_low": 0.0027666091918945312, "mix_high": 0.020452499389648438, "read_cosine": -0.5075653791427612, "beta_mean": 8.255886496044695e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009897641837596893, "mix_std": 0.2664613127708435, "beta_std": 0.0003699005173984915, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.008521921932697296, "evidence_weights": [-0.2595822513103485, -0.3219643235206604, 0.14980705082416534], "agreement_weight": -0.14708685874938965, "orthogonal_weight": -0.9985273480415344}, {"head": 3, "memory_to_active": 1.4705595970153809, "contribution_to_active": 0.24169254302978516, "mix_mean": 0.11443498730659485, "mix_low": 0.12798023223876953, "mix_high": 0.0, "read_cosine": 0.15528687834739685, "beta_mean": 0.09167852997779846, "nonfinite": 0.0, "weighted_memory_to_active": 0.38450443744659424, "mix_std": 0.1470569670200348, "beta_std": 0.14472679793834686, "beta_low": 0.2125244140625, "beta_high": 0.0, "orthogonal_to_active": 0.15095780789852142, "evidence_weights": [-0.07585945725440979, -0.09993761777877808, 0.059375107288360596], "agreement_weight": 0.05013376474380493, "orthogonal_weight": -0.49003249406814575}, {"head": 4, "memory_to_active": 0.42994603514671326, "contribution_to_active": 0.5742976069450378, "mix_mean": 0.32866525650024414, "mix_low": 0.1959829330444336, "mix_high": 0.0054683685302734375, "read_cosine": 0.5509787201881409, "beta_mean": 0.18667982518672943, "nonfinite": 0.0, "weighted_memory_to_active": 0.2786925435066223, "mix_std": 0.34246712923049927, "beta_std": 0.21546831727027893, "beta_low": 0.29949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.10603146255016327, "evidence_weights": [-0.11935073137283325, -0.25766170024871826, -0.05128270760178566], "agreement_weight": 0.030941028147935867, "orthogonal_weight": 0.5885597467422485}, {"head": 5, "memory_to_active": 0.07499364018440247, "contribution_to_active": 0.6970430016517639, "mix_mean": 0.8071625828742981, "mix_low": 0.0017404556274414062, "mix_high": 0.04522895812988281, "read_cosine": 0.17099134624004364, "beta_mean": 0.00018836540402844548, "nonfinite": 0.0, "weighted_memory_to_active": 0.05111760273575783, "mix_std": 0.22283893823623657, "beta_std": 0.0009152382845059037, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.029956649988889694, "evidence_weights": [-0.30748414993286133, -0.33134138584136963, 1.8721510171890259], "agreement_weight": 0.5082500576972961, "orthogonal_weight": -0.619003176689148}]}, {"layer": 6, "memory_to_active": 0.2713005840778351, "contribution_to_active": 0.633126437664032, "mix_mean": 0.5910155773162842, "mix_low": 0.03988329693675041, "mix_high": 0.03222418203949928, "read_cosine": 0.31918174028396606, "beta_mean": 0.06016739830374718, "nonfinite": 0.0, "weighted_memory_to_active": 0.14590232074260712, "mix_std": 0.23817408084869385, "beta_std": 0.07251568138599396, "beta_low": 0.6011455059051514, "beta_high": 0.0, "orthogonal_to_active": 0.06575414538383484, "output_influence": 0.5600765943527222, "residual_influence": 0.19814778864383698, "state_active": 0.04331423342227936, "state_memory": 0.0065629687160253525, "state_peak": 95.02825927734375, "growth_active": 1.1559640169143677, "growth_memory": 1.1227582693099976, "heads": [{"head": 0, "memory_to_active": 0.14173942804336548, "contribution_to_active": 0.7524548768997192, "mix_mean": 0.8231300711631775, "mix_low": 0.00012493133544921875, "mix_high": 0.060196876525878906, "read_cosine": 0.3202376961708069, "beta_mean": 0.014105390757322311, "nonfinite": 0.0, "weighted_memory_to_active": 0.12571391463279724, "mix_std": 0.19801568984985352, "beta_std": 0.045098721981048584, "beta_low": 0.80731201171875, "beta_high": 0.0, "orthogonal_to_active": 0.08542224764823914, "evidence_weights": [-0.3076235055923462, -0.35107433795928955, 1.221709966659546], "agreement_weight": 0.772172749042511, "orthogonal_weight": -0.7429415583610535}, {"head": 1, "memory_to_active": 0.005312054883688688, "contribution_to_active": 0.5216153264045715, "mix_mean": 0.5846863985061646, "mix_low": 0.0009603500366210938, "mix_high": 0.014046669006347656, "read_cosine": 0.45070239901542664, "beta_mean": 1.1194935723324306e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002382804173976183, "mix_std": 0.2855977416038513, "beta_std": 1.733881981635932e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013232412748038769, "evidence_weights": [-0.1927168220281601, -0.30081707239151, 1.1006205081939697], "agreement_weight": 0.12550196051597595, "orthogonal_weight": -0.7104703187942505}, {"head": 2, "memory_to_active": 0.697358250617981, "contribution_to_active": 0.09839162230491638, "mix_mean": 0.05115421116352081, "mix_low": 0.22964954376220703, "mix_high": 0.0, "read_cosine": 0.2758241295814514, "beta_mean": 0.021210920065641403, "nonfinite": 0.0, "weighted_memory_to_active": 0.06402310729026794, "mix_std": 0.07607242465019226, "beta_std": 0.02261575311422348, "beta_low": 0.2957763671875, "beta_high": 0.0, "orthogonal_to_active": 0.00908068660646677, "evidence_weights": [0.06515269726514816, -0.059305522590875626, 0.3486756384372711], "agreement_weight": 0.141112819314003, "orthogonal_weight": -0.1738366335630417}, {"head": 3, "memory_to_active": 0.12974682450294495, "contribution_to_active": 0.6718167066574097, "mix_mean": 0.6572739481925964, "mix_low": 0.00762176513671875, "mix_high": 0.021271705627441406, "read_cosine": 0.4796069264411926, "beta_mean": 0.19699345529079437, "nonfinite": 0.0, "weighted_memory_to_active": 0.09233679622411728, "mix_std": 0.325346976518631, "beta_std": 0.22788812220096588, "beta_low": 0.211181640625, "beta_high": 0.0, "orthogonal_to_active": 0.05280575901269913, "evidence_weights": [-0.16392618417739868, -0.31490427255630493, 0.14868929982185364], "agreement_weight": 0.0012427062029018998, "orthogonal_weight": 0.6607030630111694}, {"head": 4, "memory_to_active": 0.6222031712532043, "contribution_to_active": 1.1235675811767578, "mix_mean": 0.7453790307044983, "mix_low": 0.0007963180541992188, "mix_high": 0.08917236328125, "read_cosine": 0.1182553768157959, "beta_mean": 0.12855181097984314, "nonfinite": 0.0, "weighted_memory_to_active": 0.5686866044998169, "mix_std": 0.30045896768569946, "beta_std": 0.13914290070533752, "beta_low": 0.29278564453125, "beta_high": 0.0, "orthogonal_to_active": 0.22428059577941895, "evidence_weights": [-0.18863119184970856, -0.5410637259483337, -0.5767785310745239], "agreement_weight": -0.016016678884625435, "orthogonal_weight": 0.41983574628829956}, {"head": 5, "memory_to_active": 0.031443655490875244, "contribution_to_active": 0.630912184715271, "mix_mean": 0.6844695806503296, "mix_low": 0.0001468658447265625, "mix_high": 0.008657455444335938, "read_cosine": 0.270463764667511, "beta_mean": 0.00013158834190107882, "nonfinite": 0.0, "weighted_memory_to_active": 0.022270623594522476, "mix_std": 0.24355269968509674, "beta_std": 0.00033121954766102135, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.021612349897623062, "evidence_weights": [-0.23731552064418793, -0.31852227449417114, 0.37509217858314514], "agreement_weight": 0.21470221877098083, "orthogonal_weight": -1.0358684062957764}]}, {"layer": 7, "memory_to_active": 1.8074889183044434, "contribution_to_active": 0.2726809084415436, "mix_mean": 0.17343194782733917, "mix_low": 0.2294316291809082, "mix_high": 0.0011412303429096937, "read_cosine": 0.24009646475315094, "beta_mean": 0.13108661770820618, "nonfinite": 0.0, "weighted_memory_to_active": 0.21033479273319244, "mix_std": 0.14886730909347534, "beta_std": 0.09772118926048279, "beta_low": 0.223968505859375, "beta_high": 0.00019327799964230508, "orthogonal_to_active": 0.09323066473007202, "output_influence": 0.2903943657875061, "residual_influence": 0.07083010673522949, "state_active": 0.012996664270758629, "state_memory": 0.024311475455760956, "state_peak": 30.286487579345703, "growth_active": 1.1166985034942627, "growth_memory": 1.2590186595916748, "heads": [{"head": 0, "memory_to_active": 2.9534831047058105, "contribution_to_active": 0.18192525207996368, "mix_mean": 0.03764573857188225, "mix_low": 0.4172935485839844, "mix_high": 9.5367431640625e-07, "read_cosine": 0.343460351228714, "beta_mean": 0.16091251373291016, "nonfinite": 0.0, "weighted_memory_to_active": 0.20899274945259094, "mix_std": 0.07576071470975876, "beta_std": 0.12907063961029053, "beta_low": 0.007080078125, "beta_high": 0.000244140625, "orthogonal_to_active": 0.00037828937638550997, "evidence_weights": [0.04281623288989067, -0.08626965433359146, 0.023403814062476158], "agreement_weight": -0.031076418235898018, "orthogonal_weight": -0.0020734162535518408}, {"head": 1, "memory_to_active": 0.9700900316238403, "contribution_to_active": 0.3984781503677368, "mix_mean": 0.2231276035308838, "mix_low": 0.1236124038696289, "mix_high": 0.006018638610839844, "read_cosine": 0.40205878019332886, "beta_mean": 0.09408281743526459, "nonfinite": 0.0, "weighted_memory_to_active": 0.33655276894569397, "mix_std": 0.24794486165046692, "beta_std": 0.06694308668375015, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.04689516872167587, "evidence_weights": [-0.21627162396907806, -0.10308418422937393, -0.29252156615257263], "agreement_weight": 0.0527491569519043, "orthogonal_weight": 0.16766417026519775}, {"head": 2, "memory_to_active": 0.5868532061576843, "contribution_to_active": 0.3946753144264221, "mix_mean": 0.22837235033512115, "mix_low": 0.0204315185546875, "mix_high": 0.0005474090576171875, "read_cosine": 0.11139575392007828, "beta_mean": 0.03375145047903061, "nonfinite": 0.0, "weighted_memory_to_active": 0.21721696853637695, "mix_std": 0.21420404314994812, "beta_std": 0.03676866367459297, "beta_low": 0.198486328125, "beta_high": 0.0, "orthogonal_to_active": 0.12566447257995605, "evidence_weights": [-0.09417585283517838, -0.18079021573066711, 0.8437299728393555], "agreement_weight": 0.020170897245407104, "orthogonal_weight": -0.5972099304199219}, {"head": 3, "memory_to_active": 4.791052341461182, "contribution_to_active": 0.10640686750411987, "mix_mean": 0.036377400159835815, "mix_low": 0.3100461959838867, "mix_high": 0.0, "read_cosine": 0.10992459207773209, "beta_mean": 0.43311893939971924, "nonfinite": 0.0, "weighted_memory_to_active": 0.40034031867980957, "mix_std": 0.05342883989214897, "beta_std": 0.278838574886322, "beta_low": 0.002197265625, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.3485783636569977, "evidence_weights": [0.07091271132230759, -0.04115445911884308, 0.7228415608406067], "agreement_weight": 0.05386234074831009, "orthogonal_weight": -0.9418240189552307}, {"head": 4, "memory_to_active": 1.5000696182250977, "contribution_to_active": 0.10108303278684616, "mix_mean": 0.024569451808929443, "mix_low": 0.503117561340332, "mix_high": 0.0, "read_cosine": 0.23404788970947266, "beta_mean": 0.06447497010231018, "nonfinite": 0.0, "weighted_memory_to_active": 0.07673893868923187, "mix_std": 0.04841027036309242, "beta_std": 0.07427158206701279, "beta_low": 0.1361083984375, "beta_high": 0.0, "orthogonal_to_active": 0.01634717360138893, "evidence_weights": [0.12522545456886292, -0.02413122169673443, 0.18163971602916718], "agreement_weight": 0.0400126613676548, "orthogonal_weight": -0.29023024439811707}, {"head": 5, "memory_to_active": 0.04338523745536804, "contribution_to_active": 0.4535166323184967, "mix_mean": 0.4904990792274475, "mix_low": 0.0020885467529296875, "mix_high": 0.0002803802490234375, "read_cosine": 0.239691361784935, "beta_mean": 0.00017902276886161417, "nonfinite": 0.0, "weighted_memory_to_active": 0.02216697484254837, "mix_std": 0.25345516204833984, "beta_std": 0.00043456791900098324, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.021520493552088737, "evidence_weights": [-0.18058569729328156, -0.23634590208530426, 0.5026699304580688], "agreement_weight": 0.21999531984329224, "orthogonal_weight": -1.024867057800293}]}, {"layer": 8, "memory_to_active": 0.14829757809638977, "contribution_to_active": 0.5582727789878845, "mix_mean": 0.49054503440856934, "mix_low": 0.010845662094652653, "mix_high": 0.002025922294706106, "read_cosine": 0.40623828768730164, "beta_mean": 0.012711192481219769, "nonfinite": 0.0, "weighted_memory_to_active": 0.07263537496328354, "mix_std": 0.24035042524337769, "beta_std": 0.019209595397114754, "beta_low": 0.7833049297332764, "beta_high": 0.0, "orthogonal_to_active": 0.06542949378490448, "output_influence": 0.5109837651252747, "residual_influence": 0.18484145402908325, "state_active": 0.04472314193844795, "state_memory": 0.0054034460335969925, "state_peak": 62.70407485961914, "growth_active": 1.090242624282837, "growth_memory": 1.0856047868728638, "heads": [{"head": 0, "memory_to_active": 0.056049857288599014, "contribution_to_active": 0.5706989765167236, "mix_mean": 0.6021044254302979, "mix_low": 0.0005598068237304688, "mix_high": 0.0014657974243164062, "read_cosine": 0.447574257850647, "beta_mean": 0.0032010760623961687, "nonfinite": 0.0, "weighted_memory_to_active": 0.0394555926322937, "mix_std": 0.24819043278694153, "beta_std": 0.013030670583248138, "beta_low": 0.93292236328125, "beta_high": 0.0, "orthogonal_to_active": 0.02489619143307209, "evidence_weights": [-0.16146720945835114, -0.38238653540611267, 0.5843674540519714], "agreement_weight": 0.2621617615222931, "orthogonal_weight": -0.792569100856781}, {"head": 1, "memory_to_active": 0.08059459179639816, "contribution_to_active": 0.5542237758636475, "mix_mean": 0.5254153609275818, "mix_low": 6.67572021484375e-05, "mix_high": 0.001705169677734375, "read_cosine": 0.15258999168872833, "beta_mean": 0.001171146985143423, "nonfinite": 0.0, "weighted_memory_to_active": 0.036015480756759644, "mix_std": 0.2351067066192627, "beta_std": 0.006998703349381685, "beta_low": 0.98699951171875, "beta_high": 0.0, "orthogonal_to_active": 0.02578859031200409, "evidence_weights": [-0.19529618322849274, -0.2764798402786255, 0.668960452079773], "agreement_weight": 0.0829094871878624, "orthogonal_weight": -0.7661579847335815}, {"head": 2, "memory_to_active": 0.013281228020787239, "contribution_to_active": 0.6894571185112, "mix_mean": 0.6798009872436523, "mix_low": 0.00046443939208984375, "mix_high": 0.0037164688110351562, "read_cosine": 0.3981073200702667, "beta_mean": 0.0001342129398835823, "nonfinite": 0.0, "weighted_memory_to_active": 0.010469573549926281, "mix_std": 0.22250518202781677, "beta_std": 0.0034389279317110777, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.006244315765798092, "evidence_weights": [-0.22435975074768066, -0.3471362292766571, 1.010969877243042], "agreement_weight": 0.2880750596523285, "orthogonal_weight": -0.6856526136398315}, {"head": 3, "memory_to_active": 0.1033565104007721, "contribution_to_active": 0.6305056810379028, "mix_mean": 0.5751973986625671, "mix_low": 0.00070953369140625, "mix_high": 0.005054473876953125, "read_cosine": 0.7988889813423157, "beta_mean": 0.00808645412325859, "nonfinite": 0.0, "weighted_memory_to_active": 0.07744132727384567, "mix_std": 0.2558189332485199, "beta_std": 0.008077731356024742, "beta_low": 0.71881103515625, "beta_high": 0.0, "orthogonal_to_active": 0.023661309853196144, "evidence_weights": [-0.17633859813213348, -0.2975280284881592, 0.5391668081283569], "agreement_weight": 0.24795641005039215, "orthogonal_weight": -0.6009897589683533}, {"head": 4, "memory_to_active": 0.27674704790115356, "contribution_to_active": 0.2997773289680481, "mix_mean": 0.30491387844085693, "mix_low": 0.0030803680419921875, "mix_high": 1.239776611328125e-05, "read_cosine": 0.15592335164546967, "beta_mean": 0.0012793727219104767, "nonfinite": 0.0, "weighted_memory_to_active": 0.0948949083685875, "mix_std": 0.2228129506111145, "beta_std": 0.01356064435094595, "beta_low": 0.9847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.021113360300660133, "evidence_weights": [-0.12365373969078064, -0.19078543782234192, 0.6284152269363403], "agreement_weight": 0.05719651281833649, "orthogonal_weight": -0.23134836554527283}, {"head": 5, "memory_to_active": 0.3597562909126282, "contribution_to_active": 0.6049736738204956, "mix_mean": 0.25583791732788086, "mix_low": 0.06019306182861328, "mix_high": 0.00020122528076171875, "read_cosine": 0.4843456447124481, "beta_mean": 0.062394894659519196, "nonfinite": 0.0, "weighted_memory_to_active": 0.17753535509109497, "mix_std": 0.2576683461666107, "beta_std": 0.07015088945627213, "beta_low": 0.07647705078125, "beta_high": 0.0, "orthogonal_to_active": 0.2908731997013092, "evidence_weights": [-0.14527054131031036, -0.1469835340976715, -0.10498856753110886], "agreement_weight": -0.0176116731017828, "orthogonal_weight": 1.9198315143585205}]}, {"layer": 9, "memory_to_active": 0.9480717182159424, "contribution_to_active": 0.7274137139320374, "mix_mean": 0.38321298360824585, "mix_low": 0.06624715030193329, "mix_high": 0.0067180003970861435, "read_cosine": 0.3454519212245941, "beta_mean": 0.039194002747535706, "nonfinite": 0.0, "weighted_memory_to_active": 0.41347748041152954, "mix_std": 0.24153479933738708, "beta_std": 0.03427484631538391, "beta_low": 0.5968221426010132, "beta_high": 0.0, "orthogonal_to_active": 0.03461997210979462, "output_influence": 0.54883873462677, "residual_influence": 0.1618981510400772, "state_active": 0.04272140935063362, "state_memory": 0.01552653033286333, "state_peak": 56.60416793823242, "growth_active": 1.170045018196106, "growth_memory": 1.288791537284851, "heads": [{"head": 0, "memory_to_active": 0.00029695683042518795, "contribution_to_active": 0.5825271606445312, "mix_mean": 0.6540541648864746, "mix_low": 0.0013341903686523438, "mix_high": 0.027807235717773438, "read_cosine": 0.3132890462875366, "beta_mean": 8.043352863751352e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.00019304442685097456, "mix_std": 0.28580722212791443, "beta_std": 2.8592344278877135e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00020995283557567745, "evidence_weights": [-0.3061198890209198, -0.314337819814682, 0.4321777820587158], "agreement_weight": 0.10253351181745529, "orthogonal_weight": -1.145639181137085}, {"head": 1, "memory_to_active": 0.01852194033563137, "contribution_to_active": 0.6916608214378357, "mix_mean": 0.5733624696731567, "mix_low": 0.007007598876953125, "mix_high": 0.0029964447021484375, "read_cosine": 0.4338011145591736, "beta_mean": 4.828711098525673e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013505985029041767, "mix_std": 0.27978670597076416, "beta_std": 0.00011603573511820287, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009064183570444584, "evidence_weights": [-0.23795613646507263, -0.2938041090965271, 0.9920030832290649], "agreement_weight": 0.5100966691970825, "orthogonal_weight": -0.8276893496513367}, {"head": 2, "memory_to_active": 0.17826133966445923, "contribution_to_active": 0.658582866191864, "mix_mean": 0.6025685667991638, "mix_low": 0.00213623046875, "mix_high": 0.007740974426269531, "read_cosine": 0.35514605045318604, "beta_mean": 0.0007099075010046363, "nonfinite": 0.0, "weighted_memory_to_active": 0.1229727640748024, "mix_std": 0.2667076587677002, "beta_std": 0.00457469467073679, "beta_low": 0.99560546875, "beta_high": 0.0, "orthogonal_to_active": 0.07147537171840668, "evidence_weights": [-0.2037225365638733, -0.2781077027320862, 0.7654424905776978], "agreement_weight": 0.3419952094554901, "orthogonal_weight": -0.679715633392334}, {"head": 3, "memory_to_active": 0.8418476581573486, "contribution_to_active": 0.23560145497322083, "mix_mean": 0.1454477459192276, "mix_low": 0.11539745330810547, "mix_high": 0.0002193450927734375, "read_cosine": 0.353900671005249, "beta_mean": 0.03379161283373833, "nonfinite": 0.0, "weighted_memory_to_active": 0.19160452485084534, "mix_std": 0.19453436136245728, "beta_std": 0.035050682723522186, "beta_low": 0.1751708984375, "beta_high": 0.0, "orthogonal_to_active": 0.0752054825425148, "evidence_weights": [-0.014775619842112064, -0.08692502230405807, 0.35372698307037354], "agreement_weight": 0.09845664352178574, "orthogonal_weight": -0.4362267851829529}, {"head": 4, "memory_to_active": 3.3942270278930664, "contribution_to_active": 1.8059544563293457, "mix_mean": 0.14684203267097473, "mix_low": 0.13136959075927734, "mix_high": 0.0010204315185546875, "read_cosine": 0.1288352906703949, "beta_mean": 0.15723294019699097, "nonfinite": 0.0, "weighted_memory_to_active": 1.7653920650482178, "mix_std": 0.2074337899684906, "beta_std": 0.09248565137386322, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.007617373950779438, "evidence_weights": [-0.07952385395765305, -0.15088056027889252, -0.03357114642858505], "agreement_weight": -0.06355304270982742, "orthogonal_weight": -0.004538268782198429}, {"head": 5, "memory_to_active": 1.2552754878997803, "contribution_to_active": 0.3901553750038147, "mix_mean": 0.17700281739234924, "mix_low": 0.14023780822753906, "mix_high": 0.0005235671997070312, "read_cosine": 0.4877393841743469, "beta_mean": 0.04338045418262482, "nonfinite": 0.0, "weighted_memory_to_active": 0.38719621300697327, "mix_std": 0.21493904292583466, "beta_std": 0.07341916859149933, "beta_low": 0.41015625, "beta_high": 0.0, "orthogonal_to_active": 0.04414748400449753, "evidence_weights": [0.06639320403337479, -0.16684357821941376, 0.14352305233478546], "agreement_weight": 0.008280586451292038, "orthogonal_weight": 0.14423052966594696}]}, {"layer": 10, "memory_to_active": 0.2265343815088272, "contribution_to_active": 0.7717103958129883, "mix_mean": 0.5223259329795837, "mix_low": 0.0231399554759264, "mix_high": 0.0056200032122433186, "read_cosine": 0.3835541605949402, "beta_mean": 0.045173279941082, "nonfinite": 0.0, "weighted_memory_to_active": 0.10375042259693146, "mix_std": 0.2683817744255066, "beta_std": 0.04595541954040527, "beta_low": 0.6620382070541382, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.3014899492263794, "output_influence": 0.4662367105484009, "residual_influence": 0.16637194156646729, "state_active": 0.05360256880521774, "state_memory": 0.015439728274941444, "state_peak": 119.71126556396484, "growth_active": 1.0395995378494263, "growth_memory": 0.993739902973175, "heads": [{"head": 0, "memory_to_active": 0.849174976348877, "contribution_to_active": 2.316666841506958, "mix_mean": 0.5727605223655701, "mix_low": 0.10981941223144531, "mix_high": 0.010457038879394531, "read_cosine": 0.09572400152683258, "beta_mean": 0.20365923643112183, "nonfinite": 0.0, "weighted_memory_to_active": 0.500027596950531, "mix_std": 0.3955049514770508, "beta_std": 0.2091662883758545, "beta_low": 0.14453125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.718919038772583, "evidence_weights": [-0.08564609289169312, -0.37301287055015564, -1.8822189569473267], "agreement_weight": 0.015631072223186493, "orthogonal_weight": 3.4652438163757324}, {"head": 1, "memory_to_active": 0.009318577125668526, "contribution_to_active": 0.6196238994598389, "mix_mean": 0.6320098638534546, "mix_low": 0.0017948150634765625, "mix_high": 0.016061782836914062, "read_cosine": 0.321653813123703, "beta_mean": 6.55544936307706e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005287980195134878, "mix_std": 0.26412829756736755, "beta_std": 0.0001074013562174514, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008345218375325203, "evidence_weights": [-0.2513335347175598, -0.19563353061676025, 0.571392834186554], "agreement_weight": 0.6126835346221924, "orthogonal_weight": -1.7143796682357788}, {"head": 2, "memory_to_active": 0.007399730384349823, "contribution_to_active": 0.6159690618515015, "mix_mean": 0.660117506980896, "mix_low": 0.00018310546875, "mix_high": 0.005375862121582031, "read_cosine": 0.059536054730415344, "beta_mean": 1.855620212154463e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003213156946003437, "mix_std": 0.25102144479751587, "beta_std": 6.401854625437409e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013850845862179995, "evidence_weights": [-0.26740750670433044, -0.27929216623306274, 0.6216415166854858], "agreement_weight": -0.2049676775932312, "orthogonal_weight": -0.4119908809661865}, {"head": 3, "memory_to_active": 0.3407820463180542, "contribution_to_active": 0.09691434353590012, "mix_mean": 0.2763915956020355, "mix_low": 0.024517059326171875, "mix_high": 0.0002384185791015625, "read_cosine": 0.9100147485733032, "beta_mean": 0.06073754280805588, "nonfinite": 0.0, "weighted_memory_to_active": 0.037246741354465485, "mix_std": 0.22308053076267242, "beta_std": 0.05819922685623169, "beta_low": 0.0035400390625, "beta_high": 0.0, "orthogonal_to_active": 0.040239810943603516, "evidence_weights": [-0.09748274832963943, -0.14581477642059326, 0.16777439415454865], "agreement_weight": 0.01908516138792038, "orthogonal_weight": 1.4294334650039673}, {"head": 4, "memory_to_active": 0.02156681753695011, "contribution_to_active": 0.5639874935150146, "mix_mean": 0.5898078680038452, "mix_low": 0.000316619873046875, "mix_high": 0.0015783309936523438, "read_cosine": 0.0749814584851265, "beta_mean": 5.889840394956991e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012740290723741055, "mix_std": 0.2494673728942871, "beta_std": 0.00012575309665407985, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.018158160150051117, "evidence_weights": [-0.18525363504886627, -0.2562199831008911, 0.9717808365821838], "agreement_weight": 0.17291197180747986, "orthogonal_weight": -1.5028278827667236}, {"head": 5, "memory_to_active": 0.13096407055854797, "contribution_to_active": 0.41710031032562256, "mix_mean": 0.40286824107170105, "mix_low": 0.002208709716796875, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8394147157669067, "beta_mean": 0.006499891635030508, "nonfinite": 0.0, "weighted_memory_to_active": 0.06398674100637436, "mix_std": 0.2270878702402115, "beta_std": 0.00806983932852745, "beta_low": 0.82415771484375, "beta_high": 0.0, "orthogonal_to_active": 0.021892301738262177, "evidence_weights": [-0.13986286520957947, -0.1640894114971161, 0.2675297260284424], "agreement_weight": 0.0930628553032875, "orthogonal_weight": -0.6645716428756714}]}, {"layer": 11, "memory_to_active": 0.0797514021396637, "contribution_to_active": 0.6272379159927368, "mix_mean": 0.5205540657043457, "mix_low": 0.12123807519674301, "mix_high": 0.03860950469970703, "read_cosine": 0.3369366228580475, "beta_mean": 0.023078976199030876, "nonfinite": 0.0, "weighted_memory_to_active": 0.04300741106271744, "mix_std": 0.32441210746765137, "beta_std": 0.03467543050646782, "beta_low": 0.7073262929916382, "beta_high": 0.0, "orthogonal_to_active": 0.047244321554899216, "output_influence": 0.5498275756835938, "residual_influence": 0.1520071029663086, "state_active": 0.18347647786140442, "state_memory": 0.003357088193297386, "state_peak": 35.13630294799805, "growth_active": 1.0974310636520386, "growth_memory": 1.5356979370117188, "heads": [{"head": 0, "memory_to_active": 0.0047097583301365376, "contribution_to_active": 0.6133033037185669, "mix_mean": 0.6595532298088074, "mix_low": 0.0008287429809570312, "mix_high": 0.008709907531738281, "read_cosine": 0.28889036178588867, "beta_mean": 6.43314269836992e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035137361846864223, "mix_std": 0.25521543622016907, "beta_std": 0.002698064548894763, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.0028375638648867607, "evidence_weights": [-0.1763300597667694, -0.36297616362571716, 0.7957399487495422], "agreement_weight": 0.377684086561203, "orthogonal_weight": -0.8483635187149048}, {"head": 1, "memory_to_active": 0.13397753238677979, "contribution_to_active": 0.4339195191860199, "mix_mean": 0.46665632724761963, "mix_low": 0.005597114562988281, "mix_high": 0.0007047653198242188, "read_cosine": 0.46093326807022095, "beta_mean": 0.0037967716343700886, "nonfinite": 0.0, "weighted_memory_to_active": 0.06635414808988571, "mix_std": 0.27070343494415283, "beta_std": 0.012232429347932339, "beta_low": 0.9263916015625, "beta_high": 0.0, "orthogonal_to_active": 0.03651201352477074, "evidence_weights": [-0.14858722686767578, -0.26951324939727783, 0.592005729675293], "agreement_weight": 0.07890405505895615, "orthogonal_weight": -0.6457977890968323}, {"head": 2, "memory_to_active": 0.007188941817730665, "contribution_to_active": 0.9815301895141602, "mix_mean": 0.41682109236717224, "mix_low": 0.3487405776977539, "mix_high": 0.13790607452392578, "read_cosine": 0.1698201298713684, "beta_mean": 0.060884613543748856, "nonfinite": 0.0, "weighted_memory_to_active": 0.004944273270666599, "mix_std": 0.44965705275535583, "beta_std": 0.08468256890773773, "beta_low": 0.49554443359375, "beta_high": 0.0, "orthogonal_to_active": 0.04866892471909523, "evidence_weights": [0.04882796108722687, -0.3527868092060089, -1.3246647119522095], "agreement_weight": 0.0033727651461958885, "orthogonal_weight": 10.159587860107422}, {"head": 3, "memory_to_active": 0.007838506251573563, "contribution_to_active": 0.8578375577926636, "mix_mean": 0.39939743280410767, "mix_low": 0.3640575408935547, "mix_high": 0.07497692108154297, "read_cosine": 0.5024632215499878, "beta_mean": 0.04905492812395096, "nonfinite": 0.0, "weighted_memory_to_active": 0.004823426716029644, "mix_std": 0.43966734409332275, "beta_std": 0.07661588490009308, "beta_low": 0.53326416015625, "beta_high": 0.0, "orthogonal_to_active": 0.036500684916973114, "evidence_weights": [0.2070661187171936, -0.2885279357433319, -0.12134586274623871], "agreement_weight": 0.05152309685945511, "orthogonal_weight": 10.435698509216309}, {"head": 4, "memory_to_active": 0.2303265780210495, "contribution_to_active": 0.4487765431404114, "mix_mean": 0.6059542894363403, "mix_low": 0.0067348480224609375, "mix_high": 0.00250244140625, "read_cosine": 0.02847449667751789, "beta_mean": 0.01617775671184063, "nonfinite": 0.0, "weighted_memory_to_active": 0.12852917611598969, "mix_std": 0.2732999622821808, "beta_std": 0.019211985170841217, "beta_low": 0.51318359375, "beta_high": 0.0, "orthogonal_to_active": 0.12593498826026917, "evidence_weights": [-0.23486165702342987, -0.2388724982738495, 1.872610330581665], "agreement_weight": -0.2098059058189392, "orthogonal_weight": -0.9999808073043823}, {"head": 5, "memory_to_active": 0.09446709603071213, "contribution_to_active": 0.4280601739883423, "mix_mean": 0.5749422311782837, "mix_low": 0.0014696121215820312, "mix_high": 0.0068569183349609375, "read_cosine": 0.5710382461547852, "beta_mean": 0.00849545281380415, "nonfinite": 0.0, "weighted_memory_to_active": 0.04987970367074013, "mix_std": 0.25792914628982544, "beta_std": 0.012611659243702888, "beta_low": 0.77587890625, "beta_high": 0.0, "orthogonal_to_active": 0.03301176056265831, "evidence_weights": [-0.20344050228595734, -0.3134758174419403, 0.6578445434570312], "agreement_weight": 0.0811384916305542, "orthogonal_weight": -0.8606013059616089}]}, {"layer": 12, "memory_to_active": 0.20714721083641052, "contribution_to_active": 0.5806252956390381, "mix_mean": 0.5442678928375244, "mix_low": 0.00849787425249815, "mix_high": 0.006802400108426809, "read_cosine": 0.22489473223686218, "beta_mean": 0.015051065012812614, "nonfinite": 0.0, "weighted_memory_to_active": 0.1041402816772461, "mix_std": 0.25975361466407776, "beta_std": 0.014680445194244385, "beta_low": 0.8283590078353882, "beta_high": 0.0, "orthogonal_to_active": 0.06609576940536499, "output_influence": 0.5082015991210938, "residual_influence": 0.1490354686975479, "state_active": 0.033311329782009125, "state_memory": 0.008161617442965508, "state_peak": 70.99180603027344, "growth_active": 1.1114391088485718, "growth_memory": 1.1686351299285889, "heads": [{"head": 0, "memory_to_active": 0.05437571182847023, "contribution_to_active": 0.6155591011047363, "mix_mean": 0.5807362794876099, "mix_low": 0.00041103363037109375, "mix_high": 0.005806922912597656, "read_cosine": 0.29194575548171997, "beta_mean": 0.0009129969985224307, "nonfinite": 0.0, "weighted_memory_to_active": 0.030411621555685997, "mix_std": 0.26469239592552185, "beta_std": 0.0018147958908230066, "beta_low": 0.99163818359375, "beta_high": 0.0, "orthogonal_to_active": 0.027773309499025345, "evidence_weights": [-0.20815855264663696, -0.27605459094047546, -0.03704974800348282], "agreement_weight": 0.10705409198999405, "orthogonal_weight": -0.9922689199447632}, {"head": 1, "memory_to_active": 0.07580631971359253, "contribution_to_active": 0.5582558512687683, "mix_mean": 0.5594687461853027, "mix_low": 0.00031280517578125, "mix_high": 0.0024166107177734375, "read_cosine": 0.37584298849105835, "beta_mean": 0.0005175692494958639, "nonfinite": 0.0, "weighted_memory_to_active": 0.04611960053443909, "mix_std": 0.2658854126930237, "beta_std": 0.007810282055288553, "beta_low": 0.99725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.03939424827694893, "evidence_weights": [-0.20116421580314636, -0.2717486619949341, 1.0464361906051636], "agreement_weight": 0.257213830947876, "orthogonal_weight": -0.9276415109634399}, {"head": 2, "memory_to_active": 0.596466064453125, "contribution_to_active": 0.4834204912185669, "mix_mean": 0.3092252314090729, "mix_low": 0.026610374450683594, "mix_high": 0.00015544891357421875, "read_cosine": 0.5568321347236633, "beta_mean": 0.08776501566171646, "nonfinite": 0.0, "weighted_memory_to_active": 0.23107141256332397, "mix_std": 0.23271536827087402, "beta_std": 0.0631236732006073, "beta_low": 0.0013427734375, "beta_high": 0.0, "orthogonal_to_active": 0.25571635365486145, "evidence_weights": [-0.20779858529567719, -0.06036724895238876, -0.010548833757638931], "agreement_weight": 0.027117431163787842, "orthogonal_weight": 1.4073755741119385}, {"head": 3, "memory_to_active": 0.028803756460547447, "contribution_to_active": 0.751401424407959, "mix_mean": 0.7205321788787842, "mix_low": 0.000102996826171875, "mix_high": 0.020479202270507812, "read_cosine": 0.07282647490501404, "beta_mean": 0.0002597120765130967, "nonfinite": 0.0, "weighted_memory_to_active": 0.020243188366293907, "mix_std": 0.2288515716791153, "beta_std": 0.005162051413208246, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.019081387668848038, "evidence_weights": [-0.28817641735076904, -0.33535289764404297, 1.6516236066818237], "agreement_weight": 0.5730054378509521, "orthogonal_weight": -0.9649716019630432}, {"head": 4, "memory_to_active": 0.4298989176750183, "contribution_to_active": 0.5743995904922485, "mix_mean": 0.45763641595840454, "mix_low": 0.01725482940673828, "mix_high": 0.005298614501953125, "read_cosine": -0.16322682797908783, "beta_mean": 0.0005630651721730828, "nonfinite": 0.0, "weighted_memory_to_active": 0.2597605884075165, "mix_std": 0.29264625906944275, "beta_std": 0.007780313026160002, "beta_low": 0.98785400390625, "beta_high": 0.0, "orthogonal_to_active": 0.026126017794013023, "evidence_weights": [-0.15486228466033936, -0.21528339385986328, -0.1285533905029297], "agreement_weight": 0.014837884344160557, "orthogonal_weight": -0.10489976406097412}, {"head": 5, "memory_to_active": 0.057532358914613724, "contribution_to_active": 0.5007152557373047, "mix_mean": 0.6380085945129395, "mix_low": 0.006295204162597656, "mix_high": 0.006657600402832031, "read_cosine": 0.21514777839183807, "beta_mean": 0.00028803315944969654, "nonfinite": 0.0, "weighted_memory_to_active": 0.037235189229249954, "mix_std": 0.27373063564300537, "beta_std": 0.0023915469646453857, "beta_low": 0.99432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.0284833163022995, "evidence_weights": [-0.22657407820224762, -0.3175884485244751, 1.2321109771728516], "agreement_weight": 0.422921359539032, "orthogonal_weight": -0.8173580169677734}]}, {"layer": 13, "memory_to_active": 0.1180838793516159, "contribution_to_active": 0.5439861416816711, "mix_mean": 0.5370342135429382, "mix_low": 0.0055141448974609375, "mix_high": 0.009012381546199322, "read_cosine": 0.3974868059158325, "beta_mean": 0.013987835496664047, "nonfinite": 0.0, "weighted_memory_to_active": 0.0489526204764843, "mix_std": 0.25727248191833496, "beta_std": 0.014887921512126923, "beta_low": 0.8118591904640198, "beta_high": 0.0, "orthogonal_to_active": 0.051519669592380524, "output_influence": 0.5345064997673035, "residual_influence": 0.18660664558410645, "state_active": 0.12995362281799316, "state_memory": 0.04459897056221962, "state_peak": 101.7678451538086, "growth_active": 0.9215397238731384, "growth_memory": 0.9872543811798096, "heads": [{"head": 0, "memory_to_active": 0.010006980039179325, "contribution_to_active": 0.6047108173370361, "mix_mean": 0.6135216951370239, "mix_low": 0.000804901123046875, "mix_high": 0.005225181579589844, "read_cosine": 0.4277510344982147, "beta_mean": 3.4920729376608506e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006439982447773218, "mix_std": 0.2684292495250702, "beta_std": 7.096004992490634e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00620862515643239, "evidence_weights": [-0.1931937336921692, -0.266389936208725, 0.7452729344367981], "agreement_weight": 0.18796178698539734, "orthogonal_weight": -1.0662825107574463}, {"head": 1, "memory_to_active": 0.010351019911468029, "contribution_to_active": 0.6019428968429565, "mix_mean": 0.5699529647827148, "mix_low": 0.0015506744384765625, "mix_high": 0.004874229431152344, "read_cosine": 0.20899443328380585, "beta_mean": 4.149356027483009e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005839485675096512, "mix_std": 0.2763217091560364, "beta_std": 8.643300679977983e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007663327269256115, "evidence_weights": [-0.18006974458694458, -0.2634012997150421, 0.5703082084655762], "agreement_weight": 0.5163476467132568, "orthogonal_weight": -1.3591670989990234}, {"head": 2, "memory_to_active": 0.06348945945501328, "contribution_to_active": 0.4880462884902954, "mix_mean": 0.3881492018699646, "mix_low": 0.0011997222900390625, "mix_high": 0.000179290771484375, "read_cosine": 0.5364904999732971, "beta_mean": 0.0067998929880559444, "nonfinite": 0.0, "weighted_memory_to_active": 0.02954000048339367, "mix_std": 0.23114311695098877, "beta_std": 0.024636104702949524, "beta_low": 0.87664794921875, "beta_high": 0.0, "orthogonal_to_active": 0.015891781076788902, "evidence_weights": [-0.18154700100421906, -0.21000203490257263, 0.41446807980537415], "agreement_weight": 0.0055405329912900925, "orthogonal_weight": -0.8462793827056885}, {"head": 3, "memory_to_active": 0.15185119211673737, "contribution_to_active": 0.6040043234825134, "mix_mean": 0.6589694023132324, "mix_low": 0.022522926330566406, "mix_high": 0.03212738037109375, "read_cosine": 0.010003666393458843, "beta_mean": 0.0008008077275007963, "nonfinite": 0.0, "weighted_memory_to_active": 0.11126396059989929, "mix_std": 0.2880105674266815, "beta_std": 0.007545889355242252, "beta_low": 0.99444580078125, "beta_high": 0.0, "orthogonal_to_active": 0.09991813451051712, "evidence_weights": [-0.311076819896698, -0.2383660227060318, 1.5460258722305298], "agreement_weight": 6.452627658843994, "orthogonal_weight": -0.9722617864608765}, {"head": 4, "memory_to_active": 0.41227778792381287, "contribution_to_active": 0.25378870964050293, "mix_mean": 0.33124637603759766, "mix_low": 0.0049648284912109375, "mix_high": 0.0, "read_cosine": 0.8855260014533997, "beta_mean": 0.07598358392715454, "nonfinite": 0.0, "weighted_memory_to_active": 0.09887483716011047, "mix_std": 0.20972353219985962, "beta_std": 0.05482684448361397, "beta_low": 0.00054931640625, "beta_high": 0.0, "orthogonal_to_active": 0.14419886469841003, "evidence_weights": [-0.0716719850897789, -0.22856421768665314, 0.43684470653533936], "agreement_weight": 0.004232178442180157, "orthogonal_weight": 2.27345871925354}, {"head": 5, "memory_to_active": 0.060526762157678604, "contribution_to_active": 0.7114236354827881, "mix_mean": 0.6603658199310303, "mix_low": 0.0020418167114257812, "mix_high": 0.011668205261230469, "read_cosine": 0.31615495681762695, "beta_mean": 0.00026631556102074683, "nonfinite": 0.0, "weighted_memory_to_active": 0.04175746813416481, "mix_std": 0.2700067162513733, "beta_std": 0.0021612977143377066, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.03523723781108856, "evidence_weights": [-0.22216692566871643, -0.2957841455936432, 0.8630555868148804], "agreement_weight": 0.19059519469738007, "orthogonal_weight": -0.9261537790298462}]}, {"layer": 14, "memory_to_active": 0.22667188942432404, "contribution_to_active": 0.41001832485198975, "mix_mean": 0.42523154616355896, "mix_low": 0.02735789678990841, "mix_high": 0.006510893814265728, "read_cosine": 0.6129763722419739, "beta_mean": 0.0618356391787529, "nonfinite": 0.0, "weighted_memory_to_active": 0.09349013864994049, "mix_std": 0.2399844378232956, "beta_std": 0.05996917188167572, "beta_low": 0.500396728515625, "beta_high": 0.0, "orthogonal_to_active": 0.04101774841547012, "output_influence": 0.3499266505241394, "residual_influence": 0.1451769918203354, "state_active": 0.05497347563505173, "state_memory": 0.015204079449176788, "state_peak": 46.353721618652344, "growth_active": 1.092570185661316, "growth_memory": 1.2568188905715942, "heads": [{"head": 0, "memory_to_active": 0.3668467402458191, "contribution_to_active": 0.16883030533790588, "mix_mean": 0.1740265190601349, "mix_low": 0.0190277099609375, "mix_high": 9.5367431640625e-07, "read_cosine": 0.8116448521614075, "beta_mean": 0.07182515412569046, "nonfinite": 0.0, "weighted_memory_to_active": 0.07019783556461334, "mix_std": 0.16570685803890228, "beta_std": 0.06408561021089554, "beta_low": 0.00360107421875, "beta_high": 0.0, "orthogonal_to_active": 0.07993463426828384, "evidence_weights": [-0.10007025301456451, -0.04739347845315933, 0.3918440639972687], "agreement_weight": 0.022313961759209633, "orthogonal_weight": 1.821392297744751}, {"head": 1, "memory_to_active": 0.4633462429046631, "contribution_to_active": 0.27487510442733765, "mix_mean": 0.20319533348083496, "mix_low": 0.12981033325195312, "mix_high": 1.52587890625e-05, "read_cosine": 0.8752697706222534, "beta_mean": 0.25494810938835144, "nonfinite": 0.0, "weighted_memory_to_active": 0.20770004391670227, "mix_std": 0.2333160638809204, "beta_std": 0.25075286626815796, "beta_low": 0.10107421875, "beta_high": 0.0, "orthogonal_to_active": 0.016409626230597496, "evidence_weights": [0.0011355014285072684, -0.19349925220012665, -0.08839385956525803], "agreement_weight": 0.05936818942427635, "orthogonal_weight": 0.24358361959457397}, {"head": 2, "memory_to_active": 0.1283126324415207, "contribution_to_active": 0.579951286315918, "mix_mean": 0.5892598032951355, "mix_low": 0.0032072067260742188, "mix_high": 0.004509925842285156, "read_cosine": 0.3981787860393524, "beta_mean": 0.0021580529864877462, "nonfinite": 0.0, "weighted_memory_to_active": 0.08387459814548492, "mix_std": 0.2686764597892761, "beta_std": 0.002625437220558524, "beta_low": 0.978759765625, "beta_high": 0.0, "orthogonal_to_active": 0.06548836827278137, "evidence_weights": [-0.13243453204631805, -0.315950870513916, 0.9238796830177307], "agreement_weight": 0.08355187624692917, "orthogonal_weight": -0.8827147483825684}, {"head": 3, "memory_to_active": 0.047262780368328094, "contribution_to_active": 0.5989153385162354, "mix_mean": 0.676103413105011, "mix_low": 0.0042266845703125, "mix_high": 0.02455425262451172, "read_cosine": -0.0659036710858345, "beta_mean": 0.005934946704655886, "nonfinite": 0.0, "weighted_memory_to_active": 0.030413813889026642, "mix_std": 0.258367121219635, "beta_std": 0.009427458979189396, "beta_low": 0.8375244140625, "beta_high": 0.0, "orthogonal_to_active": 0.022974424064159393, "evidence_weights": [-0.3419398367404938, -0.31025275588035583, 0.27959343791007996], "agreement_weight": 0.07079888880252838, "orthogonal_weight": -0.8377713561058044}, {"head": 4, "memory_to_active": 0.16155026853084564, "contribution_to_active": 0.4988391399383545, "mix_mean": 0.529137372970581, "mix_low": 0.0036764144897460938, "mix_high": 0.008417129516601562, "read_cosine": 0.9110299348831177, "beta_mean": 0.03491790592670441, "nonfinite": 0.0, "weighted_memory_to_active": 0.09476123750209808, "mix_std": 0.25901755690574646, "beta_std": 0.026003215461969376, "beta_low": 0.10107421875, "beta_high": 0.0, "orthogonal_to_active": 0.02407265640795231, "evidence_weights": [-0.21223659813404083, -0.23970268666744232, 0.4394020438194275], "agreement_weight": 0.13664139807224274, "orthogonal_weight": -0.6202402710914612}, {"head": 5, "memory_to_active": 0.19271257519721985, "contribution_to_active": 0.33869847655296326, "mix_mean": 0.3796667158603668, "mix_low": 0.004199028015136719, "mix_high": 0.001567840576171875, "read_cosine": 0.7476382255554199, "beta_mean": 0.0012296445202082396, "nonfinite": 0.0, "weighted_memory_to_active": 0.0739932581782341, "mix_std": 0.2548225522041321, "beta_std": 0.006920455023646355, "beta_low": 0.9803466796875, "beta_high": 0.0, "orthogonal_to_active": 0.037226755172014236, "evidence_weights": [-0.17372557520866394, -0.17885726690292358, 0.3532756268978119], "agreement_weight": 0.06583283841609955, "orthogonal_weight": -0.7543607950210571}]}, {"layer": 15, "memory_to_active": 0.053122036159038544, "contribution_to_active": 0.4944881200790405, "mix_mean": 0.42766478657722473, "mix_low": 0.007780393119901419, "mix_high": 0.0029096605721861124, "read_cosine": 0.6807191967964172, "beta_mean": 0.03671543672680855, "nonfinite": 0.0, "weighted_memory_to_active": 0.026781795546412468, "mix_std": 0.2836596965789795, "beta_std": 0.04431113228201866, "beta_low": 0.7127584218978882, "beta_high": 0.0, "orthogonal_to_active": 0.008090273477137089, "output_influence": 0.3206481635570526, "residual_influence": 0.18681086599826813, "state_active": 0.17231485247612, "state_memory": 0.01052722055464983, "state_peak": 29.438207626342773, "growth_active": 1.1929937601089478, "growth_memory": 1.3071693181991577, "heads": [{"head": 0, "memory_to_active": 0.0993964746594429, "contribution_to_active": 0.49207550287246704, "mix_mean": 0.4036107063293457, "mix_low": 0.007340431213378906, "mix_high": 0.006585121154785156, "read_cosine": 0.6320946216583252, "beta_mean": 0.03637869656085968, "nonfinite": 0.0, "weighted_memory_to_active": 0.04985050857067108, "mix_std": 0.2857140898704529, "beta_std": 0.0266968235373497, "beta_low": 0.09161376953125, "beta_high": 0.0, "orthogonal_to_active": 0.027541983872652054, "evidence_weights": [-0.189875066280365, -0.19459901750087738, 0.49576374888420105], "agreement_weight": 0.09054822474718094, "orthogonal_weight": -0.8048843145370483}, {"head": 1, "memory_to_active": 0.10825036466121674, "contribution_to_active": 0.42303529381752014, "mix_mean": 0.43773314356803894, "mix_low": 0.0011911392211914062, "mix_high": 0.005970001220703125, "read_cosine": 0.9322056770324707, "beta_mean": 0.00037331299972720444, "nonfinite": 0.0, "weighted_memory_to_active": 0.046444281935691833, "mix_std": 0.28617337346076965, "beta_std": 0.001853174646385014, "beta_low": 0.99755859375, "beta_high": 0.0, "orthogonal_to_active": 0.0024145410861819983, "evidence_weights": [-0.15013855695724487, -0.15905800461769104, 0.16472391784191132], "agreement_weight": 0.12139494717121124, "orthogonal_weight": 0.1137026771903038}, {"head": 2, "memory_to_active": 0.0035946860443800688, "contribution_to_active": 0.5024664402008057, "mix_mean": 0.34807127714157104, "mix_low": 0.012249946594238281, "mix_high": 6.389617919921875e-05, "read_cosine": 0.7847799062728882, "beta_mean": 4.549765435513109e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.001897195354104042, "mix_std": 0.2679538130760193, "beta_std": 0.00011147180339321494, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005412374157458544, "evidence_weights": [-0.20388339459896088, -0.2632034718990326, 0.11000162363052368], "agreement_weight": 0.0067424955777823925, "orthogonal_weight": 0.8657308220863342}, {"head": 3, "memory_to_active": 0.0020065992139279842, "contribution_to_active": 0.3739736080169678, "mix_mean": 0.415602445602417, "mix_low": 0.0042057037353515625, "mix_high": 0.0005664825439453125, "read_cosine": 0.6218547821044922, "beta_mean": 1.0390489478595555e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0006555506261065602, "mix_std": 0.2894534766674042, "beta_std": 2.408155705779791e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00044775058631785214, "evidence_weights": [-0.27441343665122986, -0.2373187392950058, 0.21174202859401703], "agreement_weight": 0.056338123977184296, "orthogonal_weight": -0.6551899313926697}, {"head": 4, "memory_to_active": 0.0779390037059784, "contribution_to_active": 0.6345783472061157, "mix_mean": 0.37998026609420776, "mix_low": 0.012449264526367188, "mix_high": 0.0008478164672851562, "read_cosine": 0.7743340730667114, "beta_mean": 0.18078219890594482, "nonfinite": 0.0, "weighted_memory_to_active": 0.04408978298306465, "mix_std": 0.30631792545318604, "beta_std": 0.21757501363754272, "beta_low": 0.219482421875, "beta_high": 0.0, "orthogonal_to_active": 0.006587438751012087, "evidence_weights": [-0.20503894984722137, -0.2533608675003052, -0.02098054252564907], "agreement_weight": 0.013928965665400028, "orthogonal_weight": 0.3581216335296631}, {"head": 5, "memory_to_active": 0.027545059099793434, "contribution_to_active": 0.5407992601394653, "mix_mean": 0.5809909105300903, "mix_low": 0.009245872497558594, "mix_high": 0.0034246444702148438, "read_cosine": 0.33904609084129333, "beta_mean": 0.002702509518712759, "nonfinite": 0.0, "weighted_memory_to_active": 0.01775345951318741, "mix_std": 0.26634538173675537, "beta_std": 0.01960623636841774, "beta_low": 0.9678955078125, "beta_high": 0.0, "orthogonal_to_active": 0.011008689180016518, "evidence_weights": [-0.2822836637496948, -0.30407848954200745, 0.6302706003189087], "agreement_weight": 0.31300047039985657, "orthogonal_weight": 0.829083263874054}]}]}, {"step": 66000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7669514250010252, "memory_to_active": 0.7967115640640259, "contribution_to_active": 0.5721207857131958, "mix_mean": 0.42525389790534973, "mix_low": 0.12443181872367859, "mix_high": 0.010535468347370625, "read_cosine": 0.34550103545188904, "beta_mean": 0.06929665058851242, "nonfinite": 0.0, "weighted_memory_to_active": 0.20504295825958252, "mix_std": 0.22378526628017426, "beta_std": 0.061179451644420624, "beta_low": 0.582588255405426, "beta_high": 4.704793173004873e-05, "orthogonal_to_active": 0.06476949155330658, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0913705825805664, "normal_token_accuracy": 0.4338153600692749, "state_active": 0.0642542615532875, "state_memory": 0.01717604324221611, "state_peak": 128.0378875732422, "growth_active": 1.1153887510299683, "growth_memory": 1.290729284286499, "output_influence": 0.44237545132637024, "residual_influence": 0.155461385846138, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1778006553649902, "no_memory_loss_delta": 0.7489190697669983, "scrambled_loss_delta": 0.1452295035123825, "memory_only_loss_delta": 4.155021667480469, "no_evidence_loss_delta": 0.44710180163383484, "no_agreement_loss_delta": 0.03039918839931488, "no_orthogonal_loss_delta": 0.023914992809295654}, {"index": 94, "tokens": 256, "normal_loss": 3.239661693572998, "no_memory_loss_delta": 0.7099481821060181, "scrambled_loss_delta": 0.37356871366500854, "memory_only_loss_delta": 3.722350597381592, "no_evidence_loss_delta": 0.3256472647190094, "no_agreement_loss_delta": 0.034930914640426636, "no_orthogonal_loss_delta": 0.035410404205322266}, {"index": 188, "tokens": 256, "normal_loss": 3.3602566719055176, "no_memory_loss_delta": 0.8980825543403625, "scrambled_loss_delta": 0.1287723034620285, "memory_only_loss_delta": 3.7384986877441406, "no_evidence_loss_delta": 0.49198952317237854, "no_agreement_loss_delta": 0.052951984107494354, "no_orthogonal_loss_delta": 0.0561392605304718}, {"index": 283, "tokens": 131, "normal_loss": 2.1072216033935547, "no_memory_loss_delta": 0.24937878549098969, "scrambled_loss_delta": 0.20252156257629395, "memory_only_loss_delta": 2.4978530406951904, "no_evidence_loss_delta": 0.09294594079256058, "no_agreement_loss_delta": -0.02649104595184326, "no_orthogonal_loss_delta": -0.028401803225278854}], "agreement_weight_abs_mean": 0.23822236061096191, "agreement_weight_abs_max": 6.508906364440918, "agreement_weight_mean": 0.20748035609722137, "orthogonal_weight_abs_mean": 0.9204481244087219, "orthogonal_weight_abs_max": 10.530304908752441, "orthogonal_weight_mean": -0.0752587765455246, "no_memory_loss": 3.7988767623901367, "no_memory_loss_delta": 0.7075059413909912, "no_memory_token_accuracy_gain": 0.08231367915868759, "no_memory_token_win_fraction": 0.6796440482139587, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 50.71280827296098, "scrambled_loss": 3.305284023284912, "scrambled_loss_delta": 0.2139136642217636, "scrambled_token_accuracy_gain": 0.021134594455361366, "scrambled_token_win_fraction": 0.6162402629852295, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.258190891716954, "memory_only_loss": 6.763097286224365, "memory_only_loss_delta": 3.6717262268066406, "memory_only_token_accuracy_gain": 0.2914349138736725, "memory_only_token_win_fraction": 0.9132369160652161, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 97.45674702823676, "no_evidence_loss": 3.4650626182556152, "no_evidence_loss_delta": 0.3736918866634369, "no_evidence_token_accuracy_gain": 0.04449388384819031, "no_evidence_token_win_fraction": 0.6140155792236328, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 31.18110806497434, "no_agreement_loss": 3.121192693710327, "no_agreement_loss_delta": 0.02982189878821373, "no_agreement_token_accuracy_gain": -0.004449388012290001, "no_agreement_token_win_fraction": 0.4749721884727478, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.9381613533794115, "no_orthogonal_loss": 3.1201117038726807, "no_orthogonal_loss_delta": 0.028741177171468735, "no_orthogonal_token_accuracy_gain": -0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5639599561691284, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.833207823633939, "short_no_memory_loss_delta": 0.6533149480819702, "long_context_memory_gain": 0.054190993309020996, "normal_context_loss_gain": 0.13177412748336792, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.6133155822753906, "contribution_to_active": 1.2498130798339844, "mix_mean": 0.1068994551897049, "mix_low": 0.5156545639038086, "mix_high": 0.004816532135009766, "read_cosine": 0.4136347770690918, "beta_mean": 0.20304906368255615, "nonfinite": 0.0, "weighted_memory_to_active": 1.2842559814453125, "mix_std": 0.09196586906909943, "beta_std": 0.13610336184501648, "beta_low": 0.1276753842830658, "beta_high": 0.0, "orthogonal_to_active": 0.051156166940927505, "output_influence": 0.21603631973266602, "residual_influence": 0.24556681513786316, "state_active": 0.06568178534507751, "state_memory": 0.04453465715050697, "state_peak": 48.538082122802734, "growth_active": 1.2236019372940063, "growth_memory": 1.2735875844955444, "heads": [{"head": 0, "memory_to_active": 2.6921894550323486, "contribution_to_active": 0.47412383556365967, "mix_mean": 0.020648807287216187, "mix_low": 0.6201553344726562, "mix_high": 6.67572021484375e-06, "read_cosine": 0.21535301208496094, "beta_mean": 0.18575529754161835, "nonfinite": 0.0, "weighted_memory_to_active": 0.6003295183181763, "mix_std": 0.062117669731378555, "beta_std": 0.10339243710041046, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06601924449205399, "evidence_weights": [-0.007513361983001232, -0.04763123765587807, -0.056215520948171616], "agreement_weight": -0.21913599967956543, "orthogonal_weight": -0.12696702778339386}, {"head": 1, "memory_to_active": 7.181735038757324, "contribution_to_active": 3.259814739227295, "mix_mean": 0.02030862309038639, "mix_low": 0.6068706512451172, "mix_high": 0.0, "read_cosine": 0.4344249963760376, "beta_mean": 0.444377064704895, "nonfinite": 0.0, "weighted_memory_to_active": 3.5151820182800293, "mix_std": 0.05816435068845749, "beta_std": 0.2069888412952423, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05366460233926773, "evidence_weights": [0.014613749459385872, -0.11255417764186859, 0.6824740171432495], "agreement_weight": 0.026897111907601357, "orthogonal_weight": 0.032855916768312454}, {"head": 2, "memory_to_active": 2.716063976287842, "contribution_to_active": 0.5782955884933472, "mix_mean": 0.015008779242634773, "mix_low": 0.6417903900146484, "mix_high": 0.0, "read_cosine": 0.2693246901035309, "beta_mean": 0.2318812906742096, "nonfinite": 0.0, "weighted_memory_to_active": 0.6747517585754395, "mix_std": 0.032950419932603836, "beta_std": 0.19319751858711243, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.054063066840171814, "evidence_weights": [0.06042679026722908, 0.046638235449790955, 0.2396070510149002], "agreement_weight": 0.011314304545521736, "orthogonal_weight": -0.19335542619228363}, {"head": 3, "memory_to_active": 7.496129989624023, "contribution_to_active": 2.6563308238983154, "mix_mean": 0.030597884207963943, "mix_low": 0.49009037017822266, "mix_high": 0.0, "read_cosine": 0.3058462142944336, "beta_mean": 0.30517876148223877, "nonfinite": 0.0, "weighted_memory_to_active": 2.818286418914795, "mix_std": 0.07088251411914825, "beta_std": 0.2434726506471634, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11600934714078903, "evidence_weights": [-0.03651100769639015, -0.032287269830703735, -0.129576176404953], "agreement_weight": -0.006716272793710232, "orthogonal_weight": -0.050073105841875076}, {"head": 4, "memory_to_active": 1.4853169918060303, "contribution_to_active": 0.05947202444076538, "mix_mean": 0.014079675078392029, "mix_low": 0.7206077575683594, "mix_high": 0.0, "read_cosine": 0.31570756435394287, "beta_mean": 0.03246356174349785, "nonfinite": 0.0, "weighted_memory_to_active": 0.039457403123378754, "mix_std": 0.0438862219452858, "beta_std": 0.05173208937048912, "beta_low": 0.3994140625, "beta_high": 0.0, "orthogonal_to_active": 0.000963470374699682, "evidence_weights": [0.11853928864002228, 0.08174465596675873, -0.13844148814678192], "agreement_weight": -0.20260478556156158, "orthogonal_weight": 0.02083246223628521}, {"head": 5, "memory_to_active": 0.1084570661187172, "contribution_to_active": 0.47084128856658936, "mix_mean": 0.5407528877258301, "mix_low": 0.014412879943847656, "mix_high": 0.02889251708984375, "read_cosine": 0.9411521553993225, "beta_mean": 0.01863846555352211, "nonfinite": 0.0, "weighted_memory_to_active": 0.05752895027399063, "mix_std": 0.2837940454483032, "beta_std": 0.017836574465036392, "beta_low": 0.36651611328125, "beta_high": 0.0, "orthogonal_to_active": 0.016217252239584923, "evidence_weights": [-0.17410755157470703, -0.2182060182094574, 0.3784749507904053], "agreement_weight": 0.28681614995002747, "orthogonal_weight": -0.68982994556427}]}, {"layer": 1, "memory_to_active": 2.402125358581543, "contribution_to_active": 0.4672921299934387, "mix_mean": 0.21375086903572083, "mix_low": 0.351641982793808, "mix_high": 0.0010630290489643812, "read_cosine": 0.2986246645450592, "beta_mean": 0.17794151604175568, "nonfinite": 0.0, "weighted_memory_to_active": 0.31088048219680786, "mix_std": 0.1150069534778595, "beta_std": 0.13236922025680542, "beta_low": 0.3531697690486908, "beta_high": 0.00014241535973269492, "orthogonal_to_active": 0.039224773645401, "output_influence": 0.2576965093612671, "residual_influence": 0.06982654333114624, "state_active": 0.02861648239195347, "state_memory": 0.03499680012464523, "state_peak": 42.76234817504883, "growth_active": 1.1726467609405518, "growth_memory": 1.4546548128128052, "heads": [{"head": 0, "memory_to_active": 0.025470755994319916, "contribution_to_active": 0.47907012701034546, "mix_mean": 0.5260573625564575, "mix_low": 0.0047893524169921875, "mix_high": 0.0021333694458007812, "read_cosine": 0.03275047615170479, "beta_mean": 0.00016105768736451864, "nonfinite": 0.0, "weighted_memory_to_active": 0.015866104513406754, "mix_std": 0.28032273054122925, "beta_std": 0.0003047385544050485, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.015685811638832092, "evidence_weights": [-0.20702067017555237, -0.27133405208587646, 0.2739276885986328], "agreement_weight": 0.18597067892551422, "orthogonal_weight": -1.028272271156311}, {"head": 1, "memory_to_active": 2.973803758621216, "contribution_to_active": 0.26633748412132263, "mix_mean": 0.02381301112473011, "mix_low": 0.45017242431640625, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5255422592163086, "beta_mean": 0.2529299855232239, "nonfinite": 0.0, "weighted_memory_to_active": 0.2949036955833435, "mix_std": 0.04512390121817589, "beta_std": 0.18744336068630219, "beta_low": 0.00286865234375, "beta_high": 0.00048828125, "orthogonal_to_active": 0.021949056535959244, "evidence_weights": [0.021902110427618027, -0.06478085368871689, 0.05615105852484703], "agreement_weight": -0.0039981999434530735, "orthogonal_weight": 0.20838743448257446}, {"head": 2, "memory_to_active": 0.04459569230675697, "contribution_to_active": 0.536668062210083, "mix_mean": 0.6661669611930847, "mix_low": 0.00037097930908203125, "mix_high": 0.0042438507080078125, "read_cosine": 0.4374225437641144, "beta_mean": 0.0016063954681158066, "nonfinite": 0.0, "weighted_memory_to_active": 0.02967648394405842, "mix_std": 0.23032362759113312, "beta_std": 0.0038800216279923916, "beta_low": 0.97735595703125, "beta_high": 0.0, "orthogonal_to_active": 0.020398559048771858, "evidence_weights": [-0.2751789391040802, -0.3698546290397644, 0.6689361929893494], "agreement_weight": 0.04637673497200012, "orthogonal_weight": -0.8068482279777527}, {"head": 3, "memory_to_active": 3.099006175994873, "contribution_to_active": 0.16913968324661255, "mix_mean": 0.024712413549423218, "mix_low": 0.5143613815307617, "mix_high": 0.0, "read_cosine": 0.18200726807117462, "beta_mean": 0.3325391411781311, "nonfinite": 0.0, "weighted_memory_to_active": 0.2320047914981842, "mix_std": 0.06005791574716568, "beta_std": 0.25684288144111633, "beta_low": 0.0672607421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.09552047401666641, "evidence_weights": [0.037196990102529526, -0.021119702607393265, -0.022846711799502373], "agreement_weight": -0.030826322734355927, "orthogonal_weight": -0.48672881722450256}, {"head": 4, "memory_to_active": 3.8600456714630127, "contribution_to_active": 1.2747060060501099, "mix_mean": 0.03517698496580124, "mix_low": 0.29825401306152344, "mix_high": 0.0, "read_cosine": 0.33724260330200195, "beta_mean": 0.15870162844657898, "nonfinite": 0.0, "weighted_memory_to_active": 1.2397505044937134, "mix_std": 0.056329306215047836, "beta_std": 0.12822526693344116, "beta_low": 0.03326416015625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.05912327766418457, "evidence_weights": [-0.025130603462457657, -0.00679028732702136, 0.13483892381191254], "agreement_weight": -0.011775766499340534, "orthogonal_weight": 0.06900365650653839}, {"head": 5, "memory_to_active": 4.4098286628723145, "contribution_to_active": 0.07783134281635284, "mix_mean": 0.006578534375876188, "mix_low": 0.8419036865234375, "mix_high": 0.0, "read_cosine": 0.2767828106880188, "beta_mean": 0.32171082496643066, "nonfinite": 0.0, "weighted_memory_to_active": 0.05308105796575546, "mix_std": 0.0178841445595026, "beta_std": 0.21751901507377625, "beta_low": 0.03826904296875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.022671453654766083, "evidence_weights": [0.16371063888072968, 0.08074008673429489, 0.09688118100166321], "agreement_weight": -0.11383959650993347, "orthogonal_weight": 0.39042362570762634}]}, {"layer": 2, "memory_to_active": 0.9849302768707275, "contribution_to_active": 0.3564397096633911, "mix_mean": 0.406062513589859, "mix_low": 0.24955224990844727, "mix_high": 0.004309495445340872, "read_cosine": 0.305817186832428, "beta_mean": 0.08384507894515991, "nonfinite": 0.0, "weighted_memory_to_active": 0.07472817599773407, "mix_std": 0.18676120042800903, "beta_std": 0.06971054524183273, "beta_low": 0.6005859375, "beta_high": 0.000274658203125, "orthogonal_to_active": 0.016098763793706894, "output_influence": 0.39712491631507874, "residual_influence": 0.11110487580299377, "state_active": 0.03982490301132202, "state_memory": 0.014565002173185349, "state_peak": 27.63855743408203, "growth_active": 1.0487160682678223, "growth_memory": 1.5582594871520996, "heads": [{"head": 0, "memory_to_active": 0.007705723866820335, "contribution_to_active": 0.36087697744369507, "mix_mean": 0.6370419263839722, "mix_low": 0.0006036758422851562, "mix_high": 0.009012222290039062, "read_cosine": 0.140290766954422, "beta_mean": 6.289913289947435e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005063226446509361, "mix_std": 0.2677231729030609, "beta_std": 0.00010811988613568246, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005280670709908009, "evidence_weights": [-0.37516772747039795, -0.2181813269853592, 0.18771971762180328], "agreement_weight": 0.13358305394649506, "orthogonal_weight": -1.0778218507766724}, {"head": 1, "memory_to_active": 3.02016544342041, "contribution_to_active": 0.0391545444726944, "mix_mean": 0.0047535281628370285, "mix_low": 0.9007816314697266, "mix_high": 0.0, "read_cosine": 0.14092200994491577, "beta_mean": 0.24217696487903595, "nonfinite": 0.0, "weighted_memory_to_active": 0.03197526931762695, "mix_std": 0.010616089217364788, "beta_std": 0.16110381484031677, "beta_low": 0.0030517578125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.009470164775848389, "evidence_weights": [0.1429089903831482, 0.11912891268730164, 0.5823922157287598], "agreement_weight": 0.03643631935119629, "orthogonal_weight": 0.32559600472450256}, {"head": 2, "memory_to_active": 0.010886927135288715, "contribution_to_active": 0.4202578067779541, "mix_mean": 0.5456311106681824, "mix_low": 0.0012722015380859375, "mix_high": 0.0036869049072265625, "read_cosine": 0.3549653887748718, "beta_mean": 2.854380727512762e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0045770686119794846, "mix_std": 0.27581655979156494, "beta_std": 5.1421251555439085e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 6.110719550633803e-05, "evidence_weights": [-0.252420037984848, -0.29297733306884766, 0.5876931548118591], "agreement_weight": -0.018037904053926468, "orthogonal_weight": 0.016160450875759125}, {"head": 3, "memory_to_active": 0.10295305401086807, "contribution_to_active": 0.5553274154663086, "mix_mean": 0.5832947492599487, "mix_low": 0.0002651214599609375, "mix_high": 0.005927085876464844, "read_cosine": 0.6236157417297363, "beta_mean": 0.017569106072187424, "nonfinite": 0.0, "weighted_memory_to_active": 0.06498734652996063, "mix_std": 0.2536625862121582, "beta_std": 0.02609487622976303, "beta_low": 0.5428466796875, "beta_high": 0.0, "orthogonal_to_active": 0.04198651760816574, "evidence_weights": [-0.20509809255599976, -0.32657331228256226, 0.7623308300971985], "agreement_weight": 0.05503023415803909, "orthogonal_weight": -0.8870333433151245}, {"head": 4, "memory_to_active": 2.7576165199279785, "contribution_to_active": 0.24391035735607147, "mix_mean": 0.019562754780054092, "mix_low": 0.5935068130493164, "mix_high": 0.0, "read_cosine": 0.18596151471138, "beta_mean": 0.24319694936275482, "nonfinite": 0.0, "weighted_memory_to_active": 0.3361964225769043, "mix_std": 0.04893166944384575, "beta_std": 0.23085084557533264, "beta_low": 0.0576171875, "beta_high": 0.0015869140625, "orthogonal_to_active": 0.03834420442581177, "evidence_weights": [0.02620772086083889, 0.006841341964900494, 0.026325559243559837], "agreement_weight": 0.026988079771399498, "orthogonal_weight": -0.14069756865501404}, {"head": 5, "memory_to_active": 0.01025350857526064, "contribution_to_active": 0.5191110968589783, "mix_mean": 0.6460908055305481, "mix_low": 0.0008840560913085938, "mix_high": 0.0072307586669921875, "read_cosine": 0.3891476094722748, "beta_mean": 3.602130163926631e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005569731816649437, "mix_std": 0.26381716132164, "beta_std": 5.417809006758034e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014499126700684428, "evidence_weights": [-0.27962392568588257, -0.28956925868988037, 0.9030997157096863], "agreement_weight": 0.2496865689754486, "orthogonal_weight": -0.3092145621776581}]}, {"layer": 3, "memory_to_active": 0.46373116970062256, "contribution_to_active": 0.48667120933532715, "mix_mean": 0.5163143277168274, "mix_low": 0.055089473724365234, "mix_high": 0.010681470856070518, "read_cosine": 0.19473978877067566, "beta_mean": 0.023583918809890747, "nonfinite": 0.0, "weighted_memory_to_active": 0.13092926144599915, "mix_std": 0.25742214918136597, "beta_std": 0.02320818230509758, "beta_low": 0.7673543691635132, "beta_high": 0.0, "orthogonal_to_active": 0.05574209988117218, "output_influence": 0.42746472358703613, "residual_influence": 0.10219814628362656, "state_active": 0.022959420457482338, "state_memory": 0.006918356288224459, "state_peak": 22.39266586303711, "growth_active": 1.0311075448989868, "growth_memory": 1.6722462177276611, "heads": [{"head": 0, "memory_to_active": 0.4339727759361267, "contribution_to_active": 0.5997287631034851, "mix_mean": 0.6044228076934814, "mix_low": 0.002285003662109375, "mix_high": 0.022032737731933594, "read_cosine": 0.2412903904914856, "beta_mean": 0.00967784970998764, "nonfinite": 0.0, "weighted_memory_to_active": 0.2664514482021332, "mix_std": 0.3001675605773926, "beta_std": 0.011424299329519272, "beta_low": 0.698974609375, "beta_high": 0.0, "orthogonal_to_active": 0.1929132640361786, "evidence_weights": [-0.21635746955871582, -0.25036442279815674, 1.0503318309783936], "agreement_weight": 0.11612261831760406, "orthogonal_weight": -0.7653213739395142}, {"head": 1, "memory_to_active": 0.008427157066762447, "contribution_to_active": 0.4529835283756256, "mix_mean": 0.5821123123168945, "mix_low": 0.003849029541015625, "mix_high": 0.009828567504882812, "read_cosine": 0.1742788553237915, "beta_mean": 4.5606226194649935e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005229084752500057, "mix_std": 0.2950815260410309, "beta_std": 0.00010116258636116982, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004194686189293861, "evidence_weights": [-0.2689709961414337, -0.217681884765625, 0.7021477818489075], "agreement_weight": 0.5712530016899109, "orthogonal_weight": -0.8709036707878113}, {"head": 2, "memory_to_active": 2.081650733947754, "contribution_to_active": 0.29509660601615906, "mix_mean": 0.054120395332574844, "mix_low": 0.3145408630371094, "mix_high": 9.5367431640625e-07, "read_cosine": 0.20456652343273163, "beta_mean": 0.12641268968582153, "nonfinite": 0.0, "weighted_memory_to_active": 0.3240562081336975, "mix_std": 0.10564307123422623, "beta_std": 0.11453188955783844, "beta_low": 0.02032470703125, "beta_high": 0.0, "orthogonal_to_active": 0.000506267067976296, "evidence_weights": [-0.10006563365459442, -0.005107447970658541, -0.027182867750525475], "agreement_weight": -0.013216785155236721, "orthogonal_weight": 0.0019163597607985139}, {"head": 3, "memory_to_active": 0.016434580087661743, "contribution_to_active": 0.5326493382453918, "mix_mean": 0.6070976853370667, "mix_low": 0.004801750183105469, "mix_high": 0.009852409362792969, "read_cosine": 0.27213791012763977, "beta_mean": 8.380976942135021e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011036050505936146, "mix_std": 0.28709807991981506, "beta_std": 0.00018069930956698954, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028267407324165106, "evidence_weights": [-0.26334822177886963, -0.22613760828971863, 2.2687482833862305], "agreement_weight": 2.2150726318359375, "orthogonal_weight": -0.32669392228126526}, {"head": 4, "memory_to_active": 0.010786304250359535, "contribution_to_active": 0.3856031000614166, "mix_mean": 0.5241765975952148, "mix_low": 0.0024995803833007812, "mix_high": 0.0029325485229492188, "read_cosine": 0.16338929533958435, "beta_mean": 6.816569657530636e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007012483663856983, "mix_std": 0.28934523463249207, "beta_std": 0.00013233392382971942, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006788391619920731, "evidence_weights": [-0.23810353875160217, -0.22321456670761108, 0.1403307467699051], "agreement_weight": -0.044184360653162, "orthogonal_weight": -1.0272212028503418}, {"head": 5, "memory_to_active": 0.23111532628536224, "contribution_to_active": 0.6539657115936279, "mix_mean": 0.7259561419487, "mix_low": 0.0025606155395507812, "mix_high": 0.019441604614257812, "read_cosine": 0.11277572810649872, "beta_mean": 0.0052153985016047955, "nonfinite": 0.0, "weighted_memory_to_active": 0.171790212392807, "mix_std": 0.2671974301338196, "beta_std": 0.012878701090812683, "beta_low": 0.88482666015625, "beta_high": 0.0, "orthogonal_to_active": 0.12722322344779968, "evidence_weights": [-0.26015567779541016, -0.3398517370223999, 0.5572990775108337], "agreement_weight": 0.2206626683473587, "orthogonal_weight": -0.7741855382919312}]}, {"layer": 4, "memory_to_active": 0.9971376061439514, "contribution_to_active": 0.4686458706855774, "mix_mean": 0.3742331266403198, "mix_low": 0.233968585729599, "mix_high": 0.011321067810058594, "read_cosine": 0.30093616247177124, "beta_mean": 0.10935215651988983, "nonfinite": 0.0, "weighted_memory_to_active": 0.13504144549369812, "mix_std": 0.16045036911964417, "beta_std": 0.10172216594219208, "beta_low": 0.3908488154411316, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.051137834787368774, "output_influence": 0.5006146430969238, "residual_influence": 0.15199348330497742, "state_active": 0.036625903099775314, "state_memory": 0.016435956582427025, "state_peak": 41.69989013671875, "growth_active": 1.1755152940750122, "growth_memory": 1.477713704109192, "heads": [{"head": 0, "memory_to_active": 0.15417605638504028, "contribution_to_active": 0.8009560704231262, "mix_mean": 0.6863934397697449, "mix_low": 0.009884834289550781, "mix_high": 0.031670570373535156, "read_cosine": 0.44886159896850586, "beta_mean": 0.25202757120132446, "nonfinite": 0.0, "weighted_memory_to_active": 0.1312253177165985, "mix_std": 0.34713906049728394, "beta_std": 0.25285986065864563, "beta_low": 0.2498779296875, "beta_high": 0.0, "orthogonal_to_active": 0.08294175565242767, "evidence_weights": [-0.25401490926742554, -0.36473479866981506, -0.1486547738313675], "agreement_weight": 0.0027471729554235935, "orthogonal_weight": 0.7564153671264648}, {"head": 1, "memory_to_active": 0.047833122313022614, "contribution_to_active": 0.6653007864952087, "mix_mean": 0.710969090461731, "mix_low": 9.632110595703125e-05, "mix_high": 0.0182647705078125, "read_cosine": 0.5679538249969482, "beta_mean": 0.001066108699887991, "nonfinite": 0.0, "weighted_memory_to_active": 0.035619717091321945, "mix_std": 0.23389527201652527, "beta_std": 0.0073893410153687, "beta_low": 0.978271484375, "beta_high": 0.0, "orthogonal_to_active": 0.018657665699720383, "evidence_weights": [-0.241554856300354, -0.36854392290115356, 0.598307728767395], "agreement_weight": 0.34900858998298645, "orthogonal_weight": -0.719241738319397}, {"head": 2, "memory_to_active": 1.5449113845825195, "contribution_to_active": 0.3198261260986328, "mix_mean": 0.046198293566703796, "mix_low": 0.3548755645751953, "mix_high": 2.288818359375e-05, "read_cosine": 0.2233704775571823, "beta_mean": 0.059953946620225906, "nonfinite": 0.0, "weighted_memory_to_active": 0.273613303899765, "mix_std": 0.0843995213508606, "beta_std": 0.05833861231803894, "beta_low": 0.0904541015625, "beta_high": 0.0, "orthogonal_to_active": 0.09223458170890808, "evidence_weights": [0.03380994871258736, -0.04818712919950485, 0.29037952423095703], "agreement_weight": 0.13228574395179749, "orthogonal_weight": -0.4498249590396881}, {"head": 3, "memory_to_active": 1.7866981029510498, "contribution_to_active": 0.149969220161438, "mix_mean": 0.035099633038043976, "mix_low": 0.38923168182373047, "mix_high": 0.0, "read_cosine": 0.2346113622188568, "beta_mean": 0.13770902156829834, "nonfinite": 0.0, "weighted_memory_to_active": 0.2118958830833435, "mix_std": 0.05984877422451973, "beta_std": 0.12388916313648224, "beta_low": 0.02642822265625, "beta_high": 0.0, "orthogonal_to_active": 0.00027553923428058624, "evidence_weights": [-0.024546457454562187, 0.04487811401486397, -0.2523953318595886], "agreement_weight": 0.08176953345537186, "orthogonal_weight": -0.0019267555326223373}, {"head": 4, "memory_to_active": 0.009710632264614105, "contribution_to_active": 0.6378331184387207, "mix_mean": 0.7531794905662537, "mix_low": 4.482269287109375e-05, "mix_high": 0.017965316772460938, "read_cosine": 0.12408794462680817, "beta_mean": 2.1699224816984497e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007113427855074406, "mix_std": 0.20994073152542114, "beta_std": 5.9811867686221376e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004529288038611412, "evidence_weights": [-0.33346930146217346, -0.3216434419155121, 0.29326963424682617], "agreement_weight": 0.3119296133518219, "orthogonal_weight": -0.6746509671211243}, {"head": 5, "memory_to_active": 2.4394965171813965, "contribution_to_active": 0.237989604473114, "mix_mean": 0.013558804988861084, "mix_low": 0.6496782302856445, "mix_high": 2.86102294921875e-06, "read_cosine": 0.2067318558692932, "beta_mean": 0.2053346037864685, "nonfinite": 0.0, "weighted_memory_to_active": 0.1507810801267624, "mix_std": 0.02747875452041626, "beta_std": 0.16779619455337524, "beta_low": 6.103515625e-05, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.1081881895661354, "evidence_weights": [0.05674051120877266, -0.01646043173968792, -0.2884535789489746], "agreement_weight": -0.08713725954294205, "orthogonal_weight": 1.0631836652755737}]}, {"layer": 5, "memory_to_active": 0.3800017237663269, "contribution_to_active": 0.5411200523376465, "mix_mean": 0.5161876678466797, "mix_low": 0.05954599380493164, "mix_high": 0.020726840943098068, "read_cosine": 0.08603863418102264, "beta_mean": 0.051499150693416595, "nonfinite": 0.0, "weighted_memory_to_active": 0.1373371183872223, "mix_std": 0.2521519660949707, "beta_std": 0.06655289232730865, "beta_low": 0.665008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.05796678364276886, "output_influence": 0.5709977149963379, "residual_influence": 0.17842528223991394, "state_active": 0.03978557139635086, "state_memory": 0.01007291954010725, "state_peak": 59.24544906616211, "growth_active": 1.1147607564926147, "growth_memory": 1.2581990957260132, "heads": [{"head": 0, "memory_to_active": 0.21820436418056488, "contribution_to_active": 0.5806443095207214, "mix_mean": 0.48142966628074646, "mix_low": 0.008999824523925781, "mix_high": 0.0021877288818359375, "read_cosine": -0.12162017822265625, "beta_mean": 0.022582275792956352, "nonfinite": 0.0, "weighted_memory_to_active": 0.07629658281803131, "mix_std": 0.2812001705169678, "beta_std": 0.03535027801990509, "beta_low": 0.48992919921875, "beta_high": 0.0, "orthogonal_to_active": 0.033768899738788605, "evidence_weights": [-0.16647152602672577, -0.18475088477134705, 0.08849309384822845], "agreement_weight": -0.022069500759243965, "orthogonal_weight": -0.4692857265472412}, {"head": 1, "memory_to_active": 0.005663130898028612, "contribution_to_active": 0.5523967742919922, "mix_mean": 0.7096499800682068, "mix_low": 0.000995635986328125, "mix_high": 0.04045391082763672, "read_cosine": 0.22058983147144318, "beta_mean": 1.2526304089988116e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003446497255936265, "mix_std": 0.2677798867225647, "beta_std": 2.5168523279717192e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003147341776639223, "evidence_weights": [-0.2495424896478653, -0.2892139256000519, 0.9555537700653076], "agreement_weight": 0.6953561902046204, "orthogonal_weight": -0.9604032039642334}, {"head": 2, "memory_to_active": 0.012286605313420296, "contribution_to_active": 0.5940093994140625, "mix_mean": 0.667708694934845, "mix_low": 0.0024127960205078125, "mix_high": 0.02522563934326172, "read_cosine": -0.45864444971084595, "beta_mean": 7.061941141728312e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007407096214592457, "mix_std": 0.26419633626937866, "beta_std": 0.0002738522016443312, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006804581731557846, "evidence_weights": [-0.26154524087905884, -0.3234381377696991, 0.1641918569803238], "agreement_weight": -0.1494472324848175, "orthogonal_weight": -1.0285625457763672}, {"head": 3, "memory_to_active": 1.5144264698028564, "contribution_to_active": 0.2602997422218323, "mix_mean": 0.1176052987575531, "mix_low": 0.12555503845214844, "mix_high": 0.0, "read_cosine": 0.16418717801570892, "beta_mean": 0.10145266354084015, "nonfinite": 0.0, "weighted_memory_to_active": 0.4000808298587799, "mix_std": 0.14964067935943604, "beta_std": 0.15197768807411194, "beta_low": 0.19744873046875, "beta_high": 0.0, "orthogonal_to_active": 0.15993396937847137, "evidence_weights": [-0.07616011798381805, -0.09944876283407211, 0.06913399696350098], "agreement_weight": 0.0473756305873394, "orthogonal_weight": -0.4896702170372009}, {"head": 4, "memory_to_active": 0.4662858545780182, "contribution_to_active": 0.5613127946853638, "mix_mean": 0.30680322647094727, "mix_low": 0.21787548065185547, "mix_high": 0.004231452941894531, "read_cosine": 0.5333478450775146, "beta_mean": 0.18466956913471222, "nonfinite": 0.0, "weighted_memory_to_active": 0.29229670763015747, "mix_std": 0.3325144052505493, "beta_std": 0.21078084409236908, "beta_low": 0.30389404296875, "beta_high": 0.0, "orthogonal_to_active": 0.11745738238096237, "evidence_weights": [-0.11792102456092834, -0.25661367177963257, -0.051351431757211685], "agreement_weight": 0.030871577560901642, "orthogonal_weight": 0.5973362326622009}, {"head": 5, "memory_to_active": 0.06314390897750854, "contribution_to_active": 0.6980571150779724, "mix_mean": 0.8139292001724243, "mix_low": 0.0014371871948242188, "mix_high": 0.052262306213378906, "read_cosine": 0.17837156355381012, "beta_mean": 0.00020724974456243217, "nonfinite": 0.0, "weighted_memory_to_active": 0.04449491575360298, "mix_std": 0.21758025884628296, "beta_std": 0.0009094809647649527, "beta_low": 0.99884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.026688512414693832, "evidence_weights": [-0.30608370900154114, -0.33063796162605286, 1.8707865476608276], "agreement_weight": 0.5228636264801025, "orthogonal_weight": -0.6304013133049011}]}, {"layer": 6, "memory_to_active": 0.24922975897789001, "contribution_to_active": 0.6349217891693115, "mix_mean": 0.5960946679115295, "mix_low": 0.040332477539777756, "mix_high": 0.02978881448507309, "read_cosine": 0.3162584900856018, "beta_mean": 0.06758420914411545, "nonfinite": 0.0, "weighted_memory_to_active": 0.12753596901893616, "mix_std": 0.2366042286157608, "beta_std": 0.07870149612426758, "beta_low": 0.6077474355697632, "beta_high": 0.0, "orthogonal_to_active": 0.06069965288043022, "output_influence": 0.5785924196243286, "residual_influence": 0.20148393511772156, "state_active": 0.04503428190946579, "state_memory": 0.006366528570652008, "state_peak": 89.13081359863281, "growth_active": 1.2083715200424194, "growth_memory": 1.1520166397094727, "heads": [{"head": 0, "memory_to_active": 0.10248789936304092, "contribution_to_active": 0.7828790545463562, "mix_mean": 0.8197731971740723, "mix_low": 0.0002841949462890625, "mix_high": 0.04965400695800781, "read_cosine": 0.3509136438369751, "beta_mean": 0.010765786282718182, "nonfinite": 0.0, "weighted_memory_to_active": 0.08852840960025787, "mix_std": 0.19782936573028564, "beta_std": 0.030186954885721207, "beta_low": 0.81732177734375, "beta_high": 0.0, "orthogonal_to_active": 0.05914616212248802, "evidence_weights": [-0.30304738879203796, -0.3472982347011566, 1.211331844329834], "agreement_weight": 0.794498085975647, "orthogonal_weight": -0.7440639138221741}, {"head": 1, "memory_to_active": 0.004109317436814308, "contribution_to_active": 0.5272562503814697, "mix_mean": 0.5973047614097595, "mix_low": 0.00072479248046875, "mix_high": 0.017248153686523438, "read_cosine": 0.4315483272075653, "beta_mean": 1.0043962902273051e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0018936654087156057, "mix_std": 0.2829679250717163, "beta_std": 2.1824382201884873e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001046945108100772, "evidence_weights": [-0.19509875774383545, -0.3039701282978058, 1.1164703369140625], "agreement_weight": 0.11098780483007431, "orthogonal_weight": -0.7044259905815125}, {"head": 2, "memory_to_active": 0.679121732711792, "contribution_to_active": 0.09846298396587372, "mix_mean": 0.05013711750507355, "mix_low": 0.23554611206054688, "mix_high": 0.0, "read_cosine": 0.25351423025131226, "beta_mean": 0.02044041082262993, "nonfinite": 0.0, "weighted_memory_to_active": 0.06042378395795822, "mix_std": 0.07685934007167816, "beta_std": 0.02542278729379177, "beta_low": 0.3187255859375, "beta_high": 0.0, "orthogonal_to_active": 0.008784414269030094, "evidence_weights": [0.06639786064624786, -0.05892394855618477, 0.35839101672172546], "agreement_weight": 0.14557981491088867, "orthogonal_weight": -0.17656275629997253}, {"head": 3, "memory_to_active": 0.14265862107276917, "contribution_to_active": 0.704055905342102, "mix_mean": 0.6712753176689148, "mix_low": 0.0046787261962890625, "mix_high": 0.023065567016601562, "read_cosine": 0.45666995644569397, "beta_mean": 0.24117189645767212, "nonfinite": 0.0, "weighted_memory_to_active": 0.10426931828260422, "mix_std": 0.3202323615550995, "beta_std": 0.26909780502319336, "beta_low": 0.19964599609375, "beta_high": 0.0, "orthogonal_to_active": 0.06143148988485336, "evidence_weights": [-0.16974548995494843, -0.32082632184028625, 0.1523521989583969], "agreement_weight": -0.009062433615326881, "orthogonal_weight": 0.6712108850479126}, {"head": 4, "memory_to_active": 0.5399352312088013, "contribution_to_active": 1.0650490522384644, "mix_mean": 0.7490748763084412, "mix_low": 0.0005903244018554688, "mix_high": 0.08102989196777344, "read_cosine": 0.11753562092781067, "beta_mean": 0.13297805190086365, "nonfinite": 0.0, "weighted_memory_to_active": 0.4905168414115906, "mix_std": 0.2990320324897766, "beta_std": 0.1471531093120575, "beta_low": 0.3109130859375, "beta_high": 0.0, "orthogonal_to_active": 0.21478454768657684, "evidence_weights": [-0.1880505234003067, -0.5418404340744019, -0.5966922044754028], "agreement_weight": -0.01677583158016205, "orthogonal_weight": 0.46209171414375305}, {"head": 5, "memory_to_active": 0.027065575122833252, "contribution_to_active": 0.6318272352218628, "mix_mean": 0.6890024542808533, "mix_low": 0.00017070770263671875, "mix_high": 0.007735252380371094, "read_cosine": 0.28736910223960876, "beta_mean": 0.0001390388497384265, "nonfinite": 0.0, "weighted_memory_to_active": 0.01958370953798294, "mix_std": 0.2427043616771698, "beta_std": 0.00032647023908793926, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.01900434121489525, "evidence_weights": [-0.23505209386348724, -0.3166305720806122, 0.36606061458587646], "agreement_weight": 0.2118232101202011, "orthogonal_weight": -1.0413504838943481}]}, {"layer": 7, "memory_to_active": 1.73141348361969, "contribution_to_active": 0.2691456377506256, "mix_mean": 0.1737229973077774, "mix_low": 0.22827959060668945, "mix_high": 0.0011019706726074219, "read_cosine": 0.229171484708786, "beta_mean": 0.137623131275177, "nonfinite": 0.0, "weighted_memory_to_active": 0.20437905192375183, "mix_std": 0.14897029101848602, "beta_std": 0.09774813055992126, "beta_low": 0.2170918881893158, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.08681349456310272, "output_influence": 0.2885451912879944, "residual_influence": 0.06975827366113663, "state_active": 0.01317108329385519, "state_memory": 0.024277057498693466, "state_peak": 35.90825653076172, "growth_active": 1.107975959777832, "growth_memory": 1.2763150930404663, "heads": [{"head": 0, "memory_to_active": 2.4688355922698975, "contribution_to_active": 0.17851758003234863, "mix_mean": 0.039940815418958664, "mix_low": 0.40342235565185547, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3297383189201355, "beta_mean": 0.17217229306697845, "nonfinite": 0.0, "weighted_memory_to_active": 0.19895769655704498, "mix_std": 0.08178174495697021, "beta_std": 0.12737567722797394, "beta_low": 0.00274658203125, "beta_high": 0.00048828125, "orthogonal_to_active": 0.0002702973724808544, "evidence_weights": [0.042836010456085205, -0.088811956346035, 0.016507992520928383], "agreement_weight": -0.029372751712799072, "orthogonal_weight": -0.0015122325858101249}, {"head": 1, "memory_to_active": 1.0019444227218628, "contribution_to_active": 0.4116321802139282, "mix_mean": 0.2207452654838562, "mix_low": 0.1154642105102539, "mix_high": 0.005906105041503906, "read_cosine": 0.39551621675491333, "beta_mean": 0.09645889699459076, "nonfinite": 0.0, "weighted_memory_to_active": 0.35248464345932007, "mix_std": 0.2460298240184784, "beta_std": 0.06521350145339966, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04917924851179123, "evidence_weights": [-0.2152743637561798, -0.10208991169929504, -0.2955922782421112], "agreement_weight": 0.0524924136698246, "orthogonal_weight": 0.16773290932178497}, {"head": 2, "memory_to_active": 0.6168976426124573, "contribution_to_active": 0.3785727024078369, "mix_mean": 0.23050040006637573, "mix_low": 0.018035888671875, "mix_high": 0.0003757476806640625, "read_cosine": 0.12152785062789917, "beta_mean": 0.038175228983163834, "nonfinite": 0.0, "weighted_memory_to_active": 0.2366122007369995, "mix_std": 0.2145918607711792, "beta_std": 0.04098641499876976, "beta_low": 0.15386962890625, "beta_high": 0.0, "orthogonal_to_active": 0.13664431869983673, "evidence_weights": [-0.09259084612131119, -0.1796991229057312, 0.8556746244430542], "agreement_weight": 0.01824500970542431, "orthogonal_weight": -0.5948481559753418}, {"head": 3, "memory_to_active": 4.763991832733154, "contribution_to_active": 0.09771199524402618, "mix_mean": 0.03531336784362793, "mix_low": 0.3219261169433594, "mix_high": 0.0, "read_cosine": 0.07283440232276917, "beta_mean": 0.45652735233306885, "nonfinite": 0.0, "weighted_memory_to_active": 0.3456403315067291, "mix_std": 0.05127589777112007, "beta_std": 0.28366124629974365, "beta_low": 0.0018310546875, "beta_high": 0.0013427734375, "orthogonal_to_active": 0.29839208722114563, "evidence_weights": [0.07255134731531143, -0.04048603028059006, 0.7147502303123474], "agreement_weight": 0.049659501761198044, "orthogonal_weight": -0.9375627636909485}, {"head": 4, "memory_to_active": 1.4935963153839111, "contribution_to_active": 0.09857065975666046, "mix_mean": 0.02401934750378132, "mix_low": 0.5089015960693359, "mix_high": 0.0, "read_cosine": 0.23704084753990173, "beta_mean": 0.062233686447143555, "nonfinite": 0.0, "weighted_memory_to_active": 0.07073799520730972, "mix_std": 0.047269631177186966, "beta_std": 0.06877736747264862, "beta_low": 0.14434814453125, "beta_high": 0.0, "orthogonal_to_active": 0.015184944495558739, "evidence_weights": [0.12421902269124985, -0.02456505596637726, 0.18028421700000763], "agreement_weight": 0.03447190672159195, "orthogonal_weight": -0.2877142131328583}, {"head": 5, "memory_to_active": 0.043214209377765656, "contribution_to_active": 0.4498686194419861, "mix_mean": 0.49181875586509705, "mix_low": 0.0019273757934570312, "mix_high": 0.00032901763916015625, "read_cosine": 0.21837127208709717, "beta_mean": 0.0001713507081149146, "nonfinite": 0.0, "weighted_memory_to_active": 0.02184136211872101, "mix_std": 0.2528727948665619, "beta_std": 0.0004745562036987394, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.021210050210356712, "evidence_weights": [-0.18080498278141022, -0.23678477108478546, 0.5058289170265198], "agreement_weight": 0.22852030396461487, "orthogonal_weight": -1.0192716121673584}]}, {"layer": 8, "memory_to_active": 0.14836449921131134, "contribution_to_active": 0.575013279914856, "mix_mean": 0.4986494183540344, "mix_low": 0.010505994781851768, "mix_high": 0.0018043519230559468, "read_cosine": 0.41202691197395325, "beta_mean": 0.014180818572640419, "nonfinite": 0.0, "weighted_memory_to_active": 0.07787223905324936, "mix_std": 0.2429027557373047, "beta_std": 0.021303508430719376, "beta_low": 0.7858175039291382, "beta_high": 0.0, "orthogonal_to_active": 0.07238803058862686, "output_influence": 0.5216594934463501, "residual_influence": 0.18992207944393158, "state_active": 0.04313630238175392, "state_memory": 0.0051391590386629105, "state_peak": 59.74006652832031, "growth_active": 1.1004835367202759, "growth_memory": 1.0381779670715332, "heads": [{"head": 0, "memory_to_active": 0.06464563310146332, "contribution_to_active": 0.5765081644058228, "mix_mean": 0.6059492826461792, "mix_low": 0.0005540847778320312, "mix_high": 0.0015773773193359375, "read_cosine": 0.46610763669013977, "beta_mean": 0.004376235418021679, "nonfinite": 0.0, "weighted_memory_to_active": 0.04624414071440697, "mix_std": 0.24915461242198944, "beta_std": 0.01788579300045967, "beta_low": 0.9139404296875, "beta_high": 0.0, "orthogonal_to_active": 0.028829939663410187, "evidence_weights": [-0.15922631323337555, -0.3807946443557739, 0.5835248827934265], "agreement_weight": 0.25952184200286865, "orthogonal_weight": -0.7996332049369812}, {"head": 1, "memory_to_active": 0.08390837162733078, "contribution_to_active": 0.551878809928894, "mix_mean": 0.5240219831466675, "mix_low": 8.20159912109375e-05, "mix_high": 0.0013294219970703125, "read_cosine": 0.1405431181192398, "beta_mean": 0.0010957063641399145, "nonfinite": 0.0, "weighted_memory_to_active": 0.03656068444252014, "mix_std": 0.23347996175289154, "beta_std": 0.008609882555902004, "beta_low": 0.9901123046875, "beta_high": 0.0, "orthogonal_to_active": 0.026964010670781136, "evidence_weights": [-0.1934596747159958, -0.27455881237983704, 0.6363605856895447], "agreement_weight": 0.06960970163345337, "orthogonal_weight": -0.7709293365478516}, {"head": 2, "memory_to_active": 0.013634700328111649, "contribution_to_active": 0.6861757040023804, "mix_mean": 0.6828129291534424, "mix_low": 0.00040531158447265625, "mix_high": 0.00360870361328125, "read_cosine": 0.4076577425003052, "beta_mean": 0.00012193258589832112, "nonfinite": 0.0, "weighted_memory_to_active": 0.010674014687538147, "mix_std": 0.22219544649124146, "beta_std": 0.0028881405014544725, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.006374748423695564, "evidence_weights": [-0.22347711026668549, -0.3467515707015991, 1.0019551515579224], "agreement_weight": 0.30131661891937256, "orthogonal_weight": -0.6909499168395996}, {"head": 3, "memory_to_active": 0.09668862074613571, "contribution_to_active": 0.6533007621765137, "mix_mean": 0.5826210975646973, "mix_low": 0.0005865097045898438, "mix_high": 0.004027366638183594, "read_cosine": 0.8022102117538452, "beta_mean": 0.007350520230829716, "nonfinite": 0.0, "weighted_memory_to_active": 0.07329200208187103, "mix_std": 0.2553211450576782, "beta_std": 0.007319663651287556, "beta_low": 0.7618408203125, "beta_high": 0.0, "orthogonal_to_active": 0.02227909117937088, "evidence_weights": [-0.17705699801445007, -0.2990504801273346, 0.5377033948898315], "agreement_weight": 0.2457406371831894, "orthogonal_weight": -0.6021900177001953}, {"head": 4, "memory_to_active": 0.24684923887252808, "contribution_to_active": 0.32222694158554077, "mix_mean": 0.3219068646430969, "mix_low": 0.0022993087768554688, "mix_high": 9.5367431640625e-06, "read_cosine": 0.16276349127292633, "beta_mean": 0.0008734222501516342, "nonfinite": 0.0, "weighted_memory_to_active": 0.09906993806362152, "mix_std": 0.22569410502910614, "beta_std": 0.012488702312111855, "beta_low": 0.9923095703125, "beta_high": 0.0, "orthogonal_to_active": 0.021508971229195595, "evidence_weights": [-0.12936387956142426, -0.19486401975154877, 0.6584851741790771], "agreement_weight": 0.0518561415374279, "orthogonal_weight": -0.22303064167499542}, {"head": 5, "memory_to_active": 0.38446035981178284, "contribution_to_active": 0.6599892377853394, "mix_mean": 0.2745841145515442, "mix_low": 0.059108734130859375, "mix_high": 0.00027370452880859375, "read_cosine": 0.49287912249565125, "beta_mean": 0.07126709818840027, "nonfinite": 0.0, "weighted_memory_to_active": 0.20139263570308685, "mix_std": 0.27157115936279297, "beta_std": 0.07862886786460876, "beta_low": 0.0567626953125, "beta_high": 0.0, "orthogonal_to_active": 0.32837140560150146, "evidence_weights": [-0.14568090438842773, -0.1454925388097763, -0.10546011477708817], "agreement_weight": -0.003490972798317671, "orthogonal_weight": 1.9260520935058594}]}, {"layer": 9, "memory_to_active": 0.8659785985946655, "contribution_to_active": 0.6850317716598511, "mix_mean": 0.38520193099975586, "mix_low": 0.06363026797771454, "mix_high": 0.006333033554255962, "read_cosine": 0.35102221369743347, "beta_mean": 0.04004998505115509, "nonfinite": 0.0, "weighted_memory_to_active": 0.36520522832870483, "mix_std": 0.24222403764724731, "beta_std": 0.03312254697084427, "beta_low": 0.601348876953125, "beta_high": 0.0, "orthogonal_to_active": 0.03489207476377487, "output_influence": 0.5573987364768982, "residual_influence": 0.16522035002708435, "state_active": 0.042011063545942307, "state_memory": 0.015243982896208763, "state_peak": 53.46237564086914, "growth_active": 1.1761125326156616, "growth_memory": 1.2911081314086914, "heads": [{"head": 0, "memory_to_active": 0.000566915434319526, "contribution_to_active": 0.5747243165969849, "mix_mean": 0.655343770980835, "mix_low": 0.000988006591796875, "mix_high": 0.02596759796142578, "read_cosine": 0.33147501945495605, "beta_mean": 1.5573191376461182e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0003740960091818124, "mix_std": 0.2850101590156555, "beta_std": 5.591942681348883e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00039479508996009827, "evidence_weights": [-0.3065492510795593, -0.314790278673172, 0.42661187052726746], "agreement_weight": 0.11191221326589584, "orthogonal_weight": -1.1121565103530884}, {"head": 1, "memory_to_active": 0.018586769700050354, "contribution_to_active": 0.7052302956581116, "mix_mean": 0.5750023126602173, "mix_low": 0.007122039794921875, "mix_high": 0.002803802490234375, "read_cosine": 0.4390164613723755, "beta_mean": 4.5570188376586884e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014004005119204521, "mix_std": 0.28074729442596436, "beta_std": 9.901952580548823e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009334742091596127, "evidence_weights": [-0.23859569430351257, -0.29502585530281067, 0.9862239360809326], "agreement_weight": 0.5143287777900696, "orthogonal_weight": -0.8281587958335876}, {"head": 2, "memory_to_active": 0.17341765761375427, "contribution_to_active": 0.6606183648109436, "mix_mean": 0.6045097708702087, "mix_low": 0.0013246536254882812, "mix_high": 0.007525444030761719, "read_cosine": 0.3721228837966919, "beta_mean": 0.0007788858492858708, "nonfinite": 0.0, "weighted_memory_to_active": 0.11904557049274445, "mix_std": 0.2657106816768646, "beta_std": 0.0047367531806230545, "beta_low": 0.99432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.06913751363754272, "evidence_weights": [-0.20420555770397186, -0.27859437465667725, 0.7603793144226074], "agreement_weight": 0.33478039503097534, "orthogonal_weight": -0.682161808013916}, {"head": 3, "memory_to_active": 0.8418700098991394, "contribution_to_active": 0.24563300609588623, "mix_mean": 0.14666834473609924, "mix_low": 0.11483192443847656, "mix_high": 0.0001277923583984375, "read_cosine": 0.38288384675979614, "beta_mean": 0.03653206676244736, "nonfinite": 0.0, "weighted_memory_to_active": 0.19890762865543365, "mix_std": 0.19582098722457886, "beta_std": 0.038281507790088654, "beta_low": 0.148193359375, "beta_high": 0.0, "orthogonal_to_active": 0.08104348182678223, "evidence_weights": [-0.017321061342954636, -0.08914276212453842, 0.37839028239250183], "agreement_weight": 0.09462856501340866, "orthogonal_weight": -0.4530850648880005}, {"head": 4, "memory_to_active": 2.988788366317749, "contribution_to_active": 1.5777846574783325, "mix_mean": 0.146462544798851, "mix_low": 0.12591552734375, "mix_high": 0.0010833740234375, "read_cosine": 0.09976233541965485, "beta_mean": 0.1673259735107422, "nonfinite": 0.0, "weighted_memory_to_active": 1.514115333557129, "mix_std": 0.20844556391239166, "beta_std": 0.09230008721351624, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.014080760069191456, "evidence_weights": [-0.08114870637655258, -0.15230132639408112, -0.039049163460731506], "agreement_weight": -0.0639895498752594, "orthogonal_weight": -0.009702998213469982}, {"head": 5, "memory_to_active": 1.1726417541503906, "contribution_to_active": 0.34619954228401184, "mix_mean": 0.18322449922561646, "mix_low": 0.13159942626953125, "mix_high": 0.0004901885986328125, "read_cosine": 0.48087263107299805, "beta_mean": 0.03561585396528244, "nonfinite": 0.0, "weighted_memory_to_active": 0.3447846472263336, "mix_std": 0.21760942041873932, "beta_std": 0.06331232190132141, "beta_low": 0.465576171875, "beta_high": 0.0, "orthogonal_to_active": 0.035361163318157196, "evidence_weights": [0.06228933110833168, -0.1715565323829651, 0.1455795168876648], "agreement_weight": 0.008545050397515297, "orthogonal_weight": 0.13133525848388672}]}, {"layer": 10, "memory_to_active": 0.21863698959350586, "contribution_to_active": 0.7524011135101318, "mix_mean": 0.528491735458374, "mix_low": 0.022849082946777344, "mix_high": 0.004855950828641653, "read_cosine": 0.380948543548584, "beta_mean": 0.045875124633312225, "nonfinite": 0.0, "weighted_memory_to_active": 0.09837304055690765, "mix_std": 0.26887571811676025, "beta_std": 0.046036045998334885, "beta_low": 0.6599935293197632, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.28385859727859497, "output_influence": 0.4739466905593872, "residual_influence": 0.16841275990009308, "state_active": 0.05293487384915352, "state_memory": 0.014590015634894371, "state_peak": 128.0378875732422, "growth_active": 1.0362341403961182, "growth_memory": 1.0220308303833008, "heads": [{"head": 0, "memory_to_active": 0.7982149124145508, "contribution_to_active": 2.1731114387512207, "mix_mean": 0.5581178665161133, "mix_low": 0.10870647430419922, "mix_high": 0.006173133850097656, "read_cosine": 0.09295076131820679, "beta_mean": 0.20407402515411377, "nonfinite": 0.0, "weighted_memory_to_active": 0.45954838395118713, "mix_std": 0.39188769459724426, "beta_std": 0.2092476487159729, "beta_low": 0.1318359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.6051239967346191, "evidence_weights": [-0.08509869128465652, -0.3713260293006897, -1.886783480644226], "agreement_weight": 0.023120716214179993, "orthogonal_weight": 3.519263744354248}, {"head": 1, "memory_to_active": 0.011634179390966892, "contribution_to_active": 0.6203365325927734, "mix_mean": 0.6304189562797546, "mix_low": 0.0012025833129882812, "mix_high": 0.01657581329345703, "read_cosine": 0.3214957118034363, "beta_mean": 7.962949894135818e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006667983718216419, "mix_std": 0.2654423415660858, "beta_std": 0.0001395388535456732, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010543578304350376, "evidence_weights": [-0.2505352199077606, -0.19470137357711792, 0.568061113357544], "agreement_weight": 0.6190879344940186, "orthogonal_weight": -1.7228275537490845}, {"head": 2, "memory_to_active": 0.012165086343884468, "contribution_to_active": 0.6216029524803162, "mix_mean": 0.6656386852264404, "mix_low": 0.00017452239990234375, "mix_high": 0.004916191101074219, "read_cosine": 0.03688724339008331, "beta_mean": 2.546028554206714e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005857307929545641, "mix_std": 0.24779662489891052, "beta_std": 9.70493201748468e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024047279730439186, "evidence_weights": [-0.2670130133628845, -0.27835768461227417, 0.6347190141677856], "agreement_weight": -0.20692528784275055, "orthogonal_weight": -0.39888909459114075}, {"head": 3, "memory_to_active": 0.33842211961746216, "contribution_to_active": 0.10482046753168106, "mix_mean": 0.31377848982810974, "mix_low": 0.024636268615722656, "mix_high": 0.00016498565673828125, "read_cosine": 0.9183971881866455, "beta_mean": 0.0645163506269455, "nonfinite": 0.0, "weighted_memory_to_active": 0.04055025801062584, "mix_std": 0.23679065704345703, "beta_std": 0.05867650359869003, "beta_low": 0.00128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.0437682680785656, "evidence_weights": [-0.09609026461839676, -0.1454308182001114, 0.1673051118850708], "agreement_weight": 0.017712537199258804, "orthogonal_weight": 1.4415128231048584}, {"head": 4, "memory_to_active": 0.02396026812493801, "contribution_to_active": 0.567299485206604, "mix_mean": 0.595178484916687, "mix_low": 0.0002384185791015625, "mix_high": 0.0013017654418945312, "read_cosine": 0.0651547759771347, "beta_mean": 6.597152241738513e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014280700124800205, "mix_std": 0.2440476268529892, "beta_std": 0.00011764783994294703, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.020488817244768143, "evidence_weights": [-0.18531781435012817, -0.2559037208557129, 0.9651929140090942], "agreement_weight": 0.1735442727804184, "orthogonal_weight": -1.5094950199127197}, {"head": 5, "memory_to_active": 0.12742526829242706, "contribution_to_active": 0.4272354245185852, "mix_mean": 0.4078177511692047, "mix_low": 0.00213623046875, "mix_high": 3.814697265625e-06, "read_cosine": 0.8508055210113525, "beta_mean": 0.006489296909421682, "nonfinite": 0.0, "weighted_memory_to_active": 0.06333357840776443, "mix_std": 0.2272893786430359, "beta_std": 0.00793786346912384, "beta_low": 0.82684326171875, "beta_high": 0.0, "orthogonal_to_active": 0.02082216925919056, "evidence_weights": [-0.14057186245918274, -0.1651272028684616, 0.2675594985485077], "agreement_weight": 0.09396573901176453, "orthogonal_weight": -0.6689597964286804}]}, {"layer": 11, "memory_to_active": 0.07874515652656555, "contribution_to_active": 0.6246529221534729, "mix_mean": 0.5231391191482544, "mix_low": 0.11359167098999023, "mix_high": 0.04493824765086174, "read_cosine": 0.3445854187011719, "beta_mean": 0.023806128650903702, "nonfinite": 0.0, "weighted_memory_to_active": 0.04278816655278206, "mix_std": 0.32434356212615967, "beta_std": 0.03560935705900192, "beta_low": 0.70635986328125, "beta_high": 0.0, "orthogonal_to_active": 0.04726151376962662, "output_influence": 0.5650195479393005, "residual_influence": 0.15722663700580597, "state_active": 0.2071736603975296, "state_memory": 0.0034094389993697405, "state_peak": 38.09720993041992, "growth_active": 1.0780956745147705, "growth_memory": 1.4473711252212524, "heads": [{"head": 0, "memory_to_active": 0.004863742273300886, "contribution_to_active": 0.6053892374038696, "mix_mean": 0.6536575555801392, "mix_low": 0.0008602142333984375, "mix_high": 0.008286476135253906, "read_cosine": 0.3102782368659973, "beta_mean": 5.8708188589662313e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035189215559512377, "mix_std": 0.25383496284484863, "beta_std": 0.0027139014564454556, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0028766284231096506, "evidence_weights": [-0.17568367719650269, -0.36036747694015503, 0.7853963375091553], "agreement_weight": 0.3773527145385742, "orthogonal_weight": -0.856630265712738}, {"head": 1, "memory_to_active": 0.1354769766330719, "contribution_to_active": 0.43271422386169434, "mix_mean": 0.4735162556171417, "mix_low": 0.00455474853515625, "mix_high": 0.000560760498046875, "read_cosine": 0.5089896321296692, "beta_mean": 0.0038358448073267937, "nonfinite": 0.0, "weighted_memory_to_active": 0.06562922894954681, "mix_std": 0.269631952047348, "beta_std": 0.012319709174335003, "beta_low": 0.92755126953125, "beta_high": 0.0, "orthogonal_to_active": 0.036301691085100174, "evidence_weights": [-0.1479146033525467, -0.2723362445831299, 0.5960802435874939], "agreement_weight": 0.0780753642320633, "orthogonal_weight": -0.6602912545204163}, {"head": 2, "memory_to_active": 0.005538091994822025, "contribution_to_active": 0.9872159361839294, "mix_mean": 0.4366445541381836, "mix_low": 0.3166389465332031, "mix_high": 0.1698760986328125, "read_cosine": 0.14884744584560394, "beta_mean": 0.06440797448158264, "nonfinite": 0.0, "weighted_memory_to_active": 0.003907807637006044, "mix_std": 0.4506514370441437, "beta_std": 0.08859841525554657, "beta_low": 0.48291015625, "beta_high": 0.0, "orthogonal_to_active": 0.038560837507247925, "evidence_weights": [0.03792642429471016, -0.3618094325065613, -1.3270423412322998], "agreement_weight": -0.00601926539093256, "orthogonal_weight": 10.26867389678955}, {"head": 3, "memory_to_active": 0.00935918278992176, "contribution_to_active": 0.8404420018196106, "mix_mean": 0.4101138114929199, "mix_low": 0.3515634536743164, "mix_high": 0.08260536193847656, "read_cosine": 0.5415579080581665, "beta_mean": 0.049456991255283356, "nonfinite": 0.0, "weighted_memory_to_active": 0.005656772758811712, "mix_std": 0.4395865797996521, "beta_std": 0.07629664987325668, "beta_low": 0.526611328125, "beta_high": 0.0, "orthogonal_to_active": 0.045084938406944275, "evidence_weights": [0.2103094458580017, -0.2854733169078827, -0.1179649755358696], "agreement_weight": 0.055393148213624954, "orthogonal_weight": 10.530304908752441}, {"head": 4, "memory_to_active": 0.2290535271167755, "contribution_to_active": 0.4600977301597595, "mix_mean": 0.6045567989349365, "mix_low": 0.006213188171386719, "mix_high": 0.0018415451049804688, "read_cosine": 0.027875768020749092, "beta_mean": 0.0162814911454916, "nonfinite": 0.0, "weighted_memory_to_active": 0.13171732425689697, "mix_std": 0.2719084620475769, "beta_std": 0.019876498728990555, "beta_low": 0.51837158203125, "beta_high": 0.0, "orthogonal_to_active": 0.12982800602912903, "evidence_weights": [-0.23323345184326172, -0.23778314888477325, 1.8782964944839478], "agreement_weight": -0.21699194610118866, "orthogonal_weight": -0.9927965998649597}, {"head": 5, "memory_to_active": 0.08817937970161438, "contribution_to_active": 0.4220583438873291, "mix_mean": 0.5603458881378174, "mix_low": 0.0017194747924804688, "mix_high": 0.006459236145019531, "read_cosine": 0.529963493347168, "beta_mean": 0.00879575777798891, "nonfinite": 0.0, "weighted_memory_to_active": 0.046298932284116745, "mix_std": 0.2604479491710663, "beta_std": 0.013850943185389042, "beta_low": 0.7828369140625, "beta_high": 0.0, "orthogonal_to_active": 0.030916975811123848, "evidence_weights": [-0.2008524090051651, -0.3113621473312378, 0.6591458320617676], "agreement_weight": 0.08153236657381058, "orthogonal_weight": -0.859961748123169}]}, {"layer": 12, "memory_to_active": 0.21628867089748383, "contribution_to_active": 0.5922899842262268, "mix_mean": 0.5501463413238525, "mix_low": 0.00822051428258419, "mix_high": 0.0065547628328204155, "read_cosine": 0.21410983800888062, "beta_mean": 0.015681525692343712, "nonfinite": 0.0, "weighted_memory_to_active": 0.11447153985500336, "mix_std": 0.2606174647808075, "beta_std": 0.014952097088098526, "beta_low": 0.8262940049171448, "beta_high": 0.0, "orthogonal_to_active": 0.0721886083483696, "output_influence": 0.5121657848358154, "residual_influence": 0.15031768381595612, "state_active": 0.033193040639162064, "state_memory": 0.008287080563604832, "state_peak": 71.7588882446289, "growth_active": 1.1358122825622559, "growth_memory": 1.1332722902297974, "heads": [{"head": 0, "memory_to_active": 0.039259541779756546, "contribution_to_active": 0.6216860413551331, "mix_mean": 0.5855574607849121, "mix_low": 0.000396728515625, "mix_high": 0.00531005859375, "read_cosine": 0.28448307514190674, "beta_mean": 0.000866851129103452, "nonfinite": 0.0, "weighted_memory_to_active": 0.022319037467241287, "mix_std": 0.2626255452632904, "beta_std": 0.0019447552040219307, "beta_low": 0.99066162109375, "beta_high": 0.0, "orthogonal_to_active": 0.020483434200286865, "evidence_weights": [-0.2077292650938034, -0.2757221460342407, -0.04138300195336342], "agreement_weight": 0.10570136457681656, "orthogonal_weight": -0.9883934855461121}, {"head": 1, "memory_to_active": 0.07236586511135101, "contribution_to_active": 0.5518676042556763, "mix_mean": 0.5570236444473267, "mix_low": 0.0003204345703125, "mix_high": 0.0023374557495117188, "read_cosine": 0.4069855809211731, "beta_mean": 0.0004972735187038779, "nonfinite": 0.0, "weighted_memory_to_active": 0.04496483504772186, "mix_std": 0.2653428316116333, "beta_std": 0.007443808950483799, "beta_low": 0.99761962890625, "beta_high": 0.0, "orthogonal_to_active": 0.03781156614422798, "evidence_weights": [-0.20068159699440002, -0.2714603841304779, 1.0420620441436768], "agreement_weight": 0.2668913006782532, "orthogonal_weight": -0.9266281127929688}, {"head": 2, "memory_to_active": 0.6126704216003418, "contribution_to_active": 0.5412356853485107, "mix_mean": 0.34093064069747925, "mix_low": 0.025671958923339844, "mix_high": 0.0001964569091796875, "read_cosine": 0.5433589220046997, "beta_mean": 0.09138160198926926, "nonfinite": 0.0, "weighted_memory_to_active": 0.26517152786254883, "mix_std": 0.24540391564369202, "beta_std": 0.06397950649261475, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.2900698184967041, "evidence_weights": [-0.20634788274765015, -0.05994737520813942, -0.021614830940961838], "agreement_weight": 0.025001300498843193, "orthogonal_weight": 1.4093443155288696}, {"head": 3, "memory_to_active": 0.027902014553546906, "contribution_to_active": 0.7639048099517822, "mix_mean": 0.727348804473877, "mix_low": 9.632110595703125e-05, "mix_high": 0.020064353942871094, "read_cosine": 0.01861950382590294, "beta_mean": 0.0002339168859180063, "nonfinite": 0.0, "weighted_memory_to_active": 0.019111312925815582, "mix_std": 0.2254500836133957, "beta_std": 0.004210156388580799, "beta_low": 0.99822998046875, "beta_high": 0.0, "orthogonal_to_active": 0.01799621619284153, "evidence_weights": [-0.28910475969314575, -0.33613914251327515, 1.6476342678070068], "agreement_weight": 0.5856634974479675, "orthogonal_weight": -0.9586896896362305}, {"head": 4, "memory_to_active": 0.46986034512519836, "contribution_to_active": 0.5828168988227844, "mix_mean": 0.4554395079612732, "mix_low": 0.0166168212890625, "mix_high": 0.0048656463623046875, "read_cosine": -0.19321228563785553, "beta_mean": 0.0006752836634404957, "nonfinite": 0.0, "weighted_memory_to_active": 0.28476154804229736, "mix_std": 0.2902781069278717, "beta_std": 0.008091874420642853, "beta_low": 0.9820556640625, "beta_high": 0.0, "orthogonal_to_active": 0.028831932693719864, "evidence_weights": [-0.1544562131166458, -0.21444280445575714, -0.131928488612175], "agreement_weight": 0.008305715397000313, "orthogonal_weight": -0.10549827665090561}, {"head": 5, "memory_to_active": 0.07567375898361206, "contribution_to_active": 0.4922288656234741, "mix_mean": 0.6345779895782471, "mix_low": 0.006220817565917969, "mix_high": 0.006554603576660156, "read_cosine": 0.22442412376403809, "beta_mean": 0.00043423226452432573, "nonfinite": 0.0, "weighted_memory_to_active": 0.050500981509685516, "mix_std": 0.27460411190986633, "beta_std": 0.004042476881295443, "beta_low": 0.98870849609375, "beta_high": 0.0, "orthogonal_to_active": 0.037938669323921204, "evidence_weights": [-0.22449339926242828, -0.3152483105659485, 1.2415380477905273], "agreement_weight": 0.4264296293258667, "orthogonal_weight": -0.8110971450805664}]}, {"layer": 13, "memory_to_active": 0.11913909018039703, "contribution_to_active": 0.543156623840332, "mix_mean": 0.544500470161438, "mix_low": 0.006281376350671053, "mix_high": 0.010221004486083984, "read_cosine": 0.398144006729126, "beta_mean": 0.01572439633309841, "nonfinite": 0.0, "weighted_memory_to_active": 0.05356062203645706, "mix_std": 0.26066896319389343, "beta_std": 0.019540898501873016, "beta_low": 0.8071289658546448, "beta_high": 0.0, "orthogonal_to_active": 0.05546034500002861, "output_influence": 0.5393004417419434, "residual_influence": 0.19152873754501343, "state_active": 0.13703905045986176, "state_memory": 0.044367726892232895, "state_peak": 98.0675048828125, "growth_active": 0.9425451755523682, "growth_memory": 0.9639809727668762, "heads": [{"head": 0, "memory_to_active": 0.008970065042376518, "contribution_to_active": 0.6030324101448059, "mix_mean": 0.6156517863273621, "mix_low": 0.0008258819580078125, "mix_high": 0.005188941955566406, "read_cosine": 0.4436262845993042, "beta_mean": 3.176482641720213e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005788963288068771, "mix_std": 0.26827651262283325, "beta_std": 7.038580224616453e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005492410156875849, "evidence_weights": [-0.19230976700782776, -0.266131728887558, 0.7381566762924194], "agreement_weight": 0.18658262491226196, "orthogonal_weight": -1.0588977336883545}, {"head": 1, "memory_to_active": 0.013224855065345764, "contribution_to_active": 0.6021599173545837, "mix_mean": 0.5687479972839355, "mix_low": 0.0015316009521484375, "mix_high": 0.004837989807128906, "read_cosine": 0.21143457293510437, "beta_mean": 5.3641062550013885e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007416266482323408, "mix_std": 0.2775033414363861, "beta_std": 0.00010710788774304092, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009657329879701138, "evidence_weights": [-0.17907260358333588, -0.26307418942451477, 0.564551055431366], "agreement_weight": 0.5242205858230591, "orthogonal_weight": -1.3518844842910767}, {"head": 2, "memory_to_active": 0.0618077889084816, "contribution_to_active": 0.4743020534515381, "mix_mean": 0.3893672227859497, "mix_low": 0.0010480880737304688, "mix_high": 0.0001583099365234375, "read_cosine": 0.5127285122871399, "beta_mean": 0.013637766242027283, "nonfinite": 0.0, "weighted_memory_to_active": 0.029681500047445297, "mix_std": 0.23059424757957458, "beta_std": 0.04995102062821388, "beta_low": 0.8536376953125, "beta_high": 0.0, "orthogonal_to_active": 0.019878888502717018, "evidence_weights": [-0.1818608045578003, -0.21132312715053558, 0.4133293330669403], "agreement_weight": 0.0054010613821446896, "orthogonal_weight": -0.8458181619644165}, {"head": 3, "memory_to_active": 0.17933763563632965, "contribution_to_active": 0.5939007997512817, "mix_mean": 0.6579145789146423, "mix_low": 0.027344703674316406, "mix_high": 0.03797149658203125, "read_cosine": 0.005758633837103844, "beta_mean": 0.001127863535657525, "nonfinite": 0.0, "weighted_memory_to_active": 0.13743974268436432, "mix_std": 0.2901456952095032, "beta_std": 0.009991712868213654, "beta_low": 0.9896240234375, "beta_high": 0.0, "orthogonal_to_active": 0.1201237365603447, "evidence_weights": [-0.31189966201782227, -0.2383248209953308, 1.5314964056015015], "agreement_weight": 6.508906364440918, "orthogonal_weight": -0.9683555364608765}, {"head": 4, "memory_to_active": 0.4020496904850006, "contribution_to_active": 0.2666875123977661, "mix_mean": 0.37653911113739014, "mix_low": 0.004669189453125, "mix_high": 1.621246337890625e-05, "read_cosine": 0.8943073749542236, "beta_mean": 0.07928489148616791, "nonfinite": 0.0, "weighted_memory_to_active": 0.10729261487722397, "mix_std": 0.22498750686645508, "beta_std": 0.05522839352488518, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.14896106719970703, "evidence_weights": [-0.07153188437223434, -0.2305796593427658, 0.4372701942920685], "agreement_weight": 0.0044740657322108746, "orthogonal_weight": 2.278017044067383}, {"head": 5, "memory_to_active": 0.04944445192813873, "contribution_to_active": 0.7188568115234375, "mix_mean": 0.6587820649147034, "mix_low": 0.0022687911987304688, "mix_high": 0.013153076171875, "read_cosine": 0.32100844383239746, "beta_mean": 0.00021044827008154243, "nonfinite": 0.0, "weighted_memory_to_active": 0.033744629472494125, "mix_std": 0.27250638604164124, "beta_std": 0.0018967664800584316, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.028648637235164642, "evidence_weights": [-0.2209240347146988, -0.2938900589942932, 0.8559878468513489], "agreement_weight": 0.19707758724689484, "orthogonal_weight": -0.9312663078308105}]}, {"layer": 14, "memory_to_active": 0.22185452282428741, "contribution_to_active": 0.41762614250183105, "mix_mean": 0.43737468123435974, "mix_low": 0.02477630041539669, "mix_high": 0.006546815391629934, "read_cosine": 0.6223275661468506, "beta_mean": 0.06048205494880676, "nonfinite": 0.0, "weighted_memory_to_active": 0.09530071914196014, "mix_std": 0.24603009223937988, "beta_std": 0.05878988653421402, "beta_low": 0.498260498046875, "beta_high": 0.0, "orthogonal_to_active": 0.042285408824682236, "output_influence": 0.36300384998321533, "residual_influence": 0.1492261290550232, "state_active": 0.05290018022060394, "state_memory": 0.01488665584474802, "state_peak": 44.71249771118164, "growth_active": 1.0740388631820679, "growth_memory": 1.2403041124343872, "heads": [{"head": 0, "memory_to_active": 0.3498305082321167, "contribution_to_active": 0.19359396398067474, "mix_mean": 0.21142300963401794, "mix_low": 0.01629638671875, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8073053359985352, "beta_mean": 0.07174844294786453, "nonfinite": 0.0, "weighted_memory_to_active": 0.0775010958313942, "mix_std": 0.1891690194606781, "beta_std": 0.06209076941013336, "beta_low": 0.003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.09106042236089706, "evidence_weights": [-0.09861782193183899, -0.0468214750289917, 0.3902513086795807], "agreement_weight": 0.021756736561655998, "orthogonal_weight": 1.836029291152954}, {"head": 1, "memory_to_active": 0.4623726010322571, "contribution_to_active": 0.2941073775291443, "mix_mean": 0.22050297260284424, "mix_low": 0.11774921417236328, "mix_high": 1.621246337890625e-05, "read_cosine": 0.8748934864997864, "beta_mean": 0.2457127720117569, "nonfinite": 0.0, "weighted_memory_to_active": 0.21581238508224487, "mix_std": 0.2480226755142212, "beta_std": 0.24534693360328674, "beta_low": 0.1009521484375, "beta_high": 0.0, "orthogonal_to_active": 0.018316730856895447, "evidence_weights": [0.0014305423246696591, -0.1941131055355072, -0.09012530744075775], "agreement_weight": 0.05835297703742981, "orthogonal_weight": 0.2603754997253418}, {"head": 2, "memory_to_active": 0.11748342961072922, "contribution_to_active": 0.5858933925628662, "mix_mean": 0.5971052050590515, "mix_low": 0.0027551651000976562, "mix_high": 0.0058956146240234375, "read_cosine": 0.4045630693435669, "beta_mean": 0.0018711837474256754, "nonfinite": 0.0, "weighted_memory_to_active": 0.07748472690582275, "mix_std": 0.2671516537666321, "beta_std": 0.002425478771328926, "beta_low": 0.9842529296875, "beta_high": 0.0, "orthogonal_to_active": 0.06010835990309715, "evidence_weights": [-0.13315458595752716, -0.31769299507141113, 0.9317336678504944], "agreement_weight": 0.08313353359699249, "orthogonal_weight": -0.8778029680252075}, {"head": 3, "memory_to_active": 0.04652832821011543, "contribution_to_active": 0.5847067832946777, "mix_mean": 0.677300214767456, "mix_low": 0.00449371337890625, "mix_high": 0.02315998077392578, "read_cosine": -0.0062750279903411865, "beta_mean": 0.005604304373264313, "nonfinite": 0.0, "weighted_memory_to_active": 0.02946586161851883, "mix_std": 0.25750505924224854, "beta_std": 0.008402228355407715, "beta_low": 0.84796142578125, "beta_high": 0.0, "orthogonal_to_active": 0.021597621962428093, "evidence_weights": [-0.3418239951133728, -0.31078118085861206, 0.28430235385894775], "agreement_weight": 0.06837203353643417, "orthogonal_weight": -0.823427677154541}, {"head": 4, "memory_to_active": 0.15753361582756042, "contribution_to_active": 0.5051245093345642, "mix_mean": 0.5305052399635315, "mix_low": 0.0036773681640625, "mix_high": 0.008429527282714844, "read_cosine": 0.9073749780654907, "beta_mean": 0.03646988794207573, "nonfinite": 0.0, "weighted_memory_to_active": 0.09411096572875977, "mix_std": 0.25951045751571655, "beta_std": 0.02541860193014145, "beta_low": 0.07781982421875, "beta_high": 0.0, "orthogonal_to_active": 0.02408180944621563, "evidence_weights": [-0.21033035218715668, -0.23849469423294067, 0.44073134660720825], "agreement_weight": 0.13566970825195312, "orthogonal_weight": -0.6150752902030945}, {"head": 5, "memory_to_active": 0.19737869501113892, "contribution_to_active": 0.3423307538032532, "mix_mean": 0.38741135597229004, "mix_low": 0.0036859512329101562, "mix_high": 0.0017747879028320312, "read_cosine": 0.7461034059524536, "beta_mean": 0.0014856939669698477, "nonfinite": 0.0, "weighted_memory_to_active": 0.07742929458618164, "mix_std": 0.2548215687274933, "beta_std": 0.009055275470018387, "beta_low": 0.97491455078125, "beta_high": 0.0, "orthogonal_to_active": 0.03854750469326973, "evidence_weights": [-0.17500600218772888, -0.18089032173156738, 0.35701408982276917], "agreement_weight": 0.06604782491922379, "orthogonal_weight": -0.7503976225852966}]}, {"layer": 15, "memory_to_active": 0.05649187043309212, "contribution_to_active": 0.4897115230560303, "mix_mean": 0.4332931637763977, "mix_low": 0.006989002227783203, "mix_high": 0.0035041174851357937, "read_cosine": 0.6596306562423706, "beta_mean": 0.03846810758113861, "nonfinite": 0.0, "weighted_memory_to_active": 0.028028013184666634, "mix_std": 0.2855684459209442, "beta_std": 0.04340091347694397, "beta_low": 0.70672607421875, "beta_high": 0.0, "orthogonal_to_active": 0.009137691929936409, "output_influence": 0.30844053626060486, "residual_influence": 0.18517053127288818, "state_active": 0.16798058152198792, "state_memory": 0.010725371539592743, "state_peak": 33.42877960205078, "growth_active": 1.2202023267745972, "growth_memory": 1.3924298286437988, "heads": [{"head": 0, "memory_to_active": 0.12156403809785843, "contribution_to_active": 0.48168039321899414, "mix_mean": 0.40674182772636414, "mix_low": 0.007605552673339844, "mix_high": 0.006678581237792969, "read_cosine": 0.6085911989212036, "beta_mean": 0.04605800285935402, "nonfinite": 0.0, "weighted_memory_to_active": 0.06079063564538956, "mix_std": 0.28762155771255493, "beta_std": 0.03366560488939285, "beta_low": 0.0633544921875, "beta_high": 0.0, "orthogonal_to_active": 0.032909829169511795, "evidence_weights": [-0.1917065978050232, -0.19677935540676117, 0.5102981925010681], "agreement_weight": 0.09123831242322922, "orthogonal_weight": -0.8030256628990173}, {"head": 1, "memory_to_active": 0.1083483099937439, "contribution_to_active": 0.42816686630249023, "mix_mean": 0.44146478176116943, "mix_low": 0.001110076904296875, "mix_high": 0.007300376892089844, "read_cosine": 0.9281903505325317, "beta_mean": 0.00031941893394105136, "nonfinite": 0.0, "weighted_memory_to_active": 0.04658345878124237, "mix_std": 0.28789398074150085, "beta_std": 0.002081863349303603, "beta_low": 0.9981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0025795772671699524, "evidence_weights": [-0.15157580375671387, -0.16090968251228333, 0.16547797620296478], "agreement_weight": 0.12085447460412979, "orthogonal_weight": 0.11670956760644913}, {"head": 2, "memory_to_active": 0.0050330436788499355, "contribution_to_active": 0.49061742424964905, "mix_mean": 0.3584029972553253, "mix_low": 0.01032257080078125, "mix_high": 0.0001430511474609375, "read_cosine": 0.7636337280273438, "beta_mean": 6.386648601619527e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002616548677906394, "mix_std": 0.27186092734336853, "beta_std": 0.00014335005835164338, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0008629269432276487, "evidence_weights": [-0.2025197297334671, -0.2645169198513031, 0.10956776887178421], "agreement_weight": 0.005866538733243942, "orthogonal_weight": 0.8729495406150818}, {"head": 3, "memory_to_active": 0.0023897483479231596, "contribution_to_active": 0.37739822268486023, "mix_mean": 0.41279107332229614, "mix_low": 0.003922462463378906, "mix_high": 0.000690460205078125, "read_cosine": 0.6204246282577515, "beta_mean": 1.2596056876645889e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007693368243053555, "mix_std": 0.28904229402542114, "beta_std": 2.886737092921976e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005103402654640377, "evidence_weights": [-0.2721245288848877, -0.2346806675195694, 0.205510675907135], "agreement_weight": 0.059037867933511734, "orthogonal_weight": -0.6433802843093872}, {"head": 4, "memory_to_active": 0.07320355623960495, "contribution_to_active": 0.6297286748886108, "mix_mean": 0.39871227741241455, "mix_low": 0.008024215698242188, "mix_high": 0.0027008056640625, "read_cosine": 0.6888149976730347, "beta_mean": 0.18247471749782562, "nonfinite": 0.0, "weighted_memory_to_active": 0.03949431702494621, "mix_std": 0.3089771568775177, "beta_std": 0.21050801873207092, "beta_low": 0.2054443359375, "beta_high": 0.0, "orthogonal_to_active": 0.006857189815491438, "evidence_weights": [-0.20662936568260193, -0.25417694449424744, -0.021375833079218864], "agreement_weight": 0.012708820402622223, "orthogonal_weight": 0.37482133507728577}, {"head": 5, "memory_to_active": 0.02841249480843544, "contribution_to_active": 0.530677318572998, "mix_mean": 0.5816458463668823, "mix_low": 0.010949134826660156, "mix_high": 0.0035114288330078125, "read_cosine": 0.3481292128562927, "beta_mean": 0.0018800427205860615, "nonfinite": 0.0, "weighted_memory_to_active": 0.017913777381181717, "mix_std": 0.26801469922065735, "beta_std": 0.013977744616568089, "beta_low": 0.973388671875, "beta_high": 0.0, "orthogonal_to_active": 0.011106285266578197, "evidence_weights": [-0.28053370118141174, -0.30298730731010437, 0.6162346601486206], "agreement_weight": 0.3113161623477936, "orthogonal_weight": 0.8205538392066956}]}]}, {"step": 67000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.90268977300002, "memory_to_active": 0.7796476483345032, "contribution_to_active": 0.5703460574150085, "mix_mean": 0.4247451424598694, "mix_low": 0.12479721009731293, "mix_high": 0.010876188986003399, "read_cosine": 0.343526691198349, "beta_mean": 0.06685599684715271, "nonfinite": 0.0, "weighted_memory_to_active": 0.19883325695991516, "mix_std": 0.2237929105758667, "beta_std": 0.05921817943453789, "beta_low": 0.5894553661346436, "beta_high": 4.831949627259746e-05, "orthogonal_to_active": 0.06412878632545471, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0438387393951416, "normal_token_accuracy": 0.45606228709220886, "state_active": 0.06477764248847961, "state_memory": 0.016733556985855103, "state_peak": 125.1505126953125, "growth_active": 1.1350278854370117, "growth_memory": 1.2917907238006592, "output_influence": 0.4393388330936432, "residual_influence": 0.1557299643754959, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1580536365509033, "no_memory_loss_delta": 0.6233888864517212, "scrambled_loss_delta": 0.12207511812448502, "memory_only_loss_delta": 3.3694002628326416, "no_evidence_loss_delta": 0.3648003339767456, "no_agreement_loss_delta": 0.016125300899147987, "no_orthogonal_loss_delta": 0.008531718514859676}, {"index": 94, "tokens": 256, "normal_loss": 3.159374713897705, "no_memory_loss_delta": 0.6784281730651855, "scrambled_loss_delta": 0.3915952742099762, "memory_only_loss_delta": 3.4754538536071777, "no_evidence_loss_delta": 0.3219824731349945, "no_agreement_loss_delta": 0.04103792458772659, "no_orthogonal_loss_delta": 0.03871428221464157}, {"index": 188, "tokens": 256, "normal_loss": 3.353807210922241, "no_memory_loss_delta": 0.7962705492973328, "scrambled_loss_delta": 0.10339844226837158, "memory_only_loss_delta": 3.3094725608825684, "no_evidence_loss_delta": 0.4074873924255371, "no_agreement_loss_delta": 0.0513458289206028, "no_orthogonal_loss_delta": 0.040276020765304565}, {"index": 283, "tokens": 131, "normal_loss": 1.9891215562820435, "no_memory_loss_delta": 0.31840506196022034, "scrambled_loss_delta": 0.20257611572742462, "memory_only_loss_delta": 2.6856396198272705, "no_evidence_loss_delta": 0.10419285297393799, "no_agreement_loss_delta": -0.0432010255753994, "no_orthogonal_loss_delta": -0.029404276981949806}], "agreement_weight_abs_mean": 0.24037916958332062, "agreement_weight_abs_max": 6.556727886199951, "agreement_weight_mean": 0.20892299711704254, "orthogonal_weight_abs_mean": 0.925297737121582, "orthogonal_weight_abs_max": 10.628381729125977, "orthogonal_weight_mean": -0.0721040591597557, "no_memory_loss": 3.6876893043518066, "no_memory_loss_delta": 0.6438504457473755, "no_memory_token_accuracy_gain": 0.08231367915868759, "no_memory_token_win_fraction": 0.6718575954437256, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 47.473398303985, "scrambled_loss": 3.249074935913086, "scrambled_loss_delta": 0.20523591339588165, "scrambled_token_accuracy_gain": 0.041156839579343796, "scrambled_token_win_fraction": 0.604004442691803, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.554484713557724, "memory_only_loss": 6.326737880706787, "memory_only_loss_delta": 3.2828991413116455, "memory_only_token_accuracy_gain": 0.2914349138736725, "memory_only_token_win_fraction": 0.9199110269546509, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.24806748461927, "no_evidence_loss": 3.370626926422119, "no_evidence_loss_delta": 0.3267880380153656, "no_evidence_token_accuracy_gain": 0.04226918891072273, "no_evidence_token_win_fraction": 0.6106785535812378, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 27.876339843260595, "no_agreement_loss": 3.0684430599212646, "no_agreement_loss_delta": 0.024603987112641335, "no_agreement_token_accuracy_gain": 0.0033370412420481443, "no_agreement_token_win_fraction": 0.47052279114723206, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.4303776190162107, "no_orthogonal_loss": 3.0644772052764893, "no_orthogonal_loss_delta": 0.020638128742575645, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.600667417049408, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.042662011004663, "short_no_memory_loss_delta": 0.5775189399719238, "long_context_memory_gain": 0.06633150577545166, "normal_context_loss_gain": 0.13627299666404724, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.45357608795166, "contribution_to_active": 1.1907672882080078, "mix_mean": 0.10842685401439667, "mix_low": 0.5094419121742249, "mix_high": 0.005194823257625103, "read_cosine": 0.41761156916618347, "beta_mean": 0.2121237814426422, "nonfinite": 0.0, "weighted_memory_to_active": 1.2164807319641113, "mix_std": 0.09249250590801239, "beta_std": 0.13870826363563538, "beta_low": 0.1551717221736908, "beta_high": 0.0, "orthogonal_to_active": 0.04975174367427826, "output_influence": 0.21873852610588074, "residual_influence": 0.2471318393945694, "state_active": 0.06757788360118866, "state_memory": 0.04483925178647041, "state_peak": 46.86319351196289, "growth_active": 1.2540806531906128, "growth_memory": 1.2871912717819214, "heads": [{"head": 0, "memory_to_active": 2.755197286605835, "contribution_to_active": 0.48141956329345703, "mix_mean": 0.020390156656503677, "mix_low": 0.6266517639160156, "mix_high": 0.0, "read_cosine": 0.21422341465950012, "beta_mean": 0.19233593344688416, "nonfinite": 0.0, "weighted_memory_to_active": 0.6040252447128296, "mix_std": 0.06175440549850464, "beta_std": 0.09715817868709564, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06598199903964996, "evidence_weights": [-0.0066864751279354095, -0.046425752341747284, -0.053979188203811646], "agreement_weight": -0.22168777883052826, "orthogonal_weight": -0.12466936558485031}, {"head": 1, "memory_to_active": 6.488575458526611, "contribution_to_active": 2.9070043563842773, "mix_mean": 0.020366093143820763, "mix_low": 0.6141109466552734, "mix_high": 0.0, "read_cosine": 0.43372422456741333, "beta_mean": 0.4522648751735687, "nonfinite": 0.0, "weighted_memory_to_active": 3.1418018341064453, "mix_std": 0.05728096514940262, "beta_std": 0.21552006900310516, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05896180868148804, "evidence_weights": [0.016840005293488503, -0.11130768805742264, 0.6876360774040222], "agreement_weight": 0.025667114183306694, "orthogonal_weight": 0.03886667639017105}, {"head": 2, "memory_to_active": 2.4895386695861816, "contribution_to_active": 0.5805120468139648, "mix_mean": 0.01685653254389763, "mix_low": 0.6110191345214844, "mix_high": 0.0, "read_cosine": 0.3002191185951233, "beta_mean": 0.24065883457660675, "nonfinite": 0.0, "weighted_memory_to_active": 0.6814000606536865, "mix_std": 0.03586375713348389, "beta_std": 0.1928454041481018, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.05154908448457718, "evidence_weights": [0.05768333002924919, 0.04437745362520218, 0.23658743500709534], "agreement_weight": 0.014270277693867683, "orthogonal_weight": -0.2000117301940918}, {"head": 3, "memory_to_active": 7.405695915222168, "contribution_to_active": 2.6357760429382324, "mix_mean": 0.03154268488287926, "mix_low": 0.4868793487548828, "mix_high": 0.0, "read_cosine": 0.3102073073387146, "beta_mean": 0.33842891454696655, "nonfinite": 0.0, "weighted_memory_to_active": 2.776644468307495, "mix_std": 0.07153092324733734, "beta_std": 0.262035995721817, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.10516233742237091, "evidence_weights": [-0.0377206988632679, -0.03340988978743553, -0.13452313840389252], "agreement_weight": -0.01121017150580883, "orthogonal_weight": -0.04507407918572426}, {"head": 4, "memory_to_active": 1.485100269317627, "contribution_to_active": 0.05868139490485191, "mix_mean": 0.01548659335821867, "mix_low": 0.7037887573242188, "mix_high": 0.0, "read_cosine": 0.3167552351951599, "beta_mean": 0.030579911544919014, "nonfinite": 0.0, "weighted_memory_to_active": 0.043234020471572876, "mix_std": 0.04598227143287659, "beta_std": 0.04682094603776932, "beta_low": 0.48675537109375, "beta_high": 0.0, "orthogonal_to_active": 0.001033240812830627, "evidence_weights": [0.11250067502260208, 0.07613059133291245, -0.13046960532665253], "agreement_weight": -0.2012280970811844, "orthogonal_weight": 0.021536927670240402}, {"head": 5, "memory_to_active": 0.09734837710857391, "contribution_to_active": 0.4812098741531372, "mix_mean": 0.5459190607070923, "mix_low": 0.014201164245605469, "mix_high": 0.03116893768310547, "read_cosine": 0.9305398464202881, "beta_mean": 0.018474169075489044, "nonfinite": 0.0, "weighted_memory_to_active": 0.05177803337574005, "mix_std": 0.28254273533821106, "beta_std": 0.01786898262798786, "beta_low": 0.4439697265625, "beta_high": 0.0, "orthogonal_to_active": 0.015821969136595726, "evidence_weights": [-0.1746704876422882, -0.21941448748111725, 0.3779904246330261], "agreement_weight": 0.2901543080806732, "orthogonal_weight": -0.6956440210342407}]}, {"layer": 1, "memory_to_active": 2.37540340423584, "contribution_to_active": 0.47601205110549927, "mix_mean": 0.21474963426589966, "mix_low": 0.36508309841156006, "mix_high": 0.0012466113548725843, "read_cosine": 0.2980489432811737, "beta_mean": 0.17124804854393005, "nonfinite": 0.0, "weighted_memory_to_active": 0.318214476108551, "mix_std": 0.11435838788747787, "beta_std": 0.12679196894168854, "beta_low": 0.3549398183822632, "beta_high": 0.00014241535973269492, "orthogonal_to_active": 0.04004814103245735, "output_influence": 0.26199644804000854, "residual_influence": 0.07296981662511826, "state_active": 0.027546800673007965, "state_memory": 0.03485474735498428, "state_peak": 42.34883117675781, "growth_active": 1.1511873006820679, "growth_memory": 1.4639902114868164, "heads": [{"head": 0, "memory_to_active": 0.023233845829963684, "contribution_to_active": 0.4783892333507538, "mix_mean": 0.5279254913330078, "mix_low": 0.0052814483642578125, "mix_high": 0.0023050308227539062, "read_cosine": 0.029651904478669167, "beta_mean": 0.00016710232011973858, "nonfinite": 0.0, "weighted_memory_to_active": 0.013553381897509098, "mix_std": 0.2805372476577759, "beta_std": 0.00033974595135077834, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.013162494637072086, "evidence_weights": [-0.20709997415542603, -0.27098554372787476, 0.28889885544776917], "agreement_weight": 0.1998739242553711, "orthogonal_weight": -1.0114288330078125}, {"head": 1, "memory_to_active": 3.108527183532715, "contribution_to_active": 0.2945118844509125, "mix_mean": 0.022333204746246338, "mix_low": 0.48307323455810547, "mix_high": 3.814697265625e-06, "read_cosine": 0.4906001091003418, "beta_mean": 0.28159409761428833, "nonfinite": 0.0, "weighted_memory_to_active": 0.31933560967445374, "mix_std": 0.04560026153922081, "beta_std": 0.19475606083869934, "beta_low": 0.0015869140625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.025686990469694138, "evidence_weights": [0.02195572666823864, -0.06539694219827652, 0.049970079213380814], "agreement_weight": -0.0010866892989724874, "orthogonal_weight": 0.21340316534042358}, {"head": 2, "memory_to_active": 0.045196011662483215, "contribution_to_active": 0.5479428768157959, "mix_mean": 0.6756719946861267, "mix_low": 0.00041294097900390625, "mix_high": 0.0051708221435546875, "read_cosine": 0.46684396266937256, "beta_mean": 0.0016327428165823221, "nonfinite": 0.0, "weighted_memory_to_active": 0.030416339635849, "mix_std": 0.2291756570339203, "beta_std": 0.004125645384192467, "beta_low": 0.97833251953125, "beta_high": 0.0, "orthogonal_to_active": 0.021146850660443306, "evidence_weights": [-0.27616074681282043, -0.37110763788223267, 0.6706671118736267], "agreement_weight": 0.0449308380484581, "orthogonal_weight": -0.8089433908462524}, {"head": 3, "memory_to_active": 2.9707350730895996, "contribution_to_active": 0.20501244068145752, "mix_mean": 0.022857138887047768, "mix_low": 0.5458717346191406, "mix_high": 0.0, "read_cosine": 0.18684348464012146, "beta_mean": 0.31539562344551086, "nonfinite": 0.0, "weighted_memory_to_active": 0.27078208327293396, "mix_std": 0.05928456038236618, "beta_std": 0.2526053488254547, "beta_low": 0.06927490234375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.09996441006660461, "evidence_weights": [0.04115796834230423, -0.01593182608485222, -0.017540685832500458], "agreement_weight": -0.02710011787712574, "orthogonal_weight": -0.48963943123817444}, {"head": 4, "memory_to_active": 3.8548693656921387, "contribution_to_active": 1.253374457359314, "mix_mean": 0.03317330777645111, "mix_low": 0.3160085678100586, "mix_high": 0.0, "read_cosine": 0.33458977937698364, "beta_mean": 0.15598005056381226, "nonfinite": 0.0, "weighted_memory_to_active": 1.222179889678955, "mix_std": 0.05376046150922775, "beta_std": 0.11911959946155548, "beta_low": 0.04144287109375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.05829980969429016, "evidence_weights": [-0.02307470329105854, -0.007560285739600658, 0.13107776641845703], "agreement_weight": -0.0023577336687594652, "orthogonal_weight": 0.07176877558231354}, {"head": 5, "memory_to_active": 4.249858856201172, "contribution_to_active": 0.07684144377708435, "mix_mean": 0.006536572240293026, "mix_low": 0.8398504257202148, "mix_high": 0.0, "read_cosine": 0.279764324426651, "beta_mean": 0.27271875739097595, "nonfinite": 0.0, "weighted_memory_to_active": 0.05301938205957413, "mix_std": 0.01779208891093731, "beta_std": 0.18980540335178375, "beta_low": 0.03900146484375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.022028282284736633, "evidence_weights": [0.1596544086933136, 0.0758817046880722, 0.12326548248529434], "agreement_weight": -0.11068159341812134, "orthogonal_weight": 0.3929886221885681}]}, {"layer": 2, "memory_to_active": 0.9885842204093933, "contribution_to_active": 0.3521769642829895, "mix_mean": 0.40922412276268005, "mix_low": 0.2518480718135834, "mix_high": 0.0040531158447265625, "read_cosine": 0.3054756224155426, "beta_mean": 0.07744643837213516, "nonfinite": 0.0, "weighted_memory_to_active": 0.0648915022611618, "mix_std": 0.18536394834518433, "beta_std": 0.0668710470199585, "beta_low": 0.5984599590301514, "beta_high": 0.0004069010610692203, "orthogonal_to_active": 0.016456712037324905, "output_influence": 0.39811602234840393, "residual_influence": 0.11326354742050171, "state_active": 0.041953228414058685, "state_memory": 0.013989459723234177, "state_peak": 27.083660125732422, "growth_active": 1.0660651922225952, "growth_memory": 1.5661970376968384, "heads": [{"head": 0, "memory_to_active": 0.004507047589868307, "contribution_to_active": 0.3565269708633423, "mix_mean": 0.6377493143081665, "mix_low": 0.00060272216796875, "mix_high": 0.007802009582519531, "read_cosine": 0.13962143659591675, "beta_mean": 3.394803206901997e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029266169294714928, "mix_std": 0.26692670583724976, "beta_std": 6.2986058765091e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003079826245084405, "evidence_weights": [-0.3756074607372284, -0.21867850422859192, 0.1953277289867401], "agreement_weight": 0.13203951716423035, "orthogonal_weight": -1.0869139432907104}, {"head": 1, "memory_to_active": 3.0671582221984863, "contribution_to_active": 0.038205478340387344, "mix_mean": 0.0046523939818143845, "mix_low": 0.9023237228393555, "mix_high": 0.0, "read_cosine": 0.1359775960445404, "beta_mean": 0.22158893942832947, "nonfinite": 0.0, "weighted_memory_to_active": 0.031106658279895782, "mix_std": 0.009950798004865646, "beta_std": 0.15213948488235474, "beta_low": 0.0023193359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.009382026270031929, "evidence_weights": [0.14511017501354218, 0.11972691118717194, 0.5769557952880859], "agreement_weight": 0.02372000180184841, "orthogonal_weight": 0.33951929211616516}, {"head": 2, "memory_to_active": 0.007472178433090448, "contribution_to_active": 0.4377688467502594, "mix_mean": 0.549973726272583, "mix_low": 0.0011110305786132812, "mix_high": 0.0030584335327148438, "read_cosine": 0.38724201917648315, "beta_mean": 2.665554711711593e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003045528894290328, "mix_std": 0.2741275429725647, "beta_std": 5.19677996635437e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 3.184817614965141e-05, "evidence_weights": [-0.25251153111457825, -0.2942199409008026, 0.595359742641449], "agreement_weight": -0.021972736343741417, "orthogonal_weight": 0.012158186174929142}, {"head": 3, "memory_to_active": 0.11760005354881287, "contribution_to_active": 0.5539529919624329, "mix_mean": 0.5951256155967712, "mix_low": 0.00020599365234375, "mix_high": 0.007472991943359375, "read_cosine": 0.5905230045318604, "beta_mean": 0.01860734261572361, "nonfinite": 0.0, "weighted_memory_to_active": 0.07543284446001053, "mix_std": 0.2529274821281433, "beta_std": 0.02747323364019394, "beta_low": 0.53802490234375, "beta_high": 0.0, "orthogonal_to_active": 0.05254146084189415, "evidence_weights": [-0.20772816240787506, -0.32965222001075745, 0.7645525932312012], "agreement_weight": 0.05292797088623047, "orthogonal_weight": -0.8898397088050842}, {"head": 4, "memory_to_active": 2.7245473861694336, "contribution_to_active": 0.20860469341278076, "mix_mean": 0.018595751374959946, "mix_low": 0.6062154769897461, "mix_high": 0.0, "read_cosine": 0.20268122851848602, "beta_mean": 0.22438454627990723, "nonfinite": 0.0, "weighted_memory_to_active": 0.2714061141014099, "mix_std": 0.0462271012365818, "beta_std": 0.2214486300945282, "beta_low": 0.0504150390625, "beta_high": 0.00238037109375, "orthogonal_to_active": 0.032280899584293365, "evidence_weights": [0.02866651676595211, 0.008505074307322502, 0.04282350465655327], "agreement_weight": 0.021638453006744385, "orthogonal_weight": -0.1377941220998764}, {"head": 5, "memory_to_active": 0.010220333933830261, "contribution_to_active": 0.5180026888847351, "mix_mean": 0.6492481231689453, "mix_low": 0.000629425048828125, "mix_high": 0.005985260009765625, "read_cosine": 0.3768083453178406, "beta_mean": 3.720304084708914e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0054312460124492645, "mix_std": 0.2620239853858948, "beta_std": 4.9910246161744e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014242102624848485, "evidence_weights": [-0.27961111068725586, -0.29140281677246094, 0.8910530805587769], "agreement_weight": 0.25014644861221313, "orthogonal_weight": -0.3137941062450409}]}, {"layer": 3, "memory_to_active": 0.4428683817386627, "contribution_to_active": 0.4855823516845703, "mix_mean": 0.514500081539154, "mix_low": 0.05408906936645508, "mix_high": 0.010696252807974815, "read_cosine": 0.17840129137039185, "beta_mean": 0.025552481412887573, "nonfinite": 0.0, "weighted_memory_to_active": 0.12592028081417084, "mix_std": 0.25835955142974854, "beta_std": 0.024956241250038147, "beta_low": 0.7711588740348816, "beta_high": 0.0, "orthogonal_to_active": 0.051082342863082886, "output_influence": 0.42679932713508606, "residual_influence": 0.10321226716041565, "state_active": 0.024896427989006042, "state_memory": 0.0066266609355807304, "state_peak": 23.48549461364746, "growth_active": 1.114370346069336, "growth_memory": 1.5440043210983276, "heads": [{"head": 0, "memory_to_active": 0.35491132736206055, "contribution_to_active": 0.6011949777603149, "mix_mean": 0.6004438996315002, "mix_low": 0.0016374588012695312, "mix_high": 0.024456024169921875, "read_cosine": 0.25045162439346313, "beta_mean": 0.00903364084661007, "nonfinite": 0.0, "weighted_memory_to_active": 0.22786687314510345, "mix_std": 0.3035479485988617, "beta_std": 0.009592697024345398, "beta_low": 0.70684814453125, "beta_high": 0.0, "orthogonal_to_active": 0.16396130621433258, "evidence_weights": [-0.2148251086473465, -0.2496601641178131, 1.0382598638534546], "agreement_weight": 0.11156949400901794, "orthogonal_weight": -0.768446683883667}, {"head": 1, "memory_to_active": 0.007490201853215694, "contribution_to_active": 0.463712215423584, "mix_mean": 0.5813874006271362, "mix_low": 0.0035963058471679688, "mix_high": 0.010021209716796875, "read_cosine": 0.17064747214317322, "beta_mean": 4.472587170312181e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004635514225810766, "mix_std": 0.2946576476097107, "beta_std": 0.00010735999967437238, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0037005699705332518, "evidence_weights": [-0.26871591806411743, -0.21672071516513824, 0.7277653813362122], "agreement_weight": 0.5879336595535278, "orthogonal_weight": -0.8698598146438599}, {"head": 2, "memory_to_active": 2.0416016578674316, "contribution_to_active": 0.29021820425987244, "mix_mean": 0.054361432790756226, "mix_low": 0.30923938751220703, "mix_high": 7.62939453125e-06, "read_cosine": 0.21756021678447723, "beta_mean": 0.1392793357372284, "nonfinite": 0.0, "weighted_memory_to_active": 0.3351670503616333, "mix_std": 0.10792617499828339, "beta_std": 0.127461776137352, "beta_low": 0.02130126953125, "beta_high": 0.0, "orthogonal_to_active": 0.0014865349512547255, "evidence_weights": [-0.10059309005737305, -0.005887309089303017, -0.04237418621778488], "agreement_weight": -0.01840069703757763, "orthogonal_weight": -0.0056692021898925304}, {"head": 3, "memory_to_active": 0.0206650011241436, "contribution_to_active": 0.5343465209007263, "mix_mean": 0.6043341159820557, "mix_low": 0.004596710205078125, "mix_high": 0.008606910705566406, "read_cosine": 0.26968705654144287, "beta_mean": 7.950786675792187e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014051838777959347, "mix_std": 0.28548866510391235, "beta_std": 0.00022226157307159156, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0036225421354174614, "evidence_weights": [-0.2641201317310333, -0.2257297784090042, 2.2662525177001953], "agreement_weight": 2.2262766361236572, "orthogonal_weight": -0.32382437586784363}, {"head": 4, "memory_to_active": 0.016058169305324554, "contribution_to_active": 0.3827817738056183, "mix_mean": 0.5235857367515564, "mix_low": 0.002773284912109375, "mix_high": 0.002193450927734375, "read_cosine": 0.11402434855699539, "beta_mean": 7.267754699569196e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010992626659572124, "mix_std": 0.2890491485595703, "beta_std": 0.00012514961417764425, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010566363111138344, "evidence_weights": [-0.24066711962223053, -0.2249174267053604, 0.10100523382425308], "agreement_weight": -0.0697263777256012, "orthogonal_weight": -1.0465314388275146}, {"head": 5, "memory_to_active": 0.21648390591144562, "contribution_to_active": 0.6412402391433716, "mix_mean": 0.7228878736495972, "mix_low": 0.0026912689208984375, "mix_high": 0.018892288208007812, "read_cosine": 0.048037007451057434, "beta_mean": 0.004804982803761959, "nonfinite": 0.0, "weighted_memory_to_active": 0.16280776262283325, "mix_std": 0.26948773860931396, "beta_std": 0.012228199280798435, "beta_low": 0.89892578125, "beta_high": 0.0, "orthogonal_to_active": 0.12315670400857925, "evidence_weights": [-0.26185598969459534, -0.3419932425022125, 0.5525182485580444], "agreement_weight": 0.22675631940364838, "orthogonal_weight": -0.7761974930763245}]}, {"layer": 4, "memory_to_active": 0.9561909437179565, "contribution_to_active": 0.4720868170261383, "mix_mean": 0.37132570147514343, "mix_low": 0.2406148910522461, "mix_high": 0.0117645263671875, "read_cosine": 0.29964128136634827, "beta_mean": 0.10116538405418396, "nonfinite": 0.0, "weighted_memory_to_active": 0.12752079963684082, "mix_std": 0.16081272065639496, "beta_std": 0.09520649164915085, "beta_low": 0.3995565176010132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.051995210349559784, "output_influence": 0.49735045433044434, "residual_influence": 0.1553734391927719, "state_active": 0.03842925280332565, "state_memory": 0.01634426787495613, "state_peak": 40.54739761352539, "growth_active": 1.2131195068359375, "growth_memory": 1.512583613395691, "heads": [{"head": 0, "memory_to_active": 0.1566346138715744, "contribution_to_active": 0.8087337613105774, "mix_mean": 0.6905492544174194, "mix_low": 0.011462211608886719, "mix_high": 0.03507041931152344, "read_cosine": 0.43709686398506165, "beta_mean": 0.24778133630752563, "nonfinite": 0.0, "weighted_memory_to_active": 0.13495992124080658, "mix_std": 0.34742772579193115, "beta_std": 0.24718312919139862, "beta_low": 0.25830078125, "beta_high": 0.0, "orthogonal_to_active": 0.0874820351600647, "evidence_weights": [-0.25329071283340454, -0.36487770080566406, -0.1601428985595703], "agreement_weight": -0.0019033122807741165, "orthogonal_weight": 0.7695217132568359}, {"head": 1, "memory_to_active": 0.045815516263246536, "contribution_to_active": 0.6637535095214844, "mix_mean": 0.6983110904693604, "mix_low": 9.059906005859375e-05, "mix_high": 0.017608642578125, "read_cosine": 0.5855069160461426, "beta_mean": 0.0008015717612579465, "nonfinite": 0.0, "weighted_memory_to_active": 0.03411540389060974, "mix_std": 0.24000217020511627, "beta_std": 0.00569254532456398, "beta_low": 0.98126220703125, "beta_high": 0.0, "orthogonal_to_active": 0.01706111989915371, "evidence_weights": [-0.23863857984542847, -0.3658212423324585, 0.5965089797973633], "agreement_weight": 0.35344773530960083, "orthogonal_weight": -0.7139686942100525}, {"head": 2, "memory_to_active": 1.5644822120666504, "contribution_to_active": 0.3611803352832794, "mix_mean": 0.0458974614739418, "mix_low": 0.3558998107910156, "mix_high": 3.24249267578125e-05, "read_cosine": 0.2199927717447281, "beta_mean": 0.05929988995194435, "nonfinite": 0.0, "weighted_memory_to_active": 0.2691085934638977, "mix_std": 0.08289586007595062, "beta_std": 0.060313403606414795, "beta_low": 0.1063232421875, "beta_high": 0.0, "orthogonal_to_active": 0.09247900545597076, "evidence_weights": [0.03492274880409241, -0.04951516166329384, 0.2993201017379761], "agreement_weight": 0.13483798503875732, "orthogonal_weight": -0.45751088857650757}, {"head": 3, "memory_to_active": 1.6247639656066895, "contribution_to_active": 0.13329952955245972, "mix_mean": 0.03361738473176956, "mix_low": 0.3974647521972656, "mix_high": 0.0, "read_cosine": 0.20754657685756683, "beta_mean": 0.10309083014726639, "nonfinite": 0.0, "weighted_memory_to_active": 0.18656717240810394, "mix_std": 0.05684444308280945, "beta_std": 0.10545837879180908, "beta_low": 0.05120849609375, "beta_high": 0.0, "orthogonal_to_active": 0.0007289756904356182, "evidence_weights": [-0.02102413959801197, 0.047977279871702194, -0.263259619474411], "agreement_weight": 0.07870111614465714, "orthogonal_weight": -0.005987866315990686}, {"head": 4, "memory_to_active": 0.005631767213344574, "contribution_to_active": 0.6339093446731567, "mix_mean": 0.7471653819084167, "mix_low": 8.58306884765625e-05, "mix_high": 0.017871856689453125, "read_cosine": 0.1262015998363495, "beta_mean": 1.3645812941831537e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004153698682785034, "mix_std": 0.21265338361263275, "beta_std": 3.0188852178980596e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002663894323632121, "evidence_weights": [-0.33046039938926697, -0.3190576434135437, 0.2758994400501251], "agreement_weight": 0.30359724164009094, "orthogonal_weight": -0.6713597774505615}, {"head": 5, "memory_to_active": 2.3398168087005615, "contribution_to_active": 0.23164448142051697, "mix_mean": 0.01241376157850027, "mix_low": 0.6786861419677734, "mix_high": 3.814697265625e-06, "read_cosine": 0.2215028703212738, "beta_mean": 0.19600500166416168, "nonfinite": 0.0, "weighted_memory_to_active": 0.13621999323368073, "mix_std": 0.025052694603800774, "beta_std": 0.1525612473487854, "beta_low": 0.000244140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.11155621707439423, "evidence_weights": [0.059299077838659286, -0.014426183886826038, -0.27270954847335815], "agreement_weight": -0.09514341503381729, "orthogonal_weight": 1.0677963495254517}]}, {"layer": 5, "memory_to_active": 0.35503071546554565, "contribution_to_active": 0.5491690635681152, "mix_mean": 0.5171322226524353, "mix_low": 0.05485360324382782, "mix_high": 0.021352769806981087, "read_cosine": 0.06029517203569412, "beta_mean": 0.0511726476252079, "nonfinite": 0.0, "weighted_memory_to_active": 0.12835049629211426, "mix_std": 0.2546575367450714, "beta_std": 0.06689251959323883, "beta_low": 0.6741028428077698, "beta_high": 0.0, "orthogonal_to_active": 0.055680111050605774, "output_influence": 0.5762547254562378, "residual_influence": 0.18566429615020752, "state_active": 0.04195259511470795, "state_memory": 0.009771115146577358, "state_peak": 62.49836349487305, "growth_active": 1.1325746774673462, "growth_memory": 1.2052979469299316, "heads": [{"head": 0, "memory_to_active": 0.1789512038230896, "contribution_to_active": 0.5854171514511108, "mix_mean": 0.48516300320625305, "mix_low": 0.007111549377441406, "mix_high": 0.001934051513671875, "read_cosine": -0.15330110490322113, "beta_mean": 0.018092211335897446, "nonfinite": 0.0, "weighted_memory_to_active": 0.06176424026489258, "mix_std": 0.2802661955356598, "beta_std": 0.028163544833660126, "beta_low": 0.52191162109375, "beta_high": 0.0, "orthogonal_to_active": 0.02755669690668583, "evidence_weights": [-0.16882666945457458, -0.18725542724132538, 0.10156343877315521], "agreement_weight": -0.019685085862874985, "orthogonal_weight": -0.4751013517379761}, {"head": 1, "memory_to_active": 0.006332574412226677, "contribution_to_active": 0.5590106248855591, "mix_mean": 0.6981833577156067, "mix_low": 0.0008554458618164062, "mix_high": 0.04145622253417969, "read_cosine": 0.1996959149837494, "beta_mean": 1.3684413715964183e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003850917797535658, "mix_std": 0.27021998167037964, "beta_std": 3.286837454652414e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035146393347531557, "evidence_weights": [-0.24747619032859802, -0.2868129014968872, 0.9400884509086609], "agreement_weight": 0.694989025592804, "orthogonal_weight": -0.956274151802063}, {"head": 2, "memory_to_active": 0.014011344872415066, "contribution_to_active": 0.6118054389953613, "mix_mean": 0.6626757383346558, "mix_low": 0.0023784637451171875, "mix_high": 0.02402210235595703, "read_cosine": -0.527568519115448, "beta_mean": 7.462494249921292e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008526812307536602, "mix_std": 0.26403942704200745, "beta_std": 0.00023612947552464902, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007600009441375732, "evidence_weights": [-0.26150834560394287, -0.32230862975120544, 0.16742856800556183], "agreement_weight": -0.1623692363500595, "orthogonal_weight": -1.0379712581634521}, {"head": 3, "memory_to_active": 1.4178611040115356, "contribution_to_active": 0.26439929008483887, "mix_mean": 0.12288963794708252, "mix_low": 0.11801433563232422, "mix_high": 0.0, "read_cosine": 0.1439591944217682, "beta_mean": 0.09770220518112183, "nonfinite": 0.0, "weighted_memory_to_active": 0.35962486267089844, "mix_std": 0.15186837315559387, "beta_std": 0.15233373641967773, "beta_low": 0.2161865234375, "beta_high": 0.0, "orthogonal_to_active": 0.15109774470329285, "evidence_weights": [-0.07689199596643448, -0.10067451745271683, 0.08341323584318161], "agreement_weight": 0.041766855865716934, "orthogonal_weight": -0.4990364611148834}, {"head": 4, "memory_to_active": 0.42166221141815186, "contribution_to_active": 0.5776777267456055, "mix_mean": 0.3214455246925354, "mix_low": 0.1988515853881836, "mix_high": 0.0044536590576171875, "read_cosine": 0.5431394577026367, "beta_mean": 0.19091680645942688, "nonfinite": 0.0, "weighted_memory_to_active": 0.2713782787322998, "mix_std": 0.33949553966522217, "beta_std": 0.21937111020088196, "beta_low": 0.30975341796875, "beta_high": 0.0, "orthogonal_to_active": 0.10620246827602386, "evidence_weights": [-0.11954529583454132, -0.25775599479675293, -0.04844122380018234], "agreement_weight": 0.0345374196767807, "orthogonal_weight": 0.6031107902526855}, {"head": 5, "memory_to_active": 0.09136561304330826, "contribution_to_active": 0.6967037320137024, "mix_mean": 0.8124358654022217, "mix_low": 0.0019102096557617188, "mix_high": 0.056250572204589844, "read_cosine": 0.15584605932235718, "beta_mean": 0.00023636186961084604, "nonfinite": 0.0, "weighted_memory_to_active": 0.06495781242847443, "mix_std": 0.22205567359924316, "beta_std": 0.0012176883174106479, "beta_low": 0.99676513671875, "beta_high": 0.0, "orthogonal_to_active": 0.03810908645391464, "evidence_weights": [-0.30722150206565857, -0.33156993985176086, 1.8622534275054932], "agreement_weight": 0.5085890889167786, "orthogonal_weight": -0.6225188374519348}]}, {"layer": 6, "memory_to_active": 0.24398663640022278, "contribution_to_active": 0.6434850692749023, "mix_mean": 0.5982251167297363, "mix_low": 0.0340479239821434, "mix_high": 0.03294547647237778, "read_cosine": 0.31245487928390503, "beta_mean": 0.062189340591430664, "nonfinite": 0.0, "weighted_memory_to_active": 0.1322927474975586, "mix_std": 0.2380618453025818, "beta_std": 0.07295647263526917, "beta_low": 0.6103109121322632, "beta_high": 0.0, "orthogonal_to_active": 0.06655174493789673, "output_influence": 0.5783289074897766, "residual_influence": 0.2022644281387329, "state_active": 0.04482725262641907, "state_memory": 0.006136440671980381, "state_peak": 94.31452941894531, "growth_active": 1.2057820558547974, "growth_memory": 1.1486713886260986, "heads": [{"head": 0, "memory_to_active": 0.11968027800321579, "contribution_to_active": 0.7710621356964111, "mix_mean": 0.8181745409965515, "mix_low": 7.2479248046875e-05, "mix_high": 0.05568408966064453, "read_cosine": 0.28557947278022766, "beta_mean": 0.011657368391752243, "nonfinite": 0.0, "weighted_memory_to_active": 0.10588156431913376, "mix_std": 0.2023918777704239, "beta_std": 0.036647260189056396, "beta_low": 0.82415771484375, "beta_high": 0.0, "orthogonal_to_active": 0.0724632740020752, "evidence_weights": [-0.3032422661781311, -0.34839433431625366, 1.2096456289291382], "agreement_weight": 0.812474250793457, "orthogonal_weight": -0.7354819774627686}, {"head": 1, "memory_to_active": 0.005521737039089203, "contribution_to_active": 0.5544540286064148, "mix_mean": 0.5970766544342041, "mix_low": 0.0007076263427734375, "mix_high": 0.014882087707519531, "read_cosine": 0.45738255977630615, "beta_mean": 1.0968962669721805e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0026137265376746655, "mix_std": 0.28038182854652405, "beta_std": 1.723847526591271e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0014178644632920623, "evidence_weights": [-0.19399787485599518, -0.3031814396381378, 1.1252827644348145], "agreement_weight": 0.11025665700435638, "orthogonal_weight": -0.6967296600341797}, {"head": 2, "memory_to_active": 0.6277955770492554, "contribution_to_active": 0.10644041001796722, "mix_mean": 0.05818230286240578, "mix_low": 0.1982707977294922, "mix_high": 0.0, "read_cosine": 0.2656599283218384, "beta_mean": 0.018484050408005714, "nonfinite": 0.0, "weighted_memory_to_active": 0.06335318833589554, "mix_std": 0.08644472062587738, "beta_std": 0.02058941125869751, "beta_low": 0.35186767578125, "beta_high": 0.0, "orthogonal_to_active": 0.010144846513867378, "evidence_weights": [0.05675172060728073, -0.06987062096595764, 0.35963037610054016], "agreement_weight": 0.14198651909828186, "orthogonal_weight": -0.19033315777778625}, {"head": 3, "memory_to_active": 0.13195374608039856, "contribution_to_active": 0.6913967132568359, "mix_mean": 0.6742132306098938, "mix_low": 0.004520416259765625, "mix_high": 0.02212047576904297, "read_cosine": 0.47515445947647095, "beta_mean": 0.2197362780570984, "nonfinite": 0.0, "weighted_memory_to_active": 0.09629540145397186, "mix_std": 0.31984013319015503, "beta_std": 0.24458372592926025, "beta_low": 0.18182373046875, "beta_high": 0.0, "orthogonal_to_active": 0.05589304119348526, "evidence_weights": [-0.16752582788467407, -0.317387193441391, 0.13281024992465973], "agreement_weight": -0.0017388795968145132, "orthogonal_weight": 0.665504515171051}, {"head": 4, "memory_to_active": 0.5563116073608398, "contribution_to_active": 1.1073538064956665, "mix_mean": 0.7552257776260376, "mix_low": 0.000591278076171875, "mix_high": 0.09779930114746094, "read_cosine": 0.11481588333845139, "beta_mean": 0.12312906980514526, "nonfinite": 0.0, "weighted_memory_to_active": 0.5091337561607361, "mix_std": 0.2960534989833832, "beta_std": 0.1356118768453598, "beta_low": 0.3040771484375, "beta_high": 0.0, "orthogonal_to_active": 0.24326877295970917, "evidence_weights": [-0.1850995272397995, -0.5379999876022339, -0.6304180026054382], "agreement_weight": -0.013362145982682705, "orthogonal_weight": 0.5031211376190186}, {"head": 5, "memory_to_active": 0.022656898945569992, "contribution_to_active": 0.6302031874656677, "mix_mean": 0.6864781379699707, "mix_low": 0.00012493133544921875, "mix_high": 0.0071868896484375, "read_cosine": 0.27613693475723267, "beta_mean": 0.00011830576841020957, "nonfinite": 0.0, "weighted_memory_to_active": 0.016478849574923515, "mix_std": 0.24325896799564362, "beta_std": 0.00028928183019161224, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.016122661530971527, "evidence_weights": [-0.23427537083625793, -0.3160604238510132, 0.368021696805954], "agreement_weight": 0.20857228338718414, "orthogonal_weight": -1.0470366477966309}]}, {"layer": 7, "memory_to_active": 1.7541913986206055, "contribution_to_active": 0.268890917301178, "mix_mean": 0.1762237548828125, "mix_low": 0.2297968864440918, "mix_high": 0.0012033780803903937, "read_cosine": 0.23716753721237183, "beta_mean": 0.13256847858428955, "nonfinite": 0.0, "weighted_memory_to_active": 0.20746725797653198, "mix_std": 0.14919573068618774, "beta_std": 0.09607268124818802, "beta_low": 0.2147420346736908, "beta_high": 0.00019327798509038985, "orthogonal_to_active": 0.09223556518554688, "output_influence": 0.27999910712242126, "residual_influence": 0.06951723992824554, "state_active": 0.014264739118516445, "state_memory": 0.023692648857831955, "state_peak": 35.118492126464844, "growth_active": 1.1829255819320679, "growth_memory": 1.2600831985473633, "heads": [{"head": 0, "memory_to_active": 2.598706007003784, "contribution_to_active": 0.17460031807422638, "mix_mean": 0.038663677871227264, "mix_low": 0.42243099212646484, "mix_high": 0.0, "read_cosine": 0.34244436025619507, "beta_mean": 0.16590312123298645, "nonfinite": 0.0, "weighted_memory_to_active": 0.21034087240695953, "mix_std": 0.07919786125421524, "beta_std": 0.1259477287530899, "beta_low": 0.0054931640625, "beta_high": 0.000244140625, "orthogonal_to_active": 0.002828299067914486, "evidence_weights": [0.04589589685201645, -0.08675940334796906, 0.011633048765361309], "agreement_weight": -0.02597585879266262, "orthogonal_weight": -0.016235804185271263}, {"head": 1, "memory_to_active": 0.9220321178436279, "contribution_to_active": 0.39030739665031433, "mix_mean": 0.2218605875968933, "mix_low": 0.12134647369384766, "mix_high": 0.006491661071777344, "read_cosine": 0.3798704743385315, "beta_mean": 0.08503986895084381, "nonfinite": 0.0, "weighted_memory_to_active": 0.31743478775024414, "mix_std": 0.24704384803771973, "beta_std": 0.06217353790998459, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04427087679505348, "evidence_weights": [-0.21676816046237946, -0.10348478704690933, -0.2917096018791199], "agreement_weight": 0.051743507385253906, "orthogonal_weight": 0.16631165146827698}, {"head": 2, "memory_to_active": 0.6083650588989258, "contribution_to_active": 0.36880069971084595, "mix_mean": 0.2346869707107544, "mix_low": 0.01787853240966797, "mix_high": 0.0003986358642578125, "read_cosine": 0.14490509033203125, "beta_mean": 0.03706643357872963, "nonfinite": 0.0, "weighted_memory_to_active": 0.2311202585697174, "mix_std": 0.2155507206916809, "beta_std": 0.03747103363275528, "beta_low": 0.135986328125, "beta_high": 0.0, "orthogonal_to_active": 0.13596457242965698, "evidence_weights": [-0.09337323158979416, -0.18117031455039978, 0.8674324750900269], "agreement_weight": 0.020155981183052063, "orthogonal_weight": -0.607349157333374}, {"head": 3, "memory_to_active": 4.85575532913208, "contribution_to_active": 0.1089804470539093, "mix_mean": 0.036517225205898285, "mix_low": 0.31476402282714844, "mix_high": 0.0, "read_cosine": 0.1047944575548172, "beta_mean": 0.4417992830276489, "nonfinite": 0.0, "weighted_memory_to_active": 0.38432034850120544, "mix_std": 0.0530795156955719, "beta_std": 0.2761878967285156, "beta_low": 0.00262451171875, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.33245229721069336, "evidence_weights": [0.07379957288503647, -0.04024985805153847, 0.7183706164360046], "agreement_weight": 0.05343049764633179, "orthogonal_weight": -0.9368728399276733}, {"head": 4, "memory_to_active": 1.497287392616272, "contribution_to_active": 0.10776791721582413, "mix_mean": 0.024520661681890488, "mix_low": 0.5004920959472656, "mix_high": 0.0, "read_cosine": 0.21252097189426422, "beta_mean": 0.0654199868440628, "nonfinite": 0.0, "weighted_memory_to_active": 0.07967523485422134, "mix_std": 0.04818642884492874, "beta_std": 0.0742557942867279, "beta_low": 0.1444091796875, "beta_high": 0.0, "orthogonal_to_active": 0.016952291131019592, "evidence_weights": [0.12414657324552536, -0.024194832891225815, 0.18346856534481049], "agreement_weight": 0.035658035427331924, "orthogonal_weight": -0.27541810274124146}, {"head": 5, "memory_to_active": 0.04300212487578392, "contribution_to_active": 0.4628886580467224, "mix_mean": 0.5010933876037598, "mix_low": 0.00186920166015625, "mix_high": 0.0003299713134765625, "read_cosine": 0.23846977949142456, "beta_mean": 0.00018218551122117788, "nonfinite": 0.0, "weighted_memory_to_active": 0.021911999210715294, "mix_std": 0.25211599469184875, "beta_std": 0.00040007021743804216, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.02094503678381443, "evidence_weights": [-0.18296994268894196, -0.23875874280929565, 0.49250251054763794], "agreement_weight": 0.22540201246738434, "orthogonal_weight": -1.013964056968689}]}, {"layer": 8, "memory_to_active": 0.14622072875499725, "contribution_to_active": 0.5673589110374451, "mix_mean": 0.49989044666290283, "mix_low": 0.01022481918334961, "mix_high": 0.0017984708538278937, "read_cosine": 0.39782559871673584, "beta_mean": 0.01250450685620308, "nonfinite": 0.0, "weighted_memory_to_active": 0.0754411518573761, "mix_std": 0.24217870831489563, "beta_std": 0.019629525020718575, "beta_low": 0.79498291015625, "beta_high": 0.0, "orthogonal_to_active": 0.06842700392007828, "output_influence": 0.5150238871574402, "residual_influence": 0.18597304821014404, "state_active": 0.04374869167804718, "state_memory": 0.00501293083652854, "state_peak": 64.32215881347656, "growth_active": 1.1007267236709595, "growth_memory": 1.070440649986267, "heads": [{"head": 0, "memory_to_active": 0.060518741607666016, "contribution_to_active": 0.5652735233306885, "mix_mean": 0.6036127805709839, "mix_low": 0.0005741119384765625, "mix_high": 0.0014438629150390625, "read_cosine": 0.44226062297821045, "beta_mean": 0.003513611387461424, "nonfinite": 0.0, "weighted_memory_to_active": 0.04240143299102783, "mix_std": 0.24946115911006927, "beta_std": 0.014092804864048958, "beta_low": 0.9271240234375, "beta_high": 0.0, "orthogonal_to_active": 0.027072977274656296, "evidence_weights": [-0.1582740992307663, -0.37950247526168823, 0.5744033455848694], "agreement_weight": 0.2626887857913971, "orthogonal_weight": -0.8017500042915344}, {"head": 1, "memory_to_active": 0.08188202232122421, "contribution_to_active": 0.5498318672180176, "mix_mean": 0.5288888216018677, "mix_low": 6.008148193359375e-05, "mix_high": 0.0017337799072265625, "read_cosine": 0.11433106660842896, "beta_mean": 0.001096854219213128, "nonfinite": 0.0, "weighted_memory_to_active": 0.03947276622056961, "mix_std": 0.23273685574531555, "beta_std": 0.008113524876534939, "beta_low": 0.989990234375, "beta_high": 0.0, "orthogonal_to_active": 0.029399333521723747, "evidence_weights": [-0.1951710432767868, -0.27598246932029724, 0.6230242252349854], "agreement_weight": 0.07246681302785873, "orthogonal_weight": -0.7725865244865417}, {"head": 2, "memory_to_active": 0.011219470761716366, "contribution_to_active": 0.6878742575645447, "mix_mean": 0.6869751214981079, "mix_low": 0.0003662109375, "mix_high": 0.0034046173095703125, "read_cosine": 0.43270331621170044, "beta_mean": 0.00011132895451737568, "nonfinite": 0.0, "weighted_memory_to_active": 0.00898737832903862, "mix_std": 0.2203381061553955, "beta_std": 0.0030261329375207424, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005263180937618017, "evidence_weights": [-0.22333736717700958, -0.3474792540073395, 0.9958562850952148], "agreement_weight": 0.31050804257392883, "orthogonal_weight": -0.6883543133735657}, {"head": 3, "memory_to_active": 0.10097432136535645, "contribution_to_active": 0.6389475464820862, "mix_mean": 0.5793361663818359, "mix_low": 0.0006875991821289062, "mix_high": 0.0039997100830078125, "read_cosine": 0.78966224193573, "beta_mean": 0.006966212764382362, "nonfinite": 0.0, "weighted_memory_to_active": 0.07513939589262009, "mix_std": 0.25601452589035034, "beta_std": 0.007542833685874939, "beta_low": 0.7813720703125, "beta_high": 0.0, "orthogonal_to_active": 0.02385466918349266, "evidence_weights": [-0.1747925877571106, -0.29738372564315796, 0.532134473323822], "agreement_weight": 0.2500017583370209, "orthogonal_weight": -0.6026265621185303}, {"head": 4, "memory_to_active": 0.2659861445426941, "contribution_to_active": 0.32936182618141174, "mix_mean": 0.3291027843952179, "mix_low": 0.0021295547485351562, "mix_high": 1.33514404296875e-05, "read_cosine": 0.11457041651010513, "beta_mean": 0.0007408512756228447, "nonfinite": 0.0, "weighted_memory_to_active": 0.10105323046445847, "mix_std": 0.227245032787323, "beta_std": 0.013151158578693867, "beta_low": 0.9942626953125, "beta_high": 0.0, "orthogonal_to_active": 0.02103503979742527, "evidence_weights": [-0.13333359360694885, -0.19742541015148163, 0.6599174737930298], "agreement_weight": 0.036694418638944626, "orthogonal_weight": -0.21350039541721344}, {"head": 5, "memory_to_active": 0.35674360394477844, "contribution_to_active": 0.6328641772270203, "mix_mean": 0.271426796913147, "mix_low": 0.05753135681152344, "mix_high": 0.00019550323486328125, "read_cosine": 0.49342581629753113, "beta_mean": 0.06259818375110626, "nonfinite": 0.0, "weighted_memory_to_active": 0.18559269607067108, "mix_std": 0.26727646589279175, "beta_std": 0.07185068726539612, "beta_low": 0.07720947265625, "beta_high": 0.0, "orthogonal_to_active": 0.30393680930137634, "evidence_weights": [-0.1466924101114273, -0.14391103386878967, -0.11404658854007721], "agreement_weight": -0.007763590198010206, "orthogonal_weight": 1.9293348789215088}]}, {"layer": 9, "memory_to_active": 0.9043569564819336, "contribution_to_active": 0.7082134485244751, "mix_mean": 0.38742345571517944, "mix_low": 0.06050078570842743, "mix_high": 0.006176471710205078, "read_cosine": 0.3617795705795288, "beta_mean": 0.03624200448393822, "nonfinite": 0.0, "weighted_memory_to_active": 0.3890402019023895, "mix_std": 0.24451659619808197, "beta_std": 0.030803795903921127, "beta_low": 0.6086223125457764, "beta_high": 0.0, "orthogonal_to_active": 0.036659516394138336, "output_influence": 0.5469189882278442, "residual_influence": 0.1646394431591034, "state_active": 0.042159486562013626, "state_memory": 0.014838079921901226, "state_peak": 58.09552001953125, "growth_active": 1.1661689281463623, "growth_memory": 1.2994499206542969, "heads": [{"head": 0, "memory_to_active": 0.0005994727835059166, "contribution_to_active": 0.5715411305427551, "mix_mean": 0.6506369709968567, "mix_low": 0.0013275146484375, "mix_high": 0.025361061096191406, "read_cosine": 0.3247365951538086, "beta_mean": 1.7854952147899894e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0003853397793136537, "mix_std": 0.2855522930622101, "beta_std": 7.793832992319949e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00041187010356225073, "evidence_weights": [-0.3045583665370941, -0.312405526638031, 0.4198473393917084], "agreement_weight": 0.11451216042041779, "orthogonal_weight": -1.1199791431427002}, {"head": 1, "memory_to_active": 0.02266380935907364, "contribution_to_active": 0.7054433822631836, "mix_mean": 0.5742264986038208, "mix_low": 0.006610870361328125, "mix_high": 0.00304412841796875, "read_cosine": 0.43456947803497314, "beta_mean": 5.428049189504236e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.017152490094304085, "mix_std": 0.2817718982696533, "beta_std": 0.00012315806816332042, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011195160448551178, "evidence_weights": [-0.23818470537662506, -0.29528510570526123, 0.9801208972930908], "agreement_weight": 0.5364171862602234, "orthogonal_weight": -0.81868577003479}, {"head": 2, "memory_to_active": 0.17319542169570923, "contribution_to_active": 0.6578855514526367, "mix_mean": 0.6028536558151245, "mix_low": 0.0018711090087890625, "mix_high": 0.006968498229980469, "read_cosine": 0.3835446536540985, "beta_mean": 0.0006744256243109703, "nonfinite": 0.0, "weighted_memory_to_active": 0.11766910552978516, "mix_std": 0.26632848381996155, "beta_std": 0.004555049352347851, "beta_low": 0.99566650390625, "beta_high": 0.0, "orthogonal_to_active": 0.0675337165594101, "evidence_weights": [-0.2015862911939621, -0.27633363008499146, 0.7601081728935242], "agreement_weight": 0.3409249186515808, "orthogonal_weight": -0.6815940737724304}, {"head": 3, "memory_to_active": 0.8470804691314697, "contribution_to_active": 0.24828505516052246, "mix_mean": 0.15759505331516266, "mix_low": 0.1030740737915039, "mix_high": 0.0001277923583984375, "read_cosine": 0.39265620708465576, "beta_mean": 0.033622048795223236, "nonfinite": 0.0, "weighted_memory_to_active": 0.20970042049884796, "mix_std": 0.2028658390045166, "beta_std": 0.032809820026159286, "beta_low": 0.14654541015625, "beta_high": 0.0, "orthogonal_to_active": 0.09046804159879684, "evidence_weights": [-0.023333676159381866, -0.09468439966440201, 0.4011838436126709], "agreement_weight": 0.09078854322433472, "orthogonal_weight": -0.47467756271362305}, {"head": 4, "memory_to_active": 3.1689858436584473, "contribution_to_active": 1.6892462968826294, "mix_mean": 0.15115509927272797, "mix_low": 0.12446022033691406, "mix_high": 0.001178741455078125, "read_cosine": 0.14371800422668457, "beta_mean": 0.15217959880828857, "nonfinite": 0.0, "weighted_memory_to_active": 1.6314903497695923, "mix_std": 0.21046537160873413, "beta_std": 0.08806560933589935, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.014615096151828766, "evidence_weights": [-0.08042873442173004, -0.15172354876995087, -0.04125106707215309], "agreement_weight": -0.06429759413003922, "orthogonal_weight": -0.009353961795568466}, {"head": 5, "memory_to_active": 1.2136163711547852, "contribution_to_active": 0.37687909603118896, "mix_mean": 0.18807333707809448, "mix_low": 0.12566089630126953, "mix_high": 0.00037860870361328125, "read_cosine": 0.49145230650901794, "beta_mean": 0.030919872224330902, "nonfinite": 0.0, "weighted_memory_to_active": 0.3578435778617859, "mix_std": 0.22011569142341614, "beta_std": 0.059261344373226166, "beta_low": 0.509521484375, "beta_high": 0.0, "orthogonal_to_active": 0.03573320060968399, "evidence_weights": [0.059469398111104965, -0.17491205036640167, 0.14549706876277924], "agreement_weight": 0.007237807847559452, "orthogonal_weight": 0.12174857407808304}]}, {"layer": 10, "memory_to_active": 0.20998793840408325, "contribution_to_active": 0.7631505727767944, "mix_mean": 0.5233292579650879, "mix_low": 0.024570943787693977, "mix_high": 0.00494384765625, "read_cosine": 0.38672584295272827, "beta_mean": 0.041364021599292755, "nonfinite": 0.0, "weighted_memory_to_active": 0.09790384024381638, "mix_std": 0.2678626775741577, "beta_std": 0.04396424815058708, "beta_low": 0.6700338125228882, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.2937585115432739, "output_influence": 0.46855682134628296, "residual_influence": 0.16786307096481323, "state_active": 0.05426320433616638, "state_memory": 0.01449956838041544, "state_peak": 125.1505126953125, "growth_active": 1.0358150005340576, "growth_memory": 1.0095709562301636, "heads": [{"head": 0, "memory_to_active": 0.7633031606674194, "contribution_to_active": 2.260589599609375, "mix_mean": 0.5630142688751221, "mix_low": 0.11388683319091797, "mix_high": 0.006213188171386719, "read_cosine": 0.09814343601465225, "beta_mean": 0.18251334130764008, "nonfinite": 0.0, "weighted_memory_to_active": 0.47259747982025146, "mix_std": 0.39365655183792114, "beta_std": 0.19657576084136963, "beta_low": 0.16265869140625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.6797184944152832, "evidence_weights": [-0.08672969043254852, -0.3712104260921478, -1.9004188776016235], "agreement_weight": 0.031699635088443756, "orthogonal_weight": 3.5798840522766113}, {"head": 1, "memory_to_active": 0.008859990164637566, "contribution_to_active": 0.6168875694274902, "mix_mean": 0.6337482929229736, "mix_low": 0.0017261505126953125, "mix_high": 0.016572952270507812, "read_cosine": 0.3077390789985657, "beta_mean": 6.543534982483834e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0049986457452178, "mix_std": 0.26472967863082886, "beta_std": 0.00012686129775829613, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007899852469563484, "evidence_weights": [-0.2498498409986496, -0.1938905566930771, 0.5610604286193848], "agreement_weight": 0.626190185546875, "orthogonal_weight": -1.7091104984283447}, {"head": 2, "memory_to_active": 0.012381993234157562, "contribution_to_active": 0.6210203170776367, "mix_mean": 0.6609780788421631, "mix_low": 0.0003566741943359375, "mix_high": 0.0054035186767578125, "read_cosine": 0.05607493594288826, "beta_mean": 2.6134777726838365e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005418890155851841, "mix_std": 0.2520569860935211, "beta_std": 0.00010255645611323416, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023115447256714106, "evidence_weights": [-0.26471012830734253, -0.2758370637893677, 0.6402138471603394], "agreement_weight": -0.21540802717208862, "orthogonal_weight": -0.40853744745254517}, {"head": 3, "memory_to_active": 0.339433491230011, "contribution_to_active": 0.09896139800548553, "mix_mean": 0.2802865207195282, "mix_low": 0.028844833374023438, "mix_high": 7.724761962890625e-05, "read_cosine": 0.9180487990379333, "beta_mean": 0.05964533984661102, "nonfinite": 0.0, "weighted_memory_to_active": 0.038608212023973465, "mix_std": 0.22470158338546753, "beta_std": 0.0590786412358284, "beta_low": 0.00701904296875, "beta_high": 0.0, "orthogonal_to_active": 0.04132130369544029, "evidence_weights": [-0.09339653700590134, -0.14341408014297485, 0.16365306079387665], "agreement_weight": 0.021141495555639267, "orthogonal_weight": 1.4408490657806396}, {"head": 4, "memory_to_active": 0.01258184015750885, "contribution_to_active": 0.5668675303459167, "mix_mean": 0.5934655666351318, "mix_low": 0.00026226043701171875, "mix_high": 0.0013933181762695312, "read_cosine": 0.09649757295846939, "beta_mean": 3.862172889057547e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00758377555757761, "mix_std": 0.2461123913526535, "beta_std": 6.726277933921665e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010936686769127846, "evidence_weights": [-0.18446823954582214, -0.2548096477985382, 0.9640017747879028], "agreement_weight": 0.17517529428005219, "orthogonal_weight": -1.5134211778640747}, {"head": 5, "memory_to_active": 0.12336710095405579, "contribution_to_active": 0.41457706689834595, "mix_mean": 0.4084826707839966, "mix_low": 0.0023488998413085938, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8438513278961182, "beta_mean": 0.005895258858799934, "nonfinite": 0.0, "weighted_memory_to_active": 0.058216020464897156, "mix_std": 0.22591891884803772, "beta_std": 0.007834392599761486, "beta_low": 0.85052490234375, "beta_high": 0.0, "orthogonal_to_active": 0.02036307565867901, "evidence_weights": [-0.1400662213563919, -0.1649106740951538, 0.2687451243400574], "agreement_weight": 0.09405461698770523, "orthogonal_weight": -0.6711059808731079}]}, {"layer": 11, "memory_to_active": 0.07378657907247543, "contribution_to_active": 0.6252984404563904, "mix_mean": 0.5255440473556519, "mix_low": 0.11158673465251923, "mix_high": 0.04674943536520004, "read_cosine": 0.3512748181819916, "beta_mean": 0.023657385259866714, "nonfinite": 0.0, "weighted_memory_to_active": 0.039616964757442474, "mix_std": 0.3241152763366699, "beta_std": 0.03498535603284836, "beta_low": 0.7150065898895264, "beta_high": 0.0, "orthogonal_to_active": 0.0433204285800457, "output_influence": 0.55400550365448, "residual_influence": 0.1538248062133789, "state_active": 0.20262235403060913, "state_memory": 0.003120570909231901, "state_peak": 37.47494125366211, "growth_active": 1.1744589805603027, "growth_memory": 1.4518423080444336, "heads": [{"head": 0, "memory_to_active": 0.0049174171872437, "contribution_to_active": 0.6057348251342773, "mix_mean": 0.658401608467102, "mix_low": 0.0008268356323242188, "mix_high": 0.00841522216796875, "read_cosine": 0.2882545590400696, "beta_mean": 5.629000224871561e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037071043625473976, "mix_std": 0.25428715348243713, "beta_std": 0.002568250522017479, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0030655835289508104, "evidence_weights": [-0.17559531331062317, -0.36004960536956787, 0.7930880188941956], "agreement_weight": 0.3768322765827179, "orthogonal_weight": -0.8562926054000854}, {"head": 1, "memory_to_active": 0.11643633246421814, "contribution_to_active": 0.4329495429992676, "mix_mean": 0.4691980481147766, "mix_low": 0.004605293273925781, "mix_high": 0.0006046295166015625, "read_cosine": 0.5106134414672852, "beta_mean": 0.0027403784915804863, "nonfinite": 0.0, "weighted_memory_to_active": 0.05511016026139259, "mix_std": 0.26826950907707214, "beta_std": 0.009607281535863876, "beta_low": 0.94091796875, "beta_high": 0.0, "orthogonal_to_active": 0.030248232185840607, "evidence_weights": [-0.14537689089775085, -0.2731008231639862, 0.5997979044914246], "agreement_weight": 0.08276030421257019, "orthogonal_weight": -0.6544533371925354}, {"head": 2, "memory_to_active": 0.005461171269416809, "contribution_to_active": 0.9846624732017517, "mix_mean": 0.43281400203704834, "mix_low": 0.31766223907470703, "mix_high": 0.16976165771484375, "read_cosine": 0.17927174270153046, "beta_mean": 0.06384771317243576, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037786350585520267, "mix_std": 0.45214563608169556, "beta_std": 0.08673013746738434, "beta_low": 0.4832763671875, "beta_high": 0.0, "orthogonal_to_active": 0.0376911386847496, "evidence_weights": [0.041996099054813385, -0.35369667410850525, -1.3274235725402832], "agreement_weight": 0.0003277362557128072, "orthogonal_weight": 10.363668441772461}, {"head": 3, "memory_to_active": 0.008078677579760551, "contribution_to_active": 0.8469747304916382, "mix_mean": 0.4120509624481201, "mix_low": 0.33948230743408203, "mix_high": 0.0936899185180664, "read_cosine": 0.5358009338378906, "beta_mean": 0.052386097609996796, "nonfinite": 0.0, "weighted_memory_to_active": 0.004906679969280958, "mix_std": 0.44267570972442627, "beta_std": 0.08006526529788971, "beta_low": 0.5225830078125, "beta_high": 0.0, "orthogonal_to_active": 0.038434144109487534, "evidence_weights": [0.2102903425693512, -0.28389474749565125, -0.11389527469873428], "agreement_weight": 0.053394611924886703, "orthogonal_weight": 10.628381729125977}, {"head": 4, "memory_to_active": 0.2124803364276886, "contribution_to_active": 0.45310112833976746, "mix_mean": 0.6057242155075073, "mix_low": 0.005791664123535156, "mix_high": 0.0018262863159179688, "read_cosine": -0.0056412662379443645, "beta_mean": 0.014887254685163498, "nonfinite": 0.0, "weighted_memory_to_active": 0.12083755433559418, "mix_std": 0.27038633823394775, "beta_std": 0.0180408526211977, "beta_low": 0.53668212890625, "beta_high": 0.0, "orthogonal_to_active": 0.11927786469459534, "evidence_weights": [-0.23644155263900757, -0.24115265905857086, 1.8826154470443726], "agreement_weight": -0.21679814159870148, "orthogonal_weight": -0.9969369173049927}, {"head": 5, "memory_to_active": 0.09534552693367004, "contribution_to_active": 0.4283677339553833, "mix_mean": 0.5750754475593567, "mix_low": 0.00115203857421875, "mix_high": 0.006198883056640625, "read_cosine": 0.599349319934845, "beta_mean": 0.00802656915038824, "nonfinite": 0.0, "weighted_memory_to_active": 0.04936164245009422, "mix_std": 0.2569272518157959, "beta_std": 0.01290036179125309, "beta_low": 0.80670166015625, "beta_high": 0.0, "orthogonal_to_active": 0.03120558336377144, "evidence_weights": [-0.2009725272655487, -0.3121211528778076, 0.6555203199386597], "agreement_weight": 0.08410675078630447, "orthogonal_weight": -0.8665933609008789}]}, {"layer": 12, "memory_to_active": 0.18994829058647156, "contribution_to_active": 0.573772668838501, "mix_mean": 0.5456956624984741, "mix_low": 0.008745511993765831, "mix_high": 0.007024765480309725, "read_cosine": 0.22554506361484528, "beta_mean": 0.01342497207224369, "nonfinite": 0.0, "weighted_memory_to_active": 0.0934349000453949, "mix_std": 0.260827898979187, "beta_std": 0.01367259118705988, "beta_low": 0.8305359482765198, "beta_high": 0.0, "orthogonal_to_active": 0.06194251775741577, "output_influence": 0.5065079927444458, "residual_influence": 0.1486823856830597, "state_active": 0.03422772139310837, "state_memory": 0.007636969443410635, "state_peak": 70.71924591064453, "growth_active": 1.181136965751648, "growth_memory": 1.1866536140441895, "heads": [{"head": 0, "memory_to_active": 0.03625846281647682, "contribution_to_active": 0.6177548170089722, "mix_mean": 0.5879733562469482, "mix_low": 0.0003910064697265625, "mix_high": 0.006749153137207031, "read_cosine": 0.271218478679657, "beta_mean": 0.0006021107546985149, "nonfinite": 0.0, "weighted_memory_to_active": 0.02032613381743431, "mix_std": 0.2642665505409241, "beta_std": 0.0012237271293997765, "beta_low": 0.99676513671875, "beta_high": 0.0, "orthogonal_to_active": 0.018901901319622993, "evidence_weights": [-0.20726405084133148, -0.27551355957984924, -0.03960009291768074], "agreement_weight": 0.10481181740760803, "orthogonal_weight": -0.9954887628555298}, {"head": 1, "memory_to_active": 0.07134529948234558, "contribution_to_active": 0.5579386949539185, "mix_mean": 0.5621364116668701, "mix_low": 0.00025272369384765625, "mix_high": 0.0025177001953125, "read_cosine": 0.3889225721359253, "beta_mean": 0.0004686033062171191, "nonfinite": 0.0, "weighted_memory_to_active": 0.044393204152584076, "mix_std": 0.2665553689002991, "beta_std": 0.00714584905654192, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.03749376907944679, "evidence_weights": [-0.20095957815647125, -0.27170276641845703, 1.0435341596603394], "agreement_weight": 0.2619730532169342, "orthogonal_weight": -0.9232208728790283}, {"head": 2, "memory_to_active": 0.5930305123329163, "contribution_to_active": 0.47087323665618896, "mix_mean": 0.30966922640800476, "mix_low": 0.027170181274414062, "mix_high": 0.0001163482666015625, "read_cosine": 0.5652031302452087, "beta_mean": 0.07855810225009918, "nonfinite": 0.0, "weighted_memory_to_active": 0.22157563269138336, "mix_std": 0.23580098152160645, "beta_std": 0.0599973201751709, "beta_low": 0.00250244140625, "beta_high": 0.0, "orthogonal_to_active": 0.24646076560020447, "evidence_weights": [-0.20620843768119812, -0.060782089829444885, -0.02658400870859623], "agreement_weight": 0.03368633612990379, "orthogonal_weight": 1.4133151769638062}, {"head": 3, "memory_to_active": 0.0319012813270092, "contribution_to_active": 0.7524453997612, "mix_mean": 0.7202016115188599, "mix_low": 0.00012302398681640625, "mix_high": 0.020843505859375, "read_cosine": 0.09339442849159241, "beta_mean": 0.0002971127978526056, "nonfinite": 0.0, "weighted_memory_to_active": 0.02254028618335724, "mix_std": 0.22912883758544922, "beta_std": 0.005769267678260803, "beta_low": 0.9974365234375, "beta_high": 0.0, "orthogonal_to_active": 0.021267686039209366, "evidence_weights": [-0.28689682483673096, -0.3347874581813812, 1.63373601436615], "agreement_weight": 0.5884058475494385, "orthogonal_weight": -0.9632329344749451}, {"head": 4, "memory_to_active": 0.3567016124725342, "contribution_to_active": 0.5491155385971069, "mix_mean": 0.45916664600372314, "mix_low": 0.017711639404296875, "mix_high": 0.004947662353515625, "read_cosine": -0.17810273170471191, "beta_mean": 0.00039901345735415816, "nonfinite": 0.0, "weighted_memory_to_active": 0.21841298043727875, "mix_std": 0.29293641448020935, "beta_std": 0.005567105486989021, "beta_low": 0.99169921875, "beta_high": 0.0, "orthogonal_to_active": 0.02235422469675541, "evidence_weights": [-0.15451180934906006, -0.2141769826412201, -0.13631027936935425], "agreement_weight": 0.010903699323534966, "orthogonal_weight": -0.10627251863479614}, {"head": 5, "memory_to_active": 0.0504525825381279, "contribution_to_active": 0.49450790882110596, "mix_mean": 0.6350263357162476, "mix_low": 0.006824493408203125, "mix_high": 0.006974220275878906, "read_cosine": 0.21263445913791656, "beta_mean": 0.0002248921082355082, "nonfinite": 0.0, "weighted_memory_to_active": 0.033361151814460754, "mix_std": 0.2762792408466339, "beta_std": 0.0023322696797549725, "beta_low": 0.9970703125, "beta_high": 0.0, "orthogonal_to_active": 0.0251767598092556, "evidence_weights": [-0.2244580239057541, -0.3149236738681793, 1.2369458675384521], "agreement_weight": 0.4300091564655304, "orthogonal_weight": -0.8111035823822021}]}, {"layer": 13, "memory_to_active": 0.10491600632667542, "contribution_to_active": 0.5416758060455322, "mix_mean": 0.5414214730262756, "mix_low": 0.005802155006676912, "mix_high": 0.009594122879207134, "read_cosine": 0.38799893856048584, "beta_mean": 0.013076061382889748, "nonfinite": 0.0, "weighted_memory_to_active": 0.04368489235639572, "mix_std": 0.25902611017227173, "beta_std": 0.014996763318777084, "beta_low": 0.8136088252067566, "beta_high": 0.0, "orthogonal_to_active": 0.04707972705364227, "output_influence": 0.5305712819099426, "residual_influence": 0.18947960436344147, "state_active": 0.1311389058828354, "state_memory": 0.041897229850292206, "state_peak": 96.61727142333984, "growth_active": 0.9071323275566101, "growth_memory": 1.0145331621170044, "heads": [{"head": 0, "memory_to_active": 0.01166370790451765, "contribution_to_active": 0.6047235727310181, "mix_mean": 0.6202135682106018, "mix_low": 0.0007038116455078125, "mix_high": 0.005366325378417969, "read_cosine": 0.42743954062461853, "beta_mean": 4.201439151074737e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007606565952301025, "mix_std": 0.26764947175979614, "beta_std": 0.00010444821964483708, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0072633083909749985, "evidence_weights": [-0.19239117205142975, -0.2664303481578827, 0.7403460741043091], "agreement_weight": 0.20496833324432373, "orthogonal_weight": -1.0627871751785278}, {"head": 1, "memory_to_active": 0.007913210429251194, "contribution_to_active": 0.6043958067893982, "mix_mean": 0.5757710933685303, "mix_low": 0.001628875732421875, "mix_high": 0.005252838134765625, "read_cosine": 0.22555282711982727, "beta_mean": 3.158434992656112e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004501079674810171, "mix_std": 0.27825018763542175, "beta_std": 5.957285611657426e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005829239264130592, "evidence_weights": [-0.1798020899295807, -0.2630785405635834, 0.5573124289512634], "agreement_weight": 0.5264759063720703, "orthogonal_weight": -1.347135305404663}, {"head": 2, "memory_to_active": 0.055013637989759445, "contribution_to_active": 0.4804583191871643, "mix_mean": 0.38993653655052185, "mix_low": 0.0013589859008789062, "mix_high": 0.000232696533203125, "read_cosine": 0.480215847492218, "beta_mean": 0.007505698129534721, "nonfinite": 0.0, "weighted_memory_to_active": 0.02513248100876808, "mix_std": 0.23425064980983734, "beta_std": 0.02914426475763321, "beta_low": 0.885986328125, "beta_high": 0.0, "orthogonal_to_active": 0.01561524998396635, "evidence_weights": [-0.1809128075838089, -0.21102994680404663, 0.42100080847740173], "agreement_weight": 0.0065116435289382935, "orthogonal_weight": -0.8431393504142761}, {"head": 3, "memory_to_active": 0.14097483456134796, "contribution_to_active": 0.5882852673530579, "mix_mean": 0.6653240919113159, "mix_low": 0.024193763732910156, "mix_high": 0.033026695251464844, "read_cosine": -0.01855350285768509, "beta_mean": 0.0008371548028662801, "nonfinite": 0.0, "weighted_memory_to_active": 0.10412070155143738, "mix_std": 0.2850831151008606, "beta_std": 0.007509300485253334, "beta_low": 0.99462890625, "beta_high": 0.0, "orthogonal_to_active": 0.09244468808174133, "evidence_weights": [-0.3126410245895386, -0.239237979054451, 1.516631007194519], "agreement_weight": 6.556727886199951, "orthogonal_weight": -0.9705655574798584}, {"head": 4, "memory_to_active": 0.38337379693984985, "contribution_to_active": 0.25583329796791077, "mix_mean": 0.33709222078323364, "mix_low": 0.004664421081542969, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8832676410675049, "beta_mean": 0.06989938765764236, "nonfinite": 0.0, "weighted_memory_to_active": 0.09924612939357758, "mix_std": 0.2162802666425705, "beta_std": 0.052211418747901917, "beta_low": 0.00115966796875, "beta_high": 0.0, "orthogonal_to_active": 0.14324629306793213, "evidence_weights": [-0.06839966028928757, -0.22859543561935425, 0.43485304713249207], "agreement_weight": 0.009266916662454605, "orthogonal_weight": 2.277581214904785}, {"head": 5, "memory_to_active": 0.030556757003068924, "contribution_to_active": 0.7163585424423218, "mix_mean": 0.6601911783218384, "mix_low": 0.0022630691528320312, "mix_high": 0.013680458068847656, "read_cosine": 0.33007121086120605, "beta_mean": 0.00014052896585781127, "nonfinite": 0.0, "weighted_memory_to_active": 0.02150239795446396, "mix_std": 0.27264291048049927, "beta_std": 0.0009515666170045733, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.01807958073914051, "evidence_weights": [-0.22177395224571228, -0.29317015409469604, 0.8734503984451294], "agreement_weight": 0.20849181711673737, "orthogonal_weight": -0.9291789531707764}]}, {"layer": 14, "memory_to_active": 0.2205306440591812, "contribution_to_active": 0.4169892966747284, "mix_mean": 0.4301046133041382, "mix_low": 0.027339937165379524, "mix_high": 0.0062408447265625, "read_cosine": 0.6092846393585205, "beta_mean": 0.05778230354189873, "nonfinite": 0.0, "weighted_memory_to_active": 0.09314142167568207, "mix_std": 0.24388065934181213, "beta_std": 0.05754467844963074, "beta_low": 0.5139058828353882, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.04162918031215668, "output_influence": 0.35486119985580444, "residual_influence": 0.1455414891242981, "state_active": 0.053766701370477676, "state_memory": 0.014003516174852848, "state_peak": 51.54169464111328, "growth_active": 1.0900160074234009, "growth_memory": 1.2486977577209473, "heads": [{"head": 0, "memory_to_active": 0.33766263723373413, "contribution_to_active": 0.196659117937088, "mix_mean": 0.1908547580242157, "mix_low": 0.018606185913085938, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8021653890609741, "beta_mean": 0.0615178719162941, "nonfinite": 0.0, "weighted_memory_to_active": 0.0755942240357399, "mix_std": 0.18359680473804474, "beta_std": 0.057995762676000595, "beta_low": 0.016845703125, "beta_high": 0.0, "orthogonal_to_active": 0.08668726682662964, "evidence_weights": [-0.09980469942092896, -0.04871110990643501, 0.38709136843681335], "agreement_weight": 0.02290692925453186, "orthogonal_weight": 1.8544189929962158}, {"head": 1, "memory_to_active": 0.47388315200805664, "contribution_to_active": 0.2817171812057495, "mix_mean": 0.2079828679561615, "mix_low": 0.13067245483398438, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8731441497802734, "beta_mean": 0.2419479787349701, "nonfinite": 0.0, "weighted_memory_to_active": 0.212214395403862, "mix_std": 0.23882335424423218, "beta_std": 0.24376624822616577, "beta_low": 0.110107421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.018959492444992065, "evidence_weights": [0.0039048073813319206, -0.19278497993946075, -0.09414259344339371], "agreement_weight": 0.05910220369696617, "orthogonal_weight": 0.2711252272129059}, {"head": 2, "memory_to_active": 0.12286806106567383, "contribution_to_active": 0.5825364589691162, "mix_mean": 0.5894855856895447, "mix_low": 0.0027513504028320312, "mix_high": 0.0047206878662109375, "read_cosine": 0.38332149386405945, "beta_mean": 0.0019421695033088326, "nonfinite": 0.0, "weighted_memory_to_active": 0.07959374040365219, "mix_std": 0.2686917781829834, "beta_std": 0.0025199861265718937, "beta_low": 0.97991943359375, "beta_high": 0.0, "orthogonal_to_active": 0.06260120868682861, "evidence_weights": [-0.13179616630077362, -0.3153369724750519, 0.9186264872550964], "agreement_weight": 0.08696535974740982, "orthogonal_weight": -0.8800321221351624}, {"head": 3, "memory_to_active": 0.04400632530450821, "contribution_to_active": 0.5980242490768433, "mix_mean": 0.6777305603027344, "mix_low": 0.0042572021484375, "mix_high": 0.02268505096435547, "read_cosine": -0.05480619892477989, "beta_mean": 0.00523984432220459, "nonfinite": 0.0, "weighted_memory_to_active": 0.028470024466514587, "mix_std": 0.2572321891784668, "beta_std": 0.007669489830732346, "beta_low": 0.85626220703125, "beta_high": 0.0, "orthogonal_to_active": 0.021065201610326767, "evidence_weights": [-0.3420688509941101, -0.31115153431892395, 0.29412129521369934], "agreement_weight": 0.07052630186080933, "orthogonal_weight": -0.834909200668335}, {"head": 4, "memory_to_active": 0.15653447806835175, "contribution_to_active": 0.50038743019104, "mix_mean": 0.527637779712677, "mix_low": 0.003803253173828125, "mix_high": 0.008604049682617188, "read_cosine": 0.9069417715072632, "beta_mean": 0.034878432750701904, "nonfinite": 0.0, "weighted_memory_to_active": 0.09102436155080795, "mix_std": 0.26007699966430664, "beta_std": 0.02653648518025875, "beta_low": 0.14093017578125, "beta_high": 0.0, "orthogonal_to_active": 0.02365271933376789, "evidence_weights": [-0.2085818499326706, -0.23714369535446167, 0.4379233419895172], "agreement_weight": 0.1337466686964035, "orthogonal_weight": -0.6114657521247864}, {"head": 5, "memory_to_active": 0.18822921812534332, "contribution_to_active": 0.3426112234592438, "mix_mean": 0.38693615794181824, "mix_low": 0.003949165344238281, "mix_high": 0.0014276504516601562, "read_cosine": 0.7449411749839783, "beta_mean": 0.0011674941051751375, "nonfinite": 0.0, "weighted_memory_to_active": 0.07195168733596802, "mix_std": 0.2548627257347107, "beta_std": 0.006780072581022978, "beta_low": 0.9793701171875, "beta_high": 0.0, "orthogonal_to_active": 0.036809176206588745, "evidence_weights": [-0.1744949072599411, -0.18122951686382294, 0.35894808173179626], "agreement_weight": 0.06580454856157303, "orthogonal_weight": -0.7544262409210205}]}, {"layer": 15, "memory_to_active": 0.05478288233280182, "contribution_to_active": 0.4909074306488037, "mix_mean": 0.43270552158355713, "mix_low": 0.008209070190787315, "mix_high": 0.003034115070477128, "read_cosine": 0.6668963432312012, "beta_mean": 0.03817816078662872, "nonfinite": 0.0, "weighted_memory_to_active": 0.027930542826652527, "mix_std": 0.28497645258903503, "beta_std": 0.04343818128108978, "beta_low": 0.7061462998390198, "beta_high": 0.0, "orthogonal_to_active": 0.009442167356610298, "output_influence": 0.3153921067714691, "residual_influence": 0.186278834939003, "state_active": 0.17306692898273468, "state_memory": 0.010473448783159256, "state_peak": 28.789175033569336, "growth_active": 1.1848857402801514, "growth_memory": 1.3994455337524414, "heads": [{"head": 0, "memory_to_active": 0.11858128011226654, "contribution_to_active": 0.48316115140914917, "mix_mean": 0.39833399653434753, "mix_low": 0.009160995483398438, "mix_high": 0.007061004638671875, "read_cosine": 0.5460399389266968, "beta_mean": 0.03977779671549797, "nonfinite": 0.0, "weighted_memory_to_active": 0.0603325180709362, "mix_std": 0.28666725754737854, "beta_std": 0.02768203243613243, "beta_low": 0.06982421875, "beta_high": 0.0, "orthogonal_to_active": 0.03473246097564697, "evidence_weights": [-0.19007889926433563, -0.19559523463249207, 0.5047000646591187], "agreement_weight": 0.09136193245649338, "orthogonal_weight": -0.8111670017242432}, {"head": 1, "memory_to_active": 0.09881315380334854, "contribution_to_active": 0.42642050981521606, "mix_mean": 0.44102442264556885, "mix_low": 0.001068115234375, "mix_high": 0.005860328674316406, "read_cosine": 0.9320522546768188, "beta_mean": 0.00027760278317146003, "nonfinite": 0.0, "weighted_memory_to_active": 0.04250680282711983, "mix_std": 0.2844116985797882, "beta_std": 0.0016269254265353084, "beta_low": 0.99859619140625, "beta_high": 0.0, "orthogonal_to_active": 0.002552580088376999, "evidence_weights": [-0.1528308093547821, -0.16260375082492828, 0.16592706739902496], "agreement_weight": 0.12136130779981613, "orthogonal_weight": 0.13225442171096802}, {"head": 2, "memory_to_active": 0.006653851829469204, "contribution_to_active": 0.5001145005226135, "mix_mean": 0.35966551303863525, "mix_low": 0.012135505676269531, "mix_high": 0.00012683868408203125, "read_cosine": 0.7654707431793213, "beta_mean": 9.570906695444137e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035204461310058832, "mix_std": 0.27346673607826233, "beta_std": 0.0002197652356699109, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.001136637874878943, "evidence_weights": [-0.2033822387456894, -0.26649847626686096, 0.11374396830797195], "agreement_weight": 0.009080551564693451, "orthogonal_weight": 0.8913103342056274}, {"head": 3, "memory_to_active": 0.001475324621424079, "contribution_to_active": 0.3735239803791046, "mix_mean": 0.4166664183139801, "mix_low": 0.004866600036621094, "mix_high": 0.0005865097045898438, "read_cosine": 0.6267837285995483, "beta_mean": 7.827041372365784e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0004787908401340246, "mix_std": 0.2899765074253082, "beta_std": 1.8672570149647072e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003064161282964051, "evidence_weights": [-0.2741423547267914, -0.23671108484268188, 0.1907368004322052], "agreement_weight": 0.04949616640806198, "orthogonal_weight": -0.6181690692901611}, {"head": 4, "memory_to_active": 0.07476484775543213, "contribution_to_active": 0.6346508860588074, "mix_mean": 0.39242976903915405, "mix_low": 0.011877059936523438, "mix_high": 0.0011110305786132812, "read_cosine": 0.769539475440979, "beta_mean": 0.1864728331565857, "nonfinite": 0.0, "weighted_memory_to_active": 0.04207882285118103, "mix_std": 0.30913808941841125, "beta_std": 0.21400676667690277, "beta_low": 0.20062255859375, "beta_high": 0.0, "orthogonal_to_active": 0.0068211425095796585, "evidence_weights": [-0.20662084221839905, -0.2539638876914978, -0.020963545888662338], "agreement_weight": 0.01242220588028431, "orthogonal_weight": 0.37424150109291077}, {"head": 5, "memory_to_active": 0.028408829122781754, "contribution_to_active": 0.5275734066963196, "mix_mean": 0.5881128311157227, "mix_low": 0.010146141052246094, "mix_high": 0.0034589767456054688, "read_cosine": 0.3614920675754547, "beta_mean": 0.0024372022598981857, "nonfinite": 0.0, "weighted_memory_to_active": 0.018665868788957596, "mix_std": 0.26619836688041687, "beta_std": 0.017074914649128914, "beta_low": 0.9678955078125, "beta_high": 0.0, "orthogonal_to_active": 0.01110376138240099, "evidence_weights": [-0.28204345703125, -0.3040468394756317, 0.626904308795929], "agreement_weight": 0.3155219852924347, "orthogonal_weight": 0.8095738887786865}]}]}, {"step": 68000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8058648859987443, "memory_to_active": 0.7958261370658875, "contribution_to_active": 0.5712925791740417, "mix_mean": 0.42596593499183655, "mix_low": 0.12401939928531647, "mix_high": 0.010759145021438599, "read_cosine": 0.34395480155944824, "beta_mean": 0.06807580590248108, "nonfinite": 0.0, "weighted_memory_to_active": 0.20127367973327637, "mix_std": 0.22345411777496338, "beta_std": 0.06013220176100731, "beta_low": 0.5869528651237488, "beta_high": 2.733866494963877e-05, "orthogonal_to_active": 0.06339854001998901, "samples": 4, "scored_tokens": 899, "normal_loss": 3.062025785446167, "normal_token_accuracy": 0.4482758641242981, "state_active": 0.06457269191741943, "state_memory": 0.016863878816366196, "state_peak": 127.17990112304688, "growth_active": 1.133177399635315, "growth_memory": 1.289717674255371, "output_influence": 0.44045084714889526, "residual_influence": 0.1564323753118515, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.183096170425415, "no_memory_loss_delta": 0.6430470943450928, "scrambled_loss_delta": 0.15586546063423157, "memory_only_loss_delta": 3.5278635025024414, "no_evidence_loss_delta": 0.3601154386997223, "no_agreement_loss_delta": 0.01649506203830242, "no_orthogonal_loss_delta": 0.020483531057834625}, {"index": 94, "tokens": 256, "normal_loss": 3.1873114109039307, "no_memory_loss_delta": 0.6413576602935791, "scrambled_loss_delta": 0.41251349449157715, "memory_only_loss_delta": 3.746488571166992, "no_evidence_loss_delta": 0.3048664629459381, "no_agreement_loss_delta": 0.03129928931593895, "no_orthogonal_loss_delta": 0.04981200397014618}, {"index": 188, "tokens": 256, "normal_loss": 3.3773412704467773, "no_memory_loss_delta": 0.941077470779419, "scrambled_loss_delta": 0.11940321326255798, "memory_only_loss_delta": 3.6527886390686035, "no_evidence_loss_delta": 0.5097143054008484, "no_agreement_loss_delta": 0.04061039164662361, "no_orthogonal_loss_delta": 0.04484396427869797}, {"index": 283, "tokens": 131, "normal_loss": 1.9644099473953247, "no_memory_loss_delta": 0.3279843032360077, "scrambled_loss_delta": 0.18163356184959412, "memory_only_loss_delta": 2.6760590076446533, "no_evidence_loss_delta": 0.07387150824069977, "no_agreement_loss_delta": -0.027293212711811066, "no_orthogonal_loss_delta": -0.005833256524056196}], "agreement_weight_abs_mean": 0.24224638938903809, "agreement_weight_abs_max": 6.599942207336426, "agreement_weight_mean": 0.21052372455596924, "orthogonal_weight_abs_mean": 0.9293096661567688, "orthogonal_weight_abs_max": 10.7256441116333, "orthogonal_weight_mean": -0.06819771230220795, "no_memory_loss": 3.743549346923828, "no_memory_loss_delta": 0.6815232634544373, "no_memory_token_accuracy_gain": 0.09788653999567032, "no_memory_token_win_fraction": 0.6763070225715637, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 49.41541313229058, "scrambled_loss": 3.28434681892395, "scrambled_loss_delta": 0.22232064604759216, "scrambled_token_accuracy_gain": 0.023359287530183792, "scrambled_token_win_fraction": 0.6017797589302063, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.93414048431596, "memory_only_loss": 6.563596248626709, "memory_only_loss_delta": 3.501570224761963, "memory_only_token_accuracy_gain": 0.29365962743759155, "memory_only_token_win_fraction": 0.9176863431930542, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.98499960489333, "no_evidence_loss": 3.4072976112365723, "no_evidence_loss_delta": 0.34527188539505005, "no_evidence_token_accuracy_gain": 0.051167964935302734, "no_evidence_token_win_fraction": 0.6262513995170593, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 29.197217512874236, "no_agreement_loss": 3.0832231044769287, "no_agreement_loss_delta": 0.021197112277150154, "no_agreement_token_accuracy_gain": 0.0033370412420481443, "no_agreement_token_win_fraction": 0.4949944317340851, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.0974032488805614, "no_orthogonal_loss": 3.093963146209717, "no_orthogonal_loss_delta": 0.031937215477228165, "no_orthogonal_token_accuracy_gain": 0.004449388012290001, "no_orthogonal_token_win_fraction": 0.6062291264533997, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 3.1432608788328986, "short_no_memory_loss_delta": 0.6287510991096497, "long_context_memory_gain": 0.0527721643447876, "normal_context_loss_gain": 0.13291220366954803, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.554812431335449, "contribution_to_active": 1.2065694332122803, "mix_mean": 0.10762976109981537, "mix_low": 0.511313796043396, "mix_high": 0.004726727958768606, "read_cosine": 0.4100942313671112, "beta_mean": 0.20538577437400818, "nonfinite": 0.0, "weighted_memory_to_active": 1.2315242290496826, "mix_std": 0.09145838767290115, "beta_std": 0.1418696641921997, "beta_low": 0.1492716521024704, "beta_high": 0.0, "orthogonal_to_active": 0.047736890614032745, "output_influence": 0.21171072125434875, "residual_influence": 0.23619890213012695, "state_active": 0.06685144454240799, "state_memory": 0.04463936761021614, "state_peak": 49.34263229370117, "growth_active": 1.2568235397338867, "growth_memory": 1.2869458198547363, "heads": [{"head": 0, "memory_to_active": 2.747291088104248, "contribution_to_active": 0.4762299358844757, "mix_mean": 0.02053140103816986, "mix_low": 0.618556022644043, "mix_high": 9.5367431640625e-07, "read_cosine": 0.20910729467868805, "beta_mean": 0.1948842704296112, "nonfinite": 0.0, "weighted_memory_to_active": 0.5962352156639099, "mix_std": 0.05961253494024277, "beta_std": 0.10151143372058868, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06652140617370605, "evidence_weights": [-0.006103247404098511, -0.04542146995663643, -0.053265225142240524], "agreement_weight": -0.2215728610754013, "orthogonal_weight": -0.12644751369953156}, {"head": 1, "memory_to_active": 7.161638259887695, "contribution_to_active": 3.074711322784424, "mix_mean": 0.019807979464530945, "mix_low": 0.6144895553588867, "mix_high": 0.0, "read_cosine": 0.4298848509788513, "beta_mean": 0.4244423806667328, "nonfinite": 0.0, "weighted_memory_to_active": 3.317561149597168, "mix_std": 0.05546927452087402, "beta_std": 0.22081072628498077, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05679064244031906, "evidence_weights": [0.016772214323282242, -0.11194133013486862, 0.6802149415016174], "agreement_weight": 0.025123661383986473, "orthogonal_weight": 0.037464410066604614}, {"head": 2, "memory_to_active": 2.741244077682495, "contribution_to_active": 0.6146638989448547, "mix_mean": 0.015451908111572266, "mix_low": 0.6383800506591797, "mix_high": 0.0, "read_cosine": 0.24722497165203094, "beta_mean": 0.24946442246437073, "nonfinite": 0.0, "weighted_memory_to_active": 0.7138147950172424, "mix_std": 0.03360706567764282, "beta_std": 0.19145852327346802, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.058863427489995956, "evidence_weights": [0.059931688010692596, 0.045266564935445786, 0.2525938153266907], "agreement_weight": 0.013625401072204113, "orthogonal_weight": -0.20418819785118103}, {"head": 3, "memory_to_active": 7.160545349121094, "contribution_to_active": 2.5427823066711426, "mix_mean": 0.02974465675652027, "mix_low": 0.48203372955322266, "mix_high": 0.0, "read_cosine": 0.31087949872016907, "beta_mean": 0.3159666657447815, "nonfinite": 0.0, "weighted_memory_to_active": 2.6631574630737305, "mix_std": 0.068304643034935, "beta_std": 0.2603393495082855, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.08764396607875824, "evidence_weights": [-0.03845689073204994, -0.03306032717227936, -0.1256067454814911], "agreement_weight": -0.007858380675315857, "orthogonal_weight": -0.03897574916481972}, {"head": 4, "memory_to_active": 1.4159034490585327, "contribution_to_active": 0.06250554323196411, "mix_mean": 0.01630263403058052, "mix_low": 0.6999139785766602, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3252784013748169, "beta_mean": 0.030287159606814384, "nonfinite": 0.0, "weighted_memory_to_active": 0.04501553624868393, "mix_std": 0.04875790327787399, "beta_std": 0.060211751610040665, "beta_low": 0.50347900390625, "beta_high": 0.0, "orthogonal_to_active": 0.00120418646838516, "evidence_weights": [0.10832340270280838, 0.07282890379428864, -0.11601583659648895], "agreement_weight": -0.19795730710029602, "orthogonal_weight": 0.02525492012500763}, {"head": 5, "memory_to_active": 0.10225117206573486, "contribution_to_active": 0.4685229957103729, "mix_mean": 0.543940007686615, "mix_low": 0.014509201049804688, "mix_high": 0.02835845947265625, "read_cosine": 0.9381902813911438, "beta_mean": 0.01726965233683586, "nonfinite": 0.0, "weighted_memory_to_active": 0.05336131528019905, "mix_std": 0.28299885988235474, "beta_std": 0.016886206343770027, "beta_low": 0.39208984375, "beta_high": 0.0, "orthogonal_to_active": 0.015397707931697369, "evidence_weights": [-0.1757434606552124, -0.21928271651268005, 0.37705928087234497], "agreement_weight": 0.2882766127586365, "orthogonal_weight": -0.6954830288887024}]}, {"layer": 1, "memory_to_active": 2.4015824794769287, "contribution_to_active": 0.4732555150985718, "mix_mean": 0.21631649136543274, "mix_low": 0.36126360297203064, "mix_high": 0.001100699184462428, "read_cosine": 0.28032049536705017, "beta_mean": 0.17080405354499817, "nonfinite": 0.0, "weighted_memory_to_active": 0.30604487657546997, "mix_std": 0.11385923624038696, "beta_std": 0.12294803559780121, "beta_low": 0.3516438901424408, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.038378167897462845, "output_influence": 0.26225635409355164, "residual_influence": 0.07195848226547241, "state_active": 0.02900667116045952, "state_memory": 0.03458815813064575, "state_peak": 40.91484451293945, "growth_active": 1.1948436498641968, "growth_memory": 1.4544492959976196, "heads": [{"head": 0, "memory_to_active": 0.020652078092098236, "contribution_to_active": 0.4912022054195404, "mix_mean": 0.5385561585426331, "mix_low": 0.004618644714355469, "mix_high": 0.0020284652709960938, "read_cosine": -0.0005845734849572182, "beta_mean": 0.00014141474093776196, "nonfinite": 0.0, "weighted_memory_to_active": 0.01254564430564642, "mix_std": 0.27981242537498474, "beta_std": 0.00026441540103405714, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012050442397594452, "evidence_weights": [-0.2108778953552246, -0.2745194137096405, 0.27405858039855957], "agreement_weight": 0.19673894345760345, "orthogonal_weight": -0.9998761415481567}, {"head": 1, "memory_to_active": 2.9876198768615723, "contribution_to_active": 0.2714937627315521, "mix_mean": 0.022771596908569336, "mix_low": 0.46808433532714844, "mix_high": 0.0, "read_cosine": 0.48767349123954773, "beta_mean": 0.24260424077510834, "nonfinite": 0.0, "weighted_memory_to_active": 0.2924957871437073, "mix_std": 0.043217916041612625, "beta_std": 0.17587262392044067, "beta_low": 0.00177001953125, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.023742059245705605, "evidence_weights": [0.019181160256266594, -0.06641455739736557, 0.04647853597998619], "agreement_weight": -0.0014126482419669628, "orthogonal_weight": 0.21141235530376434}, {"head": 2, "memory_to_active": 0.04204477369785309, "contribution_to_active": 0.5640987753868103, "mix_mean": 0.6736438274383545, "mix_low": 0.0005006790161132812, "mix_high": 0.0045757293701171875, "read_cosine": 0.42177069187164307, "beta_mean": 0.0014498220989480615, "nonfinite": 0.0, "weighted_memory_to_active": 0.027842272073030472, "mix_std": 0.22959502041339874, "beta_std": 0.004269573837518692, "beta_low": 0.97991943359375, "beta_high": 0.0, "orthogonal_to_active": 0.019223222509026527, "evidence_weights": [-0.2751601040363312, -0.3709843158721924, 0.6672656536102295], "agreement_weight": 0.04907064884901047, "orthogonal_weight": -0.8196979761123657}, {"head": 3, "memory_to_active": 3.132370948791504, "contribution_to_active": 0.16353674232959747, "mix_mean": 0.024518445134162903, "mix_low": 0.5145244598388672, "mix_high": 0.0, "read_cosine": 0.18880504369735718, "beta_mean": 0.35533607006073, "nonfinite": 0.0, "weighted_memory_to_active": 0.22313958406448364, "mix_std": 0.061300478875637054, "beta_std": 0.26543211936950684, "beta_low": 0.0606689453125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.09878101944923401, "evidence_weights": [0.03745310381054878, -0.01915094442665577, -0.017980895936489105], "agreement_weight": -0.03275365009903908, "orthogonal_weight": -0.4947349727153778}, {"head": 4, "memory_to_active": 3.8819408416748047, "contribution_to_active": 1.262379765510559, "mix_mean": 0.0325801745057106, "mix_low": 0.31719207763671875, "mix_high": 0.0, "read_cosine": 0.3212043344974518, "beta_mean": 0.15480667352676392, "nonfinite": 0.0, "weighted_memory_to_active": 1.221562147140503, "mix_std": 0.053734760731458664, "beta_std": 0.12170487642288208, "beta_low": 0.0400390625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.053463928401470184, "evidence_weights": [-0.023681413382291794, -0.008624320849776268, 0.14820000529289246], "agreement_weight": -0.009518376551568508, "orthogonal_weight": 0.06571070104837418}, {"head": 5, "memory_to_active": 4.344865798950195, "contribution_to_active": 0.08682164549827576, "mix_mean": 0.005828568711876869, "mix_low": 0.8626613616943359, "mix_high": 0.0, "read_cosine": 0.2630539536476135, "beta_mean": 0.27048608660697937, "nonfinite": 0.0, "weighted_memory_to_active": 0.058683644980192184, "mix_std": 0.015494809485971928, "beta_std": 0.17014452815055847, "beta_low": 0.0274658203125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.023008331656455994, "evidence_weights": [0.1669912189245224, 0.08489608764648438, 0.09760445356369019], "agreement_weight": -0.11648178845643997, "orthogonal_weight": 0.4059102237224579}]}, {"layer": 2, "memory_to_active": 1.0048084259033203, "contribution_to_active": 0.3620923161506653, "mix_mean": 0.4149198532104492, "mix_low": 0.2414042353630066, "mix_high": 0.004822413437068462, "read_cosine": 0.29640042781829834, "beta_mean": 0.08110075443983078, "nonfinite": 0.0, "weighted_memory_to_active": 0.08075721561908722, "mix_std": 0.18625152111053467, "beta_std": 0.06927032768726349, "beta_low": 0.6021525263786316, "beta_high": 8.138021075865254e-05, "orthogonal_to_active": 0.017981652170419693, "output_influence": 0.4144432842731476, "residual_influence": 0.1170773133635521, "state_active": 0.04066508263349533, "state_memory": 0.014308324083685875, "state_peak": 26.891185760498047, "growth_active": 1.0529829263687134, "growth_memory": 1.5433954000473022, "heads": [{"head": 0, "memory_to_active": 0.004922861233353615, "contribution_to_active": 0.3211946487426758, "mix_mean": 0.6494675874710083, "mix_low": 0.0006380081176757812, "mix_high": 0.009801864624023438, "read_cosine": 0.14868666231632233, "beta_mean": 4.6251429012045264e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003237431403249502, "mix_std": 0.26485371589660645, "beta_std": 8.393183816224337e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003466635476797819, "evidence_weights": [-0.3752745985984802, -0.21823737025260925, 0.21880397200584412], "agreement_weight": 0.14829634130001068, "orthogonal_weight": -1.1091874837875366}, {"head": 1, "memory_to_active": 3.1176648139953613, "contribution_to_active": 0.04134261608123779, "mix_mean": 0.005379729438573122, "mix_low": 0.880305290222168, "mix_high": 0.0, "read_cosine": 0.12975719571113586, "beta_mean": 0.22011661529541016, "nonfinite": 0.0, "weighted_memory_to_active": 0.033931586891412735, "mix_std": 0.011926867067813873, "beta_std": 0.15786798298358917, "beta_low": 0.00286865234375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.010186523199081421, "evidence_weights": [0.14044561982154846, 0.11409661173820496, 0.5870698094367981], "agreement_weight": 0.02550673298537731, "orthogonal_weight": 0.3385363817214966}, {"head": 2, "memory_to_active": 0.011327984742820263, "contribution_to_active": 0.42954471707344055, "mix_mean": 0.5607692003250122, "mix_low": 0.0012063980102539062, "mix_high": 0.0040836334228515625, "read_cosine": 0.35122913122177124, "beta_mean": 3.114179344265722e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004786709323525429, "mix_std": 0.2742811143398285, "beta_std": 4.9215836043003947e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 8.522494317730889e-05, "evidence_weights": [-0.25308501720428467, -0.29575812816619873, 0.5814946293830872], "agreement_weight": -0.01343050692230463, "orthogonal_weight": 0.021533921360969543}, {"head": 3, "memory_to_active": 0.10809515416622162, "contribution_to_active": 0.5682626962661743, "mix_mean": 0.5877844095230103, "mix_low": 0.00021076202392578125, "mix_high": 0.006638526916503906, "read_cosine": 0.5532217025756836, "beta_mean": 0.019183330237865448, "nonfinite": 0.0, "weighted_memory_to_active": 0.0691380575299263, "mix_std": 0.25395020842552185, "beta_std": 0.029713131487369537, "beta_low": 0.56268310546875, "beta_high": 0.0, "orthogonal_to_active": 0.04872909188270569, "evidence_weights": [-0.20562832057476044, -0.32866716384887695, 0.7598186731338501], "agreement_weight": 0.0541854090988636, "orthogonal_weight": -0.8946354389190674}, {"head": 4, "memory_to_active": 2.7752413749694824, "contribution_to_active": 0.27940627932548523, "mix_mean": 0.021801743656396866, "mix_low": 0.5654449462890625, "mix_high": 0.0, "read_cosine": 0.2018350511789322, "beta_mean": 0.2471872717142105, "nonfinite": 0.0, "weighted_memory_to_active": 0.3670330345630646, "mix_std": 0.0539773553609848, "beta_std": 0.22785305976867676, "beta_low": 0.04736328125, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.04373515024781227, "evidence_weights": [0.022983351722359657, 0.002404445316642523, 0.053200386464595795], "agreement_weight": 0.021058538928627968, "orthogonal_weight": -0.13887335360050201}, {"head": 5, "memory_to_active": 0.01159863080829382, "contribution_to_active": 0.5328029990196228, "mix_mean": 0.6643162965774536, "mix_low": 0.0006198883056640625, "mix_high": 0.008410453796386719, "read_cosine": 0.3936728239059448, "beta_mean": 3.991015910287388e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0064164879731833935, "mix_std": 0.25851970911026, "beta_std": 5.466705624712631e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0016872776905074716, "evidence_weights": [-0.2806577980518341, -0.29240652918815613, 0.8792715668678284], "agreement_weight": 0.24877135455608368, "orthogonal_weight": -0.3202970623970032}]}, {"layer": 3, "memory_to_active": 0.4608380198478699, "contribution_to_active": 0.48764389753341675, "mix_mean": 0.519433319568634, "mix_low": 0.05169598385691643, "mix_high": 0.011552175506949425, "read_cosine": 0.20629307627677917, "beta_mean": 0.0260484516620636, "nonfinite": 0.0, "weighted_memory_to_active": 0.13369449973106384, "mix_std": 0.2574350833892822, "beta_std": 0.025052009150385857, "beta_low": 0.7716675400733948, "beta_high": 0.0, "orthogonal_to_active": 0.053946252912282944, "output_influence": 0.4282947778701782, "residual_influence": 0.10389229655265808, "state_active": 0.023309867829084396, "state_memory": 0.006726493127644062, "state_peak": 22.328441619873047, "growth_active": 1.0470460653305054, "growth_memory": 1.570250153541565, "heads": [{"head": 0, "memory_to_active": 0.366576611995697, "contribution_to_active": 0.6006633043289185, "mix_mean": 0.6124086976051331, "mix_low": 0.0019960403442382812, "mix_high": 0.025516510009765625, "read_cosine": 0.2660898268222809, "beta_mean": 0.009731827303767204, "nonfinite": 0.0, "weighted_memory_to_active": 0.23652911186218262, "mix_std": 0.3011927008628845, "beta_std": 0.013801878318190575, "beta_low": 0.7183837890625, "beta_high": 0.0, "orthogonal_to_active": 0.17199359834194183, "evidence_weights": [-0.2178855836391449, -0.2530191242694855, 1.0405789613723755], "agreement_weight": 0.11431831866502762, "orthogonal_weight": -0.7727880477905273}, {"head": 1, "memory_to_active": 0.011668476276099682, "contribution_to_active": 0.4703117907047272, "mix_mean": 0.5853209495544434, "mix_low": 0.0032701492309570312, "mix_high": 0.00994873046875, "read_cosine": 0.18747738003730774, "beta_mean": 5.8820631238631904e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00710640661418438, "mix_std": 0.2917468547821045, "beta_std": 0.00013459401088766754, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005653322208672762, "evidence_weights": [-0.27003908157348633, -0.2174660861492157, 0.702264130115509], "agreement_weight": 0.601620614528656, "orthogonal_weight": -0.8724137544631958}, {"head": 2, "memory_to_active": 2.114832878112793, "contribution_to_active": 0.31081053614616394, "mix_mean": 0.0568404346704483, "mix_low": 0.29570674896240234, "mix_high": 2.86102294921875e-06, "read_cosine": 0.215577632188797, "beta_mean": 0.14175213873386383, "nonfinite": 0.0, "weighted_memory_to_active": 0.35314106941223145, "mix_std": 0.11088612675666809, "beta_std": 0.12513843178749084, "beta_low": 0.0162353515625, "beta_high": 0.0, "orthogonal_to_active": 0.0003047374775633216, "evidence_weights": [-0.10454706102609634, -0.010503338649868965, -0.0437442921102047], "agreement_weight": -0.010963816195726395, "orthogonal_weight": -0.0010788263753056526}, {"head": 3, "memory_to_active": 0.021476756781339645, "contribution_to_active": 0.5294400453567505, "mix_mean": 0.6047145128250122, "mix_low": 0.00383758544921875, "mix_high": 0.007647514343261719, "read_cosine": 0.27417778968811035, "beta_mean": 9.3231545179151e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014494670554995537, "mix_std": 0.28331273794174194, "beta_std": 0.00021581153851002455, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0036846790462732315, "evidence_weights": [-0.26333993673324585, -0.2241583615541458, 2.284808397293091], "agreement_weight": 2.238964796066284, "orthogonal_weight": -0.32589221000671387}, {"head": 4, "memory_to_active": 0.011863397434353828, "contribution_to_active": 0.3794569671154022, "mix_mean": 0.5286276340484619, "mix_low": 0.002857208251953125, "mix_high": 0.0028553009033203125, "read_cosine": 0.14502771198749542, "beta_mean": 6.28622728982009e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007895959541201591, "mix_std": 0.28866738080978394, "beta_std": 0.00010078799823531881, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0074430061504244804, "evidence_weights": [-0.24064864218235016, -0.22335076332092285, 0.08700061589479446], "agreement_weight": -0.06386800855398178, "orthogonal_weight": -1.0189627408981323}, {"head": 5, "memory_to_active": 0.23860973119735718, "contribution_to_active": 0.6351807713508606, "mix_mean": 0.7286877632141113, "mix_low": 0.0025081634521484375, "mix_high": 0.023342132568359375, "read_cosine": 0.1494080275297165, "beta_mean": 0.004591822624206543, "nonfinite": 0.0, "weighted_memory_to_active": 0.18299974501132965, "mix_std": 0.2688046395778656, "beta_std": 0.010920542292296886, "beta_low": 0.89544677734375, "beta_high": 0.0, "orthogonal_to_active": 0.13459816575050354, "evidence_weights": [-0.26219430565834045, -0.3426167070865631, 0.5349159836769104], "agreement_weight": 0.23064392805099487, "orthogonal_weight": -0.7783788442611694}]}, {"layer": 4, "memory_to_active": 1.013169288635254, "contribution_to_active": 0.4783490300178528, "mix_mean": 0.37182483077049255, "mix_low": 0.23967315256595612, "mix_high": 0.01074981689453125, "read_cosine": 0.2964867949485779, "beta_mean": 0.11113353818655014, "nonfinite": 0.0, "weighted_memory_to_active": 0.13790325820446014, "mix_std": 0.1628131866455078, "beta_std": 0.10297264903783798, "beta_low": 0.39031982421875, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05861919745802879, "output_influence": 0.4959792494773865, "residual_influence": 0.1550815999507904, "state_active": 0.03660797327756882, "state_memory": 0.01670498214662075, "state_peak": 42.07035446166992, "growth_active": 1.1648753881454468, "growth_memory": 1.502738356590271, "heads": [{"head": 0, "memory_to_active": 0.16409465670585632, "contribution_to_active": 0.8000261783599854, "mix_mean": 0.6837100982666016, "mix_low": 0.011027336120605469, "mix_high": 0.027917861938476562, "read_cosine": 0.4565654993057251, "beta_mean": 0.2544857859611511, "nonfinite": 0.0, "weighted_memory_to_active": 0.14074677228927612, "mix_std": 0.34851887822151184, "beta_std": 0.25121963024139404, "beta_low": 0.23907470703125, "beta_high": 0.0, "orthogonal_to_active": 0.08995117992162704, "evidence_weights": [-0.25193914771080017, -0.36284786462783813, -0.17398370802402496], "agreement_weight": -0.005410479847341776, "orthogonal_weight": 0.7846199870109558}, {"head": 1, "memory_to_active": 0.04668816924095154, "contribution_to_active": 0.6648781299591064, "mix_mean": 0.7050685286521912, "mix_low": 9.822845458984375e-05, "mix_high": 0.019417762756347656, "read_cosine": 0.5301475524902344, "beta_mean": 0.0010203184792771935, "nonfinite": 0.0, "weighted_memory_to_active": 0.03479908034205437, "mix_std": 0.23861666023731232, "beta_std": 0.007638928014785051, "beta_low": 0.98016357421875, "beta_high": 0.0, "orthogonal_to_active": 0.017982760444283485, "evidence_weights": [-0.23925718665122986, -0.367527037858963, 0.5950007438659668], "agreement_weight": 0.35227400064468384, "orthogonal_weight": -0.7146176099777222}, {"head": 2, "memory_to_active": 1.569536566734314, "contribution_to_active": 0.33304518461227417, "mix_mean": 0.048971422016620636, "mix_low": 0.34450721740722656, "mix_high": 2.288818359375e-05, "read_cosine": 0.22562618553638458, "beta_mean": 0.06621667742729187, "nonfinite": 0.0, "weighted_memory_to_active": 0.2914012670516968, "mix_std": 0.08851466327905655, "beta_std": 0.0683538168668747, "beta_low": 0.07904052734375, "beta_high": 0.0, "orthogonal_to_active": 0.1005677878856659, "evidence_weights": [0.03111896477639675, -0.05448903515934944, 0.3052944242954254], "agreement_weight": 0.11966383457183838, "orthogonal_weight": -0.46967270970344543}, {"head": 3, "memory_to_active": 1.746410608291626, "contribution_to_active": 0.14417338371276855, "mix_mean": 0.03437839448451996, "mix_low": 0.39751338958740234, "mix_high": 0.0, "read_cosine": 0.24052280187606812, "beta_mean": 0.12643775343894958, "nonfinite": 0.0, "weighted_memory_to_active": 0.19760136306285858, "mix_std": 0.059799835085868835, "beta_std": 0.11926597356796265, "beta_low": 0.04351806640625, "beta_high": 0.0, "orthogonal_to_active": 8.328506373800337e-05, "evidence_weights": [-0.02213902212679386, 0.04752379655838013, -0.2629111409187317], "agreement_weight": 0.08149190247058868, "orthogonal_weight": 0.0006103626801632345}, {"head": 4, "memory_to_active": 0.006907276809215546, "contribution_to_active": 0.6386585235595703, "mix_mean": 0.7465717792510986, "mix_low": 3.528594970703125e-05, "mix_high": 0.01713085174560547, "read_cosine": 0.13131743669509888, "beta_mean": 1.6957408661255613e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004977931268513203, "mix_std": 0.21260443329811096, "beta_std": 3.7384190363809466e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003216796787455678, "evidence_weights": [-0.33004289865493774, -0.3187158703804016, 0.28346937894821167], "agreement_weight": 0.3142259120941162, "orthogonal_weight": -0.6832952499389648}, {"head": 5, "memory_to_active": 2.5453789234161377, "contribution_to_active": 0.2893124222755432, "mix_mean": 0.012248871847987175, "mix_low": 0.6848573684692383, "mix_high": 9.5367431640625e-06, "read_cosine": 0.19474121928215027, "beta_mean": 0.21862372756004333, "nonfinite": 0.0, "weighted_memory_to_active": 0.1578930914402008, "mix_std": 0.028824694454669952, "beta_std": 0.17132017016410828, "beta_low": 0.0001220703125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1399133801460266, "evidence_weights": [0.06024117395281792, -0.01410620380192995, -0.272779643535614], "agreement_weight": -0.09159009903669357, "orthogonal_weight": 1.0784529447555542}]}, {"layer": 5, "memory_to_active": 0.35423073172569275, "contribution_to_active": 0.5422823429107666, "mix_mean": 0.5144162774085999, "mix_low": 0.05641285702586174, "mix_high": 0.020545005798339844, "read_cosine": 0.08009332418441772, "beta_mean": 0.051446229219436646, "nonfinite": 0.0, "weighted_memory_to_active": 0.12718313932418823, "mix_std": 0.25192221999168396, "beta_std": 0.06538331508636475, "beta_low": 0.676788330078125, "beta_high": 0.0, "orthogonal_to_active": 0.05309662967920303, "output_influence": 0.5723391771316528, "residual_influence": 0.18426603078842163, "state_active": 0.039935898035764694, "state_memory": 0.010249595157802105, "state_peak": 67.05699157714844, "growth_active": 1.0837994813919067, "growth_memory": 1.2588154077529907, "heads": [{"head": 0, "memory_to_active": 0.20504017174243927, "contribution_to_active": 0.5790538787841797, "mix_mean": 0.4862365424633026, "mix_low": 0.006848335266113281, "mix_high": 0.0017118453979492188, "read_cosine": -0.12211120873689651, "beta_mean": 0.01838718354701996, "nonfinite": 0.0, "weighted_memory_to_active": 0.08101494610309601, "mix_std": 0.2782372534275055, "beta_std": 0.03090602718293667, "beta_low": 0.54443359375, "beta_high": 0.0, "orthogonal_to_active": 0.03673681244254112, "evidence_weights": [-0.17001333832740784, -0.18854539096355438, 0.11098036915063858], "agreement_weight": -0.020545197650790215, "orthogonal_weight": -0.4747825860977173}, {"head": 1, "memory_to_active": 0.006139541044831276, "contribution_to_active": 0.5648210644721985, "mix_mean": 0.7064802646636963, "mix_low": 0.0008487701416015625, "mix_high": 0.04217529296875, "read_cosine": 0.23440980911254883, "beta_mean": 1.3324171959538944e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003862612647935748, "mix_std": 0.26710137724876404, "beta_std": 3.576964445528574e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035827322863042355, "evidence_weights": [-0.2479744404554367, -0.2866800129413605, 0.92314213514328], "agreement_weight": 0.6988481879234314, "orthogonal_weight": -0.9654555916786194}, {"head": 2, "memory_to_active": 0.015485640615224838, "contribution_to_active": 0.5973258018493652, "mix_mean": 0.6606699228286743, "mix_low": 0.0023851394653320312, "mix_high": 0.024120330810546875, "read_cosine": -0.4933149814605713, "beta_mean": 7.577618816867471e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009077498689293861, "mix_std": 0.2647944390773773, "beta_std": 0.0002872970071621239, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.008595028892159462, "evidence_weights": [-0.2599777281284332, -0.3205230236053467, 0.1665583997964859], "agreement_weight": -0.1717761904001236, "orthogonal_weight": -1.0429688692092896}, {"head": 3, "memory_to_active": 1.412664771080017, "contribution_to_active": 0.24147549271583557, "mix_mean": 0.11546986550092697, "mix_low": 0.1238565444946289, "mix_high": 0.0, "read_cosine": 0.15976542234420776, "beta_mean": 0.09493623673915863, "nonfinite": 0.0, "weighted_memory_to_active": 0.353782594203949, "mix_std": 0.14523965120315552, "beta_std": 0.14018285274505615, "beta_low": 0.21685791015625, "beta_high": 0.0, "orthogonal_to_active": 0.13992664217948914, "evidence_weights": [-0.07660454511642456, -0.09994439780712128, 0.10077983886003494], "agreement_weight": 0.05163222551345825, "orthogonal_weight": -0.500973641872406}, {"head": 4, "memory_to_active": 0.4221552610397339, "contribution_to_active": 0.5785455703735352, "mix_mean": 0.31179308891296387, "mix_low": 0.20305252075195312, "mix_high": 0.0043735504150390625, "read_cosine": 0.5448100566864014, "beta_mean": 0.19509214162826538, "nonfinite": 0.0, "weighted_memory_to_active": 0.27143555879592896, "mix_std": 0.33197492361068726, "beta_std": 0.22009682655334473, "beta_low": 0.30078125, "beta_high": 0.0, "orthogonal_to_active": 0.10411574691534042, "evidence_weights": [-0.11419174075126648, -0.25346484780311584, -0.0563904345035553], "agreement_weight": 0.032602664083242416, "orthogonal_weight": 0.609428882598877}, {"head": 5, "memory_to_active": 0.06389878690242767, "contribution_to_active": 0.6924723386764526, "mix_mean": 0.8058479428291321, "mix_low": 0.0014858245849609375, "mix_high": 0.050889015197753906, "read_cosine": 0.1570008099079132, "beta_mean": 0.000172693922650069, "nonfinite": 0.0, "weighted_memory_to_active": 0.04392557963728905, "mix_std": 0.2241857349872589, "beta_std": 0.0007911159191280603, "beta_low": 0.998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.02562282420694828, "evidence_weights": [-0.30428388714790344, -0.3283286392688751, 1.839544415473938], "agreement_weight": 0.50128173828125, "orthogonal_weight": -0.6175826787948608}]}, {"layer": 6, "memory_to_active": 0.25854188203811646, "contribution_to_active": 0.6430321931838989, "mix_mean": 0.5958467125892639, "mix_low": 0.036137741059064865, "mix_high": 0.030791282653808594, "read_cosine": 0.3071272075176239, "beta_mean": 0.06406479328870773, "nonfinite": 0.0, "weighted_memory_to_active": 0.1366325318813324, "mix_std": 0.2379610687494278, "beta_std": 0.07506349682807922, "beta_low": 0.6168314814567566, "beta_high": 0.0, "orthogonal_to_active": 0.07203902304172516, "output_influence": 0.5754657983779907, "residual_influence": 0.2065190076828003, "state_active": 0.043504200875759125, "state_memory": 0.006304365117102861, "state_peak": 91.22766876220703, "growth_active": 1.2016034126281738, "growth_memory": 1.1610620021820068, "heads": [{"head": 0, "memory_to_active": 0.10364746302366257, "contribution_to_active": 0.7732964158058167, "mix_mean": 0.8259353637695312, "mix_low": 0.0001697540283203125, "mix_high": 0.0648202896118164, "read_cosine": 0.29512542486190796, "beta_mean": 0.00977251399308443, "nonfinite": 0.0, "weighted_memory_to_active": 0.0906752496957779, "mix_std": 0.1994904726743698, "beta_std": 0.03220367431640625, "beta_low": 0.85858154296875, "beta_high": 0.0, "orthogonal_to_active": 0.06094244867563248, "evidence_weights": [-0.3048575818538666, -0.3507358431816101, 1.1977187395095825], "agreement_weight": 0.8256927728652954, "orthogonal_weight": -0.7299757599830627}, {"head": 1, "memory_to_active": 0.004323878325521946, "contribution_to_active": 0.5263426303863525, "mix_mean": 0.5970426201820374, "mix_low": 0.0007343292236328125, "mix_high": 0.017223358154296875, "read_cosine": 0.458408385515213, "beta_mean": 7.67549681768287e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.001962493173778057, "mix_std": 0.28281328082084656, "beta_std": 1.1178653949173167e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010904386872425675, "evidence_weights": [-0.19303345680236816, -0.30190616846084595, 1.114139437675476], "agreement_weight": 0.1188339814543724, "orthogonal_weight": -0.7193030714988708}, {"head": 2, "memory_to_active": 0.6791883707046509, "contribution_to_active": 0.10220654308795929, "mix_mean": 0.05472307279706001, "mix_low": 0.2099132537841797, "mix_high": 0.0, "read_cosine": 0.2627463936805725, "beta_mean": 0.019843343645334244, "nonfinite": 0.0, "weighted_memory_to_active": 0.06737425923347473, "mix_std": 0.07899638265371323, "beta_std": 0.020956356078386307, "beta_low": 0.3199462890625, "beta_high": 0.0, "orthogonal_to_active": 0.010861369781196117, "evidence_weights": [0.060687482357025146, -0.06637285649776459, 0.34624427556991577], "agreement_weight": 0.1285126656293869, "orthogonal_weight": -0.1946823000907898}, {"head": 3, "memory_to_active": 0.1370762288570404, "contribution_to_active": 0.6798082590103149, "mix_mean": 0.6671314835548401, "mix_low": 0.0050640106201171875, "mix_high": 0.02093791961669922, "read_cosine": 0.45893752574920654, "beta_mean": 0.22333602607250214, "nonfinite": 0.0, "weighted_memory_to_active": 0.09810803830623627, "mix_std": 0.3218270242214203, "beta_std": 0.2516823410987854, "beta_low": 0.20684814453125, "beta_high": 0.0, "orthogonal_to_active": 0.05772148817777634, "evidence_weights": [-0.1674731820821762, -0.3164842426776886, 0.12453403323888779], "agreement_weight": -0.009036465547978878, "orthogonal_weight": 0.6690058708190918}, {"head": 4, "memory_to_active": 0.5927866101264954, "contribution_to_active": 1.1458187103271484, "mix_mean": 0.7385578751564026, "mix_low": 0.0008172988891601562, "mix_high": 0.07232475280761719, "read_cosine": 0.09050062298774719, "beta_mean": 0.13126493990421295, "nonfinite": 0.0, "weighted_memory_to_active": 0.5366289019584656, "mix_std": 0.3027331829071045, "beta_std": 0.1450977772474289, "beta_low": 0.31573486328125, "beta_high": 0.0, "orthogonal_to_active": 0.2773069143295288, "evidence_weights": [-0.17951449751853943, -0.5302454829216003, -0.6581389307975769], "agreement_weight": -0.014546074904501438, "orthogonal_weight": 0.541711151599884}, {"head": 5, "memory_to_active": 0.03422874957323074, "contribution_to_active": 0.6307206749916077, "mix_mean": 0.6916898488998413, "mix_low": 0.0001277923583984375, "mix_high": 0.009441375732421875, "read_cosine": 0.27704480290412903, "beta_mean": 0.000164233279065229, "nonfinite": 0.0, "weighted_memory_to_active": 0.02504618465900421, "mix_std": 0.241906076669693, "beta_std": 0.0004296206752769649, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.024311469867825508, "evidence_weights": [-0.2349272221326828, -0.316540002822876, 0.36886081099510193], "agreement_weight": 0.21031707525253296, "orthogonal_weight": -1.0398805141448975}]}, {"layer": 7, "memory_to_active": 1.8189151287078857, "contribution_to_active": 0.2831309139728546, "mix_mean": 0.1757919192314148, "mix_low": 0.23737367987632751, "mix_high": 0.0011436145287007093, "read_cosine": 0.23547129333019257, "beta_mean": 0.14038121700286865, "nonfinite": 0.0, "weighted_memory_to_active": 0.2198505848646164, "mix_std": 0.1479792445898056, "beta_std": 0.10011039674282074, "beta_low": 0.212188720703125, "beta_high": 0.000213623046875, "orthogonal_to_active": 0.09184996038675308, "output_influence": 0.2900593876838684, "residual_influence": 0.07088114321231842, "state_active": 0.013616170734167099, "state_memory": 0.024450650438666344, "state_peak": 36.54104995727539, "growth_active": 1.1631821393966675, "growth_memory": 1.2565382719039917, "heads": [{"head": 0, "memory_to_active": 2.814033269882202, "contribution_to_active": 0.20841576159000397, "mix_mean": 0.03672727942466736, "mix_low": 0.4332237243652344, "mix_high": 9.5367431640625e-07, "read_cosine": 0.349353551864624, "beta_mean": 0.18670478463172913, "nonfinite": 0.0, "weighted_memory_to_active": 0.2510261535644531, "mix_std": 0.07525394856929779, "beta_std": 0.13503563404083252, "beta_low": 0.00201416015625, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.0007075087050907314, "evidence_weights": [0.04717396944761276, -0.08506885915994644, 0.002980826422572136], "agreement_weight": -0.026270892471075058, "orthogonal_weight": -0.0038344853091984987}, {"head": 1, "memory_to_active": 0.9916341304779053, "contribution_to_active": 0.4148681163787842, "mix_mean": 0.2226429283618927, "mix_low": 0.11548137664794922, "mix_high": 0.0061283111572265625, "read_cosine": 0.3786783814430237, "beta_mean": 0.09344667196273804, "nonfinite": 0.0, "weighted_memory_to_active": 0.345520555973053, "mix_std": 0.24663034081459045, "beta_std": 0.06566362082958221, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04978945851325989, "evidence_weights": [-0.2163582146167755, -0.10285219550132751, -0.30851390957832336], "agreement_weight": 0.05437788739800453, "orthogonal_weight": 0.17098486423492432}, {"head": 2, "memory_to_active": 0.6327053308486938, "contribution_to_active": 0.38769304752349854, "mix_mean": 0.23762303590774536, "mix_low": 0.016876220703125, "mix_high": 0.00035858154296875, "read_cosine": 0.11647529900074005, "beta_mean": 0.03977928310632706, "nonfinite": 0.0, "weighted_memory_to_active": 0.24504350125789642, "mix_std": 0.2174340784549713, "beta_std": 0.04206753522157669, "beta_low": 0.13665771484375, "beta_high": 0.0, "orthogonal_to_active": 0.14369992911815643, "evidence_weights": [-0.09739310294389725, -0.18460549414157867, 0.8695797920227051], "agreement_weight": 0.018121428787708282, "orthogonal_weight": -0.6042018532752991}, {"head": 3, "memory_to_active": 4.89913272857666, "contribution_to_active": 0.11025570333003998, "mix_mean": 0.03456474095582962, "mix_low": 0.33711814880371094, "mix_high": 0.0, "read_cosine": 0.08951753377914429, "beta_mean": 0.44944068789482117, "nonfinite": 0.0, "weighted_memory_to_active": 0.3697996735572815, "mix_std": 0.051200516521930695, "beta_std": 0.2798173427581787, "beta_low": 0.0029296875, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.3165128231048584, "evidence_weights": [0.07824653387069702, -0.03585214912891388, 0.7154034972190857], "agreement_weight": 0.051814135164022446, "orthogonal_weight": -0.9436538815498352}, {"head": 4, "memory_to_active": 1.525649905204773, "contribution_to_active": 0.10972676426172256, "mix_mean": 0.02338303253054619, "mix_low": 0.5197343826293945, "mix_high": 0.0, "read_cosine": 0.24673587083816528, "beta_mean": 0.07271517813205719, "nonfinite": 0.0, "weighted_memory_to_active": 0.08229250460863113, "mix_std": 0.04508572071790695, "beta_std": 0.07751046121120453, "beta_low": 0.1322021484375, "beta_high": 0.0, "orthogonal_to_active": 0.016262976452708244, "evidence_weights": [0.1258517950773239, -0.022707322612404823, 0.18316179513931274], "agreement_weight": 0.032337527722120285, "orthogonal_weight": -0.28505364060401917}, {"head": 5, "memory_to_active": 0.05033515393733978, "contribution_to_active": 0.46782612800598145, "mix_mean": 0.4998105764389038, "mix_low": 0.00180816650390625, "mix_high": 0.00037384033203125, "read_cosine": 0.2320670783519745, "beta_mean": 0.00020065793069079518, "nonfinite": 0.0, "weighted_memory_to_active": 0.02542099542915821, "mix_std": 0.25227081775665283, "beta_std": 0.0005677727167494595, "beta_low": 0.999267578125, "beta_high": 0.0, "orthogonal_to_active": 0.024127047508955002, "evidence_weights": [-0.18389293551445007, -0.23925532400608063, 0.49212586879730225], "agreement_weight": 0.2367173284292221, "orthogonal_weight": -1.0007308721542358}]}, {"layer": 8, "memory_to_active": 0.14200124144554138, "contribution_to_active": 0.5723761320114136, "mix_mean": 0.5046979188919067, "mix_low": 0.009085496887564659, "mix_high": 0.0018966993084177375, "read_cosine": 0.4038988947868347, "beta_mean": 0.01294856145977974, "nonfinite": 0.0, "weighted_memory_to_active": 0.07369951158761978, "mix_std": 0.24362336099147797, "beta_std": 0.019996345043182373, "beta_low": 0.7856852412223816, "beta_high": 0.0, "orthogonal_to_active": 0.06970888376235962, "output_influence": 0.517378568649292, "residual_influence": 0.18992117047309875, "state_active": 0.04498346894979477, "state_memory": 0.005067694932222366, "state_peak": 59.04019546508789, "growth_active": 1.1226617097854614, "growth_memory": 1.0806869268417358, "heads": [{"head": 0, "memory_to_active": 0.06387177109718323, "contribution_to_active": 0.5710359215736389, "mix_mean": 0.605999231338501, "mix_low": 0.0005235671997070312, "mix_high": 0.0014123916625976562, "read_cosine": 0.4602479338645935, "beta_mean": 0.0038174823857843876, "nonfinite": 0.0, "weighted_memory_to_active": 0.04578786343336105, "mix_std": 0.24881184101104736, "beta_std": 0.014899187721312046, "beta_low": 0.921630859375, "beta_high": 0.0, "orthogonal_to_active": 0.028672050684690475, "evidence_weights": [-0.1568671464920044, -0.37900274991989136, 0.567218542098999], "agreement_weight": 0.26066404581069946, "orthogonal_weight": -0.7997573018074036}, {"head": 1, "memory_to_active": 0.0734659805893898, "contribution_to_active": 0.5547868609428406, "mix_mean": 0.533686637878418, "mix_low": 0.0001125335693359375, "mix_high": 0.0012331008911132812, "read_cosine": 0.12758903205394745, "beta_mean": 0.0009402831783518195, "nonfinite": 0.0, "weighted_memory_to_active": 0.03147038444876671, "mix_std": 0.2344982773065567, "beta_std": 0.00853746198117733, "beta_low": 0.991943359375, "beta_high": 0.0, "orthogonal_to_active": 0.023507047444581985, "evidence_weights": [-0.19532954692840576, -0.2757394015789032, 0.6232690215110779], "agreement_weight": 0.08590079098939896, "orthogonal_weight": -0.7761233448982239}, {"head": 2, "memory_to_active": 0.011885758489370346, "contribution_to_active": 0.6885781288146973, "mix_mean": 0.6846718788146973, "mix_low": 0.0003643035888671875, "mix_high": 0.0034198760986328125, "read_cosine": 0.4209318161010742, "beta_mean": 0.00012404602603055537, "nonfinite": 0.0, "weighted_memory_to_active": 0.00942500215023756, "mix_std": 0.2213849127292633, "beta_std": 0.0033996866550296545, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005544504150748253, "evidence_weights": [-0.22153720259666443, -0.3459993600845337, 0.9886249303817749], "agreement_weight": 0.3188336491584778, "orthogonal_weight": -0.6855908036231995}, {"head": 3, "memory_to_active": 0.09986688196659088, "contribution_to_active": 0.6327989101409912, "mix_mean": 0.5840339660644531, "mix_low": 0.00047779083251953125, "mix_high": 0.004908561706542969, "read_cosine": 0.7995702028274536, "beta_mean": 0.007449967786669731, "nonfinite": 0.0, "weighted_memory_to_active": 0.07306763529777527, "mix_std": 0.2547571659088135, "beta_std": 0.008084993809461594, "beta_low": 0.7550048828125, "beta_high": 0.0, "orthogonal_to_active": 0.022705204784870148, "evidence_weights": [-0.1749720573425293, -0.2976323068141937, 0.5320450067520142], "agreement_weight": 0.2502720057964325, "orthogonal_weight": -0.6010931730270386}, {"head": 4, "memory_to_active": 0.24031150341033936, "contribution_to_active": 0.3280637562274933, "mix_mean": 0.3345041871070862, "mix_low": 0.0020771026611328125, "mix_high": 1.239776611328125e-05, "read_cosine": 0.12144804745912552, "beta_mean": 0.0008220792515203357, "nonfinite": 0.0, "weighted_memory_to_active": 0.08685304969549179, "mix_std": 0.23066258430480957, "beta_std": 0.01374122779816389, "beta_low": 0.99127197265625, "beta_high": 0.0, "orthogonal_to_active": 0.017809338867664337, "evidence_weights": [-0.13358174264431, -0.19582295417785645, 0.6747012734413147], "agreement_weight": 0.046468641608953476, "orthogonal_weight": -0.20873281359672546}, {"head": 5, "memory_to_active": 0.3626055121421814, "contribution_to_active": 0.6589930653572083, "mix_mean": 0.28529173135757446, "mix_low": 0.050957679748535156, "mix_high": 0.00039386749267578125, "read_cosine": 0.49360644817352295, "beta_mean": 0.06453751027584076, "nonfinite": 0.0, "weighted_memory_to_active": 0.19559311866760254, "mix_std": 0.2716253399848938, "beta_std": 0.07131551206111908, "beta_low": 0.0543212890625, "beta_high": 0.0, "orthogonal_to_active": 0.32001519203186035, "evidence_weights": [-0.14914752542972565, -0.14555959403514862, -0.10935293138027191], "agreement_weight": -0.006436637602746487, "orthogonal_weight": 1.9392337799072266}]}, {"layer": 9, "memory_to_active": 0.8383017182350159, "contribution_to_active": 0.7014869451522827, "mix_mean": 0.3887946605682373, "mix_low": 0.05892467498779297, "mix_high": 0.006327311508357525, "read_cosine": 0.34209951758384705, "beta_mean": 0.03483215346932411, "nonfinite": 0.0, "weighted_memory_to_active": 0.3650498390197754, "mix_std": 0.2436423897743225, "beta_std": 0.029211822897195816, "beta_low": 0.6188456416130066, "beta_high": 0.0, "orthogonal_to_active": 0.0325692743062973, "output_influence": 0.5529027581214905, "residual_influence": 0.1666187047958374, "state_active": 0.046846386045217514, "state_memory": 0.014932242222130299, "state_peak": 58.24534225463867, "growth_active": 1.2247141599655151, "growth_memory": 1.302004337310791, "heads": [{"head": 0, "memory_to_active": 0.00044076054473407567, "contribution_to_active": 0.5759824514389038, "mix_mean": 0.6552546620368958, "mix_low": 0.0011138916015625, "mix_high": 0.026227951049804688, "read_cosine": 0.33802515268325806, "beta_mean": 1.1335347380736494e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0002867536968551576, "mix_std": 0.28419730067253113, "beta_std": 3.22958067044965e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00030537095153704286, "evidence_weights": [-0.30391934514045715, -0.3118269443511963, 0.40373125672340393], "agreement_weight": 0.09801658987998962, "orthogonal_weight": -1.1210418939590454}, {"head": 1, "memory_to_active": 0.013764264062047005, "contribution_to_active": 0.7066601514816284, "mix_mean": 0.575483500957489, "mix_low": 0.0066394805908203125, "mix_high": 0.0026960372924804688, "read_cosine": 0.4341062307357788, "beta_mean": 3.078454392380081e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010295821353793144, "mix_std": 0.2786228656768799, "beta_std": 6.386602763086557e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006874464452266693, "evidence_weights": [-0.23625245690345764, -0.2932147681713104, 0.9702832698822021], "agreement_weight": 0.533872127532959, "orthogonal_weight": -0.825736403465271}, {"head": 2, "memory_to_active": 0.15668028593063354, "contribution_to_active": 0.6601628661155701, "mix_mean": 0.6009864807128906, "mix_low": 0.0018930435180664062, "mix_high": 0.007193565368652344, "read_cosine": 0.3767668306827545, "beta_mean": 0.0005986733012832701, "nonfinite": 0.0, "weighted_memory_to_active": 0.10666593164205551, "mix_std": 0.2661214768886566, "beta_std": 0.00453045591711998, "beta_low": 0.9967041015625, "beta_high": 0.0, "orthogonal_to_active": 0.061764344573020935, "evidence_weights": [-0.20231778919696808, -0.27773144841194153, 0.7584286332130432], "agreement_weight": 0.34758028388023376, "orthogonal_weight": -0.6814675331115723}, {"head": 3, "memory_to_active": 0.791845440864563, "contribution_to_active": 0.24170993268489838, "mix_mean": 0.15632317960262299, "mix_low": 0.1024618148803711, "mix_high": 4.482269287109375e-05, "read_cosine": 0.37318694591522217, "beta_mean": 0.030357737094163895, "nonfinite": 0.0, "weighted_memory_to_active": 0.1821049451828003, "mix_std": 0.19875232875347137, "beta_std": 0.03136187791824341, "beta_low": 0.1868896484375, "beta_high": 0.0, "orthogonal_to_active": 0.08173509687185287, "evidence_weights": [-0.025469083338975906, -0.09709486365318298, 0.424978107213974], "agreement_weight": 0.09247979521751404, "orthogonal_weight": -0.4866846799850464}, {"head": 4, "memory_to_active": 2.8703365325927734, "contribution_to_active": 1.6444358825683594, "mix_mean": 0.1470762938261032, "mix_low": 0.12635040283203125, "mix_high": 0.0011816024780273438, "read_cosine": 0.07631516456604004, "beta_mean": 0.1474035233259201, "nonfinite": 0.0, "weighted_memory_to_active": 1.5389914512634277, "mix_std": 0.20914345979690552, "beta_std": 0.08054187893867493, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.012851682491600513, "evidence_weights": [-0.07898961007595062, -0.14964261651039124, -0.03605911135673523], "agreement_weight": -0.06263348460197449, "orthogonal_weight": -0.008686026558279991}, {"head": 5, "memory_to_active": 1.1967430114746094, "contribution_to_active": 0.37997037172317505, "mix_mean": 0.19764380156993866, "mix_low": 0.11508941650390625, "mix_high": 0.0006198883056640625, "read_cosine": 0.45419663190841675, "beta_mean": 0.030601050704717636, "nonfinite": 0.0, "weighted_memory_to_active": 0.35195398330688477, "mix_std": 0.22501686215400696, "beta_std": 0.05876963585615158, "beta_low": 0.52947998046875, "beta_high": 0.0, "orthogonal_to_active": 0.03188467398285866, "evidence_weights": [0.05456143990159035, -0.17910760641098022, 0.14202621579170227], "agreement_weight": 0.006584188900887966, "orthogonal_weight": 0.11064209789037704}]}, {"layer": 10, "memory_to_active": 0.2093365639448166, "contribution_to_active": 0.7208938598632812, "mix_mean": 0.526822566986084, "mix_low": 0.02361583709716797, "mix_high": 0.005053202621638775, "read_cosine": 0.38113129138946533, "beta_mean": 0.04084061458706856, "nonfinite": 0.0, "weighted_memory_to_active": 0.08859532326459885, "mix_std": 0.26739853620529175, "beta_std": 0.04347991198301315, "beta_low": 0.6621297597885132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.26179951429367065, "output_influence": 0.46706917881965637, "residual_influence": 0.16785813868045807, "state_active": 0.05206603556871414, "state_memory": 0.014475150033831596, "state_peak": 127.17990112304688, "growth_active": 1.0271391868591309, "growth_memory": 1.008284091949463, "heads": [{"head": 0, "memory_to_active": 0.7312745451927185, "contribution_to_active": 2.0015907287597656, "mix_mean": 0.5566288232803345, "mix_low": 0.11268424987792969, "mix_high": 0.005615234375, "read_cosine": 0.09113962948322296, "beta_mean": 0.17770899832248688, "nonfinite": 0.0, "weighted_memory_to_active": 0.41028594970703125, "mix_std": 0.39188525080680847, "beta_std": 0.1932702511548996, "beta_low": 0.1651611328125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.4760055541992188, "evidence_weights": [-0.08327116072177887, -0.3670445680618286, -1.8810187578201294], "agreement_weight": 0.026367031037807465, "orthogonal_weight": 3.627275228500366}, {"head": 1, "memory_to_active": 0.011979391798377037, "contribution_to_active": 0.6215695738792419, "mix_mean": 0.637361466884613, "mix_low": 0.001415252685546875, "mix_high": 0.017637252807617188, "read_cosine": 0.3025512099266052, "beta_mean": 8.451069879811257e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006667573004961014, "mix_std": 0.26315850019454956, "beta_std": 0.00015961953613441437, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010571545921266079, "evidence_weights": [-0.250233918428421, -0.19425228238105774, 0.5551781058311462], "agreement_weight": 0.6340742707252502, "orthogonal_weight": -1.7068699598312378}, {"head": 2, "memory_to_active": 0.01818833500146866, "contribution_to_active": 0.6222156882286072, "mix_mean": 0.6629079580307007, "mix_low": 0.000274658203125, "mix_high": 0.005435943603515625, "read_cosine": 0.03720138221979141, "beta_mean": 3.4256237995577976e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008319348096847534, "mix_std": 0.25034838914871216, "beta_std": 0.00013116256741341203, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035663475282490253, "evidence_weights": [-0.2672543525695801, -0.27808505296707153, 0.6421741247177124], "agreement_weight": -0.2208389937877655, "orthogonal_weight": -0.41301053762435913}, {"head": 3, "memory_to_active": 0.3612968921661377, "contribution_to_active": 0.11132632941007614, "mix_mean": 0.29974788427352905, "mix_low": 0.0249786376953125, "mix_high": 0.00021648406982421875, "read_cosine": 0.914230227470398, "beta_mean": 0.06011927127838135, "nonfinite": 0.0, "weighted_memory_to_active": 0.04376531392335892, "mix_std": 0.23202933371067047, "beta_std": 0.05832976847887039, "beta_low": 0.00384521484375, "beta_high": 0.0, "orthogonal_to_active": 0.046594515442848206, "evidence_weights": [-0.09277155250310898, -0.14325737953186035, 0.1644039899110794], "agreement_weight": 0.019827362149953842, "orthogonal_weight": 1.4444197416305542}, {"head": 4, "memory_to_active": 0.016886383295059204, "contribution_to_active": 0.5637794733047485, "mix_mean": 0.5960153341293335, "mix_low": 0.00030422210693359375, "mix_high": 0.0014123916625976562, "read_cosine": 0.10829400271177292, "beta_mean": 5.267373853712343e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010164360515773296, "mix_std": 0.2423963099718094, "beta_std": 0.00011459454253781587, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.014668038114905357, "evidence_weights": [-0.185709148645401, -0.2551652789115906, 0.9604066610336304], "agreement_weight": 0.18109233677387238, "orthogonal_weight": -1.5188580751419067}, {"head": 5, "memory_to_active": 0.1163938120007515, "contribution_to_active": 0.40488144755363464, "mix_mean": 0.4082735776901245, "mix_low": 0.0020380020141601562, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8333712220191956, "beta_mean": 0.007043958641588688, "nonfinite": 0.0, "weighted_memory_to_active": 0.05236935243010521, "mix_std": 0.22457316517829895, "beta_std": 0.00887405313551426, "beta_low": 0.80377197265625, "beta_high": 0.0, "orthogonal_to_active": 0.019390977919101715, "evidence_weights": [-0.13919678330421448, -0.16450056433677673, 0.26966455578804016], "agreement_weight": 0.09500174224376678, "orthogonal_weight": -0.668932318687439}]}, {"layer": 11, "memory_to_active": 0.0753483846783638, "contribution_to_active": 0.621062159538269, "mix_mean": 0.5263239145278931, "mix_low": 0.10750914365053177, "mix_high": 0.04842519760131836, "read_cosine": 0.3594624400138855, "beta_mean": 0.02353544346988201, "nonfinite": 0.0, "weighted_memory_to_active": 0.03977152705192566, "mix_std": 0.3238435685634613, "beta_std": 0.034667305648326874, "beta_low": 0.698333740234375, "beta_high": 0.0, "orthogonal_to_active": 0.04408013075590134, "output_influence": 0.5481617450714111, "residual_influence": 0.15279920399188995, "state_active": 0.19398431479930878, "state_memory": 0.0032927552238106728, "state_peak": 34.863929748535156, "growth_active": 1.0966758728027344, "growth_memory": 1.5046608448028564, "heads": [{"head": 0, "memory_to_active": 0.006810436956584454, "contribution_to_active": 0.6070005893707275, "mix_mean": 0.6561009287834167, "mix_low": 0.0007381439208984375, "mix_high": 0.00887298583984375, "read_cosine": 0.28574150800704956, "beta_mean": 7.157195796025917e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005116262473165989, "mix_std": 0.2541693150997162, "beta_std": 0.003481891704723239, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.004212029743939638, "evidence_weights": [-0.17458459734916687, -0.357744961977005, 0.7946822047233582], "agreement_weight": 0.38634830713272095, "orthogonal_weight": -0.850943922996521}, {"head": 1, "memory_to_active": 0.12353433668613434, "contribution_to_active": 0.431761771440506, "mix_mean": 0.4716482162475586, "mix_low": 0.0050449371337890625, "mix_high": 0.0006580352783203125, "read_cosine": 0.4959241449832916, "beta_mean": 0.003328778548166156, "nonfinite": 0.0, "weighted_memory_to_active": 0.059045515954494476, "mix_std": 0.26955896615982056, "beta_std": 0.010873801074922085, "beta_low": 0.92840576171875, "beta_high": 0.0, "orthogonal_to_active": 0.032186999917030334, "evidence_weights": [-0.1435306966304779, -0.27429595589637756, 0.5991499423980713], "agreement_weight": 0.08248435705900192, "orthogonal_weight": -0.6487157344818115}, {"head": 2, "memory_to_active": 0.005641341209411621, "contribution_to_active": 0.9869160652160645, "mix_mean": 0.44072914123535156, "mix_low": 0.30556392669677734, "mix_high": 0.1796102523803711, "read_cosine": 0.17750667035579681, "beta_mean": 0.06401589512825012, "nonfinite": 0.0, "weighted_memory_to_active": 0.004124386236071587, "mix_std": 0.4499651789665222, "beta_std": 0.08709343522787094, "beta_low": 0.47412109375, "beta_high": 0.0, "orthogonal_to_active": 0.04174226149916649, "evidence_weights": [0.04908003658056259, -0.3492047190666199, -1.3266297578811646], "agreement_weight": 0.008272198028862476, "orthogonal_weight": 10.457049369812012}, {"head": 3, "memory_to_active": 0.008110836148262024, "contribution_to_active": 0.8422655463218689, "mix_mean": 0.41730374097824097, "mix_low": 0.3256568908691406, "mix_high": 0.09451675415039062, "read_cosine": 0.5886312127113342, "beta_mean": 0.04838022589683533, "nonfinite": 0.0, "weighted_memory_to_active": 0.004956482909619808, "mix_std": 0.44093021750450134, "beta_std": 0.07377287745475769, "beta_low": 0.51336669921875, "beta_high": 0.0, "orthogonal_to_active": 0.0373295359313488, "evidence_weights": [0.21311098337173462, -0.2818457782268524, -0.11352960020303726], "agreement_weight": 0.048982564359903336, "orthogonal_weight": 10.7256441116333}, {"head": 4, "memory_to_active": 0.21863417327404022, "contribution_to_active": 0.44238144159317017, "mix_mean": 0.6019704937934875, "mix_low": 0.0066242218017578125, "mix_high": 0.0015230178833007812, "read_cosine": 0.025526266545057297, "beta_mean": 0.017112895846366882, "nonfinite": 0.0, "weighted_memory_to_active": 0.12060095369815826, "mix_std": 0.27007147669792175, "beta_std": 0.020661097019910812, "beta_low": 0.4937744140625, "beta_high": 0.0, "orthogonal_to_active": 0.11910034716129303, "evidence_weights": [-0.23350243270397186, -0.2396952211856842, 1.8879221677780151], "agreement_weight": -0.21778567135334015, "orthogonal_weight": -0.9959749579429626}, {"head": 5, "memory_to_active": 0.08935915678739548, "contribution_to_active": 0.4160475730895996, "mix_mean": 0.570190966129303, "mix_low": 0.00142669677734375, "mix_high": 0.005370140075683594, "read_cosine": 0.5834448933601379, "beta_mean": 0.008303295820951462, "nonfinite": 0.0, "weighted_memory_to_active": 0.04478555545210838, "mix_std": 0.2583661377429962, "beta_std": 0.012120726518332958, "beta_low": 0.78076171875, "beta_high": 0.0, "orthogonal_to_active": 0.029909569770097733, "evidence_weights": [-0.20188862085342407, -0.3130853474140167, 0.6603518724441528], "agreement_weight": 0.08354216068983078, "orthogonal_weight": -0.8644990921020508}]}, {"layer": 12, "memory_to_active": 0.21349866688251495, "contribution_to_active": 0.5946943163871765, "mix_mean": 0.5532135367393494, "mix_low": 0.008684635162353516, "mix_high": 0.00678253173828125, "read_cosine": 0.230592280626297, "beta_mean": 0.014364432543516159, "nonfinite": 0.0, "weighted_memory_to_active": 0.1111927479505539, "mix_std": 0.2603585124015808, "beta_std": 0.014706538058817387, "beta_low": 0.83013916015625, "beta_high": 0.0, "orthogonal_to_active": 0.06882261484861374, "output_influence": 0.5148001909255981, "residual_influence": 0.15136760473251343, "state_active": 0.03665316477417946, "state_memory": 0.007887539453804493, "state_peak": 68.31234741210938, "growth_active": 1.2160300016403198, "growth_memory": 1.1522074937820435, "heads": [{"head": 0, "memory_to_active": 0.03540807217359543, "contribution_to_active": 0.6264498233795166, "mix_mean": 0.5852839946746826, "mix_low": 0.0006685256958007812, "mix_high": 0.005774497985839844, "read_cosine": 0.30164068937301636, "beta_mean": 0.0005518469261005521, "nonfinite": 0.0, "weighted_memory_to_active": 0.0195721834897995, "mix_std": 0.26523441076278687, "beta_std": 0.001082387287169695, "beta_low": 0.99835205078125, "beta_high": 0.0, "orthogonal_to_active": 0.018025808036327362, "evidence_weights": [-0.20482386648654938, -0.27311718463897705, -0.04624956101179123], "agreement_weight": 0.09577212482690811, "orthogonal_weight": -0.9931067824363708}, {"head": 1, "memory_to_active": 0.08468083292245865, "contribution_to_active": 0.5601980686187744, "mix_mean": 0.5670262575149536, "mix_low": 0.00026035308837890625, "mix_high": 0.0025691986083984375, "read_cosine": 0.3864942193031311, "beta_mean": 0.0005140893626958132, "nonfinite": 0.0, "weighted_memory_to_active": 0.05025508254766464, "mix_std": 0.26503896713256836, "beta_std": 0.008658558130264282, "beta_low": 0.9976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.04206938296556473, "evidence_weights": [-0.20013147592544556, -0.2711479961872101, 1.0444591045379639], "agreement_weight": 0.26389482617378235, "orthogonal_weight": -0.9199580550193787}, {"head": 2, "memory_to_active": 0.5968412756919861, "contribution_to_active": 0.5234552025794983, "mix_mean": 0.33652010560035706, "mix_low": 0.026439666748046875, "mix_high": 9.1552734375e-05, "read_cosine": 0.5539736151695251, "beta_mean": 0.08409485220909119, "nonfinite": 0.0, "weighted_memory_to_active": 0.2483629286289215, "mix_std": 0.24291233718395233, "beta_std": 0.0625794529914856, "beta_low": 0.00115966796875, "beta_high": 0.0, "orthogonal_to_active": 0.27860409021377563, "evidence_weights": [-0.2070751190185547, -0.06163368746638298, -0.027313409373164177], "agreement_weight": 0.023182619363069534, "orthogonal_weight": 1.4213513135910034}, {"head": 3, "memory_to_active": 0.031763482838869095, "contribution_to_active": 0.7603771090507507, "mix_mean": 0.730758011341095, "mix_low": 0.000133514404296875, "mix_high": 0.021086692810058594, "read_cosine": 0.08985975384712219, "beta_mean": 0.00022568911663256586, "nonfinite": 0.0, "weighted_memory_to_active": 0.02229054644703865, "mix_std": 0.22431336343288422, "beta_std": 0.00513477111235261, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.020937439054250717, "evidence_weights": [-0.288867324590683, -0.33624467253685, 1.639857292175293], "agreement_weight": 0.6002127528190613, "orthogonal_weight": -0.9606883525848389}, {"head": 4, "memory_to_active": 0.4835447371006012, "contribution_to_active": 0.5873867273330688, "mix_mean": 0.4586024284362793, "mix_low": 0.018235206604003906, "mix_high": 0.004673004150390625, "read_cosine": -0.18153487145900726, "beta_mean": 0.000592778145801276, "nonfinite": 0.0, "weighted_memory_to_active": 0.29480668902397156, "mix_std": 0.2915789484977722, "beta_std": 0.00846781674772501, "beta_low": 0.9871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.02974853292107582, "evidence_weights": [-0.15467146039009094, -0.21335607767105103, -0.14274834096431732], "agreement_weight": 0.011896315030753613, "orthogonal_weight": -0.10507526248693466}, {"head": 5, "memory_to_active": 0.048753514885902405, "contribution_to_active": 0.5102988481521606, "mix_mean": 0.6410901546478271, "mix_low": 0.00637054443359375, "mix_high": 0.006500244140625, "read_cosine": 0.23312023282051086, "beta_mean": 0.00020733170094899833, "nonfinite": 0.0, "weighted_memory_to_active": 0.031869009137153625, "mix_std": 0.2730729579925537, "beta_std": 0.002316236961632967, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.02355044335126877, "evidence_weights": [-0.2254827320575714, -0.3166038691997528, 1.2221523523330688], "agreement_weight": 0.430332213640213, "orthogonal_weight": -0.8055956959724426}]}, {"layer": 13, "memory_to_active": 0.11052366346120834, "contribution_to_active": 0.545638382434845, "mix_mean": 0.5472623109817505, "mix_low": 0.005923112854361534, "mix_high": 0.009060859680175781, "read_cosine": 0.3926815390586853, "beta_mean": 0.013250522315502167, "nonfinite": 0.0, "weighted_memory_to_active": 0.04701860249042511, "mix_std": 0.26026782393455505, "beta_std": 0.014239881187677383, "beta_low": 0.811737060546875, "beta_high": 0.0, "orthogonal_to_active": 0.05115778371691704, "output_influence": 0.5307573676109314, "residual_influence": 0.192875936627388, "state_active": 0.1353214830160141, "state_memory": 0.04109735041856766, "state_peak": 98.39057159423828, "growth_active": 0.9291933178901672, "growth_memory": 0.9652732014656067, "heads": [{"head": 0, "memory_to_active": 0.009173791855573654, "contribution_to_active": 0.6049062013626099, "mix_mean": 0.6158571243286133, "mix_low": 0.0007371902465820312, "mix_high": 0.004679679870605469, "read_cosine": 0.4286961257457733, "beta_mean": 3.42296261806041e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006028390023857355, "mix_std": 0.26721906661987305, "beta_std": 7.681187707930803e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005780446343123913, "evidence_weights": [-0.19053874909877777, -0.2650182545185089, 0.7250586748123169], "agreement_weight": 0.20901432633399963, "orthogonal_weight": -1.0685787200927734}, {"head": 1, "memory_to_active": 0.009411620907485485, "contribution_to_active": 0.6145042181015015, "mix_mean": 0.5788145065307617, "mix_low": 0.0016603469848632812, "mix_high": 0.0055751800537109375, "read_cosine": 0.2364553064107895, "beta_mean": 3.820946585619822e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005443853326141834, "mix_std": 0.27663910388946533, "beta_std": 7.878048927523196e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006973169278353453, "evidence_weights": [-0.1805705577135086, -0.2638281583786011, 0.5510796904563904], "agreement_weight": 0.5447901487350464, "orthogonal_weight": -1.3355330228805542}, {"head": 2, "memory_to_active": 0.0562678761780262, "contribution_to_active": 0.48775625228881836, "mix_mean": 0.3979373574256897, "mix_low": 0.0009222030639648438, "mix_high": 0.00020599365234375, "read_cosine": 0.5493177771568298, "beta_mean": 0.006397624034434557, "nonfinite": 0.0, "weighted_memory_to_active": 0.027364764362573624, "mix_std": 0.23502862453460693, "beta_std": 0.023136060684919357, "beta_low": 0.8736572265625, "beta_high": 0.0, "orthogonal_to_active": 0.016727829352021217, "evidence_weights": [-0.18168038129806519, -0.21355779469013214, 0.4222566783428192], "agreement_weight": 0.005522603634744883, "orthogonal_weight": -0.8456776142120361}, {"head": 3, "memory_to_active": 0.15113738179206848, "contribution_to_active": 0.5711256265640259, "mix_mean": 0.6572350263595581, "mix_low": 0.026155471801757812, "mix_high": 0.031513214111328125, "read_cosine": -0.05445292592048645, "beta_mean": 0.0007510995492339134, "nonfinite": 0.0, "weighted_memory_to_active": 0.10812168568372726, "mix_std": 0.2898680865764618, "beta_std": 0.007584927137941122, "beta_low": 0.99658203125, "beta_high": 0.0, "orthogonal_to_active": 0.0991869643330574, "evidence_weights": [-0.31381434202194214, -0.24067570269107819, 1.4935539960861206], "agreement_weight": 6.599942207336426, "orthogonal_weight": -0.9766376614570618}, {"head": 4, "memory_to_active": 0.3927878141403198, "contribution_to_active": 0.2709057331085205, "mix_mean": 0.3690115511417389, "mix_low": 0.0037584304809570312, "mix_high": 1.33514404296875e-05, "read_cosine": 0.884474515914917, "beta_mean": 0.07209666073322296, "nonfinite": 0.0, "weighted_memory_to_active": 0.1040327176451683, "mix_std": 0.2225879430770874, "beta_std": 0.05252343788743019, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.15172532200813293, "evidence_weights": [-0.06719464808702469, -0.22566713392734528, 0.430910587310791], "agreement_weight": 0.008363890461623669, "orthogonal_weight": 2.289940118789673}, {"head": 5, "memory_to_active": 0.044363442808389664, "contribution_to_active": 0.7246320247650146, "mix_mean": 0.6647180914878845, "mix_low": 0.0023050308227539062, "mix_high": 0.012377738952636719, "read_cosine": 0.3115982711315155, "beta_mean": 0.00018531284877099097, "nonfinite": 0.0, "weighted_memory_to_active": 0.0311201810836792, "mix_std": 0.27026405930519104, "beta_std": 0.0020392658188939095, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.02655293419957161, "evidence_weights": [-0.22175554931163788, -0.291903018951416, 0.862617552280426], "agreement_weight": 0.20998117327690125, "orthogonal_weight": -0.9290510416030884}]}, {"layer": 14, "memory_to_active": 0.22538575530052185, "contribution_to_active": 0.4174460172653198, "mix_mean": 0.426598459482193, "mix_low": 0.027088643983006477, "mix_high": 0.006107171531766653, "read_cosine": 0.6143266558647156, "beta_mean": 0.06109089404344559, "nonfinite": 0.0, "weighted_memory_to_active": 0.09536442905664444, "mix_std": 0.24106907844543457, "beta_std": 0.060003556311130524, "beta_low": 0.5089010000228882, "beta_high": 0.0, "orthogonal_to_active": 0.043195974081754684, "output_influence": 0.35434478521347046, "residual_influence": 0.14826777577400208, "state_active": 0.05458873137831688, "state_memory": 0.01466199941933155, "state_peak": 46.56296157836914, "growth_active": 1.1037604808807373, "growth_memory": 1.24044668674469, "heads": [{"head": 0, "memory_to_active": 0.3600560128688812, "contribution_to_active": 0.20029142498970032, "mix_mean": 0.19925110042095184, "mix_low": 0.014822006225585938, "mix_high": 3.910064697265625e-05, "read_cosine": 0.8117262125015259, "beta_mean": 0.06851822882890701, "nonfinite": 0.0, "weighted_memory_to_active": 0.07838620245456696, "mix_std": 0.17966511845588684, "beta_std": 0.06169275566935539, "beta_low": 0.00616455078125, "beta_high": 0.0, "orthogonal_to_active": 0.0892469510436058, "evidence_weights": [-0.09750083833932877, -0.047040123492479324, 0.38058236241340637], "agreement_weight": 0.02055152878165245, "orthogonal_weight": 1.8635646104812622}, {"head": 1, "memory_to_active": 0.45705029368400574, "contribution_to_active": 0.2936653792858124, "mix_mean": 0.19776996970176697, "mix_low": 0.13336849212646484, "mix_high": 1.811981201171875e-05, "read_cosine": 0.8745868802070618, "beta_mean": 0.25185707211494446, "nonfinite": 0.0, "weighted_memory_to_active": 0.21068395674228668, "mix_std": 0.2303326278924942, "beta_std": 0.2538568675518036, "beta_low": 0.1031494140625, "beta_high": 0.0, "orthogonal_to_active": 0.01781347580254078, "evidence_weights": [0.006348169408738613, -0.19131863117218018, -0.09418851882219315], "agreement_weight": 0.0600130520761013, "orthogonal_weight": 0.27328911423683167}, {"head": 2, "memory_to_active": 0.13679127395153046, "contribution_to_active": 0.5748825073242188, "mix_mean": 0.5880312323570251, "mix_low": 0.0023632049560546875, "mix_high": 0.0045375823974609375, "read_cosine": 0.3976185917854309, "beta_mean": 0.0020002564415335655, "nonfinite": 0.0, "weighted_memory_to_active": 0.0881165862083435, "mix_std": 0.26701995730400085, "beta_std": 0.0026864302344620228, "beta_low": 0.9776611328125, "beta_high": 0.0, "orthogonal_to_active": 0.06957995891571045, "evidence_weights": [-0.1310638189315796, -0.31568121910095215, 0.9087908864021301], "agreement_weight": 0.08475477248430252, "orthogonal_weight": -0.8893371224403381}, {"head": 3, "memory_to_active": 0.044702887535095215, "contribution_to_active": 0.6011321544647217, "mix_mean": 0.6707226634025574, "mix_low": 0.004479408264160156, "mix_high": 0.022973060607910156, "read_cosine": -0.061239272356033325, "beta_mean": 0.005086779594421387, "nonfinite": 0.0, "weighted_memory_to_active": 0.0277317613363266, "mix_std": 0.25780147314071655, "beta_std": 0.007383865304291248, "beta_low": 0.86083984375, "beta_high": 0.0, "orthogonal_to_active": 0.02081875130534172, "evidence_weights": [-0.3414057791233063, -0.3116108477115631, 0.29851624369621277], "agreement_weight": 0.06744992733001709, "orthogonal_weight": -0.8282302021980286}, {"head": 4, "memory_to_active": 0.16031287610530853, "contribution_to_active": 0.4983054995536804, "mix_mean": 0.5240025520324707, "mix_low": 0.0040283203125, "mix_high": 0.007885932922363281, "read_cosine": 0.9015960693359375, "beta_mean": 0.037821732461452484, "nonfinite": 0.0, "weighted_memory_to_active": 0.09409857541322708, "mix_std": 0.2589099407196045, "beta_std": 0.02809128537774086, "beta_low": 0.126708984375, "beta_high": 0.0, "orthogonal_to_active": 0.024834752082824707, "evidence_weights": [-0.208065465092659, -0.2373896986246109, 0.43901392817497253], "agreement_weight": 0.1343761831521988, "orthogonal_weight": -0.6078841686248779}, {"head": 5, "memory_to_active": 0.19340114295482635, "contribution_to_active": 0.3363991379737854, "mix_mean": 0.37981313467025757, "mix_low": 0.0034704208374023438, "mix_high": 0.0011892318725585938, "read_cosine": 0.7616710662841797, "beta_mean": 0.0012612579157575965, "nonfinite": 0.0, "weighted_memory_to_active": 0.07316945493221283, "mix_std": 0.2526852488517761, "beta_std": 0.0063101123087108135, "beta_low": 0.9788818359375, "beta_high": 0.0, "orthogonal_to_active": 0.036881912499666214, "evidence_weights": [-0.17264431715011597, -0.1810891032218933, 0.36027902364730835], "agreement_weight": 0.06678721308708191, "orthogonal_weight": -0.7577050924301147}]}, {"layer": 15, "memory_to_active": 0.05192365124821663, "contribution_to_active": 0.49072781205177307, "mix_mean": 0.4255622923374176, "mix_low": 0.008203824982047081, "mix_high": 0.0030616126023232937, "read_cosine": 0.6667971611022949, "beta_mean": 0.03798539936542511, "nonfinite": 0.0, "weighted_memory_to_active": 0.026096779853105545, "mix_std": 0.285382479429245, "beta_std": 0.0431399829685688, "beta_low": 0.7046102285385132, "beta_high": 0.0, "orthogonal_to_active": 0.009394718334078789, "output_influence": 0.3112504482269287, "residual_influence": 0.18733465671539307, "state_active": 0.17522220313549042, "state_memory": 0.010435386560857296, "state_peak": 31.987377166748047, "growth_active": 1.2455074787139893, "growth_memory": 1.3477236032485962, "heads": [{"head": 0, "memory_to_active": 0.11357877403497696, "contribution_to_active": 0.47923946380615234, "mix_mean": 0.39269816875457764, "mix_low": 0.008096694946289062, "mix_high": 0.006056785583496094, "read_cosine": 0.5543601512908936, "beta_mean": 0.04948149994015694, "nonfinite": 0.0, "weighted_memory_to_active": 0.05502159148454666, "mix_std": 0.28526467084884644, "beta_std": 0.035222142934799194, "beta_low": 0.0494384765625, "beta_high": 0.0, "orthogonal_to_active": 0.03410464897751808, "evidence_weights": [-0.19020967185497284, -0.19647641479969025, 0.5094105005264282], "agreement_weight": 0.09285439550876617, "orthogonal_weight": -0.812049388885498}, {"head": 1, "memory_to_active": 0.08890234678983688, "contribution_to_active": 0.4340319037437439, "mix_mean": 0.4450533986091614, "mix_low": 0.000850677490234375, "mix_high": 0.007096290588378906, "read_cosine": 0.9351385831832886, "beta_mean": 0.00023620782303623855, "nonfinite": 0.0, "weighted_memory_to_active": 0.03840944170951843, "mix_std": 0.28532472252845764, "beta_std": 0.0016581534873694181, "beta_low": 0.9986572265625, "beta_high": 0.0, "orthogonal_to_active": 0.0023541657719761133, "evidence_weights": [-0.1541624516248703, -0.16416805982589722, 0.16353632509708405], "agreement_weight": 0.11821034550666809, "orthogonal_weight": 0.13924255967140198}, {"head": 2, "memory_to_active": 0.006359046325087547, "contribution_to_active": 0.4843751788139343, "mix_mean": 0.34645891189575195, "mix_low": 0.012978553771972656, "mix_high": 8.487701416015625e-05, "read_cosine": 0.7667132616043091, "beta_mean": 9.22028484637849e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0033743970561772585, "mix_std": 0.2712451219558716, "beta_std": 0.0002805008552968502, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0011393323075026274, "evidence_weights": [-0.20142538845539093, -0.2638186812400818, 0.11625227332115173], "agreement_weight": 0.00908235739916563, "orthogonal_weight": 0.8971673846244812}, {"head": 3, "memory_to_active": 0.0032403776422142982, "contribution_to_active": 0.3764002323150635, "mix_mean": 0.40530914068222046, "mix_low": 0.0048999786376953125, "mix_high": 0.0006475448608398438, "read_cosine": 0.6160423159599304, "beta_mean": 1.8673310478334315e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0010290660429745913, "mix_std": 0.2920418381690979, "beta_std": 3.920221934095025e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006193849258124828, "evidence_weights": [-0.27567508816719055, -0.2365058958530426, 0.1907011866569519], "agreement_weight": 0.054809581488370895, "orthogonal_weight": -0.5799505710601807}, {"head": 4, "memory_to_active": 0.06905905902385712, "contribution_to_active": 0.6375030875205994, "mix_mean": 0.38411980867385864, "mix_low": 0.012652397155761719, "mix_high": 0.0012073516845703125, "read_cosine": 0.7896328568458557, "beta_mean": 0.17563945055007935, "nonfinite": 0.0, "weighted_memory_to_active": 0.039219342172145844, "mix_std": 0.3102034330368042, "beta_std": 0.20343086123466492, "beta_low": 0.21099853515625, "beta_high": 0.0, "orthogonal_to_active": 0.006289882585406303, "evidence_weights": [-0.20595408976078033, -0.254212886095047, -0.027440255507826805], "agreement_weight": 0.012792719528079033, "orthogonal_weight": 0.3734884560108185}, {"head": 5, "memory_to_active": 0.030402284115552902, "contribution_to_active": 0.5328167676925659, "mix_mean": 0.5797343850135803, "mix_low": 0.009744644165039062, "mix_high": 0.003276824951171875, "read_cosine": 0.338895708322525, "beta_mean": 0.0024443520233035088, "nonfinite": 0.0, "weighted_memory_to_active": 0.019526833668351173, "mix_std": 0.26821497082710266, "beta_std": 0.018209019675850868, "beta_low": 0.96868896484375, "beta_high": 0.0, "orthogonal_to_active": 0.011860895901918411, "evidence_weights": [-0.28183498978614807, -0.30469557642936707, 0.6203836798667908], "agreement_weight": 0.3167659044265747, "orthogonal_weight": 0.8104283213615417}]}]}, {"step": 69000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8097536349996517, "memory_to_active": 0.8024612069129944, "contribution_to_active": 0.5717212557792664, "mix_mean": 0.4250168204307556, "mix_low": 0.12407529354095459, "mix_high": 0.010600556619465351, "read_cosine": 0.34251198172569275, "beta_mean": 0.07006411999464035, "nonfinite": 0.0, "weighted_memory_to_active": 0.20125819742679596, "mix_std": 0.22270673513412476, "beta_std": 0.0617280974984169, "beta_low": 0.5868282318115234, "beta_high": 4.38690185546875e-05, "orthogonal_to_active": 0.0640721395611763, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0915470123291016, "normal_token_accuracy": 0.43826472759246826, "state_active": 0.06409028172492981, "state_memory": 0.016981899738311768, "state_peak": 125.05995178222656, "growth_active": 1.1273231506347656, "growth_memory": 1.285893440246582, "output_influence": 0.44172948598861694, "residual_influence": 0.15710121393203735, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1799864768981934, "no_memory_loss_delta": 0.6898890733718872, "scrambled_loss_delta": 0.1367659866809845, "memory_only_loss_delta": 3.492091655731201, "no_evidence_loss_delta": 0.3781823515892029, "no_agreement_loss_delta": 0.018024306744337082, "no_orthogonal_loss_delta": 0.020495004951953888}, {"index": 94, "tokens": 256, "normal_loss": 3.2135674953460693, "no_memory_loss_delta": 0.7307375073432922, "scrambled_loss_delta": 0.38617050647735596, "memory_only_loss_delta": 3.495814085006714, "no_evidence_loss_delta": 0.33343833684921265, "no_agreement_loss_delta": 0.041173651814460754, "no_orthogonal_loss_delta": 0.041793156415224075}, {"index": 188, "tokens": 256, "normal_loss": 3.4282941818237305, "no_memory_loss_delta": 0.8620536923408508, "scrambled_loss_delta": 0.07159198820590973, "memory_only_loss_delta": 2.9091014862060547, "no_evidence_loss_delta": 0.4550924301147461, "no_agreement_loss_delta": 0.04004152864217758, "no_orthogonal_loss_delta": 0.03307260200381279}, {"index": 283, "tokens": 131, "normal_loss": 2.0221948623657227, "no_memory_loss_delta": 0.49063047766685486, "scrambled_loss_delta": 0.14188550412654877, "memory_only_loss_delta": 2.9648284912109375, "no_evidence_loss_delta": 0.17976097762584686, "no_agreement_loss_delta": 0.014621437527239323, "no_orthogonal_loss_delta": -0.01763376221060753}], "agreement_weight_abs_mean": 0.243876650929451, "agreement_weight_abs_max": 6.648873329162598, "agreement_weight_mean": 0.21267592906951904, "orthogonal_weight_abs_mean": 0.9329189658164978, "orthogonal_weight_abs_max": 10.816637992858887, "orthogonal_weight_mean": -0.06392251700162888, "no_memory_loss": 3.813058376312256, "no_memory_loss_delta": 0.7215113639831543, "no_memory_token_accuracy_gain": 0.09343715012073517, "no_memory_token_win_fraction": 0.6985539197921753, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 51.39828482234836, "scrambled_loss": 3.281520366668701, "scrambled_loss_delta": 0.1899736374616623, "scrambled_token_accuracy_gain": 0.021134594455361366, "scrambled_token_win_fraction": 0.600667417049408, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 17.30190650274014, "memory_only_loss": 6.341854572296143, "memory_only_loss_delta": 3.25030779838562, "memory_only_token_accuracy_gain": 0.28142380714416504, "memory_only_token_win_fraction": 0.9210233688354492, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.12377249703094, "no_evidence_loss": 3.4499752521514893, "no_evidence_loss_delta": 0.3584285080432892, "no_evidence_token_accuracy_gain": 0.04226918891072273, "no_evidence_token_win_fraction": 0.6106785535812378, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 30.12264192565285, "no_agreement_loss": 3.1219370365142822, "no_agreement_loss_delta": 0.030390117317438126, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.4727475047111511, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.9932980222570538, "no_orthogonal_loss": 3.1161322593688965, "no_orthogonal_loss_delta": 0.024585464969277382, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5850945711135864, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.4285704037458054, "short_no_memory_loss_delta": 0.6682384014129639, "long_context_memory_gain": 0.05327296257019043, "normal_context_loss_gain": 0.13970154523849487, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.5575950145721436, "contribution_to_active": 1.1943376064300537, "mix_mean": 0.10661493241786957, "mix_low": 0.5103449821472168, "mix_high": 0.004811128135770559, "read_cosine": 0.40191152691841125, "beta_mean": 0.21025332808494568, "nonfinite": 0.0, "weighted_memory_to_active": 1.217134714126587, "mix_std": 0.09169477969408035, "beta_std": 0.1478373110294342, "beta_low": 0.1640421599149704, "beta_high": 0.0, "orthogonal_to_active": 0.05141262710094452, "output_influence": 0.21078775823116302, "residual_influence": 0.23436906933784485, "state_active": 0.06627364456653595, "state_memory": 0.04563178867101669, "state_peak": 45.38389205932617, "growth_active": 1.262692928314209, "growth_memory": 1.2655621767044067, "heads": [{"head": 0, "memory_to_active": 2.824798822402954, "contribution_to_active": 0.4725583791732788, "mix_mean": 0.01943303644657135, "mix_low": 0.6225414276123047, "mix_high": 0.0, "read_cosine": 0.2176833599805832, "beta_mean": 0.20579202473163605, "nonfinite": 0.0, "weighted_memory_to_active": 0.5968746542930603, "mix_std": 0.058385565876960754, "beta_std": 0.1061386689543724, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06520099937915802, "evidence_weights": [-0.0020059056114405394, -0.0409797839820385, -0.05722682550549507], "agreement_weight": -0.22326093912124634, "orthogonal_weight": -0.12600156664848328}, {"head": 1, "memory_to_active": 7.019197463989258, "contribution_to_active": 2.9616668224334717, "mix_mean": 0.019814157858490944, "mix_low": 0.6139211654663086, "mix_high": 0.0, "read_cosine": 0.4141644537448883, "beta_mean": 0.4282117187976837, "nonfinite": 0.0, "weighted_memory_to_active": 3.1730127334594727, "mix_std": 0.05583857744932175, "beta_std": 0.23854467272758484, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.07548728585243225, "evidence_weights": [0.016246946528553963, -0.11089004576206207, 0.6730546355247498], "agreement_weight": 0.026711348444223404, "orthogonal_weight": 0.05017543584108353}, {"head": 2, "memory_to_active": 2.644268035888672, "contribution_to_active": 0.5669569373130798, "mix_mean": 0.015528024174273014, "mix_low": 0.6393957138061523, "mix_high": 0.0, "read_cosine": 0.2695135474205017, "beta_mean": 0.25446271896362305, "nonfinite": 0.0, "weighted_memory_to_active": 0.6661374568939209, "mix_std": 0.033570386469364166, "beta_std": 0.1992572546005249, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.054406922310590744, "evidence_weights": [0.05824843421578407, 0.04406164586544037, 0.25190111994743347], "agreement_weight": 0.01126817986369133, "orthogonal_weight": -0.20431308448314667}, {"head": 3, "memory_to_active": 7.415032863616943, "contribution_to_active": 2.6373705863952637, "mix_mean": 0.030220668762922287, "mix_low": 0.4776649475097656, "mix_high": 0.0, "read_cosine": 0.2677438259124756, "beta_mean": 0.3320099115371704, "nonfinite": 0.0, "weighted_memory_to_active": 2.7699646949768066, "mix_std": 0.06839105486869812, "beta_std": 0.28706982731819153, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.09544901549816132, "evidence_weights": [-0.03712858632206917, -0.03189367800951004, -0.1276094764471054], "agreement_weight": -0.002144985366612673, "orthogonal_weight": -0.04085724055767059}, {"head": 4, "memory_to_active": 1.333580732345581, "contribution_to_active": 0.06597226113080978, "mix_mean": 0.017081163823604584, "mix_low": 0.6935701370239258, "mix_high": 0.0, "read_cosine": 0.3061107397079468, "beta_mean": 0.022299177944660187, "nonfinite": 0.0, "weighted_memory_to_active": 0.039356302469968796, "mix_std": 0.05028486251831055, "beta_std": 0.037722356617450714, "beta_low": 0.57635498046875, "beta_high": 0.0, "orthogonal_to_active": 0.0010766178602352738, "evidence_weights": [0.10699141770601273, 0.07079616189002991, -0.12230415642261505], "agreement_weight": -0.19557538628578186, "orthogonal_weight": 0.022931700572371483}, {"head": 5, "memory_to_active": 0.10869157314300537, "contribution_to_active": 0.46150141954421997, "mix_mean": 0.5376125574111938, "mix_low": 0.01497650146484375, "mix_high": 0.02886676788330078, "read_cosine": 0.9362531900405884, "beta_mean": 0.018744267523288727, "nonfinite": 0.0, "weighted_memory_to_active": 0.05746178701519966, "mix_std": 0.2836982011795044, "beta_std": 0.018291102722287178, "beta_low": 0.40771484375, "beta_high": 0.0, "orthogonal_to_active": 0.016854915767908096, "evidence_weights": [-0.1752518117427826, -0.21810013055801392, 0.37488552927970886], "agreement_weight": 0.2866184413433075, "orthogonal_weight": -0.6951730251312256}]}, {"layer": 1, "memory_to_active": 2.4589273929595947, "contribution_to_active": 0.4846465587615967, "mix_mean": 0.21758206188678741, "mix_low": 0.36701440811157227, "mix_high": 0.0010781288146972656, "read_cosine": 0.30583304166793823, "beta_mean": 0.18529993295669556, "nonfinite": 0.0, "weighted_memory_to_active": 0.32271790504455566, "mix_std": 0.11429756879806519, "beta_std": 0.13030561804771423, "beta_low": 0.3495686948299408, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.039068352431058884, "output_influence": 0.26916658878326416, "residual_influence": 0.07472468912601471, "state_active": 0.02849949523806572, "state_memory": 0.034943588078022, "state_peak": 40.36151123046875, "growth_active": 1.1811153888702393, "growth_memory": 1.450535535812378, "heads": [{"head": 0, "memory_to_active": 0.020733943209052086, "contribution_to_active": 0.49753594398498535, "mix_mean": 0.5399720072746277, "mix_low": 0.0041446685791015625, "mix_high": 0.0024137496948242188, "read_cosine": 0.009644758887588978, "beta_mean": 0.00013398323790170252, "nonfinite": 0.0, "weighted_memory_to_active": 0.013159585185348988, "mix_std": 0.28046247363090515, "beta_std": 0.0002464975113980472, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012430285103619099, "evidence_weights": [-0.2111245095729828, -0.2734626233577728, 0.2596198618412018], "agreement_weight": 0.19039542973041534, "orthogonal_weight": -0.9883933067321777}, {"head": 1, "memory_to_active": 2.9141995906829834, "contribution_to_active": 0.2966594994068146, "mix_mean": 0.022497907280921936, "mix_low": 0.48692798614501953, "mix_high": 1.9073486328125e-06, "read_cosine": 0.5356122851371765, "beta_mean": 0.26010507345199585, "nonfinite": 0.0, "weighted_memory_to_active": 0.3304454982280731, "mix_std": 0.046049460768699646, "beta_std": 0.18850237131118774, "beta_low": 0.00396728515625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.022233812138438225, "evidence_weights": [0.021068202331662178, -0.06540714204311371, 0.05136742442846298], "agreement_weight": -0.00036123680183663964, "orthogonal_weight": 0.2125084549188614}, {"head": 2, "memory_to_active": 0.039399776607751846, "contribution_to_active": 0.5548087358474731, "mix_mean": 0.6802146434783936, "mix_low": 0.0003509521484375, "mix_high": 0.0040531158447265625, "read_cosine": 0.4954952597618103, "beta_mean": 0.00131317344494164, "nonfinite": 0.0, "weighted_memory_to_active": 0.025868995115160942, "mix_std": 0.22641152143478394, "beta_std": 0.003683187533169985, "beta_low": 0.9810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.017902256920933723, "evidence_weights": [-0.2745129466056824, -0.3703232407569885, 0.6611884236335754], "agreement_weight": 0.053609225898981094, "orthogonal_weight": -0.8229647874832153}, {"head": 3, "memory_to_active": 3.1531319618225098, "contribution_to_active": 0.18839174509048462, "mix_mean": 0.023643692955374718, "mix_low": 0.5259428024291992, "mix_high": 0.0, "read_cosine": 0.1855153739452362, "beta_mean": 0.3618709146976471, "nonfinite": 0.0, "weighted_memory_to_active": 0.2571472227573395, "mix_std": 0.06124414876103401, "beta_std": 0.26854777336120605, "beta_low": 0.05987548828125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.10472840070724487, "evidence_weights": [0.04046469181776047, -0.017102094367146492, -0.00793426763266325], "agreement_weight": -0.02627679519355297, "orthogonal_weight": -0.49198785424232483}, {"head": 4, "memory_to_active": 4.000442981719971, "contribution_to_active": 1.2878053188323975, "mix_mean": 0.03333994001150131, "mix_low": 0.3213968276977539, "mix_high": 0.0, "read_cosine": 0.33339792490005493, "beta_mean": 0.18099768459796906, "nonfinite": 0.0, "weighted_memory_to_active": 1.2539066076278687, "mix_std": 0.055678144097328186, "beta_std": 0.12395434826612473, "beta_low": 0.01849365234375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.053411807864904404, "evidence_weights": [-0.02235790342092514, -0.009892112575471401, 0.1599014550447464], "agreement_weight": -0.009773886762559414, "orthogonal_weight": 0.06299570202827454}, {"head": 5, "memory_to_active": 4.625655174255371, "contribution_to_active": 0.08267813920974731, "mix_mean": 0.005824184976518154, "mix_low": 0.8633232116699219, "mix_high": 0.0, "read_cosine": 0.27533262968063354, "beta_mean": 0.30737870931625366, "nonfinite": 0.0, "weighted_memory_to_active": 0.0557795949280262, "mix_std": 0.015939733013510704, "beta_std": 0.19689953327178955, "beta_low": 0.03399658203125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.023703543469309807, "evidence_weights": [0.1673710197210312, 0.08403299003839493, 0.10811331868171692], "agreement_weight": -0.11174839735031128, "orthogonal_weight": 0.41023629903793335}]}, {"layer": 2, "memory_to_active": 1.0569075345993042, "contribution_to_active": 0.3627493381500244, "mix_mean": 0.4078245162963867, "mix_low": 0.2418772429227829, "mix_high": 0.0040283203125, "read_cosine": 0.27891409397125244, "beta_mean": 0.07945246249437332, "nonfinite": 0.0, "weighted_memory_to_active": 0.08294636011123657, "mix_std": 0.18673831224441528, "beta_std": 0.07045415043830872, "beta_low": 0.606201171875, "beta_high": 0.00020345053053461015, "orthogonal_to_active": 0.01826559752225876, "output_influence": 0.40469998121261597, "residual_influence": 0.11332520842552185, "state_active": 0.04061516746878624, "state_memory": 0.014282799325883389, "state_peak": 31.172924041748047, "growth_active": 1.08693528175354, "growth_memory": 1.5854531526565552, "heads": [{"head": 0, "memory_to_active": 0.004778953734785318, "contribution_to_active": 0.3420495390892029, "mix_mean": 0.637384295463562, "mix_low": 0.0005998611450195312, "mix_high": 0.0073986053466796875, "read_cosine": 0.13326361775398254, "beta_mean": 3.9831167669035494e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0030920496210455894, "mix_std": 0.2665262818336487, "beta_std": 6.813864456489682e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003284074831753969, "evidence_weights": [-0.37318217754364014, -0.2151418775320053, 0.2203354686498642], "agreement_weight": 0.14901915192604065, "orthogonal_weight": -1.0956023931503296}, {"head": 1, "memory_to_active": 3.3364453315734863, "contribution_to_active": 0.05186368525028229, "mix_mean": 0.0055504729971289635, "mix_low": 0.8727874755859375, "mix_high": 0.0, "read_cosine": 0.10567494481801987, "beta_mean": 0.22334134578704834, "nonfinite": 0.0, "weighted_memory_to_active": 0.04913828521966934, "mix_std": 0.011900199577212334, "beta_std": 0.16072125732898712, "beta_low": 0.005615234375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.012412600219249725, "evidence_weights": [0.13767801225185394, 0.11013180017471313, 0.5786729454994202], "agreement_weight": 0.025117963552474976, "orthogonal_weight": 0.33185654878616333}, {"head": 2, "memory_to_active": 0.008450371213257313, "contribution_to_active": 0.4255622625350952, "mix_mean": 0.552384614944458, "mix_low": 0.0011720657348632812, "mix_high": 0.0038776397705078125, "read_cosine": 0.326435923576355, "beta_mean": 1.918496855068952e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003693562000989914, "mix_std": 0.2747461795806885, "beta_std": 3.179359555360861e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 6.488526560133323e-05, "evidence_weights": [-0.24989016354084015, -0.29318884015083313, 0.5841190814971924], "agreement_weight": -0.0035998246166855097, "orthogonal_weight": 0.021641306579113007}, {"head": 3, "memory_to_active": 0.1025528609752655, "contribution_to_active": 0.5392490029335022, "mix_mean": 0.579096794128418, "mix_low": 0.000225067138671875, "mix_high": 0.00579833984375, "read_cosine": 0.5436816811561584, "beta_mean": 0.018084418028593063, "nonfinite": 0.0, "weighted_memory_to_active": 0.06587111949920654, "mix_std": 0.25613903999328613, "beta_std": 0.02779719978570938, "beta_low": 0.576416015625, "beta_high": 0.0, "orthogonal_to_active": 0.04725109040737152, "evidence_weights": [-0.20291665196418762, -0.3262292444705963, 0.7469280362129211], "agreement_weight": 0.05460532382130623, "orthogonal_weight": -0.8959598541259766}, {"head": 4, "memory_to_active": 2.8786191940307617, "contribution_to_active": 0.2851719260215759, "mix_mean": 0.02052229270339012, "mix_low": 0.5757617950439453, "mix_high": 0.0, "read_cosine": 0.1900588572025299, "beta_mean": 0.2351909577846527, "nonfinite": 0.0, "weighted_memory_to_active": 0.37023961544036865, "mix_std": 0.050056979060173035, "beta_std": 0.23405757546424866, "beta_low": 0.05517578125, "beta_high": 0.00115966796875, "orthogonal_to_active": 0.04501007869839668, "evidence_weights": [0.026183757930994034, 0.005469673313200474, 0.0373181514441967], "agreement_weight": 0.02254820428788662, "orthogonal_weight": -0.1390809863805771}, {"head": 5, "memory_to_active": 0.010597823187708855, "contribution_to_active": 0.5325995087623596, "mix_mean": 0.6520085334777832, "mix_low": 0.0007171630859375, "mix_high": 0.0070953369140625, "read_cosine": 0.3743695616722107, "beta_mean": 3.900635783793405e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005643553100526333, "mix_std": 0.2610611617565155, "beta_std": 4.894165977020748e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015708489809185266, "evidence_weights": [-0.27769142389297485, -0.2888292968273163, 0.8866374492645264], "agreement_weight": 0.2573487162590027, "orthogonal_weight": -0.3253241777420044}]}, {"layer": 3, "memory_to_active": 0.4586184322834015, "contribution_to_active": 0.489989697933197, "mix_mean": 0.5198153853416443, "mix_low": 0.05114714801311493, "mix_high": 0.011339823715388775, "read_cosine": 0.17607083916664124, "beta_mean": 0.024115771055221558, "nonfinite": 0.0, "weighted_memory_to_active": 0.1263824701309204, "mix_std": 0.25729140639305115, "beta_std": 0.023991117253899574, "beta_low": 0.778656005859375, "beta_high": 0.0, "orthogonal_to_active": 0.04992392659187317, "output_influence": 0.43308305740356445, "residual_influence": 0.10723897814750671, "state_active": 0.023708708584308624, "state_memory": 0.006655050907284021, "state_peak": 23.8779354095459, "growth_active": 1.0739959478378296, "growth_memory": 1.5958538055419922, "heads": [{"head": 0, "memory_to_active": 0.33381664752960205, "contribution_to_active": 0.6110761165618896, "mix_mean": 0.6137148141860962, "mix_low": 0.0011835098266601562, "mix_high": 0.024133682250976562, "read_cosine": 0.2505488693714142, "beta_mean": 0.008516723290085793, "nonfinite": 0.0, "weighted_memory_to_active": 0.21126998960971832, "mix_std": 0.29899442195892334, "beta_std": 0.009349200874567032, "beta_low": 0.734130859375, "beta_high": 0.0, "orthogonal_to_active": 0.1541925072669983, "evidence_weights": [-0.21783766150474548, -0.2529848515987396, 1.056536316871643], "agreement_weight": 0.10998499393463135, "orthogonal_weight": -0.7714865207672119}, {"head": 1, "memory_to_active": 0.007320970296859741, "contribution_to_active": 0.4597981572151184, "mix_mean": 0.5855743885040283, "mix_low": 0.0034284591674804688, "mix_high": 0.010593414306640625, "read_cosine": 0.18967273831367493, "beta_mean": 3.818440382019617e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004419365432113409, "mix_std": 0.2939303517341614, "beta_std": 9.521851461613551e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0034758485853672028, "evidence_weights": [-0.27026763558387756, -0.21737684309482574, 0.7118676900863647], "agreement_weight": 0.5993242263793945, "orthogonal_weight": -0.8646164536476135}, {"head": 2, "memory_to_active": 2.1495001316070557, "contribution_to_active": 0.3045217990875244, "mix_mean": 0.058494504541158676, "mix_low": 0.2932605743408203, "mix_high": 9.5367431640625e-07, "read_cosine": 0.19280777871608734, "beta_mean": 0.13208924233913422, "nonfinite": 0.0, "weighted_memory_to_active": 0.34985506534576416, "mix_std": 0.10978089272975922, "beta_std": 0.12517865002155304, "beta_low": 0.02154541015625, "beta_high": 0.0, "orthogonal_to_active": 0.0003119518223684281, "evidence_weights": [-0.10407973825931549, -0.010259762406349182, -0.04432423785328865], "agreement_weight": -0.01138329692184925, "orthogonal_weight": -0.0011305321240797639}, {"head": 3, "memory_to_active": 0.016497883945703506, "contribution_to_active": 0.5246398448944092, "mix_mean": 0.6031293272972107, "mix_low": 0.004328727722167969, "mix_high": 0.01006317138671875, "read_cosine": 0.2747853994369507, "beta_mean": 7.637181261088699e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011146243661642075, "mix_std": 0.2848137319087982, "beta_std": 0.00016824559133965522, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028174452017992735, "evidence_weights": [-0.26326853036880493, -0.22367767989635468, 2.3039960861206055], "agreement_weight": 2.259220838546753, "orthogonal_weight": -0.3208730220794678}, {"head": 4, "memory_to_active": 0.01589777320623398, "contribution_to_active": 0.38627392053604126, "mix_mean": 0.5313019752502441, "mix_low": 0.0024700164794921875, "mix_high": 0.002593994140625, "read_cosine": 0.12439073622226715, "beta_mean": 7.861177437007427e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010504856705665588, "mix_std": 0.28819698095321655, "beta_std": 0.00011191442172275856, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00986708514392376, "evidence_weights": [-0.2429712563753128, -0.22425071895122528, 0.056654322892427444], "agreement_weight": -0.06852174550294876, "orthogonal_weight": -1.0207200050354004}, {"head": 5, "memory_to_active": 0.22867706418037415, "contribution_to_active": 0.6536281108856201, "mix_mean": 0.7266772985458374, "mix_low": 0.0022115707397460938, "mix_high": 0.020653724670410156, "read_cosine": 0.02421945333480835, "beta_mean": 0.0038954855408519506, "nonfinite": 0.0, "weighted_memory_to_active": 0.1710992306470871, "mix_std": 0.2680319547653198, "beta_std": 0.009043475612998009, "beta_low": 0.916259765625, "beta_high": 0.0, "orthogonal_to_active": 0.12887871265411377, "evidence_weights": [-0.26192912459373474, -0.3437180519104004, 0.5448493361473083], "agreement_weight": 0.23168258368968964, "orthogonal_weight": -0.7771003842353821}]}, {"layer": 4, "memory_to_active": 0.9780912399291992, "contribution_to_active": 0.47678712010383606, "mix_mean": 0.37234771251678467, "mix_low": 0.23859962821006775, "mix_high": 0.011916002258658409, "read_cosine": 0.30195194482803345, "beta_mean": 0.10787333548069, "nonfinite": 0.0, "weighted_memory_to_active": 0.13040614128112793, "mix_std": 0.16049475967884064, "beta_std": 0.09976842999458313, "beta_low": 0.3922831416130066, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.05450618267059326, "output_influence": 0.49871012568473816, "residual_influence": 0.15584418177604675, "state_active": 0.037988610565662384, "state_memory": 0.016080347821116447, "state_peak": 38.77642059326172, "growth_active": 1.203827142715454, "growth_memory": 1.5154205560684204, "heads": [{"head": 0, "memory_to_active": 0.14854368567466736, "contribution_to_active": 0.801234245300293, "mix_mean": 0.6898439526557922, "mix_low": 0.010642051696777344, "mix_high": 0.034679412841796875, "read_cosine": 0.4449687600135803, "beta_mean": 0.2640748918056488, "nonfinite": 0.0, "weighted_memory_to_active": 0.12722082436084747, "mix_std": 0.3477412462234497, "beta_std": 0.2539868652820587, "beta_low": 0.23028564453125, "beta_high": 0.0, "orthogonal_to_active": 0.08501749485731125, "evidence_weights": [-0.2514200806617737, -0.3624158799648285, -0.17187274992465973], "agreement_weight": 0.004776380490511656, "orthogonal_weight": 0.7998902201652527}, {"head": 1, "memory_to_active": 0.04218093305826187, "contribution_to_active": 0.655331015586853, "mix_mean": 0.7005013227462769, "mix_low": 8.7738037109375e-05, "mix_high": 0.017975807189941406, "read_cosine": 0.5752041935920715, "beta_mean": 0.0010588322766125202, "nonfinite": 0.0, "weighted_memory_to_active": 0.031519822776317596, "mix_std": 0.23891383409500122, "beta_std": 0.007830831222236156, "beta_low": 0.979736328125, "beta_high": 0.0, "orthogonal_to_active": 0.01630489155650139, "evidence_weights": [-0.23788706958293915, -0.36677682399749756, 0.5955670475959778], "agreement_weight": 0.35812699794769287, "orthogonal_weight": -0.7111654281616211}, {"head": 2, "memory_to_active": 1.4594184160232544, "contribution_to_active": 0.36502039432525635, "mix_mean": 0.04900767281651497, "mix_low": 0.34157562255859375, "mix_high": 2.193450927734375e-05, "read_cosine": 0.21357189118862152, "beta_mean": 0.05435991659760475, "nonfinite": 0.0, "weighted_memory_to_active": 0.2641015946865082, "mix_std": 0.08880499005317688, "beta_std": 0.05439136177301407, "beta_low": 0.11181640625, "beta_high": 0.0, "orthogonal_to_active": 0.09293577075004578, "evidence_weights": [0.03328106179833412, -0.053945429623126984, 0.3170933127403259], "agreement_weight": 0.1253938525915146, "orthogonal_weight": -0.47421324253082275}, {"head": 3, "memory_to_active": 1.8192485570907593, "contribution_to_active": 0.1444769948720932, "mix_mean": 0.030702363699674606, "mix_low": 0.4195871353149414, "mix_high": 0.0, "read_cosine": 0.22380006313323975, "beta_mean": 0.13184383511543274, "nonfinite": 0.0, "weighted_memory_to_active": 0.190326988697052, "mix_std": 0.05241174250841141, "beta_std": 0.12017937004566193, "beta_low": 0.03167724609375, "beta_high": 0.0, "orthogonal_to_active": 0.000472531421110034, "evidence_weights": [-0.016831934452056885, 0.05237855762243271, -0.27764269709587097], "agreement_weight": 0.0744958445429802, "orthogonal_weight": -0.0033910442143678665}, {"head": 4, "memory_to_active": 0.00755054596811533, "contribution_to_active": 0.6260092854499817, "mix_mean": 0.7510838508605957, "mix_low": 3.528594970703125e-05, "mix_high": 0.018815040588378906, "read_cosine": 0.1296624392271042, "beta_mean": 1.7520564142614603e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005538608878850937, "mix_std": 0.20914487540721893, "beta_std": 4.4474360038293526e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035620443522930145, "evidence_weights": [-0.33110731840133667, -0.31946080923080444, 0.2824716866016388], "agreement_weight": 0.314497172832489, "orthogonal_weight": -0.6797429919242859}, {"head": 5, "memory_to_active": 2.3916051387786865, "contribution_to_active": 0.268650621175766, "mix_mean": 0.012947017326951027, "mix_low": 0.6596698760986328, "mix_high": 3.814697265625e-06, "read_cosine": 0.22450433671474457, "beta_mean": 0.19588498771190643, "nonfinite": 0.0, "weighted_memory_to_active": 0.16372895240783691, "mix_std": 0.02595178782939911, "beta_std": 0.16217762231826782, "beta_low": 0.00018310546875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.12874437868595123, "evidence_weights": [0.05853460356593132, -0.015571050345897675, -0.270506352186203], "agreement_weight": -0.08868176490068436, "orthogonal_weight": 1.0877866744995117}]}, {"layer": 5, "memory_to_active": 0.4022439420223236, "contribution_to_active": 0.5514671802520752, "mix_mean": 0.5194836854934692, "mix_low": 0.05319515988230705, "mix_high": 0.0209808349609375, "read_cosine": 0.06901341676712036, "beta_mean": 0.05344201624393463, "nonfinite": 0.0, "weighted_memory_to_active": 0.140860453248024, "mix_std": 0.2521030306816101, "beta_std": 0.06946676224470139, "beta_low": 0.6780192255973816, "beta_high": 0.0, "orthogonal_to_active": 0.05874934047460556, "output_influence": 0.5831447243690491, "residual_influence": 0.18674132227897644, "state_active": 0.04113519936800003, "state_memory": 0.010087400674819946, "state_peak": 56.69792556762695, "growth_active": 1.117274284362793, "growth_memory": 1.2782254219055176, "heads": [{"head": 0, "memory_to_active": 0.20726792514324188, "contribution_to_active": 0.6025331020355225, "mix_mean": 0.48986580967903137, "mix_low": 0.006993293762207031, "mix_high": 0.0017261505126953125, "read_cosine": -0.14968574047088623, "beta_mean": 0.017550168558955193, "nonfinite": 0.0, "weighted_memory_to_active": 0.07745282351970673, "mix_std": 0.27769041061401367, "beta_std": 0.030865667387843132, "beta_low": 0.56512451171875, "beta_high": 0.0, "orthogonal_to_active": 0.03421171382069588, "evidence_weights": [-0.16974091529846191, -0.1884879767894745, 0.11512569338083267], "agreement_weight": -0.01673503778874874, "orthogonal_weight": -0.471447616815567}, {"head": 1, "memory_to_active": 0.005245938431471586, "contribution_to_active": 0.5583451390266418, "mix_mean": 0.710159420967102, "mix_low": 0.000782012939453125, "mix_high": 0.04272651672363281, "read_cosine": 0.21424628794193268, "beta_mean": 1.1560029633983504e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0032502319663763046, "mix_std": 0.2644246518611908, "beta_std": 2.59821808867855e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0029490587767213583, "evidence_weights": [-0.24643142521381378, -0.2852972447872162, 0.9287807941436768], "agreement_weight": 0.7000643014907837, "orthogonal_weight": -0.9550231695175171}, {"head": 2, "memory_to_active": 0.011794134974479675, "contribution_to_active": 0.5969042778015137, "mix_mean": 0.6607012748718262, "mix_low": 0.0024614334106445312, "mix_high": 0.022342681884765625, "read_cosine": -0.4860086739063263, "beta_mean": 6.773177301511168e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006916660349816084, "mix_std": 0.2634016275405884, "beta_std": 0.0002806131960824132, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00636827340349555, "evidence_weights": [-0.2590537965297699, -0.3180927336215973, 0.16780994832515717], "agreement_weight": -0.1768970936536789, "orthogonal_weight": -1.0243756771087646}, {"head": 3, "memory_to_active": 1.698301076889038, "contribution_to_active": 0.26625174283981323, "mix_mean": 0.11683009564876556, "mix_low": 0.12062835693359375, "mix_high": 0.0, "read_cosine": 0.14367961883544922, "beta_mean": 0.10614880174398422, "nonfinite": 0.0, "weighted_memory_to_active": 0.438346266746521, "mix_std": 0.1476326435804367, "beta_std": 0.16516585648059845, "beta_low": 0.21392822265625, "beta_high": 0.0, "orthogonal_to_active": 0.169889897108078, "evidence_weights": [-0.07378334552049637, -0.09738186746835709, 0.08427682518959045], "agreement_weight": 0.049071356654167175, "orthogonal_weight": -0.49546724557876587}, {"head": 4, "memory_to_active": 0.42079898715019226, "contribution_to_active": 0.5898938775062561, "mix_mean": 0.3297590911388397, "mix_low": 0.18686485290527344, "mix_high": 0.006245613098144531, "read_cosine": 0.5384541749954224, "beta_mean": 0.19667111337184906, "nonfinite": 0.0, "weighted_memory_to_active": 0.27136027812957764, "mix_std": 0.34127411246299744, "beta_std": 0.21954888105392456, "beta_low": 0.291015625, "beta_high": 0.0, "orthogonal_to_active": 0.11053164303302765, "evidence_weights": [-0.11671912670135498, -0.25550004839897156, -0.047153130173683167], "agreement_weight": 0.035033661872148514, "orthogonal_weight": 0.6170841455459595}, {"head": 5, "memory_to_active": 0.07005555927753448, "contribution_to_active": 0.6948749423027039, "mix_mean": 0.8095859885215759, "mix_low": 0.0014410018920898438, "mix_high": 0.05284404754638672, "read_cosine": 0.15339480340480804, "beta_mean": 0.00020272197434678674, "nonfinite": 0.0, "weighted_memory_to_active": 0.04783647507429123, "mix_std": 0.21819478273391724, "beta_std": 0.0009135638829320669, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.028545435518026352, "evidence_weights": [-0.30388110876083374, -0.3275260627269745, 1.8335180282592773], "agreement_weight": 0.5168485045433044, "orthogonal_weight": -0.6241624355316162}]}, {"layer": 6, "memory_to_active": 0.2687080502510071, "contribution_to_active": 0.6433922052383423, "mix_mean": 0.5939900875091553, "mix_low": 0.037263236939907074, "mix_high": 0.029364746063947678, "read_cosine": 0.31901636719703674, "beta_mean": 0.06367398798465729, "nonfinite": 0.0, "weighted_memory_to_active": 0.14204177260398865, "mix_std": 0.23850300908088684, "beta_std": 0.07456271350383759, "beta_low": 0.5948283076286316, "beta_high": 0.0, "orthogonal_to_active": 0.0796305388212204, "output_influence": 0.576984167098999, "residual_influence": 0.20927710831165314, "state_active": 0.04427000880241394, "state_memory": 0.006401236169040203, "state_peak": 94.68090057373047, "growth_active": 1.212045431137085, "growth_memory": 1.1576786041259766, "heads": [{"head": 0, "memory_to_active": 0.12363380193710327, "contribution_to_active": 0.7704883813858032, "mix_mean": 0.8240942358970642, "mix_low": 0.000232696533203125, "mix_high": 0.05900382995605469, "read_cosine": 0.3185221552848816, "beta_mean": 0.012253313325345516, "nonfinite": 0.0, "weighted_memory_to_active": 0.11098170280456543, "mix_std": 0.19875815510749817, "beta_std": 0.0362810418009758, "beta_low": 0.81329345703125, "beta_high": 0.0, "orthogonal_to_active": 0.07351058721542358, "evidence_weights": [-0.30151641368865967, -0.3479606509208679, 1.19391667842865], "agreement_weight": 0.8512141108512878, "orthogonal_weight": -0.7235347032546997}, {"head": 1, "memory_to_active": 0.004652315750718117, "contribution_to_active": 0.5245966911315918, "mix_mean": 0.5973215103149414, "mix_low": 0.00064849853515625, "mix_high": 0.015799522399902344, "read_cosine": 0.468843013048172, "beta_mean": 9.537591722619254e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021805765572935343, "mix_std": 0.28052574396133423, "beta_std": 1.5011833056632895e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011988709447905421, "evidence_weights": [-0.1929502636194229, -0.3027111887931824, 1.1209893226623535], "agreement_weight": 0.1227145791053772, "orthogonal_weight": -0.7117260098457336}, {"head": 2, "memory_to_active": 0.7070578336715698, "contribution_to_active": 0.09774309396743774, "mix_mean": 0.05309800058603287, "mix_low": 0.2147693634033203, "mix_high": 0.0, "read_cosine": 0.26947683095932007, "beta_mean": 0.021982189267873764, "nonfinite": 0.0, "weighted_memory_to_active": 0.06365060806274414, "mix_std": 0.07872426509857178, "beta_std": 0.023540934547781944, "beta_low": 0.26275634765625, "beta_high": 0.0, "orthogonal_to_active": 0.011267032474279404, "evidence_weights": [0.06298355758190155, -0.0642181932926178, 0.3513764441013336], "agreement_weight": 0.1401054710149765, "orthogonal_weight": -0.20821012556552887}, {"head": 3, "memory_to_active": 0.13605736196041107, "contribution_to_active": 0.6664754152297974, "mix_mean": 0.6649967432022095, "mix_low": 0.006740570068359375, "mix_high": 0.023446083068847656, "read_cosine": 0.46960997581481934, "beta_mean": 0.21962425112724304, "nonfinite": 0.0, "weighted_memory_to_active": 0.09592532366514206, "mix_std": 0.3256705701351166, "beta_std": 0.2486257553100586, "beta_low": 0.19586181640625, "beta_high": 0.0, "orthogonal_to_active": 0.05657727271318436, "evidence_weights": [-0.1669294834136963, -0.3154444694519043, 0.12530073523521423], "agreement_weight": -0.008227050304412842, "orthogonal_weight": 0.6739996671676636}, {"head": 4, "memory_to_active": 0.611195981502533, "contribution_to_active": 1.1696345806121826, "mix_mean": 0.7339863777160645, "mix_low": 0.0010328292846679688, "mix_high": 0.06978130340576172, "read_cosine": 0.11674466729164124, "beta_mean": 0.12804296612739563, "nonfinite": 0.0, "weighted_memory_to_active": 0.5577468872070312, "mix_std": 0.3058832883834839, "beta_std": 0.13855890929698944, "beta_low": 0.29718017578125, "beta_high": 0.0, "orthogonal_to_active": 0.314159631729126, "evidence_weights": [-0.17820601165294647, -0.5270357728004456, -0.6773134469985962], "agreement_weight": -0.017728155478835106, "orthogonal_weight": 0.5894436836242676}, {"head": 5, "memory_to_active": 0.029650919139385223, "contribution_to_active": 0.6314150094985962, "mix_mean": 0.6904433369636536, "mix_low": 0.00015544891357421875, "mix_high": 0.008157730102539062, "read_cosine": 0.27090150117874146, "beta_mean": 0.00013165510608814657, "nonfinite": 0.0, "weighted_memory_to_active": 0.021765490993857384, "mix_std": 0.24145600199699402, "beta_std": 0.00035463529638946056, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.021069824695587158, "evidence_weights": [-0.23401407897472382, -0.3153580129146576, 0.36046019196510315], "agreement_weight": 0.20914945006370544, "orthogonal_weight": -1.0357710123062134}]}, {"layer": 7, "memory_to_active": 1.7869665622711182, "contribution_to_active": 0.26903215050697327, "mix_mean": 0.17337079346179962, "mix_low": 0.2314024120569229, "mix_high": 0.0009161632042378187, "read_cosine": 0.2404450923204422, "beta_mean": 0.14321470260620117, "nonfinite": 0.0, "weighted_memory_to_active": 0.20737284421920776, "mix_std": 0.14754393696784973, "beta_std": 0.10072130709886551, "beta_low": 0.2130025327205658, "beta_high": 0.00032552084303461015, "orthogonal_to_active": 0.09466879069805145, "output_influence": 0.2885280251502991, "residual_influence": 0.0724327564239502, "state_active": 0.01400003768503666, "state_memory": 0.023963067680597305, "state_peak": 28.211580276489258, "growth_active": 1.1552488803863525, "growth_memory": 1.2664743661880493, "heads": [{"head": 0, "memory_to_active": 2.736372709274292, "contribution_to_active": 0.17418229579925537, "mix_mean": 0.038933657109737396, "mix_low": 0.4232301712036133, "mix_high": 9.5367431640625e-07, "read_cosine": 0.32921484112739563, "beta_mean": 0.19133982062339783, "nonfinite": 0.0, "weighted_memory_to_active": 0.19733421504497528, "mix_std": 0.0793418437242508, "beta_std": 0.13542291522026062, "beta_low": 0.000732421875, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.0025879316963255405, "evidence_weights": [0.043910153210163116, -0.08924012631177902, 0.010641207918524742], "agreement_weight": -0.02839658409357071, "orthogonal_weight": -0.01460570190101862}, {"head": 1, "memory_to_active": 0.9240447878837585, "contribution_to_active": 0.37656325101852417, "mix_mean": 0.21409885585308075, "mix_low": 0.12262439727783203, "mix_high": 0.004756927490234375, "read_cosine": 0.40457582473754883, "beta_mean": 0.09232577681541443, "nonfinite": 0.0, "weighted_memory_to_active": 0.31159865856170654, "mix_std": 0.24087682366371155, "beta_std": 0.06256195902824402, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.04408511146903038, "evidence_weights": [-0.21796995401382446, -0.10465969890356064, -0.31468018889427185], "agreement_weight": 0.04922519996762276, "orthogonal_weight": 0.17133164405822754}, {"head": 2, "memory_to_active": 0.6269354224205017, "contribution_to_active": 0.38685983419418335, "mix_mean": 0.2327849417924881, "mix_low": 0.017800331115722656, "mix_high": 0.000331878662109375, "read_cosine": 0.11739074438810349, "beta_mean": 0.03730577602982521, "nonfinite": 0.0, "weighted_memory_to_active": 0.24118058383464813, "mix_std": 0.21425078809261322, "beta_std": 0.03971279039978981, "beta_low": 0.15289306640625, "beta_high": 0.0, "orthogonal_to_active": 0.14193259179592133, "evidence_weights": [-0.0958508774638176, -0.18298333883285522, 0.8657434582710266], "agreement_weight": 0.012565594166517258, "orthogonal_weight": -0.6065863370895386}, {"head": 3, "memory_to_active": 4.811829566955566, "contribution_to_active": 0.0972064957022667, "mix_mean": 0.03623955696821213, "mix_low": 0.31389522552490234, "mix_high": 0.0, "read_cosine": 0.10232017934322357, "beta_mean": 0.4602939486503601, "nonfinite": 0.0, "weighted_memory_to_active": 0.38786399364471436, "mix_std": 0.0523286797106266, "beta_std": 0.28185218572616577, "beta_low": 0.00177001953125, "beta_high": 0.00103759765625, "orthogonal_to_active": 0.3396095037460327, "evidence_weights": [0.07452675700187683, -0.04027218371629715, 0.7136316299438477], "agreement_weight": 0.05117015540599823, "orthogonal_weight": -0.9429143071174622}, {"head": 4, "memory_to_active": 1.5767543315887451, "contribution_to_active": 0.11627520620822906, "mix_mean": 0.024053139612078667, "mix_low": 0.5090007781982422, "mix_high": 0.0, "read_cosine": 0.23314166069030762, "beta_mean": 0.0778248980641365, "nonfinite": 0.0, "weighted_memory_to_active": 0.08317754417657852, "mix_std": 0.04787883162498474, "beta_std": 0.08429300785064697, "beta_low": 0.12274169921875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.01784677430987358, "evidence_weights": [0.12320690602064133, -0.024841152131557465, 0.18300917744636536], "agreement_weight": 0.030824072659015656, "orthogonal_weight": -0.2951987087726593}, {"head": 5, "memory_to_active": 0.045861661434173584, "contribution_to_active": 0.463105708360672, "mix_mean": 0.4941145181655884, "mix_low": 0.0018634796142578125, "mix_high": 0.00040721893310546875, "read_cosine": 0.2560272514820099, "beta_mean": 0.00019800836162175983, "nonfinite": 0.0, "weighted_memory_to_active": 0.02308197133243084, "mix_std": 0.2505866587162018, "beta_std": 0.00048492924543097615, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.02195080742239952, "evidence_weights": [-0.18379980325698853, -0.239423006772995, 0.47725191712379456], "agreement_weight": 0.23847873508930206, "orthogonal_weight": -1.010974645614624}]}, {"layer": 8, "memory_to_active": 0.14175443351268768, "contribution_to_active": 0.5655630826950073, "mix_mean": 0.4999832808971405, "mix_low": 0.010384559631347656, "mix_high": 0.0018634798470884562, "read_cosine": 0.3900753855705261, "beta_mean": 0.013983987271785736, "nonfinite": 0.0, "weighted_memory_to_active": 0.0722324475646019, "mix_std": 0.24141232669353485, "beta_std": 0.021492671221494675, "beta_low": 0.7705078125, "beta_high": 0.0, "orthogonal_to_active": 0.06632286310195923, "output_influence": 0.5233612060546875, "residual_influence": 0.19316013157367706, "state_active": 0.0437815897166729, "state_memory": 0.0051282052882015705, "state_peak": 53.67525100708008, "growth_active": 1.1005816459655762, "growth_memory": 1.0795249938964844, "heads": [{"head": 0, "memory_to_active": 0.05879582464694977, "contribution_to_active": 0.5734710097312927, "mix_mean": 0.6011202335357666, "mix_low": 0.0004968643188476562, "mix_high": 0.0014162063598632812, "read_cosine": 0.4630293846130371, "beta_mean": 0.003918841481208801, "nonfinite": 0.0, "weighted_memory_to_active": 0.04188035428524017, "mix_std": 0.24974505603313446, "beta_std": 0.016493145376443863, "beta_low": 0.92218017578125, "beta_high": 0.0, "orthogonal_to_active": 0.026092108339071274, "evidence_weights": [-0.1567831039428711, -0.3783473074436188, 0.5743499398231506], "agreement_weight": 0.2658083736896515, "orthogonal_weight": -0.7993330359458923}, {"head": 1, "memory_to_active": 0.08118677139282227, "contribution_to_active": 0.5464122891426086, "mix_mean": 0.5241515636444092, "mix_low": 0.00018405914306640625, "mix_high": 0.0013189315795898438, "read_cosine": 0.042643576860427856, "beta_mean": 0.0012076862622052431, "nonfinite": 0.0, "weighted_memory_to_active": 0.03591585531830788, "mix_std": 0.2331642508506775, "beta_std": 0.011480358429253101, "beta_low": 0.9888916015625, "beta_high": 0.0, "orthogonal_to_active": 0.026680629700422287, "evidence_weights": [-0.19378480315208435, -0.27385011315345764, 0.6038450598716736], "agreement_weight": 0.07171406596899033, "orthogonal_weight": -0.76949542760849}, {"head": 2, "memory_to_active": 0.010262216441333294, "contribution_to_active": 0.6894763708114624, "mix_mean": 0.6869804263114929, "mix_low": 0.000316619873046875, "mix_high": 0.0038433074951171875, "read_cosine": 0.3984585404396057, "beta_mean": 0.00011372858716640621, "nonfinite": 0.0, "weighted_memory_to_active": 0.007950876839458942, "mix_std": 0.21968162059783936, "beta_std": 0.0035524696577340364, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004649214446544647, "evidence_weights": [-0.2219155728816986, -0.34699344635009766, 0.9758520126342773], "agreement_weight": 0.3129327595233917, "orthogonal_weight": -0.6701008677482605}, {"head": 3, "memory_to_active": 0.1045726090669632, "contribution_to_active": 0.6327463984489441, "mix_mean": 0.585618257522583, "mix_low": 0.0004787445068359375, "mix_high": 0.0043392181396484375, "read_cosine": 0.8072008490562439, "beta_mean": 0.008809002116322517, "nonfinite": 0.0, "weighted_memory_to_active": 0.07783962786197662, "mix_std": 0.2541889250278473, "beta_std": 0.00866942759603262, "beta_low": 0.677001953125, "beta_high": 0.0, "orthogonal_to_active": 0.023669231683015823, "evidence_weights": [-0.17433477938175201, -0.29803237318992615, 0.5291029214859009], "agreement_weight": 0.25409257411956787, "orthogonal_weight": -0.5979709625244141}, {"head": 4, "memory_to_active": 0.23426195979118347, "contribution_to_active": 0.33479684591293335, "mix_mean": 0.3406716585159302, "mix_low": 0.0017271041870117188, "mix_high": 1.049041748046875e-05, "read_cosine": 0.12218280136585236, "beta_mean": 0.0008616140112280846, "nonfinite": 0.0, "weighted_memory_to_active": 0.08528792858123779, "mix_std": 0.22967921197414398, "beta_std": 0.013625015504658222, "beta_low": 0.9908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.016421299427747726, "evidence_weights": [-0.13833960890769958, -0.1993584781885147, 0.6816613674163818], "agreement_weight": 0.04348885267972946, "orthogonal_weight": -0.19609417021274567}, {"head": 5, "memory_to_active": 0.36144721508026123, "contribution_to_active": 0.616475522518158, "mix_mean": 0.26135727763175964, "mix_low": 0.059103965759277344, "mix_high": 0.00025272369384765625, "read_cosine": 0.5069372057914734, "beta_mean": 0.06899305433034897, "nonfinite": 0.0, "weighted_memory_to_active": 0.18452000617980957, "mix_std": 0.26201480627059937, "beta_std": 0.07513561099767685, "beta_low": 0.044189453125, "beta_high": 0.0, "orthogonal_to_active": 0.30042463541030884, "evidence_weights": [-0.15004962682724, -0.14414606988430023, -0.10921710729598999], "agreement_weight": -0.008872064761817455, "orthogonal_weight": 1.9440727233886719}]}, {"layer": 9, "memory_to_active": 0.8559095859527588, "contribution_to_active": 0.7127677202224731, "mix_mean": 0.3917793929576874, "mix_low": 0.05616903305053711, "mix_high": 0.006105582229793072, "read_cosine": 0.34731239080429077, "beta_mean": 0.03625502809882164, "nonfinite": 0.0, "weighted_memory_to_active": 0.3815869092941284, "mix_std": 0.24484851956367493, "beta_std": 0.030814295634627342, "beta_low": 0.628021240234375, "beta_high": 0.0, "orthogonal_to_active": 0.03373256325721741, "output_influence": 0.549645185470581, "residual_influence": 0.16688020527362823, "state_active": 0.04428804665803909, "state_memory": 0.014675654470920563, "state_peak": 61.07002258300781, "growth_active": 1.1802281141281128, "growth_memory": 1.2760709524154663, "heads": [{"head": 0, "memory_to_active": 0.000481060822494328, "contribution_to_active": 0.5766347050666809, "mix_mean": 0.6553902626037598, "mix_low": 0.001049041748046875, "mix_high": 0.02499675750732422, "read_cosine": 0.33614832162857056, "beta_mean": 1.2140948228989146e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00031795341055840254, "mix_std": 0.28460943698883057, "beta_std": 4.624192115443293e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003344382857903838, "evidence_weights": [-0.3051196336746216, -0.31172531843185425, 0.4087839126586914], "agreement_weight": 0.09577596932649612, "orthogonal_weight": -1.1108993291854858}, {"head": 1, "memory_to_active": 0.014267627149820328, "contribution_to_active": 0.7108863592147827, "mix_mean": 0.5737419128417969, "mix_low": 0.006543159484863281, "mix_high": 0.0032396316528320312, "read_cosine": 0.43222394585609436, "beta_mean": 3.14902063109912e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010789072141051292, "mix_std": 0.27886077761650085, "beta_std": 8.916942169889808e-05, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.007245642598718405, "evidence_weights": [-0.2364480346441269, -0.29382815957069397, 0.976177990436554], "agreement_weight": 0.5503023266792297, "orthogonal_weight": -0.8257267475128174}, {"head": 2, "memory_to_active": 0.17945052683353424, "contribution_to_active": 0.6603360176086426, "mix_mean": 0.6057417988777161, "mix_low": 0.0012493133544921875, "mix_high": 0.006344795227050781, "read_cosine": 0.35270625352859497, "beta_mean": 0.0006730620516464114, "nonfinite": 0.0, "weighted_memory_to_active": 0.12560415267944336, "mix_std": 0.2645476460456848, "beta_std": 0.0048355418257415295, "beta_low": 0.99603271484375, "beta_high": 0.0, "orthogonal_to_active": 0.07279067486524582, "evidence_weights": [-0.20130792260169983, -0.27734488248825073, 0.7583239674568176], "agreement_weight": 0.34580862522125244, "orthogonal_weight": -0.6778271198272705}, {"head": 3, "memory_to_active": 0.8036264181137085, "contribution_to_active": 0.25846749544143677, "mix_mean": 0.162019282579422, "mix_low": 0.09562301635742188, "mix_high": 4.1961669921875e-05, "read_cosine": 0.37611326575279236, "beta_mean": 0.030987296253442764, "nonfinite": 0.0, "weighted_memory_to_active": 0.1947360634803772, "mix_std": 0.20280539989471436, "beta_std": 0.031998615711927414, "beta_low": 0.1842041015625, "beta_high": 0.0, "orthogonal_to_active": 0.08695883303880692, "evidence_weights": [-0.030064061284065247, -0.10233017802238464, 0.4535508155822754], "agreement_weight": 0.09290659427642822, "orthogonal_weight": -0.49492573738098145}, {"head": 4, "memory_to_active": 3.0830893516540527, "contribution_to_active": 1.7485116720199585, "mix_mean": 0.14905712008476257, "mix_low": 0.1240386962890625, "mix_high": 0.0013666152954101562, "read_cosine": 0.12153968960046768, "beta_mean": 0.15597406029701233, "nonfinite": 0.0, "weighted_memory_to_active": 1.679229974746704, "mix_std": 0.2103365659713745, "beta_std": 0.08496812731027603, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.010874366387724876, "evidence_weights": [-0.08016198128461838, -0.15044672787189484, -0.0417153500020504], "agreement_weight": -0.06703325361013412, "orthogonal_weight": -0.00677890470251441}, {"head": 5, "memory_to_active": 1.0545421838760376, "contribution_to_active": 0.3217700123786926, "mix_mean": 0.20472568273544312, "mix_low": 0.10851097106933594, "mix_high": 0.0006437301635742188, "read_cosine": 0.46514272689819336, "beta_mean": 0.02986302226781845, "nonfinite": 0.0, "weighted_memory_to_active": 0.27884405851364136, "mix_std": 0.22793123126029968, "beta_std": 0.06298968940973282, "beta_low": 0.58795166015625, "beta_high": 0.0, "orthogonal_to_active": 0.024191424250602722, "evidence_weights": [0.04967186972498894, -0.18333114683628082, 0.14160668849945068], "agreement_weight": 0.007938973605632782, "orthogonal_weight": 0.1029437929391861}]}, {"layer": 10, "memory_to_active": 0.20868615806102753, "contribution_to_active": 0.7381129264831543, "mix_mean": 0.5257724523544312, "mix_low": 0.02233266830444336, "mix_high": 0.004923661705106497, "read_cosine": 0.3791908323764801, "beta_mean": 0.04679802432656288, "nonfinite": 0.0, "weighted_memory_to_active": 0.09048079699277878, "mix_std": 0.2666102349758148, "beta_std": 0.04654728248715401, "beta_low": 0.6597900390625, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.2745016813278198, "output_influence": 0.46905624866485596, "residual_influence": 0.1679111123085022, "state_active": 0.050659339874982834, "state_memory": 0.013894467614591122, "state_peak": 125.05995178222656, "growth_active": 1.0270509719848633, "growth_memory": 1.0086724758148193, "heads": [{"head": 0, "memory_to_active": 0.7337958812713623, "contribution_to_active": 2.1038379669189453, "mix_mean": 0.5601049661636353, "mix_low": 0.10209465026855469, "mix_high": 0.005906105041503906, "read_cosine": 0.08110465109348297, "beta_mean": 0.2019391655921936, "nonfinite": 0.0, "weighted_memory_to_active": 0.42700162529945374, "mix_std": 0.39137595891952515, "beta_std": 0.20857925713062286, "beta_low": 0.14093017578125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.5624127388000488, "evidence_weights": [-0.08454278856515884, -0.36753353476524353, -1.882272720336914], "agreement_weight": 0.02866860292851925, "orthogonal_weight": 3.68410587310791}, {"head": 1, "memory_to_active": 0.009559798054397106, "contribution_to_active": 0.6287983059883118, "mix_mean": 0.6375792026519775, "mix_low": 0.00176239013671875, "mix_high": 0.016571044921875, "read_cosine": 0.3092942237854004, "beta_mean": 6.65021434542723e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005675074644386768, "mix_std": 0.2627989947795868, "beta_std": 0.00012671978038270026, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00883249007165432, "evidence_weights": [-0.24936546385288239, -0.19337239861488342, 0.5516878962516785], "agreement_weight": 0.6411445140838623, "orthogonal_weight": -1.6940008401870728}, {"head": 2, "memory_to_active": 0.006588434800505638, "contribution_to_active": 0.623946487903595, "mix_mean": 0.6679348945617676, "mix_low": 0.0003299713134765625, "mix_high": 0.005553245544433594, "read_cosine": 0.05013173446059227, "beta_mean": 1.6051864804467186e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031067226082086563, "mix_std": 0.2493128925561905, "beta_std": 5.1547729526646435e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013275579549372196, "evidence_weights": [-0.268222451210022, -0.2786807119846344, 0.6471062302589417], "agreement_weight": -0.21495342254638672, "orthogonal_weight": -0.4158015549182892}, {"head": 3, "memory_to_active": 0.3598860502243042, "contribution_to_active": 0.10073323547840118, "mix_mean": 0.27793675661087036, "mix_low": 0.027709007263183594, "mix_high": 8.678436279296875e-05, "read_cosine": 0.9192591905593872, "beta_mean": 0.07217001914978027, "nonfinite": 0.0, "weighted_memory_to_active": 0.04019477218389511, "mix_std": 0.22541570663452148, "beta_std": 0.06223774701356888, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.04301914572715759, "evidence_weights": [-0.09225446730852127, -0.1439371556043625, 0.16037501394748688], "agreement_weight": 0.01638227142393589, "orthogonal_weight": 1.4555037021636963}, {"head": 4, "memory_to_active": 0.011577443219721317, "contribution_to_active": 0.5643161535263062, "mix_mean": 0.5934118032455444, "mix_low": 0.00020313262939453125, "mix_high": 0.0014171600341796875, "read_cosine": 0.0730978474020958, "beta_mean": 3.41630911862012e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006830950733274221, "mix_std": 0.24329237639904022, "beta_std": 6.265449337661266e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009913005866110325, "evidence_weights": [-0.18424753844738007, -0.25347232818603516, 0.9501271843910217], "agreement_weight": 0.17874738574028015, "orthogonal_weight": -1.5203533172607422}, {"head": 5, "memory_to_active": 0.13070929050445557, "contribution_to_active": 0.40704554319381714, "mix_mean": 0.4176671802997589, "mix_low": 0.0018968582153320312, "mix_high": 7.62939453125e-06, "read_cosine": 0.8422573804855347, "beta_mean": 0.0065622348338365555, "nonfinite": 0.0, "weighted_memory_to_active": 0.06007564812898636, "mix_std": 0.2274654358625412, "beta_std": 0.008225761353969574, "beta_low": 0.8175048828125, "beta_high": 0.0, "orthogonal_to_active": 0.02150498703122139, "evidence_weights": [-0.13903836905956268, -0.16501255333423615, 0.2720635235309601], "agreement_weight": 0.09555414319038391, "orthogonal_weight": -0.6689020395278931}]}, {"layer": 11, "memory_to_active": 0.07659584283828735, "contribution_to_active": 0.6283788681030273, "mix_mean": 0.528504490852356, "mix_low": 0.11276658624410629, "mix_high": 0.046089015901088715, "read_cosine": 0.34445837140083313, "beta_mean": 0.0229801032692194, "nonfinite": 0.0, "weighted_memory_to_active": 0.04173101484775543, "mix_std": 0.3238937258720398, "beta_std": 0.03459370136260986, "beta_low": 0.7158712148666382, "beta_high": 0.0, "orthogonal_to_active": 0.04497109353542328, "output_influence": 0.555604100227356, "residual_influence": 0.15539205074310303, "state_active": 0.20917224884033203, "state_memory": 0.003173734527081251, "state_peak": 38.12637710571289, "growth_active": 1.136579990386963, "growth_memory": 1.4632070064544678, "heads": [{"head": 0, "memory_to_active": 0.00854397937655449, "contribution_to_active": 0.6163410544395447, "mix_mean": 0.6625599265098572, "mix_low": 0.000682830810546875, "mix_high": 0.009118080139160156, "read_cosine": 0.2602504789829254, "beta_mean": 8.597661508247256e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006315990351140499, "mix_std": 0.25336676836013794, "beta_std": 0.003996752202510834, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.005241732113063335, "evidence_weights": [-0.17623533308506012, -0.35906746983528137, 0.8101910352706909], "agreement_weight": 0.3951815068721771, "orthogonal_weight": -0.8533871173858643}, {"head": 1, "memory_to_active": 0.11550003290176392, "contribution_to_active": 0.43781882524490356, "mix_mean": 0.4727077782154083, "mix_low": 0.0044956207275390625, "mix_high": 0.000728607177734375, "read_cosine": 0.5027516484260559, "beta_mean": 0.002684080507606268, "nonfinite": 0.0, "weighted_memory_to_active": 0.056337036192417145, "mix_std": 0.2714201807975769, "beta_std": 0.010165885090827942, "beta_low": 0.951416015625, "beta_high": 0.0, "orthogonal_to_active": 0.030874688178300858, "evidence_weights": [-0.1426878273487091, -0.2763380706310272, 0.5919291377067566], "agreement_weight": 0.08556239306926727, "orthogonal_weight": -0.6536120772361755}, {"head": 2, "memory_to_active": 0.004924236796796322, "contribution_to_active": 0.9867223501205444, "mix_mean": 0.43638110160827637, "mix_low": 0.3204536437988281, "mix_high": 0.17352581024169922, "read_cosine": 0.17683616280555725, "beta_mean": 0.05974871665239334, "nonfinite": 0.0, "weighted_memory_to_active": 0.003631750587373972, "mix_std": 0.45096588134765625, "beta_std": 0.08284605294466019, "beta_low": 0.499755859375, "beta_high": 0.0, "orthogonal_to_active": 0.036771032959222794, "evidence_weights": [0.04555056616663933, -0.3437917232513428, -1.3152457475662231], "agreement_weight": 0.004448060877621174, "orthogonal_weight": 10.548952102661133}, {"head": 3, "memory_to_active": 0.007661998737603426, "contribution_to_active": 0.8550885915756226, "mix_mean": 0.4118800461292267, "mix_low": 0.3439493179321289, "mix_high": 0.0853738784790039, "read_cosine": 0.5641828775405884, "beta_mean": 0.05048133432865143, "nonfinite": 0.0, "weighted_memory_to_active": 0.004755283705890179, "mix_std": 0.4402947425842285, "beta_std": 0.07721036672592163, "beta_low": 0.53106689453125, "beta_high": 0.0, "orthogonal_to_active": 0.035344768315553665, "evidence_weights": [0.21445126831531525, -0.2787495255470276, -0.1054236963391304], "agreement_weight": 0.05415818840265274, "orthogonal_weight": 10.816637992858887}, {"head": 4, "memory_to_active": 0.22864851355552673, "contribution_to_active": 0.4573926031589508, "mix_mean": 0.6149835586547852, "mix_low": 0.005835533142089844, "mix_high": 0.0019159317016601562, "read_cosine": 0.0008267676457762718, "beta_mean": 0.016738586127758026, "nonfinite": 0.0, "weighted_memory_to_active": 0.1317921280860901, "mix_std": 0.26959046721458435, "beta_std": 0.021513111889362335, "beta_low": 0.51959228515625, "beta_high": 0.0, "orthogonal_to_active": 0.12977513670921326, "evidence_weights": [-0.23551711440086365, -0.24283546209335327, 1.8877569437026978], "agreement_weight": -0.21746346354484558, "orthogonal_weight": -0.9977057576179504}, {"head": 5, "memory_to_active": 0.09429628401994705, "contribution_to_active": 0.41690945625305176, "mix_mean": 0.5725140571594238, "mix_low": 0.00118255615234375, "mix_high": 0.005871772766113281, "read_cosine": 0.5619022250175476, "beta_mean": 0.008141914382576942, "nonfinite": 0.0, "weighted_memory_to_active": 0.047553908079862595, "mix_std": 0.2577243447303772, "beta_std": 0.01183001883327961, "beta_low": 0.79376220703125, "beta_high": 0.0, "orthogonal_to_active": 0.03181919828057289, "evidence_weights": [-0.20100529491901398, -0.3133865296840668, 0.6657106280326843], "agreement_weight": 0.0833844542503357, "orthogonal_weight": -0.8730564117431641}]}, {"layer": 12, "memory_to_active": 0.20254968106746674, "contribution_to_active": 0.5810003280639648, "mix_mean": 0.5486633777618408, "mix_low": 0.009043535217642784, "mix_high": 0.007079442963004112, "read_cosine": 0.22842872142791748, "beta_mean": 0.01637822575867176, "nonfinite": 0.0, "weighted_memory_to_active": 0.09999749064445496, "mix_std": 0.25860363245010376, "beta_std": 0.015224207192659378, "beta_low": 0.8280843496322632, "beta_high": 0.0, "orthogonal_to_active": 0.06363134831190109, "output_influence": 0.5145044326782227, "residual_influence": 0.15448960661888123, "state_active": 0.035119470208883286, "state_memory": 0.008087921887636185, "state_peak": 76.46646881103516, "growth_active": 1.174261212348938, "growth_memory": 1.146138310432434, "heads": [{"head": 0, "memory_to_active": 0.05190231651067734, "contribution_to_active": 0.6255691051483154, "mix_mean": 0.5921721458435059, "mix_low": 0.0006093978881835938, "mix_high": 0.006125450134277344, "read_cosine": 0.2735435366630554, "beta_mean": 0.000860026222653687, "nonfinite": 0.0, "weighted_memory_to_active": 0.029585890471935272, "mix_std": 0.26364409923553467, "beta_std": 0.0018939671572297812, "beta_low": 0.99066162109375, "beta_high": 0.0, "orthogonal_to_active": 0.027449579909443855, "evidence_weights": [-0.2061738222837448, -0.2753373384475708, -0.04306754097342491], "agreement_weight": 0.1044207215309143, "orthogonal_weight": -0.9926666021347046}, {"head": 1, "memory_to_active": 0.07511387765407562, "contribution_to_active": 0.5690169334411621, "mix_mean": 0.5704378485679626, "mix_low": 0.0002899169921875, "mix_high": 0.00307464599609375, "read_cosine": 0.4148988723754883, "beta_mean": 0.0005034533096477389, "nonfinite": 0.0, "weighted_memory_to_active": 0.046873148530721664, "mix_std": 0.26564204692840576, "beta_std": 0.00835423730313778, "beta_low": 0.99749755859375, "beta_high": 0.0, "orthogonal_to_active": 0.03884530067443848, "evidence_weights": [-0.20119309425354004, -0.2730500102043152, 1.040200114250183], "agreement_weight": 0.269325852394104, "orthogonal_weight": -0.9207632541656494}, {"head": 2, "memory_to_active": 0.6022072434425354, "contribution_to_active": 0.46389976143836975, "mix_mean": 0.3073229491710663, "mix_low": 0.028703689575195312, "mix_high": 9.34600830078125e-05, "read_cosine": 0.5720584392547607, "beta_mean": 0.09585443139076233, "nonfinite": 0.0, "weighted_memory_to_active": 0.22228991985321045, "mix_std": 0.23125402629375458, "beta_std": 0.0674021989107132, "beta_low": 0.00067138671875, "beta_high": 0.0, "orthogonal_to_active": 0.2478877305984497, "evidence_weights": [-0.20563986897468567, -0.0616028867661953, -0.03070729598402977], "agreement_weight": 0.03106352686882019, "orthogonal_weight": 1.4227787256240845}, {"head": 3, "memory_to_active": 0.022031884640455246, "contribution_to_active": 0.7509562373161316, "mix_mean": 0.7231919765472412, "mix_low": 0.0001392364501953125, "mix_high": 0.021803855895996094, "read_cosine": 0.07879994064569473, "beta_mean": 0.00019379411241970956, "nonfinite": 0.0, "weighted_memory_to_active": 0.016268350183963776, "mix_std": 0.2268347293138504, "beta_std": 0.004387590568512678, "beta_low": 0.99871826171875, "beta_high": 0.0, "orthogonal_to_active": 0.015329409390687943, "evidence_weights": [-0.2882728576660156, -0.3346579372882843, 1.6489474773406982], "agreement_weight": 0.6038951277732849, "orthogonal_weight": -0.9629191160202026}, {"head": 4, "memory_to_active": 0.40676403045654297, "contribution_to_active": 0.5731355547904968, "mix_mean": 0.4610532522201538, "mix_low": 0.017653465270996094, "mix_high": 0.005236625671386719, "read_cosine": -0.19924324750900269, "beta_mean": 0.0005475500365719199, "nonfinite": 0.0, "weighted_memory_to_active": 0.2476537525653839, "mix_std": 0.2911366820335388, "beta_std": 0.006638246588408947, "beta_low": 0.98736572265625, "beta_high": 0.0, "orthogonal_to_active": 0.02425108663737774, "evidence_weights": [-0.15574271976947784, -0.21427883207798004, -0.1474762260913849], "agreement_weight": 0.015783026814460754, "orthogonal_weight": -0.10193883627653122}, {"head": 5, "memory_to_active": 0.05727864056825638, "contribution_to_active": 0.5034241080284119, "mix_mean": 0.637802004814148, "mix_low": 0.006865501403808594, "mix_high": 0.006142616271972656, "read_cosine": 0.23051482439041138, "beta_mean": 0.00031009301892481744, "nonfinite": 0.0, "weighted_memory_to_active": 0.03731392323970795, "mix_std": 0.27311021089553833, "beta_std": 0.0026689968071877956, "beta_low": 0.99359130859375, "beta_high": 0.0, "orthogonal_to_active": 0.028024980798363686, "evidence_weights": [-0.22408175468444824, -0.31575343012809753, 1.2216590642929077], "agreement_weight": 0.42591243982315063, "orthogonal_weight": -0.8108371496200562}]}, {"layer": 13, "memory_to_active": 0.11355537176132202, "contribution_to_active": 0.5376513004302979, "mix_mean": 0.5395565629005432, "mix_low": 0.006056944839656353, "mix_high": 0.00950988195836544, "read_cosine": 0.42210763692855835, "beta_mean": 0.014603691175580025, "nonfinite": 0.0, "weighted_memory_to_active": 0.04774145781993866, "mix_std": 0.25791794061660767, "beta_std": 0.015370087698101997, "beta_low": 0.80792236328125, "beta_high": 0.0, "orthogonal_to_active": 0.04735809937119484, "output_influence": 0.520817756652832, "residual_influence": 0.1876937448978424, "state_active": 0.13373889029026031, "state_memory": 0.0437486432492733, "state_peak": 101.3252182006836, "growth_active": 0.9159592390060425, "growth_memory": 0.9875569343566895, "heads": [{"head": 0, "memory_to_active": 0.009021095931529999, "contribution_to_active": 0.6040971279144287, "mix_mean": 0.616279661655426, "mix_low": 0.0007429122924804688, "mix_high": 0.004590034484863281, "read_cosine": 0.42294377088546753, "beta_mean": 3.225990440114401e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005854843650013208, "mix_std": 0.26681196689605713, "beta_std": 6.712188042001799e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00556913809850812, "evidence_weights": [-0.19062724709510803, -0.26525405049324036, 0.7253007888793945], "agreement_weight": 0.21208101511001587, "orthogonal_weight": -1.05818772315979}, {"head": 1, "memory_to_active": 0.009459140710532665, "contribution_to_active": 0.5998733639717102, "mix_mean": 0.5701902508735657, "mix_low": 0.0016727447509765625, "mix_high": 0.00478363037109375, "read_cosine": 0.2227848768234253, "beta_mean": 3.5324632335687056e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005283563397824764, "mix_std": 0.276824951171875, "beta_std": 7.126265700208023e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006779615301638842, "evidence_weights": [-0.17861643433570862, -0.2615785002708435, 0.5423934459686279], "agreement_weight": 0.5516789555549622, "orthogonal_weight": -1.3347539901733398}, {"head": 2, "memory_to_active": 0.06638489663600922, "contribution_to_active": 0.4964776337146759, "mix_mean": 0.39784058928489685, "mix_low": 0.0009851455688476562, "mix_high": 0.000156402587890625, "read_cosine": 0.6811122894287109, "beta_mean": 0.008381156250834465, "nonfinite": 0.0, "weighted_memory_to_active": 0.032201461493968964, "mix_std": 0.2315186858177185, "beta_std": 0.028363768011331558, "beta_low": 0.8533935546875, "beta_high": 0.0, "orthogonal_to_active": 0.015964936465024948, "evidence_weights": [-0.17922301590442657, -0.21170493960380554, 0.4166492223739624], "agreement_weight": 0.007085059769451618, "orthogonal_weight": -0.8475747108459473}, {"head": 3, "memory_to_active": 0.15528781712055206, "contribution_to_active": 0.5813862681388855, "mix_mean": 0.661984920501709, "mix_low": 0.024835586547851562, "mix_high": 0.03304767608642578, "read_cosine": -0.04080018773674965, "beta_mean": 0.0009190606651827693, "nonfinite": 0.0, "weighted_memory_to_active": 0.11232781410217285, "mix_std": 0.2895697355270386, "beta_std": 0.008797344751656055, "beta_low": 0.99432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.10138027369976044, "evidence_weights": [-0.31401702761650085, -0.23999211192131042, 1.4984973669052124], "agreement_weight": 6.648873329162598, "orthogonal_weight": -0.9743776321411133}, {"head": 4, "memory_to_active": 0.38610032200813293, "contribution_to_active": 0.22589339315891266, "mix_mean": 0.32692626118659973, "mix_low": 0.00580596923828125, "mix_high": 0.0, "read_cosine": 0.9050334692001343, "beta_mean": 0.07800756394863129, "nonfinite": 0.0, "weighted_memory_to_active": 0.09177624434232712, "mix_std": 0.21152044832706451, "beta_std": 0.052759505808353424, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.12204812467098236, "evidence_weights": [-0.06569802761077881, -0.22535552084445953, 0.42650341987609863], "agreement_weight": 0.011938652023673058, "orthogonal_weight": 2.2894904613494873}, {"head": 5, "memory_to_active": 0.05507895350456238, "contribution_to_active": 0.7181798815727234, "mix_mean": 0.6641175746917725, "mix_low": 0.0022993087768554688, "mix_high": 0.014481544494628906, "read_cosine": 0.34157150983810425, "beta_mean": 0.00024677751935087144, "nonfinite": 0.0, "weighted_memory_to_active": 0.039004795253276825, "mix_std": 0.2712618112564087, "beta_std": 0.0021615191362798214, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.032406486570835114, "evidence_weights": [-0.22249116003513336, -0.2917279899120331, 0.864645779132843], "agreement_weight": 0.20681722462177277, "orthogonal_weight": -0.9235599040985107}]}, {"layer": 14, "memory_to_active": 0.21905672550201416, "contribution_to_active": 0.41408565640449524, "mix_mean": 0.4219359755516052, "mix_low": 0.030768396332859993, "mix_high": 0.006530126091092825, "read_cosine": 0.6174415946006775, "beta_mean": 0.0607161782681942, "nonfinite": 0.0, "weighted_memory_to_active": 0.08942976593971252, "mix_std": 0.23821553587913513, "beta_std": 0.05944376811385155, "beta_low": 0.5019022822380066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.03875184804201126, "output_influence": 0.35283318161964417, "residual_influence": 0.14808863401412964, "state_active": 0.05306272208690643, "state_memory": 0.01444415096193552, "state_peak": 49.18335723876953, "growth_active": 1.073471188545227, "growth_memory": 1.2571516036987305, "heads": [{"head": 0, "memory_to_active": 0.35074543952941895, "contribution_to_active": 0.17987994849681854, "mix_mean": 0.17527355253696442, "mix_low": 0.021538734436035156, "mix_high": 5.7220458984375e-06, "read_cosine": 0.8209013342857361, "beta_mean": 0.0689762681722641, "nonfinite": 0.0, "weighted_memory_to_active": 0.0713496059179306, "mix_std": 0.16683274507522583, "beta_std": 0.058375924825668335, "beta_low": 0.00469970703125, "beta_high": 0.0, "orthogonal_to_active": 0.07838325947523117, "evidence_weights": [-0.09762078523635864, -0.04760253056883812, 0.3775402009487152], "agreement_weight": 0.022082902491092682, "orthogonal_weight": 1.8746269941329956}, {"head": 1, "memory_to_active": 0.4726828932762146, "contribution_to_active": 0.2628934383392334, "mix_mean": 0.18623203039169312, "mix_low": 0.148529052734375, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8750908374786377, "beta_mean": 0.25126299262046814, "nonfinite": 0.0, "weighted_memory_to_active": 0.2030813992023468, "mix_std": 0.22300617396831512, "beta_std": 0.2519478499889374, "beta_low": 0.10394287109375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.018308572471141815, "evidence_weights": [0.0087428018450737, -0.1903601884841919, -0.09780760109424591], "agreement_weight": 0.05902719125151634, "orthogonal_weight": 0.2782019376754761}, {"head": 2, "memory_to_active": 0.11445295810699463, "contribution_to_active": 0.581471860408783, "mix_mean": 0.5871458053588867, "mix_low": 0.0027256011962890625, "mix_high": 0.005228996276855469, "read_cosine": 0.41178596019744873, "beta_mean": 0.001669352757744491, "nonfinite": 0.0, "weighted_memory_to_active": 0.07399414479732513, "mix_std": 0.26895982027053833, "beta_std": 0.0023223282769322395, "beta_low": 0.98486328125, "beta_high": 0.0, "orthogonal_to_active": 0.056697167456150055, "evidence_weights": [-0.13129064440727234, -0.31654199957847595, 0.9177247285842896], "agreement_weight": 0.09239614754915237, "orthogonal_weight": -0.8733202219009399}, {"head": 3, "memory_to_active": 0.0439850315451622, "contribution_to_active": 0.5988481640815735, "mix_mean": 0.6791030764579773, "mix_low": 0.004436492919921875, "mix_high": 0.02428722381591797, "read_cosine": -0.0667596161365509, "beta_mean": 0.006097039207816124, "nonfinite": 0.0, "weighted_memory_to_active": 0.027291961014270782, "mix_std": 0.25588980317115784, "beta_std": 0.009242808446288109, "beta_low": 0.8223876953125, "beta_high": 0.0, "orthogonal_to_active": 0.020454570651054382, "evidence_weights": [-0.3422318398952484, -0.3131052851676941, 0.3028485178947449], "agreement_weight": 0.06904041767120361, "orthogonal_weight": -0.8239356279373169}, {"head": 4, "memory_to_active": 0.1424735188484192, "contribution_to_active": 0.5103369951248169, "mix_mean": 0.520571768283844, "mix_low": 0.003620147705078125, "mix_high": 0.008337974548339844, "read_cosine": 0.9099968671798706, "beta_mean": 0.0349784716963768, "nonfinite": 0.0, "weighted_memory_to_active": 0.08406303822994232, "mix_std": 0.2605207860469818, "beta_std": 0.0265848059207201, "beta_low": 0.116455078125, "beta_high": 0.0, "orthogonal_to_active": 0.020899083465337753, "evidence_weights": [-0.2053826004266739, -0.23517338931560516, 0.43740350008010864], "agreement_weight": 0.1326439082622528, "orthogonal_weight": -0.6201611757278442}, {"head": 5, "memory_to_active": 0.19000044465065002, "contribution_to_active": 0.3510834276676178, "mix_mean": 0.3832895755767822, "mix_low": 0.0037603378295898438, "mix_high": 0.0013189315795898438, "read_cosine": 0.7536341547966003, "beta_mean": 0.0013129569124430418, "nonfinite": 0.0, "weighted_memory_to_active": 0.07679842412471771, "mix_std": 0.25408387184143066, "beta_std": 0.00818889681249857, "beta_low": 0.97906494140625, "beta_high": 0.0, "orthogonal_to_active": 0.03776843100786209, "evidence_weights": [-0.17411035299301147, -0.1831250637769699, 0.36265695095062256], "agreement_weight": 0.06651698797941208, "orthogonal_weight": -0.759238064289093}]}, {"layer": 15, "memory_to_active": 0.05321302264928818, "contribution_to_active": 0.4975781738758087, "mix_mean": 0.4330444931983948, "mix_low": 0.006838798522949219, "mix_high": 0.0030725798569619656, "read_cosine": 0.6580204963684082, "beta_mean": 0.04198521748185158, "nonfinite": 0.0, "weighted_memory_to_active": 0.027068622410297394, "mix_std": 0.283139169216156, "beta_std": 0.04705614224076271, "beta_low": 0.7005513906478882, "beta_high": 0.0, "orthogonal_to_active": 0.009659375995397568, "output_influence": 0.3167453408241272, "residual_influence": 0.1860506236553192, "state_active": 0.15913130342960358, "state_memory": 0.01051234733313322, "state_peak": 26.758455276489258, "growth_active": 1.1359014511108398, "growth_memory": 1.2407704591751099, "heads": [{"head": 0, "memory_to_active": 0.1089051142334938, "contribution_to_active": 0.4826790988445282, "mix_mean": 0.40201863646507263, "mix_low": 0.006833076477050781, "mix_high": 0.0065250396728515625, "read_cosine": 0.5298706293106079, "beta_mean": 0.04485701024532318, "nonfinite": 0.0, "weighted_memory_to_active": 0.05474940687417984, "mix_std": 0.28580009937286377, "beta_std": 0.03133339434862137, "beta_low": 0.05328369140625, "beta_high": 0.0, "orthogonal_to_active": 0.03385430574417114, "evidence_weights": [-0.18860173225402832, -0.1956932246685028, 0.5107269883155823], "agreement_weight": 0.0897066667675972, "orthogonal_weight": -0.8198006749153137}, {"head": 1, "memory_to_active": 0.10415444523096085, "contribution_to_active": 0.43146324157714844, "mix_mean": 0.4445115327835083, "mix_low": 0.0010013580322265625, "mix_high": 0.006778717041015625, "read_cosine": 0.9331413507461548, "beta_mean": 0.00026770844124257565, "nonfinite": 0.0, "weighted_memory_to_active": 0.045230913907289505, "mix_std": 0.28482627868652344, "beta_std": 0.0015022569568827748, "beta_low": 0.99859619140625, "beta_high": 0.0, "orthogonal_to_active": 0.002975492039695382, "evidence_weights": [-0.15569688379764557, -0.16638870537281036, 0.16277854144573212], "agreement_weight": 0.11715127527713776, "orthogonal_weight": 0.14575161039829254}, {"head": 2, "memory_to_active": 0.003941122908145189, "contribution_to_active": 0.5189961791038513, "mix_mean": 0.36456650495529175, "mix_low": 0.009439468383789062, "mix_high": 0.00013065338134765625, "read_cosine": 0.7787861227989197, "beta_mean": 5.055154542787932e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00220999657176435, "mix_std": 0.27371615171432495, "beta_std": 0.00015432998770847917, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007036954630166292, "evidence_weights": [-0.20489142835140228, -0.2685999572277069, 0.11123090982437134], "agreement_weight": 0.008079533465206623, "orthogonal_weight": 0.8912749290466309}, {"head": 3, "memory_to_active": 0.0020746560767292976, "contribution_to_active": 0.37409913539886475, "mix_mean": 0.41859835386276245, "mix_low": 0.0040683746337890625, "mix_high": 0.00054931640625, "read_cosine": 0.6309424638748169, "beta_mean": 1.1066665138059761e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0006685384432785213, "mix_std": 0.2879585027694702, "beta_std": 2.540830973885022e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0004438284086063504, "evidence_weights": [-0.2777459919452667, -0.23926624655723572, 0.17856886982917786], "agreement_weight": 0.05382414162158966, "orthogonal_weight": -0.6282879710197449}, {"head": 4, "memory_to_active": 0.06784119457006454, "contribution_to_active": 0.6429013609886169, "mix_mean": 0.38090139627456665, "mix_low": 0.009599685668945312, "mix_high": 0.0010099411010742188, "read_cosine": 0.733472466468811, "beta_mean": 0.202879399061203, "nonfinite": 0.0, "weighted_memory_to_active": 0.037642769515514374, "mix_std": 0.302706778049469, "beta_std": 0.2218344509601593, "beta_low": 0.18646240234375, "beta_high": 0.0, "orthogonal_to_active": 0.006459748838096857, "evidence_weights": [-0.20831269025802612, -0.254527747631073, -0.031642261892557144], "agreement_weight": 0.013011196628212929, "orthogonal_weight": 0.3863269090652466}, {"head": 5, "memory_to_active": 0.032361604273319244, "contribution_to_active": 0.5353301167488098, "mix_mean": 0.5876705050468445, "mix_low": 0.010090827941894531, "mix_high": 0.0034418106079101562, "read_cosine": 0.3419097065925598, "beta_mean": 0.0038455831818282604, "nonfinite": 0.0, "weighted_memory_to_active": 0.02191009558737278, "mix_std": 0.26382705569267273, "beta_std": 0.027486981824040413, "beta_low": 0.9649658203125, "beta_high": 0.0, "orthogonal_to_active": 0.01351918838918209, "evidence_weights": [-0.2807714343070984, -0.3041999638080597, 0.6243516802787781], "agreement_weight": 0.31695839762687683, "orthogonal_weight": 0.8192790150642395}]}]}, {"step": 70000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8151221770021948, "memory_to_active": 0.7911644577980042, "contribution_to_active": 0.5729438066482544, "mix_mean": 0.42698705196380615, "mix_low": 0.12256279587745667, "mix_high": 0.01063747238367796, "read_cosine": 0.3437933921813965, "beta_mean": 0.06761674582958221, "nonfinite": 0.0, "weighted_memory_to_active": 0.19794367253780365, "mix_std": 0.2231554090976715, "beta_std": 0.0597090981900692, "beta_low": 0.5923646688461304, "beta_high": 3.814697265625e-05, "orthogonal_to_active": 0.06318022310733795, "samples": 4, "scored_tokens": 899, "normal_loss": 3.095353603363037, "normal_token_accuracy": 0.4482758641242981, "state_active": 0.06618759781122208, "state_memory": 0.01671799272298813, "state_peak": 129.48069763183594, "growth_active": 1.1383936405181885, "growth_memory": 1.2944563627243042, "output_influence": 0.4449945092201233, "residual_influence": 0.15979567170143127, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.217402935028076, "no_memory_loss_delta": 0.6682246923446655, "scrambled_loss_delta": 0.1077936440706253, "memory_only_loss_delta": 3.6457455158233643, "no_evidence_loss_delta": 0.3862261176109314, "no_agreement_loss_delta": 0.012522841803729534, "no_orthogonal_loss_delta": 0.006975142285227776}, {"index": 94, "tokens": 256, "normal_loss": 3.1853742599487305, "no_memory_loss_delta": 0.8054102659225464, "scrambled_loss_delta": 0.3572726547718048, "memory_only_loss_delta": 4.100155353546143, "no_evidence_loss_delta": 0.3481237590312958, "no_agreement_loss_delta": 0.032319240272045135, "no_orthogonal_loss_delta": 0.07170096039772034}, {"index": 188, "tokens": 256, "normal_loss": 3.3913028240203857, "no_memory_loss_delta": 1.0703065395355225, "scrambled_loss_delta": 0.11186329275369644, "memory_only_loss_delta": 3.109992265701294, "no_evidence_loss_delta": 0.5770169496536255, "no_agreement_loss_delta": 0.04991453140974045, "no_orthogonal_loss_delta": 0.0573195219039917}, {"index": 283, "tokens": 131, "normal_loss": 2.102583646774292, "no_memory_loss_delta": 0.20158334076404572, "scrambled_loss_delta": 0.03104201890528202, "memory_only_loss_delta": 2.736236572265625, "no_evidence_loss_delta": -0.021200798451900482, "no_agreement_loss_delta": -0.03406688943505287, "no_orthogonal_loss_delta": -0.04819680005311966}], "agreement_weight_abs_mean": 0.24600477516651154, "agreement_weight_abs_max": 6.698206424713135, "agreement_weight_mean": 0.2147640436887741, "orthogonal_weight_abs_mean": 0.9375306963920593, "orthogonal_weight_abs_max": 10.91498851776123, "orthogonal_weight_mean": -0.05913693830370903, "no_memory_loss": 3.8491427898406982, "no_memory_loss_delta": 0.7537891268730164, "no_memory_token_accuracy_gain": 0.10567297041416168, "no_memory_token_win_fraction": 0.6840934157371521, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 52.94199173395614, "scrambled_loss": 3.2641639709472656, "scrambled_loss_delta": 0.16881033778190613, "scrambled_token_accuracy_gain": 0.032258063554763794, "scrambled_token_win_fraction": 0.5906562805175781, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 15.533090999195462, "memory_only_loss": 6.585405349731445, "memory_only_loss_delta": 3.490051031112671, "memory_only_token_accuracy_gain": 0.2925472855567932, "memory_only_token_win_fraction": 0.9110122323036194, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.95006842825127, "no_evidence_loss": 3.4656901359558105, "no_evidence_loss_delta": 0.3703365921974182, "no_evidence_token_accuracy_gain": 0.045606229454278946, "no_evidence_token_win_fraction": 0.6373748779296875, "no_evidence_probe_win_fraction": 0.75, "no_evidence_ppl_reduction_pct": 30.94981260250827, "no_agreement_loss": 3.117372512817383, "no_agreement_loss_delta": 0.022018833085894585, "no_agreement_token_accuracy_gain": 0.0055617354810237885, "no_agreement_token_win_fraction": 0.4749721884727478, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.1778188057679673, "no_orthogonal_loss": 3.127056837081909, "no_orthogonal_loss_delta": 0.03170311450958252, "no_orthogonal_token_accuracy_gain": 0.006674082484096289, "no_orthogonal_token_win_fraction": 0.5973303914070129, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 3.120583968241839, "short_no_memory_loss_delta": 0.6757513284683228, "long_context_memory_gain": 0.0780377984046936, "normal_context_loss_gain": 0.12804600596427917, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.5862436294555664, "contribution_to_active": 1.2077929973602295, "mix_mean": 0.10871199518442154, "mix_low": 0.5070109963417053, "mix_high": 0.005216916557401419, "read_cosine": 0.4181053340435028, "beta_mean": 0.21015115082263947, "nonfinite": 0.0, "weighted_memory_to_active": 1.2324665784835815, "mix_std": 0.09271681308746338, "beta_std": 0.14208245277404785, "beta_low": 0.1722005307674408, "beta_high": 0.0, "orthogonal_to_active": 0.05222953110933304, "output_influence": 0.2112588733434677, "residual_influence": 0.23550689220428467, "state_active": 0.06646277010440826, "state_memory": 0.04572176933288574, "state_peak": 46.060691833496094, "growth_active": 1.2636054754257202, "growth_memory": 1.2708250284194946, "heads": [{"head": 0, "memory_to_active": 2.846147298812866, "contribution_to_active": 0.49548161029815674, "mix_mean": 0.0206315740942955, "mix_low": 0.6023750305175781, "mix_high": 0.0, "read_cosine": 0.21227730810642242, "beta_mean": 0.1911439746618271, "nonfinite": 0.0, "weighted_memory_to_active": 0.6172649264335632, "mix_std": 0.060144368559122086, "beta_std": 0.09347984939813614, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0655067041516304, "evidence_weights": [-0.004208069294691086, -0.042820822447538376, -0.05944383516907692], "agreement_weight": -0.22856156527996063, "orthogonal_weight": -0.12131845951080322}, {"head": 1, "memory_to_active": 6.758627891540527, "contribution_to_active": 2.9073243141174316, "mix_mean": 0.01983744651079178, "mix_low": 0.6221561431884766, "mix_high": 0.0, "read_cosine": 0.46866118907928467, "beta_mean": 0.45785200595855713, "nonfinite": 0.0, "weighted_memory_to_active": 3.145071506500244, "mix_std": 0.056763723492622375, "beta_std": 0.22604766488075256, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06763184815645218, "evidence_weights": [0.021051518619060516, -0.10629674792289734, 0.6741438508033752], "agreement_weight": 0.028583411127328873, "orthogonal_weight": 0.048774104565382004}, {"head": 2, "memory_to_active": 2.817411422729492, "contribution_to_active": 0.5715048909187317, "mix_mean": 0.015329851768910885, "mix_low": 0.6380014419555664, "mix_high": 0.0, "read_cosine": 0.2945871353149414, "beta_mean": 0.2578177750110626, "nonfinite": 0.0, "weighted_memory_to_active": 0.6658538579940796, "mix_std": 0.03193037211894989, "beta_std": 0.20947876572608948, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05747213959693909, "evidence_weights": [0.0590425468981266, 0.0459638275206089, 0.25783205032348633], "agreement_weight": 0.008096452802419662, "orthogonal_weight": -0.2105942666530609}, {"head": 3, "memory_to_active": 7.616055488586426, "contribution_to_active": 2.728616714477539, "mix_mean": 0.02960762195289135, "mix_low": 0.5002412796020508, "mix_high": 0.0, "read_cosine": 0.28929638862609863, "beta_mean": 0.31074070930480957, "nonfinite": 0.0, "weighted_memory_to_active": 2.868495464324951, "mix_std": 0.06879699975252151, "beta_std": 0.2550049126148224, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.10611473768949509, "evidence_weights": [-0.03544218838214874, -0.028755122795701027, -0.1444253772497177], "agreement_weight": -0.0060392278246581554, "orthogonal_weight": -0.04385305941104889}, {"head": 4, "memory_to_active": 1.383314609527588, "contribution_to_active": 0.06934427469968796, "mix_mean": 0.019703686237335205, "mix_low": 0.6651678085327148, "mix_high": 0.0, "read_cosine": 0.3153797686100006, "beta_mean": 0.02504783123731613, "nonfinite": 0.0, "weighted_memory_to_active": 0.0474737212061882, "mix_std": 0.055001068860292435, "beta_std": 0.04900551587343216, "beta_low": 0.57232666015625, "beta_high": 0.0, "orthogonal_to_active": 0.0009296206990256906, "evidence_weights": [0.09700515121221542, 0.06123865023255348, -0.1282501518726349], "agreement_weight": -0.19127847254276276, "orthogonal_weight": 0.01719212345778942}, {"head": 5, "memory_to_active": 0.09590543806552887, "contribution_to_active": 0.4744855761528015, "mix_mean": 0.5471618175506592, "mix_low": 0.014123916625976562, "mix_high": 0.03130149841308594, "read_cosine": 0.9284300804138184, "beta_mean": 0.018304482102394104, "nonfinite": 0.0, "weighted_memory_to_active": 0.050639547407627106, "mix_std": 0.2836643159389496, "beta_std": 0.019478030502796173, "beta_low": 0.4608154296875, "beta_high": 0.0, "orthogonal_to_active": 0.015722118318080902, "evidence_weights": [-0.17714489996433258, -0.22057153284549713, 0.37762218713760376], "agreement_weight": 0.2877275049686432, "orthogonal_weight": -0.689527153968811}]}, {"layer": 1, "memory_to_active": 2.4600913524627686, "contribution_to_active": 0.48864585161209106, "mix_mean": 0.21814599633216858, "mix_low": 0.36781662702560425, "mix_high": 0.0012429555645212531, "read_cosine": 0.2974132001399994, "beta_mean": 0.17219571769237518, "nonfinite": 0.0, "weighted_memory_to_active": 0.3153514266014099, "mix_std": 0.11345899105072021, "beta_std": 0.12824603915214539, "beta_low": 0.3545735776424408, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.03816331923007965, "output_influence": 0.2767331600189209, "residual_influence": 0.07704202085733414, "state_active": 0.027979014441370964, "state_memory": 0.034626491367816925, "state_peak": 37.572898864746094, "growth_active": 1.1372876167297363, "growth_memory": 1.4604600667953491, "heads": [{"head": 0, "memory_to_active": 0.02830865979194641, "contribution_to_active": 0.5075492858886719, "mix_mean": 0.5472270250320435, "mix_low": 0.0041790008544921875, "mix_high": 0.00263214111328125, "read_cosine": 0.035467296838760376, "beta_mean": 0.0001706996699795127, "nonfinite": 0.0, "weighted_memory_to_active": 0.018147561699151993, "mix_std": 0.2794466018676758, "beta_std": 0.0004022308858111501, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.017299484461545944, "evidence_weights": [-0.21281367540359497, -0.274799108505249, 0.2417975217103958], "agreement_weight": 0.20227213203907013, "orthogonal_weight": -0.9922665953636169}, {"head": 1, "memory_to_active": 2.9337780475616455, "contribution_to_active": 0.2882438898086548, "mix_mean": 0.022280825302004814, "mix_low": 0.4728584289550781, "mix_high": 0.0, "read_cosine": 0.5314133763313293, "beta_mean": 0.26305490732192993, "nonfinite": 0.0, "weighted_memory_to_active": 0.3226649761199951, "mix_std": 0.0419059656560421, "beta_std": 0.19385753571987152, "beta_low": 0.00439453125, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.02239719219505787, "evidence_weights": [0.021172424778342247, -0.06531765311956406, 0.049116477370262146], "agreement_weight": 0.002014909405261278, "orthogonal_weight": 0.2120409905910492}, {"head": 2, "memory_to_active": 0.0371704176068306, "contribution_to_active": 0.5530937910079956, "mix_mean": 0.6792160272598267, "mix_low": 0.000396728515625, "mix_high": 0.00482177734375, "read_cosine": 0.4735279679298401, "beta_mean": 0.0013048566179350019, "nonfinite": 0.0, "weighted_memory_to_active": 0.024484632536768913, "mix_std": 0.22829118371009827, "beta_std": 0.003495696932077408, "beta_low": 0.9818115234375, "beta_high": 0.0, "orthogonal_to_active": 0.017306270077824593, "evidence_weights": [-0.2744344472885132, -0.3701312243938446, 0.6566700339317322], "agreement_weight": 0.044493064284324646, "orthogonal_weight": -0.8292482495307922}, {"head": 3, "memory_to_active": 2.9980292320251465, "contribution_to_active": 0.26675868034362793, "mix_mean": 0.023283617570996284, "mix_low": 0.5414152145385742, "mix_high": 9.5367431640625e-07, "read_cosine": 0.17244496941566467, "beta_mean": 0.3034358322620392, "nonfinite": 0.0, "weighted_memory_to_active": 0.26226216554641724, "mix_std": 0.060200802981853485, "beta_std": 0.2507878541946411, "beta_low": 0.06866455078125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.09623068571090698, "evidence_weights": [0.04217880591750145, -0.015283651649951935, -0.003401858266443014], "agreement_weight": -0.028067495673894882, "orthogonal_weight": -0.49282991886138916}, {"head": 4, "memory_to_active": 4.058297634124756, "contribution_to_active": 1.226938009262085, "mix_mean": 0.03053865395486355, "mix_low": 0.3369264602661133, "mix_high": 0.0, "read_cosine": 0.3309115171432495, "beta_mean": 0.1615470051765442, "nonfinite": 0.0, "weighted_memory_to_active": 1.204200267791748, "mix_std": 0.05077187716960907, "beta_std": 0.12377365678548813, "beta_low": 0.03985595703125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.048684000968933105, "evidence_weights": [-0.01592303439974785, -0.00402574660256505, 0.17591527104377747], "agreement_weight": -0.006407515145838261, "orthogonal_weight": 0.05937623977661133}, {"head": 5, "memory_to_active": 4.704963684082031, "contribution_to_active": 0.0892912968993187, "mix_mean": 0.006329897791147232, "mix_low": 0.8511238098144531, "mix_high": 2.86102294921875e-06, "read_cosine": 0.24071398377418518, "beta_mean": 0.3036609888076782, "nonfinite": 0.0, "weighted_memory_to_active": 0.060348931699991226, "mix_std": 0.020137496292591095, "beta_std": 0.19715917110443115, "beta_low": 0.032958984375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.02706228569149971, "evidence_weights": [0.1631649285554886, 0.08014696836471558, 0.11544278264045715], "agreement_weight": -0.10295537859201431, "orthogonal_weight": 0.41677263379096985}]}, {"layer": 2, "memory_to_active": 1.0534138679504395, "contribution_to_active": 0.3620797395706177, "mix_mean": 0.41028332710266113, "mix_low": 0.2427322268486023, "mix_high": 0.004393100738525391, "read_cosine": 0.30502521991729736, "beta_mean": 0.08310253173112869, "nonfinite": 0.0, "weighted_memory_to_active": 0.08129888772964478, "mix_std": 0.18570637702941895, "beta_std": 0.07171407341957092, "beta_low": 0.6084391474723816, "beta_high": 0.000274658203125, "orthogonal_to_active": 0.017302781343460083, "output_influence": 0.4052865505218506, "residual_influence": 0.11565136909484863, "state_active": 0.039957404136657715, "state_memory": 0.014148578979074955, "state_peak": 30.721553802490234, "growth_active": 1.0601905584335327, "growth_memory": 1.5493170022964478, "heads": [{"head": 0, "memory_to_active": 0.006565867457538843, "contribution_to_active": 0.33716151118278503, "mix_mean": 0.6436136364936829, "mix_low": 0.0006170272827148438, "mix_high": 0.008543014526367188, "read_cosine": 0.13411828875541687, "beta_mean": 5.0684146117419004e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004445030819624662, "mix_std": 0.26581597328186035, "beta_std": 0.0001170831237686798, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004762997385114431, "evidence_weights": [-0.37438955903053284, -0.21565569937229156, 0.23783856630325317], "agreement_weight": 0.14643612504005432, "orthogonal_weight": -1.105473279953003}, {"head": 1, "memory_to_active": 3.3204262256622314, "contribution_to_active": 0.04823045805096626, "mix_mean": 0.0052758026868104935, "mix_low": 0.8782491683959961, "mix_high": 0.0, "read_cosine": 0.1277754008769989, "beta_mean": 0.22900980710983276, "nonfinite": 0.0, "weighted_memory_to_active": 0.03979523107409477, "mix_std": 0.010857035405933857, "beta_std": 0.16373443603515625, "beta_low": 0.0050048828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.011148694902658463, "evidence_weights": [0.1413041651248932, 0.11348790675401688, 0.5665841698646545], "agreement_weight": 0.03315746411681175, "orthogonal_weight": 0.3243105709552765}, {"head": 2, "memory_to_active": 0.00851593166589737, "contribution_to_active": 0.42497363686561584, "mix_mean": 0.5509968400001526, "mix_low": 0.0011186599731445312, "mix_high": 0.0034580230712890625, "read_cosine": 0.3760371804237366, "beta_mean": 2.49158802034799e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003588750958442688, "mix_std": 0.2738458216190338, "beta_std": 4.6860368456691504e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001539236109238118, "evidence_weights": [-0.24831391870975494, -0.2916271984577179, 0.5859749913215637], "agreement_weight": -0.005988278426229954, "orthogonal_weight": 0.053179413080215454}, {"head": 3, "memory_to_active": 0.09863470494747162, "contribution_to_active": 0.5533946752548218, "mix_mean": 0.5838109254837036, "mix_low": 0.0003185272216796875, "mix_high": 0.0066852569580078125, "read_cosine": 0.61571204662323, "beta_mean": 0.016824249178171158, "nonfinite": 0.0, "weighted_memory_to_active": 0.06193024292588234, "mix_std": 0.25412923097610474, "beta_std": 0.026351090520620346, "beta_low": 0.588623046875, "beta_high": 0.0, "orthogonal_to_active": 0.041430726647377014, "evidence_weights": [-0.2021668553352356, -0.326348215341568, 0.7424793243408203], "agreement_weight": 0.05117259919643402, "orthogonal_weight": -0.8975561261177063}, {"head": 4, "memory_to_active": 2.8748464584350586, "contribution_to_active": 0.2880565822124481, "mix_mean": 0.020727146416902542, "mix_low": 0.5755128860473633, "mix_high": 0.0, "read_cosine": 0.19934314489364624, "beta_mean": 0.25266408920288086, "nonfinite": 0.0, "weighted_memory_to_active": 0.3716530501842499, "mix_std": 0.05080074071884155, "beta_std": 0.23998799920082092, "beta_low": 0.0570068359375, "beta_high": 0.0015869140625, "orthogonal_to_active": 0.04461085796356201, "evidence_weights": [0.02520636096596718, 0.0043296366930007935, 0.04182121902704239], "agreement_weight": 0.01865197904407978, "orthogonal_weight": -0.13741837441921234}, {"head": 5, "memory_to_active": 0.011494236998260021, "contribution_to_active": 0.520661473274231, "mix_mean": 0.6572755575180054, "mix_low": 0.0005769729614257812, "mix_high": 0.007672309875488281, "read_cosine": 0.37716516852378845, "beta_mean": 4.143285332247615e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00638099480420351, "mix_std": 0.25878942012786865, "beta_std": 4.6964665671112016e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001709482166916132, "evidence_weights": [-0.2792765200138092, -0.2890872657299042, 0.8770357370376587], "agreement_weight": 0.26703566312789917, "orthogonal_weight": -0.3235696852207184}]}, {"layer": 3, "memory_to_active": 0.43622878193855286, "contribution_to_active": 0.4875890016555786, "mix_mean": 0.5195202827453613, "mix_low": 0.05232683941721916, "mix_high": 0.01148223876953125, "read_cosine": 0.2058349847793579, "beta_mean": 0.02404310368001461, "nonfinite": 0.0, "weighted_memory_to_active": 0.11978088319301605, "mix_std": 0.2546914219856262, "beta_std": 0.023413322865962982, "beta_low": 0.7711182236671448, "beta_high": 0.0, "orthogonal_to_active": 0.05124695599079132, "output_influence": 0.43458622694015503, "residual_influence": 0.10743322968482971, "state_active": 0.023371919989585876, "state_memory": 0.006681371480226517, "state_peak": 24.351165771484375, "growth_active": 1.0785359144210815, "growth_memory": 1.623165249824524, "heads": [{"head": 0, "memory_to_active": 0.35666966438293457, "contribution_to_active": 0.6148247718811035, "mix_mean": 0.6123167276382446, "mix_low": 0.00122833251953125, "mix_high": 0.024262428283691406, "read_cosine": 0.2641163468360901, "beta_mean": 0.008975504897534847, "nonfinite": 0.0, "weighted_memory_to_active": 0.22794783115386963, "mix_std": 0.2978377640247345, "beta_std": 0.009975569322705269, "beta_low": 0.71337890625, "beta_high": 0.0, "orthogonal_to_active": 0.16643232107162476, "evidence_weights": [-0.21878443658351898, -0.25420740246772766, 1.0714067220687866], "agreement_weight": 0.12147193402051926, "orthogonal_weight": -0.7736620903015137}, {"head": 1, "memory_to_active": 0.007960603572428226, "contribution_to_active": 0.45958784222602844, "mix_mean": 0.5808846354484558, "mix_low": 0.0035181045532226562, "mix_high": 0.009327888488769531, "read_cosine": 0.18407857418060303, "beta_mean": 3.808533074334264e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00481665413826704, "mix_std": 0.2933667302131653, "beta_std": 9.161144407698885e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0038258086424320936, "evidence_weights": [-0.268873393535614, -0.21578076481819153, 0.7223469614982605], "agreement_weight": 0.6059227585792542, "orthogonal_weight": -0.8722938895225525}, {"head": 2, "memory_to_active": 1.998809814453125, "contribution_to_active": 0.26401978731155396, "mix_mean": 0.05372156947851181, "mix_low": 0.3001995086669922, "mix_high": 2.86102294921875e-06, "read_cosine": 0.21967050433158875, "beta_mean": 0.13049063086509705, "nonfinite": 0.0, "weighted_memory_to_active": 0.2932423949241638, "mix_std": 0.10169187933206558, "beta_std": 0.11983351409435272, "beta_low": 0.02423095703125, "beta_high": 0.0, "orthogonal_to_active": 1.2551818144856952e-05, "evidence_weights": [-0.10116992890834808, -0.0077426922507584095, -0.049834541976451874], "agreement_weight": -0.014546625316143036, "orthogonal_weight": -5.334662273526192e-05}, {"head": 3, "memory_to_active": 0.019900530576705933, "contribution_to_active": 0.5335066914558411, "mix_mean": 0.6062638163566589, "mix_low": 0.004479408264160156, "mix_high": 0.009008407592773438, "read_cosine": 0.29813358187675476, "beta_mean": 8.744882507016882e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013875657692551613, "mix_std": 0.28487202525138855, "beta_std": 0.00019247818272560835, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003243120852857828, "evidence_weights": [-0.26245418190956116, -0.22218354046344757, 2.318240165710449], "agreement_weight": 2.2719457149505615, "orthogonal_weight": -0.31061530113220215}, {"head": 4, "memory_to_active": 0.011523612774908543, "contribution_to_active": 0.3902086019515991, "mix_mean": 0.5287189483642578, "mix_low": 0.0025415420532226562, "mix_high": 0.0026922225952148438, "read_cosine": 0.1292220801115036, "beta_mean": 5.903470446355641e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007691982202231884, "mix_std": 0.2873712182044983, "beta_std": 0.00011075715883634984, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007219936698675156, "evidence_weights": [-0.2431011199951172, -0.2240692526102066, 0.06821459531784058], "agreement_weight": -0.06796922534704208, "orthogonal_weight": -1.0134010314941406}, {"head": 5, "memory_to_active": 0.22250840067863464, "contribution_to_active": 0.6633861064910889, "mix_mean": 0.7352159023284912, "mix_low": 0.0019941329956054688, "mix_high": 0.023599624633789062, "read_cosine": 0.13978871703147888, "beta_mean": 0.004607901908457279, "nonfinite": 0.0, "weighted_memory_to_active": 0.17111077904701233, "mix_std": 0.2630089223384857, "beta_std": 0.01027599535882473, "beta_low": 0.88909912109375, "beta_high": 0.0, "orthogonal_to_active": 0.1267479658126831, "evidence_weights": [-0.26354172825813293, -0.34680089354515076, 0.5443820357322693], "agreement_weight": 0.2383500039577484, "orthogonal_weight": -0.7818546891212463}]}, {"layer": 4, "memory_to_active": 0.942698061466217, "contribution_to_active": 0.4758170247077942, "mix_mean": 0.3731571435928345, "mix_low": 0.23170185089111328, "mix_high": 0.01110172364860773, "read_cosine": 0.30363669991493225, "beta_mean": 0.10585980862379074, "nonfinite": 0.0, "weighted_memory_to_active": 0.1294756829738617, "mix_std": 0.16195961833000183, "beta_std": 0.0975363627076149, "beta_low": 0.3970540463924408, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.056567080318927765, "output_influence": 0.5011796355247498, "residual_influence": 0.15722937881946564, "state_active": 0.04060796648263931, "state_memory": 0.01572548598051071, "state_peak": 42.17479705810547, "growth_active": 1.2541944980621338, "growth_memory": 1.5182684659957886, "heads": [{"head": 0, "memory_to_active": 0.14441590011119843, "contribution_to_active": 0.7966805100440979, "mix_mean": 0.6841892004013062, "mix_low": 0.011285781860351562, "mix_high": 0.02886199951171875, "read_cosine": 0.464805543422699, "beta_mean": 0.2367221862077713, "nonfinite": 0.0, "weighted_memory_to_active": 0.12319140136241913, "mix_std": 0.34618067741394043, "beta_std": 0.23476526141166687, "beta_low": 0.2508544921875, "beta_high": 0.0, "orthogonal_to_active": 0.08259245753288269, "evidence_weights": [-0.24985800683498383, -0.36065220832824707, -0.1826588213443756], "agreement_weight": -0.0012252215528860688, "orthogonal_weight": 0.8193867206573486}, {"head": 1, "memory_to_active": 0.05293361097574234, "contribution_to_active": 0.6670576930046082, "mix_mean": 0.7039690017700195, "mix_low": 9.34600830078125e-05, "mix_high": 0.017271041870117188, "read_cosine": 0.5327280759811401, "beta_mean": 0.00109348283149302, "nonfinite": 0.0, "weighted_memory_to_active": 0.040025532245635986, "mix_std": 0.23712857067584991, "beta_std": 0.007600440643727779, "beta_low": 0.9779052734375, "beta_high": 0.0, "orthogonal_to_active": 0.020679768174886703, "evidence_weights": [-0.23705719411373138, -0.36576560139656067, 0.5832875967025757], "agreement_weight": 0.35684090852737427, "orthogonal_weight": -0.7069013118743896}, {"head": 2, "memory_to_active": 1.515742301940918, "contribution_to_active": 0.3496434986591339, "mix_mean": 0.051551297307014465, "mix_low": 0.337493896484375, "mix_high": 3.4332275390625e-05, "read_cosine": 0.21135063469409943, "beta_mean": 0.05246421694755554, "nonfinite": 0.0, "weighted_memory_to_active": 0.2826597988605499, "mix_std": 0.09280404448509216, "beta_std": 0.05577808618545532, "beta_low": 0.13128662109375, "beta_high": 0.0, "orthogonal_to_active": 0.10295067727565765, "evidence_weights": [0.03150948882102966, -0.056366365402936935, 0.3359629213809967], "agreement_weight": 0.12899765372276306, "orthogonal_weight": -0.4845994710922241}, {"head": 3, "memory_to_active": 1.7011592388153076, "contribution_to_active": 0.14255106449127197, "mix_mean": 0.03431360796093941, "mix_low": 0.3848600387573242, "mix_high": 0.0, "read_cosine": 0.235824316740036, "beta_mean": 0.1468367874622345, "nonfinite": 0.0, "weighted_memory_to_active": 0.18011844158172607, "mix_std": 0.05590131878852844, "beta_std": 0.12666425108909607, "beta_low": 0.022216796875, "beta_high": 0.0, "orthogonal_to_active": 0.0011513509089127183, "evidence_weights": [-0.026681069284677505, 0.043513376265764236, -0.2557527422904968], "agreement_weight": 0.08210178464651108, "orthogonal_weight": 0.008554156869649887}, {"head": 4, "memory_to_active": 0.006816308479756117, "contribution_to_active": 0.634324312210083, "mix_mean": 0.7514276504516602, "mix_low": 6.389617919921875e-05, "mix_high": 0.020433425903320312, "read_cosine": 0.1505011022090912, "beta_mean": 1.5857624021009542e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00495253037661314, "mix_std": 0.21012625098228455, "beta_std": 4.017109677079134e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003175911260768771, "evidence_weights": [-0.32999253273010254, -0.31756556034088135, 0.27033138275146484], "agreement_weight": 0.3147400915622711, "orthogonal_weight": -0.681834876537323}, {"head": 5, "memory_to_active": 2.235121011734009, "contribution_to_active": 0.26464489102363586, "mix_mean": 0.013492021709680557, "mix_low": 0.6564140319824219, "mix_high": 9.5367431640625e-06, "read_cosine": 0.22661054134368896, "beta_mean": 0.19802629947662354, "nonfinite": 0.0, "weighted_memory_to_active": 0.14590629935264587, "mix_std": 0.029616862535476685, "beta_std": 0.16037000715732574, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12885230779647827, "evidence_weights": [0.059906262904405594, -0.01413985900580883, -0.27808842062950134], "agreement_weight": -0.08701203763484955, "orthogonal_weight": 1.096827745437622}]}, {"layer": 5, "memory_to_active": 0.34999725222587585, "contribution_to_active": 0.5434107780456543, "mix_mean": 0.5168717503547668, "mix_low": 0.055003803223371506, "mix_high": 0.019726913422346115, "read_cosine": 0.08264631032943726, "beta_mean": 0.05099089816212654, "nonfinite": 0.0, "weighted_memory_to_active": 0.12181618809700012, "mix_std": 0.2514958381652832, "beta_std": 0.06542927026748657, "beta_low": 0.675048828125, "beta_high": 0.0, "orthogonal_to_active": 0.05330348014831543, "output_influence": 0.582444429397583, "residual_influence": 0.1910281926393509, "state_active": 0.04059024900197983, "state_memory": 0.009468181058764458, "state_peak": 67.18173217773438, "growth_active": 1.1189981698989868, "growth_memory": 1.2330447435379028, "heads": [{"head": 0, "memory_to_active": 0.16565358638763428, "contribution_to_active": 0.5893362760543823, "mix_mean": 0.4871775507926941, "mix_low": 0.008798599243164062, "mix_high": 0.0019664764404296875, "read_cosine": -0.12391487509012222, "beta_mean": 0.017578136175870895, "nonfinite": 0.0, "weighted_memory_to_active": 0.06110987067222595, "mix_std": 0.27778488397598267, "beta_std": 0.028909407556056976, "beta_low": 0.5447998046875, "beta_high": 0.0, "orthogonal_to_active": 0.02702297270298004, "evidence_weights": [-0.1696976274251938, -0.1881747841835022, 0.12073500454425812], "agreement_weight": -0.020385315641760826, "orthogonal_weight": -0.46861565113067627}, {"head": 1, "memory_to_active": 0.006223102565854788, "contribution_to_active": 0.5490221977233887, "mix_mean": 0.7090393900871277, "mix_low": 0.0007448196411132812, "mix_high": 0.04034614562988281, "read_cosine": 0.21390873193740845, "beta_mean": 1.3288238733366597e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004007283598184586, "mix_std": 0.268271803855896, "beta_std": 3.759825267479755e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0036840750835835934, "evidence_weights": [-0.2456195056438446, -0.2850961685180664, 0.9303727746009827], "agreement_weight": 0.7100329399108887, "orthogonal_weight": -0.9570819139480591}, {"head": 2, "memory_to_active": 0.017353728413581848, "contribution_to_active": 0.5975586771965027, "mix_mean": 0.6634805202484131, "mix_low": 0.0025224685668945312, "mix_high": 0.02242755889892578, "read_cosine": -0.44236665964126587, "beta_mean": 8.301489287987351e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011352984234690666, "mix_std": 0.26327377557754517, "beta_std": 0.00035553600173443556, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.011093329638242722, "evidence_weights": [-0.2596946358680725, -0.31814080476760864, 0.17709150910377502], "agreement_weight": -0.1873108595609665, "orthogonal_weight": -1.0411441326141357}, {"head": 3, "memory_to_active": 1.416143774986267, "contribution_to_active": 0.24892795085906982, "mix_mean": 0.11626572906970978, "mix_low": 0.11468887329101562, "mix_high": 0.0, "read_cosine": 0.1497342437505722, "beta_mean": 0.09412121027708054, "nonfinite": 0.0, "weighted_memory_to_active": 0.3397822976112366, "mix_std": 0.14388062059879303, "beta_std": 0.1412661373615265, "beta_low": 0.20849609375, "beta_high": 0.0, "orthogonal_to_active": 0.13596603274345398, "evidence_weights": [-0.07668627053499222, -0.09989269822835922, 0.10526557266712189], "agreement_weight": 0.044579774141311646, "orthogonal_weight": -0.4959729015827179}, {"head": 4, "memory_to_active": 0.41248399019241333, "contribution_to_active": 0.5873453617095947, "mix_mean": 0.316267728805542, "mix_low": 0.20173931121826172, "mix_high": 0.0042743682861328125, "read_cosine": 0.528093695640564, "beta_mean": 0.19391828775405884, "nonfinite": 0.0, "weighted_memory_to_active": 0.2584279179573059, "mix_std": 0.3354361355304718, "beta_std": 0.22085854411125183, "beta_low": 0.30029296875, "beta_high": 0.0, "orthogonal_to_active": 0.10998937487602234, "evidence_weights": [-0.11351248621940613, -0.2516995668411255, -0.04999491572380066], "agreement_weight": 0.02570388838648796, "orthogonal_weight": 0.6272603869438171}, {"head": 5, "memory_to_active": 0.08212529122829437, "contribution_to_active": 0.6882740259170532, "mix_mean": 0.8089993596076965, "mix_low": 0.0015287399291992188, "mix_high": 0.049346923828125, "read_cosine": 0.17042268812656403, "beta_mean": 0.00023144461738411337, "nonfinite": 0.0, "weighted_memory_to_active": 0.05621672794222832, "mix_std": 0.22032776474952698, "beta_std": 0.0011483547277748585, "beta_low": 0.9969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.03206508606672287, "evidence_weights": [-0.3038442134857178, -0.32796967029571533, 1.840240240097046], "agreement_weight": 0.5147597193717957, "orthogonal_weight": -0.6051939725875854}]}, {"layer": 6, "memory_to_active": 0.2463351935148239, "contribution_to_active": 0.6492118835449219, "mix_mean": 0.5992394685745239, "mix_low": 0.033121585845947266, "mix_high": 0.03013976477086544, "read_cosine": 0.30495381355285645, "beta_mean": 0.06172351539134979, "nonfinite": 0.0, "weighted_memory_to_active": 0.13228276371955872, "mix_std": 0.23909877240657806, "beta_std": 0.07290419191122055, "beta_low": 0.6112162470817566, "beta_high": 0.0, "orthogonal_to_active": 0.07888229191303253, "output_influence": 0.5855706930160522, "residual_influence": 0.21317097544670105, "state_active": 0.04428461194038391, "state_memory": 0.006037022452801466, "state_peak": 90.54206085205078, "growth_active": 1.225170612335205, "growth_memory": 1.179480791091919, "heads": [{"head": 0, "memory_to_active": 0.10808732360601425, "contribution_to_active": 0.7686517834663391, "mix_mean": 0.821550190448761, "mix_low": 0.00021648406982421875, "mix_high": 0.05982208251953125, "read_cosine": 0.28481119871139526, "beta_mean": 0.012111031450331211, "nonfinite": 0.0, "weighted_memory_to_active": 0.09592209756374359, "mix_std": 0.20259934663772583, "beta_std": 0.03670203685760498, "beta_low": 0.81964111328125, "beta_high": 0.0, "orthogonal_to_active": 0.06483075022697449, "evidence_weights": [-0.30053144693374634, -0.3475417494773865, 1.1902817487716675], "agreement_weight": 0.8676679730415344, "orthogonal_weight": -0.7230128645896912}, {"head": 1, "memory_to_active": 0.0045128995552659035, "contribution_to_active": 0.5351237058639526, "mix_mean": 0.600581705570221, "mix_low": 0.0006322860717773438, "mix_high": 0.016353607177734375, "read_cosine": 0.427420973777771, "beta_mean": 1.0317898158973549e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021293293684720993, "mix_std": 0.28228580951690674, "beta_std": 1.931465885718353e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011839705985039473, "evidence_weights": [-0.1906888335943222, -0.30120235681533813, 1.114013671875], "agreement_weight": 0.11958667635917664, "orthogonal_weight": -0.7104521989822388}, {"head": 2, "memory_to_active": 0.6496006846427917, "contribution_to_active": 0.10989546775817871, "mix_mean": 0.05917244404554367, "mix_low": 0.19197940826416016, "mix_high": 9.5367431640625e-07, "read_cosine": 0.27216923236846924, "beta_mean": 0.018392808735370636, "nonfinite": 0.0, "weighted_memory_to_active": 0.0683165118098259, "mix_std": 0.0867486447095871, "beta_std": 0.019464366137981415, "beta_low": 0.35003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.012846862897276878, "evidence_weights": [0.05635755881667137, -0.07124509662389755, 0.36038967967033386], "agreement_weight": 0.1424061357975006, "orthogonal_weight": -0.22063995897769928}, {"head": 3, "memory_to_active": 0.13678468763828278, "contribution_to_active": 0.6953610181808472, "mix_mean": 0.6746114492416382, "mix_low": 0.004899024963378906, "mix_high": 0.022688865661621094, "read_cosine": 0.46814319491386414, "beta_mean": 0.2186764031648636, "nonfinite": 0.0, "weighted_memory_to_active": 0.0996878370642662, "mix_std": 0.32050514221191406, "beta_std": 0.24751633405685425, "beta_low": 0.19921875, "beta_high": 0.0, "orthogonal_to_active": 0.05969332158565521, "evidence_weights": [-0.16755573451519012, -0.3152829706668854, 0.11892286688089371], "agreement_weight": -0.005565183237195015, "orthogonal_weight": 0.6875385642051697}, {"head": 4, "memory_to_active": 0.5529556274414062, "contribution_to_active": 1.148512601852417, "mix_mean": 0.741362452507019, "mix_low": 0.0008745193481445312, "mix_high": 0.07329845428466797, "read_cosine": 0.0984344631433487, "beta_mean": 0.12102518230676651, "nonfinite": 0.0, "weighted_memory_to_active": 0.5084277987480164, "mix_std": 0.3034253716468811, "beta_std": 0.1333872675895691, "beta_low": 0.29852294921875, "beta_high": 0.0, "orthogonal_to_active": 0.3159964084625244, "evidence_weights": [-0.17419815063476562, -0.5229986310005188, -0.7031452655792236], "agreement_weight": -0.014442632906138897, "orthogonal_weight": 0.6455565094947815}, {"head": 5, "memory_to_active": 0.026069890707731247, "contribution_to_active": 0.6377264261245728, "mix_mean": 0.6981582641601562, "mix_low": 0.0001277923583984375, "mix_high": 0.00867462158203125, "read_cosine": 0.27874380350112915, "beta_mean": 0.00012531509855762124, "nonfinite": 0.0, "weighted_memory_to_active": 0.019212987273931503, "mix_std": 0.23902831971645355, "beta_std": 0.00033583526965230703, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.018742458894848824, "evidence_weights": [-0.23417994379997253, -0.3158940374851227, 0.3635760247707367], "agreement_weight": 0.2114071398973465, "orthogonal_weight": -1.046775460243225}]}, {"layer": 7, "memory_to_active": 1.7675451040267944, "contribution_to_active": 0.2795431613922119, "mix_mean": 0.18058162927627563, "mix_low": 0.22099098563194275, "mix_high": 0.001041412353515625, "read_cosine": 0.24112512171268463, "beta_mean": 0.140097975730896, "nonfinite": 0.0, "weighted_memory_to_active": 0.2060053050518036, "mix_std": 0.14993518590927124, "beta_std": 0.09744696319103241, "beta_low": 0.2179158627986908, "beta_high": 0.0001932780141942203, "orthogonal_to_active": 0.08825299143791199, "output_influence": 0.2961330711841583, "residual_influence": 0.07406134903430939, "state_active": 0.014513090252876282, "state_memory": 0.023735836148262024, "state_peak": 31.83645248413086, "growth_active": 1.168763518333435, "growth_memory": 1.2498506307601929, "heads": [{"head": 0, "memory_to_active": 2.8155744075775146, "contribution_to_active": 0.1681458055973053, "mix_mean": 0.04010142758488655, "mix_low": 0.4057283401489258, "mix_high": 0.0, "read_cosine": 0.3415195047855377, "beta_mean": 0.18283692002296448, "nonfinite": 0.0, "weighted_memory_to_active": 0.19478759169578552, "mix_std": 0.08075807243585587, "beta_std": 0.1326042115688324, "beta_low": 0.001953125, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.002068459754809737, "evidence_weights": [0.044422078877687454, -0.08935919404029846, 0.015957823023200035], "agreement_weight": -0.030914779752492905, "orthogonal_weight": -0.011675911955535412}, {"head": 1, "memory_to_active": 0.9977531433105469, "contribution_to_active": 0.4276352524757385, "mix_mean": 0.22830167412757874, "mix_low": 0.11156368255615234, "mix_high": 0.0055027008056640625, "read_cosine": 0.3920981287956238, "beta_mean": 0.09805162996053696, "nonfinite": 0.0, "weighted_memory_to_active": 0.36153823137283325, "mix_std": 0.24900983273983002, "beta_std": 0.06643044203519821, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.05348912999033928, "evidence_weights": [-0.21707049012184143, -0.10431921482086182, -0.3030647039413452], "agreement_weight": 0.05200069770216942, "orthogonal_weight": 0.176069974899292}, {"head": 2, "memory_to_active": 0.5822413563728333, "contribution_to_active": 0.3876417577266693, "mix_mean": 0.2408524751663208, "mix_low": 0.015481948852539062, "mix_high": 0.0004730224609375, "read_cosine": 0.12137965857982635, "beta_mean": 0.03422939032316208, "nonfinite": 0.0, "weighted_memory_to_active": 0.22256211936473846, "mix_std": 0.21646849811077118, "beta_std": 0.036765530705451965, "beta_low": 0.176513671875, "beta_high": 0.0, "orthogonal_to_active": 0.1313651204109192, "evidence_weights": [-0.09775881469249725, -0.1849588304758072, 0.8750878572463989], "agreement_weight": 0.017830606549978256, "orthogonal_weight": -0.6080734729766846}, {"head": 3, "memory_to_active": 4.669584274291992, "contribution_to_active": 0.10164875537157059, "mix_mean": 0.03698253631591797, "mix_low": 0.3125724792480469, "mix_high": 0.0, "read_cosine": 0.09571392089128494, "beta_mean": 0.45748162269592285, "nonfinite": 0.0, "weighted_memory_to_active": 0.35554832220077515, "mix_std": 0.053556013852357864, "beta_std": 0.2768930494785309, "beta_low": 0.0013427734375, "beta_high": 0.0006103515625, "orthogonal_to_active": 0.3064078092575073, "evidence_weights": [0.0751897543668747, -0.04238191992044449, 0.7199121713638306], "agreement_weight": 0.05142812430858612, "orthogonal_weight": -0.9438493847846985}, {"head": 4, "memory_to_active": 1.503014087677002, "contribution_to_active": 0.1202562004327774, "mix_mean": 0.02619594894349575, "mix_low": 0.47908592224121094, "mix_high": 9.5367431640625e-07, "read_cosine": 0.2359696477651596, "beta_mean": 0.06781689822673798, "nonfinite": 0.0, "weighted_memory_to_active": 0.08265328407287598, "mix_std": 0.050120946019887924, "beta_std": 0.07155047357082367, "beta_low": 0.12799072265625, "beta_high": 0.0, "orthogonal_to_active": 0.018189068883657455, "evidence_weights": [0.11964448541402817, -0.028268327936530113, 0.1923104077577591], "agreement_weight": 0.03379153087735176, "orthogonal_weight": -0.28469961881637573}, {"head": 5, "memory_to_active": 0.03710401803255081, "contribution_to_active": 0.4719310998916626, "mix_mean": 0.5110557079315186, "mix_low": 0.0015134811401367188, "mix_high": 0.00027179718017578125, "read_cosine": 0.26006972789764404, "beta_mean": 0.00017140561249107122, "nonfinite": 0.0, "weighted_memory_to_active": 0.01894216611981392, "mix_std": 0.2496977150440216, "beta_std": 0.0004380858445074409, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.017998341470956802, "evidence_weights": [-0.1870032250881195, -0.24263222515583038, 0.4742635190486908], "agreement_weight": 0.23568153381347656, "orthogonal_weight": -1.0101977586746216}]}, {"layer": 8, "memory_to_active": 0.13689637184143066, "contribution_to_active": 0.5757447481155396, "mix_mean": 0.5108906030654907, "mix_low": 0.009730180725455284, "mix_high": 0.0019798281136900187, "read_cosine": 0.40410950779914856, "beta_mean": 0.012871179729700089, "nonfinite": 0.0, "weighted_memory_to_active": 0.07288911938667297, "mix_std": 0.24457302689552307, "beta_std": 0.019692987203598022, "beta_low": 0.7905070781707764, "beta_high": 0.0, "orthogonal_to_active": 0.06750349700450897, "output_influence": 0.528761088848114, "residual_influence": 0.199504554271698, "state_active": 0.043170541524887085, "state_memory": 0.004860562738031149, "state_peak": 59.347232818603516, "growth_active": 1.081606388092041, "growth_memory": 1.054801106452942, "heads": [{"head": 0, "memory_to_active": 0.05229964107275009, "contribution_to_active": 0.5785013437271118, "mix_mean": 0.6097891926765442, "mix_low": 0.0004596710205078125, "mix_high": 0.0015172958374023438, "read_cosine": 0.4628835916519165, "beta_mean": 0.0028501616325229406, "nonfinite": 0.0, "weighted_memory_to_active": 0.037656545639038086, "mix_std": 0.2490597814321518, "beta_std": 0.011667266488075256, "beta_low": 0.937255859375, "beta_high": 0.0, "orthogonal_to_active": 0.023651784285902977, "evidence_weights": [-0.15677008032798767, -0.3787709176540375, 0.5693845748901367], "agreement_weight": 0.26836466789245605, "orthogonal_weight": -0.797857940196991}, {"head": 1, "memory_to_active": 0.07996279746294022, "contribution_to_active": 0.5579242706298828, "mix_mean": 0.5338878631591797, "mix_low": 0.00019741058349609375, "mix_high": 0.001605987548828125, "read_cosine": 0.11442022025585175, "beta_mean": 0.0011981314746662974, "nonfinite": 0.0, "weighted_memory_to_active": 0.03771002218127251, "mix_std": 0.23342734575271606, "beta_std": 0.010799920186400414, "beta_low": 0.9908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.027762308716773987, "evidence_weights": [-0.1938881278038025, -0.2741570770740509, 0.5908036828041077], "agreement_weight": 0.07304594665765762, "orthogonal_weight": -0.7657588720321655}, {"head": 2, "memory_to_active": 0.011953315697610378, "contribution_to_active": 0.6946191787719727, "mix_mean": 0.6935104131698608, "mix_low": 0.0003108978271484375, "mix_high": 0.004168510437011719, "read_cosine": 0.43890219926834106, "beta_mean": 0.00011886119318660349, "nonfinite": 0.0, "weighted_memory_to_active": 0.009566315449774265, "mix_std": 0.21911782026290894, "beta_std": 0.003154342994093895, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005453772842884064, "evidence_weights": [-0.22183071076869965, -0.3469972312450409, 0.9695860743522644], "agreement_weight": 0.3201483190059662, "orthogonal_weight": -0.6729767322540283}, {"head": 3, "memory_to_active": 0.10728232562541962, "contribution_to_active": 0.6351146697998047, "mix_mean": 0.5895653963088989, "mix_low": 0.0007572174072265625, "mix_high": 0.004065513610839844, "read_cosine": 0.7756574749946594, "beta_mean": 0.0071901967748999596, "nonfinite": 0.0, "weighted_memory_to_active": 0.07991907000541687, "mix_std": 0.25612127780914307, "beta_std": 0.007939481176435947, "beta_low": 0.76824951171875, "beta_high": 0.0, "orthogonal_to_active": 0.02613615244626999, "evidence_weights": [-0.17297013103961945, -0.2971104681491852, 0.5239413380622864], "agreement_weight": 0.25752782821655273, "orthogonal_weight": -0.604403555393219}, {"head": 4, "memory_to_active": 0.21969977021217346, "contribution_to_active": 0.3459080457687378, "mix_mean": 0.35625025629997253, "mix_low": 0.001544952392578125, "mix_high": 2.288818359375e-05, "read_cosine": 0.12610125541687012, "beta_mean": 0.0008343510562554002, "nonfinite": 0.0, "weighted_memory_to_active": 0.08476297557353973, "mix_std": 0.2340683788061142, "beta_std": 0.013219490647315979, "beta_low": 0.99102783203125, "beta_high": 0.0, "orthogonal_to_active": 0.015969160944223404, "evidence_weights": [-0.1406957358121872, -0.20107264816761017, 0.6817201972007751], "agreement_weight": 0.04602567479014397, "orthogonal_weight": -0.19165360927581787}, {"head": 5, "memory_to_active": 0.35018035769462585, "contribution_to_active": 0.6424008011817932, "mix_mean": 0.28234046697616577, "mix_low": 0.055110931396484375, "mix_high": 0.0004987716674804688, "read_cosine": 0.5066922307014465, "beta_mean": 0.0650353729724884, "nonfinite": 0.0, "weighted_memory_to_active": 0.18771977722644806, "mix_std": 0.275643527507782, "beta_std": 0.07137741893529892, "beta_low": 0.05572509765625, "beta_high": 0.0, "orthogonal_to_active": 0.3060477375984192, "evidence_weights": [-0.15229369699954987, -0.14449244737625122, -0.10978689789772034], "agreement_weight": 0.0038902515079826117, "orthogonal_weight": 1.9462289810180664}]}, {"layer": 9, "memory_to_active": 0.8190550804138184, "contribution_to_active": 0.6965695023536682, "mix_mean": 0.39672335982322693, "mix_low": 0.0538075789809227, "mix_high": 0.006587664596736431, "read_cosine": 0.33501195907592773, "beta_mean": 0.031159566715359688, "nonfinite": 0.0, "weighted_memory_to_active": 0.35548675060272217, "mix_std": 0.24737714231014252, "beta_std": 0.027421526610851288, "beta_low": 0.64544677734375, "beta_high": 0.0, "orthogonal_to_active": 0.03202842175960541, "output_influence": 0.5553374290466309, "residual_influence": 0.16953684389591217, "state_active": 0.044588807970285416, "state_memory": 0.01424413826316595, "state_peak": 62.28544235229492, "growth_active": 1.1932510137557983, "growth_memory": 1.294136881828308, "heads": [{"head": 0, "memory_to_active": 0.00038919015787541866, "contribution_to_active": 0.5808292031288147, "mix_mean": 0.6576555967330933, "mix_low": 0.001354217529296875, "mix_high": 0.028237342834472656, "read_cosine": 0.3166939914226532, "beta_mean": 9.742782367538894e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.0002564570168033242, "mix_std": 0.2851864695549011, "beta_std": 3.655453838291578e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00026735986466519535, "evidence_weights": [-0.30472511053085327, -0.31165215373039246, 0.39271605014801025], "agreement_weight": 0.09977427124977112, "orthogonal_weight": -1.0885891914367676}, {"head": 1, "memory_to_active": 0.015950260683894157, "contribution_to_active": 0.70932537317276, "mix_mean": 0.5808247327804565, "mix_low": 0.0066356658935546875, "mix_high": 0.0032405853271484375, "read_cosine": 0.42860546708106995, "beta_mean": 3.501947139739059e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012153835967183113, "mix_std": 0.27880722284317017, "beta_std": 7.414565334329382e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008130649104714394, "evidence_weights": [-0.23715005815029144, -0.2949455976486206, 0.9672703146934509], "agreement_weight": 0.5542787313461304, "orthogonal_weight": -0.8204041719436646}, {"head": 2, "memory_to_active": 0.14944076538085938, "contribution_to_active": 0.6582698225975037, "mix_mean": 0.6039345860481262, "mix_low": 0.0017604827880859375, "mix_high": 0.006190299987792969, "read_cosine": 0.3513314127922058, "beta_mean": 0.0005722856149077415, "nonfinite": 0.0, "weighted_memory_to_active": 0.09809998422861099, "mix_std": 0.26590093970298767, "beta_std": 0.004358154721558094, "beta_low": 0.99658203125, "beta_high": 0.0, "orthogonal_to_active": 0.05715237930417061, "evidence_weights": [-0.20100711286067963, -0.27656427025794983, 0.7491742372512817], "agreement_weight": 0.35166579484939575, "orthogonal_weight": -0.6780058741569519}, {"head": 3, "memory_to_active": 0.7351977229118347, "contribution_to_active": 0.26279255747795105, "mix_mean": 0.17108938097953796, "mix_low": 0.09032344818115234, "mix_high": 2.574920654296875e-05, "read_cosine": 0.3625054657459259, "beta_mean": 0.028822876513004303, "nonfinite": 0.0, "weighted_memory_to_active": 0.18560698628425598, "mix_std": 0.20824304223060608, "beta_std": 0.03200499340891838, "beta_low": 0.23626708984375, "beta_high": 0.0, "orthogonal_to_active": 0.0858464241027832, "evidence_weights": [-0.034732572734355927, -0.1064402312040329, 0.46782058477401733], "agreement_weight": 0.09314204752445221, "orthogonal_weight": -0.5132645964622498}, {"head": 4, "memory_to_active": 2.8775057792663574, "contribution_to_active": 1.623815655708313, "mix_mean": 0.15209735929965973, "mix_low": 0.12023544311523438, "mix_high": 0.0011777877807617188, "read_cosine": 0.08723602443933487, "beta_mean": 0.13483011722564697, "nonfinite": 0.0, "weighted_memory_to_active": 1.542108416557312, "mix_std": 0.21127143502235413, "beta_std": 0.07781856507062912, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.016260497272014618, "evidence_weights": [-0.07943961769342422, -0.15024840831756592, -0.050008706748485565], "agreement_weight": -0.06565950810909271, "orthogonal_weight": -0.010968981310725212}, {"head": 5, "memory_to_active": 1.135846495628357, "contribution_to_active": 0.3443843126296997, "mix_mean": 0.21473844349384308, "mix_low": 0.10253620147705078, "mix_high": 0.0006542205810546875, "read_cosine": 0.4636993408203125, "beta_mean": 0.0226961150765419, "nonfinite": 0.0, "weighted_memory_to_active": 0.2946946620941162, "mix_std": 0.23485369980335236, "beta_std": 0.05026964470744133, "beta_low": 0.63983154296875, "beta_high": 0.0, "orthogonal_to_active": 0.02451321855187416, "evidence_weights": [0.04448516294360161, -0.1877060979604721, 0.1429082751274109], "agreement_weight": 0.003287889761850238, "orthogonal_weight": 0.09606779366731644}]}, {"layer": 10, "memory_to_active": 0.19535526633262634, "contribution_to_active": 0.728319525718689, "mix_mean": 0.5234392285346985, "mix_low": 0.02539205737411976, "mix_high": 0.004715760704129934, "read_cosine": 0.3865789771080017, "beta_mean": 0.04466293007135391, "nonfinite": 0.0, "weighted_memory_to_active": 0.08608324825763702, "mix_std": 0.2654985785484314, "beta_std": 0.04521140083670616, "beta_low": 0.6664836406707764, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.26477813720703125, "output_influence": 0.47226184606552124, "residual_influence": 0.16956235468387604, "state_active": 0.05543022230267525, "state_memory": 0.01447934191673994, "state_peak": 129.48069763183594, "growth_active": 1.037857174873352, "growth_memory": 1.0144715309143066, "heads": [{"head": 0, "memory_to_active": 0.6886606216430664, "contribution_to_active": 2.041255474090576, "mix_mean": 0.5587359070777893, "mix_low": 0.1153879165649414, "mix_high": 0.004887580871582031, "read_cosine": 0.10640659928321838, "beta_mean": 0.1993020474910736, "nonfinite": 0.0, "weighted_memory_to_active": 0.40823492407798767, "mix_std": 0.39219701290130615, "beta_std": 0.20531389117240906, "beta_low": 0.16412353515625, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.5133932828903198, "evidence_weights": [-0.08425876498222351, -0.36837735772132874, -1.907702922821045], "agreement_weight": 0.030597353354096413, "orthogonal_weight": 3.741212844848633}, {"head": 1, "memory_to_active": 0.007606326602399349, "contribution_to_active": 0.6253842115402222, "mix_mean": 0.6375317573547363, "mix_low": 0.0012865066528320312, "mix_high": 0.01653575897216797, "read_cosine": 0.33903589844703674, "beta_mean": 5.635916022583842e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004359293729066849, "mix_std": 0.26377198100090027, "beta_std": 0.00010598753578960896, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006726374384015799, "evidence_weights": [-0.24910205602645874, -0.1927957981824875, 0.548083484172821], "agreement_weight": 0.6497820019721985, "orthogonal_weight": -1.696478009223938}, {"head": 2, "memory_to_active": 0.012237554416060448, "contribution_to_active": 0.6294673085212708, "mix_mean": 0.6719655990600586, "mix_low": 0.00014495849609375, "mix_high": 0.0051422119140625, "read_cosine": 0.03632762283086777, "beta_mean": 2.5796829504542984e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005675180349498987, "mix_std": 0.24702118337154388, "beta_std": 0.00010127159475814551, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002357320161536336, "evidence_weights": [-0.26941683888435364, -0.27921709418296814, 0.665401816368103], "agreement_weight": -0.21461579203605652, "orthogonal_weight": -0.3998646140098572}, {"head": 3, "memory_to_active": 0.33528706431388855, "contribution_to_active": 0.09549884498119354, "mix_mean": 0.2644001245498657, "mix_low": 0.03324317932128906, "mix_high": 0.000102996826171875, "read_cosine": 0.921661376953125, "beta_mean": 0.06211341544985771, "nonfinite": 0.0, "weighted_memory_to_active": 0.03771667554974556, "mix_std": 0.22021174430847168, "beta_std": 0.057408761233091354, "beta_low": 0.0048828125, "beta_high": 0.0, "orthogonal_to_active": 0.03657250478863716, "evidence_weights": [-0.088402159512043, -0.14107675850391388, 0.1575060337781906], "agreement_weight": 0.020214486867189407, "orthogonal_weight": 1.4580461978912354}, {"head": 4, "memory_to_active": 0.011817855760455132, "contribution_to_active": 0.5641372203826904, "mix_mean": 0.5935120582580566, "mix_low": 0.00032329559326171875, "mix_high": 0.0016222000122070312, "read_cosine": 0.0779464915394783, "beta_mean": 3.152522549498826e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0069894627667963505, "mix_std": 0.24319694936275482, "beta_std": 5.435477942228317e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010113055817782879, "evidence_weights": [-0.18296849727630615, -0.2522663176059723, 0.9469983577728271], "agreement_weight": 0.1866501122713089, "orthogonal_weight": -1.5205414295196533}, {"head": 5, "memory_to_active": 0.11652222275733948, "contribution_to_active": 0.41417399048805237, "mix_mean": 0.4144895374774933, "mix_low": 0.0019664764404296875, "mix_high": 3.814697265625e-06, "read_cosine": 0.8380956649780273, "beta_mean": 0.006448420695960522, "nonfinite": 0.0, "weighted_memory_to_active": 0.053523920476436615, "mix_std": 0.22659242153167725, "beta_std": 0.00828414037823677, "beta_low": 0.82989501953125, "beta_high": 0.0, "orthogonal_to_active": 0.019506335258483887, "evidence_weights": [-0.13855202496051788, -0.16536614298820496, 0.2730987071990967], "agreement_weight": 0.09625693410634995, "orthogonal_weight": -0.6736152172088623}]}, {"layer": 11, "memory_to_active": 0.06888297200202942, "contribution_to_active": 0.6249415874481201, "mix_mean": 0.5235393643379211, "mix_low": 0.11174377053976059, "mix_high": 0.04603227227926254, "read_cosine": 0.34312713146209717, "beta_mean": 0.023429175838828087, "nonfinite": 0.0, "weighted_memory_to_active": 0.03770482540130615, "mix_std": 0.32359257340431213, "beta_std": 0.034556739032268524, "beta_low": 0.7153727412223816, "beta_high": 0.0, "orthogonal_to_active": 0.041343383491039276, "output_influence": 0.5613610744476318, "residual_influence": 0.16052058339118958, "state_active": 0.22307269275188446, "state_memory": 0.0029743886552751064, "state_peak": 35.619998931884766, "growth_active": 1.1584391593933105, "growth_memory": 1.4861012697219849, "heads": [{"head": 0, "memory_to_active": 0.005547061562538147, "contribution_to_active": 0.6115153431892395, "mix_mean": 0.6589823365211487, "mix_low": 0.0006742477416992188, "mix_high": 0.009490966796875, "read_cosine": 0.3041050136089325, "beta_mean": 5.661397153744474e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0040855552069842815, "mix_std": 0.2529148459434509, "beta_std": 0.002901475876569748, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.0033864735160022974, "evidence_weights": [-0.17645056545734406, -0.3584078252315521, 0.8096994757652283], "agreement_weight": 0.3966390788555145, "orthogonal_weight": -0.8589178323745728}, {"head": 1, "memory_to_active": 0.10427968949079514, "contribution_to_active": 0.43079185485839844, "mix_mean": 0.46335339546203613, "mix_low": 0.0049533843994140625, "mix_high": 0.0005025863647460938, "read_cosine": 0.4789726734161377, "beta_mean": 0.00211820425465703, "nonfinite": 0.0, "weighted_memory_to_active": 0.05047622695565224, "mix_std": 0.26837655901908875, "beta_std": 0.008471664041280746, "beta_low": 0.95916748046875, "beta_high": 0.0, "orthogonal_to_active": 0.028441864997148514, "evidence_weights": [-0.14050325751304626, -0.2775184214115143, 0.5915257334709167], "agreement_weight": 0.0842423215508461, "orthogonal_weight": -0.658730685710907}, {"head": 2, "memory_to_active": 0.004953849595040083, "contribution_to_active": 0.9865410327911377, "mix_mean": 0.4353530704975128, "mix_low": 0.31857776641845703, "mix_high": 0.1745738983154297, "read_cosine": 0.15729446709156036, "beta_mean": 0.06496856361627579, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034909253008663654, "mix_std": 0.4513484835624695, "beta_std": 0.08933708816766739, "beta_low": 0.5030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.035966672003269196, "evidence_weights": [0.04387321695685387, -0.34348341822624207, -1.3154511451721191], "agreement_weight": 0.0039883810095489025, "orthogonal_weight": 10.650922775268555}, {"head": 3, "memory_to_active": 0.006790346931666136, "contribution_to_active": 0.8596208691596985, "mix_mean": 0.410749226808548, "mix_low": 0.33835697174072266, "mix_high": 0.08321666717529297, "read_cosine": 0.5549663305282593, "beta_mean": 0.04982650652527809, "nonfinite": 0.0, "weighted_memory_to_active": 0.004323918838053942, "mix_std": 0.43997353315353394, "beta_std": 0.07759356498718262, "beta_low": 0.54302978515625, "beta_high": 0.0, "orthogonal_to_active": 0.030853740870952606, "evidence_weights": [0.2186249941587448, -0.27713799476623535, -0.09806052595376968], "agreement_weight": 0.05155239254236221, "orthogonal_weight": 10.91498851776123}, {"head": 4, "memory_to_active": 0.2131813019514084, "contribution_to_active": 0.4451620578765869, "mix_mean": 0.6066572666168213, "mix_low": 0.006214141845703125, "mix_high": 0.0017251968383789062, "read_cosine": 0.038461022078990936, "beta_mean": 0.015603930689394474, "nonfinite": 0.0, "weighted_memory_to_active": 0.12273363023996353, "mix_std": 0.2704732120037079, "beta_std": 0.018070822581648827, "beta_low": 0.50738525390625, "beta_high": 0.0, "orthogonal_to_active": 0.12178283929824829, "evidence_weights": [-0.2368081957101822, -0.24395661056041718, 1.8957674503326416], "agreement_weight": -0.22060991823673248, "orthogonal_weight": -1.003008484840393}, {"head": 5, "memory_to_active": 0.07854554802179337, "contribution_to_active": 0.41601794958114624, "mix_mean": 0.56614089012146, "mix_low": 0.00168609619140625, "mix_high": 0.006684303283691406, "read_cosine": 0.5249631404876709, "beta_mean": 0.008001234382390976, "nonfinite": 0.0, "weighted_memory_to_active": 0.04111867770552635, "mix_std": 0.2584686577320099, "beta_std": 0.010965792462229729, "beta_low": 0.77984619140625, "beta_high": 0.0, "orthogonal_to_active": 0.027628695592284203, "evidence_weights": [-0.2003471851348877, -0.3128669559955597, 0.6682148575782776], "agreement_weight": 0.08256708830595016, "orthogonal_weight": -0.8628684878349304}]}, {"layer": 12, "memory_to_active": 0.2167060822248459, "contribution_to_active": 0.5885998010635376, "mix_mean": 0.551422119140625, "mix_low": 0.008572420105338097, "mix_high": 0.007061958312988281, "read_cosine": 0.2284720093011856, "beta_mean": 0.013992557302117348, "nonfinite": 0.0, "weighted_memory_to_active": 0.11249124258756638, "mix_std": 0.25875338912010193, "beta_std": 0.015030045062303543, "beta_low": 0.8271077871322632, "beta_high": 0.0, "orthogonal_to_active": 0.06712330877780914, "output_influence": 0.5165064334869385, "residual_influence": 0.15528854727745056, "state_active": 0.035414811223745346, "state_memory": 0.007735235616564751, "state_peak": 67.25886535644531, "growth_active": 1.1868174076080322, "growth_memory": 1.202125906944275, "heads": [{"head": 0, "memory_to_active": 0.04425843432545662, "contribution_to_active": 0.6300831437110901, "mix_mean": 0.5921428799629211, "mix_low": 0.0006275177001953125, "mix_high": 0.006125450134277344, "read_cosine": 0.2783149480819702, "beta_mean": 0.0008319147164002061, "nonfinite": 0.0, "weighted_memory_to_active": 0.025331920012831688, "mix_std": 0.2630239427089691, "beta_std": 0.0018877792172133923, "beta_low": 0.99224853515625, "beta_high": 0.0, "orthogonal_to_active": 0.023452091962099075, "evidence_weights": [-0.2048315703868866, -0.27337881922721863, -0.0605270080268383], "agreement_weight": 0.09900322556495667, "orthogonal_weight": -0.9878084659576416}, {"head": 1, "memory_to_active": 0.08632126450538635, "contribution_to_active": 0.5682085752487183, "mix_mean": 0.5684124827384949, "mix_low": 0.00023365020751953125, "mix_high": 0.00226593017578125, "read_cosine": 0.38737013936042786, "beta_mean": 0.0005492441705428064, "nonfinite": 0.0, "weighted_memory_to_active": 0.05616132915019989, "mix_std": 0.2645815312862396, "beta_std": 0.009382883086800575, "beta_low": 0.99725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.04725511372089386, "evidence_weights": [-0.2001696527004242, -0.27208805084228516, 1.041764259338379], "agreement_weight": 0.27974212169647217, "orthogonal_weight": -0.9204707145690918}, {"head": 2, "memory_to_active": 0.5815126895904541, "contribution_to_active": 0.48349419236183167, "mix_mean": 0.319553941488266, "mix_low": 0.0271148681640625, "mix_high": 9.059906005859375e-05, "read_cosine": 0.5743187069892883, "beta_mean": 0.08121012151241302, "nonfinite": 0.0, "weighted_memory_to_active": 0.23157495260238647, "mix_std": 0.23541216552257538, "beta_std": 0.06158187985420227, "beta_low": 0.001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.2546966075897217, "evidence_weights": [-0.20449188351631165, -0.060647208243608475, -0.030610524117946625], "agreement_weight": 0.03072328120470047, "orthogonal_weight": 1.4254869222640991}, {"head": 3, "memory_to_active": 0.024967964738607407, "contribution_to_active": 0.7568617463111877, "mix_mean": 0.7242444753646851, "mix_low": 0.0001277923583984375, "mix_high": 0.02211284637451172, "read_cosine": 0.08093096315860748, "beta_mean": 0.000233220198424533, "nonfinite": 0.0, "weighted_memory_to_active": 0.01845000684261322, "mix_std": 0.22662922739982605, "beta_std": 0.005093884654343128, "beta_low": 0.99835205078125, "beta_high": 0.0, "orthogonal_to_active": 0.01749422773718834, "evidence_weights": [-0.28789022564888, -0.33392077684402466, 1.6374633312225342], "agreement_weight": 0.6098561882972717, "orthogonal_weight": -0.9679182767868042}, {"head": 4, "memory_to_active": 0.49971872568130493, "contribution_to_active": 0.5883001685142517, "mix_mean": 0.46236205101013184, "mix_low": 0.01704692840576172, "mix_high": 0.005397796630859375, "read_cosine": -0.18604843318462372, "beta_mean": 0.0007955628097988665, "nonfinite": 0.0, "weighted_memory_to_active": 0.3009334206581116, "mix_std": 0.2910790741443634, "beta_std": 0.008769173175096512, "beta_low": 0.9796142578125, "beta_high": 0.0, "orthogonal_to_active": 0.028155561536550522, "evidence_weights": [-0.1563863605260849, -0.21475563943386078, -0.1466183066368103], "agreement_weight": 0.0164168868213892, "orthogonal_weight": -0.09679313004016876}, {"head": 5, "memory_to_active": 0.0634574294090271, "contribution_to_active": 0.5046508312225342, "mix_mean": 0.6418167352676392, "mix_low": 0.006283760070800781, "mix_high": 0.006379127502441406, "read_cosine": 0.23594564199447632, "beta_mean": 0.0003352765052113682, "nonfinite": 0.0, "weighted_memory_to_active": 0.042495809495449066, "mix_std": 0.27179428935050964, "beta_std": 0.0034646617714315653, "beta_low": 0.99346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.031686242669820786, "evidence_weights": [-0.22365042567253113, -0.3148791193962097, 1.2224324941635132], "agreement_weight": 0.43006303906440735, "orthogonal_weight": -0.808523952960968}]}, {"layer": 13, "memory_to_active": 0.11090153455734253, "contribution_to_active": 0.5426025986671448, "mix_mean": 0.5390195846557617, "mix_low": 0.006348609924316406, "mix_high": 0.010574182495474815, "read_cosine": 0.38792258501052856, "beta_mean": 0.012696457095444202, "nonfinite": 0.0, "weighted_memory_to_active": 0.04763587936758995, "mix_std": 0.25704994797706604, "beta_std": 0.015009636990725994, "beta_low": 0.8117879629135132, "beta_high": 0.0, "orthogonal_to_active": 0.049931347370147705, "output_influence": 0.5284915566444397, "residual_influence": 0.19192712008953094, "state_active": 0.13781186938285828, "state_memory": 0.042863503098487854, "state_peak": 99.02477264404297, "growth_active": 0.9483072757720947, "growth_memory": 1.0398783683776855, "heads": [{"head": 0, "memory_to_active": 0.008965461514890194, "contribution_to_active": 0.6102321743965149, "mix_mean": 0.6220308542251587, "mix_low": 0.0007333755493164062, "mix_high": 0.005589485168457031, "read_cosine": 0.4544067978858948, "beta_mean": 3.127633681287989e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005892488174140453, "mix_std": 0.2650102972984314, "beta_std": 5.97669932176359e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005516017321497202, "evidence_weights": [-0.1907108873128891, -0.2651253938674927, 0.7269814610481262], "agreement_weight": 0.21887503564357758, "orthogonal_weight": -1.0574079751968384}, {"head": 1, "memory_to_active": 0.009234115481376648, "contribution_to_active": 0.6139822602272034, "mix_mean": 0.5781565308570862, "mix_low": 0.001354217529296875, "mix_high": 0.005095481872558594, "read_cosine": 0.20171934366226196, "beta_mean": 3.923868280253373e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005234942771494389, "mix_std": 0.27579420804977417, "beta_std": 8.835898188408464e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006683992221951485, "evidence_weights": [-0.17986270785331726, -0.2624754309654236, 0.5533195734024048], "agreement_weight": 0.567919909954071, "orthogonal_weight": -1.3238388299942017}, {"head": 2, "memory_to_active": 0.06262329965829849, "contribution_to_active": 0.4863356351852417, "mix_mean": 0.388588011264801, "mix_low": 0.0011987686157226562, "mix_high": 0.00016021728515625, "read_cosine": 0.4708658754825592, "beta_mean": 0.007570735178887844, "nonfinite": 0.0, "weighted_memory_to_active": 0.032041650265455246, "mix_std": 0.2314189225435257, "beta_std": 0.028437212109565735, "beta_low": 0.8751220703125, "beta_high": 0.0, "orthogonal_to_active": 0.022016366943717003, "evidence_weights": [-0.1787164807319641, -0.212270587682724, 0.4139176905155182], "agreement_weight": 0.007381355855613947, "orthogonal_weight": -0.8519829511642456}, {"head": 3, "memory_to_active": 0.16035905480384827, "contribution_to_active": 0.5902080535888672, "mix_mean": 0.6657613515853882, "mix_low": 0.026645660400390625, "mix_high": 0.03905296325683594, "read_cosine": -0.02905138209462166, "beta_mean": 0.0008808639831840992, "nonfinite": 0.0, "weighted_memory_to_active": 0.11577904969453812, "mix_std": 0.2920496165752411, "beta_std": 0.00884396955370903, "beta_low": 0.9952392578125, "beta_high": 0.0, "orthogonal_to_active": 0.10649271309375763, "evidence_weights": [-0.3157196342945099, -0.24139174818992615, 1.491668939590454], "agreement_weight": 6.698206424713135, "orthogonal_weight": -0.9755823612213135}, {"head": 4, "memory_to_active": 0.3740391135215759, "contribution_to_active": 0.23582740128040314, "mix_mean": 0.31662723422050476, "mix_low": 0.005906105041503906, "mix_high": 0.0, "read_cosine": 0.8856245279312134, "beta_mean": 0.06741555035114288, "nonfinite": 0.0, "weighted_memory_to_active": 0.09119193255901337, "mix_std": 0.20724183320999146, "beta_std": 0.05041453242301941, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.12925782799720764, "evidence_weights": [-0.0631781667470932, -0.22426770627498627, 0.4242762625217438], "agreement_weight": 0.012275450862944126, "orthogonal_weight": 2.297340154647827}, {"head": 5, "memory_to_active": 0.050188153982162476, "contribution_to_active": 0.71902996301651, "mix_mean": 0.6629533767700195, "mix_low": 0.0022535324096679688, "mix_high": 0.013546943664550781, "read_cosine": 0.3439701795578003, "beta_mean": 0.00024107379431370646, "nonfinite": 0.0, "weighted_memory_to_active": 0.035675205290317535, "mix_std": 0.27078476548194885, "beta_std": 0.002213975414633751, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.029621154069900513, "evidence_weights": [-0.22106920182704926, -0.28999873995780945, 0.8814895153045654], "agreement_weight": 0.207826167345047, "orthogonal_weight": -0.931288480758667}]}, {"layer": 14, "memory_to_active": 0.21528524160385132, "contribution_to_active": 0.41939985752105713, "mix_mean": 0.43014097213745117, "mix_low": 0.027738889679312706, "mix_high": 0.00605646800249815, "read_cosine": 0.6094565987586975, "beta_mean": 0.05976392701268196, "nonfinite": 0.0, "weighted_memory_to_active": 0.08943650871515274, "mix_std": 0.2412455677986145, "beta_std": 0.05882467329502106, "beta_low": 0.5045675039291382, "beta_high": 0.0, "orthogonal_to_active": 0.04215620458126068, "output_influence": 0.3531680405139923, "residual_influence": 0.15044386684894562, "state_active": 0.05407822132110596, "state_memory": 0.013977057300508022, "state_peak": 48.513790130615234, "growth_active": 1.1120641231536865, "growth_memory": 1.2596948146820068, "heads": [{"head": 0, "memory_to_active": 0.34975665807724, "contribution_to_active": 0.1889992654323578, "mix_mean": 0.18903374671936035, "mix_low": 0.020263671875, "mix_high": 0.0, "read_cosine": 0.8025583028793335, "beta_mean": 0.06783068180084229, "nonfinite": 0.0, "weighted_memory_to_active": 0.07392585277557373, "mix_std": 0.17814138531684875, "beta_std": 0.06054652854800224, "beta_low": 0.0086669921875, "beta_high": 0.0, "orthogonal_to_active": 0.0891467034816742, "evidence_weights": [-0.0970337837934494, -0.04638261720538139, 0.37124213576316833], "agreement_weight": 0.021965984255075455, "orthogonal_weight": 1.8837248086929321}, {"head": 1, "memory_to_active": 0.4248516857624054, "contribution_to_active": 0.285027414560318, "mix_mean": 0.20509733259677887, "mix_low": 0.13292407989501953, "mix_high": 0.0, "read_cosine": 0.8842001557350159, "beta_mean": 0.24579188227653503, "nonfinite": 0.0, "weighted_memory_to_active": 0.18624858558177948, "mix_std": 0.23713363707065582, "beta_std": 0.24681781232357025, "beta_low": 0.09771728515625, "beta_high": 0.0, "orthogonal_to_active": 0.01723627559840679, "evidence_weights": [0.0073964763432741165, -0.19284315407276154, -0.0966094583272934], "agreement_weight": 0.05857592448592186, "orthogonal_weight": 0.2904672622680664}, {"head": 2, "memory_to_active": 0.12201245129108429, "contribution_to_active": 0.5901121497154236, "mix_mean": 0.5936930775642395, "mix_low": 0.0026311874389648438, "mix_high": 0.004466056823730469, "read_cosine": 0.40481293201446533, "beta_mean": 0.0017248704098165035, "nonfinite": 0.0, "weighted_memory_to_active": 0.08040115982294083, "mix_std": 0.2651408910751343, "beta_std": 0.0022911177948117256, "beta_low": 0.98602294921875, "beta_high": 0.0, "orthogonal_to_active": 0.06249964237213135, "evidence_weights": [-0.129372701048851, -0.31555432081222534, 0.9115470051765442], "agreement_weight": 0.08680957555770874, "orthogonal_weight": -0.8764290809631348}, {"head": 3, "memory_to_active": 0.043086085468530655, "contribution_to_active": 0.6027988791465759, "mix_mean": 0.6747596263885498, "mix_low": 0.004094123840332031, "mix_high": 0.02234935760498047, "read_cosine": -0.07166683673858643, "beta_mean": 0.004768568091094494, "nonfinite": 0.0, "weighted_memory_to_active": 0.027552610263228416, "mix_std": 0.2563375234603882, "beta_std": 0.00749425683170557, "beta_low": 0.87664794921875, "beta_high": 0.0, "orthogonal_to_active": 0.020276684314012527, "evidence_weights": [-0.34098777174949646, -0.3126550018787384, 0.2941570580005646], "agreement_weight": 0.067144013941288, "orthogonal_weight": -0.8296850323677063}, {"head": 4, "memory_to_active": 0.15833115577697754, "contribution_to_active": 0.5092238783836365, "mix_mean": 0.5305653810501099, "mix_low": 0.0032720565795898438, "mix_high": 0.008396148681640625, "read_cosine": 0.8919934630393982, "beta_mean": 0.037132956087589264, "nonfinite": 0.0, "weighted_memory_to_active": 0.0943891704082489, "mix_std": 0.2584860622882843, "beta_std": 0.027749549597501755, "beta_low": 0.08148193359375, "beta_high": 0.0, "orthogonal_to_active": 0.026326868683099747, "evidence_weights": [-0.2064894288778305, -0.23687635362148285, 0.4411121606826782], "agreement_weight": 0.13516201078891754, "orthogonal_weight": -0.6240707039833069}, {"head": 5, "memory_to_active": 0.19367341697216034, "contribution_to_active": 0.34023767709732056, "mix_mean": 0.38769638538360596, "mix_low": 0.0032482147216796875, "mix_high": 0.0011272430419921875, "read_cosine": 0.744841456413269, "beta_mean": 0.0013346028281375766, "nonfinite": 0.0, "weighted_memory_to_active": 0.07410165667533875, "mix_std": 0.2522338628768921, "beta_std": 0.008048783987760544, "beta_low": 0.97686767578125, "beta_high": 0.0, "orthogonal_to_active": 0.03745104372501373, "evidence_weights": [-0.1730470359325409, -0.18245787918567657, 0.36599788069725037], "agreement_weight": 0.06363335251808167, "orthogonal_weight": -0.7491617798805237}]}, {"layer": 15, "memory_to_active": 0.05299545079469681, "contribution_to_active": 0.4968331754207611, "mix_mean": 0.43010640144348145, "mix_low": 0.006966432556509972, "mix_high": 0.0028464002534747124, "read_cosine": 0.6472752094268799, "beta_mean": 0.035127557814121246, "nonfinite": 0.0, "weighted_memory_to_active": 0.026893364265561104, "mix_std": 0.28333333134651184, "beta_std": 0.04082585871219635, "beta_low": 0.7089945673942566, "beta_high": 0.0, "orthogonal_to_active": 0.010070839896798134, "output_influence": 0.3108321726322174, "residual_influence": 0.18882356584072113, "state_active": 0.16766734421253204, "state_memory": 0.010208919644355774, "state_peak": 30.697465896606445, "growth_active": 1.1892082691192627, "growth_memory": 1.2756797075271606, "heads": [{"head": 0, "memory_to_active": 0.1222982406616211, "contribution_to_active": 0.4696742296218872, "mix_mean": 0.3913234770298004, "mix_low": 0.007030487060546875, "mix_high": 0.005682945251464844, "read_cosine": 0.5259788036346436, "beta_mean": 0.047959405928850174, "nonfinite": 0.0, "weighted_memory_to_active": 0.06171178072690964, "mix_std": 0.2829592227935791, "beta_std": 0.033873047679662704, "beta_low": 0.06658935546875, "beta_high": 0.0, "orthogonal_to_active": 0.03831641748547554, "evidence_weights": [-0.18769030272960663, -0.19612327218055725, 0.5227729678153992], "agreement_weight": 0.09333950281143188, "orthogonal_weight": -0.8244493007659912}, {"head": 1, "memory_to_active": 0.09575046598911285, "contribution_to_active": 0.43262195587158203, "mix_mean": 0.44247013330459595, "mix_low": 0.0009183883666992188, "mix_high": 0.006707191467285156, "read_cosine": 0.9320564270019531, "beta_mean": 0.0002020071551669389, "nonfinite": 0.0, "weighted_memory_to_active": 0.0415283627808094, "mix_std": 0.28531479835510254, "beta_std": 0.0017760373884811997, "beta_low": 0.9986572265625, "beta_high": 0.0, "orthogonal_to_active": 0.002852656878530979, "evidence_weights": [-0.15663838386535645, -0.16777662932872772, 0.16174671053886414], "agreement_weight": 0.11511614173650742, "orthogonal_weight": 0.15205244719982147}, {"head": 2, "memory_to_active": 0.0057905809953808784, "contribution_to_active": 0.5085593461990356, "mix_mean": 0.3529236912727356, "mix_low": 0.011339187622070312, "mix_high": 6.771087646484375e-05, "read_cosine": 0.7594617605209351, "beta_mean": 7.772118260618299e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031156495679169893, "mix_std": 0.27034232020378113, "beta_std": 0.00019776956469286233, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.001050544437021017, "evidence_weights": [-0.2013801783323288, -0.26724061369895935, 0.10670825839042664], "agreement_weight": 0.008409466594457626, "orthogonal_weight": 0.9051846861839294}, {"head": 3, "memory_to_active": 0.00152828439604491, "contribution_to_active": 0.3898460268974304, "mix_mean": 0.4110850691795349, "mix_low": 0.0047149658203125, "mix_high": 0.0005645751953125, "read_cosine": 0.5992810726165771, "beta_mean": 7.716546861047391e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005508051253855228, "mix_std": 0.29069337248802185, "beta_std": 1.848839019658044e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003235370968468487, "evidence_weights": [-0.27698928117752075, -0.23801879584789276, 0.1823456883430481], "agreement_weight": 0.04858605936169624, "orthogonal_weight": -0.6004683971405029}, {"head": 4, "memory_to_active": 0.06303655356168747, "contribution_to_active": 0.6350365281105042, "mix_mean": 0.39242133498191833, "mix_low": 0.008358001708984375, "mix_high": 0.000934600830078125, "read_cosine": 0.730927586555481, "beta_mean": 0.16006413102149963, "nonfinite": 0.0, "weighted_memory_to_active": 0.03526613116264343, "mix_std": 0.3074997067451477, "beta_std": 0.19123633205890656, "beta_low": 0.2205810546875, "beta_high": 0.0, "orthogonal_to_active": 0.006204603705555201, "evidence_weights": [-0.2065635770559311, -0.25242865085601807, -0.03245225176215172], "agreement_weight": 0.012323934584856033, "orthogonal_weight": 0.3903639018535614}, {"head": 5, "memory_to_active": 0.02956857904791832, "contribution_to_active": 0.5452609062194824, "mix_mean": 0.5904147028923035, "mix_low": 0.00943756103515625, "mix_high": 0.0031213760375976562, "read_cosine": 0.3359454572200775, "beta_mean": 0.0024543439503759146, "nonfinite": 0.0, "weighted_memory_to_active": 0.01918744295835495, "mix_std": 0.26319050788879395, "beta_std": 0.017853466793894768, "beta_low": 0.96820068359375, "beta_high": 0.0, "orthogonal_to_active": 0.011677275411784649, "evidence_weights": [-0.28124189376831055, -0.30445393919944763, 0.6077802777290344], "agreement_weight": 0.31701791286468506, "orthogonal_weight": 0.8179729580879211}]}]}, {"step": 71000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 35.54975836499989, "memory_to_active": 0.7970561385154724, "contribution_to_active": 0.5718216896057129, "mix_mean": 0.42799103260040283, "mix_low": 0.12126702070236206, "mix_high": 0.01045836042612791, "read_cosine": 0.3437955677509308, "beta_mean": 0.06882144510746002, "nonfinite": 0.0, "weighted_memory_to_active": 0.19921641051769257, "mix_std": 0.22282303869724274, "beta_std": 0.06030057370662689, "beta_low": 0.590216338634491, "beta_high": 3.4332275390625e-05, "orthogonal_to_active": 0.06332007050514221, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0667123794555664, "normal_token_accuracy": 0.4516128897666931, "state_active": 0.06539031863212585, "state_memory": 0.016718951985239983, "state_peak": 127.28933715820312, "growth_active": 1.1287952661514282, "growth_memory": 1.2882614135742188, "output_influence": 0.4470491409301758, "residual_influence": 0.16066977381706238, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.2232301235198975, "no_memory_loss_delta": 0.6741004586219788, "scrambled_loss_delta": 0.14060789346694946, "memory_only_loss_delta": 3.6619720458984375, "no_evidence_loss_delta": 0.36478757858276367, "no_agreement_loss_delta": 0.009960575960576534, "no_orthogonal_loss_delta": -0.00706402026116848}, {"index": 94, "tokens": 256, "normal_loss": 3.181528329849243, "no_memory_loss_delta": 0.7847515940666199, "scrambled_loss_delta": 0.41861581802368164, "memory_only_loss_delta": 3.268480062484741, "no_evidence_loss_delta": 0.33600425720214844, "no_agreement_loss_delta": 0.043005652725696564, "no_orthogonal_loss_delta": 0.059484727680683136}, {"index": 188, "tokens": 256, "normal_loss": 3.37738037109375, "no_memory_loss_delta": 0.9386130571365356, "scrambled_loss_delta": 0.10352583229541779, "memory_only_loss_delta": 2.88016414642334, "no_evidence_loss_delta": 0.45891648530960083, "no_agreement_loss_delta": 0.038179777562618256, "no_orthogonal_loss_delta": 0.028735492378473282}, {"index": 283, "tokens": 131, "normal_loss": 1.9293655157089233, "no_memory_loss_delta": 0.48517924547195435, "scrambled_loss_delta": 0.18484167754650116, "memory_only_loss_delta": 2.970998525619507, "no_evidence_loss_delta": 0.17536264657974243, "no_agreement_loss_delta": -0.0014605195028707385, "no_orthogonal_loss_delta": 0.00042110998765565455}], "agreement_weight_abs_mean": 0.2478855401277542, "agreement_weight_abs_max": 6.741353511810303, "agreement_weight_mean": 0.2167145162820816, "orthogonal_weight_abs_mean": 0.9410760402679443, "orthogonal_weight_abs_max": 11.007065773010254, "orthogonal_weight_mean": -0.05490914359688759, "no_memory_loss": 3.820115566253662, "no_memory_loss_delta": 0.7534032464027405, "no_memory_token_accuracy_gain": 0.11012236028909683, "no_memory_token_win_fraction": 0.7052280306816101, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 52.923829463588845, "scrambled_loss": 3.282372236251831, "scrambled_loss_delta": 0.21565979719161987, "scrambled_token_accuracy_gain": 0.03781979903578758, "scrambled_token_win_fraction": 0.6084538102149963, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.39905380816476, "memory_only_loss": 6.293317794799805, "memory_only_loss_delta": 3.2266056537628174, "memory_only_token_accuracy_gain": 0.29032257199287415, "memory_only_token_win_fraction": 0.9010010957717896, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.03080012238105, "no_evidence_loss": 3.4225053787231445, "no_evidence_loss_delta": 0.35579293966293335, "no_evidence_token_accuracy_gain": 0.05228031054139137, "no_evidence_token_win_fraction": 0.621802031993866, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 29.93823246508336, "no_agreement_loss": 3.092453956604004, "no_agreement_loss_delta": 0.02574198879301548, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.49721914529800415, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.54134885853463, "no_orthogonal_loss": 3.08988356590271, "no_orthogonal_loss_delta": 0.02317146770656109, "no_orthogonal_token_accuracy_gain": -0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.57953280210495, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.290507081801245, "short_no_memory_loss_delta": 0.6704779267311096, "long_context_memory_gain": 0.08292531967163086, "normal_context_loss_gain": 0.142598956823349, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.566868305206299, "contribution_to_active": 1.2089000940322876, "mix_mean": 0.10905712842941284, "mix_low": 0.5023486018180847, "mix_high": 0.004807313438504934, "read_cosine": 0.41503530740737915, "beta_mean": 0.20920369029045105, "nonfinite": 0.0, "weighted_memory_to_active": 1.232635736465454, "mix_std": 0.09276637434959412, "beta_std": 0.13717520236968994, "beta_low": 0.156463623046875, "beta_high": 0.0, "orthogonal_to_active": 0.05530835688114166, "output_influence": 0.2109418362379074, "residual_influence": 0.2339497059583664, "state_active": 0.06840766221284866, "state_memory": 0.04501136764883995, "state_peak": 46.856689453125, "growth_active": 1.2486004829406738, "growth_memory": 1.2666664123535156, "heads": [{"head": 0, "memory_to_active": 2.8268468379974365, "contribution_to_active": 0.4386243224143982, "mix_mean": 0.02001318708062172, "mix_low": 0.6094083786010742, "mix_high": 0.0, "read_cosine": 0.2191440314054489, "beta_mean": 0.19465067982673645, "nonfinite": 0.0, "weighted_memory_to_active": 0.5541319847106934, "mix_std": 0.0601034089922905, "beta_std": 0.09996377676725388, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.058741308748722076, "evidence_weights": [-0.002205756027251482, -0.040517017245292664, -0.054671622812747955], "agreement_weight": -0.22230565547943115, "orthogonal_weight": -0.12312712520360947}, {"head": 1, "memory_to_active": 6.644803047180176, "contribution_to_active": 2.8744022846221924, "mix_mean": 0.020154377445578575, "mix_low": 0.5997600555419922, "mix_high": 0.0, "read_cosine": 0.442234605550766, "beta_mean": 0.43542569875717163, "nonfinite": 0.0, "weighted_memory_to_active": 3.0833606719970703, "mix_std": 0.05522415414452553, "beta_std": 0.2215878665447235, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0922485813498497, "evidence_weights": [0.018309680745005608, -0.10772080719470978, 0.6627160310745239], "agreement_weight": 0.026210753247141838, "orthogonal_weight": 0.06537337601184845}, {"head": 2, "memory_to_active": 2.713982105255127, "contribution_to_active": 0.5950578451156616, "mix_mean": 0.016076585277915, "mix_low": 0.6411418914794922, "mix_high": 0.0, "read_cosine": 0.27494877576828003, "beta_mean": 0.2513658404350281, "nonfinite": 0.0, "weighted_memory_to_active": 0.7001652717590332, "mix_std": 0.03466387838125229, "beta_std": 0.1964970827102661, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05878168344497681, "evidence_weights": [0.05656641349196434, 0.04330635070800781, 0.25887393951416016], "agreement_weight": 0.012215918861329556, "orthogonal_weight": -0.2165200114250183}, {"head": 3, "memory_to_active": 7.729405403137207, "contribution_to_active": 2.8092730045318604, "mix_mean": 0.029994484037160873, "mix_low": 0.48996448516845703, "mix_high": 0.0, "read_cosine": 0.3071388006210327, "beta_mean": 0.32949456572532654, "nonfinite": 0.0, "weighted_memory_to_active": 2.9582936763763428, "mix_std": 0.0695982426404953, "beta_std": 0.23925533890724182, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10455530881881714, "evidence_weights": [-0.03609565645456314, -0.029689257964491844, -0.15703098475933075], "agreement_weight": -0.004717306233942509, "orthogonal_weight": -0.04297696799039841}, {"head": 4, "memory_to_active": 1.3825451135635376, "contribution_to_active": 0.06130006164312363, "mix_mean": 0.01992548070847988, "mix_low": 0.6594657897949219, "mix_high": 0.0, "read_cosine": 0.3148869276046753, "beta_mean": 0.02485146000981331, "nonfinite": 0.0, "weighted_memory_to_active": 0.043990299105644226, "mix_std": 0.05462629720568657, "beta_std": 0.04561080411076546, "beta_low": 0.5421142578125, "beta_high": 0.0, "orthogonal_to_active": 0.0008410730515606701, "evidence_weights": [0.09697455912828445, 0.06252850592136383, -0.12059460580348969], "agreement_weight": -0.19328145682811737, "orthogonal_weight": 0.017972823232412338}, {"head": 5, "memory_to_active": 0.10362746566534042, "contribution_to_active": 0.4747430682182312, "mix_mean": 0.5481786131858826, "mix_low": 0.01435089111328125, "mix_high": 0.02884387969970703, "read_cosine": 0.9318586587905884, "beta_mean": 0.019433803856372833, "nonfinite": 0.0, "weighted_memory_to_active": 0.05587182939052582, "mix_std": 0.2823822498321533, "beta_std": 0.02013632468879223, "beta_low": 0.39666748046875, "beta_high": 0.0, "orthogonal_to_active": 0.016682136803865433, "evidence_weights": [-0.17629669606685638, -0.22024793922901154, 0.37541666626930237], "agreement_weight": 0.28545230627059937, "orthogonal_weight": -0.6924918293952942}]}, {"layer": 1, "memory_to_active": 2.4719443321228027, "contribution_to_active": 0.48152610659599304, "mix_mean": 0.21788433194160461, "mix_low": 0.3642038106918335, "mix_high": 0.0012365977745503187, "read_cosine": 0.29986152052879333, "beta_mean": 0.17348724603652954, "nonfinite": 0.0, "weighted_memory_to_active": 0.31639111042022705, "mix_std": 0.11370734870433807, "beta_std": 0.1276416778564453, "beta_low": 0.3528645932674408, "beta_high": 0.00014241535973269492, "orthogonal_to_active": 0.03872029483318329, "output_influence": 0.2691265344619751, "residual_influence": 0.07576639950275421, "state_active": 0.027030326426029205, "state_memory": 0.03414653241634369, "state_peak": 42.130062103271484, "growth_active": 1.1256976127624512, "growth_memory": 1.4468214511871338, "heads": [{"head": 0, "memory_to_active": 0.022166840732097626, "contribution_to_active": 0.48681795597076416, "mix_mean": 0.5419349670410156, "mix_low": 0.003887176513671875, "mix_high": 0.0026521682739257812, "read_cosine": 0.03680982068181038, "beta_mean": 0.000147074882988818, "nonfinite": 0.0, "weighted_memory_to_active": 0.012909826822578907, "mix_std": 0.28031298518180847, "beta_std": 0.0002569968346506357, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012414934113621712, "evidence_weights": [-0.21219104528427124, -0.27342361211776733, 0.22780995070934296], "agreement_weight": 0.1996525079011917, "orthogonal_weight": -1.0088727474212646}, {"head": 1, "memory_to_active": 3.041827917098999, "contribution_to_active": 0.30421751737594604, "mix_mean": 0.022702854126691818, "mix_low": 0.47713661193847656, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5227887034416199, "beta_mean": 0.25439637899398804, "nonfinite": 0.0, "weighted_memory_to_active": 0.3320429027080536, "mix_std": 0.043603599071502686, "beta_std": 0.18900156021118164, "beta_low": 0.00189208984375, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.02417854219675064, "evidence_weights": [0.019546980038285255, -0.06728504598140717, 0.05879414826631546], "agreement_weight": -0.0010877377353608608, "orthogonal_weight": 0.21477732062339783}, {"head": 2, "memory_to_active": 0.04011165350675583, "contribution_to_active": 0.5669106245040894, "mix_mean": 0.6797324419021606, "mix_low": 0.00030517578125, "mix_high": 0.0047664642333984375, "read_cosine": 0.46029868721961975, "beta_mean": 0.0013913724105805159, "nonfinite": 0.0, "weighted_memory_to_active": 0.02724025584757328, "mix_std": 0.22803057730197906, "beta_std": 0.003862934187054634, "beta_low": 0.979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.019134024158120155, "evidence_weights": [-0.2758276164531708, -0.3712424337863922, 0.6476014852523804], "agreement_weight": 0.05286397039890289, "orthogonal_weight": -0.8246886134147644}, {"head": 3, "memory_to_active": 3.0770840644836426, "contribution_to_active": 0.18549418449401855, "mix_mean": 0.024500589817762375, "mix_low": 0.5225296020507812, "mix_high": 0.0, "read_cosine": 0.17635612189769745, "beta_mean": 0.3267361521720886, "nonfinite": 0.0, "weighted_memory_to_active": 0.23616929352283478, "mix_std": 0.060118161141872406, "beta_std": 0.2571225166320801, "beta_low": 0.0679931640625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.10011443495750427, "evidence_weights": [0.04108189046382904, -0.016985835507512093, -0.000597094651311636], "agreement_weight": -0.02287345565855503, "orthogonal_weight": -0.493971586227417}, {"head": 4, "memory_to_active": 3.9882235527038574, "contribution_to_active": 1.2532376050949097, "mix_mean": 0.03232019394636154, "mix_low": 0.3214435577392578, "mix_high": 0.0, "read_cosine": 0.32724058628082275, "beta_mean": 0.1704789698123932, "nonfinite": 0.0, "weighted_memory_to_active": 1.225419282913208, "mix_std": 0.052961546927690506, "beta_std": 0.12737233936786652, "beta_low": 0.04052734375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.050215885043144226, "evidence_weights": [-0.01691947691142559, -0.004728876519948244, 0.16894745826721191], "agreement_weight": -0.002634959062561393, "orthogonal_weight": 0.0596315823495388}, {"head": 5, "memory_to_active": 4.662251949310303, "contribution_to_active": 0.09247887134552002, "mix_mean": 0.0061148833483457565, "mix_low": 0.8599205017089844, "mix_high": 0.0, "read_cosine": 0.27567511796951294, "beta_mean": 0.2877734899520874, "nonfinite": 0.0, "weighted_memory_to_active": 0.06456507742404938, "mix_std": 0.01721716858446598, "beta_std": 0.18823367357254028, "beta_low": 0.02752685546875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.026263948529958725, "evidence_weights": [0.16736046969890594, 0.0839855894446373, 0.11525825411081314], "agreement_weight": -0.11405866593122482, "orthogonal_weight": 0.4123888909816742}]}, {"layer": 2, "memory_to_active": 1.0329540967941284, "contribution_to_active": 0.36136215925216675, "mix_mean": 0.41118401288986206, "mix_low": 0.23745331168174744, "mix_high": 0.0045893993228673935, "read_cosine": 0.28322598338127136, "beta_mean": 0.07685770094394684, "nonfinite": 0.0, "weighted_memory_to_active": 0.07914430648088455, "mix_std": 0.18627408146858215, "beta_std": 0.06912684440612793, "beta_low": 0.6058553457260132, "beta_high": 0.00011189778888365254, "orthogonal_to_active": 0.016781065613031387, "output_influence": 0.41606074571609497, "residual_influence": 0.11763212084770203, "state_active": 0.04128202795982361, "state_memory": 0.013757234439253807, "state_peak": 30.61552619934082, "growth_active": 1.074021577835083, "growth_memory": 1.542288899421692, "heads": [{"head": 0, "memory_to_active": 0.00541342981159687, "contribution_to_active": 0.3513370752334595, "mix_mean": 0.6442835927009583, "mix_low": 0.0004987716674804688, "mix_high": 0.009500503540039062, "read_cosine": 0.13918590545654297, "beta_mean": 4.366702341940254e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035038667265325785, "mix_std": 0.2655409872531891, "beta_std": 7.62331037549302e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0037057066801935434, "evidence_weights": [-0.37446826696395874, -0.2153472900390625, 0.24545177817344666], "agreement_weight": 0.15606847405433655, "orthogonal_weight": -1.0989879369735718}, {"head": 1, "memory_to_active": 3.279655694961548, "contribution_to_active": 0.05085575580596924, "mix_mean": 0.006181390956044197, "mix_low": 0.853428840637207, "mix_high": 0.0, "read_cosine": 0.12148380279541016, "beta_mean": 0.2184276580810547, "nonfinite": 0.0, "weighted_memory_to_active": 0.04046639800071716, "mix_std": 0.012906827963888645, "beta_std": 0.15923693776130676, "beta_low": 0.002685546875, "beta_high": 0.0, "orthogonal_to_active": 0.011896082200109959, "evidence_weights": [0.13221612572669983, 0.1037064790725708, 0.576449990272522], "agreement_weight": 0.02642030455172062, "orthogonal_weight": 0.32203036546707153}, {"head": 2, "memory_to_active": 0.010181319899857044, "contribution_to_active": 0.4142475724220276, "mix_mean": 0.5498418807983398, "mix_low": 0.001239776611328125, "mix_high": 0.0036325454711914062, "read_cosine": 0.390811026096344, "beta_mean": 3.0263472581282258e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00421626353636384, "mix_std": 0.27453821897506714, "beta_std": 4.961723607266322e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00020705244969576597, "evidence_weights": [-0.24781440198421478, -0.2916082739830017, 0.5838583111763], "agreement_weight": -0.004636737983673811, "orthogonal_weight": 0.060586296021938324}, {"head": 3, "memory_to_active": 0.09018710255622864, "contribution_to_active": 0.5423181653022766, "mix_mean": 0.5778875350952148, "mix_low": 0.00030517578125, "mix_high": 0.005328178405761719, "read_cosine": 0.48899194598197937, "beta_mean": 0.017509181052446365, "nonfinite": 0.0, "weighted_memory_to_active": 0.057232752442359924, "mix_std": 0.25539278984069824, "beta_std": 0.02789333648979664, "beta_low": 0.57977294921875, "beta_high": 0.0, "orthogonal_to_active": 0.04191191494464874, "evidence_weights": [-0.20231638848781586, -0.3265093266963959, 0.7425163984298706], "agreement_weight": 0.059222396463155746, "orthogonal_weight": -0.8938338756561279}, {"head": 4, "memory_to_active": 2.8035390377044678, "contribution_to_active": 0.2774563729763031, "mix_mean": 0.02114233560860157, "mix_low": 0.5687389373779297, "mix_high": 0.0, "read_cosine": 0.19624949991703033, "beta_mean": 0.22510476410388947, "nonfinite": 0.0, "weighted_memory_to_active": 0.36438149213790894, "mix_std": 0.05193650722503662, "beta_std": 0.22746381163597107, "beta_low": 0.05267333984375, "beta_high": 0.00067138671875, "orthogonal_to_active": 0.041542235761880875, "evidence_weights": [0.023363862186670303, 0.0007368940860033035, 0.04144328087568283], "agreement_weight": 0.028360353782773018, "orthogonal_weight": -0.13320915400981903}, {"head": 5, "memory_to_active": 0.008747687563300133, "contribution_to_active": 0.5319578647613525, "mix_mean": 0.6677672863006592, "mix_low": 0.0005083084106445312, "mix_high": 0.009075164794921875, "read_cosine": 0.36263376474380493, "beta_mean": 3.064137854380533e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0050650485791265965, "mix_std": 0.2573291063308716, "beta_std": 4.108711800654419e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001423390582203865, "evidence_weights": [-0.2818056643009186, -0.2917667329311371, 0.8897283673286438], "agreement_weight": 0.27391907572746277, "orthogonal_weight": -0.3263248801231384}]}, {"layer": 3, "memory_to_active": 0.41410526633262634, "contribution_to_active": 0.4894108772277832, "mix_mean": 0.5201646089553833, "mix_low": 0.049910545349121094, "mix_high": 0.010844866745173931, "read_cosine": 0.18632374703884125, "beta_mean": 0.0231814906001091, "nonfinite": 0.0, "weighted_memory_to_active": 0.12739069759845734, "mix_std": 0.2561018764972687, "beta_std": 0.023429682478308678, "beta_low": 0.7749735713005066, "beta_high": 0.0, "orthogonal_to_active": 0.05108684301376343, "output_influence": 0.4378672242164612, "residual_influence": 0.10989003628492355, "state_active": 0.023835966363549232, "state_memory": 0.0064591411501169205, "state_peak": 24.793373107910156, "growth_active": 1.0924667119979858, "growth_memory": 1.609074592590332, "heads": [{"head": 0, "memory_to_active": 0.38026052713394165, "contribution_to_active": 0.6104361414909363, "mix_mean": 0.6069917678833008, "mix_low": 0.0020208358764648438, "mix_high": 0.02244853973388672, "read_cosine": 0.2127162665128708, "beta_mean": 0.009091416373848915, "nonfinite": 0.0, "weighted_memory_to_active": 0.23265215754508972, "mix_std": 0.2983466684818268, "beta_std": 0.01069555152207613, "beta_low": 0.71441650390625, "beta_high": 0.0, "orthogonal_to_active": 0.17173151671886444, "evidence_weights": [-0.21592533588409424, -0.25098538398742676, 1.055406928062439], "agreement_weight": 0.11248742789030075, "orthogonal_weight": -0.7738473415374756}, {"head": 1, "memory_to_active": 0.00900140218436718, "contribution_to_active": 0.4637828767299652, "mix_mean": 0.5955960154533386, "mix_low": 0.0026149749755859375, "mix_high": 0.010999679565429688, "read_cosine": 0.1837441772222519, "beta_mean": 4.276435720385052e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005590252578258514, "mix_std": 0.2897663712501526, "beta_std": 9.712335304357111e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0045049795880913734, "evidence_weights": [-0.2730337083339691, -0.2198219895362854, 0.724962055683136], "agreement_weight": 0.618815541267395, "orthogonal_weight": -0.8814907073974609}, {"head": 2, "memory_to_active": 1.8548007011413574, "contribution_to_active": 0.2978111505508423, "mix_mean": 0.05833873897790909, "mix_low": 0.28629398345947266, "mix_high": 0.0, "read_cosine": 0.21472305059432983, "beta_mean": 0.12511242926120758, "nonfinite": 0.0, "weighted_memory_to_active": 0.3474728465080261, "mix_std": 0.1112496554851532, "beta_std": 0.11572291702032089, "beta_low": 0.0228271484375, "beta_high": 0.0, "orthogonal_to_active": 0.00014671924873255193, "evidence_weights": [-0.10172794759273529, -0.008199772797524929, -0.05114021897315979], "agreement_weight": -0.009461471810936928, "orthogonal_weight": 0.0005464541609399021}, {"head": 3, "memory_to_active": 0.01563863269984722, "contribution_to_active": 0.5235074758529663, "mix_mean": 0.600806474685669, "mix_low": 0.0039806365966796875, "mix_high": 0.008417129516601562, "read_cosine": 0.2923641800880432, "beta_mean": 7.798770093359053e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010426798835396767, "mix_std": 0.2840369641780853, "beta_std": 0.00016734062228351831, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024740034714341164, "evidence_weights": [-0.2609793543815613, -0.2206299602985382, 2.3131184577941895], "agreement_weight": 2.2853643894195557, "orthogonal_weight": -0.3095872700214386}, {"head": 4, "memory_to_active": 0.01249045878648758, "contribution_to_active": 0.38972365856170654, "mix_mean": 0.5340524315834045, "mix_low": 0.0022430419921875, "mix_high": 0.0027875900268554688, "read_cosine": 0.1252048760652542, "beta_mean": 5.96599820710253e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008370142430067062, "mix_std": 0.28550785779953003, "beta_std": 0.0001142170513048768, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007844705134630203, "evidence_weights": [-0.24314996600151062, -0.2236497700214386, 0.07299644500017166], "agreement_weight": -0.05788563936948776, "orthogonal_weight": -1.012654185295105}, {"head": 5, "memory_to_active": 0.21243974566459656, "contribution_to_active": 0.6512037515640259, "mix_mean": 0.7252024412155151, "mix_low": 0.0023097991943359375, "mix_high": 0.020416259765625, "read_cosine": 0.08918991684913635, "beta_mean": 0.004704685881733894, "nonfinite": 0.0, "weighted_memory_to_active": 0.15983203053474426, "mix_std": 0.2677035927772522, "beta_std": 0.013780934736132622, "beta_low": 0.91259765625, "beta_high": 0.0, "orthogonal_to_active": 0.11981910467147827, "evidence_weights": [-0.26045534014701843, -0.3438275158405304, 0.5467851161956787], "agreement_weight": 0.23983798921108246, "orthogonal_weight": -0.7790552973747253}]}, {"layer": 4, "memory_to_active": 0.9894604086875916, "contribution_to_active": 0.48908722400665283, "mix_mean": 0.3789108991622925, "mix_low": 0.2309127002954483, "mix_high": 0.012924989685416222, "read_cosine": 0.3006056845188141, "beta_mean": 0.10782894492149353, "nonfinite": 0.0, "weighted_memory_to_active": 0.1371181309223175, "mix_std": 0.16102243959903717, "beta_std": 0.10025562345981598, "beta_low": 0.4029236137866974, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.06085633486509323, "output_influence": 0.5112881064414978, "residual_influence": 0.15993137657642365, "state_active": 0.03505365550518036, "state_memory": 0.01620291732251644, "state_peak": 38.840267181396484, "growth_active": 1.1399325132369995, "growth_memory": 1.5063669681549072, "heads": [{"head": 0, "memory_to_active": 0.16547444462776184, "contribution_to_active": 0.8106870651245117, "mix_mean": 0.6969335675239563, "mix_low": 0.008280754089355469, "mix_high": 0.03607463836669922, "read_cosine": 0.44004207849502563, "beta_mean": 0.2500529885292053, "nonfinite": 0.0, "weighted_memory_to_active": 0.1422477662563324, "mix_std": 0.3439127802848816, "beta_std": 0.24457979202270508, "beta_low": 0.25408935546875, "beta_high": 0.0, "orthogonal_to_active": 0.10069821774959564, "evidence_weights": [-0.2501196265220642, -0.3607233464717865, -0.19569233059883118], "agreement_weight": -0.0035316545981913805, "orthogonal_weight": 0.8339738845825195}, {"head": 1, "memory_to_active": 0.04359154775738716, "contribution_to_active": 0.6705713272094727, "mix_mean": 0.7103489637374878, "mix_low": 7.43865966796875e-05, "mix_high": 0.02028656005859375, "read_cosine": 0.5638058185577393, "beta_mean": 0.0007424966315738857, "nonfinite": 0.0, "weighted_memory_to_active": 0.03307124227285385, "mix_std": 0.2357395887374878, "beta_std": 0.005681924521923065, "beta_low": 0.9825439453125, "beta_high": 0.0, "orthogonal_to_active": 0.017064647749066353, "evidence_weights": [-0.23710942268371582, -0.3659149706363678, 0.5785672664642334], "agreement_weight": 0.35564103722572327, "orthogonal_weight": -0.6983911395072937}, {"head": 2, "memory_to_active": 1.3738691806793213, "contribution_to_active": 0.3788822889328003, "mix_mean": 0.0546238087117672, "mix_low": 0.3273935317993164, "mix_high": 2.956390380859375e-05, "read_cosine": 0.20494267344474792, "beta_mean": 0.049871817231178284, "nonfinite": 0.0, "weighted_memory_to_active": 0.25788864493370056, "mix_std": 0.09777011722326279, "beta_std": 0.05760347470641136, "beta_low": 0.15814208984375, "beta_high": 0.0, "orthogonal_to_active": 0.09764454513788223, "evidence_weights": [0.0285818912088871, -0.059441789984703064, 0.35600781440734863], "agreement_weight": 0.13208962976932526, "orthogonal_weight": -0.48645639419555664}, {"head": 3, "memory_to_active": 1.9249012470245361, "contribution_to_active": 0.1481713056564331, "mix_mean": 0.03310605511069298, "mix_low": 0.3988494873046875, "mix_high": 0.0, "read_cosine": 0.24110719561576843, "beta_mean": 0.129638671875, "nonfinite": 0.0, "weighted_memory_to_active": 0.20029737055301666, "mix_std": 0.055450186133384705, "beta_std": 0.11890237033367157, "beta_low": 0.0225830078125, "beta_high": 0.0, "orthogonal_to_active": 0.002044533146545291, "evidence_weights": [-0.02230731211602688, 0.048435475677251816, -0.25396955013275146], "agreement_weight": 0.07858708500862122, "orthogonal_weight": 0.01483670249581337}, {"head": 4, "memory_to_active": 0.005981489550322294, "contribution_to_active": 0.6268436908721924, "mix_mean": 0.7646771669387817, "mix_low": 3.719329833984375e-05, "mix_high": 0.02115631103515625, "read_cosine": 0.13029927015304565, "beta_mean": 1.46190386658418e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004402961581945419, "mix_std": 0.2044478952884674, "beta_std": 4.09596468671225e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028435897547751665, "evidence_weights": [-0.331509530544281, -0.31961917877197266, 0.2764012813568115], "agreement_weight": 0.3081437945365906, "orthogonal_weight": -0.6796377301216125}, {"head": 5, "memory_to_active": 2.4229443073272705, "contribution_to_active": 0.2993675172328949, "mix_mean": 0.013775749132037163, "mix_low": 0.6508407592773438, "mix_high": 2.86102294921875e-06, "read_cosine": 0.22343719005584717, "beta_mean": 0.21665304899215698, "nonfinite": 0.0, "weighted_memory_to_active": 0.18480077385902405, "mix_std": 0.02881399542093277, "beta_std": 0.17472520470619202, "beta_low": 0.00018310546875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1448424607515335, "evidence_weights": [0.0592644065618515, -0.014207013882696629, -0.26845085620880127], "agreement_weight": -0.08523079007863998, "orthogonal_weight": 1.1049995422363281}]}, {"layer": 5, "memory_to_active": 0.39891231060028076, "contribution_to_active": 0.551262617111206, "mix_mean": 0.5208343267440796, "mix_low": 0.05516640469431877, "mix_high": 0.019220512360334396, "read_cosine": 0.09304627776145935, "beta_mean": 0.05153973773121834, "nonfinite": 0.0, "weighted_memory_to_active": 0.1341933161020279, "mix_std": 0.2512681484222412, "beta_std": 0.06781404465436935, "beta_low": 0.68426513671875, "beta_high": 0.0, "orthogonal_to_active": 0.05858500674366951, "output_influence": 0.5913362503051758, "residual_influence": 0.18986272811889648, "state_active": 0.03820726275444031, "state_memory": 0.00992156844586134, "state_peak": 61.80978775024414, "growth_active": 1.0994493961334229, "growth_memory": 1.2465510368347168, "heads": [{"head": 0, "memory_to_active": 0.2200346291065216, "contribution_to_active": 0.5915915369987488, "mix_mean": 0.49228981137275696, "mix_low": 0.0070285797119140625, "mix_high": 0.0015039443969726562, "read_cosine": -0.050285130739212036, "beta_mean": 0.019338196143507957, "nonfinite": 0.0, "weighted_memory_to_active": 0.081316739320755, "mix_std": 0.2756091356277466, "beta_std": 0.03622356802225113, "beta_low": 0.5775146484375, "beta_high": 0.0, "orthogonal_to_active": 0.03771626576781273, "evidence_weights": [-0.16926248371601105, -0.18838845193386078, 0.1100720539689064], "agreement_weight": -0.026056893169879913, "orthogonal_weight": -0.480538934469223}, {"head": 1, "memory_to_active": 0.007435212843120098, "contribution_to_active": 0.5565520524978638, "mix_mean": 0.7102231383323669, "mix_low": 0.0006284713745117188, "mix_high": 0.03632354736328125, "read_cosine": 0.1790846437215805, "beta_mean": 1.5589710528729483e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00480406079441309, "mix_std": 0.2647359371185303, "beta_std": 6.855494575574994e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004527569282799959, "evidence_weights": [-0.2461525797843933, -0.28493472933769226, 0.942409098148346], "agreement_weight": 0.7105259895324707, "orthogonal_weight": -0.9722762107849121}, {"head": 2, "memory_to_active": 0.01764368824660778, "contribution_to_active": 0.6201149821281433, "mix_mean": 0.6669495105743408, "mix_low": 0.0019712448120117188, "mix_high": 0.021796226501464844, "read_cosine": -0.4044404923915863, "beta_mean": 9.196636528940871e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01202116347849369, "mix_std": 0.2611309885978699, "beta_std": 0.0003643682284746319, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.011628636159002781, "evidence_weights": [-0.2584337294101715, -0.3162805438041687, 0.1696486920118332], "agreement_weight": -0.188050776720047, "orthogonal_weight": -1.045262098312378}, {"head": 3, "memory_to_active": 1.6426713466644287, "contribution_to_active": 0.25571203231811523, "mix_mean": 0.11441078782081604, "mix_low": 0.1214914321899414, "mix_high": 0.0, "read_cosine": 0.1412295252084732, "beta_mean": 0.0949864387512207, "nonfinite": 0.0, "weighted_memory_to_active": 0.3790970742702484, "mix_std": 0.14596876502037048, "beta_std": 0.14898037910461426, "beta_low": 0.22198486328125, "beta_high": 0.0, "orthogonal_to_active": 0.1532561033964157, "evidence_weights": [-0.07601942867040634, -0.09861966967582703, 0.10143189877271652], "agreement_weight": 0.03919399157166481, "orthogonal_weight": -0.4939919412136078}, {"head": 4, "memory_to_active": 0.42379409074783325, "contribution_to_active": 0.5822622776031494, "mix_mean": 0.32690325379371643, "mix_low": 0.19829463958740234, "mix_high": 0.004746437072753906, "read_cosine": 0.5399199724197388, "beta_mean": 0.194588303565979, "nonfinite": 0.0, "weighted_memory_to_active": 0.270040899515152, "mix_std": 0.34237074851989746, "beta_std": 0.220174640417099, "beta_low": 0.30853271484375, "beta_high": 0.0, "orthogonal_to_active": 0.11095715314149857, "evidence_weights": [-0.11375788599252701, -0.25222742557525635, -0.05046332627534866], "agreement_weight": 0.02828715182840824, "orthogonal_weight": 0.6295017004013062}, {"head": 5, "memory_to_active": 0.08189474791288376, "contribution_to_active": 0.7013425827026367, "mix_mean": 0.8142292499542236, "mix_low": 0.0015840530395507812, "mix_high": 0.050952911376953125, "read_cosine": 0.15276914834976196, "beta_mean": 0.00021788841695524752, "nonfinite": 0.0, "weighted_memory_to_active": 0.057879913598299026, "mix_std": 0.21779325604438782, "beta_std": 0.0010727294720709324, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.03342428803443909, "evidence_weights": [-0.3042300045490265, -0.32819902896881104, 1.8402718305587769], "agreement_weight": 0.529485285282135, "orthogonal_weight": -0.608028769493103}]}, {"layer": 6, "memory_to_active": 0.22806861996650696, "contribution_to_active": 0.6319372057914734, "mix_mean": 0.5960326194763184, "mix_low": 0.033627670258283615, "mix_high": 0.028960546478629112, "read_cosine": 0.31852760910987854, "beta_mean": 0.06356054544448853, "nonfinite": 0.0, "weighted_memory_to_active": 0.11751414835453033, "mix_std": 0.2410418689250946, "beta_std": 0.07419378310441971, "beta_low": 0.607330322265625, "beta_high": 0.0, "orthogonal_to_active": 0.07331569492816925, "output_influence": 0.5792678594589233, "residual_influence": 0.2160937488079071, "state_active": 0.04441394284367561, "state_memory": 0.00581115297973156, "state_peak": 82.33012390136719, "growth_active": 1.2437154054641724, "growth_memory": 1.168487548828125, "heads": [{"head": 0, "memory_to_active": 0.11250672489404678, "contribution_to_active": 0.7762184143066406, "mix_mean": 0.8239313364028931, "mix_low": 0.000125885009765625, "mix_high": 0.059981346130371094, "read_cosine": 0.32428380846977234, "beta_mean": 0.010925225913524628, "nonfinite": 0.0, "weighted_memory_to_active": 0.0979931578040123, "mix_std": 0.19850850105285645, "beta_std": 0.032721806317567825, "beta_low": 0.82757568359375, "beta_high": 0.0, "orthogonal_to_active": 0.06512009352445602, "evidence_weights": [-0.30013108253479004, -0.3475528657436371, 1.1838663816452026], "agreement_weight": 0.8839213252067566, "orthogonal_weight": -0.7207033634185791}, {"head": 1, "memory_to_active": 0.004599877167493105, "contribution_to_active": 0.5166215896606445, "mix_mean": 0.5982140302658081, "mix_low": 0.0008153915405273438, "mix_high": 0.016292572021484375, "read_cosine": 0.4445628821849823, "beta_mean": 1.0003140232583974e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021421590354293585, "mix_std": 0.28356632590293884, "beta_std": 1.7859420040622354e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011469380697235465, "evidence_weights": [-0.19119153916835785, -0.30117267370224, 1.1016268730163574], "agreement_weight": 0.12403023988008499, "orthogonal_weight": -0.703655481338501}, {"head": 2, "memory_to_active": 0.6318384408950806, "contribution_to_active": 0.11221513897180557, "mix_mean": 0.058900415897369385, "mix_low": 0.19463253021240234, "mix_high": 1.9073486328125e-06, "read_cosine": 0.27615541219711304, "beta_mean": 0.019955944269895554, "nonfinite": 0.0, "weighted_memory_to_active": 0.06924957782030106, "mix_std": 0.08922361582517624, "beta_std": 0.023704294115304947, "beta_low": 0.32574462890625, "beta_high": 0.0, "orthogonal_to_active": 0.012573421001434326, "evidence_weights": [0.05720166489481926, -0.07049649208784103, 0.3648642599582672], "agreement_weight": 0.14252422749996185, "orthogonal_weight": -0.21764858067035675}, {"head": 3, "memory_to_active": 0.13306204974651337, "contribution_to_active": 0.6903071403503418, "mix_mean": 0.6674196124076843, "mix_low": 0.004818916320800781, "mix_high": 0.019791603088378906, "read_cosine": 0.4739428460597992, "beta_mean": 0.21839196979999542, "nonfinite": 0.0, "weighted_memory_to_active": 0.09737163037061691, "mix_std": 0.32448193430900574, "beta_std": 0.24625016748905182, "beta_low": 0.18756103515625, "beta_high": 0.0, "orthogonal_to_active": 0.05770110711455345, "evidence_weights": [-0.1671084463596344, -0.3151661455631256, 0.10940100997686386], "agreement_weight": -0.009240678511559963, "orthogonal_weight": 0.6843609809875488}, {"head": 4, "memory_to_active": 0.45977872610092163, "contribution_to_active": 1.0613930225372314, "mix_mean": 0.7353098392486572, "mix_low": 0.001178741455078125, "mix_high": 0.06813526153564453, "read_cosine": 0.11773207783699036, "beta_mean": 0.13195762038230896, "nonfinite": 0.0, "weighted_memory_to_active": 0.41904303431510925, "mix_std": 0.30848878622055054, "beta_std": 0.14220662415027618, "beta_low": 0.30316162109375, "beta_high": 0.0, "orthogonal_to_active": 0.2844088673591614, "evidence_weights": [-0.1680581271648407, -0.5161989331245422, -0.7278012037277222], "agreement_weight": -0.014375199563801289, "orthogonal_weight": 0.6990700364112854}, {"head": 5, "memory_to_active": 0.026625890284776688, "contribution_to_active": 0.6348679065704346, "mix_mean": 0.692420482635498, "mix_low": 0.000194549560546875, "mix_high": 0.009560585021972656, "read_cosine": 0.2744885981082916, "beta_mean": 0.0001225121959578246, "nonfinite": 0.0, "weighted_memory_to_active": 0.019285306334495544, "mix_std": 0.24198202788829803, "beta_std": 0.0002619359875097871, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.018943708389997482, "evidence_weights": [-0.23179581761360168, -0.3141458332538605, 0.3603704869747162], "agreement_weight": 0.2215907871723175, "orthogonal_weight": -1.051166296005249}]}, {"layer": 7, "memory_to_active": 1.8182544708251953, "contribution_to_active": 0.27584022283554077, "mix_mean": 0.17744086682796478, "mix_low": 0.21863000094890594, "mix_high": 0.0009501775493845344, "read_cosine": 0.2341175377368927, "beta_mean": 0.14726495742797852, "nonfinite": 0.0, "weighted_memory_to_active": 0.21281595528125763, "mix_std": 0.14841920137405396, "beta_std": 0.10325626283884048, "beta_low": 0.2184346616268158, "beta_high": 0.000274658203125, "orthogonal_to_active": 0.09352463483810425, "output_influence": 0.2911349833011627, "residual_influence": 0.07218807935714722, "state_active": 0.013892017304897308, "state_memory": 0.02377479150891304, "state_peak": 32.0153694152832, "growth_active": 1.1714177131652832, "growth_memory": 1.279788613319397, "heads": [{"head": 0, "memory_to_active": 2.798804998397827, "contribution_to_active": 0.18025723099708557, "mix_mean": 0.04066911339759827, "mix_low": 0.3959932327270508, "mix_high": 0.0, "read_cosine": 0.3406546115875244, "beta_mean": 0.2188722938299179, "nonfinite": 0.0, "weighted_memory_to_active": 0.20291085541248322, "mix_std": 0.08019152283668518, "beta_std": 0.1564890444278717, "beta_low": 0.00079345703125, "beta_high": 0.0006103515625, "orthogonal_to_active": 0.0020871958695352077, "evidence_weights": [0.04345990717411041, -0.09034424275159836, 0.01839449815452099], "agreement_weight": -0.030469367280602455, "orthogonal_weight": -0.01156663615256548}, {"head": 1, "memory_to_active": 1.017877459526062, "contribution_to_active": 0.4043259620666504, "mix_mean": 0.22281895577907562, "mix_low": 0.10861492156982422, "mix_high": 0.004975318908691406, "read_cosine": 0.3973538875579834, "beta_mean": 0.10545822978019714, "nonfinite": 0.0, "weighted_memory_to_active": 0.3508799076080322, "mix_std": 0.2435726523399353, "beta_std": 0.06752018630504608, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05095614492893219, "evidence_weights": [-0.2163795530796051, -0.10404586046934128, -0.30892816185951233], "agreement_weight": 0.0547151118516922, "orthogonal_weight": 0.1752641648054123}, {"head": 2, "memory_to_active": 0.5928019285202026, "contribution_to_active": 0.3932752013206482, "mix_mean": 0.24256931245326996, "mix_low": 0.014865875244140625, "mix_high": 0.0004405975341796875, "read_cosine": 0.10817708820104599, "beta_mean": 0.033254608511924744, "nonfinite": 0.0, "weighted_memory_to_active": 0.2449539452791214, "mix_std": 0.21582327783107758, "beta_std": 0.037058137357234955, "beta_low": 0.1910400390625, "beta_high": 0.0, "orthogonal_to_active": 0.14516502618789673, "evidence_weights": [-0.09725338965654373, -0.18496833741664886, 0.8833060264587402], "agreement_weight": 0.012486686930060387, "orthogonal_weight": -0.6115626096725464}, {"head": 3, "memory_to_active": 4.904134273529053, "contribution_to_active": 0.09490195661783218, "mix_mean": 0.03457674756646156, "mix_low": 0.3173389434814453, "mix_high": 0.0, "read_cosine": 0.09136487543582916, "beta_mean": 0.45352956652641296, "nonfinite": 0.0, "weighted_memory_to_active": 0.35957247018814087, "mix_std": 0.049714118242263794, "beta_std": 0.28016287088394165, "beta_low": 0.00177001953125, "beta_high": 0.00103759765625, "orthogonal_to_active": 0.3174594044685364, "evidence_weights": [0.07772646844387054, -0.04045937582850456, 0.718753457069397], "agreement_weight": 0.051629211753606796, "orthogonal_weight": -0.941747784614563}, {"head": 4, "memory_to_active": 1.540212631225586, "contribution_to_active": 0.11701641231775284, "mix_mean": 0.027023665606975555, "mix_low": 0.4733390808105469, "mix_high": 0.0, "read_cosine": 0.22743435204029083, "beta_mean": 0.07226260006427765, "nonfinite": 0.0, "weighted_memory_to_active": 0.09035329520702362, "mix_std": 0.05186131224036217, "beta_std": 0.07777124643325806, "beta_low": 0.1175537109375, "beta_high": 0.0, "orthogonal_to_active": 0.01837792806327343, "evidence_weights": [0.11549022793769836, -0.030365029349923134, 0.1885700225830078], "agreement_weight": 0.034553658217191696, "orthogonal_weight": -0.29129692912101746}, {"head": 5, "memory_to_active": 0.05569423735141754, "contribution_to_active": 0.4652646481990814, "mix_mean": 0.49698731303215027, "mix_low": 0.0016279220581054688, "mix_high": 0.00028514862060546875, "read_cosine": 0.23972028493881226, "beta_mean": 0.0002124147431459278, "nonfinite": 0.0, "weighted_memory_to_active": 0.02822517603635788, "mix_std": 0.24935221672058105, "beta_std": 0.0005360802751965821, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.027102064341306686, "evidence_weights": [-0.183430016040802, -0.23989462852478027, 0.4762469530105591], "agreement_weight": 0.2357284277677536, "orthogonal_weight": -1.012558937072754}]}, {"layer": 8, "memory_to_active": 0.14040228724479675, "contribution_to_active": 0.5716636180877686, "mix_mean": 0.5091686248779297, "mix_low": 0.010516008362174034, "mix_high": 0.0021478335838764906, "read_cosine": 0.40244758129119873, "beta_mean": 0.013964151963591576, "nonfinite": 0.0, "weighted_memory_to_active": 0.071775421500206, "mix_std": 0.24285884201526642, "beta_std": 0.019756777212023735, "beta_low": 0.77972412109375, "beta_high": 0.0, "orthogonal_to_active": 0.06789388507604599, "output_influence": 0.5293077230453491, "residual_influence": 0.1978732943534851, "state_active": 0.043141286820173264, "state_memory": 0.00498354434967041, "state_peak": 57.92400360107422, "growth_active": 1.0973316431045532, "growth_memory": 1.042953610420227, "heads": [{"head": 0, "memory_to_active": 0.06028217449784279, "contribution_to_active": 0.5753132700920105, "mix_mean": 0.6121706366539001, "mix_low": 0.0005168914794921875, "mix_high": 0.0018301010131835938, "read_cosine": 0.46520015597343445, "beta_mean": 0.0032053636386990547, "nonfinite": 0.0, "weighted_memory_to_active": 0.043776318430900574, "mix_std": 0.24932852387428284, "beta_std": 0.01272978913038969, "beta_low": 0.93115234375, "beta_high": 0.0, "orthogonal_to_active": 0.027100153267383575, "evidence_weights": [-0.1560918390750885, -0.37841710448265076, 0.5627909302711487], "agreement_weight": 0.2672275900840759, "orthogonal_weight": -0.7932679653167725}, {"head": 1, "memory_to_active": 0.07634847611188889, "contribution_to_active": 0.554894745349884, "mix_mean": 0.5371049642562866, "mix_low": 0.00011539459228515625, "mix_high": 0.0020380020141601562, "read_cosine": 0.11636290699243546, "beta_mean": 0.0009775834623724222, "nonfinite": 0.0, "weighted_memory_to_active": 0.03477589040994644, "mix_std": 0.23369084298610687, "beta_std": 0.007597545627504587, "beta_low": 0.9915771484375, "beta_high": 0.0, "orthogonal_to_active": 0.02555030956864357, "evidence_weights": [-0.19488371908664703, -0.27498865127563477, 0.5919130444526672], "agreement_weight": 0.07571988552808762, "orthogonal_weight": -0.7735257744789124}, {"head": 2, "memory_to_active": 0.01095221284776926, "contribution_to_active": 0.6935305595397949, "mix_mean": 0.6912270784378052, "mix_low": 0.00031375885009765625, "mix_high": 0.0038480758666992188, "read_cosine": 0.42051321268081665, "beta_mean": 0.00012021262955386192, "nonfinite": 0.0, "weighted_memory_to_active": 0.008710967376828194, "mix_std": 0.2198060154914856, "beta_std": 0.003501230152323842, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005033719819039106, "evidence_weights": [-0.220791757106781, -0.3455585837364197, 0.9708629250526428], "agreement_weight": 0.3289280831813812, "orthogonal_weight": -0.6719902753829956}, {"head": 3, "memory_to_active": 0.10395834594964981, "contribution_to_active": 0.6326731443405151, "mix_mean": 0.590793251991272, "mix_low": 0.000652313232421875, "mix_high": 0.0049114227294921875, "read_cosine": 0.783922553062439, "beta_mean": 0.007865586318075657, "nonfinite": 0.0, "weighted_memory_to_active": 0.0795879140496254, "mix_std": 0.25596192479133606, "beta_std": 0.007845893502235413, "beta_low": 0.73028564453125, "beta_high": 0.0, "orthogonal_to_active": 0.025250786915421486, "evidence_weights": [-0.1716688722372055, -0.29660966992378235, 0.5248873829841614], "agreement_weight": 0.260141521692276, "orthogonal_weight": -0.5958515405654907}, {"head": 4, "memory_to_active": 0.21696597337722778, "contribution_to_active": 0.3404756784439087, "mix_mean": 0.3561131954193115, "mix_low": 0.001537322998046875, "mix_high": 2.288818359375e-05, "read_cosine": 0.12448272854089737, "beta_mean": 0.0007750070653855801, "nonfinite": 0.0, "weighted_memory_to_active": 0.07357850670814514, "mix_std": 0.23136578500270844, "beta_std": 0.011986392550170422, "beta_low": 0.99169921875, "beta_high": 0.0, "orthogonal_to_active": 0.013153744861483574, "evidence_weights": [-0.14212065935134888, -0.2003730684518814, 0.6706159114837646], "agreement_weight": 0.04901959002017975, "orthogonal_weight": -0.1809796243906021}, {"head": 5, "memory_to_active": 0.3739064633846283, "contribution_to_active": 0.6330945491790771, "mix_mean": 0.267602801322937, "mix_low": 0.059960365295410156, "mix_high": 0.00023651123046875, "read_cosine": 0.5042037963867188, "beta_mean": 0.07084115594625473, "nonfinite": 0.0, "weighted_memory_to_active": 0.1902228742837906, "mix_std": 0.26699987053871155, "beta_std": 0.07487981021404266, "beta_low": 0.03369140625, "beta_high": 0.0, "orthogonal_to_active": 0.31127458810806274, "evidence_weights": [-0.15076987445354462, -0.14142021536827087, -0.11469889432191849], "agreement_weight": -0.008702373132109642, "orthogonal_weight": 1.9452764987945557}]}, {"layer": 9, "memory_to_active": 0.8346102237701416, "contribution_to_active": 0.7101132869720459, "mix_mean": 0.39933112263679504, "mix_low": 0.05186907574534416, "mix_high": 0.006392638199031353, "read_cosine": 0.3506166338920593, "beta_mean": 0.032795339822769165, "nonfinite": 0.0, "weighted_memory_to_active": 0.36788585782051086, "mix_std": 0.24686148762702942, "beta_std": 0.02782350778579712, "beta_low": 0.6529337763786316, "beta_high": 0.0, "orthogonal_to_active": 0.03255307674407959, "output_influence": 0.5609090328216553, "residual_influence": 0.1731184720993042, "state_active": 0.046553835272789, "state_memory": 0.014141532592475414, "state_peak": 60.88182067871094, "growth_active": 1.201781988143921, "growth_memory": 1.275833249092102, "heads": [{"head": 0, "memory_to_active": 0.00031373428646475077, "contribution_to_active": 0.5786645412445068, "mix_mean": 0.6578488945960999, "mix_low": 0.0012521743774414062, "mix_high": 0.02634143829345703, "read_cosine": 0.3332153558731079, "beta_mean": 7.492404279219045e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.00020734380814246833, "mix_std": 0.2851915657520294, "beta_std": 1.9984317987109534e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0002097895194310695, "evidence_weights": [-0.30296871066093445, -0.30903491377830505, 0.37195801734924316], "agreement_weight": 0.10088715702295303, "orthogonal_weight": -1.0706262588500977}, {"head": 1, "memory_to_active": 0.015317237004637718, "contribution_to_active": 0.708755373954773, "mix_mean": 0.5831912755966187, "mix_low": 0.006548881530761719, "mix_high": 0.0026836395263671875, "read_cosine": 0.4310449957847595, "beta_mean": 3.46188826370053e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011544892564415932, "mix_std": 0.2766706943511963, "beta_std": 7.600274693686515e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007536631543189287, "evidence_weights": [-0.23586909472942352, -0.29286789894104004, 0.9643425941467285], "agreement_weight": 0.5528230667114258, "orthogonal_weight": -0.8021700978279114}, {"head": 2, "memory_to_active": 0.16481676697731018, "contribution_to_active": 0.6632711887359619, "mix_mean": 0.6105550527572632, "mix_low": 0.0014810562133789062, "mix_high": 0.0072650909423828125, "read_cosine": 0.40739190578460693, "beta_mean": 0.0006674775504507124, "nonfinite": 0.0, "weighted_memory_to_active": 0.11531411856412888, "mix_std": 0.2628978490829468, "beta_std": 0.004483934957534075, "beta_low": 0.99609375, "beta_high": 0.0, "orthogonal_to_active": 0.06457239389419556, "evidence_weights": [-0.20272937417030334, -0.27847006916999817, 0.7466747760772705], "agreement_weight": 0.35036465525627136, "orthogonal_weight": -0.674966037273407}, {"head": 3, "memory_to_active": 0.7198419570922852, "contribution_to_active": 0.2737817168235779, "mix_mean": 0.17661218345165253, "mix_low": 0.08307838439941406, "mix_high": 6.29425048828125e-05, "read_cosine": 0.36612290143966675, "beta_mean": 0.02655264362692833, "nonfinite": 0.0, "weighted_memory_to_active": 0.18506455421447754, "mix_std": 0.21017713844776154, "beta_std": 0.028737030923366547, "beta_low": 0.2529296875, "beta_high": 0.0, "orthogonal_to_active": 0.08898869156837463, "evidence_weights": [-0.039269715547561646, -0.11023207753896713, 0.48894956707954407], "agreement_weight": 0.09329728782176971, "orthogonal_weight": -0.525533139705658}, {"head": 4, "memory_to_active": 3.090306282043457, "contribution_to_active": 1.6999908685684204, "mix_mean": 0.15030528604984283, "mix_low": 0.11884593963623047, "mix_high": 0.0013399124145507812, "read_cosine": 0.11941330134868622, "beta_mean": 0.14861798286437988, "nonfinite": 0.0, "weighted_memory_to_active": 1.6249659061431885, "mix_std": 0.21130605041980743, "beta_std": 0.08540630340576172, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.013336778618395329, "evidence_weights": [-0.07952089607715607, -0.15033937990665436, -0.05348122864961624], "agreement_weight": -0.06601901352405548, "orthogonal_weight": -0.008580753579735756}, {"head": 5, "memory_to_active": 1.0170657634735107, "contribution_to_active": 0.3362157940864563, "mix_mean": 0.21747387945652008, "mix_low": 0.10000801086425781, "mix_high": 0.0006628036499023438, "read_cosine": 0.44651108980178833, "beta_mean": 0.020898547023534775, "nonfinite": 0.0, "weighted_memory_to_active": 0.2702183127403259, "mix_std": 0.23492556810379028, "beta_std": 0.04823576658964157, "beta_low": 0.6685791015625, "beta_high": 0.0, "orthogonal_to_active": 0.020674165338277817, "evidence_weights": [0.04203519970178604, -0.19063302874565125, 0.14183175563812256], "agreement_weight": 0.002651556860655546, "orthogonal_weight": 0.08991053700447083}]}, {"layer": 10, "memory_to_active": 0.19376979768276215, "contribution_to_active": 0.7147162556648254, "mix_mean": 0.5259557962417603, "mix_low": 0.02325550839304924, "mix_high": 0.004816214554011822, "read_cosine": 0.3800480365753174, "beta_mean": 0.04576058313250542, "nonfinite": 0.0, "weighted_memory_to_active": 0.08336705714464188, "mix_std": 0.2655826210975647, "beta_std": 0.0459805428981781, "beta_low": 0.6564432978630066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.25409406423568726, "output_influence": 0.4742831289768219, "residual_influence": 0.171846404671669, "state_active": 0.056994322687387466, "state_memory": 0.01446776557713747, "state_peak": 127.28933715820312, "growth_active": 1.031875729560852, "growth_memory": 1.0195190906524658, "heads": [{"head": 0, "memory_to_active": 0.6606247425079346, "contribution_to_active": 1.9524970054626465, "mix_mean": 0.5556254386901855, "mix_low": 0.10943031311035156, "mix_high": 0.004755973815917969, "read_cosine": 0.09270511567592621, "beta_mean": 0.19816195964813232, "nonfinite": 0.0, "weighted_memory_to_active": 0.3836488723754883, "mix_std": 0.3900771141052246, "beta_std": 0.2070600688457489, "beta_low": 0.1484375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.4428794384002686, "evidence_weights": [-0.08401014655828476, -0.3668059706687927, -1.9193940162658691], "agreement_weight": 0.04057539626955986, "orthogonal_weight": 3.7957401275634766}, {"head": 1, "memory_to_active": 0.007873302325606346, "contribution_to_active": 0.6247156858444214, "mix_mean": 0.6379066705703735, "mix_low": 0.0010356903076171875, "mix_high": 0.01654529571533203, "read_cosine": 0.3180478811264038, "beta_mean": 5.352219159249216e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004534482024610043, "mix_std": 0.2637249827384949, "beta_std": 9.83346690190956e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007077377289533615, "evidence_weights": [-0.2483636438846588, -0.19207340478897095, 0.5470715761184692], "agreement_weight": 0.6598898768424988, "orthogonal_weight": -1.6908621788024902}, {"head": 2, "memory_to_active": 0.015653003007173538, "contribution_to_active": 0.6263760328292847, "mix_mean": 0.6717132925987244, "mix_low": 0.0001888275146484375, "mix_high": 0.005972862243652344, "read_cosine": 0.019171694293618202, "beta_mean": 3.0141060051391833e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006880008615553379, "mix_std": 0.2475859820842743, "beta_std": 0.0001389241951983422, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002820858731865883, "evidence_weights": [-0.26852303743362427, -0.2778645157814026, 0.6937289237976074], "agreement_weight": -0.2076406031847, "orthogonal_weight": -0.3934576213359833}, {"head": 3, "memory_to_active": 0.34023317694664, "contribution_to_active": 0.09868249297142029, "mix_mean": 0.28027376532554626, "mix_low": 0.026717185974121094, "mix_high": 6.771087646484375e-05, "read_cosine": 0.9269890785217285, "beta_mean": 0.06886470317840576, "nonfinite": 0.0, "weighted_memory_to_active": 0.038824956864118576, "mix_std": 0.22317001223564148, "beta_std": 0.059166572988033295, "beta_low": 0.00177001953125, "beta_high": 0.0, "orthogonal_to_active": 0.03868424519896507, "evidence_weights": [-0.08867713809013367, -0.14112155139446259, 0.15905950963497162], "agreement_weight": 0.02191407047212124, "orthogonal_weight": 1.4610774517059326}, {"head": 4, "memory_to_active": 0.014274412766098976, "contribution_to_active": 0.5722523331642151, "mix_mean": 0.5980271697044373, "mix_low": 0.0002460479736328125, "mix_high": 0.00154876708984375, "read_cosine": 0.08685387670993805, "beta_mean": 3.601940261432901e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00848989188671112, "mix_std": 0.24326634407043457, "beta_std": 6.323603156488389e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.012352535501122475, "evidence_weights": [-0.18338170647621155, -0.2523309290409088, 0.9440561532974243], "agreement_weight": 0.18833863735198975, "orthogonal_weight": -1.5276174545288086}, {"head": 5, "memory_to_active": 0.12396016716957092, "contribution_to_active": 0.41377389430999756, "mix_mean": 0.4121880829334259, "mix_low": 0.00191497802734375, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8365204334259033, "beta_mean": 0.00741715868934989, "nonfinite": 0.0, "weighted_memory_to_active": 0.05782408267259598, "mix_std": 0.22567126154899597, "beta_std": 0.00935608521103859, "beta_low": 0.7884521484375, "beta_high": 0.0, "orthogonal_to_active": 0.020749663934111595, "evidence_weights": [-0.13753965497016907, -0.16505545377731323, 0.2755027711391449], "agreement_weight": 0.09606598317623138, "orthogonal_weight": -0.6687542796134949}]}, {"layer": 11, "memory_to_active": 0.07692153006792068, "contribution_to_active": 0.6261880397796631, "mix_mean": 0.5272711515426636, "mix_low": 0.11262258887290955, "mix_high": 0.04516410827636719, "read_cosine": 0.33383819460868835, "beta_mean": 0.02535005286335945, "nonfinite": 0.0, "weighted_memory_to_active": 0.042007993906736374, "mix_std": 0.32279351353645325, "beta_std": 0.03701112046837807, "beta_low": 0.699462890625, "beta_high": 0.0, "orthogonal_to_active": 0.045697472989559174, "output_influence": 0.5612784028053284, "residual_influence": 0.16095495223999023, "state_active": 0.21208113431930542, "state_memory": 0.0032272597309201956, "state_peak": 37.244659423828125, "growth_active": 1.1358697414398193, "growth_memory": 1.4819856882095337, "heads": [{"head": 0, "memory_to_active": 0.005875404924154282, "contribution_to_active": 0.6143444776535034, "mix_mean": 0.6610523462295532, "mix_low": 0.0006198883056640625, "mix_high": 0.008617401123046875, "read_cosine": 0.26905372738838196, "beta_mean": 8.20606728666462e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004336056299507618, "mix_std": 0.25188523530960083, "beta_std": 0.00357613293454051, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.003598289331421256, "evidence_weights": [-0.17635908722877502, -0.35760506987571716, 0.8168210387229919], "agreement_weight": 0.40233486890792847, "orthogonal_weight": -0.8609232902526855}, {"head": 1, "memory_to_active": 0.1272704303264618, "contribution_to_active": 0.44352084398269653, "mix_mean": 0.47451069951057434, "mix_low": 0.004273414611816406, "mix_high": 0.0005950927734375, "read_cosine": 0.480286180973053, "beta_mean": 0.003215379547327757, "nonfinite": 0.0, "weighted_memory_to_active": 0.06425352394580841, "mix_std": 0.26910966634750366, "beta_std": 0.011956790462136269, "beta_low": 0.935546875, "beta_high": 0.0, "orthogonal_to_active": 0.03549306094646454, "evidence_weights": [-0.1381700485944748, -0.27782294154167175, 0.5980743765830994], "agreement_weight": 0.086578369140625, "orthogonal_weight": -0.6601399183273315}, {"head": 2, "memory_to_active": 0.0049736276268959045, "contribution_to_active": 0.9870521426200867, "mix_mean": 0.4351997375488281, "mix_low": 0.32140159606933594, "mix_high": 0.17262554168701172, "read_cosine": 0.16002613306045532, "beta_mean": 0.06829231232404709, "nonfinite": 0.0, "weighted_memory_to_active": 0.003523025196045637, "mix_std": 0.45148444175720215, "beta_std": 0.09080637991428375, "beta_low": 0.48858642578125, "beta_high": 0.0, "orthogonal_to_active": 0.036443933844566345, "evidence_weights": [0.045228227972984314, -0.3385143280029297, -1.3225116729736328], "agreement_weight": 0.0035931190941482782, "orthogonal_weight": 10.74072265625}, {"head": 3, "memory_to_active": 0.008425087668001652, "contribution_to_active": 0.8449766635894775, "mix_mean": 0.4098522961139679, "mix_low": 0.341461181640625, "mix_high": 0.08123397827148438, "read_cosine": 0.5771549940109253, "beta_mean": 0.05421406775712967, "nonfinite": 0.0, "weighted_memory_to_active": 0.005177753046154976, "mix_std": 0.4389621615409851, "beta_std": 0.08199163526296616, "beta_low": 0.52685546875, "beta_high": 0.0, "orthogonal_to_active": 0.03983645886182785, "evidence_weights": [0.22014406323432922, -0.27635079622268677, -0.09027083963155746], "agreement_weight": 0.05430357903242111, "orthogonal_weight": 11.007065773010254}, {"head": 4, "memory_to_active": 0.2271336317062378, "contribution_to_active": 0.455534428358078, "mix_mean": 0.6097092628479004, "mix_low": 0.006676673889160156, "mix_high": 0.00179290771484375, "read_cosine": 0.012015918269753456, "beta_mean": 0.017319079488515854, "nonfinite": 0.0, "weighted_memory_to_active": 0.12736593186855316, "mix_std": 0.26954108476638794, "beta_std": 0.020366720855236053, "beta_low": 0.47332763671875, "beta_high": 0.0, "orthogonal_to_active": 0.12576670944690704, "evidence_weights": [-0.23376113176345825, -0.24099360406398773, 1.879500150680542], "agreement_weight": -0.22394967079162598, "orthogonal_weight": -0.9993577003479004}, {"head": 5, "memory_to_active": 0.08785102516412735, "contribution_to_active": 0.41169971227645874, "mix_mean": 0.5733024477958679, "mix_low": 0.0013027191162109375, "mix_high": 0.006119728088378906, "read_cosine": 0.5044921636581421, "beta_mean": 0.00897741038352251, "nonfinite": 0.0, "weighted_memory_to_active": 0.04739166051149368, "mix_std": 0.25577834248542786, "beta_std": 0.013369069434702396, "beta_low": 0.77294921875, "beta_high": 0.0, "orthogonal_to_active": 0.033046379685401917, "evidence_weights": [-0.19909541308879852, -0.3117334544658661, 0.6652124524116516], "agreement_weight": 0.08268541842699051, "orthogonal_weight": -0.8673705458641052}]}, {"layer": 12, "memory_to_active": 0.2087727189064026, "contribution_to_active": 0.5840461850166321, "mix_mean": 0.5495045185089111, "mix_low": 0.008492152206599712, "mix_high": 0.0066699986346066, "read_cosine": 0.22947336733341217, "beta_mean": 0.01599709317088127, "nonfinite": 0.0, "weighted_memory_to_active": 0.10583676397800446, "mix_std": 0.2571994662284851, "beta_std": 0.015164612792432308, "beta_low": 0.8271281123161316, "beta_high": 0.0, "orthogonal_to_active": 0.06557707488536835, "output_influence": 0.5160003900527954, "residual_influence": 0.15434396266937256, "state_active": 0.03590202331542969, "state_memory": 0.008031642064452171, "state_peak": 76.31436157226562, "growth_active": 1.17264723777771, "growth_memory": 1.1684170961380005, "heads": [{"head": 0, "memory_to_active": 0.04137514531612396, "contribution_to_active": 0.6223036050796509, "mix_mean": 0.5904794931411743, "mix_low": 0.0006818771362304688, "mix_high": 0.007060050964355469, "read_cosine": 0.27060258388519287, "beta_mean": 0.0008209858788177371, "nonfinite": 0.0, "weighted_memory_to_active": 0.02355196699500084, "mix_std": 0.2629784941673279, "beta_std": 0.0019297418184578419, "beta_low": 0.99102783203125, "beta_high": 0.0, "orthogonal_to_active": 0.02180319093167782, "evidence_weights": [-0.20324601233005524, -0.2711426019668579, -0.06242357939481735], "agreement_weight": 0.10600409656763077, "orthogonal_weight": -0.9860966801643372}, {"head": 1, "memory_to_active": 0.09378036111593246, "contribution_to_active": 0.5699021816253662, "mix_mean": 0.5712935924530029, "mix_low": 0.00029850006103515625, "mix_high": 0.0021114349365234375, "read_cosine": 0.3996509313583374, "beta_mean": 0.0007473843870684505, "nonfinite": 0.0, "weighted_memory_to_active": 0.05776183679699898, "mix_std": 0.26296117901802063, "beta_std": 0.009640170261263847, "beta_low": 0.99371337890625, "beta_high": 0.0, "orthogonal_to_active": 0.04799406975507736, "evidence_weights": [-0.19985735416412354, -0.27250954508781433, 1.0415260791778564], "agreement_weight": 0.27983543276786804, "orthogonal_weight": -0.9160767197608948}, {"head": 2, "memory_to_active": 0.5869473218917847, "contribution_to_active": 0.47169655561447144, "mix_mean": 0.3059583902359009, "mix_low": 0.0278472900390625, "mix_high": 6.389617919921875e-05, "read_cosine": 0.5706583261489868, "beta_mean": 0.09321114420890808, "nonfinite": 0.0, "weighted_memory_to_active": 0.22243431210517883, "mix_std": 0.23525601625442505, "beta_std": 0.06285837292671204, "beta_low": 0.0006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.25124630331993103, "evidence_weights": [-0.20218297839164734, -0.05908916890621185, -0.030111124739050865], "agreement_weight": 0.0278666652739048, "orthogonal_weight": 1.4296091794967651}, {"head": 3, "memory_to_active": 0.0318790003657341, "contribution_to_active": 0.7523733377456665, "mix_mean": 0.7238309383392334, "mix_low": 9.918212890625e-05, "mix_high": 0.019266128540039062, "read_cosine": 0.08581647276878357, "beta_mean": 0.00029332871781662107, "nonfinite": 0.0, "weighted_memory_to_active": 0.022593088448047638, "mix_std": 0.22269414365291595, "beta_std": 0.005974302999675274, "beta_low": 0.99786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.021302126348018646, "evidence_weights": [-0.2866707146167755, -0.333522230386734, 1.669190764427185], "agreement_weight": 0.6132023334503174, "orthogonal_weight": -0.9639832377433777}, {"head": 4, "memory_to_active": 0.4492511451244354, "contribution_to_active": 0.5828509330749512, "mix_mean": 0.46416544914245605, "mix_low": 0.016234397888183594, "mix_high": 0.0051059722900390625, "read_cosine": -0.1866043508052826, "beta_mean": 0.0006736753857694566, "nonfinite": 0.0, "weighted_memory_to_active": 0.2763051688671112, "mix_std": 0.28911858797073364, "beta_std": 0.008169458247721195, "beta_low": 0.98272705078125, "beta_high": 0.0, "orthogonal_to_active": 0.02678517997264862, "evidence_weights": [-0.15578582882881165, -0.21395061910152435, -0.14886760711669922], "agreement_weight": 0.015850979834794998, "orthogonal_weight": -0.10052364319562912}, {"head": 5, "memory_to_active": 0.04940337687730789, "contribution_to_active": 0.505150556564331, "mix_mean": 0.6412989497184753, "mix_low": 0.005791664123535156, "mix_high": 0.006412506103515625, "read_cosine": 0.23671618103981018, "beta_mean": 0.00023603143927175552, "nonfinite": 0.0, "weighted_memory_to_active": 0.032374151051044464, "mix_std": 0.27018848061561584, "beta_std": 0.0024156279396265745, "beta_low": 0.996826171875, "beta_high": 0.0, "orthogonal_to_active": 0.024331573396921158, "evidence_weights": [-0.22477629780769348, -0.3161408007144928, 1.235995888710022], "agreement_weight": 0.42841267585754395, "orthogonal_weight": -0.8076109290122986}]}, {"layer": 13, "memory_to_active": 0.11381904780864716, "contribution_to_active": 0.5436422824859619, "mix_mean": 0.5459993481636047, "mix_low": 0.005819797981530428, "mix_high": 0.009331703186035156, "read_cosine": 0.41658562421798706, "beta_mean": 0.014697759412229061, "nonfinite": 0.0, "weighted_memory_to_active": 0.04798576235771179, "mix_std": 0.25770604610443115, "beta_std": 0.014305810444056988, "beta_low": 0.8112183213233948, "beta_high": 0.0, "orthogonal_to_active": 0.04898618534207344, "output_influence": 0.5346580147743225, "residual_influence": 0.193684384226799, "state_active": 0.13322347402572632, "state_memory": 0.042701929807662964, "state_peak": 99.93223571777344, "growth_active": 0.914186954498291, "growth_memory": 0.9607622027397156, "heads": [{"head": 0, "memory_to_active": 0.010812914930284023, "contribution_to_active": 0.6106258034706116, "mix_mean": 0.6192666292190552, "mix_low": 0.0007915496826171875, "mix_high": 0.0055828094482421875, "read_cosine": 0.4556678533554077, "beta_mean": 3.8650578062515706e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0071192095056176186, "mix_std": 0.26538270711898804, "beta_std": 8.012079342734069e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006718446500599384, "evidence_weights": [-0.18898607790470123, -0.26343461871147156, 0.7243868708610535], "agreement_weight": 0.2282436490058899, "orthogonal_weight": -1.0663707256317139}, {"head": 1, "memory_to_active": 0.01000154297798872, "contribution_to_active": 0.6106572151184082, "mix_mean": 0.5765013694763184, "mix_low": 0.0015115737915039062, "mix_high": 0.005179405212402344, "read_cosine": 0.21576815843582153, "beta_mean": 3.9907085010781884e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005705662537366152, "mix_std": 0.2750353515148163, "beta_std": 8.241491741500795e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007228445261716843, "evidence_weights": [-0.17921443283557892, -0.26184189319610596, 0.5478305220603943], "agreement_weight": 0.5695222020149231, "orthogonal_weight": -1.3180752992630005}, {"head": 2, "memory_to_active": 0.06219230592250824, "contribution_to_active": 0.4999723732471466, "mix_mean": 0.40084150433540344, "mix_low": 0.0011625289916992188, "mix_high": 0.0001697540283203125, "read_cosine": 0.6124405860900879, "beta_mean": 0.0061537278816103935, "nonfinite": 0.0, "weighted_memory_to_active": 0.030378326773643494, "mix_std": 0.2329552173614502, "beta_std": 0.02182181179523468, "beta_low": 0.87225341796875, "beta_high": 0.0, "orthogonal_to_active": 0.015946663916110992, "evidence_weights": [-0.17943383753299713, -0.21470023691654205, 0.4133341908454895], "agreement_weight": 0.0067515987902879715, "orthogonal_weight": -0.8520534038543701}, {"head": 3, "memory_to_active": 0.15083450078964233, "contribution_to_active": 0.567808210849762, "mix_mean": 0.6647960543632507, "mix_low": 0.02470684051513672, "mix_high": 0.03302764892578125, "read_cosine": -0.0387071818113327, "beta_mean": 0.0008170802611857653, "nonfinite": 0.0, "weighted_memory_to_active": 0.11115041375160217, "mix_std": 0.2877420485019684, "beta_std": 0.007819859310984612, "beta_low": 0.99493408203125, "beta_high": 0.0, "orthogonal_to_active": 0.09936210513114929, "evidence_weights": [-0.3159942328929901, -0.24117159843444824, 1.4737443923950195], "agreement_weight": 6.741353511810303, "orthogonal_weight": -0.9733535051345825}, {"head": 4, "memory_to_active": 0.40342608094215393, "contribution_to_active": 0.25005289912223816, "mix_mean": 0.3524829149246216, "mix_low": 0.0045452117919921875, "mix_high": 0.0, "read_cosine": 0.9013711810112, "beta_mean": 0.0809415876865387, "nonfinite": 0.0, "weighted_memory_to_active": 0.10195781290531158, "mix_std": 0.21450258791446686, "beta_std": 0.05401191487908363, "beta_low": 0.00079345703125, "beta_high": 0.0, "orthogonal_to_active": 0.13836461305618286, "evidence_weights": [-0.06357041746377945, -0.22502432763576508, 0.42554613947868347], "agreement_weight": 0.006655104458332062, "orthogonal_weight": 2.3028314113616943}, {"head": 5, "memory_to_active": 0.04564690589904785, "contribution_to_active": 0.7227373123168945, "mix_mean": 0.6621072292327881, "mix_low": 0.002201080322265625, "mix_high": 0.012030601501464844, "read_cosine": 0.35297301411628723, "beta_mean": 0.00019560048531275243, "nonfinite": 0.0, "weighted_memory_to_active": 0.0316031239926815, "mix_std": 0.27061837911605835, "beta_std": 0.0020187378395348787, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.026296833530068398, "evidence_weights": [-0.220231831073761, -0.28865522146224976, 0.8906460404396057], "agreement_weight": 0.20974446833133698, "orthogonal_weight": -0.9295416474342346}]}, {"layer": 14, "memory_to_active": 0.21584217250347137, "contribution_to_active": 0.41530388593673706, "mix_mean": 0.42636460065841675, "mix_low": 0.028285663574934006, "mix_high": 0.006162961479276419, "read_cosine": 0.6147245168685913, "beta_mean": 0.0632026195526123, "nonfinite": 0.0, "weighted_memory_to_active": 0.08699361234903336, "mix_std": 0.23745223879814148, "beta_std": 0.059284843504428864, "beta_low": 0.5072225332260132, "beta_high": 0.0, "orthogonal_to_active": 0.04081571847200394, "output_influence": 0.35691046714782715, "residual_influence": 0.15205956995487213, "state_active": 0.05313410237431526, "state_memory": 0.014552482403814793, "state_peak": 46.71366500854492, "growth_active": 1.0958305597305298, "growth_memory": 1.2513511180877686, "heads": [{"head": 0, "memory_to_active": 0.36395394802093506, "contribution_to_active": 0.18243467807769775, "mix_mean": 0.18605126440525055, "mix_low": 0.014978408813476562, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8057329654693604, "beta_mean": 0.08022305369377136, "nonfinite": 0.0, "weighted_memory_to_active": 0.0732877105474472, "mix_std": 0.1725374460220337, "beta_std": 0.06471503525972366, "beta_low": 0.00238037109375, "beta_high": 0.0, "orthogonal_to_active": 0.08855622261762619, "evidence_weights": [-0.09735164046287537, -0.04728658124804497, 0.3686351180076599], "agreement_weight": 0.02116849087178707, "orthogonal_weight": 1.8967691659927368}, {"head": 1, "memory_to_active": 0.43635180592536926, "contribution_to_active": 0.273671954870224, "mix_mean": 0.19057069718837738, "mix_low": 0.1414947509765625, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8805555701255798, "beta_mean": 0.2531786262989044, "nonfinite": 0.0, "weighted_memory_to_active": 0.18681460618972778, "mix_std": 0.22347770631313324, "beta_std": 0.24646040797233582, "beta_low": 0.09942626953125, "beta_high": 0.0, "orthogonal_to_active": 0.017261482775211334, "evidence_weights": [0.011509791016578674, -0.18941856920719147, -0.09902889281511307], "agreement_weight": 0.05789550766348839, "orthogonal_weight": 0.29305028915405273}, {"head": 2, "memory_to_active": 0.12207856774330139, "contribution_to_active": 0.5839248895645142, "mix_mean": 0.5912669897079468, "mix_low": 0.0026760101318359375, "mix_high": 0.004574775695800781, "read_cosine": 0.39959773421287537, "beta_mean": 0.0019541573710739613, "nonfinite": 0.0, "weighted_memory_to_active": 0.07957863807678223, "mix_std": 0.2648772597312927, "beta_std": 0.002821494359523058, "beta_low": 0.97589111328125, "beta_high": 0.0, "orthogonal_to_active": 0.06182020157575607, "evidence_weights": [-0.12923192977905273, -0.3154307007789612, 0.9101715683937073], "agreement_weight": 0.08737663924694061, "orthogonal_weight": -0.8780362606048584}, {"head": 3, "memory_to_active": 0.03679706156253815, "contribution_to_active": 0.6072596311569214, "mix_mean": 0.67718505859375, "mix_low": 0.0038175582885742188, "mix_high": 0.02348804473876953, "read_cosine": -0.053029291331768036, "beta_mean": 0.004503224976360798, "nonfinite": 0.0, "weighted_memory_to_active": 0.023760180920362473, "mix_std": 0.2545928955078125, "beta_std": 0.006961041130125523, "beta_low": 0.885986328125, "beta_high": 0.0, "orthogonal_to_active": 0.017971543595194817, "evidence_weights": [-0.3412724733352661, -0.31338050961494446, 0.29738232493400574], "agreement_weight": 0.06545960903167725, "orthogonal_weight": -0.8352530598640442}, {"head": 4, "memory_to_active": 0.15135107934474945, "contribution_to_active": 0.5032966136932373, "mix_mean": 0.5290039777755737, "mix_low": 0.0033025741577148438, "mix_high": 0.007739067077636719, "read_cosine": 0.8963741660118103, "beta_mean": 0.0380578488111496, "nonfinite": 0.0, "weighted_memory_to_active": 0.08855622261762619, "mix_std": 0.2586614489555359, "beta_std": 0.028547070920467377, "beta_low": 0.1033935546875, "beta_high": 0.0, "orthogonal_to_active": 0.024338608607649803, "evidence_weights": [-0.20589156448841095, -0.23676598072052002, 0.4407989978790283], "agreement_weight": 0.13295719027519226, "orthogonal_weight": -0.6177732348442078}, {"head": 5, "memory_to_active": 0.18452048301696777, "contribution_to_active": 0.34123560786247253, "mix_mean": 0.38410964608192444, "mix_low": 0.003444671630859375, "mix_high": 0.001171112060546875, "read_cosine": 0.7591155767440796, "beta_mean": 0.0012988196685910225, "nonfinite": 0.0, "weighted_memory_to_active": 0.06996428966522217, "mix_std": 0.25056666135787964, "beta_std": 0.006204016972333193, "beta_low": 0.97625732421875, "beta_high": 0.0, "orthogonal_to_active": 0.03494623675942421, "evidence_weights": [-0.1725521683692932, -0.18238413333892822, 0.3684731423854828], "agreement_weight": 0.0631178691983223, "orthogonal_weight": -0.7508220672607422}]}, {"layer": 15, "memory_to_active": 0.04819341003894806, "contribution_to_active": 0.49414604902267456, "mix_mean": 0.4327525198459625, "mix_low": 0.007158438675105572, "mix_high": 0.0031139058992266655, "read_cosine": 0.6422514915466309, "beta_mean": 0.03645121306180954, "nonfinite": 0.0, "weighted_memory_to_active": 0.024406541138887405, "mix_std": 0.28411298990249634, "beta_std": 0.04258885234594345, "beta_low": 0.7062175273895264, "beta_high": 0.0, "orthogonal_to_active": 0.009325435385107994, "output_influence": 0.31241530179977417, "residual_influence": 0.1915210485458374, "state_active": 0.17309211194515228, "state_memory": 0.010312382131814957, "state_peak": 30.7298526763916, "growth_active": 1.2158982753753662, "growth_memory": 1.3453153371810913, "heads": [{"head": 0, "memory_to_active": 0.10682673007249832, "contribution_to_active": 0.47458818554878235, "mix_mean": 0.39267444610595703, "mix_low": 0.008473396301269531, "mix_high": 0.0066375732421875, "read_cosine": 0.4892861247062683, "beta_mean": 0.05406183749437332, "nonfinite": 0.0, "weighted_memory_to_active": 0.0529392808675766, "mix_std": 0.2857275605201721, "beta_std": 0.038808345794677734, "beta_low": 0.05126953125, "beta_high": 0.0, "orthogonal_to_active": 0.03455466777086258, "evidence_weights": [-0.1870478242635727, -0.1961088925600052, 0.5205676555633545], "agreement_weight": 0.09029874205589294, "orthogonal_weight": -0.8233715891838074}, {"head": 1, "memory_to_active": 0.08395298570394516, "contribution_to_active": 0.4417538046836853, "mix_mean": 0.4460090398788452, "mix_low": 0.0008497238159179688, "mix_high": 0.007206916809082031, "read_cosine": 0.9342621564865112, "beta_mean": 0.00018059411377180368, "nonfinite": 0.0, "weighted_memory_to_active": 0.03702769801020622, "mix_std": 0.2850034236907959, "beta_std": 0.001414179801940918, "beta_low": 0.99896240234375, "beta_high": 0.0, "orthogonal_to_active": 0.0027563818730413914, "evidence_weights": [-0.15751215815544128, -0.1690676361322403, 0.16138815879821777], "agreement_weight": 0.11487439274787903, "orthogonal_weight": 0.1667332798242569}, {"head": 2, "memory_to_active": 0.005396704189479351, "contribution_to_active": 0.49890977144241333, "mix_mean": 0.3543645143508911, "mix_low": 0.011487960815429688, "mix_high": 6.389617919921875e-05, "read_cosine": 0.7707545757293701, "beta_mean": 6.928971561137587e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029013887979090214, "mix_std": 0.27191174030303955, "beta_std": 0.00020958046661689878, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00096841249614954, "evidence_weights": [-0.19711749255657196, -0.2643226981163025, 0.11218298226594925], "agreement_weight": 0.010631073266267776, "orthogonal_weight": 0.9130101799964905}, {"head": 3, "memory_to_active": 0.0024858596734702587, "contribution_to_active": 0.3778464198112488, "mix_mean": 0.4183022677898407, "mix_low": 0.003966331481933594, "mix_high": 0.0007572174072265625, "read_cosine": 0.6168855428695679, "beta_mean": 1.2557802619994618e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0008540837443433702, "mix_std": 0.29095757007598877, "beta_std": 3.637548797996715e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0005117353284731507, "evidence_weights": [-0.2764137089252472, -0.23870401084423065, 0.19409212470054626], "agreement_weight": 0.059110693633556366, "orthogonal_weight": -0.5842382311820984}, {"head": 4, "memory_to_active": 0.06404025852680206, "contribution_to_active": 0.635332465171814, "mix_mean": 0.39609676599502563, "mix_low": 0.007415771484375, "mix_high": 0.000926971435546875, "read_cosine": 0.7142308354377747, "beta_mean": 0.16199466586112976, "nonfinite": 0.0, "weighted_memory_to_active": 0.035438764840364456, "mix_std": 0.30717912316322327, "beta_std": 0.19803132116794586, "beta_low": 0.21826171875, "beta_high": 0.0, "orthogonal_to_active": 0.006948900409042835, "evidence_weights": [-0.20626738667488098, -0.25291627645492554, -0.02623717673122883], "agreement_weight": 0.013002545572817326, "orthogonal_weight": 0.39361265301704407}, {"head": 5, "memory_to_active": 0.02645791880786419, "contribution_to_active": 0.5364454984664917, "mix_mean": 0.5890681147575378, "mix_low": 0.0107574462890625, "mix_high": 0.0030908584594726562, "read_cosine": 0.32808902859687805, "beta_mean": 0.00238831271417439, "nonfinite": 0.0, "weighted_memory_to_active": 0.017278039827942848, "mix_std": 0.2638985514640808, "beta_std": 0.017033301293849945, "beta_low": 0.9688720703125, "beta_high": 0.0, "orthogonal_to_active": 0.010212510824203491, "evidence_weights": [-0.2803133428096771, -0.3044982850551605, 0.6027690172195435], "agreement_weight": 0.3141023814678192, "orthogonal_weight": 0.8052838444709778}]}]}, {"step": 72000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7969962219995068, "memory_to_active": 0.7976392507553101, "contribution_to_active": 0.573098361492157, "mix_mean": 0.42936134338378906, "mix_low": 0.12161885201931, "mix_high": 0.010521244257688522, "read_cosine": 0.34696152806282043, "beta_mean": 0.06816262751817703, "nonfinite": 0.0, "weighted_memory_to_active": 0.1977786272764206, "mix_std": 0.22289158403873444, "beta_std": 0.05936076492071152, "beta_low": 0.5920016765594482, "beta_high": 2.9881795853725635e-05, "orthogonal_to_active": 0.06253492832183838, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0524508953094482, "normal_token_accuracy": 0.43826472759246826, "state_active": 0.06320454180240631, "state_memory": 0.016506794840097427, "state_peak": 128.48922729492188, "growth_active": 1.123136281967163, "growth_memory": 1.293269395828247, "output_influence": 0.4481784403324127, "residual_influence": 0.16227731108665466, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.198324680328369, "no_memory_loss_delta": 0.6646747589111328, "scrambled_loss_delta": 0.13533353805541992, "memory_only_loss_delta": 3.7320427894592285, "no_evidence_loss_delta": 0.3640799820423126, "no_agreement_loss_delta": 0.01580825448036194, "no_orthogonal_loss_delta": 0.01635180227458477}, {"index": 94, "tokens": 256, "normal_loss": 3.1290102005004883, "no_memory_loss_delta": 0.8325456380844116, "scrambled_loss_delta": 0.4633035957813263, "memory_only_loss_delta": 4.288342475891113, "no_evidence_loss_delta": 0.3644631803035736, "no_agreement_loss_delta": 0.03802741318941116, "no_orthogonal_loss_delta": 0.06700608134269714}, {"index": 188, "tokens": 256, "normal_loss": 3.3617327213287354, "no_memory_loss_delta": 0.922319233417511, "scrambled_loss_delta": 0.09554608166217804, "memory_only_loss_delta": 3.3002071380615234, "no_evidence_loss_delta": 0.44759154319763184, "no_agreement_loss_delta": 0.037404101341962814, "no_orthogonal_loss_delta": 0.044293858110904694}, {"index": 283, "tokens": 131, "normal_loss": 2.013371706008911, "no_memory_loss_delta": 0.39418020844459534, "scrambled_loss_delta": 0.1327144056558609, "memory_only_loss_delta": 2.928774118423462, "no_evidence_loss_delta": 0.1573563665151596, "no_agreement_loss_delta": 0.023783409968018532, "no_orthogonal_loss_delta": -0.03435015678405762}], "agreement_weight_abs_mean": 0.2495747059583664, "agreement_weight_abs_max": 6.780755996704102, "agreement_weight_mean": 0.21798233687877655, "orthogonal_weight_abs_mean": 0.944366991519928, "orthogonal_weight_abs_max": 11.086341857910156, "orthogonal_weight_mean": -0.049927130341529846, "no_memory_loss": 3.798879384994507, "no_memory_loss_delta": 0.7464291453361511, "no_memory_token_accuracy_gain": 0.09566184878349304, "no_memory_token_win_fraction": 0.6796440482139587, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 52.59436797895781, "scrambled_loss": 3.269465446472168, "scrambled_loss_delta": 0.21701501309871674, "scrambled_token_accuracy_gain": 0.022246941924095154, "scrambled_token_win_fraction": 0.5928809642791748, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.50821150973754, "memory_only_loss": 6.70288610458374, "memory_only_loss_delta": 3.650434970855713, "memory_only_token_accuracy_gain": 0.28476083278656006, "memory_only_token_win_fraction": 0.9176863431930542, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 97.40201741463707, "no_evidence_loss": 3.4102976322174072, "no_evidence_loss_delta": 0.3578466773033142, "no_evidence_token_accuracy_gain": 0.04338153451681137, "no_evidence_token_win_fraction": 0.6051167845726013, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 30.081973300716168, "no_agreement_loss": 3.081897735595703, "no_agreement_loss_delta": 0.029447173699736595, "no_agreement_token_accuracy_gain": -0.013348164968192577, "no_agreement_token_win_fraction": 0.4805339276790619, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.9017830317956, "no_orthogonal_loss": 3.0837953090667725, "no_orthogonal_loss_delta": 0.03134480491280556, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5862069129943848, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 3.0858649240356, "short_no_memory_loss_delta": 0.7095816731452942, "long_context_memory_gain": 0.036847472190856934, "normal_context_loss_gain": 0.15276913344860077, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.6528403759002686, "contribution_to_active": 1.227166771888733, "mix_mean": 0.10896985232830048, "mix_low": 0.4955415725708008, "mix_high": 0.004829565994441509, "read_cosine": 0.41612425446510315, "beta_mean": 0.21301937103271484, "nonfinite": 0.0, "weighted_memory_to_active": 1.2449430227279663, "mix_std": 0.09248204529285431, "beta_std": 0.1429821401834488, "beta_low": 0.1740010678768158, "beta_high": 0.0, "orthogonal_to_active": 0.053945720195770264, "output_influence": 0.21650996804237366, "residual_influence": 0.23632459342479706, "state_active": 0.06451081484556198, "state_memory": 0.04494394361972809, "state_peak": 46.01905059814453, "growth_active": 1.2262088060379028, "growth_memory": 1.2651944160461426, "heads": [{"head": 0, "memory_to_active": 3.006789207458496, "contribution_to_active": 0.4358934462070465, "mix_mean": 0.020003698766231537, "mix_low": 0.6027612686157227, "mix_high": 0.0, "read_cosine": 0.2182365357875824, "beta_mean": 0.19750243425369263, "nonfinite": 0.0, "weighted_memory_to_active": 0.5415785908699036, "mix_std": 0.05752082169055939, "beta_std": 0.10095243901014328, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05593295767903328, "evidence_weights": [-0.001631824765354395, -0.03950091451406479, -0.05224690958857536], "agreement_weight": -0.22929343581199646, "orthogonal_weight": -0.11873563379049301}, {"head": 1, "memory_to_active": 7.139298439025879, "contribution_to_active": 3.1115217208862305, "mix_mean": 0.02052120678126812, "mix_low": 0.5985774993896484, "mix_high": 0.0, "read_cosine": 0.4383726119995117, "beta_mean": 0.43091994524002075, "nonfinite": 0.0, "weighted_memory_to_active": 3.320540189743042, "mix_std": 0.05621347576379776, "beta_std": 0.22156962752342224, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.09938029944896698, "evidence_weights": [0.019271304830908775, -0.10717477649450302, 0.6688255667686462], "agreement_weight": 0.027187464758753777, "orthogonal_weight": 0.06739933788776398}, {"head": 2, "memory_to_active": 2.8118338584899902, "contribution_to_active": 0.6679782271385193, "mix_mean": 0.015954099595546722, "mix_low": 0.6342735290527344, "mix_high": 0.0, "read_cosine": 0.28581273555755615, "beta_mean": 0.2654329836368561, "nonfinite": 0.0, "weighted_memory_to_active": 0.7784404158592224, "mix_std": 0.03437255322933197, "beta_std": 0.20338276028633118, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06127127259969711, "evidence_weights": [0.05851056054234505, 0.044325828552246094, 0.2586636245250702], "agreement_weight": 0.01264537125825882, "orthogonal_weight": -0.20543411374092102}, {"head": 3, "memory_to_active": 7.577168941497803, "contribution_to_active": 2.608419895172119, "mix_mean": 0.030477819964289665, "mix_low": 0.47661876678466797, "mix_high": 0.0, "read_cosine": 0.3046630620956421, "beta_mean": 0.3459285497665405, "nonfinite": 0.0, "weighted_memory_to_active": 2.731114387512207, "mix_std": 0.06858561187982559, "beta_std": 0.28008171916007996, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.08932846784591675, "evidence_weights": [-0.03934565559029579, -0.03226986154913902, -0.1553751826286316], "agreement_weight": -0.008502334356307983, "orthogonal_weight": -0.038702357560396194}, {"head": 4, "memory_to_active": 1.2781264781951904, "contribution_to_active": 0.06326309591531754, "mix_mean": 0.02046564593911171, "mix_low": 0.6467571258544922, "mix_high": 0.0, "read_cosine": 0.3203369975090027, "beta_mean": 0.02010396495461464, "nonfinite": 0.0, "weighted_memory_to_active": 0.04178917407989502, "mix_std": 0.05506915599107742, "beta_std": 0.03142602741718292, "beta_low": 0.58111572265625, "beta_high": 0.0, "orthogonal_to_active": 0.0007582238176837564, "evidence_weights": [0.09651355445384979, 0.062454693019390106, -0.1324036568403244], "agreement_weight": -0.20038339495658875, "orthogonal_weight": 0.016083059832453728}, {"head": 5, "memory_to_active": 0.10382407158613205, "contribution_to_active": 0.47592368721961975, "mix_mean": 0.5463966727256775, "mix_low": 0.014261245727539062, "mix_high": 0.028977394104003906, "read_cosine": 0.9293234348297119, "beta_mean": 0.01822833903133869, "nonfinite": 0.0, "weighted_memory_to_active": 0.05619533360004425, "mix_std": 0.28313058614730835, "beta_std": 0.020480185747146606, "beta_low": 0.46282958984375, "beta_high": 0.0, "orthogonal_to_active": 0.017003094777464867, "evidence_weights": [-0.17402109503746033, -0.2171960324048996, 0.37163853645324707], "agreement_weight": 0.28348562121391296, "orthogonal_weight": -0.7015130519866943}]}, {"layer": 1, "memory_to_active": 2.5026345252990723, "contribution_to_active": 0.4933324456214905, "mix_mean": 0.21718502044677734, "mix_low": 0.37407052516937256, "mix_high": 0.001276175258681178, "read_cosine": 0.29992443323135376, "beta_mean": 0.17671066522598267, "nonfinite": 0.0, "weighted_memory_to_active": 0.33089619874954224, "mix_std": 0.11265864223241806, "beta_std": 0.1279507577419281, "beta_low": 0.3542379140853882, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.03841889277100563, "output_influence": 0.27108633518218994, "residual_influence": 0.07685843110084534, "state_active": 0.026760054752230644, "state_memory": 0.03455342352390289, "state_peak": 40.279937744140625, "growth_active": 1.120841383934021, "growth_memory": 1.443397879600525, "heads": [{"head": 0, "memory_to_active": 0.020576633512973785, "contribution_to_active": 0.49367767572402954, "mix_mean": 0.5399192571640015, "mix_low": 0.004706382751464844, "mix_high": 0.002719879150390625, "read_cosine": 0.08626729249954224, "beta_mean": 0.00014970127085689455, "nonfinite": 0.0, "weighted_memory_to_active": 0.01158879417926073, "mix_std": 0.2801963686943054, "beta_std": 0.0002408346044830978, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011038258671760559, "evidence_weights": [-0.21004033088684082, -0.2709304094314575, 0.21663007140159607], "agreement_weight": 0.21098145842552185, "orthogonal_weight": -1.0055090188980103}, {"head": 1, "memory_to_active": 3.09995174407959, "contribution_to_active": 0.30452096462249756, "mix_mean": 0.02076001465320587, "mix_low": 0.5073432922363281, "mix_high": 2.86102294921875e-06, "read_cosine": 0.49815964698791504, "beta_mean": 0.2565001845359802, "nonfinite": 0.0, "weighted_memory_to_active": 0.3323918581008911, "mix_std": 0.04137926548719406, "beta_std": 0.18452519178390503, "beta_low": 0.00189208984375, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.023245198652148247, "evidence_weights": [0.025074953213334084, -0.06314385682344437, 0.05614408105611801], "agreement_weight": -0.00024874930386431515, "orthogonal_weight": 0.21103322505950928}, {"head": 2, "memory_to_active": 0.03885916620492935, "contribution_to_active": 0.5509809255599976, "mix_mean": 0.6811131834983826, "mix_low": 0.00028228759765625, "mix_high": 0.0049343109130859375, "read_cosine": 0.440290629863739, "beta_mean": 0.0012688272399827838, "nonfinite": 0.0, "weighted_memory_to_active": 0.025585517287254333, "mix_std": 0.22630709409713745, "beta_std": 0.0033560595475137234, "beta_low": 0.980712890625, "beta_high": 0.0, "orthogonal_to_active": 0.018421076238155365, "evidence_weights": [-0.2740524411201477, -0.3696102797985077, 0.6378172636032104], "agreement_weight": 0.05158944055438042, "orthogonal_weight": -0.8332329988479614}, {"head": 3, "memory_to_active": 3.094144344329834, "contribution_to_active": 0.18578998744487762, "mix_mean": 0.02364569902420044, "mix_low": 0.5295381546020508, "mix_high": 0.0, "read_cosine": 0.1841561645269394, "beta_mean": 0.3478263020515442, "nonfinite": 0.0, "weighted_memory_to_active": 0.2504730224609375, "mix_std": 0.05995798856019974, "beta_std": 0.263029545545578, "beta_low": 0.06915283203125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.09907646477222443, "evidence_weights": [0.04103359207510948, -0.0170833058655262, -0.010698705911636353], "agreement_weight": -0.02406005747616291, "orthogonal_weight": -0.48929038643836975}, {"head": 4, "memory_to_active": 4.254247188568115, "contribution_to_active": 1.331115484237671, "mix_mean": 0.032203588634729385, "mix_low": 0.32839107513427734, "mix_high": 0.0, "read_cosine": 0.32679444551467896, "beta_mean": 0.17390820384025574, "nonfinite": 0.0, "weighted_memory_to_active": 1.3000112771987915, "mix_std": 0.05289266258478165, "beta_std": 0.1322908103466034, "beta_low": 0.039794921875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.052475858479738235, "evidence_weights": [-0.017516767606139183, -0.006897442974150181, 0.17006975412368774], "agreement_weight": -0.01215916033834219, "orthogonal_weight": 0.06001058220863342}, {"head": 5, "memory_to_active": 4.508028030395508, "contribution_to_active": 0.09390956163406372, "mix_mean": 0.0054683019407093525, "mix_low": 0.8741617202758789, "mix_high": 0.0, "read_cosine": 0.26387834548950195, "beta_mean": 0.28061074018478394, "nonfinite": 0.0, "weighted_memory_to_active": 0.06532661616802216, "mix_std": 0.015218428336083889, "beta_std": 0.18426206707954407, "beta_low": 0.03387451171875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.02625647932291031, "evidence_weights": [0.17043012380599976, 0.08644652366638184, 0.08905798941850662], "agreement_weight": -0.113528773188591, "orthogonal_weight": 0.4187130630016327}]}, {"layer": 2, "memory_to_active": 1.0118470191955566, "contribution_to_active": 0.35311877727508545, "mix_mean": 0.4115239381790161, "mix_low": 0.23920473456382751, "mix_high": 0.0042476654052734375, "read_cosine": 0.2966998517513275, "beta_mean": 0.0742531269788742, "nonfinite": 0.0, "weighted_memory_to_active": 0.0709591805934906, "mix_std": 0.1850317418575287, "beta_std": 0.06663048267364502, "beta_low": 0.6082357168197632, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.016305383294820786, "output_influence": 0.4100716710090637, "residual_influence": 0.11698531359434128, "state_active": 0.040528006851673126, "state_memory": 0.013718821108341217, "state_peak": 26.77788543701172, "growth_active": 1.0956460237503052, "growth_memory": 1.5721981525421143, "heads": [{"head": 0, "memory_to_active": 0.00565075920894742, "contribution_to_active": 0.3528395891189575, "mix_mean": 0.6408547163009644, "mix_low": 0.0005159378051757812, "mix_high": 0.008497238159179688, "read_cosine": 0.1342218518257141, "beta_mean": 4.811226244783029e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003729724558070302, "mix_std": 0.26456987857818604, "beta_std": 8.042895933613181e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003907574340701103, "evidence_weights": [-0.37292882800102234, -0.21458108723163605, 0.2133445143699646], "agreement_weight": 0.14215007424354553, "orthogonal_weight": -1.088355302810669}, {"head": 1, "memory_to_active": 3.100107431411743, "contribution_to_active": 0.04774841293692589, "mix_mean": 0.0058166105300188065, "mix_low": 0.8662595748901367, "mix_high": 0.0, "read_cosine": 0.13168497383594513, "beta_mean": 0.2053610235452652, "nonfinite": 0.0, "weighted_memory_to_active": 0.04128169268369675, "mix_std": 0.012186450883746147, "beta_std": 0.14566978812217712, "beta_low": 0.00616455078125, "beta_high": 0.0, "orthogonal_to_active": 0.011077767238020897, "evidence_weights": [0.13495133817195892, 0.10654689371585846, 0.5602031946182251], "agreement_weight": 0.037119291722774506, "orthogonal_weight": 0.3179066777229309}, {"head": 2, "memory_to_active": 0.00900697149336338, "contribution_to_active": 0.419169545173645, "mix_mean": 0.5524128079414368, "mix_low": 0.000957489013671875, "mix_high": 0.0032606124877929688, "read_cosine": 0.3788435161113739, "beta_mean": 2.808915996865835e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037756585516035557, "mix_std": 0.2728620767593384, "beta_std": 5.3148422011872754e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000224187970161438, "evidence_weights": [-0.24764509499073029, -0.2907337248325348, 0.5774524807929993], "agreement_weight": -0.006760065443813801, "orthogonal_weight": 0.07167224586009979}, {"head": 3, "memory_to_active": 0.10627792030572891, "contribution_to_active": 0.5448570251464844, "mix_mean": 0.5879230499267578, "mix_low": 0.0002613067626953125, "mix_high": 0.005539894104003906, "read_cosine": 0.5858474969863892, "beta_mean": 0.017064787447452545, "nonfinite": 0.0, "weighted_memory_to_active": 0.06803075969219208, "mix_std": 0.25368550419807434, "beta_std": 0.027057316154241562, "beta_low": 0.5784912109375, "beta_high": 0.0, "orthogonal_to_active": 0.04725685343146324, "evidence_weights": [-0.20306815207004547, -0.32760488986968994, 0.7375988364219666], "agreement_weight": 0.05575190484523773, "orthogonal_weight": -0.8947427272796631}, {"head": 4, "memory_to_active": 2.8407483100891113, "contribution_to_active": 0.22536160051822662, "mix_mean": 0.020913081243634224, "mix_low": 0.5667104721069336, "mix_high": 0.0, "read_cosine": 0.18835382163524628, "beta_mean": 0.22298172116279602, "nonfinite": 0.0, "weighted_memory_to_active": 0.30379340052604675, "mix_std": 0.04964691400527954, "beta_std": 0.22687506675720215, "beta_low": 0.06475830078125, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.03398284688591957, "evidence_weights": [0.02382836677134037, 0.0009857930708676577, 0.04129045456647873], "agreement_weight": 0.028524460271000862, "orthogonal_weight": -0.13326720893383026}, {"head": 5, "memory_to_active": 0.009289815090596676, "contribution_to_active": 0.5287364721298218, "mix_mean": 0.6612231731414795, "mix_low": 0.0005235671997070312, "mix_high": 0.008188247680664062, "read_cosine": 0.3612474501132965, "beta_mean": 3.504968844936229e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005143855232745409, "mix_std": 0.25723958015441895, "beta_std": 4.714525130111724e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013830614043399692, "evidence_weights": [-0.27821406722068787, -0.2889127731323242, 0.8900387287139893], "agreement_weight": 0.26988932490348816, "orthogonal_weight": -0.3127685785293579}]}, {"layer": 3, "memory_to_active": 0.41454648971557617, "contribution_to_active": 0.4932209551334381, "mix_mean": 0.5217725038528442, "mix_low": 0.051324211061000824, "mix_high": 0.010639827698469162, "read_cosine": 0.18490543961524963, "beta_mean": 0.02341834455728531, "nonfinite": 0.0, "weighted_memory_to_active": 0.12336982786655426, "mix_std": 0.25660228729248047, "beta_std": 0.02298698015511036, "beta_low": 0.7744039297103882, "beta_high": 0.0, "orthogonal_to_active": 0.05128258094191551, "output_influence": 0.4408104419708252, "residual_influence": 0.11102811992168427, "state_active": 0.023367054760456085, "state_memory": 0.006465820129960775, "state_peak": 25.05748176574707, "growth_active": 1.0970736742019653, "growth_memory": 1.6536638736724854, "heads": [{"head": 0, "memory_to_active": 0.3718707263469696, "contribution_to_active": 0.6195128560066223, "mix_mean": 0.6075502038002014, "mix_low": 0.0018911361694335938, "mix_high": 0.021553993225097656, "read_cosine": 0.2161928415298462, "beta_mean": 0.009254522621631622, "nonfinite": 0.0, "weighted_memory_to_active": 0.23440012335777283, "mix_std": 0.2991858422756195, "beta_std": 0.01072113960981369, "beta_low": 0.70892333984375, "beta_high": 0.0, "orthogonal_to_active": 0.1734280288219452, "evidence_weights": [-0.21605946123600006, -0.25111886858940125, 1.0599726438522339], "agreement_weight": 0.11308496445417404, "orthogonal_weight": -0.7732795476913452}, {"head": 1, "memory_to_active": 0.008073728531599045, "contribution_to_active": 0.4579128921031952, "mix_mean": 0.5942270755767822, "mix_low": 0.003086090087890625, "mix_high": 0.010004997253417969, "read_cosine": 0.16498640179634094, "beta_mean": 4.02281220885925e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005055200774222612, "mix_std": 0.2908153831958771, "beta_std": 0.00010491718421690166, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004062905441969633, "evidence_weights": [-0.27146318554878235, -0.2183864414691925, 0.7128167748451233], "agreement_weight": 0.6147971749305725, "orthogonal_weight": -0.872277557849884}, {"head": 2, "memory_to_active": 1.8696438074111938, "contribution_to_active": 0.28160834312438965, "mix_mean": 0.05752132087945938, "mix_low": 0.2941246032714844, "mix_high": 9.5367431640625e-07, "read_cosine": 0.2113066464662552, "beta_mean": 0.12702161073684692, "nonfinite": 0.0, "weighted_memory_to_active": 0.3236289620399475, "mix_std": 0.10994969308376312, "beta_std": 0.11635109037160873, "beta_low": 0.02362060546875, "beta_high": 0.0, "orthogonal_to_active": 0.001538737560622394, "evidence_weights": [-0.10156027227640152, -0.008982845582067966, -0.043887946754693985], "agreement_weight": -0.013100649230182171, "orthogonal_weight": 0.006098071113228798}, {"head": 3, "memory_to_active": 0.016131635755300522, "contribution_to_active": 0.5412022471427917, "mix_mean": 0.6065995693206787, "mix_low": 0.003951072692871094, "mix_high": 0.008758544921875, "read_cosine": 0.2879413962364197, "beta_mean": 7.742609886918217e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010857188142836094, "mix_std": 0.2834186255931854, "beta_std": 0.00015329945017583668, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025409937370568514, "evidence_weights": [-0.26218438148498535, -0.22138580679893494, 2.315420389175415], "agreement_weight": 2.3007261753082275, "orthogonal_weight": -0.3043365776538849}, {"head": 4, "memory_to_active": 0.007393051404505968, "contribution_to_active": 0.4049677848815918, "mix_mean": 0.5400508046150208, "mix_low": 0.0025949478149414062, "mix_high": 0.0026721954345703125, "read_cosine": 0.15403857827186584, "beta_mean": 4.252725557307713e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004889294505119324, "mix_std": 0.2865156829357147, "beta_std": 6.970616232138127e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004586342256516218, "evidence_weights": [-0.24539844691753387, -0.22497114539146423, 0.0657162293791771], "agreement_weight": -0.04964654520153999, "orthogonal_weight": -1.0196088552474976}, {"head": 5, "memory_to_active": 0.21416613459587097, "contribution_to_active": 0.6541213989257812, "mix_mean": 0.7246860861778259, "mix_low": 0.0022974014282226562, "mix_high": 0.02084827423095703, "read_cosine": 0.07496671378612518, "beta_mean": 0.004073737189173698, "nonfinite": 0.0, "weighted_memory_to_active": 0.16138814389705658, "mix_std": 0.26972830295562744, "beta_std": 0.01052172016352415, "beta_low": 0.91387939453125, "beta_high": 0.0, "orthogonal_to_active": 0.12153849005699158, "evidence_weights": [-0.25851693749427795, -0.34157195687294006, 0.531563937664032], "agreement_weight": 0.24243152141571045, "orthogonal_weight": -0.7761245369911194}]}, {"layer": 4, "memory_to_active": 0.9822761416435242, "contribution_to_active": 0.46671774983406067, "mix_mean": 0.37616056203842163, "mix_low": 0.2294890284538269, "mix_high": 0.012354692444205284, "read_cosine": 0.30230236053466797, "beta_mean": 0.10417922586202621, "nonfinite": 0.0, "weighted_memory_to_active": 0.12132644653320312, "mix_std": 0.16218465566635132, "beta_std": 0.09523965418338776, "beta_low": 0.3948466181755066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.050964366644620895, "output_influence": 0.5101217031478882, "residual_influence": 0.16165626049041748, "state_active": 0.03760472685098648, "state_memory": 0.015455605462193489, "state_peak": 44.15131759643555, "growth_active": 1.202957034111023, "growth_memory": 1.529828429222107, "heads": [{"head": 0, "memory_to_active": 0.14668560028076172, "contribution_to_active": 0.8140286207199097, "mix_mean": 0.6966855525970459, "mix_low": 0.009531021118164062, "mix_high": 0.037654876708984375, "read_cosine": 0.427644819021225, "beta_mean": 0.24917355179786682, "nonfinite": 0.0, "weighted_memory_to_active": 0.12549848854541779, "mix_std": 0.34706994891166687, "beta_std": 0.24041126668453217, "beta_low": 0.23187255859375, "beta_high": 0.0, "orthogonal_to_active": 0.09171292185783386, "evidence_weights": [-0.24830850958824158, -0.3594566881656647, -0.204006627202034], "agreement_weight": 0.003660466754809022, "orthogonal_weight": 0.8478700518608093}, {"head": 1, "memory_to_active": 0.04737282171845436, "contribution_to_active": 0.6678210496902466, "mix_mean": 0.7080187797546387, "mix_low": 7.534027099609375e-05, "mix_high": 0.01857280731201172, "read_cosine": 0.5568325519561768, "beta_mean": 0.0009265531552955508, "nonfinite": 0.0, "weighted_memory_to_active": 0.03576112166047096, "mix_std": 0.23616990447044373, "beta_std": 0.007224170491099358, "beta_low": 0.98175048828125, "beta_high": 0.0, "orthogonal_to_active": 0.017762329429388046, "evidence_weights": [-0.23579657077789307, -0.36503294110298157, 0.5726735591888428], "agreement_weight": 0.3597172498703003, "orthogonal_weight": -0.6915204524993896}, {"head": 2, "memory_to_active": 1.4060792922973633, "contribution_to_active": 0.35693359375, "mix_mean": 0.05332441255450249, "mix_low": 0.3230094909667969, "mix_high": 2.47955322265625e-05, "read_cosine": 0.21270500123500824, "beta_mean": 0.05242793262004852, "nonfinite": 0.0, "weighted_memory_to_active": 0.255964070558548, "mix_std": 0.09430666267871857, "beta_std": 0.056328438222408295, "beta_low": 0.13323974609375, "beta_high": 0.0, "orthogonal_to_active": 0.09679785370826721, "evidence_weights": [0.029003433883190155, -0.0595657154917717, 0.35674676299095154], "agreement_weight": 0.1249578669667244, "orthogonal_weight": -0.4942574203014374}, {"head": 3, "memory_to_active": 1.8470098972320557, "contribution_to_active": 0.14515532553195953, "mix_mean": 0.03308190405368805, "mix_low": 0.40110206604003906, "mix_high": 0.0, "read_cosine": 0.25935104489326477, "beta_mean": 0.12686990201473236, "nonfinite": 0.0, "weighted_memory_to_active": 0.1956809163093567, "mix_std": 0.05599154159426689, "beta_std": 0.11249672621488571, "beta_low": 0.02215576171875, "beta_high": 0.0, "orthogonal_to_active": 0.002499714959412813, "evidence_weights": [-0.024904651567339897, 0.046424977481365204, -0.24942483007907867], "agreement_weight": 0.08104350417852402, "orthogonal_weight": 0.018363110721111298}, {"head": 4, "memory_to_active": 0.0050265369936823845, "contribution_to_active": 0.6257950067520142, "mix_mean": 0.7519168853759766, "mix_low": 8.106231689453125e-05, "mix_high": 0.017871856689453125, "read_cosine": 0.13438275456428528, "beta_mean": 1.1590595022425987e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0036129255313426256, "mix_std": 0.20999857783317566, "beta_std": 3.0234938094508834e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002347427187487483, "evidence_weights": [-0.32814332842826843, -0.3160216510295868, 0.2570785880088806], "agreement_weight": 0.3008761703968048, "orthogonal_weight": -0.6874034404754639}, {"head": 5, "memory_to_active": 2.4414827823638916, "contribution_to_active": 0.19057294726371765, "mix_mean": 0.013935903087258339, "mix_low": 0.6431350708007812, "mix_high": 3.814697265625e-06, "read_cosine": 0.22289788722991943, "beta_mean": 0.19566582143306732, "nonfinite": 0.0, "weighted_memory_to_active": 0.11144115030765533, "mix_std": 0.029571298509836197, "beta_std": 0.15494710206985474, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.09466594457626343, "evidence_weights": [0.057783905416727066, -0.015636786818504333, -0.27734649181365967], "agreement_weight": -0.09512024372816086, "orthogonal_weight": 1.1127668619155884}]}, {"layer": 5, "memory_to_active": 0.36184996366500854, "contribution_to_active": 0.5472050905227661, "mix_mean": 0.5218774080276489, "mix_low": 0.055147647857666016, "mix_high": 0.019684158265590668, "read_cosine": 0.07586242258548737, "beta_mean": 0.05127444863319397, "nonfinite": 0.0, "weighted_memory_to_active": 0.12272588908672333, "mix_std": 0.2515733242034912, "beta_std": 0.06587136536836624, "beta_low": 0.6778361201286316, "beta_high": 0.0, "orthogonal_to_active": 0.05330147221684456, "output_influence": 0.589619517326355, "residual_influence": 0.19520975649356842, "state_active": 0.039083365350961685, "state_memory": 0.009624559432268143, "state_peak": 63.16720962524414, "growth_active": 1.1134482622146606, "growth_memory": 1.2448914051055908, "heads": [{"head": 0, "memory_to_active": 0.18947386741638184, "contribution_to_active": 0.5909759402275085, "mix_mean": 0.5020487904548645, "mix_low": 0.0074787139892578125, "mix_high": 0.00205230712890625, "read_cosine": -0.09126322716474533, "beta_mean": 0.017684925347566605, "nonfinite": 0.0, "weighted_memory_to_active": 0.0674058124423027, "mix_std": 0.27662739157676697, "beta_std": 0.029533252120018005, "beta_low": 0.57635498046875, "beta_high": 0.0, "orthogonal_to_active": 0.03121257573366165, "evidence_weights": [-0.1701144576072693, -0.1892346441745758, 0.11400400102138519], "agreement_weight": -0.026863636448979378, "orthogonal_weight": -0.48278912901878357}, {"head": 1, "memory_to_active": 0.0044596632942557335, "contribution_to_active": 0.5677599310874939, "mix_mean": 0.7148049473762512, "mix_low": 0.0009241104125976562, "mix_high": 0.037383079528808594, "read_cosine": 0.1963723748922348, "beta_mean": 9.820446393860038e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0028542927466332912, "mix_std": 0.2641974091529846, "beta_std": 2.9740791433141567e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0026696871500462294, "evidence_weights": [-0.24589288234710693, -0.2850063443183899, 0.9397838711738586], "agreement_weight": 0.715310275554657, "orthogonal_weight": -0.972951352596283}, {"head": 2, "memory_to_active": 0.012934524565935135, "contribution_to_active": 0.6077426671981812, "mix_mean": 0.6651952266693115, "mix_low": 0.0019826889038085938, "mix_high": 0.021828651428222656, "read_cosine": -0.4951862692832947, "beta_mean": 6.645466055488214e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00810188427567482, "mix_std": 0.2624482214450836, "beta_std": 0.0002513564541004598, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007571285590529442, "evidence_weights": [-0.256587415933609, -0.31356626749038696, 0.17108142375946045], "agreement_weight": -0.18633712828159332, "orthogonal_weight": -1.0531907081604004}, {"head": 3, "memory_to_active": 1.4913935661315918, "contribution_to_active": 0.2381766140460968, "mix_mean": 0.11799265444278717, "mix_low": 0.11657142639160156, "mix_high": 0.0, "read_cosine": 0.15391764044761658, "beta_mean": 0.09633708000183105, "nonfinite": 0.0, "weighted_memory_to_active": 0.35857582092285156, "mix_std": 0.14792171120643616, "beta_std": 0.1486077606678009, "beta_low": 0.20123291015625, "beta_high": 0.0, "orthogonal_to_active": 0.1444152593612671, "evidence_weights": [-0.07700704783201218, -0.09884021431207657, 0.11962263286113739], "agreement_weight": 0.050368499010801315, "orthogonal_weight": -0.5028753280639648}, {"head": 4, "memory_to_active": 0.4106784164905548, "contribution_to_active": 0.583799421787262, "mix_mean": 0.3203679323196411, "mix_low": 0.20238208770751953, "mix_high": 0.0046844482421875, "read_cosine": 0.5408362746238708, "beta_mean": 0.19337081909179688, "nonfinite": 0.0, "weighted_memory_to_active": 0.2562777101993561, "mix_std": 0.3383336663246155, "beta_std": 0.21597038209438324, "beta_low": 0.2908935546875, "beta_high": 0.0, "orthogonal_to_active": 0.10880528390407562, "evidence_weights": [-0.11284665018320084, -0.2508939504623413, -0.05216676741838455], "agreement_weight": 0.029241852462291718, "orthogonal_weight": 0.6373533010482788}, {"head": 5, "memory_to_active": 0.06215987727046013, "contribution_to_active": 0.6947759985923767, "mix_mean": 0.8108547925949097, "mix_low": 0.0015468597412109375, "mix_high": 0.05215644836425781, "read_cosine": 0.15049776434898376, "beta_mean": 0.0001775905693648383, "nonfinite": 0.0, "weighted_memory_to_active": 0.043139759451150894, "mix_std": 0.2199113965034485, "beta_std": 0.0008356936159543693, "beta_low": 0.99853515625, "beta_high": 0.0, "orthogonal_to_active": 0.02513473480939865, "evidence_weights": [-0.3047797977924347, -0.3283156156539917, 1.8301318883895874], "agreement_weight": 0.5269080400466919, "orthogonal_weight": -0.6064828634262085}]}, {"layer": 6, "memory_to_active": 0.2305249571800232, "contribution_to_active": 0.6413800716400146, "mix_mean": 0.6017162203788757, "mix_low": 0.032588642090559006, "mix_high": 0.03202978894114494, "read_cosine": 0.3229736387729645, "beta_mean": 0.06232941523194313, "nonfinite": 0.0, "weighted_memory_to_active": 0.11760934442281723, "mix_std": 0.23945334553718567, "beta_std": 0.07283297181129456, "beta_low": 0.6052144765853882, "beta_high": 0.0, "orthogonal_to_active": 0.07645914703607559, "output_influence": 0.5929627418518066, "residual_influence": 0.21791495382785797, "state_active": 0.043743375688791275, "state_memory": 0.005562459118664265, "state_peak": 80.83499908447266, "growth_active": 1.2567346096038818, "growth_memory": 1.209647536277771, "heads": [{"head": 0, "memory_to_active": 0.11088622361421585, "contribution_to_active": 0.7904945611953735, "mix_mean": 0.8291853070259094, "mix_low": 0.00022411346435546875, "mix_high": 0.06860160827636719, "read_cosine": 0.3191683888435364, "beta_mean": 0.009994544088840485, "nonfinite": 0.0, "weighted_memory_to_active": 0.09923972189426422, "mix_std": 0.19852012395858765, "beta_std": 0.03120468556880951, "beta_low": 0.85015869140625, "beta_high": 0.0, "orthogonal_to_active": 0.06499265134334564, "evidence_weights": [-0.29822036623954773, -0.3463761806488037, 1.1766865253448486], "agreement_weight": 0.8961901068687439, "orthogonal_weight": -0.7154816389083862}, {"head": 1, "memory_to_active": 0.0025219861418008804, "contribution_to_active": 0.5275588631629944, "mix_mean": 0.6047627925872803, "mix_low": 0.0005245208740234375, "mix_high": 0.017925262451171875, "read_cosine": 0.4899170696735382, "beta_mean": 5.321900971466675e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0012035105610266328, "mix_std": 0.28166401386260986, "beta_std": 8.657928447064478e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006399796111509204, "evidence_weights": [-0.19126087427139282, -0.3019099235534668, 1.1179841756820679], "agreement_weight": 0.12510140240192413, "orthogonal_weight": -0.707129716873169}, {"head": 2, "memory_to_active": 0.6598761677742004, "contribution_to_active": 0.10996342450380325, "mix_mean": 0.060178112238645554, "mix_low": 0.18793296813964844, "mix_high": 1.9073486328125e-06, "read_cosine": 0.26771384477615356, "beta_mean": 0.022701436653733253, "nonfinite": 0.0, "weighted_memory_to_active": 0.07279932498931885, "mix_std": 0.08581823110580444, "beta_std": 0.02761622704565525, "beta_low": 0.29132080078125, "beta_high": 0.0, "orthogonal_to_active": 0.014110046438872814, "evidence_weights": [0.05516170710325241, -0.0732785165309906, 0.37263575196266174], "agreement_weight": 0.13888399302959442, "orthogonal_weight": -0.2295800745487213}, {"head": 3, "memory_to_active": 0.12771405279636383, "contribution_to_active": 0.6947729587554932, "mix_mean": 0.6765804290771484, "mix_low": 0.0054798126220703125, "mix_high": 0.024700164794921875, "read_cosine": 0.4743533730506897, "beta_mean": 0.21334128081798553, "nonfinite": 0.0, "weighted_memory_to_active": 0.09406232833862305, "mix_std": 0.3228693902492523, "beta_std": 0.2415473908185959, "beta_low": 0.1912841796875, "beta_high": 0.0, "orthogonal_to_active": 0.05684785544872284, "evidence_weights": [-0.16758567094802856, -0.31453078985214233, 0.11420513689517975], "agreement_weight": -0.007106904871761799, "orthogonal_weight": 0.6946948766708374}, {"head": 4, "memory_to_active": 0.45965662598609924, "contribution_to_active": 1.0804919004440308, "mix_mean": 0.7431414723396301, "mix_low": 0.0010890960693359375, "mix_high": 0.0711832046508789, "read_cosine": 0.10112550854682922, "beta_mean": 0.12782740592956543, "nonfinite": 0.0, "weighted_memory_to_active": 0.4217373728752136, "mix_std": 0.30620524287223816, "beta_std": 0.13635315001010895, "beta_low": 0.29864501953125, "beta_high": 0.0, "orthogonal_to_active": 0.305979460477829, "evidence_weights": [-0.16543032228946686, -0.5113271474838257, -0.7487329840660095], "agreement_weight": -0.011457826010882854, "orthogonal_weight": 0.7474417090415955}, {"head": 5, "memory_to_active": 0.022494588047266006, "contribution_to_active": 0.6449984908103943, "mix_mean": 0.6964491009712219, "mix_low": 0.00028133392333984375, "mix_high": 0.009766578674316406, "read_cosine": 0.28556352853775024, "beta_mean": 0.00010650663170963526, "nonfinite": 0.0, "weighted_memory_to_active": 0.01661382056772709, "mix_std": 0.2416429966688156, "beta_std": 0.00026771542616188526, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.016184870153665543, "evidence_weights": [-0.23233719170093536, -0.3143175542354584, 0.35792070627212524], "agreement_weight": 0.2238202542066574, "orthogonal_weight": -1.0468616485595703}]}, {"layer": 7, "memory_to_active": 1.8003684282302856, "contribution_to_active": 0.27480950951576233, "mix_mean": 0.1777193248271942, "mix_low": 0.2228231430053711, "mix_high": 0.000956376432441175, "read_cosine": 0.25192469358444214, "beta_mean": 0.13975617289543152, "nonfinite": 0.0, "weighted_memory_to_active": 0.2029828131198883, "mix_std": 0.14759111404418945, "beta_std": 0.09651971608400345, "beta_low": 0.2282002866268158, "beta_high": 0.00020345053053461015, "orthogonal_to_active": 0.08969150483608246, "output_influence": 0.2935482859611511, "residual_influence": 0.07408971339464188, "state_active": 0.01364146452397108, "state_memory": 0.023813409730792046, "state_peak": 29.869115829467773, "growth_active": 1.1573805809020996, "growth_memory": 1.2688019275665283, "heads": [{"head": 0, "memory_to_active": 2.9102439880371094, "contribution_to_active": 0.17561541497707367, "mix_mean": 0.03738544136285782, "mix_low": 0.4133930206298828, "mix_high": 9.5367431640625e-07, "read_cosine": 0.35085564851760864, "beta_mean": 0.20086097717285156, "nonfinite": 0.0, "weighted_memory_to_active": 0.20731601119041443, "mix_std": 0.07474604994058609, "beta_std": 0.13954411447048187, "beta_low": 0.00103759765625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.0011434275656938553, "evidence_weights": [0.04677092283964157, -0.08867374807596207, 0.02602105587720871], "agreement_weight": -0.02903055027127266, "orthogonal_weight": -0.006374710705131292}, {"head": 1, "memory_to_active": 0.9660512804985046, "contribution_to_active": 0.3901579678058624, "mix_mean": 0.214542418718338, "mix_low": 0.11831474304199219, "mix_high": 0.004879951477050781, "read_cosine": 0.39794686436653137, "beta_mean": 0.09375744313001633, "nonfinite": 0.0, "weighted_memory_to_active": 0.3322470188140869, "mix_std": 0.24096262454986572, "beta_std": 0.05940799415111542, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04774443432688713, "evidence_weights": [-0.2166556864976883, -0.10494464635848999, -0.31013476848602295], "agreement_weight": 0.057478148490190506, "orthogonal_weight": 0.17360782623291016}, {"head": 2, "memory_to_active": 0.5389220118522644, "contribution_to_active": 0.39657968282699585, "mix_mean": 0.2447575181722641, "mix_low": 0.01473236083984375, "mix_high": 0.0004749298095703125, "read_cosine": 0.13480274379253387, "beta_mean": 0.030716829001903534, "nonfinite": 0.0, "weighted_memory_to_active": 0.21426522731781006, "mix_std": 0.21711677312850952, "beta_std": 0.03486453741788864, "beta_low": 0.2099609375, "beta_high": 0.0, "orthogonal_to_active": 0.12755468487739563, "evidence_weights": [-0.09755101054906845, -0.18499083817005157, 0.8855751752853394], "agreement_weight": 0.013267723843455315, "orthogonal_weight": -0.6136147379875183}, {"head": 3, "memory_to_active": 4.943282127380371, "contribution_to_active": 0.10455013811588287, "mix_mean": 0.035756830126047134, "mix_low": 0.3071756362915039, "mix_high": 0.0, "read_cosine": 0.09884125739336014, "beta_mean": 0.45880451798439026, "nonfinite": 0.0, "weighted_memory_to_active": 0.3800230622291565, "mix_std": 0.051120348274707794, "beta_std": 0.28135746717453003, "beta_low": 0.00103759765625, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.3302639126777649, "evidence_weights": [0.07476146519184113, -0.04372752457857132, 0.7165699601173401], "agreement_weight": 0.05299907922744751, "orthogonal_weight": -0.9392150640487671}, {"head": 4, "memory_to_active": 1.4099745750427246, "contribution_to_active": 0.11762380599975586, "mix_mean": 0.026036079972982407, "mix_low": 0.48153209686279297, "mix_high": 0.0, "read_cosine": 0.2588231563568115, "beta_mean": 0.05425111949443817, "nonfinite": 0.0, "weighted_memory_to_active": 0.06703955680131912, "mix_std": 0.05090102553367615, "beta_std": 0.06362980604171753, "beta_low": 0.1571044921875, "beta_high": 0.0, "orthogonal_to_active": 0.01538278441876173, "evidence_weights": [0.11590873450040817, -0.029222600162029266, 0.1968994438648224], "agreement_weight": 0.03695744648575783, "orthogonal_weight": -0.29634130001068115}, {"head": 5, "memory_to_active": 0.03373650088906288, "contribution_to_active": 0.4643300175666809, "mix_mean": 0.5078375339508057, "mix_low": 0.0017910003662109375, "mix_high": 0.00038242340087890625, "read_cosine": 0.2702784240245819, "beta_mean": 0.00014611815277021378, "nonfinite": 0.0, "weighted_memory_to_active": 0.01700599677860737, "mix_std": 0.2506997585296631, "beta_std": 0.000314353674184531, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01605977490544319, "evidence_weights": [-0.18519453704357147, -0.24146096408367157, 0.47989964485168457], "agreement_weight": 0.2375859171152115, "orthogonal_weight": -1.015397310256958}]}, {"layer": 8, "memory_to_active": 0.13595865666866302, "contribution_to_active": 0.5655717253684998, "mix_mean": 0.5063419342041016, "mix_low": 0.0100377406924963, "mix_high": 0.001965681789442897, "read_cosine": 0.39956027269363403, "beta_mean": 0.013018349185585976, "nonfinite": 0.0, "weighted_memory_to_active": 0.06954172998666763, "mix_std": 0.24164538085460663, "beta_std": 0.018725533038377762, "beta_low": 0.7744852900505066, "beta_high": 0.0, "orthogonal_to_active": 0.0625225305557251, "output_influence": 0.5227583646774292, "residual_influence": 0.19734500348567963, "state_active": 0.04085497930645943, "state_memory": 0.004709252621978521, "state_peak": 63.12621307373047, "growth_active": 1.090728998184204, "growth_memory": 1.1065346002578735, "heads": [{"head": 0, "memory_to_active": 0.05588473752140999, "contribution_to_active": 0.5694496035575867, "mix_mean": 0.6079521179199219, "mix_low": 0.00048065185546875, "mix_high": 0.0015773773193359375, "read_cosine": 0.4494123160839081, "beta_mean": 0.003196398261934519, "nonfinite": 0.0, "weighted_memory_to_active": 0.040091995149850845, "mix_std": 0.24923893809318542, "beta_std": 0.013826994225382805, "beta_low": 0.933349609375, "beta_high": 0.0, "orthogonal_to_active": 0.02527642250061035, "evidence_weights": [-0.1545121967792511, -0.3767612874507904, 0.5576308965682983], "agreement_weight": 0.2675541937351227, "orthogonal_weight": -0.7921473979949951}, {"head": 1, "memory_to_active": 0.06808459758758545, "contribution_to_active": 0.5430595874786377, "mix_mean": 0.5316308736801147, "mix_low": 0.00016021728515625, "mix_high": 0.001926422119140625, "read_cosine": 0.15154409408569336, "beta_mean": 0.0009866516338661313, "nonfinite": 0.0, "weighted_memory_to_active": 0.030403319746255875, "mix_std": 0.2323693335056305, "beta_std": 0.007255940698087215, "beta_low": 0.99090576171875, "beta_high": 0.0, "orthogonal_to_active": 0.022333763539791107, "evidence_weights": [-0.19318212568759918, -0.2734375596046448, 0.5909886360168457], "agreement_weight": 0.08453197777271271, "orthogonal_weight": -0.7714747786521912}, {"head": 2, "memory_to_active": 0.009966874495148659, "contribution_to_active": 0.6905588507652283, "mix_mean": 0.6887911558151245, "mix_low": 0.00030231475830078125, "mix_high": 0.003914833068847656, "read_cosine": 0.42078036069869995, "beta_mean": 0.0001144642592407763, "nonfinite": 0.0, "weighted_memory_to_active": 0.007940798997879028, "mix_std": 0.22059696912765503, "beta_std": 0.0034591122530400753, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004585030488669872, "evidence_weights": [-0.22053106129169464, -0.3453429043292999, 0.9703701734542847], "agreement_weight": 0.3299431800842285, "orthogonal_weight": -0.6687431931495667}, {"head": 3, "memory_to_active": 0.10248495638370514, "contribution_to_active": 0.6373738646507263, "mix_mean": 0.5851681232452393, "mix_low": 0.000560760498046875, "mix_high": 0.00415802001953125, "read_cosine": 0.7901284098625183, "beta_mean": 0.00857603270560503, "nonfinite": 0.0, "weighted_memory_to_active": 0.07668187469244003, "mix_std": 0.25610703229904175, "beta_std": 0.008356538601219654, "beta_low": 0.69000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.023823779076337814, "evidence_weights": [-0.1711055338382721, -0.296588659286499, 0.5239983797073364], "agreement_weight": 0.26042675971984863, "orthogonal_weight": -0.5924391746520996}, {"head": 4, "memory_to_active": 0.23843224346637726, "contribution_to_active": 0.35011059045791626, "mix_mean": 0.36362507939338684, "mix_low": 0.001338958740234375, "mix_high": 1.239776611328125e-05, "read_cosine": 0.0857207179069519, "beta_mean": 0.0006172486464492977, "nonfinite": 0.0, "weighted_memory_to_active": 0.09001312404870987, "mix_std": 0.23227012157440186, "beta_std": 0.013388551771640778, "beta_low": 0.9949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.016468286514282227, "evidence_weights": [-0.14668960869312286, -0.20508623123168945, 0.668445348739624], "agreement_weight": 0.03903483971953392, "orthogonal_weight": -0.18575851619243622}, {"head": 5, "memory_to_active": 0.3408985435962677, "contribution_to_active": 0.6028777360916138, "mix_mean": 0.26088452339172363, "mix_low": 0.05738353729248047, "mix_high": 0.00020503997802734375, "read_cosine": 0.49977558851242065, "beta_mean": 0.0646192878484726, "nonfinite": 0.0, "weighted_memory_to_active": 0.17211921513080597, "mix_std": 0.25928986072540283, "beta_std": 0.06606606394052505, "beta_low": 0.0377197265625, "beta_high": 0.0, "orthogonal_to_active": 0.28264790773391724, "evidence_weights": [-0.15005998313426971, -0.13919131457805634, -0.11880228668451309], "agreement_weight": -0.0025976491160690784, "orthogonal_weight": 1.9488955736160278}]}, {"layer": 9, "memory_to_active": 0.79709792137146, "contribution_to_active": 0.7117336988449097, "mix_mean": 0.40138673782348633, "mix_low": 0.049332145601511, "mix_high": 0.0061620082706213, "read_cosine": 0.34183141589164734, "beta_mean": 0.033786602318286896, "nonfinite": 0.0, "weighted_memory_to_active": 0.3571268320083618, "mix_std": 0.24781900644302368, "beta_std": 0.027895942330360413, "beta_low": 0.6620687246322632, "beta_high": 0.0, "orthogonal_to_active": 0.03092089481651783, "output_influence": 0.560364305973053, "residual_influence": 0.17425622045993805, "state_active": 0.0444604754447937, "state_memory": 0.013939479365944862, "state_peak": 58.48977279663086, "growth_active": 1.2006609439849854, "growth_memory": 1.2391142845153809, "heads": [{"head": 0, "memory_to_active": 0.00030514728859998286, "contribution_to_active": 0.5834959149360657, "mix_mean": 0.6600910425186157, "mix_low": 0.0010805130004882812, "mix_high": 0.026102066040039062, "read_cosine": 0.3391561806201935, "beta_mean": 7.003367841207364e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.0002073466603178531, "mix_std": 0.282642126083374, "beta_std": 1.832786779232265e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00020336592569947243, "evidence_weights": [-0.3039787709712982, -0.3105539381504059, 0.3574167788028717], "agreement_weight": 0.10281798988580704, "orthogonal_weight": -1.0409901142120361}, {"head": 1, "memory_to_active": 0.016050267964601517, "contribution_to_active": 0.7195386290550232, "mix_mean": 0.583629846572876, "mix_low": 0.006854057312011719, "mix_high": 0.0026788711547851562, "read_cosine": 0.430428147315979, "beta_mean": 3.4714645153144374e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012214677408337593, "mix_std": 0.2768482267856598, "beta_std": 7.50647159293294e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0079445019364357, "evidence_weights": [-0.23545362055301666, -0.2919338047504425, 0.9583538174629211], "agreement_weight": 0.5629534721374512, "orthogonal_weight": -0.7980186939239502}, {"head": 2, "memory_to_active": 0.15310345590114594, "contribution_to_active": 0.6579024195671082, "mix_mean": 0.6013607382774353, "mix_low": 0.001956939697265625, "mix_high": 0.00632476806640625, "read_cosine": 0.37900984287261963, "beta_mean": 0.0005694313440471888, "nonfinite": 0.0, "weighted_memory_to_active": 0.10397794842720032, "mix_std": 0.2666347324848175, "beta_std": 0.004137915093451738, "beta_low": 0.99676513671875, "beta_high": 0.0, "orthogonal_to_active": 0.05936379358172417, "evidence_weights": [-0.200152188539505, -0.27559593319892883, 0.7492070198059082], "agreement_weight": 0.35364070534706116, "orthogonal_weight": -0.6699449419975281}, {"head": 3, "memory_to_active": 0.7098370790481567, "contribution_to_active": 0.2838762104511261, "mix_mean": 0.18602511286735535, "mix_low": 0.07696723937988281, "mix_high": 4.291534423828125e-05, "read_cosine": 0.3703003525733948, "beta_mean": 0.023461366072297096, "nonfinite": 0.0, "weighted_memory_to_active": 0.1858283132314682, "mix_std": 0.21361684799194336, "beta_std": 0.025832243263721466, "beta_low": 0.3035888671875, "beta_high": 0.0, "orthogonal_to_active": 0.08928556740283966, "evidence_weights": [-0.04358566552400589, -0.11464336514472961, 0.5002461671829224], "agreement_weight": 0.09403008222579956, "orthogonal_weight": -0.5316132307052612}, {"head": 4, "memory_to_active": 2.943021774291992, "contribution_to_active": 1.6862536668777466, "mix_mean": 0.15046045184135437, "mix_low": 0.11730098724365234, "mix_high": 0.001148223876953125, "read_cosine": 0.08701951056718826, "beta_mean": 0.15839418768882751, "nonfinite": 0.0, "weighted_memory_to_active": 1.5641183853149414, "mix_std": 0.20909638702869415, "beta_std": 0.08821557462215424, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.009082411415874958, "evidence_weights": [-0.08001895993947983, -0.1501573771238327, -0.04258056730031967], "agreement_weight": -0.06667818129062653, "orthogonal_weight": -0.005979621782898903}, {"head": 5, "memory_to_active": 0.9602699875831604, "contribution_to_active": 0.33933523297309875, "mix_mean": 0.2267531156539917, "mix_low": 0.09183311462402344, "mix_high": 0.000675201416015625, "read_cosine": 0.4450743496417999, "beta_mean": 0.020259205251932144, "nonfinite": 0.0, "weighted_memory_to_active": 0.27641430497169495, "mix_std": 0.23807571828365326, "beta_std": 0.04911300912499428, "beta_low": 0.67205810546875, "beta_high": 0.0, "orthogonal_to_active": 0.019645731896162033, "evidence_weights": [0.03577306121587753, -0.19592225551605225, 0.1462157666683197], "agreement_weight": 0.007671545259654522, "orthogonal_weight": 0.0831708163022995}]}, {"layer": 10, "memory_to_active": 0.19236387312412262, "contribution_to_active": 0.7081905603408813, "mix_mean": 0.5284596085548401, "mix_low": 0.023008346557617188, "mix_high": 0.0052355132065713406, "read_cosine": 0.3880537748336792, "beta_mean": 0.0434885211288929, "nonfinite": 0.0, "weighted_memory_to_active": 0.08022148907184601, "mix_std": 0.26671600341796875, "beta_std": 0.04354667663574219, "beta_low": 0.6653035879135132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.25009816884994507, "output_influence": 0.47562313079833984, "residual_influence": 0.1730455458164215, "state_active": 0.05321573093533516, "state_memory": 0.01355038769543171, "state_peak": 128.48922729492188, "growth_active": 1.0056946277618408, "growth_memory": 1.0312186479568481, "heads": [{"head": 0, "memory_to_active": 0.6496960520744324, "contribution_to_active": 1.9158241748809814, "mix_mean": 0.5608553886413574, "mix_low": 0.1123514175415039, "mix_high": 0.005715370178222656, "read_cosine": 0.0989179015159607, "beta_mean": 0.18834227323532104, "nonfinite": 0.0, "weighted_memory_to_active": 0.37214505672454834, "mix_std": 0.3913152813911438, "beta_std": 0.19656364619731903, "beta_low": 0.14190673828125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.4203795194625854, "evidence_weights": [-0.080411896109581, -0.3622666299343109, -1.9297467470169067], "agreement_weight": 0.04401830583810806, "orthogonal_weight": 3.847449779510498}, {"head": 1, "memory_to_active": 0.006713537499308586, "contribution_to_active": 0.6231212615966797, "mix_mean": 0.6372697353363037, "mix_low": 0.0009698867797851562, "mix_high": 0.016959190368652344, "read_cosine": 0.3321845233440399, "beta_mean": 4.675993841374293e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003879174590110779, "mix_std": 0.2624287009239197, "beta_std": 8.683891064720228e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00597947183996439, "evidence_weights": [-0.2477884143590927, -0.19125090539455414, 0.5412085652351379], "agreement_weight": 0.6648608446121216, "orthogonal_weight": -1.6888446807861328}, {"head": 2, "memory_to_active": 0.011210212484002113, "contribution_to_active": 0.6216098070144653, "mix_mean": 0.6710578203201294, "mix_low": 0.00011157989501953125, "mix_high": 0.006808280944824219, "read_cosine": 0.03990943357348442, "beta_mean": 2.5045384973054752e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0052040377631783485, "mix_std": 0.24811840057373047, "beta_std": 8.89749571797438e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020729941315948963, "evidence_weights": [-0.26819249987602234, -0.27704766392707825, 0.6876951456069946], "agreement_weight": -0.21184730529785156, "orthogonal_weight": -0.3874240219593048}, {"head": 3, "memory_to_active": 0.36429908871650696, "contribution_to_active": 0.10435705631971359, "mix_mean": 0.2945738732814789, "mix_low": 0.02257823944091797, "mix_high": 5.054473876953125e-05, "read_cosine": 0.9260436296463013, "beta_mean": 0.06655286252498627, "nonfinite": 0.0, "weighted_memory_to_active": 0.04208659380674362, "mix_std": 0.2293948233127594, "beta_std": 0.05691758543252945, "beta_low": 0.0025634765625, "beta_high": 0.0, "orthogonal_to_active": 0.04335351288318634, "evidence_weights": [-0.08985240012407303, -0.14293278753757477, 0.1597002148628235], "agreement_weight": 0.018692392855882645, "orthogonal_weight": 1.468692660331726}, {"head": 4, "memory_to_active": 0.011829760856926441, "contribution_to_active": 0.5702744722366333, "mix_mean": 0.5966020226478577, "mix_low": 0.00035762786865234375, "mix_high": 0.0018625259399414062, "read_cosine": 0.09747105836868286, "beta_mean": 2.9178667318774387e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007007396314293146, "mix_std": 0.2446688860654831, "beta_std": 6.039869549567811e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010219338349997997, "evidence_weights": [-0.18189924955368042, -0.25067856907844543, 0.9263137578964233], "agreement_weight": 0.18207615613937378, "orthogonal_weight": -1.5331358909606934}, {"head": 5, "memory_to_active": 0.11043454706668854, "contribution_to_active": 0.4139562249183655, "mix_mean": 0.41039854288101196, "mix_low": 0.0016813278198242188, "mix_high": 1.71661376953125e-05, "read_cosine": 0.8337960243225098, "beta_mean": 0.005934976041316986, "nonfinite": 0.0, "weighted_memory_to_active": 0.051006682217121124, "mix_std": 0.2243700474500656, "beta_std": 0.007562614977359772, "beta_low": 0.84735107421875, "beta_high": 0.0, "orthogonal_to_active": 0.018584270030260086, "evidence_weights": [-0.1373603790998459, -0.16583140194416046, 0.27684998512268066], "agreement_weight": 0.09680729359388351, "orthogonal_weight": -0.6715239882469177}]}, {"layer": 11, "memory_to_active": 0.07598458975553513, "contribution_to_active": 0.629031240940094, "mix_mean": 0.5265514254570007, "mix_low": 0.11407868564128876, "mix_high": 0.04191955178976059, "read_cosine": 0.3539082407951355, "beta_mean": 0.02251391112804413, "nonfinite": 0.0, "weighted_memory_to_active": 0.041404277086257935, "mix_std": 0.32165849208831787, "beta_std": 0.033268388360738754, "beta_low": 0.6996562480926514, "beta_high": 0.0, "orthogonal_to_active": 0.04492878168821335, "output_influence": 0.5604322552680969, "residual_influence": 0.1614370048046112, "state_active": 0.206589937210083, "state_memory": 0.0031583046074956656, "state_peak": 36.43867874145508, "growth_active": 1.1284055709838867, "growth_memory": 1.4942078590393066, "heads": [{"head": 0, "memory_to_active": 0.008626284077763557, "contribution_to_active": 0.6091568470001221, "mix_mean": 0.6618062257766724, "mix_low": 0.0006914138793945312, "mix_high": 0.008932113647460938, "read_cosine": 0.24268263578414917, "beta_mean": 8.521367271896452e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006538871210068464, "mix_std": 0.25143957138061523, "beta_std": 0.003931368235498667, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.005457783583551645, "evidence_weights": [-0.1767202764749527, -0.3570628762245178, 0.8143777847290039], "agreement_weight": 0.41200029850006104, "orthogonal_weight": -0.8592674732208252}, {"head": 1, "memory_to_active": 0.10838192701339722, "contribution_to_active": 0.43533241748809814, "mix_mean": 0.4718608260154724, "mix_low": 0.0045223236083984375, "mix_high": 0.0005340576171875, "read_cosine": 0.5247117280960083, "beta_mean": 0.002646317007020116, "nonfinite": 0.0, "weighted_memory_to_active": 0.05365374684333801, "mix_std": 0.268062949180603, "beta_std": 0.01070928294211626, "beta_low": 0.9498291015625, "beta_high": 0.0, "orthogonal_to_active": 0.029373595491051674, "evidence_weights": [-0.13595175743103027, -0.2777426242828369, 0.5883470177650452], "agreement_weight": 0.08604973554611206, "orthogonal_weight": -0.6649666428565979}, {"head": 2, "memory_to_active": 0.005078199319541454, "contribution_to_active": 0.9863803386688232, "mix_mean": 0.43361896276474, "mix_low": 0.3277149200439453, "mix_high": 0.15879249572753906, "read_cosine": 0.1746998131275177, "beta_mean": 0.05783667042851448, "nonfinite": 0.0, "weighted_memory_to_active": 0.00351673923432827, "mix_std": 0.4499611258506775, "beta_std": 0.07820369303226471, "beta_low": 0.4918212890625, "beta_high": 0.0, "orthogonal_to_active": 0.036339107900857925, "evidence_weights": [0.04319993034005165, -0.3366241455078125, -1.3131992816925049], "agreement_weight": 0.00041084695840254426, "orthogonal_weight": 10.823314666748047}, {"head": 3, "memory_to_active": 0.008126361295580864, "contribution_to_active": 0.8681617975234985, "mix_mean": 0.41051292419433594, "mix_low": 0.34487152099609375, "mix_high": 0.07588863372802734, "read_cosine": 0.6324568390846252, "beta_mean": 0.048038166016340256, "nonfinite": 0.0, "weighted_memory_to_active": 0.005297558382153511, "mix_std": 0.4374154806137085, "beta_std": 0.07325489073991776, "beta_low": 0.5264892578125, "beta_high": 0.0, "orthogonal_to_active": 0.03540806099772453, "evidence_weights": [0.2205902636051178, -0.27443474531173706, -0.09035739302635193], "agreement_weight": 0.05271533131599426, "orthogonal_weight": 11.086341857910156}, {"head": 4, "memory_to_active": 0.2349185347557068, "contribution_to_active": 0.45165887475013733, "mix_mean": 0.6078948974609375, "mix_low": 0.005847930908203125, "mix_high": 0.0013427734375, "read_cosine": -0.001574439462274313, "beta_mean": 0.017329007387161255, "nonfinite": 0.0, "weighted_memory_to_active": 0.13187874853610992, "mix_std": 0.26817166805267334, "beta_std": 0.02062339335680008, "beta_low": 0.4698486328125, "beta_high": 0.0, "orthogonal_to_active": 0.13113297522068024, "evidence_weights": [-0.23223935067653656, -0.23980864882469177, 1.879830002784729], "agreement_weight": -0.2217121422290802, "orthogonal_weight": -1.004538655281067}, {"head": 5, "memory_to_active": 0.09077621251344681, "contribution_to_active": 0.4234968423843384, "mix_mean": 0.5736148357391357, "mix_low": 0.000823974609375, "mix_high": 0.0060272216796875, "read_cosine": 0.5504727363586426, "beta_mean": 0.009148078970611095, "nonfinite": 0.0, "weighted_memory_to_active": 0.047540001571178436, "mix_std": 0.25490015745162964, "beta_std": 0.012887704186141491, "beta_low": 0.76043701171875, "beta_high": 0.0, "orthogonal_to_active": 0.03186117485165596, "evidence_weights": [-0.19733761250972748, -0.3105608820915222, 0.6600187420845032], "agreement_weight": 0.08139503747224808, "orthogonal_weight": -0.8698155879974365}]}, {"layer": 12, "memory_to_active": 0.21422232687473297, "contribution_to_active": 0.6028046607971191, "mix_mean": 0.5588269233703613, "mix_low": 0.008191904053092003, "mix_high": 0.00727717112749815, "read_cosine": 0.22561505436897278, "beta_mean": 0.016889607533812523, "nonfinite": 0.0, "weighted_memory_to_active": 0.11375155299901962, "mix_std": 0.25943881273269653, "beta_std": 0.015833266079425812, "beta_low": 0.8284200429916382, "beta_high": 0.0, "orthogonal_to_active": 0.07579152286052704, "output_influence": 0.5227582454681396, "residual_influence": 0.15878954529762268, "state_active": 0.032619476318359375, "state_memory": 0.008127737790346146, "state_peak": 76.30374145507812, "growth_active": 1.1337645053863525, "growth_memory": 1.1592835187911987, "heads": [{"head": 0, "memory_to_active": 0.03948822617530823, "contribution_to_active": 0.6326360106468201, "mix_mean": 0.5967457294464111, "mix_low": 0.0008592605590820312, "mix_high": 0.008183479309082031, "read_cosine": 0.2892957925796509, "beta_mean": 0.0006497932481579483, "nonfinite": 0.0, "weighted_memory_to_active": 0.0223563089966774, "mix_std": 0.26385438442230225, "beta_std": 0.0013878901954740286, "beta_low": 0.996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.02038446255028248, "evidence_weights": [-0.2041890174150467, -0.27256667613983154, -0.06807266175746918], "agreement_weight": 0.10209785401821136, "orthogonal_weight": -0.981959342956543}, {"head": 1, "memory_to_active": 0.08764471113681793, "contribution_to_active": 0.5719550848007202, "mix_mean": 0.5733035802841187, "mix_low": 0.00030803680419921875, "mix_high": 0.0025281906127929688, "read_cosine": 0.3987506330013275, "beta_mean": 0.0005850729066878557, "nonfinite": 0.0, "weighted_memory_to_active": 0.056189995259046555, "mix_std": 0.2638911306858063, "beta_std": 0.008898806758224964, "beta_low": 0.996826171875, "beta_high": 0.0, "orthogonal_to_active": 0.04664580151438713, "evidence_weights": [-0.19969846308231354, -0.2726577818393707, 1.0386505126953125], "agreement_weight": 0.28303736448287964, "orthogonal_weight": -0.9152157306671143}, {"head": 2, "memory_to_active": 0.6275636553764343, "contribution_to_active": 0.5624244213104248, "mix_mean": 0.3388066291809082, "mix_low": 0.02422046661376953, "mix_high": 0.0001163482666015625, "read_cosine": 0.5515545010566711, "beta_mean": 0.09889086335897446, "nonfinite": 0.0, "weighted_memory_to_active": 0.27539101243019104, "mix_std": 0.24415922164916992, "beta_std": 0.067882239818573, "beta_low": 0.00030517578125, "beta_high": 0.0, "orthogonal_to_active": 0.30865901708602905, "evidence_weights": [-0.2049333155155182, -0.06349396705627441, -0.02412421815097332], "agreement_weight": 0.02432309091091156, "orthogonal_weight": 1.4460639953613281}, {"head": 3, "memory_to_active": 0.03414618596434593, "contribution_to_active": 0.7561476230621338, "mix_mean": 0.7301148176193237, "mix_low": 0.00010776519775390625, "mix_high": 0.019865989685058594, "read_cosine": 0.06805842369794846, "beta_mean": 0.00031716690864413977, "nonfinite": 0.0, "weighted_memory_to_active": 0.02421339973807335, "mix_std": 0.2223016321659088, "beta_std": 0.00609754491597414, "beta_low": 0.9976806640625, "beta_high": 0.0, "orthogonal_to_active": 0.022971592843532562, "evidence_weights": [-0.2863207161426544, -0.33341342210769653, 1.6760696172714233], "agreement_weight": 0.6182141900062561, "orthogonal_weight": -0.9656760096549988}, {"head": 4, "memory_to_active": 0.4305467903614044, "contribution_to_active": 0.5808330774307251, "mix_mean": 0.46932148933410645, "mix_low": 0.01743030548095703, "mix_high": 0.0056285858154296875, "read_cosine": -0.18851017951965332, "beta_mean": 0.0005652995314449072, "nonfinite": 0.0, "weighted_memory_to_active": 0.2610763907432556, "mix_std": 0.2914074659347534, "beta_std": 0.007473080884665251, "beta_low": 0.98626708984375, "beta_high": 0.0, "orthogonal_to_active": 0.02428329735994339, "evidence_weights": [-0.1553700715303421, -0.21328315138816833, -0.14669984579086304], "agreement_weight": 0.01861589029431343, "orthogonal_weight": -0.09647447615861893}, {"head": 5, "memory_to_active": 0.0659443736076355, "contribution_to_active": 0.5128316879272461, "mix_mean": 0.6446695327758789, "mix_low": 0.0062255859375, "mix_high": 0.007340431213378906, "read_cosine": 0.23454107344150543, "beta_mean": 0.00032945015118457377, "nonfinite": 0.0, "weighted_memory_to_active": 0.0432821586728096, "mix_std": 0.2710190713405609, "beta_std": 0.0032600320409983397, "beta_low": 0.99310302734375, "beta_high": 0.0, "orthogonal_to_active": 0.031804922968149185, "evidence_weights": [-0.22255150973796844, -0.31354811787605286, 1.2280316352844238], "agreement_weight": 0.4324784278869629, "orthogonal_weight": -0.8022804856300354}]}, {"layer": 13, "memory_to_active": 0.1162852868437767, "contribution_to_active": 0.5401989817619324, "mix_mean": 0.5457896590232849, "mix_low": 0.006609917152673006, "mix_high": 0.009828091599047184, "read_cosine": 0.4143829941749573, "beta_mean": 0.0137418732047081, "nonfinite": 0.0, "weighted_memory_to_active": 0.0516660213470459, "mix_std": 0.25922691822052, "beta_std": 0.014199662953615189, "beta_low": 0.8119202256202698, "beta_high": 0.0, "orthogonal_to_active": 0.05360385403037071, "output_influence": 0.5328475832939148, "residual_influence": 0.19559919834136963, "state_active": 0.1293717324733734, "state_memory": 0.04203600063920021, "state_peak": 98.64944458007812, "growth_active": 0.8853925466537476, "growth_memory": 0.9899982213973999, "heads": [{"head": 0, "memory_to_active": 0.008977802470326424, "contribution_to_active": 0.6111933588981628, "mix_mean": 0.6188209056854248, "mix_low": 0.0008993148803710938, "mix_high": 0.0047016143798828125, "read_cosine": 0.43508243560791016, "beta_mean": 3.168952389387414e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00595134450122714, "mix_std": 0.26491960883140564, "beta_std": 6.646775000263005e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0055898744612932205, "evidence_weights": [-0.18940702080726624, -0.2641761600971222, 0.7281502485275269], "agreement_weight": 0.23230582475662231, "orthogonal_weight": -1.0520511865615845}, {"head": 1, "memory_to_active": 0.01126858964562416, "contribution_to_active": 0.6059504747390747, "mix_mean": 0.5739878416061401, "mix_low": 0.0015077590942382812, "mix_high": 0.004535675048828125, "read_cosine": 0.24697545170783997, "beta_mean": 4.5330940338317305e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00625733844935894, "mix_std": 0.2740364670753479, "beta_std": 0.00010228165774606168, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007911528460681438, "evidence_weights": [-0.17871062457561493, -0.2609492540359497, 0.5399022698402405], "agreement_weight": 0.5795291662216187, "orthogonal_weight": -1.3156046867370605}, {"head": 2, "memory_to_active": 0.059791356325149536, "contribution_to_active": 0.48778092861175537, "mix_mean": 0.40324822068214417, "mix_low": 0.0011577606201171875, "mix_high": 0.0001621246337890625, "read_cosine": 0.6071549654006958, "beta_mean": 0.005282715894281864, "nonfinite": 0.0, "weighted_memory_to_active": 0.03031591884791851, "mix_std": 0.23283793032169342, "beta_std": 0.019208697602152824, "beta_low": 0.8812255859375, "beta_high": 0.0, "orthogonal_to_active": 0.017178023234009743, "evidence_weights": [-0.1781512200832367, -0.21472984552383423, 0.4040006399154663], "agreement_weight": 0.00483026634901762, "orthogonal_weight": -0.8500954508781433}, {"head": 3, "memory_to_active": 0.18718968331813812, "contribution_to_active": 0.5594764947891235, "mix_mean": 0.6583479642868042, "mix_low": 0.029860496520996094, "mix_high": 0.03609275817871094, "read_cosine": -0.04023583605885506, "beta_mean": 0.0010955256875604391, "nonfinite": 0.0, "weighted_memory_to_active": 0.137715682387352, "mix_std": 0.29366517066955566, "beta_std": 0.010678822174668312, "beta_low": 0.99041748046875, "beta_high": 0.0, "orthogonal_to_active": 0.12374041229486465, "evidence_weights": [-0.3155495226383209, -0.24059273302555084, 1.4726508855819702], "agreement_weight": 6.780755996704102, "orthogonal_weight": -0.9737973809242249}, {"head": 4, "memory_to_active": 0.38856130838394165, "contribution_to_active": 0.25510746240615845, "mix_mean": 0.3575458228588104, "mix_low": 0.004070281982421875, "mix_high": 0.0, "read_cosine": 0.8939653635025024, "beta_mean": 0.07583296298980713, "nonfinite": 0.0, "weighted_memory_to_active": 0.10112625360488892, "mix_std": 0.21919813752174377, "beta_std": 0.053227752447128296, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.14331212639808655, "evidence_weights": [-0.06200912222266197, -0.22448822855949402, 0.4225212037563324], "agreement_weight": 0.009857630357146263, "orthogonal_weight": 2.3085670471191406}, {"head": 5, "memory_to_active": 0.04192298650741577, "contribution_to_active": 0.721684992313385, "mix_mean": 0.6627868413925171, "mix_low": 0.0021638870239257812, "mix_high": 0.013476371765136719, "read_cosine": 0.3433554172515869, "beta_mean": 0.00016301198047585785, "nonfinite": 0.0, "weighted_memory_to_active": 0.02862960286438465, "mix_std": 0.2707042098045349, "beta_std": 0.0019139547366648912, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.023891162127256393, "evidence_weights": [-0.22086283564567566, -0.2886905372142792, 0.8987998366355896], "agreement_weight": 0.21812260150909424, "orthogonal_weight": -0.9232299327850342}]}, {"layer": 14, "memory_to_active": 0.22080200910568237, "contribution_to_active": 0.4149577021598816, "mix_mean": 0.4297683537006378, "mix_low": 0.02830616757273674, "mix_high": 0.006430785171687603, "read_cosine": 0.6113245487213135, "beta_mean": 0.06254232674837112, "nonfinite": 0.0, "weighted_memory_to_active": 0.08883320540189743, "mix_std": 0.2402879297733307, "beta_std": 0.059212956577539444, "beta_low": 0.5082703232765198, "beta_high": 0.0, "orthogonal_to_active": 0.04252086207270622, "output_influence": 0.35452017188072205, "residual_influence": 0.15204526484012604, "state_active": 0.050360456109046936, "state_memory": 0.014007631689310074, "state_peak": 44.91233825683594, "growth_active": 1.0756224393844604, "growth_memory": 1.2794835567474365, "heads": [{"head": 0, "memory_to_active": 0.3849007785320282, "contribution_to_active": 0.1985500007867813, "mix_mean": 0.19007618725299835, "mix_low": 0.018083572387695312, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8171775341033936, "beta_mean": 0.0704236552119255, "nonfinite": 0.0, "weighted_memory_to_active": 0.08342006057500839, "mix_std": 0.17815260589122772, "beta_std": 0.059123095124959946, "beta_low": 0.00640869140625, "beta_high": 0.0, "orthogonal_to_active": 0.09548795968294144, "evidence_weights": [-0.09784431010484695, -0.04887537658214569, 0.36874428391456604], "agreement_weight": 0.023705413565039635, "orthogonal_weight": 1.9105324745178223}, {"head": 1, "memory_to_active": 0.4422115385532379, "contribution_to_active": 0.2740618884563446, "mix_mean": 0.20334891974925995, "mix_low": 0.1384878158569336, "mix_high": 0.0, "read_cosine": 0.8809638023376465, "beta_mean": 0.261160671710968, "nonfinite": 0.0, "weighted_memory_to_active": 0.18559136986732483, "mix_std": 0.23300325870513916, "beta_std": 0.2520063817501068, "beta_low": 0.08514404296875, "beta_high": 0.0, "orthogonal_to_active": 0.019053420051932335, "evidence_weights": [0.009611200541257858, -0.19113437831401825, -0.08795612305402756], "agreement_weight": 0.058990828692913055, "orthogonal_weight": 0.3077407777309418}, {"head": 2, "memory_to_active": 0.11970019340515137, "contribution_to_active": 0.580453634262085, "mix_mean": 0.5904542207717896, "mix_low": 0.00269317626953125, "mix_high": 0.0052642822265625, "read_cosine": 0.3858376741409302, "beta_mean": 0.001666599535383284, "nonfinite": 0.0, "weighted_memory_to_active": 0.0774848461151123, "mix_std": 0.26630839705467224, "beta_std": 0.0023279215674847364, "beta_low": 0.9862060546875, "beta_high": 0.0, "orthogonal_to_active": 0.06016217917203903, "evidence_weights": [-0.12831264734268188, -0.3151981830596924, 0.9118788838386536], "agreement_weight": 0.08495279401540756, "orthogonal_weight": -0.8686152696609497}, {"head": 3, "memory_to_active": 0.041851870715618134, "contribution_to_active": 0.5933151841163635, "mix_mean": 0.6812835335731506, "mix_low": 0.003841400146484375, "mix_high": 0.02398681640625, "read_cosine": -0.06479645520448685, "beta_mean": 0.0049871960654854774, "nonfinite": 0.0, "weighted_memory_to_active": 0.02670995704829693, "mix_std": 0.25431597232818604, "beta_std": 0.008654220029711723, "beta_low": 0.87139892578125, "beta_high": 0.0, "orthogonal_to_active": 0.020495083183050156, "evidence_weights": [-0.3415355682373047, -0.31410613656044006, 0.3045567274093628], "agreement_weight": 0.0668305903673172, "orthogonal_weight": -0.8402879238128662}, {"head": 4, "memory_to_active": 0.14795762300491333, "contribution_to_active": 0.49958521127700806, "mix_mean": 0.5253745913505554, "mix_low": 0.0037755966186523438, "mix_high": 0.008141517639160156, "read_cosine": 0.8958508968353271, "beta_mean": 0.035926301032304764, "nonfinite": 0.0, "weighted_memory_to_active": 0.08704514801502228, "mix_std": 0.25937357544898987, "beta_std": 0.027408331632614136, "beta_low": 0.11749267578125, "beta_high": 0.0, "orthogonal_to_active": 0.023870278149843216, "evidence_weights": [-0.2047574371099472, -0.23564057052135468, 0.4419143497943878], "agreement_weight": 0.13423952460289001, "orthogonal_weight": -0.6146180033683777}, {"head": 5, "memory_to_active": 0.18818995356559753, "contribution_to_active": 0.34378015995025635, "mix_mean": 0.38807255029678345, "mix_low": 0.0029554367065429688, "mix_high": 0.001190185546875, "read_cosine": 0.7529140710830688, "beta_mean": 0.0010895250597968698, "nonfinite": 0.0, "weighted_memory_to_active": 0.0727478414773941, "mix_std": 0.2505737245082855, "beta_std": 0.005757765844464302, "beta_low": 0.98297119140625, "beta_high": 0.0, "orthogonal_to_active": 0.03605623170733452, "evidence_weights": [-0.17497575283050537, -0.18520230054855347, 0.3730478584766388], "agreement_weight": 0.06282585859298706, "orthogonal_weight": -0.7511493563652039}]}, {"layer": 15, "memory_to_active": 0.05262535810470581, "contribution_to_active": 0.500134289264679, "mix_mean": 0.4357324242591858, "mix_low": 0.0061472258530557156, "mix_high": 0.0035028457641601562, "read_cosine": 0.6659910678863525, "beta_mean": 0.039680127054452896, "nonfinite": 0.0, "weighted_memory_to_active": 0.02710023708641529, "mix_std": 0.2818955183029175, "beta_std": 0.04607576131820679, "beta_low": 0.7049255967140198, "beta_high": 0.0, "orthogonal_to_active": 0.009803267195820808, "output_influence": 0.31682032346725464, "residual_influence": 0.19385212659835815, "state_active": 0.16456110775470734, "state_memory": 0.010441884398460388, "state_peak": 26.002954483032227, "growth_active": 1.179619550704956, "growth_memory": 1.2048454284667969, "heads": [{"head": 0, "memory_to_active": 0.1117473840713501, "contribution_to_active": 0.48314040899276733, "mix_mean": 0.4057142436504364, "mix_low": 0.007069587707519531, "mix_high": 0.007648468017578125, "read_cosine": 0.5398685336112976, "beta_mean": 0.05382107198238373, "nonfinite": 0.0, "weighted_memory_to_active": 0.05653828755021095, "mix_std": 0.28680047392845154, "beta_std": 0.038711126893758774, "beta_low": 0.05694580078125, "beta_high": 0.0, "orthogonal_to_active": 0.03511471673846245, "evidence_weights": [-0.18784473836421967, -0.19728510081768036, 0.5282963514328003], "agreement_weight": 0.0904296338558197, "orthogonal_weight": -0.8334451913833618}, {"head": 1, "memory_to_active": 0.09121955186128616, "contribution_to_active": 0.4390178918838501, "mix_mean": 0.45032715797424316, "mix_low": 0.0007572174072265625, "mix_high": 0.007999420166015625, "read_cosine": 0.9312660098075867, "beta_mean": 0.00018422765424475074, "nonfinite": 0.0, "weighted_memory_to_active": 0.039933864027261734, "mix_std": 0.2851509153842926, "beta_std": 0.0014618558343499899, "beta_low": 0.99884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.0031847096979618073, "evidence_weights": [-0.1600455641746521, -0.17199958860874176, 0.16222193837165833], "agreement_weight": 0.11408842355012894, "orthogonal_weight": 0.17635582387447357}, {"head": 2, "memory_to_active": 0.0040739355608820915, "contribution_to_active": 0.5109472870826721, "mix_mean": 0.3548380732536316, "mix_low": 0.009412765502929688, "mix_high": 6.198883056640625e-05, "read_cosine": 0.7798581123352051, "beta_mean": 4.7816003643674776e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002207417506724596, "mix_std": 0.2690494656562805, "beta_std": 0.00011231053213123232, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00071433448465541, "evidence_weights": [-0.197830930352211, -0.26473215222358704, 0.10690385848283768], "agreement_weight": 0.009642871096730232, "orthogonal_weight": 0.9233660101890564}, {"head": 3, "memory_to_active": 0.0022186781279742718, "contribution_to_active": 0.3802257776260376, "mix_mean": 0.42066359519958496, "mix_low": 0.0030736923217773438, "mix_high": 0.0007867813110351562, "read_cosine": 0.6282720565795898, "beta_mean": 1.1538978469616268e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0007496928446926177, "mix_std": 0.2861175239086151, "beta_std": 2.5707948225317523e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00043075892608612776, "evidence_weights": [-0.27385520935058594, -0.2370213121175766, 0.1810862421989441], "agreement_weight": 0.05651969462633133, "orthogonal_weight": -0.5763424038887024}, {"head": 4, "memory_to_active": 0.07527604699134827, "contribution_to_active": 0.6413903832435608, "mix_mean": 0.3892408609390259, "mix_low": 0.0081329345703125, "mix_high": 0.0011157989501953125, "read_cosine": 0.7687045335769653, "beta_mean": 0.1807200163602829, "nonfinite": 0.0, "weighted_memory_to_active": 0.04200362414121628, "mix_std": 0.305366575717926, "beta_std": 0.21284125745296478, "beta_low": 0.20794677734375, "beta_high": 0.0, "orthogonal_to_active": 0.007069516461342573, "evidence_weights": [-0.20517514646053314, -0.2509836256504059, -0.02673858404159546], "agreement_weight": 0.01184887159615755, "orthogonal_weight": 0.40254145860671997}, {"head": 5, "memory_to_active": 0.031216565519571304, "contribution_to_active": 0.5460838079452515, "mix_mean": 0.5936105251312256, "mix_low": 0.008437156677246094, "mix_high": 0.0034046173095703125, "read_cosine": 0.34797680377960205, "beta_mean": 0.0032960856333374977, "nonfinite": 0.0, "weighted_memory_to_active": 0.021168529987335205, "mix_std": 0.25888803601264954, "beta_std": 0.023302285000681877, "beta_low": 0.9658203125, "beta_high": 0.0, "orthogonal_to_active": 0.012305562384426594, "evidence_weights": [-0.28105080127716064, -0.3060109615325928, 0.5953677296638489], "agreement_weight": 0.3181324303150177, "orthogonal_weight": 0.7990632057189941}]}]}, {"step": 73000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7880670910017216, "memory_to_active": 0.803092896938324, "contribution_to_active": 0.5789111852645874, "mix_mean": 0.43093305826187134, "mix_low": 0.12026340514421463, "mix_high": 0.01061873696744442, "read_cosine": 0.34346407651901245, "beta_mean": 0.0672130435705185, "nonfinite": 0.0, "weighted_memory_to_active": 0.19935286045074463, "mix_std": 0.22332943975925446, "beta_std": 0.05911014601588249, "beta_low": 0.5969499349594116, "beta_high": 3.496805948088877e-05, "orthogonal_to_active": 0.06536826491355896, "samples": 4, "scored_tokens": 899, "normal_loss": 3.027010202407837, "normal_token_accuracy": 0.45828697085380554, "state_active": 0.06506258994340897, "state_memory": 0.016353042796254158, "state_peak": 128.87705993652344, "growth_active": 1.1174495220184326, "growth_memory": 1.2661042213439941, "output_influence": 0.4494823217391968, "residual_influence": 0.16439440846443176, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.172151565551758, "no_memory_loss_delta": 0.7035772800445557, "scrambled_loss_delta": 0.14521682262420654, "memory_only_loss_delta": 4.332127571105957, "no_evidence_loss_delta": 0.4479878842830658, "no_agreement_loss_delta": 0.026535579934716225, "no_orthogonal_loss_delta": 0.012476634234189987}, {"index": 94, "tokens": 256, "normal_loss": 3.1388609409332275, "no_memory_loss_delta": 0.848242998123169, "scrambled_loss_delta": 0.46268314123153687, "memory_only_loss_delta": 4.4591193199157715, "no_evidence_loss_delta": 0.3815656900405884, "no_agreement_loss_delta": 0.03662629425525665, "no_orthogonal_loss_delta": 0.07040411233901978}, {"index": 188, "tokens": 256, "normal_loss": 3.324439287185669, "no_memory_loss_delta": 0.9019259214401245, "scrambled_loss_delta": 0.12735074758529663, "memory_only_loss_delta": 3.534942150115967, "no_evidence_loss_delta": 0.5010510683059692, "no_agreement_loss_delta": 0.03377116471529007, "no_orthogonal_loss_delta": 0.03492718189954758}, {"index": 283, "tokens": 131, "normal_loss": 1.9435607194900513, "no_memory_loss_delta": 0.44033873081207275, "scrambled_loss_delta": 0.13750790059566498, "memory_only_loss_delta": 3.360806465148926, "no_evidence_loss_delta": 0.21083931624889374, "no_agreement_loss_delta": 0.01050172746181488, "no_orthogonal_loss_delta": -0.03624855726957321}], "agreement_weight_abs_mean": 0.2516111731529236, "agreement_weight_abs_max": 6.8144001960754395, "agreement_weight_mean": 0.2190866470336914, "orthogonal_weight_abs_mean": 0.9474877715110779, "orthogonal_weight_abs_max": 11.17026424407959, "orthogonal_weight_mean": -0.045555371791124344, "no_memory_loss": 3.7899062633514404, "no_memory_loss_delta": 0.7628958821296692, "no_memory_token_accuracy_gain": 0.12347052246332169, "no_memory_token_win_fraction": 0.697441577911377, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 53.36859207737446, "scrambled_loss": 3.256417989730835, "scrambled_loss_delta": 0.2294079214334488, "scrambled_token_accuracy_gain": 0.02002224698662758, "scrambled_token_win_fraction": 0.6173526048660278, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.49958318880602, "memory_only_loss": 7.026754379272461, "memory_only_loss_delta": 3.999744176864624, "memory_only_token_accuracy_gain": 0.3181312680244446, "memory_only_token_win_fraction": 0.9176863431930542, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 98.16796749477099, "no_evidence_loss": 3.4366371631622314, "no_evidence_loss_delta": 0.4096270203590393, "no_evidence_token_accuracy_gain": 0.07007786631584167, "no_evidence_token_win_fraction": 0.621802031993866, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 33.610217566446345, "no_agreement_loss": 3.056143045425415, "no_agreement_loss_delta": 0.029133014380931854, "no_agreement_token_accuracy_gain": -0.006674082484096289, "no_agreement_token_win_fraction": 0.48164626955986023, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 2.871273929987321, "no_orthogonal_loss": 3.0552752017974854, "no_orthogonal_loss_delta": 0.028265034779906273, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5895439386367798, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.7869315786599342, "short_no_memory_loss_delta": 0.7124328017234802, "long_context_memory_gain": 0.050463080406188965, "normal_context_loss_gain": 0.1420411467552185, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.679039478302002, "contribution_to_active": 1.2323379516601562, "mix_mean": 0.1098683550953865, "mix_low": 0.493543803691864, "mix_high": 0.005129019729793072, "read_cosine": 0.41344135999679565, "beta_mean": 0.20912140607833862, "nonfinite": 0.0, "weighted_memory_to_active": 1.2498102188110352, "mix_std": 0.0924716666340828, "beta_std": 0.14321288466453552, "beta_low": 0.1820984035730362, "beta_high": 0.0, "orthogonal_to_active": 0.05393829941749573, "output_influence": 0.21552389860153198, "residual_influence": 0.23417921364307404, "state_active": 0.06539028882980347, "state_memory": 0.04538746923208237, "state_peak": 47.86204147338867, "growth_active": 1.2113202810287476, "growth_memory": 1.2686280012130737, "heads": [{"head": 0, "memory_to_active": 2.9663102626800537, "contribution_to_active": 0.48923152685165405, "mix_mean": 0.02019030600786209, "mix_low": 0.5966272354125977, "mix_high": 0.0, "read_cosine": 0.22455571591854095, "beta_mean": 0.20039314031600952, "nonfinite": 0.0, "weighted_memory_to_active": 0.6102471351623535, "mix_std": 0.05829595401883125, "beta_std": 0.10134618729352951, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06208818778395653, "evidence_weights": [-0.003442879067733884, -0.04068272188305855, -0.05102169141173363], "agreement_weight": -0.22798489034175873, "orthogonal_weight": -0.11691007018089294}, {"head": 1, "memory_to_active": 7.095543384552002, "contribution_to_active": 3.164083242416382, "mix_mean": 0.02097741700708866, "mix_low": 0.5993204116821289, "mix_high": 0.0, "read_cosine": 0.43729519844055176, "beta_mean": 0.4260425567626953, "nonfinite": 0.0, "weighted_memory_to_active": 3.377671241760254, "mix_std": 0.05776526406407356, "beta_std": 0.22843337059020996, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10146595537662506, "evidence_weights": [0.020262671634554863, -0.10526411980390549, 0.6739788055419922], "agreement_weight": 0.028887392953038216, "orthogonal_weight": 0.068265400826931}, {"head": 2, "memory_to_active": 2.8763222694396973, "contribution_to_active": 0.6361541748046875, "mix_mean": 0.015149453654885292, "mix_low": 0.6437911987304688, "mix_high": 0.0, "read_cosine": 0.29299330711364746, "beta_mean": 0.2466268241405487, "nonfinite": 0.0, "weighted_memory_to_active": 0.7344502210617065, "mix_std": 0.032657112926244736, "beta_std": 0.19810685515403748, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.059960003942251205, "evidence_weights": [0.06137651950120926, 0.047377292066812515, 0.25731414556503296], "agreement_weight": 0.009702431038022041, "orthogonal_weight": -0.20296767354011536}, {"head": 3, "memory_to_active": 7.697226524353027, "contribution_to_active": 2.5648534297943115, "mix_mean": 0.028665386140346527, "mix_low": 0.4915885925292969, "mix_high": 0.0, "read_cosine": 0.2839996814727783, "beta_mean": 0.3418436050415039, "nonfinite": 0.0, "weighted_memory_to_active": 2.678102493286133, "mix_std": 0.06487954407930374, "beta_std": 0.2712910771369934, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.0831640213727951, "evidence_weights": [-0.03600373864173889, -0.028009844943881035, -0.14870236814022064], "agreement_weight": -0.008830117993056774, "orthogonal_weight": -0.03648931160569191}, {"head": 4, "memory_to_active": 1.3407942056655884, "contribution_to_active": 0.06712895631790161, "mix_mean": 0.022684719413518906, "mix_low": 0.6164360046386719, "mix_high": 0.0, "read_cosine": 0.3104058504104614, "beta_mean": 0.022689545527100563, "nonfinite": 0.0, "weighted_memory_to_active": 0.046329569071531296, "mix_std": 0.059987396001815796, "beta_std": 0.043091535568237305, "beta_low": 0.62359619140625, "beta_high": 0.0, "orthogonal_to_active": 0.0008256735745817423, "evidence_weights": [0.09277231991291046, 0.05952462553977966, -0.12958084046840668], "agreement_weight": -0.1942894011735916, "orthogonal_weight": 0.016280796378850937}, {"head": 5, "memory_to_active": 0.0980406403541565, "contribution_to_active": 0.4725766181945801, "mix_mean": 0.5515428781509399, "mix_low": 0.013499259948730469, "mix_high": 0.03077411651611328, "read_cosine": 0.9313985109329224, "beta_mean": 0.01713266223669052, "nonfinite": 0.0, "weighted_memory_to_active": 0.052060436457395554, "mix_std": 0.28124478459358215, "beta_std": 0.017008185386657715, "beta_low": 0.46881103515625, "beta_high": 0.0, "orthogonal_to_active": 0.016125936061143875, "evidence_weights": [-0.17417162656784058, -0.21802301704883575, 0.37044453620910645], "agreement_weight": 0.28441107273101807, "orthogonal_weight": -0.704686164855957}]}, {"layer": 1, "memory_to_active": 2.5295047760009766, "contribution_to_active": 0.49202975630760193, "mix_mean": 0.21784687042236328, "mix_low": 0.36316922307014465, "mix_high": 0.0012658437481150031, "read_cosine": 0.29844337701797485, "beta_mean": 0.17685477435588837, "nonfinite": 0.0, "weighted_memory_to_active": 0.32949548959732056, "mix_std": 0.11379943788051605, "beta_std": 0.12751483917236328, "beta_low": 0.3514709770679474, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.03810583055019379, "output_influence": 0.27421462535858154, "residual_influence": 0.07846900820732117, "state_active": 0.027850791811943054, "state_memory": 0.03431515395641327, "state_peak": 43.55857849121094, "growth_active": 1.1458429098129272, "growth_memory": 1.446524739265442, "heads": [{"head": 0, "memory_to_active": 0.016558531671762466, "contribution_to_active": 0.4847126603126526, "mix_mean": 0.5415689945220947, "mix_low": 0.004300117492675781, "mix_high": 0.0029039382934570312, "read_cosine": 0.05069796368479729, "beta_mean": 0.00011626559717115015, "nonfinite": 0.0, "weighted_memory_to_active": 0.009561092592775822, "mix_std": 0.2823689579963684, "beta_std": 0.00018927408382296562, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009285672567784786, "evidence_weights": [-0.2097058892250061, -0.27044209837913513, 0.21661023795604706], "agreement_weight": 0.21125827729701996, "orthogonal_weight": -1.0126246213912964}, {"head": 1, "memory_to_active": 3.0814356803894043, "contribution_to_active": 0.31643444299697876, "mix_mean": 0.022292420268058777, "mix_low": 0.4891986846923828, "mix_high": 2.86102294921875e-06, "read_cosine": 0.5139674544334412, "beta_mean": 0.2667573392391205, "nonfinite": 0.0, "weighted_memory_to_active": 0.34905654191970825, "mix_std": 0.04378686845302582, "beta_std": 0.18288683891296387, "beta_low": 0.00115966796875, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.023350507020950317, "evidence_weights": [0.02446495182812214, -0.06353002041578293, 0.05572207272052765], "agreement_weight": 0.0006997068412601948, "orthogonal_weight": 0.21225778758525848}, {"head": 2, "memory_to_active": 0.03748537227511406, "contribution_to_active": 0.5610284805297852, "mix_mean": 0.6798605918884277, "mix_low": 0.00032901763916015625, "mix_high": 0.004688262939453125, "read_cosine": 0.4240303337574005, "beta_mean": 0.001372263883240521, "nonfinite": 0.0, "weighted_memory_to_active": 0.025182649493217468, "mix_std": 0.22635206580162048, "beta_std": 0.003917661961168051, "beta_low": 0.97235107421875, "beta_high": 0.0, "orthogonal_to_active": 0.01804104447364807, "evidence_weights": [-0.27381929755210876, -0.36938977241516113, 0.6386566162109375], "agreement_weight": 0.054349396377801895, "orthogonal_weight": -0.8341024518013}, {"head": 3, "memory_to_active": 3.209367275238037, "contribution_to_active": 0.19794555008411407, "mix_mean": 0.02482428029179573, "mix_low": 0.5136995315551758, "mix_high": 0.0, "read_cosine": 0.1865176558494568, "beta_mean": 0.3489193022251129, "nonfinite": 0.0, "weighted_memory_to_active": 0.2507292330265045, "mix_std": 0.06154158338904381, "beta_std": 0.2657107710838318, "beta_low": 0.064453125, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.10094252228736877, "evidence_weights": [0.03862112760543823, -0.01892048306763172, -0.016511941328644753], "agreement_weight": -0.02728160470724106, "orthogonal_weight": -0.48172837495803833}, {"head": 4, "memory_to_active": 4.069874286651611, "contribution_to_active": 1.2842267751693726, "mix_mean": 0.032370857894420624, "mix_low": 0.3187389373779297, "mix_high": 0.0, "read_cosine": 0.33797162771224976, "beta_mean": 0.1664539873600006, "nonfinite": 0.0, "weighted_memory_to_active": 1.2668365240097046, "mix_std": 0.05310998111963272, "beta_std": 0.12789246439933777, "beta_low": 0.03875732421875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.04667116701602936, "evidence_weights": [-0.015415020287036896, -0.005397681146860123, 0.16372588276863098], "agreement_weight": -0.004982451908290386, "orthogonal_weight": 0.05672581121325493}, {"head": 5, "memory_to_active": 4.762307167053223, "contribution_to_active": 0.10783053934574127, "mix_mean": 0.006164028774946928, "mix_low": 0.8527488708496094, "mix_high": 0.0, "read_cosine": 0.2774752378463745, "beta_mean": 0.27750951051712036, "nonfinite": 0.0, "weighted_memory_to_active": 0.0756068229675293, "mix_std": 0.015637168660759926, "beta_std": 0.18449199199676514, "beta_low": 0.0321044921875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.03034409135580063, "evidence_weights": [0.1641637682914734, 0.08082268387079239, 0.09886645525693893], "agreement_weight": -0.11854609847068787, "orthogonal_weight": 0.4175306260585785}]}, {"layer": 2, "memory_to_active": 1.0067381858825684, "contribution_to_active": 0.35820597410202026, "mix_mean": 0.41484200954437256, "mix_low": 0.23849520087242126, "mix_high": 0.004511356353759766, "read_cosine": 0.2971046566963196, "beta_mean": 0.07137580960988998, "nonfinite": 0.0, "weighted_memory_to_active": 0.07078981399536133, "mix_std": 0.18507027626037598, "beta_std": 0.06465505063533783, "beta_low": 0.6127116084098816, "beta_high": 0.00014241537428461015, "orthogonal_to_active": 0.016305942088365555, "output_influence": 0.41267967224121094, "residual_influence": 0.11869756132364273, "state_active": 0.03919730335474014, "state_memory": 0.01344936154782772, "state_peak": 29.68842315673828, "growth_active": 1.0444788932800293, "growth_memory": 1.5546705722808838, "heads": [{"head": 0, "memory_to_active": 0.004304754082113504, "contribution_to_active": 0.36497020721435547, "mix_mean": 0.649905800819397, "mix_low": 0.0005950927734375, "mix_high": 0.008985519409179688, "read_cosine": 0.14354130625724792, "beta_mean": 3.630136779975146e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0028522040229290724, "mix_std": 0.26300573348999023, "beta_std": 5.931988562224433e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002969579305499792, "evidence_weights": [-0.37497347593307495, -0.21585536003112793, 0.19952256977558136], "agreement_weight": 0.13153497874736786, "orthogonal_weight": -1.0799819231033325}, {"head": 1, "memory_to_active": 3.239664077758789, "contribution_to_active": 0.04556760936975479, "mix_mean": 0.005920318886637688, "mix_low": 0.8647422790527344, "mix_high": 0.0, "read_cosine": 0.10986433923244476, "beta_mean": 0.19522430002689362, "nonfinite": 0.0, "weighted_memory_to_active": 0.037930481135845184, "mix_std": 0.013344199396669865, "beta_std": 0.14141975343227386, "beta_low": 0.00360107421875, "beta_high": 0.0, "orthogonal_to_active": 0.010611210949718952, "evidence_weights": [0.13264063000679016, 0.10424702614545822, 0.5634738206863403], "agreement_weight": 0.03265358507633209, "orthogonal_weight": 0.32353055477142334}, {"head": 2, "memory_to_active": 0.0066826483234763145, "contribution_to_active": 0.4226629137992859, "mix_mean": 0.5535169839859009, "mix_low": 0.001018524169921875, "mix_high": 0.003681182861328125, "read_cosine": 0.3937842845916748, "beta_mean": 2.1518262656172737e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0027931146323680878, "mix_std": 0.2737841010093689, "beta_std": 3.740147076314315e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00015917557175271213, "evidence_weights": [-0.24764016270637512, -0.28986304998397827, 0.5703829526901245], "agreement_weight": -0.02160029672086239, "orthogonal_weight": 0.0693405494093895}, {"head": 3, "memory_to_active": 0.10346846282482147, "contribution_to_active": 0.5550514459609985, "mix_mean": 0.5958530902862549, "mix_low": 0.00020885467529296875, "mix_high": 0.006855010986328125, "read_cosine": 0.5854194164276123, "beta_mean": 0.015223797410726547, "nonfinite": 0.0, "weighted_memory_to_active": 0.06644099950790405, "mix_std": 0.25261592864990234, "beta_std": 0.023837976157665253, "beta_low": 0.60888671875, "beta_high": 0.0, "orthogonal_to_active": 0.04765152558684349, "evidence_weights": [-0.20203574001789093, -0.32790860533714294, 0.736263632774353], "agreement_weight": 0.05831523984670639, "orthogonal_weight": -0.901964545249939}, {"head": 4, "memory_to_active": 2.6779961585998535, "contribution_to_active": 0.23649263381958008, "mix_mean": 0.021267613396048546, "mix_low": 0.563939094543457, "mix_high": 0.0, "read_cosine": 0.1910567432641983, "beta_mean": 0.21771924197673798, "nonfinite": 0.0, "weighted_memory_to_active": 0.31013041734695435, "mix_std": 0.051364295184612274, "beta_std": 0.22253648936748505, "beta_low": 0.06378173828125, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.0351787768304348, "evidence_weights": [0.022313280031085014, -0.0014054898638278246, 0.04070030897855759], "agreement_weight": 0.023909544572234154, "orthogonal_weight": -0.13259413838386536}, {"head": 5, "memory_to_active": 0.008313192054629326, "contribution_to_active": 0.5244911909103394, "mix_mean": 0.6625880002975464, "mix_low": 0.0004673004150390625, "mix_high": 0.007546424865722656, "read_cosine": 0.3589617609977722, "beta_mean": 2.970574496430345e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004591633565723896, "mix_std": 0.2563073933124542, "beta_std": 3.932361141778529e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012653849553316832, "evidence_weights": [-0.27774104475975037, -0.28833404183387756, 0.8921564221382141], "agreement_weight": 0.2779477834701538, "orthogonal_weight": -0.32260939478874207}]}, {"layer": 3, "memory_to_active": 0.42622682452201843, "contribution_to_active": 0.4904981851577759, "mix_mean": 0.5242393016815186, "mix_low": 0.05054124444723129, "mix_high": 0.011551380157470703, "read_cosine": 0.1861247718334198, "beta_mean": 0.024170130491256714, "nonfinite": 0.0, "weighted_memory_to_active": 0.11951575428247452, "mix_std": 0.25636643171310425, "beta_std": 0.023556530475616455, "beta_low": 0.774993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0487343966960907, "output_influence": 0.4363824725151062, "residual_influence": 0.11052058637142181, "state_active": 0.02297045662999153, "state_memory": 0.006266866344958544, "state_peak": 25.164031982421875, "growth_active": 1.0701583623886108, "growth_memory": 1.585153341293335, "heads": [{"head": 0, "memory_to_active": 0.33176636695861816, "contribution_to_active": 0.617266058921814, "mix_mean": 0.6119015216827393, "mix_low": 0.0014371871948242188, "mix_high": 0.022866249084472656, "read_cosine": 0.23147548735141754, "beta_mean": 0.009111134335398674, "nonfinite": 0.0, "weighted_memory_to_active": 0.2127493917942047, "mix_std": 0.2979680597782135, "beta_std": 0.010452652350068092, "beta_low": 0.72430419921875, "beta_high": 0.0, "orthogonal_to_active": 0.15705332159996033, "evidence_weights": [-0.21472512185573578, -0.2491326630115509, 1.058775544166565], "agreement_weight": 0.11242656409740448, "orthogonal_weight": -0.7736710906028748}, {"head": 1, "memory_to_active": 0.005345349200069904, "contribution_to_active": 0.45742207765579224, "mix_mean": 0.5961952805519104, "mix_low": 0.0026769638061523438, "mix_high": 0.011654853820800781, "read_cosine": 0.1727706491947174, "beta_mean": 2.7417925593908876e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0033522990997880697, "mix_std": 0.2915651500225067, "beta_std": 6.916242273291573e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0027051526121795177, "evidence_weights": [-0.2707328498363495, -0.21826456487178802, 0.6984242796897888], "agreement_weight": 0.6051317453384399, "orthogonal_weight": -0.8779008388519287}, {"head": 2, "memory_to_active": 1.9793815612792969, "contribution_to_active": 0.2865843176841736, "mix_mean": 0.058692917227745056, "mix_low": 0.29025840759277344, "mix_high": 9.5367431640625e-07, "read_cosine": 0.20647430419921875, "beta_mean": 0.1312173753976822, "nonfinite": 0.0, "weighted_memory_to_active": 0.3205535411834717, "mix_std": 0.10877220332622528, "beta_std": 0.11832557618618011, "beta_low": 0.0218505859375, "beta_high": 0.0, "orthogonal_to_active": 0.000958206772338599, "evidence_weights": [-0.0995069220662117, -0.0070654721930623055, -0.028396321460604668], "agreement_weight": -0.01276476588100195, "orthogonal_weight": 0.003778887912631035}, {"head": 3, "memory_to_active": 0.01315456721931696, "contribution_to_active": 0.544002890586853, "mix_mean": 0.607567548751831, "mix_low": 0.00400543212890625, "mix_high": 0.00870513916015625, "read_cosine": 0.2659609317779541, "beta_mean": 6.678383215330541e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008849499747157097, "mix_std": 0.284305214881897, "beta_std": 0.0001428004034096375, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002144047524780035, "evidence_weights": [-0.26226261258125305, -0.22127202153205872, 2.3255434036254883], "agreement_weight": 2.3254499435424805, "orthogonal_weight": -0.30880123376846313}, {"head": 4, "memory_to_active": 0.00970850046724081, "contribution_to_active": 0.3871232867240906, "mix_mean": 0.5384243726730347, "mix_low": 0.00234222412109375, "mix_high": 0.0028228759765625, "read_cosine": 0.15554608404636383, "beta_mean": 5.228819645708427e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006406875792890787, "mix_std": 0.2873302400112152, "beta_std": 8.203242759918794e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005997112486511469, "evidence_weights": [-0.2448069453239441, -0.22392567992210388, 0.07232195883989334], "agreement_weight": -0.052247874438762665, "orthogonal_weight": -1.0151480436325073}, {"head": 5, "memory_to_active": 0.21800439059734344, "contribution_to_active": 0.6505903601646423, "mix_mean": 0.7326539754867554, "mix_low": 0.0025272369384765625, "mix_high": 0.023258209228515625, "read_cosine": 0.08452114462852478, "beta_mean": 0.004545794799923897, "nonfinite": 0.0, "weighted_memory_to_active": 0.16518288850784302, "mix_std": 0.26825767755508423, "beta_std": 0.01226696278899908, "beta_low": 0.90380859375, "beta_high": 0.0, "orthogonal_to_active": 0.12354852259159088, "evidence_weights": [-0.2616203725337982, -0.3452795147895813, 0.5401359796524048], "agreement_weight": 0.24159106612205505, "orthogonal_weight": -0.7743722796440125}]}, {"layer": 4, "memory_to_active": 0.9982355833053589, "contribution_to_active": 0.4779331088066101, "mix_mean": 0.37553146481513977, "mix_low": 0.22795328497886658, "mix_high": 0.01179361343383789, "read_cosine": 0.29796043038368225, "beta_mean": 0.1022961288690567, "nonfinite": 0.0, "weighted_memory_to_active": 0.12840870022773743, "mix_std": 0.16372713446617126, "beta_std": 0.09475207328796387, "beta_low": 0.4019266963005066, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.056193843483924866, "output_influence": 0.5071498155593872, "residual_influence": 0.1647954285144806, "state_active": 0.03810743987560272, "state_memory": 0.015206210315227509, "state_peak": 38.48539352416992, "growth_active": 1.2071659564971924, "growth_memory": 1.485174298286438, "heads": [{"head": 0, "memory_to_active": 0.14025720953941345, "contribution_to_active": 0.8144247531890869, "mix_mean": 0.686995267868042, "mix_low": 0.01035308837890625, "mix_high": 0.03110504150390625, "read_cosine": 0.4316007196903229, "beta_mean": 0.2255038619041443, "nonfinite": 0.0, "weighted_memory_to_active": 0.12043121457099915, "mix_std": 0.347898930311203, "beta_std": 0.22567662596702576, "beta_low": 0.26251220703125, "beta_high": 0.0, "orthogonal_to_active": 0.08797823637723923, "evidence_weights": [-0.2454201877117157, -0.3573513925075531, -0.2184276282787323], "agreement_weight": 0.0028872594702988863, "orthogonal_weight": 0.8599894046783447}, {"head": 1, "memory_to_active": 0.04113943129777908, "contribution_to_active": 0.6669142246246338, "mix_mean": 0.7067508697509766, "mix_low": 6.389617919921875e-05, "mix_high": 0.020198822021484375, "read_cosine": 0.5702007412910461, "beta_mean": 0.000809550634585321, "nonfinite": 0.0, "weighted_memory_to_active": 0.0309080071747303, "mix_std": 0.23687046766281128, "beta_std": 0.006475403904914856, "beta_low": 0.98199462890625, "beta_high": 0.0, "orthogonal_to_active": 0.015284966677427292, "evidence_weights": [-0.23567797243595123, -0.36484336853027344, 0.5670979619026184], "agreement_weight": 0.365509033203125, "orthogonal_weight": -0.6862329840660095}, {"head": 2, "memory_to_active": 1.4413291215896606, "contribution_to_active": 0.3692808449268341, "mix_mean": 0.05822112038731575, "mix_low": 0.3137636184692383, "mix_high": 3.4332275390625e-05, "read_cosine": 0.21414834260940552, "beta_mean": 0.051756370812654495, "nonfinite": 0.0, "weighted_memory_to_active": 0.2866625487804413, "mix_std": 0.10277310758829117, "beta_std": 0.05581501126289368, "beta_low": 0.13995361328125, "beta_high": 0.0, "orthogonal_to_active": 0.1085524782538414, "evidence_weights": [0.025087689980864525, -0.06476565450429916, 0.36730828881263733], "agreement_weight": 0.12226319313049316, "orthogonal_weight": -0.49169525504112244}, {"head": 3, "memory_to_active": 1.865442156791687, "contribution_to_active": 0.13892832398414612, "mix_mean": 0.03191304951906204, "mix_low": 0.4041614532470703, "mix_high": 0.0, "read_cosine": 0.2160937786102295, "beta_mean": 0.12952737510204315, "nonfinite": 0.0, "weighted_memory_to_active": 0.18792980909347534, "mix_std": 0.05427778139710426, "beta_std": 0.11950257420539856, "beta_low": 0.027099609375, "beta_high": 0.0, "orthogonal_to_active": 0.002981565659865737, "evidence_weights": [-0.022532297298312187, 0.04872729256749153, -0.2531719207763672], "agreement_weight": 0.08080173283815384, "orthogonal_weight": 0.02277604676783085}, {"head": 4, "memory_to_active": 0.00527748791500926, "contribution_to_active": 0.635227382183075, "mix_mean": 0.7548850774765015, "mix_low": 2.384185791015625e-05, "mix_high": 0.019418716430664062, "read_cosine": 0.12497549504041672, "beta_mean": 1.1869749869219959e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037982792127877474, "mix_std": 0.2092656046152115, "beta_std": 3.1003295589471236e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002425496932119131, "evidence_weights": [-0.3265359103679657, -0.31453830003738403, 0.24114541709423065], "agreement_weight": 0.3080221712589264, "orthogonal_weight": -0.6762704253196716}, {"head": 5, "memory_to_active": 2.4959678649902344, "contribution_to_active": 0.24282336235046387, "mix_mean": 0.014423273503780365, "mix_low": 0.6393537521362305, "mix_high": 4.76837158203125e-06, "read_cosine": 0.23074348270893097, "beta_mean": 0.20616771280765533, "nonfinite": 0.0, "weighted_memory_to_active": 0.14072231948375702, "mix_std": 0.03127680718898773, "beta_std": 0.16101185977458954, "beta_low": 0.0, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.11994029581546783, "evidence_weights": [0.056975170969963074, -0.01622042804956436, -0.26629844307899475], "agreement_weight": -0.09345249831676483, "orthogonal_weight": 1.1142241954803467}]}, {"layer": 5, "memory_to_active": 0.3593733012676239, "contribution_to_active": 0.5470695495605469, "mix_mean": 0.5212572813034058, "mix_low": 0.05611690133810043, "mix_high": 0.02020263671875, "read_cosine": 0.07993379980325699, "beta_mean": 0.0511164553463459, "nonfinite": 0.0, "weighted_memory_to_active": 0.1250474601984024, "mix_std": 0.25110167264938354, "beta_std": 0.06653839349746704, "beta_low": 0.6867777705192566, "beta_high": 0.0, "orthogonal_to_active": 0.05438031256198883, "output_influence": 0.5860871076583862, "residual_influence": 0.1966341882944107, "state_active": 0.03863309323787689, "state_memory": 0.009512708522379398, "state_peak": 62.09172058105469, "growth_active": 1.0843288898468018, "growth_memory": 1.2507644891738892, "heads": [{"head": 0, "memory_to_active": 0.16972076892852783, "contribution_to_active": 0.5968351364135742, "mix_mean": 0.5026957988739014, "mix_low": 0.006743431091308594, "mix_high": 0.0023632049560546875, "read_cosine": -0.09331019222736359, "beta_mean": 0.015929220244288445, "nonfinite": 0.0, "weighted_memory_to_active": 0.06430570036172867, "mix_std": 0.2786751389503479, "beta_std": 0.029072605073451996, "beta_low": 0.61236572265625, "beta_high": 0.0, "orthogonal_to_active": 0.029016781598329544, "evidence_weights": [-0.170503169298172, -0.18944062292575836, 0.1161060556769371], "agreement_weight": -0.023296676576137543, "orthogonal_weight": -0.4776376187801361}, {"head": 1, "memory_to_active": 0.0040669627487659454, "contribution_to_active": 0.5570017099380493, "mix_mean": 0.7138750553131104, "mix_low": 0.000759124755859375, "mix_high": 0.038130760192871094, "read_cosine": 0.2036704272031784, "beta_mean": 9.465284165344201e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002562384121119976, "mix_std": 0.2623450458049774, "beta_std": 2.2451960830949247e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023933483753353357, "evidence_weights": [-0.24588777124881744, -0.2852119207382202, 0.9469109773635864], "agreement_weight": 0.7223126292228699, "orthogonal_weight": -0.978123128414154}, {"head": 2, "memory_to_active": 0.014555234462022781, "contribution_to_active": 0.6111674308776855, "mix_mean": 0.6658796072006226, "mix_low": 0.0017757415771484375, "mix_high": 0.02126789093017578, "read_cosine": -0.48906856775283813, "beta_mean": 7.729485514573753e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009252992458641529, "mix_std": 0.2611832916736603, "beta_std": 0.0003064608608838171, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.008703681640326977, "evidence_weights": [-0.2570749819278717, -0.31300726532936096, 0.1802918016910553], "agreement_weight": -0.19801971316337585, "orthogonal_weight": -1.054661512374878}, {"head": 3, "memory_to_active": 1.4685142040252686, "contribution_to_active": 0.24344627559185028, "mix_mean": 0.11448319256305695, "mix_low": 0.12064075469970703, "mix_high": 0.0, "read_cosine": 0.14858108758926392, "beta_mean": 0.09561579674482346, "nonfinite": 0.0, "weighted_memory_to_active": 0.3488600254058838, "mix_std": 0.14564117789268494, "beta_std": 0.1506851315498352, "beta_low": 0.21148681640625, "beta_high": 0.0, "orthogonal_to_active": 0.14238111674785614, "evidence_weights": [-0.07522231340408325, -0.09704533219337463, 0.12268329411745071], "agreement_weight": 0.049152206629514694, "orthogonal_weight": -0.5032440423965454}, {"head": 4, "memory_to_active": 0.423111230134964, "contribution_to_active": 0.5869808197021484, "mix_mean": 0.3209012746810913, "mix_low": 0.20514583587646484, "mix_high": 0.004845619201660156, "read_cosine": 0.5472712516784668, "beta_mean": 0.19486168026924133, "nonfinite": 0.0, "weighted_memory_to_active": 0.27294737100601196, "mix_std": 0.3377448320388794, "beta_std": 0.21814587712287903, "beta_low": 0.2994384765625, "beta_high": 0.0, "orthogonal_to_active": 0.11373568326234818, "evidence_weights": [-0.11210674792528152, -0.2511681020259857, -0.05538015067577362], "agreement_weight": 0.028637630864977837, "orthogonal_weight": 0.6478517055511475}, {"head": 5, "memory_to_active": 0.07627126574516296, "contribution_to_active": 0.6869858503341675, "mix_mean": 0.809708833694458, "mix_low": 0.001636505126953125, "mix_high": 0.05460834503173828, "read_cosine": 0.16245882213115692, "beta_mean": 0.00020528133609332144, "nonfinite": 0.0, "weighted_memory_to_active": 0.05235621705651283, "mix_std": 0.22102059423923492, "beta_std": 0.000997815397568047, "beta_low": 0.9974365234375, "beta_high": 0.0, "orthogonal_to_active": 0.030051257461309433, "evidence_weights": [-0.3030068576335907, -0.3265782296657562, 1.8272510766983032], "agreement_weight": 0.5405126214027405, "orthogonal_weight": -0.6031757593154907}]}, {"layer": 6, "memory_to_active": 0.22546106576919556, "contribution_to_active": 0.6373029947280884, "mix_mean": 0.5979416966438293, "mix_low": 0.03270991891622543, "mix_high": 0.02847910113632679, "read_cosine": 0.3192555904388428, "beta_mean": 0.060927171260118484, "nonfinite": 0.0, "weighted_memory_to_active": 0.11532275378704071, "mix_std": 0.2402752786874771, "beta_std": 0.07235598564147949, "beta_low": 0.6151937246322632, "beta_high": 0.0, "orthogonal_to_active": 0.0788663700222969, "output_influence": 0.5870196223258972, "residual_influence": 0.22218360006809235, "state_active": 0.04167154058814049, "state_memory": 0.005314905662089586, "state_peak": 76.2930908203125, "growth_active": 1.1906263828277588, "growth_memory": 1.1323925256729126, "heads": [{"head": 0, "memory_to_active": 0.1135338693857193, "contribution_to_active": 0.7822862267494202, "mix_mean": 0.8229531645774841, "mix_low": 0.00021839141845703125, "mix_high": 0.06377887725830078, "read_cosine": 0.30703502893447876, "beta_mean": 0.011379214003682137, "nonfinite": 0.0, "weighted_memory_to_active": 0.10068453848361969, "mix_std": 0.2034328728914261, "beta_std": 0.033796753734350204, "beta_low": 0.8291015625, "beta_high": 0.0, "orthogonal_to_active": 0.06589900702238083, "evidence_weights": [-0.2970380485057831, -0.3455522954463959, 1.1757185459136963], "agreement_weight": 0.9166325330734253, "orthogonal_weight": -0.7088794708251953}, {"head": 1, "memory_to_active": 0.004205701872706413, "contribution_to_active": 0.533240795135498, "mix_mean": 0.5993390679359436, "mix_low": 0.0005445480346679688, "mix_high": 0.016303062438964844, "read_cosine": 0.48506152629852295, "beta_mean": 9.174220394925214e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0019556416664272547, "mix_std": 0.2818487286567688, "beta_std": 1.7667342035565525e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0010646390728652477, "evidence_weights": [-0.18921072781085968, -0.3006449341773987, 1.1116955280303955], "agreement_weight": 0.12524385750293732, "orthogonal_weight": -0.6990795135498047}, {"head": 2, "memory_to_active": 0.629830539226532, "contribution_to_active": 0.10550122708082199, "mix_mean": 0.06005645915865898, "mix_low": 0.18731021881103516, "mix_high": 0.0, "read_cosine": 0.2680523097515106, "beta_mean": 0.0200765710324049, "nonfinite": 0.0, "weighted_memory_to_active": 0.06753268837928772, "mix_std": 0.0863715335726738, "beta_std": 0.023596838116645813, "beta_low": 0.33038330078125, "beta_high": 0.0, "orthogonal_to_active": 0.013413088396191597, "evidence_weights": [0.054316338151693344, -0.07404253631830215, 0.37307414412498474], "agreement_weight": 0.13723281025886536, "orthogonal_weight": -0.23605401813983917}, {"head": 3, "memory_to_active": 0.13252246379852295, "contribution_to_active": 0.6794997453689575, "mix_mean": 0.6694722175598145, "mix_low": 0.0072040557861328125, "mix_high": 0.02098369598388672, "read_cosine": 0.46333047747612, "beta_mean": 0.21577128767967224, "nonfinite": 0.0, "weighted_memory_to_active": 0.0958314910531044, "mix_std": 0.32328328490257263, "beta_std": 0.2458958476781845, "beta_low": 0.212158203125, "beta_high": 0.0, "orthogonal_to_active": 0.05862632393836975, "evidence_weights": [-0.16568103432655334, -0.3116907775402069, 0.10797078162431717], "agreement_weight": -0.01270799245685339, "orthogonal_weight": 0.699845552444458}, {"head": 4, "memory_to_active": 0.4517044723033905, "contribution_to_active": 1.0802726745605469, "mix_mean": 0.7380249500274658, "mix_low": 0.000843048095703125, "mix_high": 0.06051349639892578, "read_cosine": 0.10117964446544647, "beta_mean": 0.11822289228439331, "nonfinite": 0.0, "weighted_memory_to_active": 0.4104900062084198, "mix_std": 0.3068064749240875, "beta_std": 0.13057225942611694, "beta_low": 0.319580078125, "beta_high": 0.0, "orthogonal_to_active": 0.3192889392375946, "evidence_weights": [-0.16137835383415222, -0.5063310265541077, -0.7762395739555359], "agreement_weight": -0.015748122707009315, "orthogonal_weight": 0.7942737936973572}, {"head": 5, "memory_to_active": 0.020969295874238014, "contribution_to_active": 0.643017053604126, "mix_mean": 0.6978039741516113, "mix_low": 0.0001392364501953125, "mix_high": 0.009295463562011719, "read_cosine": 0.2908743619918823, "beta_mean": 0.00010386385838501155, "nonfinite": 0.0, "weighted_memory_to_active": 0.015442167408764362, "mix_std": 0.2399086356163025, "beta_std": 0.0002565559116192162, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01490618847310543, "evidence_weights": [-0.23258507251739502, -0.31538328528404236, 0.3536098897457123], "agreement_weight": 0.2262965738773346, "orthogonal_weight": -1.0382949113845825}]}, {"layer": 7, "memory_to_active": 1.7933396100997925, "contribution_to_active": 0.27519142627716064, "mix_mean": 0.18054872751235962, "mix_low": 0.2166072577238083, "mix_high": 0.0010093053570017219, "read_cosine": 0.24286118149757385, "beta_mean": 0.13795779645442963, "nonfinite": 0.0, "weighted_memory_to_active": 0.20011967420578003, "mix_std": 0.14889594912528992, "beta_std": 0.0965951681137085, "beta_low": 0.2327372282743454, "beta_high": 0.0002543131704442203, "orthogonal_to_active": 0.08702709525823593, "output_influence": 0.2900925576686859, "residual_influence": 0.07346523553133011, "state_active": 0.013563258573412895, "state_memory": 0.023379923775792122, "state_peak": 30.879776000976562, "growth_active": 1.1364657878875732, "growth_memory": 1.263602375984192, "heads": [{"head": 0, "memory_to_active": 2.7265994548797607, "contribution_to_active": 0.17129357159137726, "mix_mean": 0.03930790722370148, "mix_low": 0.4036121368408203, "mix_high": 0.0, "read_cosine": 0.3557392358779907, "beta_mean": 0.1858658492565155, "nonfinite": 0.0, "weighted_memory_to_active": 0.20623792707920074, "mix_std": 0.07732073217630386, "beta_std": 0.13154414296150208, "beta_low": 0.00115966796875, "beta_high": 0.0006103515625, "orthogonal_to_active": 0.0008923721616156399, "evidence_weights": [0.04537768289446831, -0.08978157490491867, 0.029499245807528496], "agreement_weight": -0.027910400182008743, "orthogonal_weight": -0.005193524528294802}, {"head": 1, "memory_to_active": 0.959412693977356, "contribution_to_active": 0.4043457806110382, "mix_mean": 0.2285979688167572, "mix_low": 0.10859966278076172, "mix_high": 0.005280494689941406, "read_cosine": 0.387680321931839, "beta_mean": 0.09425956010818481, "nonfinite": 0.0, "weighted_memory_to_active": 0.33663564920425415, "mix_std": 0.24766655266284943, "beta_std": 0.06332656741142273, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04704669490456581, "evidence_weights": [-0.2181016355752945, -0.10683918744325638, -0.2994421720504761], "agreement_weight": 0.058939579874277115, "orthogonal_weight": 0.16659821569919586}, {"head": 2, "memory_to_active": 0.5435245037078857, "contribution_to_active": 0.38975563645362854, "mix_mean": 0.24638332426548004, "mix_low": 0.012644767761230469, "mix_high": 0.00044918060302734375, "read_cosine": 0.11894504725933075, "beta_mean": 0.028981313109397888, "nonfinite": 0.0, "weighted_memory_to_active": 0.21557073295116425, "mix_std": 0.21750469505786896, "beta_std": 0.03433798998594284, "beta_low": 0.23260498046875, "beta_high": 0.0, "orthogonal_to_active": 0.12991926074028015, "evidence_weights": [-0.09949228167533875, -0.18728004395961761, 0.883553147315979], "agreement_weight": 0.014884761534631252, "orthogonal_weight": -0.620468020439148}, {"head": 3, "memory_to_active": 4.963380336761475, "contribution_to_active": 0.10114817321300507, "mix_mean": 0.0343455970287323, "mix_low": 0.3147926330566406, "mix_high": 0.0, "read_cosine": 0.09333857893943787, "beta_mean": 0.45800575613975525, "nonfinite": 0.0, "weighted_memory_to_active": 0.35470375418663025, "mix_std": 0.04875361546874046, "beta_std": 0.27934902906417847, "beta_low": 0.00091552734375, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.3079974055290222, "evidence_weights": [0.07763189822435379, -0.04054545983672142, 0.7098954916000366], "agreement_weight": 0.049573712050914764, "orthogonal_weight": -0.9365943670272827}, {"head": 4, "memory_to_active": 1.5274169445037842, "contribution_to_active": 0.10977178066968918, "mix_mean": 0.027581527829170227, "mix_low": 0.458587646484375, "mix_high": 0.0, "read_cosine": 0.2510652542114258, "beta_mean": 0.060482144355773926, "nonfinite": 0.0, "weighted_memory_to_active": 0.06719331443309784, "mix_std": 0.052161335945129395, "beta_std": 0.0705864280462265, "beta_low": 0.161865234375, "beta_high": 0.0, "orthogonal_to_active": 0.016859466210007668, "evidence_weights": [0.11236516386270523, -0.03172457590699196, 0.19656482338905334], "agreement_weight": 0.034521616995334625, "orthogonal_weight": -0.29276493191719055}, {"head": 5, "memory_to_active": 0.0397033765912056, "contribution_to_active": 0.47483357787132263, "mix_mean": 0.5070761442184448, "mix_low": 0.0014066696166992188, "mix_high": 0.0003261566162109375, "read_cosine": 0.2503984570503235, "beta_mean": 0.00015215248276945204, "nonfinite": 0.0, "weighted_memory_to_active": 0.02037663199007511, "mix_std": 0.24996870756149292, "beta_std": 0.00042685301741585135, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.01944733038544655, "evidence_weights": [-0.18544568121433258, -0.24161063134670258, 0.48195865750312805], "agreement_weight": 0.2430890053510666, "orthogonal_weight": -1.0156803131103516}]}, {"layer": 8, "memory_to_active": 0.13802668452262878, "contribution_to_active": 0.5740593671798706, "mix_mean": 0.5107163190841675, "mix_low": 0.009700457565486431, "mix_high": 0.0019903182983398438, "read_cosine": 0.38132160902023315, "beta_mean": 0.013051982969045639, "nonfinite": 0.0, "weighted_memory_to_active": 0.0712694302201271, "mix_std": 0.24189093708992004, "beta_std": 0.018827546387910843, "beta_low": 0.7825927734375, "beta_high": 0.0, "orthogonal_to_active": 0.0655214861035347, "output_influence": 0.5282236337661743, "residual_influence": 0.20221394300460815, "state_active": 0.04144669696688652, "state_memory": 0.0048047685995697975, "state_peak": 64.68185424804688, "growth_active": 1.1104549169540405, "growth_memory": 1.052427053451538, "heads": [{"head": 0, "memory_to_active": 0.06033772602677345, "contribution_to_active": 0.5789099335670471, "mix_mean": 0.6114538908004761, "mix_low": 0.000469207763671875, "mix_high": 0.0014848709106445312, "read_cosine": 0.448957622051239, "beta_mean": 0.003049787599593401, "nonfinite": 0.0, "weighted_memory_to_active": 0.043556053191423416, "mix_std": 0.24822530150413513, "beta_std": 0.012461996637284756, "beta_low": 0.93505859375, "beta_high": 0.0, "orthogonal_to_active": 0.027358975261449814, "evidence_weights": [-0.15457181632518768, -0.3757227063179016, 0.5549018979072571], "agreement_weight": 0.27165618538856506, "orthogonal_weight": -0.7931786179542542}, {"head": 1, "memory_to_active": 0.05710805952548981, "contribution_to_active": 0.5515608787536621, "mix_mean": 0.5322678089141846, "mix_low": 0.00017642974853515625, "mix_high": 0.00217437744140625, "read_cosine": 0.056148726493120193, "beta_mean": 0.0006753149209544063, "nonfinite": 0.0, "weighted_memory_to_active": 0.0249934121966362, "mix_std": 0.23270291090011597, "beta_std": 0.005167161114513874, "beta_low": 0.9932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.01846270263195038, "evidence_weights": [-0.1929061859846115, -0.2729039788246155, 0.5857837200164795], "agreement_weight": 0.07779968529939651, "orthogonal_weight": -0.7752218842506409}, {"head": 2, "memory_to_active": 0.01035250909626484, "contribution_to_active": 0.6974828243255615, "mix_mean": 0.6936479806900024, "mix_low": 0.00026416778564453125, "mix_high": 0.0037641525268554688, "read_cosine": 0.4074823260307312, "beta_mean": 0.0001176122241304256, "nonfinite": 0.0, "weighted_memory_to_active": 0.00846308283507824, "mix_std": 0.21843105554580688, "beta_std": 0.0034332829527556896, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.004966463427990675, "evidence_weights": [-0.2201903909444809, -0.3454907536506653, 0.9717903137207031], "agreement_weight": 0.336409330368042, "orthogonal_weight": -0.6701814532279968}, {"head": 3, "memory_to_active": 0.10476724803447723, "contribution_to_active": 0.6366778016090393, "mix_mean": 0.59097820520401, "mix_low": 0.0006580352783203125, "mix_high": 0.004238128662109375, "read_cosine": 0.7856220602989197, "beta_mean": 0.007835743948817253, "nonfinite": 0.0, "weighted_memory_to_active": 0.07804043591022491, "mix_std": 0.2564241290092468, "beta_std": 0.008086880668997765, "beta_low": 0.7320556640625, "beta_high": 0.0, "orthogonal_to_active": 0.024495378136634827, "evidence_weights": [-0.17033028602600098, -0.2972283959388733, 0.5238490104675293], "agreement_weight": 0.2607610523700714, "orthogonal_weight": -0.5882977247238159}, {"head": 4, "memory_to_active": 0.23800130188465118, "contribution_to_active": 0.36011040210723877, "mix_mean": 0.37206485867500305, "mix_low": 0.0009822845458984375, "mix_high": 1.430511474609375e-05, "read_cosine": 0.09620624035596848, "beta_mean": 0.0006386588793247938, "nonfinite": 0.0, "weighted_memory_to_active": 0.09017384052276611, "mix_std": 0.23455289006233215, "beta_std": 0.013767893426120281, "beta_low": 0.99432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.016489887610077858, "evidence_weights": [-0.14712835848331451, -0.2049221694469452, 0.6771925091743469], "agreement_weight": 0.03772152587771416, "orthogonal_weight": -0.18631848692893982}, {"head": 5, "memory_to_active": 0.3575931191444397, "contribution_to_active": 0.6196138858795166, "mix_mean": 0.2638852894306183, "mix_low": 0.055652618408203125, "mix_high": 0.00026607513427734375, "read_cosine": 0.4935126006603241, "beta_mean": 0.06599477678537369, "nonfinite": 0.0, "weighted_memory_to_active": 0.1823897510766983, "mix_std": 0.26100921630859375, "beta_std": 0.07004804909229279, "beta_low": 0.041015625, "beta_high": 0.0, "orthogonal_to_active": 0.3013554811477661, "evidence_weights": [-0.15022557973861694, -0.13921351730823517, -0.1148596704006195], "agreement_weight": -0.004735434427857399, "orthogonal_weight": 1.9556176662445068}]}, {"layer": 9, "memory_to_active": 0.7921416759490967, "contribution_to_active": 0.7175946235656738, "mix_mean": 0.4043712019920349, "mix_low": 0.0492456778883934, "mix_high": 0.006099383346736431, "read_cosine": 0.34660542011260986, "beta_mean": 0.0312076173722744, "nonfinite": 0.0, "weighted_memory_to_active": 0.36076176166534424, "mix_std": 0.2483023703098297, "beta_std": 0.025018714368343353, "beta_low": 0.6804403066635132, "beta_high": 0.0, "orthogonal_to_active": 0.02964678592979908, "output_influence": 0.5612349510192871, "residual_influence": 0.17636168003082275, "state_active": 0.045738860964775085, "state_memory": 0.013659648597240448, "state_peak": 61.875057220458984, "growth_active": 1.2018076181411743, "growth_memory": 1.2484720945358276, "heads": [{"head": 0, "memory_to_active": 0.0003649880236480385, "contribution_to_active": 0.5778389573097229, "mix_mean": 0.660828709602356, "mix_low": 0.0010433197021484375, "mix_high": 0.024505615234375, "read_cosine": 0.3391568958759308, "beta_mean": 8.689595460964483e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.00024370526080019772, "mix_std": 0.28181180357933044, "beta_std": 2.2003396225045435e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00023843521194066852, "evidence_weights": [-0.3037258982658386, -0.30927613377571106, 0.34208357334136963], "agreement_weight": 0.107882060110569, "orthogonal_weight": -1.0392751693725586}, {"head": 1, "memory_to_active": 0.020868318155407906, "contribution_to_active": 0.7195152044296265, "mix_mean": 0.584847092628479, "mix_low": 0.00714874267578125, "mix_high": 0.0028133392333984375, "read_cosine": 0.423696368932724, "beta_mean": 4.6123412175802514e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01602216809988022, "mix_std": 0.27641695737838745, "beta_std": 0.00010145148553419858, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010535955429077148, "evidence_weights": [-0.23490187525749207, -0.2910991609096527, 0.9511144161224365], "agreement_weight": 0.5715007185935974, "orthogonal_weight": -0.8002206087112427}, {"head": 2, "memory_to_active": 0.1508934199810028, "contribution_to_active": 0.6658499240875244, "mix_mean": 0.608451247215271, "mix_low": 0.0015306472778320312, "mix_high": 0.007218360900878906, "read_cosine": 0.3951086103916168, "beta_mean": 0.0005481840926222503, "nonfinite": 0.0, "weighted_memory_to_active": 0.10532818734645844, "mix_std": 0.26445144414901733, "beta_std": 0.0042399256490170956, "beta_low": 0.9969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.059282515197992325, "evidence_weights": [-0.20124219357967377, -0.2768681049346924, 0.7524335384368896], "agreement_weight": 0.3607279062271118, "orthogonal_weight": -0.6707603335380554}, {"head": 3, "memory_to_active": 0.6735521554946899, "contribution_to_active": 0.2865184545516968, "mix_mean": 0.18999852240085602, "mix_low": 0.0742330551147461, "mix_high": 4.1961669921875e-05, "read_cosine": 0.36994630098342896, "beta_mean": 0.019502872601151466, "nonfinite": 0.0, "weighted_memory_to_active": 0.18047019839286804, "mix_std": 0.21662741899490356, "beta_std": 0.021351352334022522, "beta_low": 0.3780517578125, "beta_high": 0.0, "orthogonal_to_active": 0.08661510795354843, "evidence_weights": [-0.045459531247615814, -0.11680197715759277, 0.5140840411186218], "agreement_weight": 0.09481372684240341, "orthogonal_weight": -0.5347179770469666}, {"head": 4, "memory_to_active": 2.9722368717193604, "contribution_to_active": 1.7332789897918701, "mix_mean": 0.14926812052726746, "mix_low": 0.12171173095703125, "mix_high": 0.0013360977172851562, "read_cosine": 0.08119842410087585, "beta_mean": 0.15010233223438263, "nonfinite": 0.0, "weighted_memory_to_active": 1.6199414730072021, "mix_std": 0.21008367836475372, "beta_std": 0.08124426752328873, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.004563332535326481, "evidence_weights": [-0.07951083779335022, -0.14907245337963104, -0.045640889555215836], "agreement_weight": -0.06786863505840302, "orthogonal_weight": -0.0029355764854699373}, {"head": 5, "memory_to_active": 0.9349343776702881, "contribution_to_active": 0.3225663900375366, "mix_mean": 0.2328333705663681, "mix_low": 0.08980655670166016, "mix_high": 0.0006809234619140625, "read_cosine": 0.4705258011817932, "beta_mean": 0.017045309767127037, "nonfinite": 0.0, "weighted_memory_to_active": 0.2425646036863327, "mix_std": 0.2404228150844574, "beta_std": 0.04317309707403183, "beta_low": 0.7076416015625, "beta_high": 0.0, "orthogonal_to_active": 0.016645358875393867, "evidence_weights": [0.0343833826482296, -0.1964251846075058, 0.1454034298658371], "agreement_weight": 0.004548391792923212, "orthogonal_weight": 0.07600211352109909}]}, {"layer": 10, "memory_to_active": 0.20992642641067505, "contribution_to_active": 0.7649000287055969, "mix_mean": 0.5321155190467834, "mix_low": 0.02405548095703125, "mix_high": 0.00539398193359375, "read_cosine": 0.37986239790916443, "beta_mean": 0.04194897413253784, "nonfinite": 0.0, "weighted_memory_to_active": 0.09156083315610886, "mix_std": 0.2681592106819153, "beta_std": 0.0434652604162693, "beta_low": 0.6677551865577698, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.2969570755958557, "output_influence": 0.48039454221725464, "residual_influence": 0.17562733590602875, "state_active": 0.05216740444302559, "state_memory": 0.014106172136962414, "state_peak": 128.87705993652344, "growth_active": 1.0493184328079224, "growth_memory": 1.0140514373779297, "heads": [{"head": 0, "memory_to_active": 0.7485997676849365, "contribution_to_active": 2.2538626194000244, "mix_mean": 0.5678799152374268, "mix_low": 0.1150197982788086, "mix_high": 0.006821632385253906, "read_cosine": 0.07928766310214996, "beta_mean": 0.18190480768680573, "nonfinite": 0.0, "weighted_memory_to_active": 0.43979156017303467, "mix_std": 0.39621126651763916, "beta_std": 0.1939348429441452, "beta_low": 0.16595458984375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.700201153755188, "evidence_weights": [-0.08301568776369095, -0.3629668354988098, -1.9533411264419556], "agreement_weight": 0.034068793058395386, "orthogonal_weight": 3.8928327560424805}, {"head": 1, "memory_to_active": 0.005907742772251368, "contribution_to_active": 0.6302834153175354, "mix_mean": 0.6434621214866638, "mix_low": 0.0009555816650390625, "mix_high": 0.018021583557128906, "read_cosine": 0.33789217472076416, "beta_mean": 4.2032039345940575e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0034058510791510344, "mix_std": 0.2618185877799988, "beta_std": 7.935653411550447e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005228694994002581, "evidence_weights": [-0.24841143190860748, -0.19160567224025726, 0.5418835878372192], "agreement_weight": 0.6731013059616089, "orthogonal_weight": -1.6780835390090942}, {"head": 2, "memory_to_active": 0.010239759460091591, "contribution_to_active": 0.6193810701370239, "mix_mean": 0.6689950823783875, "mix_low": 0.00021076202392578125, "mix_high": 0.005657196044921875, "read_cosine": 0.031200997531414032, "beta_mean": 2.2529302441398613e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004558325279504061, "mix_std": 0.24893105030059814, "beta_std": 8.02462745923549e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001850611180998385, "evidence_weights": [-0.2666545808315277, -0.2752591073513031, 0.7083168029785156], "agreement_weight": -0.22120153903961182, "orthogonal_weight": -0.39066624641418457}, {"head": 3, "memory_to_active": 0.3624967932701111, "contribution_to_active": 0.10201803594827652, "mix_mean": 0.29465606808662415, "mix_low": 0.026264190673828125, "mix_high": 5.14984130859375e-05, "read_cosine": 0.9235200881958008, "beta_mean": 0.06347128003835678, "nonfinite": 0.0, "weighted_memory_to_active": 0.03954610973596573, "mix_std": 0.23269537091255188, "beta_std": 0.05831030756235123, "beta_low": 0.00347900390625, "beta_high": 0.0, "orthogonal_to_active": 0.042107876390218735, "evidence_weights": [-0.08836670964956284, -0.14161692559719086, 0.1544688642024994], "agreement_weight": 0.018880978226661682, "orthogonal_weight": 1.4765044450759888}, {"head": 4, "memory_to_active": 0.013034586794674397, "contribution_to_active": 0.5709006786346436, "mix_mean": 0.5983730554580688, "mix_low": 0.00022602081298828125, "mix_high": 0.0018072128295898438, "read_cosine": 0.0916089415550232, "beta_mean": 3.2535026548430324e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007720382418483496, "mix_std": 0.24294602870941162, "beta_std": 5.910784238949418e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01130539271980524, "evidence_weights": [-0.1813269853591919, -0.2496151179075241, 0.9229357838630676], "agreement_weight": 0.1897519826889038, "orthogonal_weight": -1.5382428169250488}, {"head": 5, "memory_to_active": 0.11927999556064606, "contribution_to_active": 0.4129544496536255, "mix_mean": 0.4193266034126282, "mix_low": 0.0016565322875976562, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8156642913818359, "beta_mean": 0.006220657378435135, "nonfinite": 0.0, "weighted_memory_to_active": 0.05434276536107063, "mix_std": 0.22635291516780853, "beta_std": 0.008327698335051537, "beta_low": 0.83709716796875, "beta_high": 0.0, "orthogonal_to_active": 0.021048568189144135, "evidence_weights": [-0.1371496468782425, -0.16668829321861267, 0.2788306772708893], "agreement_weight": 0.09711567312479019, "orthogonal_weight": -0.674597442150116}]}, {"layer": 11, "memory_to_active": 0.07190181314945221, "contribution_to_active": 0.6316037178039551, "mix_mean": 0.5287224054336548, "mix_low": 0.11277103424072266, "mix_high": 0.04491695016622543, "read_cosine": 0.3317781686782837, "beta_mean": 0.022890720516443253, "nonfinite": 0.0, "weighted_memory_to_active": 0.03920982778072357, "mix_std": 0.3225191831588745, "beta_std": 0.03384163975715637, "beta_low": 0.7149454951286316, "beta_high": 0.0, "orthogonal_to_active": 0.042414188385009766, "output_influence": 0.5708791017532349, "residual_influence": 0.1677921712398529, "state_active": 0.2214486300945282, "state_memory": 0.002987511456012726, "state_peak": 36.38642883300781, "growth_active": 1.0853463411331177, "growth_memory": 1.4182995557785034, "heads": [{"head": 0, "memory_to_active": 0.006701439619064331, "contribution_to_active": 0.6137865781784058, "mix_mean": 0.6673249006271362, "mix_low": 0.0005550384521484375, "mix_high": 0.008717536926269531, "read_cosine": 0.2539852261543274, "beta_mean": 6.585175287909806e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005012920591980219, "mix_std": 0.24893498420715332, "beta_std": 0.0030792690813541412, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.004140076693147421, "evidence_weights": [-0.1772322803735733, -0.35706156492233276, 0.8093592524528503], "agreement_weight": 0.41448497772216797, "orthogonal_weight": -0.8535344004631042}, {"head": 1, "memory_to_active": 0.11965488642454147, "contribution_to_active": 0.4503018260002136, "mix_mean": 0.47633805871009827, "mix_low": 0.00469970703125, "mix_high": 0.0006799697875976562, "read_cosine": 0.49891266226768494, "beta_mean": 0.0021158959716558456, "nonfinite": 0.0, "weighted_memory_to_active": 0.060897041112184525, "mix_std": 0.2689165771007538, "beta_std": 0.008364680223166943, "beta_low": 0.958740234375, "beta_high": 0.0, "orthogonal_to_active": 0.03376784548163414, "evidence_weights": [-0.13577447831630707, -0.27963659167289734, 0.5958436131477356], "agreement_weight": 0.09103600680828094, "orthogonal_weight": -0.662203848361969}, {"head": 2, "memory_to_active": 0.0048264311626553535, "contribution_to_active": 0.987163782119751, "mix_mean": 0.4350031614303589, "mix_low": 0.3244132995605469, "mix_high": 0.1696004867553711, "read_cosine": 0.16364474594593048, "beta_mean": 0.06160828471183777, "nonfinite": 0.0, "weighted_memory_to_active": 0.0033821898978203535, "mix_std": 0.45061761140823364, "beta_std": 0.08267708867788315, "beta_low": 0.489013671875, "beta_high": 0.0, "orthogonal_to_active": 0.03541180118918419, "evidence_weights": [0.04580359160900116, -0.3337041139602661, -1.3243178129196167], "agreement_weight": -0.007991492748260498, "orthogonal_weight": 10.907698631286621}, {"head": 3, "memory_to_active": 0.007264215499162674, "contribution_to_active": 0.8709571361541748, "mix_mean": 0.41166073083877563, "mix_low": 0.3402109146118164, "mix_high": 0.08241558074951172, "read_cosine": 0.5554912090301514, "beta_mean": 0.048925478011369705, "nonfinite": 0.0, "weighted_memory_to_active": 0.004567987285554409, "mix_std": 0.4389452636241913, "beta_std": 0.07448878884315491, "beta_low": 0.533203125, "beta_high": 0.0, "orthogonal_to_active": 0.03443888574838638, "evidence_weights": [0.22352929413318634, -0.2712070047855377, -0.08619493991136551], "agreement_weight": 0.0527404248714447, "orthogonal_weight": 11.17026424407959}, {"head": 4, "memory_to_active": 0.20711606740951538, "contribution_to_active": 0.4507524371147156, "mix_mean": 0.6069941520690918, "mix_low": 0.00566864013671875, "mix_high": 0.0017557144165039062, "read_cosine": 0.0038195070810616016, "beta_mean": 0.015898898243904114, "nonfinite": 0.0, "weighted_memory_to_active": 0.11644534766674042, "mix_std": 0.2699061334133148, "beta_std": 0.01984945312142372, "beta_low": 0.51739501953125, "beta_high": 0.0, "orthogonal_to_active": 0.11550840735435486, "evidence_weights": [-0.2319081872701645, -0.239485964179039, 1.8797800540924072], "agreement_weight": -0.2197156548500061, "orthogonal_weight": -1.0030226707458496}, {"head": 5, "memory_to_active": 0.085847869515419, "contribution_to_active": 0.41666048765182495, "mix_mean": 0.5750131011009216, "mix_low": 0.0010786056518554688, "mix_high": 0.0063323974609375, "read_cosine": 0.5148155689239502, "beta_mean": 0.008729908615350723, "nonfinite": 0.0, "weighted_memory_to_active": 0.04495347663760185, "mix_std": 0.2577945590019226, "beta_std": 0.014590560458600521, "beta_low": 0.79180908203125, "beta_high": 0.0, "orthogonal_to_active": 0.031218085438013077, "evidence_weights": [-0.197281152009964, -0.31086376309394836, 0.6660539507865906], "agreement_weight": 0.0821017324924469, "orthogonal_weight": -0.8666010499000549}]}, {"layer": 12, "memory_to_active": 0.23093055188655853, "contribution_to_active": 0.6018973588943481, "mix_mean": 0.555641770362854, "mix_low": 0.008606275543570518, "mix_high": 0.007349968422204256, "read_cosine": 0.22379961609840393, "beta_mean": 0.015598948113620281, "nonfinite": 0.0, "weighted_memory_to_active": 0.11989125609397888, "mix_std": 0.25838804244995117, "beta_std": 0.015788085758686066, "beta_low": 0.8271891474723816, "beta_high": 0.0, "orthogonal_to_active": 0.07170464098453522, "output_influence": 0.5245950222015381, "residual_influence": 0.16061195731163025, "state_active": 0.033376649022102356, "state_memory": 0.0076189604587852955, "state_peak": 74.4134750366211, "growth_active": 1.1433542966842651, "growth_memory": 1.1018486022949219, "heads": [{"head": 0, "memory_to_active": 0.03450129181146622, "contribution_to_active": 0.6342539191246033, "mix_mean": 0.5993600487709045, "mix_low": 0.000926971435546875, "mix_high": 0.007847785949707031, "read_cosine": 0.26925134658813477, "beta_mean": 0.0005823112442158163, "nonfinite": 0.0, "weighted_memory_to_active": 0.019757866859436035, "mix_std": 0.26263272762298584, "beta_std": 0.0013069064589217305, "beta_low": 0.9967041015625, "beta_high": 0.0, "orthogonal_to_active": 0.01815078780055046, "evidence_weights": [-0.20443597435951233, -0.27305537462234497, -0.08017981797456741], "agreement_weight": 0.10112689435482025, "orthogonal_weight": -0.9800267219543457}, {"head": 1, "memory_to_active": 0.08790057897567749, "contribution_to_active": 0.5711494088172913, "mix_mean": 0.5718777179718018, "mix_low": 0.00030517578125, "mix_high": 0.0026035308837890625, "read_cosine": 0.39530718326568604, "beta_mean": 0.0004972468595951796, "nonfinite": 0.0, "weighted_memory_to_active": 0.05491935834288597, "mix_std": 0.2642595171928406, "beta_std": 0.008178580552339554, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.04523797705769539, "evidence_weights": [-0.19856439530849457, -0.2715967297554016, 1.04158616065979], "agreement_weight": 0.2880537509918213, "orthogonal_weight": -0.910355269908905}, {"head": 2, "memory_to_active": 0.6028990745544434, "contribution_to_active": 0.5031173229217529, "mix_mean": 0.3212798237800598, "mix_low": 0.026007652282714844, "mix_high": 5.626678466796875e-05, "read_cosine": 0.5681725740432739, "beta_mean": 0.0909612849354744, "nonfinite": 0.0, "weighted_memory_to_active": 0.23391209542751312, "mix_std": 0.23883801698684692, "beta_std": 0.06487676501274109, "beta_low": 0.0008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.2705841362476349, "evidence_weights": [-0.20126023888587952, -0.060984108597040176, -0.02073562890291214], "agreement_weight": 0.02722916565835476, "orthogonal_weight": 1.448277473449707}, {"head": 3, "memory_to_active": 0.03413280472159386, "contribution_to_active": 0.7681773900985718, "mix_mean": 0.7314304113388062, "mix_low": 9.059906005859375e-05, "mix_high": 0.021440505981445312, "read_cosine": 0.09839262068271637, "beta_mean": 0.000294801575364545, "nonfinite": 0.0, "weighted_memory_to_active": 0.025440027937293053, "mix_std": 0.22137877345085144, "beta_std": 0.005853720474988222, "beta_low": 0.997802734375, "beta_high": 0.0, "orthogonal_to_active": 0.02390664629638195, "evidence_weights": [-0.28637611865997314, -0.33327823877334595, 1.678378701210022], "agreement_weight": 0.629589855670929, "orthogonal_weight": -0.9625114798545837}, {"head": 4, "memory_to_active": 0.5429924726486206, "contribution_to_active": 0.6247729063034058, "mix_mean": 0.46929848194122314, "mix_low": 0.01769733428955078, "mix_high": 0.0054950714111328125, "read_cosine": -0.1959308236837387, "beta_mean": 0.0008327220566570759, "nonfinite": 0.0, "weighted_memory_to_active": 0.3300514817237854, "mix_std": 0.2915511727333069, "beta_std": 0.009183134883642197, "beta_low": 0.9791259765625, "beta_high": 0.0, "orthogonal_to_active": 0.03121643140912056, "evidence_weights": [-0.15714620053768158, -0.2145019769668579, -0.14577394723892212], "agreement_weight": 0.021262312307953835, "orthogonal_weight": -0.09781757742166519}, {"head": 5, "memory_to_active": 0.0831570029258728, "contribution_to_active": 0.5099135041236877, "mix_mean": 0.6406042575836182, "mix_low": 0.006609916687011719, "mix_high": 0.006656646728515625, "read_cosine": 0.20760485529899597, "beta_mean": 0.00042531942017376423, "nonfinite": 0.0, "weighted_memory_to_active": 0.05526671186089516, "mix_std": 0.2716679871082306, "beta_std": 0.00532941147685051, "beta_low": 0.99090576171875, "beta_high": 0.0, "orthogonal_to_active": 0.04113185778260231, "evidence_weights": [-0.2208404541015625, -0.31146934628486633, 1.2263978719711304], "agreement_weight": 0.4359056055545807, "orthogonal_weight": -0.8010056614875793}]}, {"layer": 13, "memory_to_active": 0.11661311984062195, "contribution_to_active": 0.5476676821708679, "mix_mean": 0.5507226586341858, "mix_low": 0.005958716385066509, "mix_high": 0.010562419891357422, "read_cosine": 0.4186857342720032, "beta_mean": 0.014416959136724472, "nonfinite": 0.0, "weighted_memory_to_active": 0.052692629396915436, "mix_std": 0.2585519850254059, "beta_std": 0.016538525000214577, "beta_low": 0.8092753887176514, "beta_high": 0.0, "orthogonal_to_active": 0.054452501237392426, "output_influence": 0.5426598787307739, "residual_influence": 0.2002008557319641, "state_active": 0.1355675458908081, "state_memory": 0.04137556254863739, "state_peak": 101.01654815673828, "growth_active": 0.8976069688796997, "growth_memory": 0.9631282091140747, "heads": [{"head": 0, "memory_to_active": 0.008319146931171417, "contribution_to_active": 0.6131798624992371, "mix_mean": 0.6235114336013794, "mix_low": 0.00089263916015625, "mix_high": 0.004609107971191406, "read_cosine": 0.43918928503990173, "beta_mean": 2.9755365176242776e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005525465589016676, "mix_std": 0.2640439569950104, "beta_std": 6.403766747098416e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005154437385499477, "evidence_weights": [-0.18968431651592255, -0.2647594213485718, 0.7302705645561218], "agreement_weight": 0.24037034809589386, "orthogonal_weight": -1.045432209968567}, {"head": 1, "memory_to_active": 0.008861861191689968, "contribution_to_active": 0.6084025502204895, "mix_mean": 0.5779399275779724, "mix_low": 0.0015687942504882812, "mix_high": 0.0044689178466796875, "read_cosine": 0.2320801019668579, "beta_mean": 3.371743150637485e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004974547307938337, "mix_std": 0.2728422284126282, "beta_std": 6.56914635328576e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0062757451087236404, "evidence_weights": [-0.1778455674648285, -0.2600347697734833, 0.5247450470924377], "agreement_weight": 0.584356427192688, "orthogonal_weight": -1.3134323358535767}, {"head": 2, "memory_to_active": 0.062323227524757385, "contribution_to_active": 0.4865020513534546, "mix_mean": 0.4014483094215393, "mix_low": 0.0010938644409179688, "mix_high": 0.00014495849609375, "read_cosine": 0.5957630276679993, "beta_mean": 0.009489411488175392, "nonfinite": 0.0, "weighted_memory_to_active": 0.03102380782365799, "mix_std": 0.23244409263134003, "beta_std": 0.03447546064853668, "beta_low": 0.86029052734375, "beta_high": 0.0, "orthogonal_to_active": 0.018253380432724953, "evidence_weights": [-0.17535275220870972, -0.2134454846382141, 0.40760916471481323], "agreement_weight": 0.00529715558513999, "orthogonal_weight": -0.8538388609886169}, {"head": 3, "memory_to_active": 0.19759459793567657, "contribution_to_active": 0.5834051370620728, "mix_mean": 0.6681225895881653, "mix_low": 0.02588939666748047, "mix_high": 0.04073810577392578, "read_cosine": 0.01677633821964264, "beta_mean": 0.0008778118644841015, "nonfinite": 0.0, "weighted_memory_to_active": 0.1486886441707611, "mix_std": 0.29268473386764526, "beta_std": 0.009387314319610596, "beta_low": 0.99517822265625, "beta_high": 0.0, "orthogonal_to_active": 0.13031932711601257, "evidence_weights": [-0.3149721026420593, -0.23991481959819794, 1.4707897901535034], "agreement_weight": 6.8144001960754395, "orthogonal_weight": -0.9752696752548218}, {"head": 4, "memory_to_active": 0.384236216545105, "contribution_to_active": 0.26415878534317017, "mix_mean": 0.36538228392601013, "mix_low": 0.004244804382324219, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8889086842536926, "beta_mean": 0.07591262459754944, "nonfinite": 0.0, "weighted_memory_to_active": 0.09903095662593842, "mix_std": 0.22082650661468506, "beta_std": 0.05378333479166031, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.14428505301475525, "evidence_weights": [-0.059710800647735596, -0.2222501039505005, 0.41901516914367676], "agreement_weight": 0.007723147980868816, "orthogonal_weight": 2.312767505645752}, {"head": 5, "memory_to_active": 0.0383436493575573, "contribution_to_active": 0.7303576469421387, "mix_mean": 0.667931318283081, "mix_low": 0.0020627975463867188, "mix_high": 0.013411521911621094, "read_cosine": 0.33939695358276367, "beta_mean": 0.00015843768778722733, "nonfinite": 0.0, "weighted_memory_to_active": 0.026912344619631767, "mix_std": 0.26847022771835327, "beta_std": 0.0014553125947713852, "beta_low": 0.99969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.02242705225944519, "evidence_weights": [-0.22058981657028198, -0.2874484062194824, 0.8870013356208801], "agreement_weight": 0.21728792786598206, "orthogonal_weight": -0.9242603182792664}]}, {"layer": 14, "memory_to_active": 0.2212965041399002, "contribution_to_active": 0.4153951406478882, "mix_mean": 0.4284091889858246, "mix_low": 0.028447788208723068, "mix_high": 0.006202380172908306, "read_cosine": 0.614675760269165, "beta_mean": 0.06533337384462357, "nonfinite": 0.0, "weighted_memory_to_active": 0.08990757167339325, "mix_std": 0.2394210547208786, "beta_std": 0.06079910695552826, "beta_low": 0.504669189453125, "beta_high": 0.0, "orthogonal_to_active": 0.04172884672880173, "output_influence": 0.3529227077960968, "residual_influence": 0.15334415435791016, "state_active": 0.05179475620388985, "state_memory": 0.014204480685293674, "state_peak": 49.54151153564453, "growth_active": 1.092759609222412, "growth_memory": 1.224099040031433, "heads": [{"head": 0, "memory_to_active": 0.37820562720298767, "contribution_to_active": 0.18987499177455902, "mix_mean": 0.18898576498031616, "mix_low": 0.019878387451171875, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8194997310638428, "beta_mean": 0.07309088110923767, "nonfinite": 0.0, "weighted_memory_to_active": 0.07900401949882507, "mix_std": 0.17603935301303864, "beta_std": 0.0637960359454155, "beta_low": 0.0037841796875, "beta_high": 0.0, "orthogonal_to_active": 0.09303322434425354, "evidence_weights": [-0.09529273211956024, -0.047445960342884064, 0.3644302785396576], "agreement_weight": 0.02195751667022705, "orthogonal_weight": 1.921067714691162}, {"head": 1, "memory_to_active": 0.4531272053718567, "contribution_to_active": 0.28495579957962036, "mix_mean": 0.1949853152036667, "mix_low": 0.1380147933959961, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8916388750076294, "beta_mean": 0.2742064595222473, "nonfinite": 0.0, "weighted_memory_to_active": 0.19985491037368774, "mix_std": 0.22881247103214264, "beta_std": 0.25717663764953613, "beta_low": 0.08892822265625, "beta_high": 0.0, "orthogonal_to_active": 0.018513178452849388, "evidence_weights": [0.013557088561356068, -0.18799592554569244, -0.09298625588417053], "agreement_weight": 0.06026461347937584, "orthogonal_weight": 0.3183121085166931}, {"head": 2, "memory_to_active": 0.11788580566644669, "contribution_to_active": 0.5816466808319092, "mix_mean": 0.5876464247703552, "mix_low": 0.0026197433471679688, "mix_high": 0.0046520233154296875, "read_cosine": 0.38951966166496277, "beta_mean": 0.001803041435778141, "nonfinite": 0.0, "weighted_memory_to_active": 0.07566539198160172, "mix_std": 0.2658417522907257, "beta_std": 0.002518205903470516, "beta_low": 0.98406982421875, "beta_high": 0.0, "orthogonal_to_active": 0.05889453366398811, "evidence_weights": [-0.1258445829153061, -0.31349313259124756, 0.9030781388282776], "agreement_weight": 0.08313591033220291, "orthogonal_weight": -0.8713396191596985}, {"head": 3, "memory_to_active": 0.042174044996500015, "contribution_to_active": 0.59297776222229, "mix_mean": 0.6805286407470703, "mix_low": 0.0036020278930664062, "mix_high": 0.0233917236328125, "read_cosine": -0.059470534324645996, "beta_mean": 0.005011250730603933, "nonfinite": 0.0, "weighted_memory_to_active": 0.027375373989343643, "mix_std": 0.2546730935573578, "beta_std": 0.007683349307626486, "beta_low": 0.86126708984375, "beta_high": 0.0, "orthogonal_to_active": 0.02043391577899456, "evidence_weights": [-0.3407478928565979, -0.3134298622608185, 0.3075271248817444], "agreement_weight": 0.06773180514574051, "orthogonal_weight": -0.8362877368927002}, {"head": 4, "memory_to_active": 0.1514909267425537, "contribution_to_active": 0.5022947788238525, "mix_mean": 0.5295680165290833, "mix_low": 0.0034542083740234375, "mix_high": 0.007846832275390625, "read_cosine": 0.8956590294837952, "beta_mean": 0.036840878427028656, "nonfinite": 0.0, "weighted_memory_to_active": 0.08943469822406769, "mix_std": 0.25918230414390564, "beta_std": 0.028008487075567245, "beta_low": 0.107177734375, "beta_high": 0.0, "orthogonal_to_active": 0.024594131857156754, "evidence_weights": [-0.20349758863449097, -0.23488864302635193, 0.4450327157974243], "agreement_weight": 0.13397997617721558, "orthogonal_weight": -0.6202967166900635}, {"head": 5, "memory_to_active": 0.1848953366279602, "contribution_to_active": 0.34062066674232483, "mix_mean": 0.3887408375740051, "mix_low": 0.0031175613403320312, "mix_high": 0.001316070556640625, "read_cosine": 0.7512075901031494, "beta_mean": 0.0010477129835635424, "nonfinite": 0.0, "weighted_memory_to_active": 0.06811104714870453, "mix_std": 0.2519773244857788, "beta_std": 0.005611889995634556, "beta_low": 0.9827880859375, "beta_high": 0.0, "orthogonal_to_active": 0.03490408509969711, "evidence_weights": [-0.17377310991287231, -0.18476514518260956, 0.37284934520721436], "agreement_weight": 0.060992494225502014, "orthogonal_weight": -0.7533813118934631}]}, {"layer": 15, "memory_to_active": 0.05073025822639465, "contribution_to_active": 0.49889206886291504, "mix_mean": 0.44215404987335205, "mix_low": 0.006292184814810753, "mix_high": 0.0034421286545693874, "read_cosine": 0.6635715961456299, "beta_mean": 0.037140414118766785, "nonfinite": 0.0, "weighted_memory_to_active": 0.025842681527137756, "mix_std": 0.2843303680419922, "beta_std": 0.042302440851926804, "beta_low": 0.7064208984375, "beta_high": 0.0, "orthogonal_to_active": 0.009914645925164223, "output_influence": 0.32165777683258057, "residual_influence": 0.1952136754989624, "state_active": 0.17207671701908112, "state_memory": 0.010058973915874958, "state_peak": 27.78771209716797, "growth_active": 1.2081581354141235, "growth_memory": 1.2484301328659058, "heads": [{"head": 0, "memory_to_active": 0.11917197704315186, "contribution_to_active": 0.4738408029079437, "mix_mean": 0.4052104353904724, "mix_low": 0.006432533264160156, "mix_high": 0.007143974304199219, "read_cosine": 0.536119818687439, "beta_mean": 0.04659319669008255, "nonfinite": 0.0, "weighted_memory_to_active": 0.05934818461537361, "mix_std": 0.28812897205352783, "beta_std": 0.031099747866392136, "beta_low": 0.045166015625, "beta_high": 0.0, "orthogonal_to_active": 0.037124477326869965, "evidence_weights": [-0.18588930368423462, -0.19676196575164795, 0.5350155234336853], "agreement_weight": 0.09025760740041733, "orthogonal_weight": -0.8394774794578552}, {"head": 1, "memory_to_active": 0.08592410385608673, "contribution_to_active": 0.43760788440704346, "mix_mean": 0.4497607946395874, "mix_low": 0.0008029937744140625, "mix_high": 0.008500099182128906, "read_cosine": 0.9341753721237183, "beta_mean": 0.00017009700241032988, "nonfinite": 0.0, "weighted_memory_to_active": 0.037228699773550034, "mix_std": 0.28421783447265625, "beta_std": 0.0013629726599901915, "beta_low": 0.9989013671875, "beta_high": 0.0, "orthogonal_to_active": 0.0030502353329211473, "evidence_weights": [-0.16082100570201874, -0.17332524061203003, 0.16067922115325928], "agreement_weight": 0.11123049259185791, "orthogonal_weight": 0.18573762476444244}, {"head": 2, "memory_to_active": 0.004907188005745411, "contribution_to_active": 0.5340733528137207, "mix_mean": 0.37292489409446716, "mix_low": 0.008656501770019531, "mix_high": 6.29425048828125e-05, "read_cosine": 0.8001902103424072, "beta_mean": 5.9801273891935125e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0026981853879988194, "mix_std": 0.2744026184082031, "beta_std": 0.00022893049754202366, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.0008556979009881616, "evidence_weights": [-0.20052282512187958, -0.2689673900604248, 0.11128810793161392], "agreement_weight": 0.0076371305622160435, "orthogonal_weight": 0.9427520036697388}, {"head": 3, "memory_to_active": 0.0012450384674593806, "contribution_to_active": 0.36166733503341675, "mix_mean": 0.42025989294052124, "mix_low": 0.0036859512329101562, "mix_high": 0.0006265640258789062, "read_cosine": 0.627859354019165, "beta_mean": 6.643307642661966e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0004060280043631792, "mix_std": 0.28771349787712097, "beta_std": 1.5516339772148058e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00022467720555141568, "evidence_weights": [-0.2732890844345093, -0.23659585416316986, 0.1805768609046936], "agreement_weight": 0.05372758209705353, "orthogonal_weight": -0.5526787638664246}, {"head": 4, "memory_to_active": 0.06191662326455116, "contribution_to_active": 0.635875940322876, "mix_mean": 0.40604621171951294, "mix_low": 0.007916450500488281, "mix_high": 0.0011882781982421875, "read_cosine": 0.7310377955436707, "beta_mean": 0.17380808293819427, "nonfinite": 0.0, "weighted_memory_to_active": 0.03399771451950073, "mix_std": 0.31011995673179626, "beta_std": 0.20424015820026398, "beta_low": 0.2235107421875, "beta_high": 0.0, "orthogonal_to_active": 0.0061169154942035675, "evidence_weights": [-0.20487284660339355, -0.2502986788749695, -0.028172723948955536], "agreement_weight": 0.011832289397716522, "orthogonal_weight": 0.40034812688827515}, {"head": 5, "memory_to_active": 0.03121662512421608, "contribution_to_active": 0.5502868890762329, "mix_mean": 0.598721981048584, "mix_low": 0.010258674621582031, "mix_high": 0.0031309127807617188, "read_cosine": 0.35204702615737915, "beta_mean": 0.0022046570666134357, "nonfinite": 0.0, "weighted_memory_to_active": 0.021377263590693474, "mix_std": 0.26139920949935913, "beta_std": 0.01686730794608593, "beta_low": 0.97119140625, "beta_high": 0.0, "orthogonal_to_active": 0.012115871533751488, "evidence_weights": [-0.2797980010509491, -0.30531206727027893, 0.5925122499465942], "agreement_weight": 0.31626006960868835, "orthogonal_weight": 0.8113045692443848}]}]}, {"step": 74000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8208100160009053, "memory_to_active": 0.8033082485198975, "contribution_to_active": 0.5810912847518921, "mix_mean": 0.43176937103271484, "mix_low": 0.12100274115800858, "mix_high": 0.010867983102798462, "read_cosine": 0.343597412109375, "beta_mean": 0.06816691160202026, "nonfinite": 0.0, "weighted_memory_to_active": 0.1988314688205719, "mix_std": 0.22263731062412262, "beta_std": 0.059945981949567795, "beta_low": 0.596771240234375, "beta_high": 3.115335857728496e-05, "orthogonal_to_active": 0.06545382738113403, "samples": 4, "scored_tokens": 899, "normal_loss": 3.014084577560425, "normal_token_accuracy": 0.4660734236240387, "state_active": 0.06454683840274811, "state_memory": 0.016155220568180084, "state_peak": 123.85283660888672, "growth_active": 1.1273632049560547, "growth_memory": 1.2694441080093384, "output_influence": 0.45172446966171265, "residual_influence": 0.16577576100826263, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.14909029006958, "no_memory_loss_delta": 0.7878242135047913, "scrambled_loss_delta": 0.12314531952142715, "memory_only_loss_delta": 3.4290313720703125, "no_evidence_loss_delta": 0.44264891743659973, "no_agreement_loss_delta": 0.018604867160320282, "no_orthogonal_loss_delta": 0.016359787434339523}, {"index": 94, "tokens": 256, "normal_loss": 3.11134934425354, "no_memory_loss_delta": 0.8454809188842773, "scrambled_loss_delta": 0.4601144790649414, "memory_only_loss_delta": 3.815120220184326, "no_evidence_loss_delta": 0.35536521673202515, "no_agreement_loss_delta": 0.04440638795495033, "no_orthogonal_loss_delta": 0.06647661328315735}, {"index": 188, "tokens": 256, "normal_loss": 3.340548038482666, "no_memory_loss_delta": 1.0544670820236206, "scrambled_loss_delta": 0.11975245177745819, "memory_only_loss_delta": 3.0871331691741943, "no_evidence_loss_delta": 0.5174100995063782, "no_agreement_loss_delta": 0.04811229556798935, "no_orthogonal_loss_delta": 0.039285384118556976}, {"index": 283, "tokens": 131, "normal_loss": 1.9222089052200317, "no_memory_loss_delta": 0.4842206537723541, "scrambled_loss_delta": 0.14369656145572662, "memory_only_loss_delta": 3.122498035430908, "no_evidence_loss_delta": 0.18270790576934814, "no_agreement_loss_delta": -0.004977579228579998, "no_orthogonal_loss_delta": -0.028915923088788986}], "agreement_weight_abs_mean": 0.25397804379463196, "agreement_weight_abs_max": 6.850212574005127, "agreement_weight_mean": 0.22040967643260956, "orthogonal_weight_abs_mean": 0.9506719708442688, "orthogonal_weight_abs_max": 11.242267608642578, "orthogonal_weight_mean": -0.04164661467075348, "no_memory_loss": 3.8500163555145264, "no_memory_loss_delta": 0.835931658744812, "no_memory_token_accuracy_gain": 0.13793103396892548, "no_memory_token_win_fraction": 0.7107897400856018, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 56.65295532813247, "scrambled_loss": 3.2352144718170166, "scrambled_loss_delta": 0.22112944722175598, "scrambled_token_accuracy_gain": 0.041156839579343796, "scrambled_token_win_fraction": 0.6084538102149963, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.83870929899774, "memory_only_loss": 6.411032676696777, "memory_only_loss_delta": 3.3969476222991943, "memory_only_token_accuracy_gain": 0.31701889634132385, "memory_only_token_win_fraction": 0.9276974201202393, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 96.65247065868006, "no_evidence_loss": 3.415289878845215, "no_evidence_loss_delta": 0.40120503306388855, "no_evidence_token_accuracy_gain": 0.06451612710952759, "no_evidence_token_win_fraction": 0.6195772886276245, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 33.04872252912139, "no_agreement_loss": 3.0450026988983154, "no_agreement_loss_delta": 0.030918316915631294, "no_agreement_token_accuracy_gain": -0.0055617354810237885, "no_agreement_token_win_fraction": 0.48832035064697266, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 3.04452339346253, "no_orthogonal_loss": 3.0446465015411377, "no_orthogonal_loss_delta": 0.03056194633245468, "no_orthogonal_token_accuracy_gain": 0.01001112349331379, "no_orthogonal_token_win_fraction": 0.5739710927009583, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 3.0099651563182706, "short_no_memory_loss_delta": 0.7517711520195007, "long_context_memory_gain": 0.08416050672531128, "normal_context_loss_gain": 0.14379768073558807, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.6297340393066406, "contribution_to_active": 1.2255452871322632, "mix_mean": 0.10953706502914429, "mix_low": 0.5011155009269714, "mix_high": 0.004944324493408203, "read_cosine": 0.4097515344619751, "beta_mean": 0.21817217767238617, "nonfinite": 0.0, "weighted_memory_to_active": 1.244783639907837, "mix_std": 0.0926768109202385, "beta_std": 0.1458202749490738, "beta_low": 0.1704203337430954, "beta_high": 0.0, "orthogonal_to_active": 0.05207210034132004, "output_influence": 0.21677625179290771, "residual_influence": 0.23224246501922607, "state_active": 0.06545264273881912, "state_memory": 0.044876858592033386, "state_peak": 44.77373504638672, "growth_active": 1.2254644632339478, "growth_memory": 1.2540035247802734, "heads": [{"head": 0, "memory_to_active": 2.9115242958068848, "contribution_to_active": 0.4817684590816498, "mix_mean": 0.01984637789428234, "mix_low": 0.6124248504638672, "mix_high": 1.9073486328125e-06, "read_cosine": 0.21519768238067627, "beta_mean": 0.200699120759964, "nonfinite": 0.0, "weighted_memory_to_active": 0.6011285185813904, "mix_std": 0.058508288115262985, "beta_std": 0.09936411678791046, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06094709411263466, "evidence_weights": [-0.003037806833162904, -0.040233153849840164, -0.052948880940675735], "agreement_weight": -0.22881105542182922, "orthogonal_weight": -0.11621633917093277}, {"head": 1, "memory_to_active": 6.80659294128418, "contribution_to_active": 3.0034189224243164, "mix_mean": 0.020019782707095146, "mix_low": 0.6094827651977539, "mix_high": 0.0, "read_cosine": 0.4410943388938904, "beta_mean": 0.4629721939563751, "nonfinite": 0.0, "weighted_memory_to_active": 3.2242448329925537, "mix_std": 0.05745130032300949, "beta_std": 0.22167575359344482, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0935974270105362, "evidence_weights": [0.0231998972594738, -0.10157202929258347, 0.6719063520431519], "agreement_weight": 0.027773937210440636, "orthogonal_weight": 0.06608518213033676}, {"head": 2, "memory_to_active": 2.8221988677978516, "contribution_to_active": 0.6235920190811157, "mix_mean": 0.01519795972853899, "mix_low": 0.6448802947998047, "mix_high": 0.0, "read_cosine": 0.28750401735305786, "beta_mean": 0.26924124360084534, "nonfinite": 0.0, "weighted_memory_to_active": 0.7231019735336304, "mix_std": 0.03332115709781647, "beta_std": 0.20951329171657562, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05908297747373581, "evidence_weights": [0.06113595515489578, 0.04615658149123192, 0.25910061597824097], "agreement_weight": 0.009845110587775707, "orthogonal_weight": -0.204400897026062}, {"head": 3, "memory_to_active": 7.776923179626465, "contribution_to_active": 2.712467670440674, "mix_mean": 0.02838171273469925, "mix_low": 0.5166158676147461, "mix_high": 0.0, "read_cosine": 0.2836487889289856, "beta_mean": 0.33335429430007935, "nonfinite": 0.0, "weighted_memory_to_active": 2.8226425647735596, "mix_std": 0.06642795354127884, "beta_std": 0.27135127782821655, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0800776258111, "evidence_weights": [-0.03581594303250313, -0.027330517768859863, -0.148146390914917], "agreement_weight": -0.006150755565613508, "orthogonal_weight": -0.03321049362421036}, {"head": 4, "memory_to_active": 1.3545138835906982, "contribution_to_active": 0.059257037937641144, "mix_mean": 0.022895710542798042, "mix_low": 0.6097421646118164, "mix_high": 0.0, "read_cosine": 0.3014971613883972, "beta_mean": 0.024590691551566124, "nonfinite": 0.0, "weighted_memory_to_active": 0.040054675191640854, "mix_std": 0.05951807647943497, "beta_std": 0.05370446667075157, "beta_low": 0.57861328125, "beta_high": 0.0, "orthogonal_to_active": 0.000509093573782593, "evidence_weights": [0.09210488200187683, 0.05888481065630913, -0.14209434390068054], "agreement_weight": -0.2040165811777115, "orthogonal_weight": 0.011508517898619175}, {"head": 5, "memory_to_active": 0.10664993524551392, "contribution_to_active": 0.47276771068573, "mix_mean": 0.5508807897567749, "mix_low": 0.013546943664550781, "mix_high": 0.029664039611816406, "read_cosine": 0.9295672178268433, "beta_mean": 0.018175479024648666, "nonfinite": 0.0, "weighted_memory_to_active": 0.05752921476960182, "mix_std": 0.2808340787887573, "beta_std": 0.019312694668769836, "beta_low": 0.44390869140625, "beta_high": 0.0, "orthogonal_to_active": 0.018218375742435455, "evidence_weights": [-0.17335925996303558, -0.21732471883296967, 0.36678004264831543], "agreement_weight": 0.28414186835289, "orthogonal_weight": -0.7152606844902039}]}, {"layer": 1, "memory_to_active": 2.6050357818603516, "contribution_to_active": 0.478829026222229, "mix_mean": 0.21660137176513672, "mix_low": 0.3733522295951843, "mix_high": 0.0012095769634470344, "read_cosine": 0.3027368187904358, "beta_mean": 0.17830203473567963, "nonfinite": 0.0, "weighted_memory_to_active": 0.3161841630935669, "mix_std": 0.11211123317480087, "beta_std": 0.13013269007205963, "beta_low": 0.35247802734375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.03649954870343208, "output_influence": 0.27433428168296814, "residual_influence": 0.07848464697599411, "state_active": 0.027522467076778412, "state_memory": 0.03430015593767166, "state_peak": 43.627376556396484, "growth_active": 1.1502867937088013, "growth_memory": 1.4460947513580322, "heads": [{"head": 0, "memory_to_active": 0.014528404921293259, "contribution_to_active": 0.49467167258262634, "mix_mean": 0.540655791759491, "mix_low": 0.004097938537597656, "mix_high": 0.00279998779296875, "read_cosine": 0.04051997512578964, "beta_mean": 0.00010448000830365345, "nonfinite": 0.0, "weighted_memory_to_active": 0.008425590582191944, "mix_std": 0.28079020977020264, "beta_std": 0.00016462447820231318, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008093860931694508, "evidence_weights": [-0.20885765552520752, -0.2694082260131836, 0.20396405458450317], "agreement_weight": 0.2071831375360489, "orthogonal_weight": -1.0128567218780518}, {"head": 1, "memory_to_active": 3.1713695526123047, "contribution_to_active": 0.2923610806465149, "mix_mean": 0.020492034032940865, "mix_low": 0.5116138458251953, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5045202970504761, "beta_mean": 0.26440948247909546, "nonfinite": 0.0, "weighted_memory_to_active": 0.3176254630088806, "mix_std": 0.040654800832271576, "beta_std": 0.1874711662530899, "beta_low": 0.00140380859375, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.0218320582062006, "evidence_weights": [0.02900032140314579, -0.05949554964900017, 0.047330498695373535], "agreement_weight": -0.003347009187564254, "orthogonal_weight": 0.21168948709964752}, {"head": 2, "memory_to_active": 0.0304148867726326, "contribution_to_active": 0.5479418039321899, "mix_mean": 0.6776109933853149, "mix_low": 0.000255584716796875, "mix_high": 0.004456520080566406, "read_cosine": 0.4707483649253845, "beta_mean": 0.0009958782466128469, "nonfinite": 0.0, "weighted_memory_to_active": 0.01980941742658615, "mix_std": 0.22564946115016937, "beta_std": 0.0028894618153572083, "beta_low": 0.9827880859375, "beta_high": 0.0, "orthogonal_to_active": 0.014224259182810783, "evidence_weights": [-0.27166861295700073, -0.3677501082420349, 0.6363439559936523], "agreement_weight": 0.05135834217071533, "orthogonal_weight": -0.8360176086425781}, {"head": 3, "memory_to_active": 3.176199436187744, "contribution_to_active": 0.20870693027973175, "mix_mean": 0.02442421019077301, "mix_low": 0.5199337005615234, "mix_high": 0.0, "read_cosine": 0.18869853019714355, "beta_mean": 0.33580154180526733, "nonfinite": 0.0, "weighted_memory_to_active": 0.2742445468902588, "mix_std": 0.060023367404937744, "beta_std": 0.263083815574646, "beta_low": 0.06939697265625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.10109420120716095, "evidence_weights": [0.0393223762512207, -0.01779424212872982, -0.014793957583606243], "agreement_weight": -0.029432963579893112, "orthogonal_weight": -0.47811615467071533}, {"head": 4, "memory_to_active": 4.1668596267700195, "contribution_to_active": 1.2180448770523071, "mix_mean": 0.030816329643130302, "mix_low": 0.33113670349121094, "mix_high": 0.0, "read_cosine": 0.35516926646232605, "beta_mean": 0.16047579050064087, "nonfinite": 0.0, "weighted_memory_to_active": 1.2013187408447266, "mix_std": 0.05016982555389404, "beta_std": 0.1278812289237976, "beta_low": 0.0345458984375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.04318052530288696, "evidence_weights": [-0.013360927812755108, -0.003752598073333502, 0.1591740846633911], "agreement_weight": -0.01055177766829729, "orthogonal_weight": 0.05362997204065323}, {"head": 5, "memory_to_active": 5.070842266082764, "contribution_to_active": 0.11124744266271591, "mix_mean": 0.005608804523944855, "mix_low": 0.8730754852294922, "mix_high": 0.0, "read_cosine": 0.2567644715309143, "beta_mean": 0.308025062084198, "nonfinite": 0.0, "weighted_memory_to_active": 0.07568138092756271, "mix_std": 0.015379734337329865, "beta_std": 0.19930583238601685, "beta_low": 0.0267333984375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.03057238832116127, "evidence_weights": [0.1677694171667099, 0.08383800089359283, 0.09279816597700119], "agreement_weight": -0.11722055077552795, "orthogonal_weight": 0.4186115264892578}]}, {"layer": 2, "memory_to_active": 1.039689302444458, "contribution_to_active": 0.36613357067108154, "mix_mean": 0.41505035758018494, "mix_low": 0.2377074658870697, "mix_high": 0.004520257469266653, "read_cosine": 0.2957208752632141, "beta_mean": 0.0750918909907341, "nonfinite": 0.0, "weighted_memory_to_active": 0.07550182938575745, "mix_std": 0.1843232661485672, "beta_std": 0.06604981422424316, "beta_low": 0.61785888671875, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.016208697110414505, "output_influence": 0.4128679931163788, "residual_influence": 0.11957130581140518, "state_active": 0.03967806324362755, "state_memory": 0.013172733597457409, "state_peak": 28.17778968811035, "growth_active": 1.0445423126220703, "growth_memory": 1.5423165559768677, "heads": [{"head": 0, "memory_to_active": 0.004384931642562151, "contribution_to_active": 0.3665819466114044, "mix_mean": 0.6504659652709961, "mix_low": 0.0005807876586914062, "mix_high": 0.009472846984863281, "read_cosine": 0.1473512500524521, "beta_mean": 3.691393430926837e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029261973686516285, "mix_std": 0.2629803419113159, "beta_std": 6.44786050543189e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030139070004224777, "evidence_weights": [-0.37410926818847656, -0.21512283384799957, 0.20525585114955902], "agreement_weight": 0.14265860617160797, "orthogonal_weight": -1.0673999786376953}, {"head": 1, "memory_to_active": 3.303851842880249, "contribution_to_active": 0.05064323917031288, "mix_mean": 0.0060399724170565605, "mix_low": 0.8579645156860352, "mix_high": 0.0, "read_cosine": 0.1127370297908783, "beta_mean": 0.20892637968063354, "nonfinite": 0.0, "weighted_memory_to_active": 0.03957829251885414, "mix_std": 0.012873618863523006, "beta_std": 0.1480381339788437, "beta_low": 0.006591796875, "beta_high": 0.0, "orthogonal_to_active": 0.011696086265146732, "evidence_weights": [0.13069260120391846, 0.1022171899676323, 0.5690054893493652], "agreement_weight": 0.03266870230436325, "orthogonal_weight": 0.32610610127449036}, {"head": 2, "memory_to_active": 0.007013032212853432, "contribution_to_active": 0.4238738715648651, "mix_mean": 0.5579558610916138, "mix_low": 0.0009174346923828125, "mix_high": 0.00403594970703125, "read_cosine": 0.39951902627944946, "beta_mean": 2.2905223886482418e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002974054077640176, "mix_std": 0.2735516428947449, "beta_std": 4.018977051600814e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00021073492825962603, "evidence_weights": [-0.24855513870716095, -0.2913148105144501, 0.5717697143554688], "agreement_weight": -0.03189047798514366, "orthogonal_weight": 0.08629109710454941}, {"head": 3, "memory_to_active": 0.09329842776060104, "contribution_to_active": 0.5492026805877686, "mix_mean": 0.5927072763442993, "mix_low": 0.00016689300537109375, "mix_high": 0.006390571594238281, "read_cosine": 0.574994683265686, "beta_mean": 0.013965191319584846, "nonfinite": 0.0, "weighted_memory_to_active": 0.05946378409862518, "mix_std": 0.2522898018360138, "beta_std": 0.023198513314127922, "beta_low": 0.63983154296875, "beta_high": 0.0, "orthogonal_to_active": 0.042457882314920425, "evidence_weights": [-0.20033395290374756, -0.3266257643699646, 0.7277677059173584], "agreement_weight": 0.05466434359550476, "orthogonal_weight": -0.8950182795524597}, {"head": 4, "memory_to_active": 2.8195431232452393, "contribution_to_active": 0.2694564163684845, "mix_mean": 0.02047988772392273, "mix_low": 0.5661725997924805, "mix_high": 0.0, "read_cosine": 0.18947285413742065, "beta_mean": 0.22756525874137878, "nonfinite": 0.0, "weighted_memory_to_active": 0.34236380457878113, "mix_std": 0.04785085469484329, "beta_std": 0.2249096930027008, "beta_low": 0.06072998046875, "beta_high": 0.00091552734375, "orthogonal_to_active": 0.038294531404972076, "evidence_weights": [0.02318422682583332, 0.0001317029382335022, 0.04184725508093834], "agreement_weight": 0.024547910317778587, "orthogonal_weight": -0.12871231138706207}, {"head": 5, "memory_to_active": 0.010044523514807224, "contribution_to_active": 0.5370430946350098, "mix_mean": 0.6626532077789307, "mix_low": 0.0004425048828125, "mix_high": 0.007222175598144531, "read_cosine": 0.35025036334991455, "beta_mean": 3.4713331842795014e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005704818293452263, "mix_std": 0.2563933730125427, "beta_std": 4.783464828506112e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015790450852364302, "evidence_weights": [-0.2766909897327423, -0.28720784187316895, 0.8990294337272644], "agreement_weight": 0.2898334860801697, "orthogonal_weight": -0.3221774399280548}]}, {"layer": 3, "memory_to_active": 0.4229697585105896, "contribution_to_active": 0.49851304292678833, "mix_mean": 0.5262705683708191, "mix_low": 0.05220763385295868, "mix_high": 0.01185623835772276, "read_cosine": 0.18200448155403137, "beta_mean": 0.0231255404651165, "nonfinite": 0.0, "weighted_memory_to_active": 0.1173286885023117, "mix_std": 0.25521320104599, "beta_std": 0.022361161187291145, "beta_low": 0.7760722637176514, "beta_high": 0.0, "orthogonal_to_active": 0.04859800636768341, "output_influence": 0.4449434280395508, "residual_influence": 0.11499576270580292, "state_active": 0.02409527637064457, "state_memory": 0.00621173158288002, "state_peak": 24.131494522094727, "growth_active": 1.1256883144378662, "growth_memory": 1.5910561084747314, "heads": [{"head": 0, "memory_to_active": 0.3546193838119507, "contribution_to_active": 0.6164394617080688, "mix_mean": 0.615074872970581, "mix_low": 0.0013780593872070312, "mix_high": 0.02373790740966797, "read_cosine": 0.2074572890996933, "beta_mean": 0.008772268891334534, "nonfinite": 0.0, "weighted_memory_to_active": 0.2161523848772049, "mix_std": 0.2982538938522339, "beta_std": 0.009891756810247898, "beta_low": 0.72283935546875, "beta_high": 0.0, "orthogonal_to_active": 0.15994040668010712, "evidence_weights": [-0.21426217257976532, -0.24855458736419678, 1.0628105401992798], "agreement_weight": 0.11716524511575699, "orthogonal_weight": -0.7730820178985596}, {"head": 1, "memory_to_active": 0.006665200926363468, "contribution_to_active": 0.47381386160850525, "mix_mean": 0.5949937105178833, "mix_low": 0.0027589797973632812, "mix_high": 0.010769844055175781, "read_cosine": 0.16883587837219238, "beta_mean": 3.042194657609798e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004126561339944601, "mix_std": 0.2908460795879364, "beta_std": 7.803185872035101e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0033444976434111595, "evidence_weights": [-0.2692869305610657, -0.216880202293396, 0.7005600333213806], "agreement_weight": 0.61101233959198, "orthogonal_weight": -0.8793554902076721}, {"head": 2, "memory_to_active": 1.9450328350067139, "contribution_to_active": 0.27423498034477234, "mix_mean": 0.055866438895463943, "mix_low": 0.3009815216064453, "mix_high": 9.5367431640625e-07, "read_cosine": 0.21330112218856812, "beta_mean": 0.12583176791667938, "nonfinite": 0.0, "weighted_memory_to_active": 0.3096303939819336, "mix_std": 0.10653510689735413, "beta_std": 0.11495549231767654, "beta_low": 0.0250244140625, "beta_high": 0.0, "orthogonal_to_active": 0.002890282776206732, "evidence_weights": [-0.09888648241758347, -0.007104894146323204, -0.028962645679712296], "agreement_weight": -0.012817799113690853, "orthogonal_weight": 0.0120760602876544}, {"head": 3, "memory_to_active": 0.01600237563252449, "contribution_to_active": 0.5405200123786926, "mix_mean": 0.6088861227035522, "mix_low": 0.003911018371582031, "mix_high": 0.009415626525878906, "read_cosine": 0.27357861399650574, "beta_mean": 7.271580398082733e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010743662714958191, "mix_std": 0.2834320664405823, "beta_std": 0.00016757557750679553, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025473523419350386, "evidence_weights": [-0.26183900237083435, -0.22062228620052338, 2.3378639221191406], "agreement_weight": 2.3457841873168945, "orthogonal_weight": -0.3080298900604248}, {"head": 4, "memory_to_active": 0.010903793387115002, "contribution_to_active": 0.41550761461257935, "mix_mean": 0.5462754964828491, "mix_low": 0.0019702911376953125, "mix_high": 0.003345489501953125, "read_cosine": 0.1420546919107437, "beta_mean": 5.5442586017306894e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007277791388332844, "mix_std": 0.28678658604621887, "beta_std": 0.00010780961019918323, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006877291016280651, "evidence_weights": [-0.24620628356933594, -0.22453351318836212, 0.06691338121891022], "agreement_weight": -0.05947510525584221, "orthogonal_weight": -1.021532416343689}, {"head": 5, "memory_to_active": 0.2045949399471283, "contribution_to_active": 0.6705622673034668, "mix_mean": 0.7365265488624573, "mix_low": 0.0022459030151367188, "mix_high": 0.02386760711669922, "read_cosine": 0.08679921925067902, "beta_mean": 0.003990623634308577, "nonfinite": 0.0, "weighted_memory_to_active": 0.15604132413864136, "mix_std": 0.2654254734516144, "beta_std": 0.00896630808711052, "beta_low": 0.9085693359375, "beta_high": 0.0, "orthogonal_to_active": 0.11598820984363556, "evidence_weights": [-0.26116710901260376, -0.34478357434272766, 0.5531365275382996], "agreement_weight": 0.24732640385627747, "orthogonal_weight": -0.7728132605552673}]}, {"layer": 4, "memory_to_active": 1.0035005807876587, "contribution_to_active": 0.5004063248634338, "mix_mean": 0.37557268142700195, "mix_low": 0.2274438589811325, "mix_high": 0.011501789093017578, "read_cosine": 0.3028593063354492, "beta_mean": 0.10310791432857513, "nonfinite": 0.0, "weighted_memory_to_active": 0.13725285232067108, "mix_std": 0.16402682662010193, "beta_std": 0.0944170355796814, "beta_low": 0.41253662109375, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.06556940078735352, "output_influence": 0.5081943869590759, "residual_influence": 0.164740651845932, "state_active": 0.03634374588727951, "state_memory": 0.015356356278061867, "state_peak": 41.2098274230957, "growth_active": 1.1483262777328491, "growth_memory": 1.5012742280960083, "heads": [{"head": 0, "memory_to_active": 0.1523555964231491, "contribution_to_active": 0.81215500831604, "mix_mean": 0.6859177947044373, "mix_low": 0.011736869812011719, "mix_high": 0.030806541442871094, "read_cosine": 0.44105279445648193, "beta_mean": 0.2320438027381897, "nonfinite": 0.0, "weighted_memory_to_active": 0.1315685659646988, "mix_std": 0.34983155131340027, "beta_std": 0.23070871829986572, "beta_low": 0.262939453125, "beta_high": 0.0, "orthogonal_to_active": 0.09597823768854141, "evidence_weights": [-0.24271416664123535, -0.3555808663368225, -0.22232738137245178], "agreement_weight": -0.0004721881414297968, "orthogonal_weight": 0.875519335269928}, {"head": 1, "memory_to_active": 0.04048266261816025, "contribution_to_active": 0.6701679229736328, "mix_mean": 0.7075017690658569, "mix_low": 6.771087646484375e-05, "mix_high": 0.019431114196777344, "read_cosine": 0.555587649345398, "beta_mean": 0.0007997270440682769, "nonfinite": 0.0, "weighted_memory_to_active": 0.030591540038585663, "mix_std": 0.23632381856441498, "beta_std": 0.006109423004090786, "beta_low": 0.98193359375, "beta_high": 0.0, "orthogonal_to_active": 0.015368004329502583, "evidence_weights": [-0.23447923362255096, -0.3639981746673584, 0.5643834471702576], "agreement_weight": 0.3748261630535126, "orthogonal_weight": -0.6879124045372009}, {"head": 2, "memory_to_active": 1.3322021961212158, "contribution_to_active": 0.3774448037147522, "mix_mean": 0.06005723029375076, "mix_low": 0.29966163635253906, "mix_high": 1.33514404296875e-05, "read_cosine": 0.21351318061351776, "beta_mean": 0.04268088936805725, "nonfinite": 0.0, "weighted_memory_to_active": 0.25189292430877686, "mix_std": 0.1047927737236023, "beta_std": 0.0507914200425148, "beta_low": 0.1959228515625, "beta_high": 0.0, "orthogonal_to_active": 0.09915823489427567, "evidence_weights": [0.023718129843473434, -0.06765259802341461, 0.3755531907081604], "agreement_weight": 0.1277867704629898, "orthogonal_weight": -0.5008899569511414}, {"head": 3, "memory_to_active": 1.916621208190918, "contribution_to_active": 0.14308401942253113, "mix_mean": 0.03238941729068756, "mix_low": 0.4048786163330078, "mix_high": 0.0, "read_cosine": 0.24158701300621033, "beta_mean": 0.12103340774774551, "nonfinite": 0.0, "weighted_memory_to_active": 0.19391360878944397, "mix_std": 0.05504661053419113, "beta_std": 0.11347928643226624, "beta_low": 0.034423828125, "beta_high": 0.0, "orthogonal_to_active": 0.002684945473447442, "evidence_weights": [-0.022201739251613617, 0.048618514090776443, -0.25323405861854553], "agreement_weight": 0.07897668331861496, "orthogonal_weight": 0.02051166445016861}, {"head": 4, "memory_to_active": 0.007297391537576914, "contribution_to_active": 0.6367600560188293, "mix_mean": 0.7537684440612793, "mix_low": 3.4332275390625e-05, "mix_high": 0.018754005432128906, "read_cosine": 0.12970468401908875, "beta_mean": 1.5083540347404778e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0053505669347941875, "mix_std": 0.2089601755142212, "beta_std": 5.6188295275205746e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003444805508479476, "evidence_weights": [-0.3251142203807831, -0.3132987916469574, 0.24302619695663452], "agreement_weight": 0.32145220041275024, "orthogonal_weight": -0.6806871891021729}, {"head": 5, "memory_to_active": 2.5720443725585938, "contribution_to_active": 0.3628259599208832, "mix_mean": 0.013801360502839088, "mix_low": 0.6482839584350586, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2357105016708374, "beta_mean": 0.22207459807395935, "nonfinite": 0.0, "weighted_memory_to_active": 0.21019992232322693, "mix_std": 0.02920600399374962, "beta_std": 0.16535711288452148, "beta_low": 0.0, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.17678216099739075, "evidence_weights": [0.05803840979933739, -0.015726441517472267, -0.26248249411582947], "agreement_weight": -0.09300723671913147, "orthogonal_weight": 1.1178544759750366}]}, {"layer": 5, "memory_to_active": 0.35718053579330444, "contribution_to_active": 0.550413191318512, "mix_mean": 0.5243451595306396, "mix_low": 0.05603933334350586, "mix_high": 0.02133003994822502, "read_cosine": 0.08667491376399994, "beta_mean": 0.049281638115644455, "nonfinite": 0.0, "weighted_memory_to_active": 0.12619030475616455, "mix_std": 0.2498207986354828, "beta_std": 0.06501457095146179, "beta_low": 0.6909383535385132, "beta_high": 0.0, "orthogonal_to_active": 0.05502305179834366, "output_influence": 0.5983456373214722, "residual_influence": 0.20070165395736694, "state_active": 0.03831629082560539, "state_memory": 0.009614828042685986, "state_peak": 62.134342193603516, "growth_active": 1.0793625116348267, "growth_memory": 1.2346471548080444, "heads": [{"head": 0, "memory_to_active": 0.1727197766304016, "contribution_to_active": 0.5988371968269348, "mix_mean": 0.5109528303146362, "mix_low": 0.006290435791015625, "mix_high": 0.0026493072509765625, "read_cosine": -0.12028811126947403, "beta_mean": 0.013525218702852726, "nonfinite": 0.0, "weighted_memory_to_active": 0.06769698858261108, "mix_std": 0.2768166661262512, "beta_std": 0.023956960067152977, "beta_low": 0.623779296875, "beta_high": 0.0, "orthogonal_to_active": 0.030946867540478706, "evidence_weights": [-0.1704254299402237, -0.1897004395723343, 0.12225169688463211], "agreement_weight": -0.025698047131299973, "orthogonal_weight": -0.4790031909942627}, {"head": 1, "memory_to_active": 0.0043456340208649635, "contribution_to_active": 0.5614566206932068, "mix_mean": 0.7161287069320679, "mix_low": 0.00058746337890625, "mix_high": 0.03758525848388672, "read_cosine": 0.22171638906002045, "beta_mean": 9.855773896561004e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0027547315694391727, "mix_std": 0.2631518542766571, "beta_std": 2.566098191891797e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0025744736194610596, "evidence_weights": [-0.24413177371025085, -0.28375890851020813, 0.94949871301651], "agreement_weight": 0.728000819683075, "orthogonal_weight": -0.9790634512901306}, {"head": 2, "memory_to_active": 0.014686751179397106, "contribution_to_active": 0.6257920861244202, "mix_mean": 0.6710968017578125, "mix_low": 0.0018720626831054688, "mix_high": 0.022416114807128906, "read_cosine": -0.40284788608551025, "beta_mean": 7.56832305341959e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009742246009409428, "mix_std": 0.2601529061794281, "beta_std": 0.0003069300146307796, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.009428054094314575, "evidence_weights": [-0.25674447417259216, -0.31207239627838135, 0.178133025765419], "agreement_weight": -0.2000432014465332, "orthogonal_weight": -1.054810643196106}, {"head": 3, "memory_to_active": 1.450128197669983, "contribution_to_active": 0.23881566524505615, "mix_mean": 0.11718595772981644, "mix_low": 0.11618518829345703, "mix_high": 0.0, "read_cosine": 0.14165857434272766, "beta_mean": 0.09613613784313202, "nonfinite": 0.0, "weighted_memory_to_active": 0.34644702076911926, "mix_std": 0.14693422615528107, "beta_std": 0.15286500751972198, "beta_low": 0.2103271484375, "beta_high": 0.0, "orthogonal_to_active": 0.14314822852611542, "evidence_weights": [-0.07697267085313797, -0.09819193184375763, 0.13011905550956726], "agreement_weight": 0.0530567541718483, "orthogonal_weight": -0.501157283782959}, {"head": 4, "memory_to_active": 0.4344591200351715, "contribution_to_active": 0.5812850594520569, "mix_mean": 0.31515979766845703, "mix_low": 0.20989131927490234, "mix_high": 0.004767417907714844, "read_cosine": 0.5351916551589966, "beta_mean": 0.185757577419281, "nonfinite": 0.0, "weighted_memory_to_active": 0.2833937406539917, "mix_std": 0.33463555574417114, "beta_std": 0.21203070878982544, "beta_low": 0.31365966796875, "beta_high": 0.0, "orthogonal_to_active": 0.11685390025377274, "evidence_weights": [-0.11099253594875336, -0.2501528263092041, -0.06728733330965042], "agreement_weight": 0.03456486389040947, "orthogonal_weight": 0.65370774269104}, {"head": 5, "memory_to_active": 0.06674390286207199, "contribution_to_active": 0.6962924003601074, "mix_mean": 0.8155466318130493, "mix_low": 0.0014095306396484375, "mix_high": 0.0605621337890625, "read_cosine": 0.14461879432201385, "beta_mean": 0.00018534567789174616, "nonfinite": 0.0, "weighted_memory_to_active": 0.047107089310884476, "mix_std": 0.2172335386276245, "beta_std": 0.0009021639707498252, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.027186771854758263, "evidence_weights": [-0.303284227848053, -0.3269888460636139, 1.8214244842529297], "agreement_weight": 0.5476317405700684, "orthogonal_weight": -0.6079598069190979}]}, {"layer": 6, "memory_to_active": 0.2265041470527649, "contribution_to_active": 0.6443321704864502, "mix_mean": 0.5966118574142456, "mix_low": 0.03256305307149887, "mix_high": 0.027716955170035362, "read_cosine": 0.31266695261001587, "beta_mean": 0.06275847554206848, "nonfinite": 0.0, "weighted_memory_to_active": 0.11836718022823334, "mix_std": 0.2396390438079834, "beta_std": 0.07421562075614929, "beta_low": 0.6270955801010132, "beta_high": 0.0, "orthogonal_to_active": 0.08480563759803772, "output_influence": 0.5810763835906982, "residual_influence": 0.21684744954109192, "state_active": 0.04262765869498253, "state_memory": 0.005458001047372818, "state_peak": 86.26890563964844, "growth_active": 1.2078322172164917, "growth_memory": 1.1677697896957397, "heads": [{"head": 0, "memory_to_active": 0.10706295073032379, "contribution_to_active": 0.7713016867637634, "mix_mean": 0.8241336345672607, "mix_low": 0.0001125335693359375, "mix_high": 0.06424236297607422, "read_cosine": 0.29370611906051636, "beta_mean": 0.010292893275618553, "nonfinite": 0.0, "weighted_memory_to_active": 0.09636030346155167, "mix_std": 0.20005352795124054, "beta_std": 0.03212226927280426, "beta_low": 0.85223388671875, "beta_high": 0.0, "orthogonal_to_active": 0.06313013285398483, "evidence_weights": [-0.29549384117126465, -0.34423619508743286, 1.1720749139785767], "agreement_weight": 0.9297555088996887, "orthogonal_weight": -0.7025808095932007}, {"head": 1, "memory_to_active": 0.0036566671915352345, "contribution_to_active": 0.5380255579948425, "mix_mean": 0.6011278033256531, "mix_low": 0.00042247772216796875, "mix_high": 0.01692676544189453, "read_cosine": 0.4801683723926544, "beta_mean": 7.952745363581926e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.001763397129252553, "mix_std": 0.28140324354171753, "beta_std": 1.4882360119372606e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009324049460701644, "evidence_weights": [-0.1878519207239151, -0.29974105954170227, 1.1125671863555908], "agreement_weight": 0.13003212213516235, "orthogonal_weight": -0.6888880729675293}, {"head": 2, "memory_to_active": 0.6089094877243042, "contribution_to_active": 0.10329940915107727, "mix_mean": 0.05894035845994949, "mix_low": 0.18581485748291016, "mix_high": 0.0, "read_cosine": 0.27360397577285767, "beta_mean": 0.019002610817551613, "nonfinite": 0.0, "weighted_memory_to_active": 0.0632377564907074, "mix_std": 0.0830797478556633, "beta_std": 0.023252513259649277, "beta_low": 0.36956787109375, "beta_high": 0.0, "orthogonal_to_active": 0.012929140590131283, "evidence_weights": [0.05579021945595741, -0.07314937561750412, 0.3787982165813446], "agreement_weight": 0.1336943656206131, "orthogonal_weight": -0.2425146996974945}, {"head": 3, "memory_to_active": 0.13467741012573242, "contribution_to_active": 0.6880333423614502, "mix_mean": 0.6661648154258728, "mix_low": 0.007956504821777344, "mix_high": 0.021440505981445312, "read_cosine": 0.4544607102870941, "beta_mean": 0.2228020429611206, "nonfinite": 0.0, "weighted_memory_to_active": 0.0975358784198761, "mix_std": 0.32532772421836853, "beta_std": 0.25244614481925964, "beta_low": 0.21533203125, "beta_high": 0.0, "orthogonal_to_active": 0.060877107083797455, "evidence_weights": [-0.16456906497478485, -0.31082987785339355, 0.10328243672847748], "agreement_weight": -0.010943662375211716, "orthogonal_weight": 0.7083432674407959}, {"head": 4, "memory_to_active": 0.4816749691963196, "contribution_to_active": 1.1219160556793213, "mix_mean": 0.7297027707099915, "mix_low": 0.0009279251098632812, "mix_high": 0.054984092712402344, "read_cosine": 0.08882806450128555, "beta_mean": 0.12433698773384094, "nonfinite": 0.0, "weighted_memory_to_active": 0.4342055022716522, "mix_std": 0.31011542677879333, "beta_std": 0.1371728777885437, "beta_low": 0.32550048828125, "beta_high": 0.0, "orthogonal_to_active": 0.3544522225856781, "evidence_weights": [-0.15831100940704346, -0.5024921894073486, -0.7838472723960876], "agreement_weight": -0.014320183545351028, "orthogonal_weight": 0.8385603427886963}, {"head": 5, "memory_to_active": 0.02304336056113243, "contribution_to_active": 0.6434168219566345, "mix_mean": 0.6996011137962341, "mix_low": 0.00014400482177734375, "mix_high": 0.008708000183105469, "read_cosine": 0.2852344810962677, "beta_mean": 0.00010836226283572614, "nonfinite": 0.0, "weighted_memory_to_active": 0.017100173979997635, "mix_std": 0.237854465842247, "beta_std": 0.00028499195468612015, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.016512740403413773, "evidence_weights": [-0.23163244128227234, -0.3145325183868408, 0.35464173555374146], "agreement_weight": 0.23175925016403198, "orthogonal_weight": -1.0374852418899536}]}, {"layer": 7, "memory_to_active": 1.7618417739868164, "contribution_to_active": 0.27848225831985474, "mix_mean": 0.18121100962162018, "mix_low": 0.2151743620634079, "mix_high": 0.0010725657921284437, "read_cosine": 0.24326077103614807, "beta_mean": 0.1345994621515274, "nonfinite": 0.0, "weighted_memory_to_active": 0.20783008635044098, "mix_std": 0.1488378942012787, "beta_std": 0.09755649417638779, "beta_low": 0.2268269956111908, "beta_high": 0.00020345053053461015, "orthogonal_to_active": 0.0912054181098938, "output_influence": 0.29593029618263245, "residual_influence": 0.07571583241224289, "state_active": 0.013499225489795208, "state_memory": 0.023239295929670334, "state_peak": 31.67401123046875, "growth_active": 1.123016357421875, "growth_memory": 1.2591663599014282, "heads": [{"head": 0, "memory_to_active": 2.7538933753967285, "contribution_to_active": 0.1743040829896927, "mix_mean": 0.039540018886327744, "mix_low": 0.4066009521484375, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3443794548511505, "beta_mean": 0.17422601580619812, "nonfinite": 0.0, "weighted_memory_to_active": 0.19639654457569122, "mix_std": 0.07848627865314484, "beta_std": 0.12765620648860931, "beta_low": 0.002685546875, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.0001614074280951172, "evidence_weights": [0.04466629400849342, -0.0918450653553009, 0.03220782056450844], "agreement_weight": -0.029619211331009865, "orthogonal_weight": 0.0009095377754420042}, {"head": 1, "memory_to_active": 0.9765812158584595, "contribution_to_active": 0.39495402574539185, "mix_mean": 0.22584353387355804, "mix_low": 0.10948371887207031, "mix_high": 0.005532264709472656, "read_cosine": 0.4114197790622711, "beta_mean": 0.10129787027835846, "nonfinite": 0.0, "weighted_memory_to_active": 0.34130969643592834, "mix_std": 0.2459535002708435, "beta_std": 0.0670415386557579, "beta_low": 0.000244140625, "beta_high": 0.0, "orthogonal_to_active": 0.04561956599354744, "evidence_weights": [-0.21617747843265533, -0.1046113595366478, -0.3040188252925873], "agreement_weight": 0.05002793297171593, "orthogonal_weight": 0.16270381212234497}, {"head": 2, "memory_to_active": 0.578619122505188, "contribution_to_active": 0.4097376763820648, "mix_mean": 0.24860936403274536, "mix_low": 0.013178825378417969, "mix_high": 0.0004425048828125, "read_cosine": 0.1203889399766922, "beta_mean": 0.030801959335803986, "nonfinite": 0.0, "weighted_memory_to_active": 0.2421310842037201, "mix_std": 0.2193635255098343, "beta_std": 0.034567490220069885, "beta_low": 0.20733642578125, "beta_high": 0.0, "orthogonal_to_active": 0.14487487077713013, "evidence_weights": [-0.0998893603682518, -0.1875784695148468, 0.8875676989555359], "agreement_weight": 0.014320703223347664, "orthogonal_weight": -0.6196920275688171}, {"head": 3, "memory_to_active": 4.7234954833984375, "contribution_to_active": 0.10746090859174728, "mix_mean": 0.03668540343642235, "mix_low": 0.2924346923828125, "mix_high": 0.0, "read_cosine": 0.0988081619143486, "beta_mean": 0.43565624952316284, "nonfinite": 0.0, "weighted_memory_to_active": 0.3778722882270813, "mix_std": 0.05147023871541023, "beta_std": 0.2794245183467865, "beta_low": 0.00140380859375, "beta_high": 0.0008544921875, "orthogonal_to_active": 0.3250230550765991, "evidence_weights": [0.07202325016260147, -0.04751455783843994, 0.7103683352470398], "agreement_weight": 0.04960675537586212, "orthogonal_weight": -0.9420607686042786}, {"head": 4, "memory_to_active": 1.5071918964385986, "contribution_to_active": 0.11141546815633774, "mix_mean": 0.026193805038928986, "mix_low": 0.46793460845947266, "mix_high": 0.0, "read_cosine": 0.2428269386291504, "beta_mean": 0.06549516320228577, "nonfinite": 0.0, "weighted_memory_to_active": 0.07335640490055084, "mix_std": 0.04837788641452789, "beta_std": 0.07630762457847595, "beta_low": 0.1494140625, "beta_high": 0.0, "orthogonal_to_active": 0.016482630744576454, "evidence_weights": [0.11414866894483566, -0.030871693044900894, 0.19468645751476288], "agreement_weight": 0.03472239151597023, "orthogonal_weight": -0.2910600006580353}, {"head": 5, "memory_to_active": 0.031269147992134094, "contribution_to_active": 0.47302132844924927, "mix_mean": 0.5103939175605774, "mix_low": 0.0014133453369140625, "mix_high": 0.0004596710205078125, "read_cosine": 0.24174129962921143, "beta_mean": 0.00011945120058953762, "nonfinite": 0.0, "weighted_memory_to_active": 0.015914499759674072, "mix_std": 0.24937590956687927, "beta_std": 0.0003415785322431475, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.015070988796651363, "evidence_weights": [-0.1861748844385147, -0.2421572506427765, 0.48896414041519165], "agreement_weight": 0.2487718015909195, "orthogonal_weight": -1.0100303888320923}]}, {"layer": 8, "memory_to_active": 0.1431984007358551, "contribution_to_active": 0.5827853679656982, "mix_mean": 0.5160317420959473, "mix_low": 0.009020647034049034, "mix_high": 0.0021985373459756374, "read_cosine": 0.3927820324897766, "beta_mean": 0.013100828975439072, "nonfinite": 0.0, "weighted_memory_to_active": 0.07310459762811661, "mix_std": 0.2425161451101303, "beta_std": 0.02002999559044838, "beta_low": 0.7809957265853882, "beta_high": 0.0, "orthogonal_to_active": 0.06831403076648712, "output_influence": 0.5373797416687012, "residual_influence": 0.2075803279876709, "state_active": 0.04344169795513153, "state_memory": 0.004781324416399002, "state_peak": 64.55787658691406, "growth_active": 1.1430951356887817, "growth_memory": 1.0629661083221436, "heads": [{"head": 0, "memory_to_active": 0.0564492866396904, "contribution_to_active": 0.5815577507019043, "mix_mean": 0.6142652034759521, "mix_low": 0.00046443939208984375, "mix_high": 0.001434326171875, "read_cosine": 0.4577065706253052, "beta_mean": 0.0036421469412744045, "nonfinite": 0.0, "weighted_memory_to_active": 0.04085677117109299, "mix_std": 0.24685221910476685, "beta_std": 0.015311092138290405, "beta_low": 0.9249267578125, "beta_high": 0.0, "orthogonal_to_active": 0.02548515610396862, "evidence_weights": [-0.15243560075759888, -0.3742855191230774, 0.5558279156684875], "agreement_weight": 0.27363860607147217, "orthogonal_weight": -0.7936104536056519}, {"head": 1, "memory_to_active": 0.0691421777009964, "contribution_to_active": 0.5563453435897827, "mix_mean": 0.535568118095398, "mix_low": 0.000133514404296875, "mix_high": 0.00182342529296875, "read_cosine": 0.09259305894374847, "beta_mean": 0.0011027163127437234, "nonfinite": 0.0, "weighted_memory_to_active": 0.031505633145570755, "mix_std": 0.23147526383399963, "beta_std": 0.010108381509780884, "beta_low": 0.9903564453125, "beta_high": 0.0, "orthogonal_to_active": 0.023419838398694992, "evidence_weights": [-0.19229932129383087, -0.2724085748195648, 0.575947105884552], "agreement_weight": 0.06874468177556992, "orthogonal_weight": -0.7787696719169617}, {"head": 2, "memory_to_active": 0.010027156211435795, "contribution_to_active": 0.6970348358154297, "mix_mean": 0.6923307180404663, "mix_low": 0.00024127960205078125, "mix_high": 0.0039424896240234375, "read_cosine": 0.4112218916416168, "beta_mean": 0.0001039669441524893, "nonfinite": 0.0, "weighted_memory_to_active": 0.007923541590571404, "mix_std": 0.21779413521289825, "beta_std": 0.0032394693698734045, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00463471282273531, "evidence_weights": [-0.21863023936748505, -0.34419506788253784, 0.9741971492767334], "agreement_weight": 0.34616947174072266, "orthogonal_weight": -0.6684538722038269}, {"head": 3, "memory_to_active": 0.10293323546648026, "contribution_to_active": 0.6523710489273071, "mix_mean": 0.5922151207923889, "mix_low": 0.0009059906005859375, "mix_high": 0.005644798278808594, "read_cosine": 0.7975721955299377, "beta_mean": 0.007732304744422436, "nonfinite": 0.0, "weighted_memory_to_active": 0.07756318151950836, "mix_std": 0.2542121708393097, "beta_std": 0.0080137113109231, "beta_low": 0.7366943359375, "beta_high": 0.0, "orthogonal_to_active": 0.02324029989540577, "evidence_weights": [-0.16756878793239594, -0.2958887815475464, 0.5223612189292908], "agreement_weight": 0.2662840783596039, "orthogonal_weight": -0.5881407856941223}, {"head": 4, "memory_to_active": 0.26001882553100586, "contribution_to_active": 0.3611983358860016, "mix_mean": 0.37521040439605713, "mix_low": 0.0012149810791015625, "mix_high": 6.67572021484375e-06, "read_cosine": 0.10002781450748444, "beta_mean": 0.000815067789517343, "nonfinite": 0.0, "weighted_memory_to_active": 0.08929187059402466, "mix_std": 0.23411954939365387, "beta_std": 0.014083722606301308, "beta_low": 0.9906005859375, "beta_high": 0.0, "orthogonal_to_active": 0.016589822247624397, "evidence_weights": [-0.14725807309150696, -0.2047852724790573, 0.6743549108505249], "agreement_weight": 0.03292388468980789, "orthogonal_weight": -0.1877417415380478}, {"head": 5, "memory_to_active": 0.3606196641921997, "contribution_to_active": 0.6482049226760864, "mix_mean": 0.2866005301475525, "mix_low": 0.051163673400878906, "mix_high": 0.000339508056640625, "read_cosine": 0.49757057428359985, "beta_mean": 0.06520877033472061, "nonfinite": 0.0, "weighted_memory_to_active": 0.19148653745651245, "mix_std": 0.27064353227615356, "beta_std": 0.06942359358072281, "beta_low": 0.04345703125, "beta_high": 0.0, "orthogonal_to_active": 0.3165143132209778, "evidence_weights": [-0.15209482610225677, -0.14081959426403046, -0.11278937757015228], "agreement_weight": -0.011074935086071491, "orthogonal_weight": 1.961438775062561}]}, {"layer": 9, "memory_to_active": 0.7970410585403442, "contribution_to_active": 0.7264509797096252, "mix_mean": 0.4040601849555969, "mix_low": 0.04847876355051994, "mix_high": 0.0065356893464922905, "read_cosine": 0.34202492237091064, "beta_mean": 0.03322548791766167, "nonfinite": 0.0, "weighted_memory_to_active": 0.3695300221443176, "mix_std": 0.24882705509662628, "beta_std": 0.02828405238687992, "beta_low": 0.6594747304916382, "beta_high": 0.0, "orthogonal_to_active": 0.030266471207141876, "output_influence": 0.5700575709342957, "residual_influence": 0.1797286719083786, "state_active": 0.045665234327316284, "state_memory": 0.013654444366693497, "state_peak": 60.534000396728516, "growth_active": 1.221542477607727, "growth_memory": 1.2655649185180664, "heads": [{"head": 0, "memory_to_active": 0.00035977212246507406, "contribution_to_active": 0.5853155255317688, "mix_mean": 0.6605066657066345, "mix_low": 0.0011415481567382812, "mix_high": 0.02718067169189453, "read_cosine": 0.32791489362716675, "beta_mean": 9.22501328659564e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.0002392177702859044, "mix_std": 0.283627450466156, "beta_std": 3.534506049618358e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0002366936969337985, "evidence_weights": [-0.30311617255210876, -0.30803972482681274, 0.32407066226005554], "agreement_weight": 0.10316949337720871, "orthogonal_weight": -1.043264627456665}, {"head": 1, "memory_to_active": 0.018112607300281525, "contribution_to_active": 0.7182487845420837, "mix_mean": 0.5831907987594604, "mix_low": 0.0068378448486328125, "mix_high": 0.0030298233032226562, "read_cosine": 0.4483819603919983, "beta_mean": 3.9734783058520406e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013883156701922417, "mix_std": 0.2760850191116333, "beta_std": 9.888214844977483e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008779807016253471, "evidence_weights": [-0.23456472158432007, -0.2907409369945526, 0.9421428442001343], "agreement_weight": 0.5695982575416565, "orthogonal_weight": -0.7848199009895325}, {"head": 2, "memory_to_active": 0.1480981707572937, "contribution_to_active": 0.6647415161132812, "mix_mean": 0.6040732264518738, "mix_low": 0.0017728805541992188, "mix_high": 0.007069587707519531, "read_cosine": 0.3982047438621521, "beta_mean": 0.0005210680537857115, "nonfinite": 0.0, "weighted_memory_to_active": 0.10212138295173645, "mix_std": 0.2643401324748993, "beta_std": 0.004095809534192085, "beta_low": 0.99700927734375, "beta_high": 0.0, "orthogonal_to_active": 0.05713493749499321, "evidence_weights": [-0.19943207502365112, -0.2750576138496399, 0.7488088607788086], "agreement_weight": 0.36391398310661316, "orthogonal_weight": -0.6694902777671814}, {"head": 3, "memory_to_active": 0.6888182163238525, "contribution_to_active": 0.28648218512535095, "mix_mean": 0.18857544660568237, "mix_low": 0.07366371154785156, "mix_high": 2.47955322265625e-05, "read_cosine": 0.37445181608200073, "beta_mean": 0.024984359741210938, "nonfinite": 0.0, "weighted_memory_to_active": 0.1973360776901245, "mix_std": 0.214748352766037, "beta_std": 0.026380905881524086, "beta_low": 0.26025390625, "beta_high": 0.0, "orthogonal_to_active": 0.09592831134796143, "evidence_weights": [-0.04603796824812889, -0.1180352047085762, 0.5254080891609192], "agreement_weight": 0.09392359107732773, "orthogonal_weight": -0.5443941950798035}, {"head": 4, "memory_to_active": 3.031146287918091, "contribution_to_active": 1.766232967376709, "mix_mean": 0.15142467617988586, "mix_low": 0.11869430541992188, "mix_high": 0.0011453628540039062, "read_cosine": 0.06766137480735779, "beta_mean": 0.15409277379512787, "nonfinite": 0.0, "weighted_memory_to_active": 1.6450096368789673, "mix_std": 0.21030908823013306, "beta_std": 0.08472558110952377, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035010408610105515, "evidence_weights": [-0.07918649911880493, -0.14819110929965973, -0.04382982477545738], "agreement_weight": -0.06712546944618225, "orthogonal_weight": -0.0022128422278910875}, {"head": 5, "memory_to_active": 0.8957110047340393, "contribution_to_active": 0.33768486976623535, "mix_mean": 0.236590176820755, "mix_low": 0.08876228332519531, "mix_high": 0.0007638931274414062, "read_cosine": 0.43553444743156433, "beta_mean": 0.01971404440701008, "nonfinite": 0.0, "weighted_memory_to_active": 0.2585904598236084, "mix_std": 0.24385224282741547, "beta_std": 0.05439959466457367, "beta_low": 0.6995849609375, "beta_high": 0.0, "orthogonal_to_active": 0.01601802371442318, "evidence_weights": [0.030569661408662796, -0.19991786777973175, 0.14464475214481354], "agreement_weight": 0.00809355266392231, "orthogonal_weight": 0.071483314037323}]}, {"layer": 10, "memory_to_active": 0.20857787132263184, "contribution_to_active": 0.7545801401138306, "mix_mean": 0.5345239639282227, "mix_low": 0.02547089383006096, "mix_high": 0.005341371521353722, "read_cosine": 0.3788406252861023, "beta_mean": 0.04286237061023712, "nonfinite": 0.0, "weighted_memory_to_active": 0.08772935718297958, "mix_std": 0.2657865881919861, "beta_std": 0.04380805045366287, "beta_low": 0.6621806025505066, "beta_high": 0.0, "orthogonal_to_active": 0.2863522171974182, "output_influence": 0.4842817187309265, "residual_influence": 0.17857064306735992, "state_active": 0.05343830958008766, "state_memory": 0.013676390051841736, "state_peak": 123.85283660888672, "growth_active": 1.037886381149292, "growth_memory": 1.0039937496185303, "heads": [{"head": 0, "memory_to_active": 0.7407093644142151, "contribution_to_active": 2.171921491622925, "mix_mean": 0.562909722328186, "mix_low": 0.12810420989990234, "mix_high": 0.0062885284423828125, "read_cosine": 0.07964184135198593, "beta_mean": 0.18585294485092163, "nonfinite": 0.0, "weighted_memory_to_active": 0.4185762405395508, "mix_std": 0.3969319760799408, "beta_std": 0.1954304575920105, "beta_low": 0.16064453125, "beta_high": 0.0, "orthogonal_to_active": 1.637412190437317, "evidence_weights": [-0.0804629847407341, -0.3590903878211975, -1.9679116010665894], "agreement_weight": 0.04130435734987259, "orthogonal_weight": 3.9360885620117188}, {"head": 1, "memory_to_active": 0.005370550323277712, "contribution_to_active": 0.633954644203186, "mix_mean": 0.643444299697876, "mix_low": 0.0010623931884765625, "mix_high": 0.01768779754638672, "read_cosine": 0.3346850275993347, "beta_mean": 3.562666097423062e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031452083494514227, "mix_std": 0.26053473353385925, "beta_std": 6.299822416622192e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004844333045184612, "evidence_weights": [-0.24786706268787384, -0.19118312001228333, 0.5414668917655945], "agreement_weight": 0.6742144227027893, "orthogonal_weight": -1.6785695552825928}, {"head": 2, "memory_to_active": 0.012015610933303833, "contribution_to_active": 0.6239705085754395, "mix_mean": 0.6786196231842041, "mix_low": 0.0001316070556640625, "mix_high": 0.006167411804199219, "read_cosine": 0.026047412306070328, "beta_mean": 2.4577504518674687e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005101828835904598, "mix_std": 0.2459249049425125, "beta_std": 9.846340981312096e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020770553965121508, "evidence_weights": [-0.26771557331085205, -0.27580246329307556, 0.7122902870178223], "agreement_weight": -0.22629964351654053, "orthogonal_weight": -0.38710570335388184}, {"head": 3, "memory_to_active": 0.36812686920166016, "contribution_to_active": 0.10580934584140778, "mix_mean": 0.2946423292160034, "mix_low": 0.021787643432617188, "mix_high": 6.67572021484375e-05, "read_cosine": 0.9257864952087402, "beta_mean": 0.06425800919532776, "nonfinite": 0.0, "weighted_memory_to_active": 0.04088955372571945, "mix_std": 0.22731053829193115, "beta_std": 0.057933151721954346, "beta_low": 0.0048828125, "beta_high": 0.0, "orthogonal_to_active": 0.043509528040885925, "evidence_weights": [-0.08735942840576172, -0.141252338886261, 0.15188221633434296], "agreement_weight": 0.019893577322363853, "orthogonal_weight": 1.4874054193496704}, {"head": 4, "memory_to_active": 0.011891382746398449, "contribution_to_active": 0.5810768008232117, "mix_mean": 0.6088426113128662, "mix_low": 0.00016880035400390625, "mix_high": 0.0018310546875, "read_cosine": 0.08556132018566132, "beta_mean": 2.8349808417260647e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00724272895604372, "mix_std": 0.24020051956176758, "beta_std": 5.141214205650613e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010622859001159668, "evidence_weights": [-0.18220162391662598, -0.25023505091667175, 0.9289290904998779], "agreement_weight": 0.18707233667373657, "orthogonal_weight": -1.541149377822876}, {"head": 5, "memory_to_active": 0.11335338652133942, "contribution_to_active": 0.41074803471565247, "mix_mean": 0.4186848998069763, "mix_low": 0.0015707015991210938, "mix_high": 6.67572021484375e-06, "read_cosine": 0.8213214874267578, "beta_mean": 0.006974692922085524, "nonfinite": 0.0, "weighted_memory_to_active": 0.05142059177160263, "mix_std": 0.22381684184074402, "beta_std": 0.009271814487874508, "beta_low": 0.80755615234375, "beta_high": 0.0, "orthogonal_to_active": 0.01964736171066761, "evidence_weights": [-0.1354392021894455, -0.16580981016159058, 0.2782699763774872], "agreement_weight": 0.09819642454385757, "orthogonal_weight": -0.6803607940673828}]}, {"layer": 11, "memory_to_active": 0.07044236361980438, "contribution_to_active": 0.6335442066192627, "mix_mean": 0.5315667390823364, "mix_low": 0.11158768832683563, "mix_high": 0.04881175607442856, "read_cosine": 0.3444426655769348, "beta_mean": 0.023631617426872253, "nonfinite": 0.0, "weighted_memory_to_active": 0.03908885270357132, "mix_std": 0.32332518696784973, "beta_std": 0.035171158611774445, "beta_low": 0.7216796875, "beta_high": 0.0, "orthogonal_to_active": 0.04161979630589485, "output_influence": 0.569674015045166, "residual_influence": 0.16816535592079163, "state_active": 0.22264018654823303, "state_memory": 0.0028936052694916725, "state_peak": 36.085784912109375, "growth_active": 1.1608084440231323, "growth_memory": 1.4459799528121948, "heads": [{"head": 0, "memory_to_active": 0.008280038833618164, "contribution_to_active": 0.6072332859039307, "mix_mean": 0.6696216464042664, "mix_low": 0.000514984130859375, "mix_high": 0.009251594543457031, "read_cosine": 0.22947271168231964, "beta_mean": 8.465485007036477e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0062960609793663025, "mix_std": 0.24937812983989716, "beta_std": 0.0037728415336459875, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.0052291941829025745, "evidence_weights": [-0.1764722317457199, -0.3557209074497223, 0.8037052154541016], "agreement_weight": 0.42108574509620667, "orthogonal_weight": -0.8549239635467529}, {"head": 1, "memory_to_active": 0.11179564893245697, "contribution_to_active": 0.4449790120124817, "mix_mean": 0.4793654680252075, "mix_low": 0.00440216064453125, "mix_high": 0.0006589889526367188, "read_cosine": 0.4984661638736725, "beta_mean": 0.0019256045343354344, "nonfinite": 0.0, "weighted_memory_to_active": 0.056424159556627274, "mix_std": 0.26933109760284424, "beta_std": 0.008340015076100826, "beta_low": 0.9632568359375, "beta_high": 0.0, "orthogonal_to_active": 0.03119855746626854, "evidence_weights": [-0.13279959559440613, -0.27880093455314636, 0.6008504033088684], "agreement_weight": 0.08962623029947281, "orthogonal_weight": -0.662303626537323}, {"head": 2, "memory_to_active": 0.004426075145602226, "contribution_to_active": 0.9895907044410706, "mix_mean": 0.43725281953811646, "mix_low": 0.3206319808959961, "mix_high": 0.1845703125, "read_cosine": 0.15335938334465027, "beta_mean": 0.06410916149616241, "nonfinite": 0.0, "weighted_memory_to_active": 0.003180771367624402, "mix_std": 0.45375216007232666, "beta_std": 0.08737803250551224, "beta_low": 0.49481201171875, "beta_high": 0.0, "orthogonal_to_active": 0.03367660567164421, "evidence_weights": [0.04523934796452522, -0.3325824439525604, -1.3317238092422485], "agreement_weight": -0.0013842502376064658, "orthogonal_weight": 10.976059913635254}, {"head": 3, "memory_to_active": 0.007392848376184702, "contribution_to_active": 0.879412055015564, "mix_mean": 0.4143257439136505, "mix_low": 0.33650970458984375, "mix_high": 0.09003925323486328, "read_cosine": 0.6164302229881287, "beta_mean": 0.053107358515262604, "nonfinite": 0.0, "weighted_memory_to_active": 0.005006837658584118, "mix_std": 0.4426190257072449, "beta_std": 0.0823957547545433, "beta_low": 0.53546142578125, "beta_high": 0.0, "orthogonal_to_active": 0.0328165739774704, "evidence_weights": [0.22358787059783936, -0.26919230818748474, -0.08369691669940948], "agreement_weight": 0.05147611349821091, "orthogonal_weight": 11.242267608642578}, {"head": 4, "memory_to_active": 0.2059711515903473, "contribution_to_active": 0.4490276575088501, "mix_mean": 0.6100540161132812, "mix_low": 0.006389617919921875, "mix_high": 0.0016546249389648438, "read_cosine": 0.016861142590641975, "beta_mean": 0.014719473198056221, "nonfinite": 0.0, "weighted_memory_to_active": 0.11912766098976135, "mix_std": 0.26924753189086914, "beta_std": 0.018194306641817093, "beta_low": 0.54095458984375, "beta_high": 0.0, "orthogonal_to_active": 0.11721409857273102, "evidence_weights": [-0.2308240830898285, -0.23867195844650269, 1.8867980241775513], "agreement_weight": -0.2275785207748413, "orthogonal_weight": -0.9929466843605042}, {"head": 5, "memory_to_active": 0.08478838205337524, "contribution_to_active": 0.4310222864151001, "mix_mean": 0.5787805318832397, "mix_low": 0.0010776519775390625, "mix_high": 0.006695747375488281, "read_cosine": 0.5520663261413574, "beta_mean": 0.007843444123864174, "nonfinite": 0.0, "weighted_memory_to_active": 0.044497620314359665, "mix_std": 0.2556231617927551, "beta_std": 0.010945990681648254, "beta_low": 0.796142578125, "beta_high": 0.0, "orthogonal_to_active": 0.029583724215626717, "evidence_weights": [-0.19531303644180298, -0.30982691049575806, 0.6724047660827637], "agreement_weight": 0.0822736993432045, "orthogonal_weight": -0.8666819334030151}]}, {"layer": 12, "memory_to_active": 0.19705232977867126, "contribution_to_active": 0.5915811061859131, "mix_mean": 0.5559424757957458, "mix_low": 0.007957936264574528, "mix_high": 0.007185300346463919, "read_cosine": 0.21986904740333557, "beta_mean": 0.015073894523084164, "nonfinite": 0.0, "weighted_memory_to_active": 0.0996939018368721, "mix_std": 0.25709858536720276, "beta_std": 0.015003676526248455, "beta_low": 0.8308004140853882, "beta_high": 0.0, "orthogonal_to_active": 0.06605615466833115, "output_influence": 0.5213644504547119, "residual_influence": 0.1602887660264969, "state_active": 0.033434633165597916, "state_memory": 0.007345057558268309, "state_peak": 75.61991882324219, "growth_active": 1.1688090562820435, "growth_memory": 1.1247667074203491, "heads": [{"head": 0, "memory_to_active": 0.030711647123098373, "contribution_to_active": 0.6356085538864136, "mix_mean": 0.5974884033203125, "mix_low": 0.00077056884765625, "mix_high": 0.007618904113769531, "read_cosine": 0.25529706478118896, "beta_mean": 0.0005516032688319683, "nonfinite": 0.0, "weighted_memory_to_active": 0.01775580458343029, "mix_std": 0.26243069767951965, "beta_std": 0.0012871089857071638, "beta_low": 0.9969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.016379624605178833, "evidence_weights": [-0.20233005285263062, -0.27107518911361694, -0.08540302515029907], "agreement_weight": 0.10189542174339294, "orthogonal_weight": -0.9821766018867493}, {"head": 1, "memory_to_active": 0.07477634400129318, "contribution_to_active": 0.5696089863777161, "mix_mean": 0.5726543664932251, "mix_low": 0.00028324127197265625, "mix_high": 0.0022106170654296875, "read_cosine": 0.3913458585739136, "beta_mean": 0.0004356222343631089, "nonfinite": 0.0, "weighted_memory_to_active": 0.046651486307382584, "mix_std": 0.2622946798801422, "beta_std": 0.008053760975599289, "beta_low": 0.99798583984375, "beta_high": 0.0, "orthogonal_to_active": 0.038723334670066833, "evidence_weights": [-0.19806703925132751, -0.27129441499710083, 1.0368808507919312], "agreement_weight": 0.28610116243362427, "orthogonal_weight": -0.9073203206062317}, {"head": 2, "memory_to_active": 0.5938625335693359, "contribution_to_active": 0.49597835540771484, "mix_mean": 0.32599228620529175, "mix_low": 0.02333831787109375, "mix_high": 2.86102294921875e-05, "read_cosine": 0.5850621461868286, "beta_mean": 0.08847212791442871, "nonfinite": 0.0, "weighted_memory_to_active": 0.2333155870437622, "mix_std": 0.23471081256866455, "beta_std": 0.06251366436481476, "beta_low": 0.00103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.2651822865009308, "evidence_weights": [-0.2009754329919815, -0.0620734766125679, -0.0244277510792017], "agreement_weight": 0.025062108412384987, "orthogonal_weight": 1.454911231994629}, {"head": 3, "memory_to_active": 0.028221217915415764, "contribution_to_active": 0.7690916657447815, "mix_mean": 0.728408694267273, "mix_low": 9.5367431640625e-05, "mix_high": 0.02166748046875, "read_cosine": 0.06323245912790298, "beta_mean": 0.00026699856971390545, "nonfinite": 0.0, "weighted_memory_to_active": 0.021189844235777855, "mix_std": 0.22334781289100647, "beta_std": 0.005515058059245348, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.01992882788181305, "evidence_weights": [-0.28425440192222595, -0.33136439323425293, 1.6980173587799072], "agreement_weight": 0.6364864706993103, "orthogonal_weight": -0.9577200412750244}, {"head": 4, "memory_to_active": 0.38577067852020264, "contribution_to_active": 0.5744821429252625, "mix_mean": 0.4733198583126068, "mix_low": 0.016429901123046875, "mix_high": 0.005374908447265625, "read_cosine": -0.1960894763469696, "beta_mean": 0.00043092400301247835, "nonfinite": 0.0, "weighted_memory_to_active": 0.23412366211414337, "mix_std": 0.2893295884132385, "beta_std": 0.007807611487805843, "beta_low": 0.99432373046875, "beta_high": 0.0, "orthogonal_to_active": 0.022567812353372574, "evidence_weights": [-0.15698091685771942, -0.2142125368118286, -0.1494855284690857], "agreement_weight": 0.021473532542586327, "orthogonal_weight": -0.09973093867301941}, {"head": 5, "memory_to_active": 0.06897153705358505, "contribution_to_active": 0.5047168135643005, "mix_mean": 0.6377911567687988, "mix_low": 0.0068302154541015625, "mix_high": 0.006211280822753906, "read_cosine": 0.2203662395477295, "beta_mean": 0.0002860873064491898, "nonfinite": 0.0, "weighted_memory_to_active": 0.04512700065970421, "mix_std": 0.27047789096832275, "beta_std": 0.004844861105084419, "beta_low": 0.9964599609375, "beta_high": 0.0, "orthogonal_to_active": 0.033555008471012115, "evidence_weights": [-0.22129835188388824, -0.3116506338119507, 1.2298508882522583], "agreement_weight": 0.4352051913738251, "orthogonal_weight": -0.7976309061050415}]}, {"layer": 13, "memory_to_active": 0.11584999412298203, "contribution_to_active": 0.5434114933013916, "mix_mean": 0.5489236116409302, "mix_low": 0.005802790634334087, "mix_high": 0.01015838049352169, "read_cosine": 0.4160010814666748, "beta_mean": 0.014185193926095963, "nonfinite": 0.0, "weighted_memory_to_active": 0.05056474357843399, "mix_std": 0.25694575905799866, "beta_std": 0.015720929950475693, "beta_low": 0.8114929795265198, "beta_high": 0.0, "orthogonal_to_active": 0.051667772233486176, "output_influence": 0.5357074737548828, "residual_influence": 0.19836001098155975, "state_active": 0.12635429203510284, "state_memory": 0.040239330381155014, "state_peak": 99.4449462890625, "growth_active": 0.924924373626709, "growth_memory": 0.9727548956871033, "heads": [{"head": 0, "memory_to_active": 0.007056820206344128, "contribution_to_active": 0.6089809536933899, "mix_mean": 0.620292067527771, "mix_low": 0.0008077621459960938, "mix_high": 0.0044097900390625, "read_cosine": 0.43880194425582886, "beta_mean": 2.5000967070809565e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004609611816704273, "mix_std": 0.2637782096862793, "beta_std": 5.782009975519031e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004281003959476948, "evidence_weights": [-0.1877341866493225, -0.26295408606529236, 0.7229118347167969], "agreement_weight": 0.23570382595062256, "orthogonal_weight": -1.0419163703918457}, {"head": 1, "memory_to_active": 0.008011461235582829, "contribution_to_active": 0.6173394918441772, "mix_mean": 0.5807051658630371, "mix_low": 0.0013790130615234375, "mix_high": 0.004576683044433594, "read_cosine": 0.24575552344322205, "beta_mean": 3.098853267147206e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0045913876965641975, "mix_std": 0.2733097970485687, "beta_std": 6.104633212089539e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005713122896850109, "evidence_weights": [-0.1779717057943344, -0.2605586647987366, 0.5133578777313232], "agreement_weight": 0.5927776098251343, "orthogonal_weight": -1.3008629083633423}, {"head": 2, "memory_to_active": 0.06236695498228073, "contribution_to_active": 0.49717628955841064, "mix_mean": 0.3974192440509796, "mix_low": 0.0011310577392578125, "mix_high": 0.00018978118896484375, "read_cosine": 0.5853613615036011, "beta_mean": 0.007349896244704723, "nonfinite": 0.0, "weighted_memory_to_active": 0.03032374382019043, "mix_std": 0.2294226437807083, "beta_std": 0.027364933863282204, "beta_low": 0.87493896484375, "beta_high": 0.0, "orthogonal_to_active": 0.017402511090040207, "evidence_weights": [-0.1743834912776947, -0.21339456737041473, 0.40936121344566345], "agreement_weight": 0.005830342881381512, "orthogonal_weight": -0.8533831834793091}, {"head": 3, "memory_to_active": 0.19127175211906433, "contribution_to_active": 0.5622481107711792, "mix_mean": 0.6724899411201477, "mix_low": 0.025701522827148438, "mix_high": 0.03790760040283203, "read_cosine": -0.010451924987137318, "beta_mean": 0.000984563841484487, "nonfinite": 0.0, "weighted_memory_to_active": 0.1420491635799408, "mix_std": 0.29073837399482727, "beta_std": 0.011795274913311005, "beta_low": 0.99395751953125, "beta_high": 0.0, "orthogonal_to_active": 0.1288812905550003, "evidence_weights": [-0.31648942828178406, -0.24072864651679993, 1.4567028284072876], "agreement_weight": 6.850212574005127, "orthogonal_weight": -0.9769964814186096}, {"head": 4, "memory_to_active": 0.3896951973438263, "contribution_to_active": 0.248231902718544, "mix_mean": 0.35634326934814453, "mix_low": 0.00372314453125, "mix_high": 0.0, "read_cosine": 0.903900146484375, "beta_mean": 0.07656604796648026, "nonfinite": 0.0, "weighted_memory_to_active": 0.09664521366357803, "mix_std": 0.21490755677223206, "beta_std": 0.05302636697888374, "beta_low": 0.00048828125, "beta_high": 0.0, "orthogonal_to_active": 0.13246449828147888, "evidence_weights": [-0.0579976849257946, -0.22127152979373932, 0.4155266582965851], "agreement_weight": 0.00872422568500042, "orthogonal_weight": 2.3190383911132812}, {"head": 5, "memory_to_active": 0.03669772297143936, "contribution_to_active": 0.7264919281005859, "mix_mean": 0.6662921905517578, "mix_low": 0.0020742416381835938, "mix_high": 0.013866424560546875, "read_cosine": 0.33263954520225525, "beta_mean": 0.0001546642743051052, "nonfinite": 0.0, "weighted_memory_to_active": 0.025169337168335915, "mix_std": 0.26951801776885986, "beta_std": 0.002020134124904871, "beta_low": 0.99957275390625, "beta_high": 0.0, "orthogonal_to_active": 0.021264204755425453, "evidence_weights": [-0.2202182412147522, -0.2865947186946869, 0.8784124255180359], "agreement_weight": 0.21597935259342194, "orthogonal_weight": -0.932091474533081}]}, {"layer": 14, "memory_to_active": 0.22313611209392548, "contribution_to_active": 0.417148619890213, "mix_mean": 0.4316408038139343, "mix_low": 0.025988738983869553, "mix_high": 0.006269455421715975, "read_cosine": 0.6224792003631592, "beta_mean": 0.06214288994669914, "nonfinite": 0.0, "weighted_memory_to_active": 0.09129449725151062, "mix_std": 0.23778030276298523, "beta_std": 0.05879301205277443, "beta_low": 0.5069580078125, "beta_high": 0.0, "orthogonal_to_active": 0.042453423142433167, "output_influence": 0.35201478004455566, "residual_influence": 0.15601542592048645, "state_active": 0.05069965869188309, "state_memory": 0.013504250906407833, "state_peak": 52.082706451416016, "growth_active": 1.0944483280181885, "growth_memory": 1.2247695922851562, "heads": [{"head": 0, "memory_to_active": 0.37589478492736816, "contribution_to_active": 0.20127886533737183, "mix_mean": 0.20348185300827026, "mix_low": 0.012607574462890625, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8218221664428711, "beta_mean": 0.06776060163974762, "nonfinite": 0.0, "weighted_memory_to_active": 0.0819455236196518, "mix_std": 0.1780667006969452, "beta_std": 0.059521861374378204, "beta_low": 0.0079345703125, "beta_high": 0.0, "orthogonal_to_active": 0.09442012012004852, "evidence_weights": [-0.09543370455503464, -0.047572117298841476, 0.3601553440093994], "agreement_weight": 0.02291879430413246, "orthogonal_weight": 1.933018684387207}, {"head": 1, "memory_to_active": 0.4442475736141205, "contribution_to_active": 0.28982043266296387, "mix_mean": 0.1957220435142517, "mix_low": 0.13072490692138672, "mix_high": 8.58306884765625e-06, "read_cosine": 0.8851779699325562, "beta_mean": 0.26010456681251526, "nonfinite": 0.0, "weighted_memory_to_active": 0.19541746377944946, "mix_std": 0.2257251888513565, "beta_std": 0.2491016387939453, "beta_low": 0.08843994140625, "beta_high": 0.0, "orthogonal_to_active": 0.018673880025744438, "evidence_weights": [0.014355803839862347, -0.18772947788238525, -0.09481265395879745], "agreement_weight": 0.0596936009824276, "orthogonal_weight": 0.3273049294948578}, {"head": 2, "memory_to_active": 0.11767062544822693, "contribution_to_active": 0.5760151743888855, "mix_mean": 0.5916522741317749, "mix_low": 0.0023851394653320312, "mix_high": 0.0050792694091796875, "read_cosine": 0.4203639030456543, "beta_mean": 0.0015962732722982764, "nonfinite": 0.0, "weighted_memory_to_active": 0.07543627917766571, "mix_std": 0.2643634080886841, "beta_std": 0.002150928135961294, "beta_low": 0.9884033203125, "beta_high": 0.0, "orthogonal_to_active": 0.05773235112428665, "evidence_weights": [-0.12566810846328735, -0.31372585892677307, 0.903195321559906], "agreement_weight": 0.08418266475200653, "orthogonal_weight": -0.8718317747116089}, {"head": 3, "memory_to_active": 0.040018945932388306, "contribution_to_active": 0.5963883399963379, "mix_mean": 0.6836251020431519, "mix_low": 0.00380706787109375, "mix_high": 0.023443222045898438, "read_cosine": -0.04738076031208038, "beta_mean": 0.004669741727411747, "nonfinite": 0.0, "weighted_memory_to_active": 0.02582915499806404, "mix_std": 0.2526031732559204, "beta_std": 0.0071422564797103405, "beta_low": 0.8759765625, "beta_high": 0.0, "orthogonal_to_active": 0.019804393872618675, "evidence_weights": [-0.3402175009250641, -0.3131939470767975, 0.3059271574020386], "agreement_weight": 0.0681534856557846, "orthogonal_weight": -0.8362855315208435}, {"head": 4, "memory_to_active": 0.15219056606292725, "contribution_to_active": 0.5015069246292114, "mix_mean": 0.5314509868621826, "mix_low": 0.0033283233642578125, "mix_high": 0.008187294006347656, "read_cosine": 0.9054102897644043, "beta_mean": 0.03760042414069176, "nonfinite": 0.0, "weighted_memory_to_active": 0.08957839012145996, "mix_std": 0.25747615098953247, "beta_std": 0.027899501845240593, "beta_low": 0.09954833984375, "beta_high": 0.0, "orthogonal_to_active": 0.023964498192071915, "evidence_weights": [-0.20224620401859283, -0.23421548306941986, 0.44817686080932617], "agreement_weight": 0.13303978741168976, "orthogonal_weight": -0.616557240486145}, {"head": 5, "memory_to_active": 0.20879413187503815, "contribution_to_active": 0.3378819525241852, "mix_mean": 0.38391250371932983, "mix_low": 0.0030794143676757812, "mix_high": 0.000896453857421875, "read_cosine": 0.7494815587997437, "beta_mean": 0.0011257348814979196, "nonfinite": 0.0, "weighted_memory_to_active": 0.07956009358167648, "mix_std": 0.24844712018966675, "beta_std": 0.006941862404346466, "beta_low": 0.9814453125, "beta_high": 0.0, "orthogonal_to_active": 0.040125295519828796, "evidence_weights": [-0.171853169798851, -0.18354371190071106, 0.37464937567710876], "agreement_weight": 0.06218346580862999, "orthogonal_weight": -0.7599567770957947}]}, {"layer": 15, "memory_to_active": 0.05117792263627052, "contribution_to_active": 0.5053038597106934, "mix_mean": 0.44042015075683594, "mix_low": 0.006132921203970909, "mix_high": 0.0032354993745684624, "read_cosine": 0.6454439163208008, "beta_mean": 0.04200911894440651, "nonfinite": 0.0, "weighted_memory_to_active": 0.026858750730752945, "mix_std": 0.28326839208602905, "beta_std": 0.04675712808966637, "beta_low": 0.7005310654640198, "beta_high": 0.0, "orthogonal_to_active": 0.010549565777182579, "output_influence": 0.32464325428009033, "residual_influence": 0.20040321350097656, "state_active": 0.16954007744789124, "state_memory": 0.010159144178032875, "state_peak": 25.247936248779297, "growth_active": 1.1817762851715088, "growth_memory": 1.2139816284179688, "heads": [{"head": 0, "memory_to_active": 0.12105816602706909, "contribution_to_active": 0.49097466468811035, "mix_mean": 0.404938280582428, "mix_low": 0.0068569183349609375, "mix_high": 0.006833076477050781, "read_cosine": 0.5030133724212646, "beta_mean": 0.056281790137290955, "nonfinite": 0.0, "weighted_memory_to_active": 0.06496255844831467, "mix_std": 0.2863354980945587, "beta_std": 0.039327751845121384, "beta_low": 0.0391845703125, "beta_high": 0.0, "orthogonal_to_active": 0.04060585796833038, "evidence_weights": [-0.184975266456604, -0.1962723433971405, 0.540091335773468], "agreement_weight": 0.08975175023078918, "orthogonal_weight": -0.8442371487617493}, {"head": 1, "memory_to_active": 0.08767156302928925, "contribution_to_active": 0.43643906712532043, "mix_mean": 0.45071423053741455, "mix_low": 0.0006704330444335938, "mix_high": 0.007754325866699219, "read_cosine": 0.9326592087745667, "beta_mean": 0.0001619207760086283, "nonfinite": 0.0, "weighted_memory_to_active": 0.03810787573456764, "mix_std": 0.28278297185897827, "beta_std": 0.0016591098392382264, "beta_low": 0.99908447265625, "beta_high": 0.0, "orthogonal_to_active": 0.0032315817661583424, "evidence_weights": [-0.16205137968063354, -0.17504961788654327, 0.15887126326560974], "agreement_weight": 0.10795779526233673, "orthogonal_weight": 0.19176946580410004}, {"head": 2, "memory_to_active": 0.006092539057135582, "contribution_to_active": 0.5231934189796448, "mix_mean": 0.3634299337863922, "mix_low": 0.008542060852050781, "mix_high": 8.0108642578125e-05, "read_cosine": 0.7830642461776733, "beta_mean": 7.156039646361023e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0033114436082541943, "mix_std": 0.2726396918296814, "beta_std": 0.00026877468917518854, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.0011231985408812761, "evidence_weights": [-0.1979384571313858, -0.26738056540489197, 0.11993105709552765], "agreement_weight": 0.007851558737456799, "orthogonal_weight": 0.9592897891998291}, {"head": 3, "memory_to_active": 0.0017873060423880816, "contribution_to_active": 0.3833150267601013, "mix_mean": 0.4212494492530823, "mix_low": 0.0031709671020507812, "mix_high": 0.0005702972412109375, "read_cosine": 0.6019874811172485, "beta_mean": 9.704727744974662e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0006146744126453996, "mix_std": 0.2889723777770996, "beta_std": 2.3989612600416876e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00032346646185033023, "evidence_weights": [-0.27385663986206055, -0.2377672642469406, 0.1858617216348648], "agreement_weight": 0.06200333312153816, "orthogonal_weight": -0.5376564860343933}, {"head": 4, "memory_to_active": 0.0600830502808094, "contribution_to_active": 0.6465098857879639, "mix_mean": 0.4016825556755066, "mix_low": 0.0076732635498046875, "mix_high": 0.0009660720825195312, "read_cosine": 0.7081260681152344, "beta_mean": 0.19231100380420685, "nonfinite": 0.0, "weighted_memory_to_active": 0.03365221619606018, "mix_std": 0.3091295659542084, "beta_std": 0.2164376825094223, "beta_low": 0.19805908203125, "beta_high": 0.0, "orthogonal_to_active": 0.006044717039912939, "evidence_weights": [-0.2059696465730667, -0.25258898735046387, -0.02947154827415943], "agreement_weight": 0.012333359569311142, "orthogonal_weight": 0.40170061588287354}, {"head": 5, "memory_to_active": 0.030374862253665924, "contribution_to_active": 0.5513908863067627, "mix_mean": 0.6005061864852905, "mix_low": 0.009883880615234375, "mix_high": 0.0032091140747070312, "read_cosine": 0.34381306171417236, "beta_mean": 0.0032187215983867645, "nonfinite": 0.0, "weighted_memory_to_active": 0.020503738895058632, "mix_std": 0.259750097990036, "beta_std": 0.022825445979833603, "beta_low": 0.967041015625, "beta_high": 0.0, "orthogonal_to_active": 0.011968567967414856, "evidence_weights": [-0.2806013226509094, -0.307086706161499, 0.5833624005317688], "agreement_weight": 0.31853607296943665, "orthogonal_weight": 0.8073306083679199}]}]}, {"step": 75000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.794744296999852, "memory_to_active": 0.8117629289627075, "contribution_to_active": 0.5817658305168152, "mix_mean": 0.4322547912597656, "mix_low": 0.12081689387559891, "mix_high": 0.011198749765753746, "read_cosine": 0.3424557149410248, "beta_mean": 0.06830387562513351, "nonfinite": 0.0, "weighted_memory_to_active": 0.19989794492721558, "mix_std": 0.22186458110809326, "beta_std": 0.06000691279768944, "beta_low": 0.5975825190544128, "beta_high": 4.00543212890625e-05, "orthogonal_to_active": 0.0646098181605339, "samples": 4, "scored_tokens": 899, "normal_loss": 3.015655994415283, "normal_token_accuracy": 0.4538376033306122, "state_active": 0.0641336515545845, "state_memory": 0.01600758172571659, "state_peak": 127.48783111572266, "growth_active": 1.1223604679107666, "growth_memory": 1.2636488676071167, "output_influence": 0.4531322121620178, "residual_influence": 0.16808852553367615, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.159586191177368, "no_memory_loss_delta": 0.8455187678337097, "scrambled_loss_delta": 0.11708906292915344, "memory_only_loss_delta": 3.634923219680786, "no_evidence_loss_delta": 0.4692333936691284, "no_agreement_loss_delta": 0.0248008631169796, "no_orthogonal_loss_delta": 0.012881528586149216}, {"index": 94, "tokens": 256, "normal_loss": 3.113355875015259, "no_memory_loss_delta": 0.8320097923278809, "scrambled_loss_delta": 0.48315244913101196, "memory_only_loss_delta": 4.324416637420654, "no_evidence_loss_delta": 0.3873569965362549, "no_agreement_loss_delta": 0.035634927451610565, "no_orthogonal_loss_delta": 0.04449174180626869}, {"index": 188, "tokens": 256, "normal_loss": 3.3123388290405273, "no_memory_loss_delta": 1.0581187009811401, "scrambled_loss_delta": 0.09132973849773407, "memory_only_loss_delta": 3.272434711456299, "no_evidence_loss_delta": 0.5773877501487732, "no_agreement_loss_delta": 0.05667370557785034, "no_orthogonal_loss_delta": 0.043060801923274994}, {"index": 283, "tokens": 131, "normal_loss": 1.9636833667755127, "no_memory_loss_delta": 0.425976037979126, "scrambled_loss_delta": 0.11099220812320709, "memory_only_loss_delta": 3.101118803024292, "no_evidence_loss_delta": 0.21594840288162231, "no_agreement_loss_delta": 0.008815423585474491, "no_orthogonal_loss_delta": -0.048810649663209915}], "agreement_weight_abs_mean": 0.2552308142185211, "agreement_weight_abs_max": 6.878977298736572, "agreement_weight_mean": 0.22160518169403076, "orthogonal_weight_abs_mean": 0.9533535838127136, "orthogonal_weight_abs_max": 11.311408996582031, "orthogonal_weight_mean": -0.0371854305267334, "no_memory_loss": 3.8567330837249756, "no_memory_loss_delta": 0.8410773277282715, "no_memory_token_accuracy_gain": 0.11012236028909683, "no_memory_token_win_fraction": 0.7107897400856018, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 56.87543198491611, "scrambled_loss": 3.2287614345550537, "scrambled_loss_delta": 0.2131059318780899, "scrambled_token_accuracy_gain": 0.013348164968192577, "scrambled_token_win_fraction": 0.6129032373428345, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.192946774668954, "memory_only_loss": 6.665912628173828, "memory_only_loss_delta": 3.650256633758545, "memory_only_token_accuracy_gain": 0.2969966530799866, "memory_only_token_win_fraction": 0.9087875485420227, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 97.40155405664858, "no_evidence_loss": 3.4554641246795654, "no_evidence_loss_delta": 0.43980827927589417, "no_evidence_token_accuracy_gain": 0.06896551698446274, "no_evidence_token_win_fraction": 0.6273637413978577, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 35.5840091950731, "no_agreement_loss": 3.050288677215576, "no_agreement_loss_delta": 0.034632761031389236, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.4783092439174652, "no_agreement_probe_win_fraction": 1.0, "no_agreement_ppl_reduction_pct": 3.4039910684384176, "no_orthogonal_loss": 3.037142753601074, "no_orthogonal_loss_delta": 0.021487124264240265, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5928809642791748, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.1257920587590786, "short_no_memory_loss_delta": 0.7845230102539062, "long_context_memory_gain": 0.056554317474365234, "normal_context_loss_gain": 0.13636940717697144, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.7372188568115234, "contribution_to_active": 1.264271855354309, "mix_mean": 0.1097264513373375, "mix_low": 0.5020658373832703, "mix_high": 0.004835606086999178, "read_cosine": 0.4099190831184387, "beta_mean": 0.22034963965415955, "nonfinite": 0.0, "weighted_memory_to_active": 1.2803027629852295, "mix_std": 0.09243906289339066, "beta_std": 0.1513579785823822, "beta_low": 0.1760762631893158, "beta_high": 0.0, "orthogonal_to_active": 0.05096174031496048, "output_influence": 0.21458271145820618, "residual_influence": 0.22705477476119995, "state_active": 0.06909871101379395, "state_memory": 0.045920029282569885, "state_peak": 47.70231628417969, "growth_active": 1.263546347618103, "growth_memory": 1.2582536935806274, "heads": [{"head": 0, "memory_to_active": 2.9977777004241943, "contribution_to_active": 0.4554317593574524, "mix_mean": 0.019645825028419495, "mix_low": 0.6093330383300781, "mix_high": 0.0, "read_cosine": 0.2224099487066269, "beta_mean": 0.2059030830860138, "nonfinite": 0.0, "weighted_memory_to_active": 0.5707579851150513, "mix_std": 0.05807332322001457, "beta_std": 0.10991980135440826, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05787475034594536, "evidence_weights": [-0.0030247638933360577, -0.04007190838456154, -0.05698069557547569], "agreement_weight": -0.23201586306095123, "orthogonal_weight": -0.11830132454633713}, {"head": 1, "memory_to_active": 7.0208306312561035, "contribution_to_active": 3.068488121032715, "mix_mean": 0.01966744102537632, "mix_low": 0.6130990982055664, "mix_high": 0.0, "read_cosine": 0.4417538344860077, "beta_mean": 0.456190288066864, "nonfinite": 0.0, "weighted_memory_to_active": 3.284860610961914, "mix_std": 0.05697404965758324, "beta_std": 0.21966741979122162, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0957522988319397, "evidence_weights": [0.023505043238401413, -0.10055616497993469, 0.6755731105804443], "agreement_weight": 0.028514215722680092, "orthogonal_weight": 0.0658184140920639}, {"head": 2, "memory_to_active": 3.0120902061462402, "contribution_to_active": 0.6688461899757385, "mix_mean": 0.014133675955235958, "mix_low": 0.6618328094482422, "mix_high": 0.0, "read_cosine": 0.2854657769203186, "beta_mean": 0.2651490569114685, "nonfinite": 0.0, "weighted_memory_to_active": 0.7682387828826904, "mix_std": 0.031372953206300735, "beta_std": 0.22626222670078278, "beta_low": 0.00042724609375, "beta_high": 0.0, "orthogonal_to_active": 0.05644918605685234, "evidence_weights": [0.06426331400871277, 0.04872078076004982, 0.2622072994709015], "agreement_weight": 0.010464298538863659, "orthogonal_weight": -0.19696488976478577}, {"head": 3, "memory_to_active": 7.945838928222656, "contribution_to_active": 2.8560779094696045, "mix_mean": 0.028303341940045357, "mix_low": 0.5100679397583008, "mix_high": 0.0, "read_cosine": 0.2771492898464203, "beta_mean": 0.353535532951355, "nonfinite": 0.0, "weighted_memory_to_active": 2.9628968238830566, "mix_std": 0.06655894964933395, "beta_std": 0.28363022208213806, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.07853815704584122, "evidence_weights": [-0.033493828028440475, -0.02494766376912594, -0.1429058164358139], "agreement_weight": -0.005527922883629799, "orthogonal_weight": -0.0307901781052351}, {"head": 4, "memory_to_active": 1.3493802547454834, "contribution_to_active": 0.0634094774723053, "mix_mean": 0.023907845839858055, "mix_low": 0.6044483184814453, "mix_high": 0.0, "read_cosine": 0.2994147837162018, "beta_mean": 0.02385932207107544, "nonfinite": 0.0, "weighted_memory_to_active": 0.043367065489292145, "mix_std": 0.0619509220123291, "beta_std": 0.05119156837463379, "beta_low": 0.6239013671875, "beta_high": 0.0, "orthogonal_to_active": 0.0007918855990283191, "evidence_weights": [0.08726052939891815, 0.05466232821345329, -0.14635640382766724], "agreement_weight": -0.20892906188964844, "orthogonal_weight": 0.01691383868455887}, {"head": 5, "memory_to_active": 0.09739397466182709, "contribution_to_active": 0.4733772873878479, "mix_mean": 0.5527005791664124, "mix_low": 0.013613700866699219, "mix_high": 0.029013633728027344, "read_cosine": 0.9333209991455078, "beta_mean": 0.017460480332374573, "nonfinite": 0.0, "weighted_memory_to_active": 0.05169442296028137, "mix_std": 0.27970418334007263, "beta_std": 0.017476525157690048, "beta_low": 0.43194580078125, "beta_high": 0.0, "orthogonal_to_active": 0.016364147886633873, "evidence_weights": [-0.17300374805927277, -0.21697284281253815, 0.3651866912841797], "agreement_weight": 0.28510400652885437, "orthogonal_weight": -0.7159236073493958}]}, {"layer": 1, "memory_to_active": 2.581237316131592, "contribution_to_active": 0.48212072253227234, "mix_mean": 0.21847528219223022, "mix_low": 0.3753206133842468, "mix_high": 0.0012297630310058594, "read_cosine": 0.3023909032344818, "beta_mean": 0.1777874231338501, "nonfinite": 0.0, "weighted_memory_to_active": 0.3205353319644928, "mix_std": 0.11225198209285736, "beta_std": 0.12955006957054138, "beta_low": 0.3542785942554474, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.039070338010787964, "output_influence": 0.2747410535812378, "residual_influence": 0.07897018641233444, "state_active": 0.027818214148283005, "state_memory": 0.033781517297029495, "state_peak": 44.19798278808594, "growth_active": 1.1472084522247314, "growth_memory": 1.4422202110290527, "heads": [{"head": 0, "memory_to_active": 0.019026968628168106, "contribution_to_active": 0.4941663146018982, "mix_mean": 0.5484299659729004, "mix_low": 0.0036668777465820312, "mix_high": 0.002765655517578125, "read_cosine": 0.021567227318882942, "beta_mean": 0.00011857051140395924, "nonfinite": 0.0, "weighted_memory_to_active": 0.01202782616019249, "mix_std": 0.2801769971847534, "beta_std": 0.00022101156355347484, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011656641960144043, "evidence_weights": [-0.21042433381080627, -0.270498663187027, 0.20727850496768951], "agreement_weight": 0.20986175537109375, "orthogonal_weight": -1.0081725120544434}, {"head": 1, "memory_to_active": 3.137406587600708, "contribution_to_active": 0.3050891160964966, "mix_mean": 0.02033204585313797, "mix_low": 0.5123014450073242, "mix_high": 9.5367431640625e-07, "read_cosine": 0.49776968359947205, "beta_mean": 0.2657608985900879, "nonfinite": 0.0, "weighted_memory_to_active": 0.32690462470054626, "mix_std": 0.04013412073254585, "beta_std": 0.1937602162361145, "beta_low": 0.00189208984375, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.023839641362428665, "evidence_weights": [0.028053877875208855, -0.06107885017991066, 0.04822593927383423], "agreement_weight": -0.0024834363721311092, "orthogonal_weight": 0.21408109366893768}, {"head": 2, "memory_to_active": 0.03168371692299843, "contribution_to_active": 0.5474281907081604, "mix_mean": 0.6818989515304565, "mix_low": 0.00023746490478515625, "mix_high": 0.004611968994140625, "read_cosine": 0.5086398124694824, "beta_mean": 0.0011044754646718502, "nonfinite": 0.0, "weighted_memory_to_active": 0.020595014095306396, "mix_std": 0.22448581457138062, "beta_std": 0.003142180386930704, "beta_low": 0.9814453125, "beta_high": 0.0, "orthogonal_to_active": 0.014629439450800419, "evidence_weights": [-0.27163636684417725, -0.3677956759929657, 0.6335277557373047], "agreement_weight": 0.052438635379076004, "orthogonal_weight": -0.8434364199638367}, {"head": 3, "memory_to_active": 3.1512327194213867, "contribution_to_active": 0.2207811176776886, "mix_mean": 0.02400589920580387, "mix_low": 0.5315265655517578, "mix_high": 0.0, "read_cosine": 0.17626747488975525, "beta_mean": 0.33353808522224426, "nonfinite": 0.0, "weighted_memory_to_active": 0.28477486968040466, "mix_std": 0.061459362506866455, "beta_std": 0.26171889901161194, "beta_low": 0.076171875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.11063341796398163, "evidence_weights": [0.04144757613539696, -0.016058381646871567, -0.011216714978218079], "agreement_weight": -0.027973417192697525, "orthogonal_weight": -0.47613298892974854}, {"head": 4, "memory_to_active": 4.309325218200684, "contribution_to_active": 1.2265937328338623, "mix_mean": 0.030087951570749283, "mix_low": 0.34497928619384766, "mix_high": 0.0, "read_cosine": 0.33772164583206177, "beta_mean": 0.16134825348854065, "nonfinite": 0.0, "weighted_memory_to_active": 1.2101026773452759, "mix_std": 0.0489884614944458, "beta_std": 0.12546765804290771, "beta_low": 0.04119873046875, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.04499766230583191, "evidence_weights": [-0.010986823588609695, -0.0027743971440941095, 0.1727266162633896], "agreement_weight": -0.009086688049137592, "orthogonal_weight": 0.0569612942636013}, {"head": 5, "memory_to_active": 4.838748931884766, "contribution_to_active": 0.0986657589673996, "mix_mean": 0.006096839904785156, "mix_low": 0.8592119216918945, "mix_high": 0.0, "read_cosine": 0.27237945795059204, "beta_mean": 0.304854154586792, "nonfinite": 0.0, "weighted_memory_to_active": 0.06880665570497513, "mix_std": 0.01826714351773262, "beta_std": 0.19299042224884033, "beta_low": 0.02496337890625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.0286652073264122, "evidence_weights": [0.16078370809555054, 0.07708030939102173, 0.09152982383966446], "agreement_weight": -0.11804535984992981, "orthogonal_weight": 0.4200300872325897}]}, {"layer": 2, "memory_to_active": 1.0400090217590332, "contribution_to_active": 0.3612869381904602, "mix_mean": 0.4166925251483917, "mix_low": 0.2397807538509369, "mix_high": 0.004443963523954153, "read_cosine": 0.28748202323913574, "beta_mean": 0.07750073075294495, "nonfinite": 0.0, "weighted_memory_to_active": 0.0730743259191513, "mix_std": 0.18411023914813995, "beta_std": 0.06737418472766876, "beta_low": 0.615753173828125, "beta_high": 0.00020345053053461015, "orthogonal_to_active": 0.015454091131687164, "output_influence": 0.41534432768821716, "residual_influence": 0.12313608080148697, "state_active": 0.04125054180622101, "state_memory": 0.013265886344015598, "state_peak": 29.503355026245117, "growth_active": 1.0623573064804077, "growth_memory": 1.5426595211029053, "heads": [{"head": 0, "memory_to_active": 0.004073010291904211, "contribution_to_active": 0.36061155796051025, "mix_mean": 0.6543126106262207, "mix_low": 0.0005311965942382812, "mix_high": 0.009631156921386719, "read_cosine": 0.13779516518115997, "beta_mean": 3.4278331440873444e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0027404376305639744, "mix_std": 0.26284974813461304, "beta_std": 6.148953252704814e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0028197625651955605, "evidence_weights": [-0.37409621477127075, -0.2147655338048935, 0.20647311210632324], "agreement_weight": 0.13663336634635925, "orthogonal_weight": -1.0609146356582642}, {"head": 1, "memory_to_active": 3.2647390365600586, "contribution_to_active": 0.04564616456627846, "mix_mean": 0.005596593953669071, "mix_low": 0.8716115951538086, "mix_high": 0.0, "read_cosine": 0.11815492808818817, "beta_mean": 0.21595534682273865, "nonfinite": 0.0, "weighted_memory_to_active": 0.041219331324100494, "mix_std": 0.011587582528591156, "beta_std": 0.1484045684337616, "beta_low": 0.00518798828125, "beta_high": 0.0, "orthogonal_to_active": 0.01055622287094593, "evidence_weights": [0.13444510102272034, 0.10541627556085587, 0.572456419467926], "agreement_weight": 0.0227037500590086, "orthogonal_weight": 0.3194611966609955}, {"head": 2, "memory_to_active": 0.009357599541544914, "contribution_to_active": 0.43240290880203247, "mix_mean": 0.5646626353263855, "mix_low": 0.0008268356323242188, "mix_high": 0.003963470458984375, "read_cosine": 0.3955298662185669, "beta_mean": 3.0130571758490987e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038733412511646748, "mix_std": 0.27362069487571716, "beta_std": 6.06381036050152e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00031986687099561095, "evidence_weights": [-0.2495495080947876, -0.2925993800163269, 0.5764438509941101], "agreement_weight": -0.03964933007955551, "orthogonal_weight": 0.09896437078714371}, {"head": 3, "memory_to_active": 0.08930151164531708, "contribution_to_active": 0.5521681904792786, "mix_mean": 0.5901190638542175, "mix_low": 0.0001697540283203125, "mix_high": 0.0060367584228515625, "read_cosine": 0.5111263990402222, "beta_mean": 0.014480924233794212, "nonfinite": 0.0, "weighted_memory_to_active": 0.057022660970687866, "mix_std": 0.252096563577652, "beta_std": 0.022892983630299568, "beta_low": 0.62811279296875, "beta_high": 0.0, "orthogonal_to_active": 0.04305297136306763, "evidence_weights": [-0.1996617317199707, -0.3266066014766693, 0.7263823747634888], "agreement_weight": 0.05511162430047989, "orthogonal_weight": -0.8988599181175232}, {"head": 4, "memory_to_active": 2.86403226852417, "contribution_to_active": 0.24809741973876953, "mix_mean": 0.02070353366434574, "mix_low": 0.5651283264160156, "mix_high": 0.0, "read_cosine": 0.18823787569999695, "beta_mean": 0.23447373509407043, "nonfinite": 0.0, "weighted_memory_to_active": 0.3288784623146057, "mix_std": 0.04899681359529495, "beta_std": 0.23278771340847015, "beta_low": 0.06121826171875, "beta_high": 0.001220703125, "orthogonal_to_active": 0.03469626232981682, "evidence_weights": [0.02319050021469593, 0.00026249815709888935, 0.03794648125767708], "agreement_weight": 0.027638476341962814, "orthogonal_weight": -0.12871232628822327}, {"head": 5, "memory_to_active": 0.00855053961277008, "contribution_to_active": 0.5287953615188599, "mix_mean": 0.6647605895996094, "mix_low": 0.00041675567626953125, "mix_high": 0.0070323944091796875, "read_cosine": 0.37404775619506836, "beta_mean": 2.9913648177171126e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0047116829082369804, "mix_std": 0.25551000237464905, "beta_std": 3.769245813600719e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0012794509530067444, "evidence_weights": [-0.2751576900482178, -0.2858314514160156, 0.8946546912193298], "agreement_weight": 0.28513872623443604, "orthogonal_weight": -0.3198363780975342}]}, {"layer": 3, "memory_to_active": 0.42488789558410645, "contribution_to_active": 0.4949396252632141, "mix_mean": 0.5290470123291016, "mix_low": 0.050858497619628906, "mix_high": 0.011894702911376953, "read_cosine": 0.18095526099205017, "beta_mean": 0.023293614387512207, "nonfinite": 0.0, "weighted_memory_to_active": 0.1153317242860794, "mix_std": 0.25480449199676514, "beta_std": 0.022795364260673523, "beta_low": 0.7721151113510132, "beta_high": 0.0, "orthogonal_to_active": 0.05045049265027046, "output_influence": 0.44805628061294556, "residual_influence": 0.1171804890036583, "state_active": 0.022413119673728943, "state_memory": 0.006178848911076784, "state_peak": 25.063692092895508, "growth_active": 1.0837444067001343, "growth_memory": 1.6129556894302368, "heads": [{"head": 0, "memory_to_active": 0.36402860283851624, "contribution_to_active": 0.622473955154419, "mix_mean": 0.6158736348152161, "mix_low": 0.0011682510375976562, "mix_high": 0.02328968048095703, "read_cosine": 0.21676518023014069, "beta_mean": 0.009424296207726002, "nonfinite": 0.0, "weighted_memory_to_active": 0.22388045489788055, "mix_std": 0.297465443611145, "beta_std": 0.010196949355304241, "beta_low": 0.7030029296875, "beta_high": 0.0, "orthogonal_to_active": 0.16497904062271118, "evidence_weights": [-0.2147742062807083, -0.24875976145267487, 1.0592961311340332], "agreement_weight": 0.11319447308778763, "orthogonal_weight": -0.77223140001297}, {"head": 1, "memory_to_active": 0.006179467774927616, "contribution_to_active": 0.47612664103507996, "mix_mean": 0.6028814315795898, "mix_low": 0.0026750564575195312, "mix_high": 0.012456893920898438, "read_cosine": 0.16911202669143677, "beta_mean": 2.9251848900457844e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003856745082885027, "mix_std": 0.2904072403907776, "beta_std": 7.201054540928453e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00313419196754694, "evidence_weights": [-0.2687681019306183, -0.2167682945728302, 0.6904683709144592], "agreement_weight": 0.6129173040390015, "orthogonal_weight": -0.8843955397605896}, {"head": 2, "memory_to_active": 1.9360146522521973, "contribution_to_active": 0.26413872838020325, "mix_mean": 0.056346211582422256, "mix_low": 0.2928590774536133, "mix_high": 0.0, "read_cosine": 0.20958957076072693, "beta_mean": 0.1258486807346344, "nonfinite": 0.0, "weighted_memory_to_active": 0.28181374073028564, "mix_std": 0.10538986325263977, "beta_std": 0.11548073589801788, "beta_low": 0.02471923828125, "beta_high": 0.0, "orthogonal_to_active": 0.0036474009975790977, "evidence_weights": [-0.09978708624839783, -0.007808718364685774, -0.03065345622599125], "agreement_weight": -0.014076774008572102, "orthogonal_weight": 0.015980834141373634}, {"head": 3, "memory_to_active": 0.01826261542737484, "contribution_to_active": 0.541954755783081, "mix_mean": 0.6136853098869324, "mix_low": 0.0038251876831054688, "mix_high": 0.009593963623046875, "read_cosine": 0.2614717483520508, "beta_mean": 7.940546493045986e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012373496778309345, "mix_std": 0.28353801369667053, "beta_std": 0.00020580276031978428, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0030242830980569124, "evidence_weights": [-0.262206494808197, -0.22042222321033478, 2.347451686859131], "agreement_weight": 2.3648767471313477, "orthogonal_weight": -0.3109484910964966}, {"head": 4, "memory_to_active": 0.01050182618200779, "contribution_to_active": 0.40730008482933044, "mix_mean": 0.5484867691993713, "mix_low": 0.0022382736206054688, "mix_high": 0.003223419189453125, "read_cosine": 0.14195296168327332, "beta_mean": 5.391804006649181e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007119667250663042, "mix_std": 0.28721004724502563, "beta_std": 0.00010262380237691104, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006750193890184164, "evidence_weights": [-0.24588796496391296, -0.22391152381896973, 0.06931371241807938], "agreement_weight": -0.05285133421421051, "orthogonal_weight": -1.0209040641784668}, {"head": 5, "memory_to_active": 0.21434026956558228, "contribution_to_active": 0.6576433181762695, "mix_mean": 0.737008810043335, "mix_low": 0.0023851394653320312, "mix_high": 0.02280426025390625, "read_cosine": 0.08684004098176956, "beta_mean": 0.00432613305747509, "nonfinite": 0.0, "weighted_memory_to_active": 0.16294622421264648, "mix_std": 0.2648162841796875, "beta_std": 0.010714058764278889, "beta_low": 0.905029296875, "beta_high": 0.0, "orthogonal_to_active": 0.12116780877113342, "evidence_weights": [-0.26022833585739136, -0.34365034103393555, 0.5502712726593018], "agreement_weight": 0.24718144536018372, "orthogonal_weight": -0.7699138522148132}]}, {"layer": 4, "memory_to_active": 1.0267577171325684, "contribution_to_active": 0.48124074935913086, "mix_mean": 0.37734389305114746, "mix_low": 0.22459349036216736, "mix_high": 0.012291908264160156, "read_cosine": 0.295835942029953, "beta_mean": 0.10532136261463165, "nonfinite": 0.0, "weighted_memory_to_active": 0.1287262737751007, "mix_std": 0.16216695308685303, "beta_std": 0.09617146849632263, "beta_low": 0.4031982421875, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05749493092298508, "output_influence": 0.5073983669281006, "residual_influence": 0.16700831055641174, "state_active": 0.03843609243631363, "state_memory": 0.015522263944149017, "state_peak": 41.56439971923828, "growth_active": 1.2065460681915283, "growth_memory": 1.5028278827667236, "heads": [{"head": 0, "memory_to_active": 0.13627177476882935, "contribution_to_active": 0.8164118528366089, "mix_mean": 0.6957904100418091, "mix_low": 0.009660720825195312, "mix_high": 0.033934593200683594, "read_cosine": 0.44768625497817993, "beta_mean": 0.23844337463378906, "nonfinite": 0.0, "weighted_memory_to_active": 0.11682891845703125, "mix_std": 0.34480372071266174, "beta_std": 0.23274509608745575, "beta_low": 0.2496337890625, "beta_high": 0.0, "orthogonal_to_active": 0.08741440623998642, "evidence_weights": [-0.24282778799533844, -0.355849951505661, -0.22992849349975586], "agreement_weight": 0.0022046752274036407, "orthogonal_weight": 0.8878567814826965}, {"head": 1, "memory_to_active": 0.03900046646595001, "contribution_to_active": 0.6605220437049866, "mix_mean": 0.7032819986343384, "mix_low": 7.152557373046875e-05, "mix_high": 0.01975250244140625, "read_cosine": 0.5555632710456848, "beta_mean": 0.0007888404652476311, "nonfinite": 0.0, "weighted_memory_to_active": 0.029219770804047585, "mix_std": 0.23746570944786072, "beta_std": 0.006401053164154291, "beta_low": 0.9825439453125, "beta_high": 0.0, "orthogonal_to_active": 0.014464059844613075, "evidence_weights": [-0.23263411223888397, -0.3625953793525696, 0.5590317845344543], "agreement_weight": 0.37634986639022827, "orthogonal_weight": -0.6814653873443604}, {"head": 2, "memory_to_active": 1.4217689037322998, "contribution_to_active": 0.3551095426082611, "mix_mean": 0.060240112245082855, "mix_low": 0.29825687408447266, "mix_high": 5.626678466796875e-05, "read_cosine": 0.20552030205726624, "beta_mean": 0.04838139936327934, "nonfinite": 0.0, "weighted_memory_to_active": 0.26388370990753174, "mix_std": 0.10562769323587418, "beta_std": 0.0573190338909626, "beta_low": 0.16558837890625, "beta_high": 0.0, "orthogonal_to_active": 0.10488785803318024, "evidence_weights": [0.02340051531791687, -0.06890415400266647, 0.3812916576862335], "agreement_weight": 0.11885170638561249, "orthogonal_weight": -0.5043317079544067}, {"head": 3, "memory_to_active": 1.9971364736557007, "contribution_to_active": 0.1431008279323578, "mix_mean": 0.03164718300104141, "mix_low": 0.3991889953613281, "mix_high": 0.0, "read_cosine": 0.21051445603370667, "beta_mean": 0.14062052965164185, "nonfinite": 0.0, "weighted_memory_to_active": 0.19625142216682434, "mix_std": 0.051903992891311646, "beta_std": 0.1225942075252533, "beta_low": 0.02142333984375, "beta_high": 0.0, "orthogonal_to_active": 0.0029085250571370125, "evidence_weights": [-0.024150455370545387, 0.047324419021606445, -0.2504095733165741], "agreement_weight": 0.0785982757806778, "orthogonal_weight": 0.022123178467154503}, {"head": 4, "memory_to_active": 0.006629847921431065, "contribution_to_active": 0.6427298188209534, "mix_mean": 0.7594312429428101, "mix_low": 2.288818359375e-05, "mix_high": 0.020000457763671875, "read_cosine": 0.12256575375795364, "beta_mean": 1.3616700925922487e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005002101417630911, "mix_std": 0.20545926690101624, "beta_std": 4.534308391157538e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003214519936591387, "evidence_weights": [-0.3266630172729492, -0.3146677017211914, 0.239652618765831], "agreement_weight": 0.3215814232826233, "orthogonal_weight": -0.6765575408935547}, {"head": 5, "memory_to_active": 2.5597386360168457, "contribution_to_active": 0.26957032084465027, "mix_mean": 0.013672337867319584, "mix_low": 0.6403598785400391, "mix_high": 7.62939453125e-06, "read_cosine": 0.23316551744937897, "beta_mean": 0.2036803662776947, "nonfinite": 0.0, "weighted_memory_to_active": 0.16117170453071594, "mix_std": 0.027741344645619392, "beta_std": 0.15792404115200043, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.13208019733428955, "evidence_weights": [0.056518178433179855, -0.017559271305799484, -0.2588532269001007], "agreement_weight": -0.09332006424665451, "orthogonal_weight": 1.1238151788711548}]}, {"layer": 5, "memory_to_active": 0.3521321415901184, "contribution_to_active": 0.5442932844161987, "mix_mean": 0.5243850946426392, "mix_low": 0.055741310119628906, "mix_high": 0.0212961845099926, "read_cosine": 0.08162008970975876, "beta_mean": 0.049866318702697754, "nonfinite": 0.0, "weighted_memory_to_active": 0.12124890834093094, "mix_std": 0.2492465376853943, "beta_std": 0.06335916370153427, "beta_low": 0.6950480341911316, "beta_high": 0.0, "orthogonal_to_active": 0.052059005945920944, "output_influence": 0.5944666862487793, "residual_influence": 0.20387119054794312, "state_active": 0.03900854289531708, "state_memory": 0.009394127875566483, "state_peak": 64.83404541015625, "growth_active": 1.1096696853637695, "growth_memory": 1.2583791017532349, "heads": [{"head": 0, "memory_to_active": 0.15098202228546143, "contribution_to_active": 0.5880318880081177, "mix_mean": 0.5128560066223145, "mix_low": 0.006079673767089844, "mix_high": 0.0023097991943359375, "read_cosine": -0.1584431529045105, "beta_mean": 0.01187948975712061, "nonfinite": 0.0, "weighted_memory_to_active": 0.05576899275183678, "mix_std": 0.27651727199554443, "beta_std": 0.021578475832939148, "beta_low": 0.656005859375, "beta_high": 0.0, "orthogonal_to_active": 0.025050431489944458, "evidence_weights": [-0.17003488540649414, -0.188945934176445, 0.12261360883712769], "agreement_weight": -0.024290237575769424, "orthogonal_weight": -0.4778847396373749}, {"head": 1, "memory_to_active": 0.0040462426841259, "contribution_to_active": 0.5667769908905029, "mix_mean": 0.7174540162086487, "mix_low": 0.0005807876586914062, "mix_high": 0.039206504821777344, "read_cosine": 0.23892167210578918, "beta_mean": 8.868515578797087e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002576324390247464, "mix_std": 0.26262903213500977, "beta_std": 2.1101574020576663e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002388502238318324, "evidence_weights": [-0.24547581374645233, -0.28472965955734253, 0.948794960975647], "agreement_weight": 0.7381411790847778, "orthogonal_weight": -0.9765679836273193}, {"head": 2, "memory_to_active": 0.014635898172855377, "contribution_to_active": 0.6144189834594727, "mix_mean": 0.6668895483016968, "mix_low": 0.0021190643310546875, "mix_high": 0.022800445556640625, "read_cosine": -0.4462552070617676, "beta_mean": 8.057821105467156e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009448354132473469, "mix_std": 0.2595796585083008, "beta_std": 0.0003263085091020912, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.009126709774136543, "evidence_weights": [-0.2549097537994385, -0.30961427092552185, 0.18043068051338196], "agreement_weight": -0.20422935485839844, "orthogonal_weight": -1.0603207349777222}, {"head": 3, "memory_to_active": 1.466484785079956, "contribution_to_active": 0.23441456258296967, "mix_mean": 0.11632251739501953, "mix_low": 0.1135110855102539, "mix_high": 0.0, "read_cosine": 0.14264488220214844, "beta_mean": 0.09309031069278717, "nonfinite": 0.0, "weighted_memory_to_active": 0.3524613678455353, "mix_std": 0.14542213082313538, "beta_std": 0.14246144890785217, "beta_low": 0.2125244140625, "beta_high": 0.0, "orthogonal_to_active": 0.1410280019044876, "evidence_weights": [-0.07750716060400009, -0.09917011111974716, 0.12224754691123962], "agreement_weight": 0.04693064093589783, "orthogonal_weight": -0.49670401215553284}, {"head": 4, "memory_to_active": 0.41401413083076477, "contribution_to_active": 0.5700271725654602, "mix_mean": 0.3172355592250824, "mix_low": 0.21055030822753906, "mix_high": 0.003627777099609375, "read_cosine": 0.547942042350769, "beta_mean": 0.19397643208503723, "nonfinite": 0.0, "weighted_memory_to_active": 0.26413336396217346, "mix_std": 0.3351171016693115, "beta_std": 0.21504075825214386, "beta_low": 0.3028564453125, "beta_high": 0.0, "orthogonal_to_active": 0.11007372289896011, "evidence_weights": [-0.11092134565114975, -0.2503758370876312, -0.06693033128976822], "agreement_weight": 0.028034988790750504, "orthogonal_weight": 0.6628946661949158}, {"head": 5, "memory_to_active": 0.06262964755296707, "contribution_to_active": 0.6920901536941528, "mix_mean": 0.815552830696106, "mix_low": 0.0016069412231445312, "mix_high": 0.05983257293701172, "read_cosine": 0.16491027176380157, "beta_mean": 0.0001622306153876707, "nonfinite": 0.0, "weighted_memory_to_active": 0.04310503974556923, "mix_std": 0.21621398627758026, "beta_std": 0.0007268757908605039, "beta_low": 0.9990234375, "beta_high": 0.0, "orthogonal_to_active": 0.024686655029654503, "evidence_weights": [-0.30218109488487244, -0.32577037811279297, 1.8149415254592896], "agreement_weight": 0.5502824783325195, "orthogonal_weight": -0.6041487455368042}]}, {"layer": 6, "memory_to_active": 0.2309347689151764, "contribution_to_active": 0.6557732820510864, "mix_mean": 0.6035789847373962, "mix_low": 0.02995665930211544, "mix_high": 0.03093608282506466, "read_cosine": 0.31861233711242676, "beta_mean": 0.06008334457874298, "nonfinite": 0.0, "weighted_memory_to_active": 0.12291532754898071, "mix_std": 0.23911091685295105, "beta_std": 0.0718807727098465, "beta_low": 0.621917724609375, "beta_high": 0.0, "orthogonal_to_active": 0.0907854437828064, "output_influence": 0.6040054559707642, "residual_influence": 0.22766512632369995, "state_active": 0.041443467140197754, "state_memory": 0.005393126979470253, "state_peak": 89.391357421875, "growth_active": 1.1833316087722778, "growth_memory": 1.1495893001556396, "heads": [{"head": 0, "memory_to_active": 0.10933437198400497, "contribution_to_active": 0.7912337183952332, "mix_mean": 0.833783745765686, "mix_low": 0.00013637542724609375, "mix_high": 0.07161235809326172, "read_cosine": 0.32597965002059937, "beta_mean": 0.010128265246748924, "nonfinite": 0.0, "weighted_memory_to_active": 0.09845008701086044, "mix_std": 0.1945488154888153, "beta_std": 0.030828088521957397, "beta_low": 0.85028076171875, "beta_high": 0.0, "orthogonal_to_active": 0.06290017068386078, "evidence_weights": [-0.2933729887008667, -0.3427034318447113, 1.1720311641693115], "agreement_weight": 0.9440776705741882, "orthogonal_weight": -0.6978133916854858}, {"head": 1, "memory_to_active": 0.0030215890146791935, "contribution_to_active": 0.531355082988739, "mix_mean": 0.6082247495651245, "mix_low": 0.00043392181396484375, "mix_high": 0.018400192260742188, "read_cosine": 0.48661357164382935, "beta_mean": 6.447597115766257e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014192442176863551, "mix_std": 0.28045454621315, "beta_std": 1.3354216207517311e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007528322748839855, "evidence_weights": [-0.18772359192371368, -0.2991369962692261, 1.097396969795227], "agreement_weight": 0.13613109290599823, "orthogonal_weight": -0.6935583353042603}, {"head": 2, "memory_to_active": 0.6119208335876465, "contribution_to_active": 0.11134203523397446, "mix_mean": 0.06405362486839294, "mix_low": 0.1732196807861328, "mix_high": 3.814697265625e-06, "read_cosine": 0.2745712399482727, "beta_mean": 0.020241474732756615, "nonfinite": 0.0, "weighted_memory_to_active": 0.07094283401966095, "mix_std": 0.09061847627162933, "beta_std": 0.025737445801496506, "beta_low": 0.34820556640625, "beta_high": 0.0, "orthogonal_to_active": 0.014720429666340351, "evidence_weights": [0.05226133391261101, -0.07667712867259979, 0.3899660110473633], "agreement_weight": 0.13193778693675995, "orthogonal_weight": -0.24188098311424255}, {"head": 3, "memory_to_active": 0.13667763769626617, "contribution_to_active": 0.6980643272399902, "mix_mean": 0.6740080118179321, "mix_low": 0.004802703857421875, "mix_high": 0.021997451782226562, "read_cosine": 0.45079168677330017, "beta_mean": 0.20837363600730896, "nonfinite": 0.0, "weighted_memory_to_active": 0.09900463372468948, "mix_std": 0.3226313591003418, "beta_std": 0.23932859301567078, "beta_low": 0.2088623046875, "beta_high": 0.0, "orthogonal_to_active": 0.06296445429325104, "evidence_weights": [-0.16232620179653168, -0.3082839250564575, 0.10000568628311157], "agreement_weight": -0.00568099832162261, "orthogonal_weight": 0.7193763852119446}, {"head": 4, "memory_to_active": 0.4998404383659363, "contribution_to_active": 1.1527087688446045, "mix_mean": 0.7353625893592834, "mix_low": 0.001003265380859375, "mix_high": 0.06398677825927734, "read_cosine": 0.09968548268079758, "beta_mean": 0.12163476645946503, "nonfinite": 0.0, "weighted_memory_to_active": 0.4492007791996002, "mix_std": 0.3100321888923645, "beta_std": 0.13506962358951569, "beta_low": 0.32421875, "beta_high": 0.0, "orthogonal_to_active": 0.385517954826355, "evidence_weights": [-0.15516887605190277, -0.4993380010128021, -0.8033536672592163], "agreement_weight": -0.014435958117246628, "orthogonal_weight": 0.8829192519187927}, {"head": 5, "memory_to_active": 0.024813659489154816, "contribution_to_active": 0.6499354839324951, "mix_mean": 0.7060410976409912, "mix_low": 0.00014400482177734375, "mix_high": 0.009615898132324219, "read_cosine": 0.2740324139595032, "beta_mean": 0.00011546110908966511, "nonfinite": 0.0, "weighted_memory_to_active": 0.018474381417036057, "mix_std": 0.23638004064559937, "beta_std": 0.0003075309796258807, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.017856793478131294, "evidence_weights": [-0.2312450110912323, -0.31434527039527893, 0.35826098918914795], "agreement_weight": 0.23901322484016418, "orthogonal_weight": -1.0349997282028198}]}, {"layer": 7, "memory_to_active": 1.7740353345870972, "contribution_to_active": 0.2796132266521454, "mix_mean": 0.18348507583141327, "mix_low": 0.2122534215450287, "mix_high": 0.0010970433941110969, "read_cosine": 0.2514222264289856, "beta_mean": 0.13805601000785828, "nonfinite": 0.0, "weighted_memory_to_active": 0.1983688473701477, "mix_std": 0.1489029973745346, "beta_std": 0.09639979898929596, "beta_low": 0.237884521484375, "beta_high": 0.00029500326490961015, "orthogonal_to_active": 0.0861869752407074, "output_influence": 0.29647278785705566, "residual_influence": 0.07686823606491089, "state_active": 0.014492679387331009, "state_memory": 0.023367343470454216, "state_peak": 35.54972457885742, "growth_active": 1.1615519523620605, "growth_memory": 1.2431745529174805, "heads": [{"head": 0, "memory_to_active": 2.81441068649292, "contribution_to_active": 0.16664159297943115, "mix_mean": 0.03814411535859108, "mix_low": 0.4144563674926758, "mix_high": 0.0, "read_cosine": 0.3497070372104645, "beta_mean": 0.1888902336359024, "nonfinite": 0.0, "weighted_memory_to_active": 0.19531720876693726, "mix_std": 0.075709268450737, "beta_std": 0.13443179428577423, "beta_low": 0.0008544921875, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.00017619381833355874, "evidence_weights": [0.045112818479537964, -0.09156755357980728, 0.026802530512213707], "agreement_weight": -0.02811546064913273, "orthogonal_weight": -0.00101173366419971}, {"head": 1, "memory_to_active": 0.9520712494850159, "contribution_to_active": 0.3985055387020111, "mix_mean": 0.2274097502231598, "mix_low": 0.10859012603759766, "mix_high": 0.005614280700683594, "read_cosine": 0.3991910517215729, "beta_mean": 0.09031683206558228, "nonfinite": 0.0, "weighted_memory_to_active": 0.3377615809440613, "mix_std": 0.24686934053897858, "beta_std": 0.06207982823252678, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04646217077970505, "evidence_weights": [-0.2172260880470276, -0.10482852905988693, -0.301553875207901], "agreement_weight": 0.05773421749472618, "orthogonal_weight": 0.1662013828754425}, {"head": 2, "memory_to_active": 0.5339271426200867, "contribution_to_active": 0.4101473093032837, "mix_mean": 0.2536143362522125, "mix_low": 0.012022972106933594, "mix_high": 0.00042724609375, "read_cosine": 0.12204541265964508, "beta_mean": 0.02772543951869011, "nonfinite": 0.0, "weighted_memory_to_active": 0.22178655862808228, "mix_std": 0.22018849849700928, "beta_std": 0.034981731325387955, "beta_low": 0.26153564453125, "beta_high": 0.0, "orthogonal_to_active": 0.13314634561538696, "evidence_weights": [-0.10109260678291321, -0.18924248218536377, 0.8826566338539124], "agreement_weight": 0.014011421240866184, "orthogonal_weight": -0.6197664141654968}, {"head": 3, "memory_to_active": 4.809700012207031, "contribution_to_active": 0.09965721517801285, "mix_mean": 0.03721703961491585, "mix_low": 0.2904396057128906, "mix_high": 0.0, "read_cosine": 0.1044788584113121, "beta_mean": 0.4647953510284424, "nonfinite": 0.0, "weighted_memory_to_active": 0.35061711072921753, "mix_std": 0.051057759672403336, "beta_std": 0.28026553988456726, "beta_low": 0.000732421875, "beta_high": 0.001220703125, "orthogonal_to_active": 0.3045535087585449, "evidence_weights": [0.07183250039815903, -0.04763948544859886, 0.7100063562393188], "agreement_weight": 0.04990360885858536, "orthogonal_weight": -0.9413169622421265}, {"head": 4, "memory_to_active": 1.4995372295379639, "contribution_to_active": 0.11093839257955551, "mix_mean": 0.027853883802890778, "mix_low": 0.44667625427246094, "mix_high": 0.0, "read_cosine": 0.2827402949333191, "beta_mean": 0.056475166231393814, "nonfinite": 0.0, "weighted_memory_to_active": 0.06680455058813095, "mix_std": 0.05087697133421898, "beta_std": 0.06625740975141525, "beta_low": 0.164306640625, "beta_high": 0.0, "orthogonal_to_active": 0.01598072610795498, "evidence_weights": [0.1090201884508133, -0.03574720397591591, 0.1961410790681839], "agreement_weight": 0.03322663530707359, "orthogonal_weight": -0.2850320041179657}, {"head": 5, "memory_to_active": 0.034565187990665436, "contribution_to_active": 0.4917892813682556, "mix_mean": 0.5166712999343872, "mix_low": 0.00133514404296875, "mix_high": 0.0005407333374023438, "read_cosine": 0.2503707706928253, "beta_mean": 0.00013294548261910677, "nonfinite": 0.0, "weighted_memory_to_active": 0.01792607456445694, "mix_std": 0.24871611595153809, "beta_std": 0.0003824664745479822, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.016802875325083733, "evidence_weights": [-0.18738429248332977, -0.24326235055923462, 0.4816286563873291], "agreement_weight": 0.2553558051586151, "orthogonal_weight": -1.00059175491333}]}, {"layer": 8, "memory_to_active": 0.14321684837341309, "contribution_to_active": 0.5770739912986755, "mix_mean": 0.5144495964050293, "mix_low": 0.009211381897330284, "mix_high": 0.001991113182157278, "read_cosine": 0.3917713463306427, "beta_mean": 0.012876500375568867, "nonfinite": 0.0, "weighted_memory_to_active": 0.07184197753667831, "mix_std": 0.241326242685318, "beta_std": 0.0198358204215765, "beta_low": 0.78387451171875, "beta_high": 0.0, "orthogonal_to_active": 0.06464175134897232, "output_influence": 0.5341663956642151, "residual_influence": 0.20735830068588257, "state_active": 0.04127414897084236, "state_memory": 0.004551961086690426, "state_peak": 71.74882507324219, "growth_active": 1.102390170097351, "growth_memory": 1.0294007062911987, "heads": [{"head": 0, "memory_to_active": 0.06012195348739624, "contribution_to_active": 0.5782231092453003, "mix_mean": 0.6135134100914001, "mix_low": 0.00039577484130859375, "mix_high": 0.001499176025390625, "read_cosine": 0.4547126293182373, "beta_mean": 0.0037918956950306892, "nonfinite": 0.0, "weighted_memory_to_active": 0.04372234269976616, "mix_std": 0.24740999937057495, "beta_std": 0.016215022653341293, "beta_low": 0.9268798828125, "beta_high": 0.0, "orthogonal_to_active": 0.02727777324616909, "evidence_weights": [-0.15087120234966278, -0.37296268343925476, 0.5576136708259583], "agreement_weight": 0.2771242558956146, "orthogonal_weight": -0.7950679063796997}, {"head": 1, "memory_to_active": 0.06427834928035736, "contribution_to_active": 0.5532530546188354, "mix_mean": 0.5324246883392334, "mix_low": 0.0001220703125, "mix_high": 0.001605987548828125, "read_cosine": 0.09203067421913147, "beta_mean": 0.0008812973974272609, "nonfinite": 0.0, "weighted_memory_to_active": 0.028618983924388885, "mix_std": 0.22990703582763672, "beta_std": 0.008169446140527725, "beta_low": 0.99237060546875, "beta_high": 0.0, "orthogonal_to_active": 0.021209562197327614, "evidence_weights": [-0.1910167634487152, -0.2706899642944336, 0.5781629085540771], "agreement_weight": 0.07115975767374039, "orthogonal_weight": -0.7758967876434326}, {"head": 2, "memory_to_active": 0.011000220663845539, "contribution_to_active": 0.7023504972457886, "mix_mean": 0.6953207850456238, "mix_low": 0.00020599365234375, "mix_high": 0.004094123840332031, "read_cosine": 0.4160904586315155, "beta_mean": 0.00011468434968264773, "nonfinite": 0.0, "weighted_memory_to_active": 0.008811194449663162, "mix_std": 0.21749918162822723, "beta_std": 0.003427827963605523, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.005073372274637222, "evidence_weights": [-0.21779751777648926, -0.34353670477867126, 0.9761016964912415], "agreement_weight": 0.35432741045951843, "orthogonal_weight": -0.6624237895011902}, {"head": 3, "memory_to_active": 0.1062786802649498, "contribution_to_active": 0.6379610896110535, "mix_mean": 0.5912575721740723, "mix_low": 0.0006017684936523438, "mix_high": 0.004474639892578125, "read_cosine": 0.7989446520805359, "beta_mean": 0.007621461525559425, "nonfinite": 0.0, "weighted_memory_to_active": 0.07995827496051788, "mix_std": 0.25381389260292053, "beta_std": 0.0073573472909629345, "beta_low": 0.7406005859375, "beta_high": 0.0, "orthogonal_to_active": 0.02400059811770916, "evidence_weights": [-0.16569554805755615, -0.2944759428501129, 0.5201321840286255], "agreement_weight": 0.2648295760154724, "orthogonal_weight": -0.5831384062767029}, {"head": 4, "memory_to_active": 0.2790137529373169, "contribution_to_active": 0.3652108907699585, "mix_mean": 0.3764250874519348, "mix_low": 0.0012235641479492188, "mix_high": 6.67572021484375e-06, "read_cosine": 0.09267138689756393, "beta_mean": 0.0007513486780226231, "nonfinite": 0.0, "weighted_memory_to_active": 0.09335272014141083, "mix_std": 0.2330368608236313, "beta_std": 0.014431898482143879, "beta_low": 0.991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.01742066815495491, "evidence_weights": [-0.14749331772327423, -0.20446142554283142, 0.6668802499771118], "agreement_weight": 0.027530979365110397, "orthogonal_weight": -0.18772241473197937}, {"head": 5, "memory_to_active": 0.33860814571380615, "contribution_to_active": 0.6254450082778931, "mix_mean": 0.2777557373046875, "mix_low": 0.0527191162109375, "mix_high": 0.00026607513427734375, "read_cosine": 0.4961782395839691, "beta_mean": 0.0640983134508133, "nonfinite": 0.0, "weighted_memory_to_active": 0.17658838629722595, "mix_std": 0.2662903666496277, "beta_std": 0.06941337138414383, "beta_low": 0.052001953125, "beta_high": 0.0, "orthogonal_to_active": 0.29286855459213257, "evidence_weights": [-0.1516280323266983, -0.14058905839920044, -0.1115468442440033], "agreement_weight": -0.010257801041007042, "orthogonal_weight": 1.9674615859985352}]}, {"layer": 9, "memory_to_active": 0.809990406036377, "contribution_to_active": 0.7261843085289001, "mix_mean": 0.40671172738075256, "mix_low": 0.04655647650361061, "mix_high": 0.006699562072753906, "read_cosine": 0.34134936332702637, "beta_mean": 0.03185347467660904, "nonfinite": 0.0, "weighted_memory_to_active": 0.3685198724269867, "mix_std": 0.24887356162071228, "beta_std": 0.0267330352216959, "beta_low": 0.6628316640853882, "beta_high": 0.0, "orthogonal_to_active": 0.02999897487461567, "output_influence": 0.568696141242981, "residual_influence": 0.18156689405441284, "state_active": 0.044817473739385605, "state_memory": 0.013380391523241997, "state_peak": 58.7337532043457, "growth_active": 1.1834932565689087, "growth_memory": 1.2393078804016113, "heads": [{"head": 0, "memory_to_active": 0.0002550588105805218, "contribution_to_active": 0.5909941792488098, "mix_mean": 0.6677002310752869, "mix_low": 0.0009975433349609375, "mix_high": 0.027469635009765625, "read_cosine": 0.33718276023864746, "beta_mean": 6.090739930186828e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.00017177319386973977, "mix_std": 0.28112107515335083, "beta_std": 1.7382234318574774e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00016852306725922972, "evidence_weights": [-0.3035944402217865, -0.30867093801498413, 0.32466816902160645], "agreement_weight": 0.10486698150634766, "orthogonal_weight": -1.0365012884140015}, {"head": 1, "memory_to_active": 0.014356948435306549, "contribution_to_active": 0.7242234945297241, "mix_mean": 0.582491397857666, "mix_low": 0.007008552551269531, "mix_high": 0.0031585693359375, "read_cosine": 0.4339306056499481, "beta_mean": 3.136432496830821e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010925847105681896, "mix_std": 0.27671313285827637, "beta_std": 7.509141141781583e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0068924748338758945, "evidence_weights": [-0.2334824949502945, -0.2895857095718384, 0.9395545721054077], "agreement_weight": 0.5755427479743958, "orthogonal_weight": -0.774205207824707}, {"head": 2, "memory_to_active": 0.15194401144981384, "contribution_to_active": 0.6635330319404602, "mix_mean": 0.6036576628684998, "mix_low": 0.0014429092407226562, "mix_high": 0.007422447204589844, "read_cosine": 0.37437719106674194, "beta_mean": 0.0005236915312707424, "nonfinite": 0.0, "weighted_memory_to_active": 0.10327505320310593, "mix_std": 0.2635737359523773, "beta_std": 0.004355866927653551, "beta_low": 0.9970703125, "beta_high": 0.0, "orthogonal_to_active": 0.05844492092728615, "evidence_weights": [-0.1981554925441742, -0.2739906907081604, 0.7503747344017029], "agreement_weight": 0.3681563138961792, "orthogonal_weight": -0.6658800840377808}, {"head": 3, "memory_to_active": 0.6845594644546509, "contribution_to_active": 0.29457059502601624, "mix_mean": 0.1981962025165558, "mix_low": 0.06566047668457031, "mix_high": 6.771087646484375e-05, "read_cosine": 0.38086655735969543, "beta_mean": 0.024865254759788513, "nonfinite": 0.0, "weighted_memory_to_active": 0.20073950290679932, "mix_std": 0.2188451886177063, "beta_std": 0.026034565642476082, "beta_low": 0.2799072265625, "beta_high": 0.0, "orthogonal_to_active": 0.09746256470680237, "evidence_weights": [-0.050975698977708817, -0.12308979034423828, 0.5380715131759644], "agreement_weight": 0.09560370445251465, "orthogonal_weight": -0.5459278225898743}, {"head": 4, "memory_to_active": 3.009852409362793, "contribution_to_active": 1.7307876348495483, "mix_mean": 0.15119734406471252, "mix_low": 0.1176443099975586, "mix_high": 0.00130462646484375, "read_cosine": 0.09830009937286377, "beta_mean": 0.14720210433006287, "nonfinite": 0.0, "weighted_memory_to_active": 1.6188334226608276, "mix_std": 0.2110971361398697, "beta_std": 0.08150957524776459, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.00015624046500306576, "evidence_weights": [-0.07900245487689972, -0.14806850254535675, -0.04082460328936577], "agreement_weight": -0.06746353954076767, "orthogonal_weight": -9.969143866328523e-05}, {"head": 5, "memory_to_active": 0.998974621295929, "contribution_to_active": 0.3529968857765198, "mix_mean": 0.23702746629714966, "mix_low": 0.08658504486083984, "mix_high": 0.000774383544921875, "read_cosine": 0.42343878746032715, "beta_mean": 0.018497804179787636, "nonfinite": 0.0, "weighted_memory_to_active": 0.2771736979484558, "mix_std": 0.24189110100269318, "beta_std": 0.04842135310173035, "beta_low": 0.70001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.016869118437170982, "evidence_weights": [0.029665032401680946, -0.20102213323116302, 0.14691293239593506], "agreement_weight": 0.00667486572638154, "orthogonal_weight": 0.0688486099243164}]}, {"layer": 10, "memory_to_active": 0.2043886035680771, "contribution_to_active": 0.755857527256012, "mix_mean": 0.5321237444877625, "mix_low": 0.022908847779035568, "mix_high": 0.006040891632437706, "read_cosine": 0.3831224739551544, "beta_mean": 0.042929716408252716, "nonfinite": 0.0, "weighted_memory_to_active": 0.08703956753015518, "mix_std": 0.2648797333240509, "beta_std": 0.044428788125514984, "beta_low": 0.6659647822380066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.28885743021965027, "output_influence": 0.4838477075099945, "residual_influence": 0.1799551248550415, "state_active": 0.05322934687137604, "state_memory": 0.013355479575693607, "state_peak": 127.48783111572266, "growth_active": 1.0160781145095825, "growth_memory": 1.0120105743408203, "heads": [{"head": 0, "memory_to_active": 0.7420890927314758, "contribution_to_active": 2.195293664932251, "mix_mean": 0.5757607221603394, "mix_low": 0.11406707763671875, "mix_high": 0.009554862976074219, "read_cosine": 0.08012550324201584, "beta_mean": 0.1883000284433365, "nonfinite": 0.0, "weighted_memory_to_active": 0.41973406076431274, "mix_std": 0.3995473384857178, "beta_std": 0.1976020187139511, "beta_low": 0.158203125, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.658207654953003, "evidence_weights": [-0.07981602102518082, -0.35822102427482605, -1.9898490905761719], "agreement_weight": 0.04125673323869705, "orthogonal_weight": 3.978564977645874}, {"head": 1, "memory_to_active": 0.0055207605473697186, "contribution_to_active": 0.6359764337539673, "mix_mean": 0.6437444686889648, "mix_low": 0.0009431838989257812, "mix_high": 0.0186767578125, "read_cosine": 0.34214991331100464, "beta_mean": 3.7723661080235615e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0032934672199189663, "mix_std": 0.2601649761199951, "beta_std": 6.664935790468007e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005042091943323612, "evidence_weights": [-0.24824459850788116, -0.19130127131938934, 0.5487029552459717], "agreement_weight": 0.6858129501342773, "orthogonal_weight": -1.6723980903625488}, {"head": 2, "memory_to_active": 0.008458788506686687, "contribution_to_active": 0.619702160358429, "mix_mean": 0.674083948135376, "mix_low": 0.00013446807861328125, "mix_high": 0.006501197814941406, "read_cosine": 0.041472844779491425, "beta_mean": 1.7790058336686343e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003841614117845893, "mix_std": 0.24634580314159393, "beta_std": 5.915920701227151e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0015614409931004047, "evidence_weights": [-0.2668004035949707, -0.2741706073284149, 0.7241878509521484], "agreement_weight": -0.22481279075145721, "orthogonal_weight": -0.39024269580841064}, {"head": 3, "memory_to_active": 0.34745609760284424, "contribution_to_active": 0.096387580037117, "mix_mean": 0.2802961468696594, "mix_low": 0.020549774169921875, "mix_high": 5.817413330078125e-05, "read_cosine": 0.9331696033477783, "beta_mean": 0.06273771822452545, "nonfinite": 0.0, "weighted_memory_to_active": 0.03787820041179657, "mix_std": 0.21905773878097534, "beta_std": 0.06009533256292343, "beta_low": 0.00799560546875, "beta_high": 0.0, "orthogonal_to_active": 0.03794591873884201, "evidence_weights": [-0.08492573350667953, -0.14011149108409882, 0.15021325647830963], "agreement_weight": 0.017687644809484482, "orthogonal_weight": 1.491442084312439}, {"head": 4, "memory_to_active": 0.012189241126179695, "contribution_to_active": 0.5760683417320251, "mix_mean": 0.6029179692268372, "mix_low": 0.000255584716796875, "mix_high": 0.0014390945434570312, "read_cosine": 0.08566013723611832, "beta_mean": 3.022623059223406e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0073787132278084755, "mix_std": 0.24087229371070862, "beta_std": 5.645464625558816e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.010833281092345715, "evidence_weights": [-0.181134432554245, -0.24892127513885498, 0.9202154278755188], "agreement_weight": 0.1944548636674881, "orthogonal_weight": -1.543552041053772}, {"head": 5, "memory_to_active": 0.11061754822731018, "contribution_to_active": 0.4117169678211212, "mix_mean": 0.4159387946128845, "mix_low": 0.00150299072265625, "mix_high": 1.52587890625e-05, "read_cosine": 0.8161568641662598, "beta_mean": 0.00645481375977397, "nonfinite": 0.0, "weighted_memory_to_active": 0.05011138319969177, "mix_std": 0.22329021990299225, "beta_std": 0.008693118579685688, "beta_low": 0.82958984375, "beta_high": 0.0, "orthogonal_to_active": 0.019554054364562035, "evidence_weights": [-0.1348775178194046, -0.16585978865623474, 0.27789101004600525], "agreement_weight": 0.09836950898170471, "orthogonal_weight": -0.6805094480514526}]}, {"layer": 11, "memory_to_active": 0.06785233318805695, "contribution_to_active": 0.6306591033935547, "mix_mean": 0.5326410531997681, "mix_low": 0.11358325183391571, "mix_high": 0.05044826120138168, "read_cosine": 0.3542994260787964, "beta_mean": 0.023788776248693466, "nonfinite": 0.0, "weighted_memory_to_active": 0.037722863256931305, "mix_std": 0.32277345657348633, "beta_std": 0.03557109087705612, "beta_low": 0.720733642578125, "beta_high": 0.0, "orthogonal_to_active": 0.04083111137151718, "output_influence": 0.5670095682144165, "residual_influence": 0.16901767253875732, "state_active": 0.20360003411769867, "state_memory": 0.0027055356185883284, "state_peak": 33.52972412109375, "growth_active": 1.0565638542175293, "growth_memory": 1.4197911024093628, "heads": [{"head": 0, "memory_to_active": 0.007277489639818668, "contribution_to_active": 0.6131772994995117, "mix_mean": 0.6705811023712158, "mix_low": 0.000457763671875, "mix_high": 0.009149551391601562, "read_cosine": 0.2290680855512619, "beta_mean": 7.815613935235888e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005498366896063089, "mix_std": 0.24791638553142548, "beta_std": 0.003484383225440979, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.004522962961345911, "evidence_weights": [-0.17666684091091156, -0.35505181550979614, 0.8023494482040405], "agreement_weight": 0.4254774749279022, "orthogonal_weight": -0.8513422608375549}, {"head": 1, "memory_to_active": 0.10041727870702744, "contribution_to_active": 0.4534413814544678, "mix_mean": 0.4806019067764282, "mix_low": 0.004302024841308594, "mix_high": 0.0006008148193359375, "read_cosine": 0.5143742561340332, "beta_mean": 0.0016278749099001288, "nonfinite": 0.0, "weighted_memory_to_active": 0.05200684070587158, "mix_std": 0.26860249042510986, "beta_std": 0.007834101095795631, "beta_low": 0.96795654296875, "beta_high": 0.0, "orthogonal_to_active": 0.028742706403136253, "evidence_weights": [-0.13160471618175507, -0.27916213870048523, 0.5981535911560059], "agreement_weight": 0.08789699524641037, "orthogonal_weight": -0.6653534770011902}, {"head": 2, "memory_to_active": 0.004587188828736544, "contribution_to_active": 0.9887202978134155, "mix_mean": 0.43967580795288086, "mix_low": 0.3267850875854492, "mix_high": 0.1875286102294922, "read_cosine": 0.15525157749652863, "beta_mean": 0.0648634135723114, "nonfinite": 0.0, "weighted_memory_to_active": 0.003314598696306348, "mix_std": 0.454071044921875, "beta_std": 0.08859876543283463, "beta_low": 0.5009765625, "beta_high": 0.0, "orthogonal_to_active": 0.03538542985916138, "evidence_weights": [0.049793656915426254, -0.32860618829727173, -1.323847770690918], "agreement_weight": -0.00024357529764529318, "orthogonal_weight": 11.037020683288574}, {"head": 3, "memory_to_active": 0.006756233051419258, "contribution_to_active": 0.8699953556060791, "mix_mean": 0.418116956949234, "mix_low": 0.3422355651855469, "mix_high": 0.09785938262939453, "read_cosine": 0.6460736393928528, "beta_mean": 0.05226226523518562, "nonfinite": 0.0, "weighted_memory_to_active": 0.004425012972205877, "mix_std": 0.4435241222381592, "beta_std": 0.08158469200134277, "beta_low": 0.5338134765625, "beta_high": 0.0, "orthogonal_to_active": 0.02994050644338131, "evidence_weights": [0.22554302215576172, -0.266946017742157, -0.07597600668668747], "agreement_weight": 0.04820266366004944, "orthogonal_weight": 11.311408996582031}, {"head": 4, "memory_to_active": 0.20834314823150635, "contribution_to_active": 0.4371527433395386, "mix_mean": 0.6124951839447021, "mix_low": 0.0068149566650390625, "mix_high": 0.0015125274658203125, "read_cosine": 0.023343311622738838, "beta_mean": 0.015995070338249207, "nonfinite": 0.0, "weighted_memory_to_active": 0.12050144374370575, "mix_std": 0.26751071214675903, "beta_std": 0.01951824687421322, "beta_low": 0.5147705078125, "beta_high": 0.0, "orthogonal_to_active": 0.11942541599273682, "evidence_weights": [-0.23025177419185638, -0.23790976405143738, 1.8852338790893555], "agreement_weight": -0.23053979873657227, "orthogonal_weight": -1.001247525215149}, {"head": 5, "memory_to_active": 0.07973265647888184, "contribution_to_active": 0.4214671552181244, "mix_mean": 0.574375331401825, "mix_low": 0.000904083251953125, "mix_high": 0.006038665771484375, "read_cosine": 0.5576857328414917, "beta_mean": 0.007905859500169754, "nonfinite": 0.0, "weighted_memory_to_active": 0.04059088975191116, "mix_std": 0.2550160586833954, "beta_std": 0.012406336143612862, "beta_low": 0.80755615234375, "beta_high": 0.0, "orthogonal_to_active": 0.026969630271196365, "evidence_weights": [-0.19211596250534058, -0.3077434301376343, 0.6695886254310608], "agreement_weight": 0.0825800895690918, "orthogonal_weight": -0.8661364316940308}]}, {"layer": 12, "memory_to_active": 0.22343039512634277, "contribution_to_active": 0.5983119606971741, "mix_mean": 0.5555850863456726, "mix_low": 0.008397102355957031, "mix_high": 0.007435163017362356, "read_cosine": 0.22123321890830994, "beta_mean": 0.014309084974229336, "nonfinite": 0.0, "weighted_memory_to_active": 0.11417597532272339, "mix_std": 0.2556149661540985, "beta_std": 0.015451776795089245, "beta_low": 0.8268840312957764, "beta_high": 0.0, "orthogonal_to_active": 0.06782986223697662, "output_influence": 0.5291467905044556, "residual_influence": 0.16577595472335815, "state_active": 0.03374719247221947, "state_memory": 0.007272019051015377, "state_peak": 72.96269989013672, "growth_active": 1.1613881587982178, "growth_memory": 1.1070715188980103, "heads": [{"head": 0, "memory_to_active": 0.02934197336435318, "contribution_to_active": 0.6384367346763611, "mix_mean": 0.5993664264678955, "mix_low": 0.0010967254638671875, "mix_high": 0.007500648498535156, "read_cosine": 0.26828068494796753, "beta_mean": 0.0005411685560829937, "nonfinite": 0.0, "weighted_memory_to_active": 0.01692291721701622, "mix_std": 0.26131394505500793, "beta_std": 0.0012318069348111749, "beta_low": 0.99700927734375, "beta_high": 0.0, "orthogonal_to_active": 0.015576514415442944, "evidence_weights": [-0.20139503479003906, -0.27053627371788025, -0.0938061848282814], "agreement_weight": 0.10083343833684921, "orthogonal_weight": -0.9828289747238159}, {"head": 1, "memory_to_active": 0.07440265268087387, "contribution_to_active": 0.5799422264099121, "mix_mean": 0.5771424174308777, "mix_low": 0.0002574920654296875, "mix_high": 0.0025663375854492188, "read_cosine": 0.41659772396087646, "beta_mean": 0.0004175283538643271, "nonfinite": 0.0, "weighted_memory_to_active": 0.04700033366680145, "mix_std": 0.2620994746685028, "beta_std": 0.0069619445130229, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.038245998322963715, "evidence_weights": [-0.1972220540046692, -0.2708425223827362, 1.031934380531311], "agreement_weight": 0.29145511984825134, "orthogonal_weight": -0.9071030616760254}, {"head": 2, "memory_to_active": 0.5926347970962524, "contribution_to_active": 0.4732323884963989, "mix_mean": 0.30764418840408325, "mix_low": 0.025142669677734375, "mix_high": 1.049041748046875e-05, "read_cosine": 0.5895429849624634, "beta_mean": 0.08331143856048584, "nonfinite": 0.0, "weighted_memory_to_active": 0.2249668687582016, "mix_std": 0.22639447450637817, "beta_std": 0.0637553408741951, "beta_low": 0.00311279296875, "beta_high": 0.0, "orthogonal_to_active": 0.25182515382766724, "evidence_weights": [-0.19902540743350983, -0.06083456054329872, -0.026762843132019043], "agreement_weight": 0.027237024158239365, "orthogonal_weight": 1.4569628238677979}, {"head": 3, "memory_to_active": 0.040384601801633835, "contribution_to_active": 0.7793074250221252, "mix_mean": 0.7320964336395264, "mix_low": 9.822845458984375e-05, "mix_high": 0.02266407012939453, "read_cosine": 0.0786956176161766, "beta_mean": 0.0003464480978436768, "nonfinite": 0.0, "weighted_memory_to_active": 0.029128316789865494, "mix_std": 0.2232866734266281, "beta_std": 0.006644526496529579, "beta_low": 0.99725341796875, "beta_high": 0.0, "orthogonal_to_active": 0.027343302965164185, "evidence_weights": [-0.28403037786483765, -0.330896258354187, 1.7055177688598633], "agreement_weight": 0.6435731053352356, "orthogonal_weight": -0.9606040120124817}, {"head": 4, "memory_to_active": 0.5134411454200745, "contribution_to_active": 0.6148284673690796, "mix_mean": 0.474814236164093, "mix_low": 0.016801834106445312, "mix_high": 0.0050106048583984375, "read_cosine": -0.219441756606102, "beta_mean": 0.0007052430300973356, "nonfinite": 0.0, "weighted_memory_to_active": 0.3068881034851074, "mix_std": 0.28905951976776123, "beta_std": 0.008219601586461067, "beta_low": 0.9805908203125, "beta_high": 0.0, "orthogonal_to_active": 0.028995951637625694, "evidence_weights": [-0.15669161081314087, -0.21370111405849457, -0.15276272594928741], "agreement_weight": 0.020047519356012344, "orthogonal_weight": -0.0979817807674408}, {"head": 5, "memory_to_active": 0.09037718921899796, "contribution_to_active": 0.5041244029998779, "mix_mean": 0.642446756362915, "mix_low": 0.006985664367675781, "mix_high": 0.00685882568359375, "read_cosine": 0.193724125623703, "beta_mean": 0.0005326760001480579, "nonfinite": 0.0, "weighted_memory_to_active": 0.060149289667606354, "mix_std": 0.27153560519218445, "beta_std": 0.00589743722230196, "beta_low": 0.9852294921875, "beta_high": 0.0, "orthogonal_to_active": 0.044992201030254364, "evidence_weights": [-0.21959801018238068, -0.30992910265922546, 1.2222590446472168], "agreement_weight": 0.43506404757499695, "orthogonal_weight": -0.800006628036499}]}, {"layer": 13, "memory_to_active": 0.10925182700157166, "contribution_to_active": 0.5409932732582092, "mix_mean": 0.5451615452766418, "mix_low": 0.005856037605553865, "mix_high": 0.009574731811881065, "read_cosine": 0.3973652422428131, "beta_mean": 0.013381152413785458, "nonfinite": 0.0, "weighted_memory_to_active": 0.046730924397706985, "mix_std": 0.2549899220466614, "beta_std": 0.0161174014210701, "beta_low": 0.8109741806983948, "beta_high": 0.0, "orthogonal_to_active": 0.048925891518592834, "output_influence": 0.5316215753555298, "residual_influence": 0.20134469866752625, "state_active": 0.1304323524236679, "state_memory": 0.039059873670339584, "state_peak": 100.83169555664062, "growth_active": 0.9203671216964722, "growth_memory": 0.9610810279846191, "heads": [{"head": 0, "memory_to_active": 0.005662983749061823, "contribution_to_active": 0.6118020415306091, "mix_mean": 0.6224408149719238, "mix_low": 0.0008554458618164062, "mix_high": 0.004486083984375, "read_cosine": 0.43008506298065186, "beta_mean": 1.963893555512186e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003733969759196043, "mix_std": 0.26293420791625977, "beta_std": 4.630810872185975e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0034766013268381357, "evidence_weights": [-0.18760178983211517, -0.2625982463359833, 0.7297071814537048], "agreement_weight": 0.24247309565544128, "orthogonal_weight": -1.0330647230148315}, {"head": 1, "memory_to_active": 0.008186427876353264, "contribution_to_active": 0.6156461834907532, "mix_mean": 0.5837656259536743, "mix_low": 0.00125885009765625, "mix_high": 0.0048980712890625, "read_cosine": 0.22903195023536682, "beta_mean": 3.2680767617421225e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004589228890836239, "mix_std": 0.2731418311595917, "beta_std": 6.842421862529591e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005729789845645428, "evidence_weights": [-0.177494615316391, -0.2597152590751648, 0.5099509954452515], "agreement_weight": 0.5993973016738892, "orthogonal_weight": -1.300489068031311}, {"head": 2, "memory_to_active": 0.0548374205827713, "contribution_to_active": 0.49037447571754456, "mix_mean": 0.3939104378223419, "mix_low": 0.0013208389282226562, "mix_high": 0.00018024444580078125, "read_cosine": 0.5573855638504028, "beta_mean": 0.0077379788272082806, "nonfinite": 0.0, "weighted_memory_to_active": 0.026678364723920822, "mix_std": 0.2288648635149002, "beta_std": 0.029287029057741165, "beta_low": 0.87225341796875, "beta_high": 0.0, "orthogonal_to_active": 0.01649775728583336, "evidence_weights": [-0.1728089153766632, -0.21330419182777405, 0.40660837292671204], "agreement_weight": 0.004492286592721939, "orthogonal_weight": -0.8512725234031677}, {"head": 3, "memory_to_active": 0.1728198230266571, "contribution_to_active": 0.5593884587287903, "mix_mean": 0.6739516854286194, "mix_low": 0.02517414093017578, "mix_high": 0.03527641296386719, "read_cosine": -0.024808572605252266, "beta_mean": 0.0009662563097663224, "nonfinite": 0.0, "weighted_memory_to_active": 0.128133162856102, "mix_std": 0.28882864117622375, "beta_std": 0.010550707578659058, "beta_low": 0.99298095703125, "beta_high": 0.0, "orthogonal_to_active": 0.11541459709405899, "evidence_weights": [-0.31743478775024414, -0.24128589034080505, 1.4529874324798584], "agreement_weight": 6.878977298736572, "orthogonal_weight": -0.9775751829147339}, {"head": 4, "memory_to_active": 0.370431125164032, "contribution_to_active": 0.2399798035621643, "mix_mean": 0.3354448080062866, "mix_low": 0.0041904449462890625, "mix_high": 0.0, "read_cosine": 0.8944733142852783, "beta_mean": 0.07136067003011703, "nonfinite": 0.0, "weighted_memory_to_active": 0.089243583381176, "mix_std": 0.20544172823429108, "beta_std": 0.05428827181458473, "beta_low": 0.00115966796875, "beta_high": 0.0, "orthogonal_to_active": 0.1281694620847702, "evidence_weights": [-0.0563633069396019, -0.2209068238735199, 0.41209617257118225], "agreement_weight": 0.006051402539014816, "orthogonal_weight": 2.325399398803711}, {"head": 5, "memory_to_active": 0.04357315972447395, "contribution_to_active": 0.7287685871124268, "mix_mean": 0.6614557504653931, "mix_low": 0.0023365020751953125, "mix_high": 0.012607574462890625, "read_cosine": 0.29802405834198, "beta_mean": 0.00016968937416095287, "nonfinite": 0.0, "weighted_memory_to_active": 0.028007227927446365, "mix_std": 0.2707282304763794, "beta_std": 0.002463665558025241, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.02426714450120926, "evidence_weights": [-0.21854563057422638, -0.2840380370616913, 0.878924548625946], "agreement_weight": 0.2178906500339508, "orthogonal_weight": -0.927683413028717}]}, {"layer": 14, "memory_to_active": 0.2137598991394043, "contribution_to_active": 0.41578587889671326, "mix_mean": 0.42613685131073, "mix_low": 0.029549282044172287, "mix_high": 0.0061152782291173935, "read_cosine": 0.6123711466789246, "beta_mean": 0.060114536434412, "nonfinite": 0.0, "weighted_memory_to_active": 0.08664419502019882, "mix_std": 0.23580050468444824, "beta_std": 0.0583636611700058, "beta_low": 0.5126953125, "beta_high": 0.0, "orthogonal_to_active": 0.039803631603717804, "output_influence": 0.35416117310523987, "residual_influence": 0.15897564589977264, "state_active": 0.051459915935993195, "state_memory": 0.013195425271987915, "state_peak": 53.805145263671875, "growth_active": 1.105158805847168, "growth_memory": 1.2171733379364014, "heads": [{"head": 0, "memory_to_active": 0.35610121488571167, "contribution_to_active": 0.18960285186767578, "mix_mean": 0.18334388732910156, "mix_low": 0.01760387420654297, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8142706155776978, "beta_mean": 0.06688617169857025, "nonfinite": 0.0, "weighted_memory_to_active": 0.07333450019359589, "mix_std": 0.1682160645723343, "beta_std": 0.06195773929357529, "beta_low": 0.01300048828125, "beta_high": 0.0, "orthogonal_to_active": 0.08438363671302795, "evidence_weights": [-0.0931815579533577, -0.04563497379422188, 0.3557087779045105], "agreement_weight": 0.020823480561375618, "orthogonal_weight": 1.943830966949463}, {"head": 1, "memory_to_active": 0.44774898886680603, "contribution_to_active": 0.28477808833122253, "mix_mean": 0.18626484274864197, "mix_low": 0.14709854125976562, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8889428377151489, "beta_mean": 0.25267326831817627, "nonfinite": 0.0, "weighted_memory_to_active": 0.19419850409030914, "mix_std": 0.2203890085220337, "beta_std": 0.2473399043083191, "beta_low": 0.0916748046875, "beta_high": 0.0, "orthogonal_to_active": 0.018798107281327248, "evidence_weights": [0.017058005556464195, -0.18527784943580627, -0.09769124537706375], "agreement_weight": 0.05923702195286751, "orthogonal_weight": 0.3365529775619507}, {"head": 2, "memory_to_active": 0.12041017413139343, "contribution_to_active": 0.587104082107544, "mix_mean": 0.5925594568252563, "mix_low": 0.0025339126586914062, "mix_high": 0.0050945281982421875, "read_cosine": 0.3891545534133911, "beta_mean": 0.0017554524820297956, "nonfinite": 0.0, "weighted_memory_to_active": 0.07766793668270111, "mix_std": 0.2642310559749603, "beta_std": 0.0023992147762328386, "beta_low": 0.98468017578125, "beta_high": 0.0, "orthogonal_to_active": 0.060438286513090134, "evidence_weights": [-0.12511304020881653, -0.31385350227355957, 0.8996532559394836], "agreement_weight": 0.0859759971499443, "orthogonal_weight": -0.8733295798301697}, {"head": 3, "memory_to_active": 0.03638644888997078, "contribution_to_active": 0.5880446434020996, "mix_mean": 0.6780473589897156, "mix_low": 0.004039764404296875, "mix_high": 0.02266407012939453, "read_cosine": -0.06954845041036606, "beta_mean": 0.00485954200848937, "nonfinite": 0.0, "weighted_memory_to_active": 0.023278672248125076, "mix_std": 0.25496765971183777, "beta_std": 0.007576936390250921, "beta_low": 0.87042236328125, "beta_high": 0.0, "orthogonal_to_active": 0.017813334241509438, "evidence_weights": [-0.3382292687892914, -0.31130343675613403, 0.30830875039100647], "agreement_weight": 0.06853897869586945, "orthogonal_weight": -0.8328889608383179}, {"head": 4, "memory_to_active": 0.14430394768714905, "contribution_to_active": 0.4973785877227783, "mix_mean": 0.5300946235656738, "mix_low": 0.00315093994140625, "mix_high": 0.0080413818359375, "read_cosine": 0.9063514471054077, "beta_mean": 0.033666256815195084, "nonfinite": 0.0, "weighted_memory_to_active": 0.08442584425210953, "mix_std": 0.2580541670322418, "beta_std": 0.02549373358488083, "beta_low": 0.12884521484375, "beta_high": 0.0, "orthogonal_to_active": 0.02285951003432274, "evidence_weights": [-0.20111961662769318, -0.23381608724594116, 0.44927382469177246], "agreement_weight": 0.13325271010398865, "orthogonal_weight": -0.6280697584152222}, {"head": 5, "memory_to_active": 0.17760850489139557, "contribution_to_active": 0.3478071689605713, "mix_mean": 0.38651084899902344, "mix_low": 0.00286865234375, "mix_high": 0.000885009765625, "read_cosine": 0.7450559139251709, "beta_mean": 0.0008465494611300528, "nonfinite": 0.0, "weighted_memory_to_active": 0.06695972383022308, "mix_std": 0.2489449679851532, "beta_std": 0.005414410028606653, "beta_low": 0.987548828125, "beta_high": 0.0, "orthogonal_to_active": 0.03452891856431961, "evidence_weights": [-0.17352961003780365, -0.18567955493927002, 0.3791051506996155], "agreement_weight": 0.06121456250548363, "orthogonal_weight": -0.7584744691848755}]}, {"layer": 15, "memory_to_active": 0.04910358041524887, "contribution_to_active": 0.49984750151634216, "mix_mean": 0.4405325949192047, "mix_low": 0.006437460891902447, "mix_high": 0.0028497378807514906, "read_cosine": 0.6495412588119507, "beta_mean": 0.0413503460586071, "nonfinite": 0.0, "weighted_memory_to_active": 0.025188155472278595, "mix_std": 0.2825416922569275, "beta_std": 0.04472016915678978, "beta_low": 0.7010905742645264, "beta_high": 0.0, "orthogonal_to_active": 0.010405406355857849, "output_influence": 0.3263983130455017, "residual_influence": 0.2036675214767456, "state_active": 0.17361660301685333, "state_memory": 0.009777482599020004, "state_peak": 25.052139282226562, "growth_active": 1.1943734884262085, "growth_memory": 1.2224857807159424, "heads": [{"head": 0, "memory_to_active": 0.11608317494392395, "contribution_to_active": 0.48616480827331543, "mix_mean": 0.40555471181869507, "mix_low": 0.0070590972900390625, "mix_high": 0.0057697296142578125, "read_cosine": 0.4902041554450989, "beta_mean": 0.05247030779719353, "nonfinite": 0.0, "weighted_memory_to_active": 0.06030821055173874, "mix_std": 0.2860887050628662, "beta_std": 0.036139823496341705, "beta_low": 0.04443359375, "beta_high": 0.0, "orthogonal_to_active": 0.04064919799566269, "evidence_weights": [-0.18241465091705322, -0.19507652521133423, 0.5482242703437805], "agreement_weight": 0.08907977491617203, "orthogonal_weight": -0.8492689728736877}, {"head": 1, "memory_to_active": 0.08451013267040253, "contribution_to_active": 0.4351556897163391, "mix_mean": 0.4490397572517395, "mix_low": 0.0007839202880859375, "mix_high": 0.007023811340332031, "read_cosine": 0.9335224032402039, "beta_mean": 0.0001571319007780403, "nonfinite": 0.0, "weighted_memory_to_active": 0.036261994391679764, "mix_std": 0.28224632143974304, "beta_std": 0.001289660343900323, "beta_low": 0.9990234375, "beta_high": 0.0, "orthogonal_to_active": 0.003120013978332281, "evidence_weights": [-0.16226144134998322, -0.17585523426532745, 0.15746797621250153], "agreement_weight": 0.1048862412571907, "orthogonal_weight": 0.19456152617931366}, {"head": 2, "memory_to_active": 0.004252323880791664, "contribution_to_active": 0.5162123441696167, "mix_mean": 0.3622732162475586, "mix_low": 0.010254859924316406, "mix_high": 4.9591064453125e-05, "read_cosine": 0.7820397615432739, "beta_mean": 4.9301987019134685e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002284308895468712, "mix_std": 0.27171534299850464, "beta_std": 0.00015350151807069778, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0007802336476743221, "evidence_weights": [-0.19649150967597961, -0.2665180265903473, 0.12998263537883759], "agreement_weight": 0.009519707411527634, "orthogonal_weight": 0.9718893766403198}, {"head": 3, "memory_to_active": 0.0016706124879419804, "contribution_to_active": 0.3752157688140869, "mix_mean": 0.42565545439720154, "mix_low": 0.0036039352416992188, "mix_high": 0.00061798095703125, "read_cosine": 0.6277289390563965, "beta_mean": 8.68915049068164e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005676576402038336, "mix_std": 0.28820064663887024, "beta_std": 2.287738061568234e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00029299187008291483, "evidence_weights": [-0.2758500576019287, -0.2399836778640747, 0.18413211405277252], "agreement_weight": 0.0584283210337162, "orthogonal_weight": -0.5171934962272644}, {"head": 4, "memory_to_active": 0.05698481202125549, "contribution_to_active": 0.6349123120307922, "mix_mean": 0.4046712815761566, "mix_low": 0.008017539978027344, "mix_high": 0.0006475448608398438, "read_cosine": 0.7023046016693115, "beta_mean": 0.1925748884677887, "nonfinite": 0.0, "weighted_memory_to_active": 0.03088897466659546, "mix_std": 0.30758136510849, "beta_std": 0.21207493543624878, "beta_low": 0.19781494140625, "beta_high": 0.0, "orthogonal_to_active": 0.005574074573814869, "evidence_weights": [-0.20402349531650543, -0.251058965921402, -0.026983169838786125], "agreement_weight": 0.011346310377120972, "orthogonal_weight": 0.4041126072406769}, {"head": 5, "memory_to_active": 0.0311204195022583, "contribution_to_active": 0.5514242053031921, "mix_mean": 0.5960010290145874, "mix_low": 0.008905410766601562, "mix_high": 0.0029897689819335938, "read_cosine": 0.3614475131034851, "beta_mean": 0.0028417338617146015, "nonfinite": 0.0, "weighted_memory_to_active": 0.02081778272986412, "mix_std": 0.25941765308380127, "beta_std": 0.018640197813510895, "beta_low": 0.96533203125, "beta_high": 0.0, "orthogonal_to_active": 0.012015927582979202, "evidence_weights": [-0.2786259651184082, -0.30641570687294006, 0.5708335638046265], "agreement_weight": 0.31663185358047485, "orthogonal_weight": 0.8146146535873413}]}]}, {"step": 76000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8252802440001688, "memory_to_active": 0.8053848743438721, "contribution_to_active": 0.5847471952438354, "mix_mean": 0.43310824036598206, "mix_low": 0.12001513689756393, "mix_high": 0.010997235774993896, "read_cosine": 0.34530365467071533, "beta_mean": 0.0679982379078865, "nonfinite": 0.0, "weighted_memory_to_active": 0.20112285017967224, "mix_std": 0.22176210582256317, "beta_std": 0.05924630165100098, "beta_low": 0.6001708507537842, "beta_high": 4.704793173004873e-05, "orthogonal_to_active": 0.06530135869979858, "samples": 4, "scored_tokens": 899, "normal_loss": 3.0252554416656494, "normal_token_accuracy": 0.4549499452114105, "state_active": 0.06480426341295242, "state_memory": 0.015879090875387192, "state_peak": 122.4492416381836, "growth_active": 1.125327229499817, "growth_memory": 1.261296272277832, "output_influence": 0.45453786849975586, "residual_influence": 0.16983771324157715, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.111499309539795, "no_memory_loss_delta": 0.8334683775901794, "scrambled_loss_delta": 0.1455167680978775, "memory_only_loss_delta": 3.878600835800171, "no_evidence_loss_delta": 0.44060319662094116, "no_agreement_loss_delta": 0.021765101701021194, "no_orthogonal_loss_delta": 0.013684012927114964}, {"index": 94, "tokens": 256, "normal_loss": 3.126046895980835, "no_memory_loss_delta": 0.8684221506118774, "scrambled_loss_delta": 0.46015310287475586, "memory_only_loss_delta": 4.362908840179443, "no_evidence_loss_delta": 0.36865028738975525, "no_agreement_loss_delta": 0.038077060133218765, "no_orthogonal_loss_delta": 0.04506060481071472}, {"index": 188, "tokens": 256, "normal_loss": 3.343839168548584, "no_memory_loss_delta": 1.098931074142456, "scrambled_loss_delta": 0.11814695596694946, "memory_only_loss_delta": 3.369267702102661, "no_evidence_loss_delta": 0.5878468751907349, "no_agreement_loss_delta": 0.05153685808181763, "no_orthogonal_loss_delta": 0.04677809029817581}, {"index": 283, "tokens": 131, "normal_loss": 2.0371735095977783, "no_memory_loss_delta": 0.44206568598747253, "scrambled_loss_delta": 0.02191092260181904, "memory_only_loss_delta": 3.0545053482055664, "no_evidence_loss_delta": 0.20406295359134674, "no_agreement_loss_delta": -0.01029330026358366, "no_orthogonal_loss_delta": -0.07559105008840561}], "agreement_weight_abs_mean": 0.25645706057548523, "agreement_weight_abs_max": 6.899858474731445, "agreement_weight_mean": 0.22284121811389923, "orthogonal_weight_abs_mean": 0.9551419615745544, "orthogonal_weight_abs_max": 11.371125221252441, "orthogonal_weight_mean": -0.033186718821525574, "no_memory_loss": 3.8872363567352295, "no_memory_loss_delta": 0.8619810938835144, "no_memory_token_accuracy_gain": 0.11902113258838654, "no_memory_token_win_fraction": 0.6874305009841919, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 57.76754116442716, "scrambled_loss": 3.234562873840332, "scrambled_loss_delta": 0.2093074917793274, "scrambled_token_accuracy_gain": 0.017797552049160004, "scrambled_token_win_fraction": 0.607341468334198, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 18.885422336607142, "memory_only_loss": 6.776644706726074, "memory_only_loss_delta": 3.751389503479004, "memory_only_token_accuracy_gain": 0.300333708524704, "memory_only_token_win_fraction": 0.9121245741844177, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 97.65149094410977, "no_evidence_loss": 3.4528300762176514, "no_evidence_loss_delta": 0.427575021982193, "no_evidence_token_accuracy_gain": 0.05672970041632652, "no_evidence_token_win_fraction": 0.6195772886276245, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 34.79115208092548, "no_agreement_loss": 3.055471420288086, "no_agreement_loss_delta": 0.03021647036075592, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.489432692527771, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.976451641061759, "no_orthogonal_loss": 3.0442886352539062, "no_orthogonal_loss_delta": 0.019033798947930336, "no_orthogonal_token_accuracy_gain": -0.0011123470030725002, "no_orthogonal_token_win_fraction": 0.5706340670585632, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 1.8853800026940686, "short_no_memory_loss_delta": 0.7922658324241638, "long_context_memory_gain": 0.06971526145935059, "normal_context_loss_gain": 0.11902404576539993, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.6555252075195312, "contribution_to_active": 1.245983600616455, "mix_mean": 0.1107020154595375, "mix_low": 0.49766814708709717, "mix_high": 0.004997094627469778, "read_cosine": 0.4126039147377014, "beta_mean": 0.22211512923240662, "nonfinite": 0.0, "weighted_memory_to_active": 1.2600442171096802, "mix_std": 0.09207773953676224, "beta_std": 0.14137955009937286, "beta_low": 0.187591552734375, "beta_high": 0.0, "orthogonal_to_active": 0.05062401667237282, "output_influence": 0.2121344357728958, "residual_influence": 0.22187983989715576, "state_active": 0.06429296731948853, "state_memory": 0.04527098685503006, "state_peak": 47.43202590942383, "growth_active": 1.1972306966781616, "growth_memory": 1.2423001527786255, "heads": [{"head": 0, "memory_to_active": 2.919518232345581, "contribution_to_active": 0.44021835923194885, "mix_mean": 0.018733300268650055, "mix_low": 0.6264820098876953, "mix_high": 0.0, "read_cosine": 0.2273280918598175, "beta_mean": 0.19426503777503967, "nonfinite": 0.0, "weighted_memory_to_active": 0.5484960079193115, "mix_std": 0.05542881041765213, "beta_std": 0.09839092195034027, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.056985702365636826, "evidence_weights": [-0.0007500178762711585, -0.03747519478201866, -0.05988210067152977], "agreement_weight": -0.23506242036819458, "orthogonal_weight": -0.12026071548461914}, {"head": 1, "memory_to_active": 6.990445137023926, "contribution_to_active": 3.0560317039489746, "mix_mean": 0.020317360758781433, "mix_low": 0.6036787033081055, "mix_high": 0.0, "read_cosine": 0.4402366280555725, "beta_mean": 0.48881423473358154, "nonfinite": 0.0, "weighted_memory_to_active": 3.264108180999756, "mix_std": 0.05757909640669823, "beta_std": 0.21633704006671906, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.09896206855773926, "evidence_weights": [0.02287561260163784, -0.10091681778430939, 0.6803271174430847], "agreement_weight": 0.02775605209171772, "orthogonal_weight": 0.06727659702301025}, {"head": 2, "memory_to_active": 2.892699956893921, "contribution_to_active": 0.6759576797485352, "mix_mean": 0.015245135873556137, "mix_low": 0.63995361328125, "mix_high": 0.0, "read_cosine": 0.2856634855270386, "beta_mean": 0.2683527171611786, "nonfinite": 0.0, "weighted_memory_to_active": 0.7796995043754578, "mix_std": 0.032878316938877106, "beta_std": 0.20502880215644836, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05919121950864792, "evidence_weights": [0.061406880617141724, 0.04463515803217888, 0.2649269998073578], "agreement_weight": 0.010202720761299133, "orthogonal_weight": -0.19432634115219116}, {"head": 3, "memory_to_active": 7.732421875, "contribution_to_active": 2.772630214691162, "mix_mean": 0.02851148694753647, "mix_low": 0.501643180847168, "mix_high": 0.0, "read_cosine": 0.28404566645622253, "beta_mean": 0.34406185150146484, "nonfinite": 0.0, "weighted_memory_to_active": 2.872443675994873, "mix_std": 0.0664558932185173, "beta_std": 0.2680503726005554, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.07073430716991425, "evidence_weights": [-0.03416391834616661, -0.02487105503678322, -0.1460321992635727], "agreement_weight": -0.005750768817961216, "orthogonal_weight": -0.028744321316480637}, {"head": 4, "memory_to_active": 1.2954012155532837, "contribution_to_active": 0.058744367212057114, "mix_mean": 0.023818835616111755, "mix_low": 0.6011972427368164, "mix_high": 0.0, "read_cosine": 0.30336660146713257, "beta_mean": 0.0201520174741745, "nonfinite": 0.0, "weighted_memory_to_active": 0.04085554555058479, "mix_std": 0.06075394153594971, "beta_std": 0.04174913465976715, "beta_low": 0.6424560546875, "beta_high": 0.0, "orthogonal_to_active": 0.0007723408634774387, "evidence_weights": [0.08667395263910294, 0.054520707577466965, -0.1476845145225525], "agreement_weight": -0.20819340646266937, "orthogonal_weight": 0.018007425591349602}, {"head": 5, "memory_to_active": 0.10266372561454773, "contribution_to_active": 0.47231897711753845, "mix_mean": 0.5575859546661377, "mix_low": 0.01305389404296875, "mix_high": 0.029982566833496094, "read_cosine": 0.9349830746650696, "beta_mean": 0.017044786363840103, "nonfinite": 0.0, "weighted_memory_to_active": 0.05466172844171524, "mix_std": 0.27937036752700806, "beta_std": 0.018720993772149086, "beta_low": 0.48272705078125, "beta_high": 0.0, "orthogonal_to_active": 0.01709846779704094, "evidence_weights": [-0.1728094220161438, -0.21626894176006317, 0.3640320301055908], "agreement_weight": 0.28461918234825134, "orthogonal_weight": -0.7215695977210999}]}, {"layer": 1, "memory_to_active": 2.564631700515747, "contribution_to_active": 0.5049691796302795, "mix_mean": 0.2190832644701004, "mix_low": 0.36760222911834717, "mix_high": 0.0013251304626464844, "read_cosine": 0.29699429869651794, "beta_mean": 0.1785743236541748, "nonfinite": 0.0, "weighted_memory_to_active": 0.344728022813797, "mix_std": 0.11406642198562622, "beta_std": 0.1307964026927948, "beta_low": 0.3541158437728882, "beta_high": 0.00016276042151730508, "orthogonal_to_active": 0.03767380118370056, "output_influence": 0.278873473405838, "residual_influence": 0.08059300482273102, "state_active": 0.027056757360696793, "state_memory": 0.03412560373544693, "state_peak": 41.76907730102539, "growth_active": 1.1203442811965942, "growth_memory": 1.442534327507019, "heads": [{"head": 0, "memory_to_active": 0.02078751102089882, "contribution_to_active": 0.4958590269088745, "mix_mean": 0.5486572980880737, "mix_low": 0.0038557052612304688, "mix_high": 0.0030384063720703125, "read_cosine": 0.027300992980599403, "beta_mean": 0.00012200677156215534, "nonfinite": 0.0, "weighted_memory_to_active": 0.013112816028296947, "mix_std": 0.28273165225982666, "beta_std": 0.00024033308727666736, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.01275317370891571, "evidence_weights": [-0.20978812873363495, -0.26965197920799255, 0.186723992228508], "agreement_weight": 0.20426660776138306, "orthogonal_weight": -1.0120306015014648}, {"head": 1, "memory_to_active": 3.0863566398620605, "contribution_to_active": 0.2970053553581238, "mix_mean": 0.02123444527387619, "mix_low": 0.5032377243041992, "mix_high": 9.5367431640625e-07, "read_cosine": 0.539709210395813, "beta_mean": 0.28051167726516724, "nonfinite": 0.0, "weighted_memory_to_active": 0.328349769115448, "mix_std": 0.042690690606832504, "beta_std": 0.1944214105606079, "beta_low": 0.001220703125, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.021783463656902313, "evidence_weights": [0.026619724929332733, -0.06277341395616531, 0.050816863775253296], "agreement_weight": -0.0008083684951998293, "orthogonal_weight": 0.2122454047203064}, {"head": 2, "memory_to_active": 0.03383807837963104, "contribution_to_active": 0.5502524971961975, "mix_mean": 0.681397557258606, "mix_low": 0.000244140625, "mix_high": 0.0049114227294921875, "read_cosine": 0.44638797640800476, "beta_mean": 0.001197322621010244, "nonfinite": 0.0, "weighted_memory_to_active": 0.022605400532484055, "mix_std": 0.22558721899986267, "beta_std": 0.0037134182639420033, "beta_low": 0.98028564453125, "beta_high": 0.0, "orthogonal_to_active": 0.016540737822651863, "evidence_weights": [-0.2705039083957672, -0.3667372763156891, 0.6343333125114441], "agreement_weight": 0.055560506880283356, "orthogonal_weight": -0.840793251991272}, {"head": 3, "memory_to_active": 3.156240463256836, "contribution_to_active": 0.22379948198795319, "mix_mean": 0.025352444499731064, "mix_low": 0.5103063583374023, "mix_high": 0.0, "read_cosine": 0.18492630124092102, "beta_mean": 0.32698214054107666, "nonfinite": 0.0, "weighted_memory_to_active": 0.28674155473709106, "mix_std": 0.06327693909406662, "beta_std": 0.2611028552055359, "beta_low": 0.073974609375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.10056319832801819, "evidence_weights": [0.03801239654421806, -0.0201663039624691, -0.009966558776795864], "agreement_weight": -0.023286687210202217, "orthogonal_weight": -0.4758894145488739}, {"head": 4, "memory_to_active": 4.385845184326172, "contribution_to_active": 1.3693146705627441, "mix_mean": 0.031546879559755325, "mix_low": 0.33905601501464844, "mix_high": 0.0, "read_cosine": 0.31875112652778625, "beta_mean": 0.17472310364246368, "nonfinite": 0.0, "weighted_memory_to_active": 1.353334903717041, "mix_std": 0.05215475708246231, "beta_std": 0.13474103808403015, "beta_low": 0.03485107421875, "beta_high": 0.000244140625, "orthogonal_to_active": 0.047186125069856644, "evidence_weights": [-0.011326435022056103, -0.003490072675049305, 0.17982973158359528], "agreement_weight": -0.0037365336902439594, "orthogonal_weight": 0.05649355426430702}, {"head": 5, "memory_to_active": 4.7047224044799805, "contribution_to_active": 0.09358391910791397, "mix_mean": 0.006310859229415655, "mix_low": 0.8489131927490234, "mix_high": 0.0, "read_cosine": 0.2648901343345642, "beta_mean": 0.28790968656539917, "nonfinite": 0.0, "weighted_memory_to_active": 0.0642237514257431, "mix_std": 0.01795727200806141, "beta_std": 0.1905592828989029, "beta_low": 0.03436279296875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.02721608430147171, "evidence_weights": [0.16046591103076935, 0.07675714790821075, 0.08212210237979889], "agreement_weight": -0.11319532245397568, "orthogonal_weight": 0.41715964674949646}]}, {"layer": 2, "memory_to_active": 1.0249583721160889, "contribution_to_active": 0.3546407222747803, "mix_mean": 0.4170190691947937, "mix_low": 0.2361912876367569, "mix_high": 0.0043422384187579155, "read_cosine": 0.29315924644470215, "beta_mean": 0.07575712352991104, "nonfinite": 0.0, "weighted_memory_to_active": 0.06527262926101685, "mix_std": 0.18361589312553406, "beta_std": 0.06659763306379318, "beta_low": 0.6171163320541382, "beta_high": 0.000274658203125, "orthogonal_to_active": 0.014828979969024658, "output_influence": 0.41320598125457764, "residual_influence": 0.12165580689907074, "state_active": 0.040848277509212494, "state_memory": 0.012747855857014656, "state_peak": 26.926149368286133, "growth_active": 1.0672214031219482, "growth_memory": 1.553184986114502, "heads": [{"head": 0, "memory_to_active": 0.003605542704463005, "contribution_to_active": 0.3578994572162628, "mix_mean": 0.6561738848686218, "mix_low": 0.00045871734619140625, "mix_high": 0.009699821472167969, "read_cosine": 0.13660569489002228, "beta_mean": 3.078308509429917e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0023717174772173166, "mix_std": 0.2622120976448059, "beta_std": 5.900370524614118e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002446997445076704, "evidence_weights": [-0.3734363913536072, -0.21473191678524017, 0.2145799845457077], "agreement_weight": 0.14048044383525848, "orthogonal_weight": -1.068084716796875}, {"head": 1, "memory_to_active": 3.188086748123169, "contribution_to_active": 0.04718836396932602, "mix_mean": 0.005959029775112867, "mix_low": 0.8583364486694336, "mix_high": 0.0, "read_cosine": 0.12451175600290298, "beta_mean": 0.20224399864673615, "nonfinite": 0.0, "weighted_memory_to_active": 0.03826737403869629, "mix_std": 0.012049579992890358, "beta_std": 0.14045874774456024, "beta_low": 0.003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.010870033875107765, "evidence_weights": [0.12977707386016846, 0.10028733313083649, 0.5749569535255432], "agreement_weight": 0.032041508704423904, "orthogonal_weight": 0.3194849491119385}, {"head": 2, "memory_to_active": 0.006959268823266029, "contribution_to_active": 0.41488122940063477, "mix_mean": 0.5623472929000854, "mix_low": 0.000911712646484375, "mix_high": 0.0039691925048828125, "read_cosine": 0.4080074727535248, "beta_mean": 2.1891333744861186e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0028672791086137295, "mix_std": 0.27424880862236023, "beta_std": 4.2500287236180156e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00023800089547876269, "evidence_weights": [-0.2469100058078766, -0.2904810607433319, 0.5694066286087036], "agreement_weight": -0.039282072335481644, "orthogonal_weight": 0.10012222081422806}, {"head": 3, "memory_to_active": 0.09034425020217896, "contribution_to_active": 0.5514732003211975, "mix_mean": 0.59068763256073, "mix_low": 0.000171661376953125, "mix_high": 0.005476951599121094, "read_cosine": 0.5219668745994568, "beta_mean": 0.014396759681403637, "nonfinite": 0.0, "weighted_memory_to_active": 0.05803427845239639, "mix_std": 0.25007617473602295, "beta_std": 0.02381550334393978, "beta_low": 0.6370849609375, "beta_high": 0.0, "orthogonal_to_active": 0.043286941945552826, "evidence_weights": [-0.1981520801782608, -0.3256988823413849, 0.722183883190155], "agreement_weight": 0.055261678993701935, "orthogonal_weight": -0.893795371055603}, {"head": 4, "memory_to_active": 2.852095603942871, "contribution_to_active": 0.22219529747962952, "mix_mean": 0.020781921222805977, "mix_low": 0.556849479675293, "mix_high": 0.0, "read_cosine": 0.1933794915676117, "beta_mean": 0.23781901597976685, "nonfinite": 0.0, "weighted_memory_to_active": 0.28524136543273926, "mix_std": 0.04842882603406906, "beta_std": 0.2351709008216858, "beta_low": 0.06195068359375, "beta_high": 0.00164794921875, "orthogonal_to_active": 0.03082982636988163, "evidence_weights": [0.023194264620542526, -0.000510149693582207, 0.03326907008886337], "agreement_weight": 0.018872033804655075, "orthogonal_weight": -0.12640686333179474}, {"head": 5, "memory_to_active": 0.008658397942781448, "contribution_to_active": 0.5342066287994385, "mix_mean": 0.6661645770072937, "mix_low": 0.00041961669921875, "mix_high": 0.006907463073730469, "read_cosine": 0.37448424100875854, "beta_mean": 3.027714410563931e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004853720776736736, "mix_std": 0.25467991828918457, "beta_std": 3.913127147825435e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0013020718470215797, "evidence_weights": [-0.27449214458465576, -0.2849820852279663, 0.89741051197052], "agreement_weight": 0.2895313501358032, "orthogonal_weight": -0.31260332465171814}]}, {"layer": 3, "memory_to_active": 0.4026554226875305, "contribution_to_active": 0.5000159740447998, "mix_mean": 0.5311768054962158, "mix_low": 0.05148490518331528, "mix_high": 0.011683623306453228, "read_cosine": 0.18455269932746887, "beta_mean": 0.022757017984986305, "nonfinite": 0.0, "weighted_memory_to_active": 0.11466100066900253, "mix_std": 0.2541322708129883, "beta_std": 0.022431571036577225, "beta_low": 0.7762655019760132, "beta_high": 0.0, "orthogonal_to_active": 0.047771014273166656, "output_influence": 0.45180490612983704, "residual_influence": 0.11833707988262177, "state_active": 0.023335179314017296, "state_memory": 0.0061372676864266396, "state_peak": 23.83917236328125, "growth_active": 1.1049044132232666, "growth_memory": 1.5972950458526611, "heads": [{"head": 0, "memory_to_active": 0.347664475440979, "contribution_to_active": 0.6204115152359009, "mix_mean": 0.6210665702819824, "mix_low": 0.001129150390625, "mix_high": 0.02255535125732422, "read_cosine": 0.24694311618804932, "beta_mean": 0.008383484557271004, "nonfinite": 0.0, "weighted_memory_to_active": 0.21142663061618805, "mix_std": 0.2945106029510498, "beta_std": 0.009040549397468567, "beta_low": 0.737060546875, "beta_high": 0.0, "orthogonal_to_active": 0.1552528738975525, "evidence_weights": [-0.21371635794639587, -0.24753500521183014, 1.0626392364501953], "agreement_weight": 0.11507070064544678, "orthogonal_weight": -0.7733712196350098}, {"head": 1, "memory_to_active": 0.006019849330186844, "contribution_to_active": 0.4755735695362091, "mix_mean": 0.5984815359115601, "mix_low": 0.0025386810302734375, "mix_high": 0.011662483215332031, "read_cosine": 0.16861021518707275, "beta_mean": 2.6194202291662805e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037245952989906073, "mix_std": 0.2905462384223938, "beta_std": 6.78968062857166e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003008305560797453, "evidence_weights": [-0.2665609121322632, -0.21467016637325287, 0.6822701096534729], "agreement_weight": 0.6115965247154236, "orthogonal_weight": -0.8789628148078918}, {"head": 2, "memory_to_active": 1.8328583240509033, "contribution_to_active": 0.2678930163383484, "mix_mean": 0.05798892304301262, "mix_low": 0.29841041564941406, "mix_high": 9.5367431640625e-07, "read_cosine": 0.22410424053668976, "beta_mean": 0.12352263927459717, "nonfinite": 0.0, "weighted_memory_to_active": 0.30134767293930054, "mix_std": 0.10959120094776154, "beta_std": 0.11452658474445343, "beta_low": 0.0225830078125, "beta_high": 0.0, "orthogonal_to_active": 0.004381926730275154, "evidence_weights": [-0.09954559057950974, -0.007721771020442247, -0.02664203941822052], "agreement_weight": -0.0061592962592840195, "orthogonal_weight": 0.018883682787418365}, {"head": 3, "memory_to_active": 0.014111045747995377, "contribution_to_active": 0.5502305030822754, "mix_mean": 0.6172305941581726, "mix_low": 0.0032901763916015625, "mix_high": 0.009537696838378906, "read_cosine": 0.26383957266807556, "beta_mean": 6.370319897541776e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00946741458028555, "mix_std": 0.2813328206539154, "beta_std": 0.0001466902031097561, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0022549876011908054, "evidence_weights": [-0.2622968852519989, -0.22017847001552582, 2.3477160930633545], "agreement_weight": 2.3723692893981934, "orthogonal_weight": -0.3062311112880707}, {"head": 4, "memory_to_active": 0.011228651739656925, "contribution_to_active": 0.4174943268299103, "mix_mean": 0.5535658597946167, "mix_low": 0.0016422271728515625, "mix_high": 0.0031871795654296875, "read_cosine": 0.12685872614383698, "beta_mean": 5.372246232582256e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007666615769267082, "mix_std": 0.2857552766799927, "beta_std": 9.210294228978455e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007139138877391815, "evidence_weights": [-0.24649803340435028, -0.22366872429847717, 0.05873987823724747], "agreement_weight": -0.0579448863863945, "orthogonal_weight": -1.01638662815094}, {"head": 5, "memory_to_active": 0.20405013859272003, "contribution_to_active": 0.6684930324554443, "mix_mean": 0.7387272715568542, "mix_low": 0.0018987655639648438, "mix_high": 0.02315807342529297, "read_cosine": 0.07696031033992767, "beta_mean": 0.004492353182286024, "nonfinite": 0.0, "weighted_memory_to_active": 0.15433306992053986, "mix_std": 0.2630574703216553, "beta_std": 0.010715601965785027, "beta_low": 0.89794921875, "beta_high": 0.0, "orthogonal_to_active": 0.11458887159824371, "evidence_weights": [-0.2595416307449341, -0.3428340256214142, 0.5573930740356445], "agreement_weight": 0.24539636075496674, "orthogonal_weight": -0.7659368515014648}]}, {"layer": 4, "memory_to_active": 0.999731183052063, "contribution_to_active": 0.4764523208141327, "mix_mean": 0.37764185667037964, "mix_low": 0.22362741827964783, "mix_high": 0.012201627716422081, "read_cosine": 0.2977074086666107, "beta_mean": 0.10365476459264755, "nonfinite": 0.0, "weighted_memory_to_active": 0.12517890334129333, "mix_std": 0.1609419286251068, "beta_std": 0.09545335918664932, "beta_low": 0.4002583920955658, "beta_high": 2.0345052689663135e-05, "orthogonal_to_active": 0.055086925625801086, "output_influence": 0.5096921324729919, "residual_influence": 0.16797921061515808, "state_active": 0.03749099373817444, "state_memory": 0.015268249437212944, "state_peak": 43.19480514526367, "growth_active": 1.1960749626159668, "growth_memory": 1.4982798099517822, "heads": [{"head": 0, "memory_to_active": 0.13839252293109894, "contribution_to_active": 0.8197129964828491, "mix_mean": 0.6957437992095947, "mix_low": 0.008261680603027344, "mix_high": 0.03480339050292969, "read_cosine": 0.4309912323951721, "beta_mean": 0.2387436330318451, "nonfinite": 0.0, "weighted_memory_to_active": 0.11953020095825195, "mix_std": 0.34414514899253845, "beta_std": 0.23355400562286377, "beta_low": 0.25592041015625, "beta_high": 0.0, "orthogonal_to_active": 0.09150514006614685, "evidence_weights": [-0.24096103012561798, -0.3545636534690857, -0.23813125491142273], "agreement_weight": 0.0049950135871768, "orthogonal_weight": 0.8994216322898865}, {"head": 1, "memory_to_active": 0.039247091859579086, "contribution_to_active": 0.6620360016822815, "mix_mean": 0.7079075574874878, "mix_low": 7.2479248046875e-05, "mix_high": 0.019346237182617188, "read_cosine": 0.5658698081970215, "beta_mean": 0.0008847024291753769, "nonfinite": 0.0, "weighted_memory_to_active": 0.029394183307886124, "mix_std": 0.23511475324630737, "beta_std": 0.006847360637038946, "beta_low": 0.981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.014387852512300014, "evidence_weights": [-0.23249684274196625, -0.3627997040748596, 0.5544681549072266], "agreement_weight": 0.3757643699645996, "orthogonal_weight": -0.6726067066192627}, {"head": 2, "memory_to_active": 1.439003348350525, "contribution_to_active": 0.37369322776794434, "mix_mean": 0.05910825729370117, "mix_low": 0.29143810272216797, "mix_high": 2.288818359375e-05, "read_cosine": 0.20726755261421204, "beta_mean": 0.05109521746635437, "nonfinite": 0.0, "weighted_memory_to_active": 0.2700555920600891, "mix_std": 0.10098867118358612, "beta_std": 0.057474441826343536, "beta_low": 0.13861083984375, "beta_high": 0.0, "orthogonal_to_active": 0.10724148154258728, "evidence_weights": [0.02375965006649494, -0.06954076141119003, 0.38938117027282715], "agreement_weight": 0.121895432472229, "orthogonal_weight": -0.5092936754226685}, {"head": 3, "memory_to_active": 1.8814866542816162, "contribution_to_active": 0.13566823303699493, "mix_mean": 0.03200923651456833, "mix_low": 0.39696693420410156, "mix_high": 0.0, "read_cosine": 0.22606784105300903, "beta_mean": 0.13209322094917297, "nonfinite": 0.0, "weighted_memory_to_active": 0.18050608038902283, "mix_std": 0.0527212880551815, "beta_std": 0.12059132009744644, "beta_low": 0.02532958984375, "beta_high": 0.0, "orthogonal_to_active": 0.003100199857726693, "evidence_weights": [-0.024164684116840363, 0.04746849462389946, -0.2571481764316559], "agreement_weight": 0.07914422452449799, "orthogonal_weight": 0.024620629847049713}, {"head": 4, "memory_to_active": 0.009212843142449856, "contribution_to_active": 0.6365705728530884, "mix_mean": 0.7576496601104736, "mix_low": 1.239776611328125e-05, "mix_high": 0.01903247833251953, "read_cosine": 0.12321397662162781, "beta_mean": 1.863715806393884e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006968243978917599, "mix_std": 0.20669162273406982, "beta_std": 6.602021312573925e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004434832837432623, "evidence_weights": [-0.32525041699409485, -0.3134108781814575, 0.2342660129070282], "agreement_weight": 0.3226027488708496, "orthogonal_weight": -0.6721239686012268}, {"head": 5, "memory_to_active": 2.491044282913208, "contribution_to_active": 0.23103290796279907, "mix_mean": 0.013432437554001808, "mix_low": 0.6450128555297852, "mix_high": 4.76837158203125e-06, "read_cosine": 0.2328340858221054, "beta_mean": 0.19909314811229706, "nonfinite": 0.0, "weighted_memory_to_active": 0.14461904764175415, "mix_std": 0.02599009871482849, "beta_std": 0.15418699383735657, "beta_low": 0.0, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.10985204577445984, "evidence_weights": [0.05644936487078667, -0.01778215728700161, -0.2589312493801117], "agreement_weight": -0.09278010576963425, "orthogonal_weight": 1.1254968643188477}]}, {"layer": 5, "memory_to_active": 0.3513571321964264, "contribution_to_active": 0.5460363626480103, "mix_mean": 0.5232385396957397, "mix_low": 0.05529928579926491, "mix_high": 0.020250003784894943, "read_cosine": 0.08669695258140564, "beta_mean": 0.04934900254011154, "nonfinite": 0.0, "weighted_memory_to_active": 0.12354087084531784, "mix_std": 0.2495061308145523, "beta_std": 0.06399956345558167, "beta_low": 0.6957194209098816, "beta_high": 0.0, "orthogonal_to_active": 0.05412645637989044, "output_influence": 0.5964449644088745, "residual_influence": 0.20639851689338684, "state_active": 0.038554273545742035, "state_memory": 0.009196199476718903, "state_peak": 63.84632110595703, "growth_active": 1.0979636907577515, "growth_memory": 1.2440907955169678, "heads": [{"head": 0, "memory_to_active": 0.16022413969039917, "contribution_to_active": 0.5848620533943176, "mix_mean": 0.5122239589691162, "mix_low": 0.005550384521484375, "mix_high": 0.00225830078125, "read_cosine": -0.1221398264169693, "beta_mean": 0.013148671947419643, "nonfinite": 0.0, "weighted_memory_to_active": 0.05610806494951248, "mix_std": 0.27496352791786194, "beta_std": 0.02433602139353752, "beta_low": 0.64691162109375, "beta_high": 0.0, "orthogonal_to_active": 0.025312455371022224, "evidence_weights": [-0.16922229528427124, -0.18776707351207733, 0.12229068577289581], "agreement_weight": -0.024165458977222443, "orthogonal_weight": -0.47598379850387573}, {"head": 1, "memory_to_active": 0.003350584302097559, "contribution_to_active": 0.5613414645195007, "mix_mean": 0.7114712595939636, "mix_low": 0.0006256103515625, "mix_high": 0.0389862060546875, "read_cosine": 0.23612771928310394, "beta_mean": 7.406441000057384e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0021319244988262653, "mix_std": 0.26375842094421387, "beta_std": 1.660273119341582e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019624708220362663, "evidence_weights": [-0.24453198909759521, -0.28401944041252136, 0.9426496624946594], "agreement_weight": 0.7390313744544983, "orthogonal_weight": -0.9734899997711182}, {"head": 2, "memory_to_active": 0.014129213988780975, "contribution_to_active": 0.6138378381729126, "mix_mean": 0.6669162511825562, "mix_low": 0.0018243789672851562, "mix_high": 0.020421981811523438, "read_cosine": -0.4330882430076599, "beta_mean": 7.746065239189193e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009052561596035957, "mix_std": 0.2587295174598694, "beta_std": 0.00032632582588121295, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.008752438239753246, "evidence_weights": [-0.25529271364212036, -0.3095855414867401, 0.1807929128408432], "agreement_weight": -0.2062070369720459, "orthogonal_weight": -1.0625591278076172}, {"head": 3, "memory_to_active": 1.4597221612930298, "contribution_to_active": 0.24130217730998993, "mix_mean": 0.11551456898450851, "mix_low": 0.11766529083251953, "mix_high": 0.0, "read_cosine": 0.15273040533065796, "beta_mean": 0.09289802610874176, "nonfinite": 0.0, "weighted_memory_to_active": 0.36782610416412354, "mix_std": 0.1453838348388672, "beta_std": 0.1430651694536209, "beta_low": 0.21063232421875, "beta_high": 0.0, "orthogonal_to_active": 0.14995354413986206, "evidence_weights": [-0.07517272979021072, -0.09728831052780151, 0.1302335560321808], "agreement_weight": 0.04983043670654297, "orthogonal_weight": -0.4974173605442047}, {"head": 4, "memory_to_active": 0.40481698513031006, "contribution_to_active": 0.5821878910064697, "mix_mean": 0.32128703594207764, "mix_low": 0.2046346664428711, "mix_high": 0.004326820373535156, "read_cosine": 0.5397096872329712, "beta_mean": 0.1897861510515213, "nonfinite": 0.0, "weighted_memory_to_active": 0.2607996463775635, "mix_std": 0.3363618552684784, "beta_std": 0.21532368659973145, "beta_low": 0.31903076171875, "beta_high": 0.0, "orthogonal_to_active": 0.1127135306596756, "evidence_weights": [-0.11192964017391205, -0.25110533833503723, -0.06812459230422974], "agreement_weight": 0.030225414782762527, "orthogonal_weight": 0.6704161167144775}, {"head": 5, "memory_to_active": 0.06589950621128082, "contribution_to_active": 0.6926865577697754, "mix_mean": 0.8120180368423462, "mix_low": 0.001495361328125, "mix_high": 0.05550670623779297, "read_cosine": 0.146841898560524, "beta_mean": 0.00017627933993935585, "nonfinite": 0.0, "weighted_memory_to_active": 0.04532691463828087, "mix_std": 0.21783959865570068, "beta_std": 0.0009295344352722168, "beta_low": 0.99786376953125, "beta_high": 0.0, "orthogonal_to_active": 0.026064280420541763, "evidence_weights": [-0.3001110255718231, -0.3234710097312927, 1.809713363647461], "agreement_weight": 0.5569530725479126, "orthogonal_weight": -0.6031768321990967}]}, {"layer": 6, "memory_to_active": 0.2304561883211136, "contribution_to_active": 0.6573867797851562, "mix_mean": 0.6027097105979919, "mix_low": 0.030356885865330696, "mix_high": 0.029415449127554893, "read_cosine": 0.31634262204170227, "beta_mean": 0.06062394380569458, "nonfinite": 0.0, "weighted_memory_to_active": 0.12399666756391525, "mix_std": 0.23889532685279846, "beta_std": 0.07267209887504578, "beta_low": 0.63433837890625, "beta_high": 0.0, "orthogonal_to_active": 0.09485242515802383, "output_influence": 0.5997086763381958, "residual_influence": 0.230926975607872, "state_active": 0.04064527899026871, "state_memory": 0.005065558943897486, "state_peak": 87.13285064697266, "growth_active": 1.1899900436401367, "growth_memory": 1.099897027015686, "heads": [{"head": 0, "memory_to_active": 0.11361247301101685, "contribution_to_active": 0.7850967645645142, "mix_mean": 0.8302654027938843, "mix_low": 0.00015544891357421875, "mix_high": 0.06925773620605469, "read_cosine": 0.30606111884117126, "beta_mean": 0.010633560828864574, "nonfinite": 0.0, "weighted_memory_to_active": 0.10198549926280975, "mix_std": 0.19747905433177948, "beta_std": 0.03499674052000046, "beta_low": 0.85723876953125, "beta_high": 0.0, "orthogonal_to_active": 0.06535164266824722, "evidence_weights": [-0.2915857136249542, -0.3411070704460144, 1.1657207012176514], "agreement_weight": 0.9583890438079834, "orthogonal_weight": -0.6932470202445984}, {"head": 1, "memory_to_active": 0.0031943535432219505, "contribution_to_active": 0.5284352898597717, "mix_mean": 0.6112110018730164, "mix_low": 0.00033283233642578125, "mix_high": 0.017557144165039062, "read_cosine": 0.4888366162776947, "beta_mean": 6.870924607937923e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0015148125821724534, "mix_std": 0.27883684635162354, "beta_std": 1.421817341906717e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0007929992862045765, "evidence_weights": [-0.1876247674226761, -0.2992106080055237, 1.0935859680175781], "agreement_weight": 0.1390191614627838, "orthogonal_weight": -0.6873287558555603}, {"head": 2, "memory_to_active": 0.6016958951950073, "contribution_to_active": 0.11067377775907516, "mix_mean": 0.06333626806735992, "mix_low": 0.17521190643310547, "mix_high": 1.9073486328125e-06, "read_cosine": 0.28732410073280334, "beta_mean": 0.017545318230986595, "nonfinite": 0.0, "weighted_memory_to_active": 0.07038015127182007, "mix_std": 0.08970142155885696, "beta_std": 0.021163148805499077, "beta_low": 0.413330078125, "beta_high": 0.0, "orthogonal_to_active": 0.014358525164425373, "evidence_weights": [0.05322574824094772, -0.0761357918381691, 0.3855225741863251], "agreement_weight": 0.13663528859615326, "orthogonal_weight": -0.24502989649772644}, {"head": 3, "memory_to_active": 0.13173595070838928, "contribution_to_active": 0.682583749294281, "mix_mean": 0.6706490516662598, "mix_low": 0.005253791809082031, "mix_high": 0.02114582061767578, "read_cosine": 0.446496844291687, "beta_mean": 0.21076750755310059, "nonfinite": 0.0, "weighted_memory_to_active": 0.09446268528699875, "mix_std": 0.32387104630470276, "beta_std": 0.24224358797073364, "beta_low": 0.215087890625, "beta_high": 0.0, "orthogonal_to_active": 0.06046875938773155, "evidence_weights": [-0.16119185090065002, -0.3070017993450165, 0.09741610288619995], "agreement_weight": -0.009757411666214466, "orthogonal_weight": 0.7249564528465271}, {"head": 4, "memory_to_active": 0.5147186517715454, "contribution_to_active": 1.1859043836593628, "mix_mean": 0.7374275326728821, "mix_low": 0.0010385513305664062, "mix_high": 0.05975341796875, "read_cosine": 0.08652448654174805, "beta_mean": 0.12471050769090652, "nonfinite": 0.0, "weighted_memory_to_active": 0.4624502956867218, "mix_std": 0.3066006600856781, "beta_std": 0.13740792870521545, "beta_low": 0.3204345703125, "beta_high": 0.0, "orthogonal_to_active": 0.41544032096862793, "evidence_weights": [-0.15206575393676758, -0.4955770671367645, -0.8178468346595764], "agreement_weight": -0.015332498587667942, "orthogonal_weight": 0.916660726070404}, {"head": 5, "memory_to_active": 0.017779700458049774, "contribution_to_active": 0.6516261696815491, "mix_mean": 0.7033688426017761, "mix_low": 0.000148773193359375, "mix_high": 0.008776664733886719, "read_cosine": 0.2828124165534973, "beta_mean": 7.990480662556365e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013186517171561718, "mix_std": 0.23688280582427979, "beta_std": 0.00020697647414635867, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01270224153995514, "evidence_weights": [-0.2296825796365738, -0.3128318786621094, 0.35331037640571594], "agreement_weight": 0.241072416305542, "orthogonal_weight": -1.0325664281845093}]}, {"layer": 7, "memory_to_active": 1.802804708480835, "contribution_to_active": 0.2751542925834656, "mix_mean": 0.18299201130867004, "mix_low": 0.21067985892295837, "mix_high": 0.0009848277550190687, "read_cosine": 0.2516257166862488, "beta_mean": 0.1384304314851761, "nonfinite": 0.0, "weighted_memory_to_active": 0.19969996809959412, "mix_std": 0.1485845148563385, "beta_std": 0.09866345673799515, "beta_low": 0.229827880859375, "beta_high": 0.000274658203125, "orthogonal_to_active": 0.08849263191223145, "output_influence": 0.2981144189834595, "residual_influence": 0.07790388911962509, "state_active": 0.014791548252105713, "state_memory": 0.023330798372626305, "state_peak": 35.0481071472168, "growth_active": 1.1854090690612793, "growth_memory": 1.2581835985183716, "heads": [{"head": 0, "memory_to_active": 2.8661694526672363, "contribution_to_active": 0.1559085100889206, "mix_mean": 0.03797721490263939, "mix_low": 0.4154815673828125, "mix_high": 9.5367431640625e-07, "read_cosine": 0.35490891337394714, "beta_mean": 0.18790845572948456, "nonfinite": 0.0, "weighted_memory_to_active": 0.17956730723381042, "mix_std": 0.0760214775800705, "beta_std": 0.13657723367214203, "beta_low": 0.000244140625, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.0003522561746649444, "evidence_weights": [0.047254785895347595, -0.09002181887626648, 0.028932694345712662], "agreement_weight": -0.02752665989100933, "orthogonal_weight": -0.002104134764522314}, {"head": 1, "memory_to_active": 0.9422740936279297, "contribution_to_active": 0.38489246368408203, "mix_mean": 0.22636368870735168, "mix_low": 0.11140251159667969, "mix_high": 0.004973411560058594, "read_cosine": 0.42019224166870117, "beta_mean": 0.09320226311683655, "nonfinite": 0.0, "weighted_memory_to_active": 0.32755011320114136, "mix_std": 0.2454284280538559, "beta_std": 0.06383173167705536, "beta_low": 0.0001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.04381006211042404, "evidence_weights": [-0.21869397163391113, -0.10566941648721695, -0.30620020627975464], "agreement_weight": 0.05712215602397919, "orthogonal_weight": 0.1629979908466339}, {"head": 2, "memory_to_active": 0.561245322227478, "contribution_to_active": 0.4061693251132965, "mix_mean": 0.25005877017974854, "mix_low": 0.013118743896484375, "mix_high": 0.00040912628173828125, "read_cosine": 0.10394526273012161, "beta_mean": 0.03068455308675766, "nonfinite": 0.0, "weighted_memory_to_active": 0.22935664653778076, "mix_std": 0.21850846707820892, "beta_std": 0.03748282790184021, "beta_low": 0.2369384765625, "beta_high": 0.0, "orthogonal_to_active": 0.1379617601633072, "evidence_weights": [-0.09980572760105133, -0.1876441091299057, 0.8888975977897644], "agreement_weight": 0.01187163032591343, "orthogonal_weight": -0.6205644011497498}, {"head": 3, "memory_to_active": 4.852757453918457, "contribution_to_active": 0.10377154499292374, "mix_mean": 0.037889450788497925, "mix_low": 0.28920650482177734, "mix_high": 0.0, "read_cosine": 0.10574851930141449, "beta_mean": 0.45329415798187256, "nonfinite": 0.0, "weighted_memory_to_active": 0.3702778220176697, "mix_std": 0.05257337540388107, "beta_std": 0.27988219261169434, "beta_low": 0.00115966796875, "beta_high": 0.0010986328125, "orthogonal_to_active": 0.31852173805236816, "evidence_weights": [0.07173966616392136, -0.048165444284677505, 0.7069230675697327], "agreement_weight": 0.04774782806634903, "orthogonal_weight": -0.934339702129364}, {"head": 4, "memory_to_active": 1.5670626163482666, "contribution_to_active": 0.11337387561798096, "mix_mean": 0.028492238372564316, "mix_low": 0.4334421157836914, "mix_high": 0.0, "read_cosine": 0.2643013596534729, "beta_mean": 0.06537818163633347, "nonfinite": 0.0, "weighted_memory_to_active": 0.07753371447324753, "mix_std": 0.050335004925727844, "beta_std": 0.07392612844705582, "beta_low": 0.14056396484375, "beta_high": 0.0, "orthogonal_to_active": 0.01721259206533432, "evidence_weights": [0.10488348454236984, -0.03888047859072685, 0.19349904358386993], "agreement_weight": 0.030872507020831108, "orthogonal_weight": -0.28256699442863464}, {"head": 5, "memory_to_active": 0.0273200161755085, "contribution_to_active": 0.4868101477622986, "mix_mean": 0.5171706676483154, "mix_low": 0.0014276504516601562, "mix_high": 0.0005254745483398438, "read_cosine": 0.26065805554389954, "beta_mean": 0.00011493921920191497, "nonfinite": 0.0, "weighted_memory_to_active": 0.013914153911173344, "mix_std": 0.2486402988433838, "beta_std": 0.00028059276519343257, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0130973681807518, "evidence_weights": [-0.18694286048412323, -0.24280521273612976, 0.4711242616176605], "agreement_weight": 0.2540276050567627, "orthogonal_weight": -1.0013662576675415}]}, {"layer": 8, "memory_to_active": 0.1396963745355606, "contribution_to_active": 0.5732444524765015, "mix_mean": 0.515055239200592, "mix_low": 0.009567579254508018, "mix_high": 0.002032756805419922, "read_cosine": 0.3904472887516022, "beta_mean": 0.012701265513896942, "nonfinite": 0.0, "weighted_memory_to_active": 0.0701608806848526, "mix_std": 0.24158348143100739, "beta_std": 0.019587256014347076, "beta_low": 0.7847290635108948, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.0618380606174469, "output_influence": 0.5334012508392334, "residual_influence": 0.2106829136610031, "state_active": 0.04233293607831001, "state_memory": 0.00448951032012701, "state_peak": 68.58998107910156, "growth_active": 1.1284456253051758, "growth_memory": 1.0270940065383911, "heads": [{"head": 0, "memory_to_active": 0.05899981036782265, "contribution_to_active": 0.5763267278671265, "mix_mean": 0.6130361557006836, "mix_low": 0.00040340423583984375, "mix_high": 0.0015621185302734375, "read_cosine": 0.4462277293205261, "beta_mean": 0.0033094349782913923, "nonfinite": 0.0, "weighted_memory_to_active": 0.04283207654953003, "mix_std": 0.248274564743042, "beta_std": 0.014334005303680897, "beta_low": 0.93341064453125, "beta_high": 0.0, "orthogonal_to_active": 0.026851866394281387, "evidence_weights": [-0.1489202231168747, -0.37168896198272705, 0.5522887110710144], "agreement_weight": 0.2703407406806946, "orthogonal_weight": -0.7916897535324097}, {"head": 1, "memory_to_active": 0.06507954001426697, "contribution_to_active": 0.553494930267334, "mix_mean": 0.5341534614562988, "mix_low": 0.00011539459228515625, "mix_high": 0.0021343231201171875, "read_cosine": 0.08131805807352066, "beta_mean": 0.0008445230778306723, "nonfinite": 0.0, "weighted_memory_to_active": 0.02887938730418682, "mix_std": 0.23055359721183777, "beta_std": 0.00808902271091938, "beta_low": 0.99346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.021542202681303024, "evidence_weights": [-0.18994168937206268, -0.2694452702999115, 0.5689172148704529], "agreement_weight": 0.07263673841953278, "orthogonal_weight": -0.7797406315803528}, {"head": 2, "memory_to_active": 0.008166980929672718, "contribution_to_active": 0.7011567950248718, "mix_mean": 0.6927176117897034, "mix_low": 0.00024700164794921875, "mix_high": 0.004123687744140625, "read_cosine": 0.4246659278869629, "beta_mean": 8.92401731107384e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006557649467140436, "mix_std": 0.21762889623641968, "beta_std": 0.002968671265989542, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0037669457960873842, "evidence_weights": [-0.21588508784770966, -0.3422001004219055, 0.9822943806648254], "agreement_weight": 0.3601643741130829, "orthogonal_weight": -0.6619415283203125}, {"head": 3, "memory_to_active": 0.10266654193401337, "contribution_to_active": 0.6357074975967407, "mix_mean": 0.5940368175506592, "mix_low": 0.0006170272827148438, "mix_high": 0.004180908203125, "read_cosine": 0.7931997776031494, "beta_mean": 0.0077611664310097694, "nonfinite": 0.0, "weighted_memory_to_active": 0.07657168805599213, "mix_std": 0.2542803883552551, "beta_std": 0.007920154370367527, "beta_low": 0.73223876953125, "beta_high": 0.0, "orthogonal_to_active": 0.023841962218284607, "evidence_weights": [-0.1640738993883133, -0.29380813241004944, 0.5182899832725525], "agreement_weight": 0.2696375548839569, "orthogonal_weight": -0.5848242044448853}, {"head": 4, "memory_to_active": 0.27355995774269104, "contribution_to_active": 0.368553102016449, "mix_mean": 0.3850550055503845, "mix_low": 0.0011854171752929688, "mix_high": 1.049041748046875e-05, "read_cosine": 0.09266266226768494, "beta_mean": 0.0007110610604286194, "nonfinite": 0.0, "weighted_memory_to_active": 0.09837465733289719, "mix_std": 0.23625749349594116, "beta_std": 0.014694912359118462, "beta_low": 0.99285888671875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.017397496849298477, "evidence_weights": [-0.15002122521400452, -0.2068924903869629, 0.6744470000267029], "agreement_weight": 0.029366515576839447, "orthogonal_weight": -0.17871011793613434}, {"head": 5, "memory_to_active": 0.3297054171562195, "contribution_to_active": 0.6042274832725525, "mix_mean": 0.2713322043418884, "mix_low": 0.05483722686767578, "mix_high": 0.0001850128173828125, "read_cosine": 0.5046096444129944, "beta_mean": 0.06349216401576996, "nonfinite": 0.0, "weighted_memory_to_active": 0.16774982213974, "mix_std": 0.2625059485435486, "beta_std": 0.06951676309108734, "beta_low": 0.05645751953125, "beta_high": 0.0, "orthogonal_to_active": 0.2776278555393219, "evidence_weights": [-0.15277965366840363, -0.14147226512432098, -0.10735008120536804], "agreement_weight": -0.010178493335843086, "orthogonal_weight": 1.9743118286132812}]}, {"layer": 9, "memory_to_active": 0.8436805009841919, "contribution_to_active": 0.753448486328125, "mix_mean": 0.4081849455833435, "mix_low": 0.046733222901821136, "mix_high": 0.0067043304443359375, "read_cosine": 0.3498428761959076, "beta_mean": 0.031828995794057846, "nonfinite": 0.0, "weighted_memory_to_active": 0.3937530815601349, "mix_std": 0.24912309646606445, "beta_std": 0.026467066258192062, "beta_low": 0.675506591796875, "beta_high": 0.0, "orthogonal_to_active": 0.02962474897503853, "output_influence": 0.5757322311401367, "residual_influence": 0.18534225225448608, "state_active": 0.04476712644100189, "state_memory": 0.01320380624383688, "state_peak": 60.256256103515625, "growth_active": 1.189135193824768, "growth_memory": 1.2227115631103516, "heads": [{"head": 0, "memory_to_active": 0.00024045727332122624, "contribution_to_active": 0.5870997905731201, "mix_mean": 0.6629594564437866, "mix_low": 0.0009031295776367188, "mix_high": 0.027055740356445312, "read_cosine": 0.35373952984809875, "beta_mean": 5.530544058274245e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.00016342112212441862, "mix_std": 0.28143373131752014, "beta_std": 1.5895795968390303e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00015860928397160023, "evidence_weights": [-0.30243417620658875, -0.3071924149990082, 0.3201993703842163], "agreement_weight": 0.10639294981956482, "orthogonal_weight": -1.0301886796951294}, {"head": 1, "memory_to_active": 0.01653985306620598, "contribution_to_active": 0.7285560369491577, "mix_mean": 0.5876284837722778, "mix_low": 0.006712913513183594, "mix_high": 0.0032072067260742188, "read_cosine": 0.4392320513725281, "beta_mean": 3.4796878026099876e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012840483337640762, "mix_std": 0.27695992588996887, "beta_std": 7.452482532244176e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007916608825325966, "evidence_weights": [-0.2340334802865982, -0.29008644819259644, 0.938157320022583], "agreement_weight": 0.5785567164421082, "orthogonal_weight": -0.766535758972168}, {"head": 2, "memory_to_active": 0.15920917689800262, "contribution_to_active": 0.6682617664337158, "mix_mean": 0.6075355410575867, "mix_low": 0.0018558502197265625, "mix_high": 0.007647514343261719, "read_cosine": 0.3565283417701721, "beta_mean": 0.0005192427197471261, "nonfinite": 0.0, "weighted_memory_to_active": 0.10884161293506622, "mix_std": 0.26357901096343994, "beta_std": 0.004410390742123127, "beta_low": 0.996826171875, "beta_high": 0.0, "orthogonal_to_active": 0.0613400936126709, "evidence_weights": [-0.1977180391550064, -0.27343496680259705, 0.746931254863739], "agreement_weight": 0.37326422333717346, "orthogonal_weight": -0.6613147854804993}, {"head": 3, "memory_to_active": 0.6631314754486084, "contribution_to_active": 0.2987462878227234, "mix_mean": 0.1965317726135254, "mix_low": 0.06782245635986328, "mix_high": 4.38690185546875e-05, "read_cosine": 0.37138357758522034, "beta_mean": 0.02192509174346924, "nonfinite": 0.0, "weighted_memory_to_active": 0.1842224895954132, "mix_std": 0.21708229184150696, "beta_std": 0.024677960202097893, "beta_low": 0.348388671875, "beta_high": 0.0, "orthogonal_to_active": 0.09193968772888184, "evidence_weights": [-0.05006072670221329, -0.12250864505767822, 0.5439391136169434], "agreement_weight": 0.0931834876537323, "orthogonal_weight": -0.5557745695114136}, {"head": 4, "memory_to_active": 3.225346565246582, "contribution_to_active": 1.8975750207901, "mix_mean": 0.15109215676784515, "mix_low": 0.12006855010986328, "mix_high": 0.0013246536254882812, "read_cosine": 0.11389431357383728, "beta_mean": 0.15016084909439087, "nonfinite": 0.0, "weighted_memory_to_active": 1.7889173030853271, "mix_std": 0.21140283346176147, "beta_std": 0.08346962928771973, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 8.505704317940399e-05, "evidence_weights": [-0.07828608900308609, -0.14752432703971863, -0.048294760286808014], "agreement_weight": -0.06842547655105591, "orthogonal_weight": -4.909576819045469e-05}, {"head": 5, "memory_to_active": 0.9976151585578918, "contribution_to_active": 0.3404518961906433, "mix_mean": 0.24336211383342743, "mix_low": 0.08303642272949219, "mix_high": 0.0009469985961914062, "read_cosine": 0.46427929401397705, "beta_mean": 0.01833343505859375, "nonfinite": 0.0, "weighted_memory_to_active": 0.2675331234931946, "mix_std": 0.24428072571754456, "beta_std": 0.046168290078639984, "beta_low": 0.70782470703125, "beta_high": 0.0, "orthogonal_to_active": 0.016308438032865524, "evidence_weights": [0.026956643909215927, -0.20278936624526978, 0.14638960361480713], "agreement_weight": 0.004713344853371382, "orthogonal_weight": 0.06774864345788956}]}, {"layer": 10, "memory_to_active": 0.20888207852840424, "contribution_to_active": 0.7740365266799927, "mix_mean": 0.5325465202331543, "mix_low": 0.024075031280517578, "mix_high": 0.005837440490722656, "read_cosine": 0.37916991114616394, "beta_mean": 0.04340355843305588, "nonfinite": 0.0, "weighted_memory_to_active": 0.08933176100254059, "mix_std": 0.2648911476135254, "beta_std": 0.044546257704496384, "beta_low": 0.664093017578125, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.30333542823791504, "output_influence": 0.4819347858428955, "residual_influence": 0.17922009527683258, "state_active": 0.053771357983350754, "state_memory": 0.01325032114982605, "state_peak": 122.4492416381836, "growth_active": 1.0143240690231323, "growth_memory": 0.9969042539596558, "heads": [{"head": 0, "memory_to_active": 0.784075140953064, "contribution_to_active": 2.3013389110565186, "mix_mean": 0.5691289305686951, "mix_low": 0.1185445785522461, "mix_high": 0.008839607238769531, "read_cosine": 0.07159674912691116, "beta_mean": 0.19039523601531982, "nonfinite": 0.0, "weighted_memory_to_active": 0.43963003158569336, "mix_std": 0.3985920548439026, "beta_std": 0.20095741748809814, "beta_low": 0.15496826171875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.7531673908233643, "evidence_weights": [-0.07897941023111343, -0.35700181126594543, -2.0067920684814453], "agreement_weight": 0.04203720763325691, "orthogonal_weight": 4.015294551849365}, {"head": 1, "memory_to_active": 0.00464135967195034, "contribution_to_active": 0.6418465375900269, "mix_mean": 0.6472787261009216, "mix_low": 0.000823974609375, "mix_high": 0.01821422576904297, "read_cosine": 0.32821738719940186, "beta_mean": 3.023384851985611e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00269639166072011, "mix_std": 0.2592158913612366, "beta_std": 5.5587865062989295e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0041459775529801846, "evidence_weights": [-0.24808159470558167, -0.1908254325389862, 0.5422565340995789], "agreement_weight": 0.6831569671630859, "orthogonal_weight": -1.6726588010787964}, {"head": 2, "memory_to_active": 0.010125505737960339, "contribution_to_active": 0.6229182481765747, "mix_mean": 0.6762301921844482, "mix_low": 0.000274658203125, "mix_high": 0.0064296722412109375, "read_cosine": 0.032463133335113525, "beta_mean": 2.1495350665645674e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004430096130818129, "mix_std": 0.24584676325321198, "beta_std": 7.1668517193757e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017754703294485807, "evidence_weights": [-0.2662552297115326, -0.2731558084487915, 0.7265321612358093], "agreement_weight": -0.23254090547561646, "orthogonal_weight": -0.3826964795589447}, {"head": 3, "memory_to_active": 0.337942898273468, "contribution_to_active": 0.09124888479709625, "mix_mean": 0.28187549114227295, "mix_low": 0.023168563842773438, "mix_high": 1.1444091796875e-05, "read_cosine": 0.9336503148078918, "beta_mean": 0.06338221579790115, "nonfinite": 0.0, "weighted_memory_to_active": 0.03508295863866806, "mix_std": 0.22179993987083435, "beta_std": 0.05706425756216049, "beta_low": 0.0029296875, "beta_high": 0.0, "orthogonal_to_active": 0.03489842265844345, "evidence_weights": [-0.08377579599618912, -0.13973583281040192, 0.14941097795963287], "agreement_weight": 0.017636021599173546, "orthogonal_weight": 1.4988245964050293}, {"head": 4, "memory_to_active": 0.00829989556223154, "contribution_to_active": 0.574817419052124, "mix_mean": 0.6031789779663086, "mix_low": 0.000194549560546875, "mix_high": 0.001522064208984375, "read_cosine": 0.08283351361751556, "beta_mean": 2.0193550881231204e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005002305377274752, "mix_std": 0.2400582879781723, "beta_std": 3.676165943033993e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007339823059737682, "evidence_weights": [-0.17996422946453094, -0.24743513762950897, 0.9104414582252502], "agreement_weight": 0.19311529397964478, "orthogonal_weight": -1.542162537574768}, {"head": 5, "memory_to_active": 0.10820755362510681, "contribution_to_active": 0.4120487570762634, "mix_mean": 0.4175868034362793, "mix_low": 0.0014438629150390625, "mix_high": 7.62939453125e-06, "read_cosine": 0.8262583017349243, "beta_mean": 0.0065719494596123695, "nonfinite": 0.0, "weighted_memory_to_active": 0.0491488017141819, "mix_std": 0.22383403778076172, "beta_std": 0.009091857820749283, "beta_low": 0.82666015625, "beta_high": 0.0, "orthogonal_to_active": 0.018685368821024895, "evidence_weights": [-0.13456988334655762, -0.1660502851009369, 0.27959495782852173], "agreement_weight": 0.09794578701257706, "orthogonal_weight": -0.6804265379905701}]}, {"layer": 11, "memory_to_active": 0.06761446595191956, "contribution_to_active": 0.6353710889816284, "mix_mean": 0.5336699485778809, "mix_low": 0.11639818549156189, "mix_high": 0.048933982849121094, "read_cosine": 0.34459227323532104, "beta_mean": 0.023067034780979156, "nonfinite": 0.0, "weighted_memory_to_active": 0.0373484268784523, "mix_std": 0.3222293555736542, "beta_std": 0.0336577482521534, "beta_low": 0.7266032695770264, "beta_high": 0.0, "orthogonal_to_active": 0.04009111225605011, "output_influence": 0.567232072353363, "residual_influence": 0.17095035314559937, "state_active": 0.21531176567077637, "state_memory": 0.002638256410136819, "state_peak": 36.465240478515625, "growth_active": 1.1028571128845215, "growth_memory": 1.4386682510375977, "heads": [{"head": 0, "memory_to_active": 0.006224914453923702, "contribution_to_active": 0.6099053621292114, "mix_mean": 0.6725831031799316, "mix_low": 0.000415802001953125, "mix_high": 0.008713722229003906, "read_cosine": 0.23611685633659363, "beta_mean": 6.147847307147458e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004579281434416771, "mix_std": 0.24704833328723907, "beta_std": 0.0028359838761389256, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.003784399712458253, "evidence_weights": [-0.17592792212963104, -0.35380905866622925, 0.7978562116622925], "agreement_weight": 0.4294584095478058, "orthogonal_weight": -0.8515772819519043}, {"head": 1, "memory_to_active": 0.10928668081760406, "contribution_to_active": 0.4546690881252289, "mix_mean": 0.4823312759399414, "mix_low": 0.004276275634765625, "mix_high": 0.0006055831909179688, "read_cosine": 0.47629401087760925, "beta_mean": 0.001758559257723391, "nonfinite": 0.0, "weighted_memory_to_active": 0.05641698092222214, "mix_std": 0.2679179310798645, "beta_std": 0.008106691762804985, "beta_low": 0.9678955078125, "beta_high": 0.0, "orthogonal_to_active": 0.03171813115477562, "evidence_weights": [-0.12982280552387238, -0.2785029113292694, 0.5978152751922607], "agreement_weight": 0.09029868990182877, "orthogonal_weight": -0.6642346382141113}, {"head": 2, "memory_to_active": 0.004764140583574772, "contribution_to_active": 0.9874954223632812, "mix_mean": 0.4370346665382385, "mix_low": 0.33786869049072266, "mix_high": 0.18348312377929688, "read_cosine": 0.15702876448631287, "beta_mean": 0.06313871592283249, "nonfinite": 0.0, "weighted_memory_to_active": 0.0033567468635737896, "mix_std": 0.4542355239391327, "beta_std": 0.08443634957075119, "beta_low": 0.50140380859375, "beta_high": 0.0, "orthogonal_to_active": 0.035827893763780594, "evidence_weights": [0.04703786224126816, -0.326846718788147, -1.3201013803482056], "agreement_weight": -1.1033844202756882e-05, "orthogonal_weight": 11.0907564163208}, {"head": 3, "memory_to_active": 0.006266910582780838, "contribution_to_active": 0.8826968669891357, "mix_mean": 0.41621896624565125, "mix_low": 0.34916210174560547, "mix_high": 0.0927743911743164, "read_cosine": 0.5826923251152039, "beta_mean": 0.051125116646289825, "nonfinite": 0.0, "weighted_memory_to_active": 0.004056441131979227, "mix_std": 0.44330155849456787, "beta_std": 0.07763996720314026, "beta_low": 0.537353515625, "beta_high": 0.0, "orthogonal_to_active": 0.029193997383117676, "evidence_weights": [0.22594794631004333, -0.26485592126846313, -0.06974993646144867], "agreement_weight": 0.04903936758637428, "orthogonal_weight": 11.371125221252441}, {"head": 4, "memory_to_active": 0.198579341173172, "contribution_to_active": 0.4485768675804138, "mix_mean": 0.6125360727310181, "mix_low": 0.005878448486328125, "mix_high": 0.00148773193359375, "read_cosine": 0.0157766230404377, "beta_mean": 0.015044474974274635, "nonfinite": 0.0, "weighted_memory_to_active": 0.1149282455444336, "mix_std": 0.2672242522239685, "beta_std": 0.01865728572010994, "beta_low": 0.53924560546875, "beta_high": 0.0, "orthogonal_to_active": 0.1134738102555275, "evidence_weights": [-0.22890841960906982, -0.23684625327587128, 1.8906917572021484], "agreement_weight": -0.23321561515331268, "orthogonal_weight": -0.9964666962623596}, {"head": 5, "memory_to_active": 0.08056479692459106, "contribution_to_active": 0.4288828372955322, "mix_mean": 0.5813153982162476, "mix_low": 0.0007877349853515625, "mix_high": 0.006539344787597656, "read_cosine": 0.5996450781822205, "beta_mean": 0.0072738537564873695, "nonfinite": 0.0, "weighted_memory_to_active": 0.04075285792350769, "mix_std": 0.25364840030670166, "beta_std": 0.010270214639604092, "beta_low": 0.814208984375, "beta_high": 0.0, "orthogonal_to_active": 0.026548434048891068, "evidence_weights": [-0.18971922993659973, -0.30599793791770935, 0.6675757765769958], "agreement_weight": 0.08354691416025162, "orthogonal_weight": -0.8644591569900513}]}, {"layer": 12, "memory_to_active": 0.2178005576133728, "contribution_to_active": 0.5953251719474792, "mix_mean": 0.5560641288757324, "mix_low": 0.008408705703914165, "mix_high": 0.0074512166902422905, "read_cosine": 0.2209940254688263, "beta_mean": 0.014663050882518291, "nonfinite": 0.0, "weighted_memory_to_active": 0.11034204810857773, "mix_std": 0.25633642077445984, "beta_std": 0.01545882411301136, "beta_low": 0.8290303945541382, "beta_high": 0.0, "orthogonal_to_active": 0.06632034480571747, "output_influence": 0.5295637845993042, "residual_influence": 0.16834545135498047, "state_active": 0.034841954708099365, "state_memory": 0.007250530645251274, "state_peak": 74.85856628417969, "growth_active": 1.186257004737854, "growth_memory": 1.1164374351501465, "heads": [{"head": 0, "memory_to_active": 0.025701982900500298, "contribution_to_active": 0.6382632255554199, "mix_mean": 0.6008628606796265, "mix_low": 0.00093841552734375, "mix_high": 0.0076084136962890625, "read_cosine": 0.2890890836715698, "beta_mean": 0.00042319318163208663, "nonfinite": 0.0, "weighted_memory_to_active": 0.014658326283097267, "mix_std": 0.2614539861679077, "beta_std": 0.0009284642292186618, "beta_low": 0.99896240234375, "beta_high": 0.0, "orthogonal_to_active": 0.013346858322620392, "evidence_weights": [-0.2008257508277893, -0.2702080011367798, -0.0962022989988327], "agreement_weight": 0.10103492438793182, "orthogonal_weight": -0.982399582862854}, {"head": 1, "memory_to_active": 0.06783229857683182, "contribution_to_active": 0.5797495245933533, "mix_mean": 0.5774955153465271, "mix_low": 0.000301361083984375, "mix_high": 0.0026121139526367188, "read_cosine": 0.4015507400035858, "beta_mean": 0.00037901190808042884, "nonfinite": 0.0, "weighted_memory_to_active": 0.04365137591958046, "mix_std": 0.2612802982330322, "beta_std": 0.006784331984817982, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.03574872389435768, "evidence_weights": [-0.19615846872329712, -0.2699235677719116, 1.03005051612854], "agreement_weight": 0.29441919922828674, "orthogonal_weight": -0.9058621525764465}, {"head": 2, "memory_to_active": 0.5885648727416992, "contribution_to_active": 0.46896809339523315, "mix_mean": 0.3118838667869568, "mix_low": 0.02569103240966797, "mix_high": 2.47955322265625e-05, "read_cosine": 0.5887904763221741, "beta_mean": 0.08585681021213531, "nonfinite": 0.0, "weighted_memory_to_active": 0.2221786230802536, "mix_std": 0.23059652745723724, "beta_std": 0.06344617903232574, "beta_low": 0.00177001953125, "beta_high": 0.0, "orthogonal_to_active": 0.2521842122077942, "evidence_weights": [-0.1996675282716751, -0.06184803694486618, -0.02555146999657154], "agreement_weight": 0.03602255508303642, "orthogonal_weight": 1.4666874408721924}, {"head": 3, "memory_to_active": 0.030391620472073555, "contribution_to_active": 0.7763142585754395, "mix_mean": 0.7324154376983643, "mix_low": 0.0001068115234375, "mix_high": 0.02292919158935547, "read_cosine": 0.07044874131679535, "beta_mean": 0.00027312952443026006, "nonfinite": 0.0, "weighted_memory_to_active": 0.022631283849477768, "mix_std": 0.22379428148269653, "beta_std": 0.0060593243688344955, "beta_low": 0.9981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.021373584866523743, "evidence_weights": [-0.2829338014125824, -0.329784095287323, 1.7090246677398682], "agreement_weight": 0.6487702131271362, "orthogonal_weight": -0.9637271165847778}, {"head": 4, "memory_to_active": 0.5000597238540649, "contribution_to_active": 0.6041712760925293, "mix_mean": 0.47294995188713074, "mix_low": 0.016661643981933594, "mix_high": 0.005156517028808594, "read_cosine": -0.22344596683979034, "beta_mean": 0.0005968152545392513, "nonfinite": 0.0, "weighted_memory_to_active": 0.29633498191833496, "mix_std": 0.289314866065979, "beta_std": 0.009052447974681854, "beta_low": 0.98638916015625, "beta_high": 0.0, "orthogonal_to_active": 0.02849123626947403, "evidence_weights": [-0.15536129474639893, -0.21267905831336975, -0.15726590156555176], "agreement_weight": 0.022694239392876625, "orthogonal_weight": -0.09936697781085968}, {"head": 5, "memory_to_active": 0.09425296634435654, "contribution_to_active": 0.5044845342636108, "mix_mean": 0.640777051448822, "mix_low": 0.006752967834472656, "mix_high": 0.0063762664794921875, "read_cosine": 0.19953098893165588, "beta_mean": 0.00044933712342754006, "nonfinite": 0.0, "weighted_memory_to_active": 0.06259768456220627, "mix_std": 0.27157852053642273, "beta_std": 0.006482190452516079, "beta_low": 0.990478515625, "beta_high": 0.0, "orthogonal_to_active": 0.04677741974592209, "evidence_weights": [-0.2180427461862564, -0.30797576904296875, 1.222226619720459], "agreement_weight": 0.4369666576385498, "orthogonal_weight": -0.8016532063484192}]}, {"layer": 13, "memory_to_active": 0.10998375713825226, "contribution_to_active": 0.5518894195556641, "mix_mean": 0.5527776479721069, "mix_low": 0.005794207565486431, "mix_high": 0.010533174499869347, "read_cosine": 0.4281424880027771, "beta_mean": 0.014032160863280296, "nonfinite": 0.0, "weighted_memory_to_active": 0.04810599982738495, "mix_std": 0.25541529059410095, "beta_std": 0.016095705330371857, "beta_low": 0.8104553818702698, "beta_high": 0.0, "orthogonal_to_active": 0.0492902547121048, "output_influence": 0.5489948987960815, "residual_influence": 0.20865073800086975, "state_active": 0.13267090916633606, "state_memory": 0.03952399641275406, "state_peak": 100.26785278320312, "growth_active": 0.9158235788345337, "growth_memory": 0.9620345234870911, "heads": [{"head": 0, "memory_to_active": 0.0060306088998913765, "contribution_to_active": 0.6192479133605957, "mix_mean": 0.629707396030426, "mix_low": 0.0008783340454101562, "mix_high": 0.004787445068359375, "read_cosine": 0.4471755027770996, "beta_mean": 2.0836883777519688e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004010848235338926, "mix_std": 0.26188305020332336, "beta_std": 4.8518824769416824e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0036608714144676924, "evidence_weights": [-0.18756212294101715, -0.26256853342056274, 0.7225585579872131], "agreement_weight": 0.2458503395318985, "orthogonal_weight": -1.0249594449996948}, {"head": 1, "memory_to_active": 0.007409284822642803, "contribution_to_active": 0.6185927391052246, "mix_mean": 0.5850052833557129, "mix_low": 0.0013856887817382812, "mix_high": 0.0049896240234375, "read_cosine": 0.2443881332874298, "beta_mean": 2.9822975193383172e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004127940628677607, "mix_std": 0.2724623382091522, "beta_std": 6.157909956527874e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005126584321260452, "evidence_weights": [-0.175913468003273, -0.2586926519870758, 0.5016596913337708], "agreement_weight": 0.6051570177078247, "orthogonal_weight": -1.2899476289749146}, {"head": 2, "memory_to_active": 0.06353078037500381, "contribution_to_active": 0.5087456703186035, "mix_mean": 0.4032403230667114, "mix_low": 0.0010967254638671875, "mix_high": 0.000156402587890625, "read_cosine": 0.6323728561401367, "beta_mean": 0.007977882400155067, "nonfinite": 0.0, "weighted_memory_to_active": 0.03165993466973305, "mix_std": 0.2299812138080597, "beta_std": 0.029575005173683167, "beta_low": 0.8670654296875, "beta_high": 0.0, "orthogonal_to_active": 0.01767469011247158, "evidence_weights": [-0.17263752222061157, -0.21400168538093567, 0.4092289209365845], "agreement_weight": 0.005599961616098881, "orthogonal_weight": -0.8543979525566101}, {"head": 3, "memory_to_active": 0.1784818172454834, "contribution_to_active": 0.5911530256271362, "mix_mean": 0.682466983795166, "mix_low": 0.024736404418945312, "mix_high": 0.03976726531982422, "read_cosine": 0.0075877052731812, "beta_mean": 0.000927415385376662, "nonfinite": 0.0, "weighted_memory_to_active": 0.13638824224472046, "mix_std": 0.28542560338974, "beta_std": 0.010925667360424995, "beta_low": 0.99493408203125, "beta_high": 0.0, "orthogonal_to_active": 0.12171509861946106, "evidence_weights": [-0.3162935972213745, -0.23974594473838806, 1.4472403526306152], "agreement_weight": 6.899858474731445, "orthogonal_weight": -0.9734373092651367}, {"head": 4, "memory_to_active": 0.37131819128990173, "contribution_to_active": 0.2405794858932495, "mix_mean": 0.34588438272476196, "mix_low": 0.004542350769042969, "mix_high": 0.0, "read_cosine": 0.9012635350227356, "beta_mean": 0.07509855180978775, "nonfinite": 0.0, "weighted_memory_to_active": 0.08940787613391876, "mix_std": 0.2149801403284073, "beta_std": 0.05459441989660263, "beta_low": 0.0009765625, "beta_high": 0.0, "orthogonal_to_active": 0.12819012999534607, "evidence_weights": [-0.05503927916288376, -0.21991346776485443, 0.41020217537879944], "agreement_weight": 0.008396890945732594, "orthogonal_weight": 2.3321850299835205}, {"head": 5, "memory_to_active": 0.03313180431723595, "contribution_to_active": 0.7330175042152405, "mix_mean": 0.6703612804412842, "mix_low": 0.0021257400512695312, "mix_high": 0.013498306274414062, "read_cosine": 0.33606716990470886, "beta_mean": 0.00013845099601894617, "nonfinite": 0.0, "weighted_memory_to_active": 0.02304115518927574, "mix_std": 0.2677592933177948, "beta_std": 0.001369047211483121, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.019374173134565353, "evidence_weights": [-0.2185257226228714, -0.2833460867404938, 0.8793097734451294], "agreement_weight": 0.22222992777824402, "orthogonal_weight": -0.9272913932800293}]}, {"layer": 14, "memory_to_active": 0.21822111308574677, "contribution_to_active": 0.4070740342140198, "mix_mean": 0.4244542121887207, "mix_low": 0.030405839905142784, "mix_high": 0.006216844543814659, "read_cosine": 0.6117444634437561, "beta_mean": 0.059230655431747437, "nonfinite": 0.0, "weighted_memory_to_active": 0.08657678961753845, "mix_std": 0.23442573845386505, "beta_std": 0.057671159505844116, "beta_low": 0.5124410390853882, "beta_high": 0.0, "orthogonal_to_active": 0.04044021666049957, "output_influence": 0.3445628881454468, "residual_influence": 0.15792560577392578, "state_active": 0.051253724843263626, "state_memory": 0.012869937345385551, "state_peak": 54.366825103759766, "growth_active": 1.1108063459396362, "growth_memory": 1.2378283739089966, "heads": [{"head": 0, "memory_to_active": 0.35014620423316956, "contribution_to_active": 0.18099230527877808, "mix_mean": 0.1736762821674347, "mix_low": 0.022643089294433594, "mix_high": 0.0, "read_cosine": 0.8082855343818665, "beta_mean": 0.06713354587554932, "nonfinite": 0.0, "weighted_memory_to_active": 0.06876315176486969, "mix_std": 0.164282888174057, "beta_std": 0.06027491018176079, "beta_low": 0.00823974609375, "beta_high": 0.0, "orthogonal_to_active": 0.08234899491071701, "evidence_weights": [-0.09139732271432877, -0.04404617100954056, 0.3521047830581665], "agreement_weight": 0.02075241133570671, "orthogonal_weight": 1.9521509408950806}, {"head": 1, "memory_to_active": 0.4591933488845825, "contribution_to_active": 0.2695618271827698, "mix_mean": 0.18469378352165222, "mix_low": 0.14807605743408203, "mix_high": 0.0, "read_cosine": 0.8874610066413879, "beta_mean": 0.24660253524780273, "nonfinite": 0.0, "weighted_memory_to_active": 0.19258931279182434, "mix_std": 0.21938779950141907, "beta_std": 0.2445715367794037, "beta_low": 0.09344482421875, "beta_high": 0.0, "orthogonal_to_active": 0.020086193457245827, "evidence_weights": [0.017493056133389473, -0.18605825304985046, -0.100619837641716], "agreement_weight": 0.05774718523025513, "orthogonal_weight": 0.3462328612804413}, {"head": 2, "memory_to_active": 0.1255081743001938, "contribution_to_active": 0.5811440944671631, "mix_mean": 0.5914744734764099, "mix_low": 0.00238037109375, "mix_high": 0.0054912567138671875, "read_cosine": 0.4020225703716278, "beta_mean": 0.0018140344182029366, "nonfinite": 0.0, "weighted_memory_to_active": 0.08004377037286758, "mix_std": 0.26408466696739197, "beta_std": 0.0025087553076446056, "beta_low": 0.98291015625, "beta_high": 0.0, "orthogonal_to_active": 0.06156571954488754, "evidence_weights": [-0.12423792481422424, -0.3133065700531006, 0.8973739743232727], "agreement_weight": 0.08630901575088501, "orthogonal_weight": -0.8700296878814697}, {"head": 3, "memory_to_active": 0.03798164799809456, "contribution_to_active": 0.577527642250061, "mix_mean": 0.6795024275779724, "mix_low": 0.0034971237182617188, "mix_high": 0.02306652069091797, "read_cosine": -0.06981701403856277, "beta_mean": 0.0046118092723190784, "nonfinite": 0.0, "weighted_memory_to_active": 0.023966945707798004, "mix_std": 0.2540871500968933, "beta_std": 0.0072132013738155365, "beta_low": 0.87835693359375, "beta_high": 0.0, "orthogonal_to_active": 0.018479324877262115, "evidence_weights": [-0.33804768323898315, -0.31143075227737427, 0.3092406392097473], "agreement_weight": 0.06952422857284546, "orthogonal_weight": -0.8378927111625671}, {"head": 4, "memory_to_active": 0.14545942842960358, "contribution_to_active": 0.4929410219192505, "mix_mean": 0.5337417721748352, "mix_low": 0.0030345916748046875, "mix_high": 0.007853507995605469, "read_cosine": 0.9079605937004089, "beta_mean": 0.034289367496967316, "nonfinite": 0.0, "weighted_memory_to_active": 0.08377007395029068, "mix_std": 0.256521075963974, "beta_std": 0.02558712288737297, "beta_low": 0.126953125, "beta_high": 0.0, "orthogonal_to_active": 0.022773608565330505, "evidence_weights": [-0.20092831552028656, -0.2344197779893875, 0.4518057405948639], "agreement_weight": 0.13357973098754883, "orthogonal_weight": -0.6255228519439697}, {"head": 5, "memory_to_active": 0.19103790819644928, "contribution_to_active": 0.3402773141860962, "mix_mean": 0.3836366534233093, "mix_low": 0.002803802490234375, "mix_high": 0.0008897781372070312, "read_cosine": 0.7345540523529053, "beta_mean": 0.0009326463914476335, "nonfinite": 0.0, "weighted_memory_to_active": 0.07032746821641922, "mix_std": 0.2481907159090042, "beta_std": 0.00587142538279295, "beta_low": 0.9847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.03738745301961899, "evidence_weights": [-0.17153646051883698, -0.18461361527442932, 0.37781471014022827], "agreement_weight": 0.05965343490242958, "orthogonal_weight": -0.7609143853187561}]}, {"layer": 15, "memory_to_active": 0.048159319907426834, "contribution_to_active": 0.5049264430999756, "mix_mean": 0.4424159526824951, "mix_low": 0.005949338432401419, "mix_high": 0.003046035999432206, "read_cosine": 0.6602427363395691, "beta_mean": 0.037783343344926834, "nonfinite": 0.0, "weighted_memory_to_active": 0.025224540382623672, "mix_std": 0.2823689579963684, "beta_std": 0.04246317595243454, "beta_low": 0.7046407461166382, "beta_high": 0.0, "orthogonal_to_active": 0.010425306856632233, "output_influence": 0.3312048316001892, "residual_influence": 0.2106119692325592, "state_active": 0.17490310966968536, "state_memory": 0.009696578606963158, "state_peak": 28.869632720947266, "growth_active": 1.1984491348266602, "growth_memory": 1.2432971000671387, "heads": [{"head": 0, "memory_to_active": 0.11818587779998779, "contribution_to_active": 0.4942024350166321, "mix_mean": 0.41337835788726807, "mix_low": 0.0054187774658203125, "mix_high": 0.006230354309082031, "read_cosine": 0.5206468105316162, "beta_mean": 0.054184440523386, "nonfinite": 0.0, "weighted_memory_to_active": 0.0631721168756485, "mix_std": 0.28731849789619446, "beta_std": 0.03815634176135063, "beta_low": 0.045166015625, "beta_high": 0.0, "orthogonal_to_active": 0.040104806423187256, "evidence_weights": [-0.1814662218093872, -0.19535978138446808, 0.5547425150871277], "agreement_weight": 0.08900842815637589, "orthogonal_weight": -0.8552345037460327}, {"head": 1, "memory_to_active": 0.07575622946023941, "contribution_to_active": 0.4334959089756012, "mix_mean": 0.45255187153816223, "mix_low": 0.000568389892578125, "mix_high": 0.0077056884765625, "read_cosine": 0.9290237426757812, "beta_mean": 0.000121696139103733, "nonfinite": 0.0, "weighted_memory_to_active": 0.03217438608407974, "mix_std": 0.2814705967903137, "beta_std": 0.00144061422906816, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.002989342203363776, "evidence_weights": [-0.16354435682296753, -0.17721104621887207, 0.15576347708702087], "agreement_weight": 0.10196340084075928, "orthogonal_weight": 0.1999734491109848}, {"head": 2, "memory_to_active": 0.006098038516938686, "contribution_to_active": 0.5311167240142822, "mix_mean": 0.36470332741737366, "mix_low": 0.00952911376953125, "mix_high": 6.771087646484375e-05, "read_cosine": 0.7912027835845947, "beta_mean": 7.635843212483451e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003426485462114215, "mix_std": 0.27303826808929443, "beta_std": 0.0002345836255699396, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.001160630490630865, "evidence_weights": [-0.19559191167354584, -0.26713046431541443, 0.12880688905715942], "agreement_weight": 0.008651582524180412, "orthogonal_weight": 0.9856621623039246}, {"head": 3, "memory_to_active": 0.0016728207701817155, "contribution_to_active": 0.37746185064315796, "mix_mean": 0.4223601818084717, "mix_low": 0.003765106201171875, "mix_high": 0.0005369186401367188, "read_cosine": 0.6353778839111328, "beta_mean": 8.58891144162044e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.000568102637771517, "mix_std": 0.28843432664871216, "beta_std": 2.0019564544782043e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00028364339959807694, "evidence_weights": [-0.27388930320739746, -0.239255890250206, 0.17827410995960236], "agreement_weight": 0.059241581708192825, "orthogonal_weight": -0.500529408454895}, {"head": 4, "memory_to_active": 0.05586130917072296, "contribution_to_active": 0.6387918591499329, "mix_mean": 0.40125572681427, "mix_low": 0.007930755615234375, "mix_high": 0.0008192062377929688, "read_cosine": 0.7134737372398376, "beta_mean": 0.16975471377372742, "nonfinite": 0.0, "weighted_memory_to_active": 0.03086932934820652, "mix_std": 0.3075180947780609, "beta_std": 0.19693467020988464, "beta_low": 0.21478271484375, "beta_high": 0.0, "orthogonal_to_active": 0.005758346989750862, "evidence_weights": [-0.2027607560157776, -0.24990734457969666, -0.028711741790175438], "agreement_weight": 0.011284742504358292, "orthogonal_weight": 0.40572649240493774}, {"head": 5, "memory_to_active": 0.03138163685798645, "contribution_to_active": 0.5544899702072144, "mix_mean": 0.6002461910247803, "mix_low": 0.00848388671875, "mix_high": 0.0029163360595703125, "read_cosine": 0.3717314600944519, "beta_mean": 0.00255427579395473, "nonfinite": 0.0, "weighted_memory_to_active": 0.021136818453669548, "mix_std": 0.2564338743686676, "beta_std": 0.017992816865444183, "beta_low": 0.96868896484375, "beta_high": 0.0, "orthogonal_to_active": 0.012255072593688965, "evidence_weights": [-0.27900904417037964, -0.308063805103302, 0.5576397776603699], "agreement_weight": 0.3168834447860718, "orthogonal_weight": 0.8129275441169739}]}]}, {"step": 77000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8242403490003198, "memory_to_active": 0.8133994340896606, "contribution_to_active": 0.590080201625824, "mix_mean": 0.43488985300064087, "mix_low": 0.11938588321208954, "mix_high": 0.011135379783809185, "read_cosine": 0.3437080681324005, "beta_mean": 0.06724218279123306, "nonfinite": 0.0, "weighted_memory_to_active": 0.20301876962184906, "mix_std": 0.2218358814716339, "beta_std": 0.058701109141111374, "beta_low": 0.6008008718490601, "beta_high": 3.4968063118867576e-05, "orthogonal_to_active": 0.06702099740505219, "samples": 4, "scored_tokens": 899, "normal_loss": 3.009998321533203, "normal_token_accuracy": 0.4605116844177246, "state_active": 0.06373949348926544, "state_memory": 0.015743503347039223, "state_peak": 126.75067901611328, "growth_active": 1.1164387464523315, "growth_memory": 1.2587389945983887, "output_influence": 0.4570486545562744, "residual_influence": 0.17221595346927643, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.090047836303711, "no_memory_loss_delta": 0.869600236415863, "scrambled_loss_delta": 0.1609189510345459, "memory_only_loss_delta": 3.9023733139038086, "no_evidence_loss_delta": 0.47911977767944336, "no_agreement_loss_delta": 0.021365899592638016, "no_orthogonal_loss_delta": 0.02182476967573166}, {"index": 94, "tokens": 256, "normal_loss": 3.1144652366638184, "no_memory_loss_delta": 0.8455256223678589, "scrambled_loss_delta": 0.4733516573905945, "memory_only_loss_delta": 4.972308158874512, "no_evidence_loss_delta": 0.36051228642463684, "no_agreement_loss_delta": 0.03725804388523102, "no_orthogonal_loss_delta": 0.04896606504917145}, {"index": 188, "tokens": 256, "normal_loss": 3.3131532669067383, "no_memory_loss_delta": 1.0311861038208008, "scrambled_loss_delta": 0.11040034890174866, "memory_only_loss_delta": 3.179779052734375, "no_evidence_loss_delta": 0.5553587675094604, "no_agreement_loss_delta": 0.054815955460071564, "no_orthogonal_loss_delta": 0.04597965627908707}, {"index": 283, "tokens": 131, "normal_loss": 2.0569915771484375, "no_memory_loss_delta": 0.33805638551712036, "scrambled_loss_delta": 0.002813477534800768, "memory_only_loss_delta": 3.0635077953338623, "no_evidence_loss_delta": 0.15846431255340576, "no_agreement_loss_delta": -0.007641682866960764, "no_orthogonal_loss_delta": -0.08495204150676727}], "agreement_weight_abs_mean": 0.2576700747013092, "agreement_weight_abs_max": 6.915136337280273, "agreement_weight_mean": 0.22348244488239288, "orthogonal_weight_abs_mean": 0.9565069079399109, "orthogonal_weight_abs_max": 11.415931701660156, "orthogonal_weight_mean": -0.029795685783028603, "no_memory_loss": 3.8413009643554688, "no_memory_loss_delta": 0.8313027620315552, "no_memory_token_accuracy_gain": 0.12124583125114441, "no_memory_token_win_fraction": 0.7119021415710449, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 56.451841226419845, "scrambled_loss": 3.222461700439453, "scrambled_loss_delta": 0.2124631106853485, "scrambled_token_accuracy_gain": 0.023359287530183792, "scrambled_token_win_fraction": 0.607341468334198, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.140985589246412, "memory_only_loss": 6.889042377471924, "memory_only_loss_delta": 3.879044771194458, "memory_only_token_accuracy_gain": 0.31368187069892883, "memory_only_token_win_fraction": 0.9054505228996277, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 97.93294391306677, "no_evidence_loss": 3.430328369140625, "no_evidence_loss_delta": 0.42032983899116516, "no_evidence_token_accuracy_gain": 0.060066740959882736, "no_evidence_token_win_fraction": 0.6229143738746643, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 34.316986410777155, "no_agreement_loss": 3.0411880016326904, "no_agreement_loss_delta": 0.03118971548974514, "no_agreement_token_accuracy_gain": -0.0011123470030725002, "no_agreement_token_win_fraction": 0.4749721884727478, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 3.070833401155293, "no_orthogonal_loss": 3.0308709144592285, "no_orthogonal_loss_delta": 0.02087266556918621, "no_orthogonal_token_accuracy_gain": 0.007786429487168789, "no_orthogonal_token_win_fraction": 0.5706340670585632, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.0656339202151743, "short_no_memory_loss_delta": 0.7854679226875305, "long_context_memory_gain": 0.04583483934402466, "normal_context_loss_gain": 0.12825454771518707, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.731049060821533, "contribution_to_active": 1.2698060274124146, "mix_mean": 0.1106574684381485, "mix_low": 0.5015777349472046, "mix_high": 0.004685878753662109, "read_cosine": 0.41005390882492065, "beta_mean": 0.21778500080108643, "nonfinite": 0.0, "weighted_memory_to_active": 1.2855420112609863, "mix_std": 0.09083371609449387, "beta_std": 0.1381683498620987, "beta_low": 0.1906636655330658, "beta_high": 0.0, "orthogonal_to_active": 0.050273098051548004, "output_influence": 0.21223604679107666, "residual_influence": 0.21863669157028198, "state_active": 0.06607387959957123, "state_memory": 0.045787226408720016, "state_peak": 46.351436614990234, "growth_active": 1.197562336921692, "growth_memory": 1.2524621486663818, "heads": [{"head": 0, "memory_to_active": 2.9712648391723633, "contribution_to_active": 0.4381404221057892, "mix_mean": 0.018487028777599335, "mix_low": 0.6293621063232422, "mix_high": 0.0, "read_cosine": 0.22274959087371826, "beta_mean": 0.19208166003227234, "nonfinite": 0.0, "weighted_memory_to_active": 0.549660325050354, "mix_std": 0.05573613941669464, "beta_std": 0.09564455598592758, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05555387958884239, "evidence_weights": [-0.0018101244932040572, -0.03813804313540459, -0.05891171470284462], "agreement_weight": -0.23556315898895264, "orthogonal_weight": -0.11823973804712296}, {"head": 1, "memory_to_active": 7.080869197845459, "contribution_to_active": 3.183657169342041, "mix_mean": 0.019484518095850945, "mix_low": 0.6124162673950195, "mix_high": 0.0, "read_cosine": 0.44409799575805664, "beta_mean": 0.47090309858322144, "nonfinite": 0.0, "weighted_memory_to_active": 3.396566867828369, "mix_std": 0.05682053416967392, "beta_std": 0.20667235553264618, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.10345044732093811, "evidence_weights": [0.024103058502078056, -0.09911137819290161, 0.6745812892913818], "agreement_weight": 0.029009437188506126, "orthogonal_weight": 0.068940669298172}, {"head": 2, "memory_to_active": 3.0261728763580322, "contribution_to_active": 0.6800722479820251, "mix_mean": 0.014505506493151188, "mix_low": 0.6493721008300781, "mix_high": 0.0, "read_cosine": 0.2780783176422119, "beta_mean": 0.2652524411678314, "nonfinite": 0.0, "weighted_memory_to_active": 0.7856673002243042, "mix_std": 0.03135047107934952, "beta_std": 0.21286612749099731, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.05687648802995682, "evidence_weights": [0.06304576992988586, 0.04651555046439171, 0.274008184671402], "agreement_weight": 0.010903313755989075, "orthogonal_weight": -0.19596858322620392}, {"head": 3, "memory_to_active": 7.935516357421875, "contribution_to_active": 2.789151668548584, "mix_mean": 0.02791156992316246, "mix_low": 0.5012073516845703, "mix_high": 0.0, "read_cosine": 0.27413374185562134, "beta_mean": 0.34332555532455444, "nonfinite": 0.0, "weighted_memory_to_active": 2.8865175247192383, "mix_std": 0.06431924551725388, "beta_std": 0.26058733463287354, "beta_low": 0.00018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.06690838187932968, "evidence_weights": [-0.03377959132194519, -0.02425246499478817, -0.14755691587924957], "agreement_weight": -0.006568525452166796, "orthogonal_weight": -0.02732226252555847}, {"head": 4, "memory_to_active": 1.2684698104858398, "contribution_to_active": 0.06031251326203346, "mix_mean": 0.02310355007648468, "mix_low": 0.6044378280639648, "mix_high": 0.0, "read_cosine": 0.3101697564125061, "beta_mean": 0.017704248428344727, "nonfinite": 0.0, "weighted_memory_to_active": 0.0392119325697422, "mix_std": 0.05980962887406349, "beta_std": 0.03432542085647583, "beta_low": 0.65582275390625, "beta_high": 0.0, "orthogonal_to_active": 0.0008178635616786778, "evidence_weights": [0.08602578192949295, 0.054312605410814285, -0.14516356587409973], "agreement_weight": -0.21358680725097656, "orthogonal_weight": 0.01934380643069744}, {"head": 5, "memory_to_active": 0.10400121659040451, "contribution_to_active": 0.467502236366272, "mix_mean": 0.5604525804519653, "mix_low": 0.012670516967773438, "mix_high": 0.028115272521972656, "read_cosine": 0.9310940504074097, "beta_mean": 0.017442915588617325, "nonfinite": 0.0, "weighted_memory_to_active": 0.055627502501010895, "mix_std": 0.27696627378463745, "beta_std": 0.01891428418457508, "beta_low": 0.48797607421875, "beta_high": 0.0, "orthogonal_to_active": 0.01803150586783886, "evidence_weights": [-0.17280200123786926, -0.2159552127122879, 0.36193719506263733], "agreement_weight": 0.28374889492988586, "orthogonal_weight": -0.7254526615142822}]}, {"layer": 1, "memory_to_active": 2.5994203090667725, "contribution_to_active": 0.5073838233947754, "mix_mean": 0.22021308541297913, "mix_low": 0.3684002757072449, "mix_high": 0.0012715657940134406, "read_cosine": 0.29121172428131104, "beta_mean": 0.1815488487482071, "nonfinite": 0.0, "weighted_memory_to_active": 0.34380412101745605, "mix_std": 0.11286691576242447, "beta_std": 0.13098499178886414, "beta_low": 0.3540751338005066, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.03873012214899063, "output_influence": 0.281046986579895, "residual_influence": 0.08104206621646881, "state_active": 0.02680392935872078, "state_memory": 0.03389769420027733, "state_peak": 40.35038757324219, "growth_active": 1.1041672229766846, "growth_memory": 1.428667664527893, "heads": [{"head": 0, "memory_to_active": 0.01786082610487938, "contribution_to_active": 0.505943238735199, "mix_mean": 0.5576751828193665, "mix_low": 0.0037689208984375, "mix_high": 0.0031795501708984375, "read_cosine": 0.021165892481803894, "beta_mean": 0.0001087916680262424, "nonfinite": 0.0, "weighted_memory_to_active": 0.011418281123042107, "mix_std": 0.2816759943962097, "beta_std": 0.00020234713156241924, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.011098450981080532, "evidence_weights": [-0.2114335298538208, -0.2712472677230835, 0.18407438695430756], "agreement_weight": 0.20337733626365662, "orthogonal_weight": -1.0123653411865234}, {"head": 1, "memory_to_active": 3.145613670349121, "contribution_to_active": 0.3019160032272339, "mix_mean": 0.020351890474557877, "mix_low": 0.5096950531005859, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5286223888397217, "beta_mean": 0.28162842988967896, "nonfinite": 0.0, "weighted_memory_to_active": 0.3262862265110016, "mix_std": 0.0397598072886467, "beta_std": 0.1910897046327591, "beta_low": 0.00152587890625, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.022916046902537346, "evidence_weights": [0.028380045667290688, -0.06119678169488907, 0.05251523479819298], "agreement_weight": -0.0022231689654290676, "orthogonal_weight": 0.2121935337781906}, {"head": 2, "memory_to_active": 0.03650127351284027, "contribution_to_active": 0.5519037246704102, "mix_mean": 0.680783748626709, "mix_low": 0.0002536773681640625, "mix_high": 0.004448890686035156, "read_cosine": 0.4418787360191345, "beta_mean": 0.0012447347398847342, "nonfinite": 0.0, "weighted_memory_to_active": 0.023899998515844345, "mix_std": 0.22502930462360382, "beta_std": 0.003822174621745944, "beta_low": 0.97869873046875, "beta_high": 0.0, "orthogonal_to_active": 0.01756214164197445, "evidence_weights": [-0.2670764625072479, -0.3638775944709778, 0.6316932439804077], "agreement_weight": 0.053556982427835464, "orthogonal_weight": -0.84097820520401}, {"head": 3, "memory_to_active": 3.2635278701782227, "contribution_to_active": 0.21497851610183716, "mix_mean": 0.02500295639038086, "mix_low": 0.5079288482666016, "mix_high": 0.0, "read_cosine": 0.17491598427295685, "beta_mean": 0.3492836058139801, "nonfinite": 0.0, "weighted_memory_to_active": 0.28085193037986755, "mix_std": 0.06221194565296173, "beta_std": 0.2678631842136383, "beta_low": 0.0716552734375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.1072201281785965, "evidence_weights": [0.038200315088033676, -0.019961580634117126, -0.012793431058526039], "agreement_weight": -0.025435127317905426, "orthogonal_weight": -0.474635511636734}, {"head": 4, "memory_to_active": 4.382058143615723, "contribution_to_active": 1.3737597465515137, "mix_mean": 0.03107614256441593, "mix_low": 0.3408832550048828, "mix_high": 0.0, "read_cosine": 0.3149642050266266, "beta_mean": 0.16534030437469482, "nonfinite": 0.0, "weighted_memory_to_active": 1.3556122779846191, "mix_std": 0.05104001611471176, "beta_std": 0.1315898299217224, "beta_low": 0.03997802734375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.04613681137561798, "evidence_weights": [-0.010068618692457676, -0.0027937572449445724, 0.17595171928405762], "agreement_weight": -0.004020635038614273, "orthogonal_weight": 0.05361136049032211}, {"head": 5, "memory_to_active": 4.750958442687988, "contribution_to_active": 0.09580167382955551, "mix_mean": 0.006388479378074408, "mix_low": 0.8478717803955078, "mix_high": 0.0, "read_cosine": 0.26572293043136597, "beta_mean": 0.29168716073036194, "nonfinite": 0.0, "weighted_memory_to_active": 0.0647561326622963, "mix_std": 0.017484433948993683, "beta_std": 0.19134275615215302, "beta_low": 0.0325927734375, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.027447152882814407, "evidence_weights": [0.15874645113945007, 0.07480110228061676, 0.08466481417417526], "agreement_weight": -0.11679230630397797, "orthogonal_weight": 0.4190366566181183}]}, {"layer": 2, "memory_to_active": 1.0471713542938232, "contribution_to_active": 0.36460423469543457, "mix_mean": 0.41617339849472046, "mix_low": 0.2379682958126068, "mix_high": 0.0044147176668047905, "read_cosine": 0.2851954698562622, "beta_mean": 0.07877620309591293, "nonfinite": 0.0, "weighted_memory_to_active": 0.074320487678051, "mix_std": 0.18368081748485565, "beta_std": 0.06795882433652878, "beta_low": 0.61529541015625, "beta_high": 0.00013224284339230508, "orthogonal_to_active": 0.016083789989352226, "output_influence": 0.41764140129089355, "residual_influence": 0.12331155687570572, "state_active": 0.0401298850774765, "state_memory": 0.012928832322359085, "state_peak": 28.24431037902832, "growth_active": 1.0634753704071045, "growth_memory": 1.5495178699493408, "heads": [{"head": 0, "memory_to_active": 0.003382519818842411, "contribution_to_active": 0.3725755512714386, "mix_mean": 0.6502540707588196, "mix_low": 0.0004119873046875, "mix_high": 0.009173393249511719, "read_cosine": 0.14213094115257263, "beta_mean": 2.697426316444762e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022262190468609333, "mix_std": 0.2626975178718567, "beta_std": 4.629528120858595e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0022978552151471376, "evidence_weights": [-0.37067121267318726, -0.21195393800735474, 0.21584372222423553], "agreement_weight": 0.14064240455627441, "orthogonal_weight": -1.0683585405349731}, {"head": 1, "memory_to_active": 3.2961254119873047, "contribution_to_active": 0.04602750018239021, "mix_mean": 0.005653552711009979, "mix_low": 0.8685874938964844, "mix_high": 0.0, "read_cosine": 0.12473894655704498, "beta_mean": 0.21405376493930817, "nonfinite": 0.0, "weighted_memory_to_active": 0.03657353296875954, "mix_std": 0.011309511959552765, "beta_std": 0.14602260291576385, "beta_low": 0.00341796875, "beta_high": 0.0, "orthogonal_to_active": 0.010409233160316944, "evidence_weights": [0.1314183920621872, 0.10199745744466782, 0.5702667832374573], "agreement_weight": 0.030041871592402458, "orthogonal_weight": 0.3139357566833496}, {"head": 2, "memory_to_active": 0.006363529246300459, "contribution_to_active": 0.4228421747684479, "mix_mean": 0.562776505947113, "mix_low": 0.0008029937744140625, "mix_high": 0.003692626953125, "read_cosine": 0.39054325222969055, "beta_mean": 1.771563256625086e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0026884565595537424, "mix_std": 0.27311837673187256, "beta_std": 3.451191514614038e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00020335279987193644, "evidence_weights": [-0.24592997133731842, -0.2896009087562561, 0.5660450458526611], "agreement_weight": -0.0405820868909359, "orthogonal_weight": 0.09417743235826492}, {"head": 3, "memory_to_active": 0.09567977488040924, "contribution_to_active": 0.5476361513137817, "mix_mean": 0.5909743309020996, "mix_low": 0.00017070770263671875, "mix_high": 0.005756378173828125, "read_cosine": 0.47774964570999146, "beta_mean": 0.015313880518078804, "nonfinite": 0.0, "weighted_memory_to_active": 0.06172310188412666, "mix_std": 0.2510988116264343, "beta_std": 0.02525969408452511, "beta_low": 0.62506103515625, "beta_high": 0.0, "orthogonal_to_active": 0.04750512167811394, "evidence_weights": [-0.19731293618679047, -0.32550111413002014, 0.7190608978271484], "agreement_weight": 0.054355520755052567, "orthogonal_weight": -0.8943755030632019}, {"head": 4, "memory_to_active": 2.8737008571624756, "contribution_to_active": 0.2643090486526489, "mix_mean": 0.020834971219301224, "mix_low": 0.5574254989624023, "mix_high": 0.0, "read_cosine": 0.19690559804439545, "beta_mean": 0.24321770668029785, "nonfinite": 0.0, "weighted_memory_to_active": 0.3383425176143646, "mix_std": 0.04914945736527443, "beta_std": 0.23635420203208923, "beta_low": 0.06329345703125, "beta_high": 0.00079345703125, "orthogonal_to_active": 0.03492216765880585, "evidence_weights": [0.02308342233300209, -0.0006932559190317988, 0.034936562180519104], "agreement_weight": 0.020889030769467354, "orthogonal_weight": -0.1222308948636055}, {"head": 5, "memory_to_active": 0.0077758971601724625, "contribution_to_active": 0.5342347621917725, "mix_mean": 0.6665467619895935, "mix_low": 0.00041103363037109375, "mix_high": 0.00786590576171875, "read_cosine": 0.3791043162345886, "beta_mean": 2.7157766453456134e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00436907634139061, "mix_std": 0.2547111511230469, "beta_std": 3.563556674635038e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0011650030501186848, "evidence_weights": [-0.27380964159965515, -0.2841557562351227, 0.891791582107544], "agreement_weight": 0.29397132992744446, "orthogonal_weight": -0.31317636370658875}]}, {"layer": 3, "memory_to_active": 0.41973328590393066, "contribution_to_active": 0.5012589693069458, "mix_mean": 0.5333482027053833, "mix_low": 0.05120055004954338, "mix_high": 0.01263030432164669, "read_cosine": 0.18224003911018372, "beta_mean": 0.02378520742058754, "nonfinite": 0.0, "weighted_memory_to_active": 0.11865082383155823, "mix_std": 0.25412648916244507, "beta_std": 0.02364649437367916, "beta_low": 0.7752482295036316, "beta_high": 0.0, "orthogonal_to_active": 0.04923499748110771, "output_influence": 0.45509690046310425, "residual_influence": 0.12023638188838959, "state_active": 0.02219306118786335, "state_memory": 0.00604106392711401, "state_peak": 24.979124069213867, "growth_active": 1.086837649345398, "growth_memory": 1.6062513589859009, "heads": [{"head": 0, "memory_to_active": 0.3328157961368561, "contribution_to_active": 0.6286606192588806, "mix_mean": 0.6210328340530396, "mix_low": 0.0009603500366210938, "mix_high": 0.023616790771484375, "read_cosine": 0.22700586915016174, "beta_mean": 0.008967136964201927, "nonfinite": 0.0, "weighted_memory_to_active": 0.21363210678100586, "mix_std": 0.29517555236816406, "beta_std": 0.01050405204296112, "beta_low": 0.718505859375, "beta_high": 0.0, "orthogonal_to_active": 0.15720057487487793, "evidence_weights": [-0.21336475014686584, -0.24697446823120117, 1.0590566396713257], "agreement_weight": 0.11115013808012009, "orthogonal_weight": -0.7722619771957397}, {"head": 1, "memory_to_active": 0.005900029558688402, "contribution_to_active": 0.47303077578544617, "mix_mean": 0.6047199964523315, "mix_low": 0.0022840499877929688, "mix_high": 0.013128280639648438, "read_cosine": 0.1661781370639801, "beta_mean": 2.739529372774996e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00373232364654541, "mix_std": 0.29062825441360474, "beta_std": 6.716331699863076e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003026902209967375, "evidence_weights": [-0.26681774854660034, -0.21532858908176422, 0.6834726929664612], "agreement_weight": 0.6167875528335571, "orthogonal_weight": -0.8759510517120361}, {"head": 2, "memory_to_active": 1.9392924308776855, "contribution_to_active": 0.28088101744651794, "mix_mean": 0.056296661496162415, "mix_low": 0.2968311309814453, "mix_high": 0.0, "read_cosine": 0.20823976397514343, "beta_mean": 0.12901532649993896, "nonfinite": 0.0, "weighted_memory_to_active": 0.31421977281570435, "mix_std": 0.10645841807126999, "beta_std": 0.11875726282596588, "beta_low": 0.02691650390625, "beta_high": 0.0, "orthogonal_to_active": 0.00552552193403244, "evidence_weights": [-0.1002589538693428, -0.008511610329151154, -0.028922423720359802], "agreement_weight": -0.011737904511392117, "orthogonal_weight": 0.022662967443466187}, {"head": 3, "memory_to_active": 0.016017453745007515, "contribution_to_active": 0.5518711805343628, "mix_mean": 0.624038577079773, "mix_low": 0.0033006668090820312, "mix_high": 0.01049041748046875, "read_cosine": 0.2754054665565491, "beta_mean": 6.942253821762279e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010924058966338634, "mix_std": 0.2806427776813507, "beta_std": 0.0001636288216104731, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002518017077818513, "evidence_weights": [-0.26382601261138916, -0.22156432271003723, 2.351919174194336], "agreement_weight": 2.3806395530700684, "orthogonal_weight": -0.3010297119617462}, {"head": 4, "memory_to_active": 0.013781992718577385, "contribution_to_active": 0.4121326208114624, "mix_mean": 0.5544654130935669, "mix_low": 0.0016803741455078125, "mix_high": 0.0033197402954101562, "read_cosine": 0.13849586248397827, "beta_mean": 6.90794549882412e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009376048110425472, "mix_std": 0.28678467869758606, "beta_std": 0.00012095018610125408, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008752809837460518, "evidence_weights": [-0.24598290026187897, -0.22287532687187195, 0.045122649520635605], "agreement_weight": -0.06621304154396057, "orthogonal_weight": -1.0167815685272217}, {"head": 5, "memory_to_active": 0.21059201657772064, "contribution_to_active": 0.6609776020050049, "mix_mean": 0.739535927772522, "mix_low": 0.0021467208862304688, "mix_high": 0.025226593017578125, "read_cosine": 0.07811514288187027, "beta_mean": 0.00456286920234561, "nonfinite": 0.0, "weighted_memory_to_active": 0.1600206047296524, "mix_std": 0.2650691270828247, "beta_std": 0.012265892699360847, "beta_low": 0.90606689453125, "beta_high": 0.0, "orthogonal_to_active": 0.1183861568570137, "evidence_weights": [-0.259427934885025, -0.3424246311187744, 0.5581758618354797], "agreement_weight": 0.24475829303264618, "orthogonal_weight": -0.7640513777732849}]}, {"layer": 4, "memory_to_active": 0.9932137727737427, "contribution_to_active": 0.48667147755622864, "mix_mean": 0.37799522280693054, "mix_low": 0.21957510709762573, "mix_high": 0.012113571166992188, "read_cosine": 0.2943648099899292, "beta_mean": 0.10167908668518066, "nonfinite": 0.0, "weighted_memory_to_active": 0.12947943806648254, "mix_std": 0.16221290826797485, "beta_std": 0.09363679587841034, "beta_low": 0.4011942744255066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05752909183502197, "output_influence": 0.5151141881942749, "residual_influence": 0.17285969853401184, "state_active": 0.03832899406552315, "state_memory": 0.015143116004765034, "state_peak": 41.94532012939453, "growth_active": 1.2138220071792603, "growth_memory": 1.5201942920684814, "heads": [{"head": 0, "memory_to_active": 0.13440163433551788, "contribution_to_active": 0.8177262544631958, "mix_mean": 0.6895550489425659, "mix_low": 0.009401321411132812, "mix_high": 0.033100128173828125, "read_cosine": 0.42627957463264465, "beta_mean": 0.22718295454978943, "nonfinite": 0.0, "weighted_memory_to_active": 0.11590000987052917, "mix_std": 0.34741464257240295, "beta_std": 0.22442832589149475, "beta_low": 0.26422119140625, "beta_high": 0.0, "orthogonal_to_active": 0.09039971232414246, "evidence_weights": [-0.23849113285541534, -0.35222095251083374, -0.24480043351650238], "agreement_weight": 0.004391887225210667, "orthogonal_weight": 0.9099389910697937}, {"head": 1, "memory_to_active": 0.04413044452667236, "contribution_to_active": 0.6684078574180603, "mix_mean": 0.7135437726974487, "mix_low": 5.340576171875e-05, "mix_high": 0.021109580993652344, "read_cosine": 0.5392928719520569, "beta_mean": 0.0009923225734382868, "nonfinite": 0.0, "weighted_memory_to_active": 0.033692143857479095, "mix_std": 0.23374232649803162, "beta_std": 0.007996760308742523, "beta_low": 0.980712890625, "beta_high": 0.0, "orthogonal_to_active": 0.01633916050195694, "evidence_weights": [-0.232654869556427, -0.36329787969589233, 0.5526218414306641], "agreement_weight": 0.37748873233795166, "orthogonal_weight": -0.6661978960037231}, {"head": 2, "memory_to_active": 1.3789573907852173, "contribution_to_active": 0.38634756207466125, "mix_mean": 0.06308574974536896, "mix_low": 0.2734708786010742, "mix_high": 2.384185791015625e-05, "read_cosine": 0.21650010347366333, "beta_mean": 0.049960438162088394, "nonfinite": 0.0, "weighted_memory_to_active": 0.27822011709213257, "mix_std": 0.10600512474775314, "beta_std": 0.055789485573768616, "beta_low": 0.13116455078125, "beta_high": 0.0, "orthogonal_to_active": 0.111122265458107, "evidence_weights": [0.020753633230924606, -0.07338910549879074, 0.3986215889453888], "agreement_weight": 0.12186374515295029, "orthogonal_weight": -0.5126467347145081}, {"head": 3, "memory_to_active": 1.8358547687530518, "contribution_to_active": 0.13621193170547485, "mix_mean": 0.032240837812423706, "mix_low": 0.3963813781738281, "mix_high": 0.0, "read_cosine": 0.22138911485671997, "beta_mean": 0.12585937976837158, "nonfinite": 0.0, "weighted_memory_to_active": 0.18422582745552063, "mix_std": 0.05285324528813362, "beta_std": 0.11665242910385132, "beta_low": 0.031005859375, "beta_high": 0.0, "orthogonal_to_active": 0.0036024674773216248, "evidence_weights": [-0.02359519526362419, 0.04828624427318573, -0.2578754127025604], "agreement_weight": 0.07754357159137726, "orthogonal_weight": 0.0285395085811615}, {"head": 4, "memory_to_active": 0.008310895413160324, "contribution_to_active": 0.6590144038200378, "mix_mean": 0.7557684779167175, "mix_low": 2.002716064453125e-05, "mix_high": 0.018444061279296875, "read_cosine": 0.12736305594444275, "beta_mean": 1.6837217117426917e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00622224248945713, "mix_std": 0.20659244060516357, "beta_std": 5.330290150595829e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003924774471670389, "evidence_weights": [-0.3232799172401428, -0.31147536635398865, 0.23353785276412964], "agreement_weight": 0.32305923104286194, "orthogonal_weight": -0.6651689410209656}, {"head": 5, "memory_to_active": 2.5576274394989014, "contribution_to_active": 0.2523207664489746, "mix_mean": 0.013777380809187889, "mix_low": 0.6381235122680664, "mix_high": 3.814697265625e-06, "read_cosine": 0.23536401987075806, "beta_mean": 0.20606254041194916, "nonfinite": 0.0, "weighted_memory_to_active": 0.15861627459526062, "mix_std": 0.026669710874557495, "beta_std": 0.15690039098262787, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.11978615820407867, "evidence_weights": [0.0558316670358181, -0.018777858465909958, -0.2637822926044464], "agreement_weight": -0.09414038062095642, "orthogonal_weight": 1.1322053670883179}]}, {"layer": 5, "memory_to_active": 0.3660098910331726, "contribution_to_active": 0.554753303527832, "mix_mean": 0.5271745920181274, "mix_low": 0.05529944226145744, "mix_high": 0.0203196220099926, "read_cosine": 0.1003570556640625, "beta_mean": 0.048347391188144684, "nonfinite": 0.0, "weighted_memory_to_active": 0.12476956844329834, "mix_std": 0.2495557963848114, "beta_std": 0.06346137821674347, "beta_low": 0.696380615234375, "beta_high": 0.0, "orthogonal_to_active": 0.055577121675014496, "output_influence": 0.6045230627059937, "residual_influence": 0.2152496576309204, "state_active": 0.0384875163435936, "state_memory": 0.009159613400697708, "state_peak": 62.176368713378906, "growth_active": 1.1105091571807861, "growth_memory": 1.2549500465393066, "heads": [{"head": 0, "memory_to_active": 0.14485935866832733, "contribution_to_active": 0.5889555811882019, "mix_mean": 0.5177582502365112, "mix_low": 0.0052318572998046875, "mix_high": 0.0022144317626953125, "read_cosine": -0.132888525724411, "beta_mean": 0.013759113848209381, "nonfinite": 0.0, "weighted_memory_to_active": 0.05207778513431549, "mix_std": 0.2750691771507263, "beta_std": 0.025144409388303757, "beta_low": 0.63720703125, "beta_high": 0.0, "orthogonal_to_active": 0.02352728135883808, "evidence_weights": [-0.16955336928367615, -0.18787112832069397, 0.12047524005174637], "agreement_weight": -0.020907234400510788, "orthogonal_weight": -0.4798848330974579}, {"head": 1, "memory_to_active": 0.0035464861430227757, "contribution_to_active": 0.5765109062194824, "mix_mean": 0.7204948663711548, "mix_low": 0.0006160736083984375, "mix_high": 0.037499427795410156, "read_cosine": 0.27124178409576416, "beta_mean": 7.856055162847042e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002256231615319848, "mix_std": 0.2597823441028595, "beta_std": 1.7336986275040545e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002075118012726307, "evidence_weights": [-0.2442891001701355, -0.2835310101509094, 0.9403817653656006], "agreement_weight": 0.7453923225402832, "orthogonal_weight": -0.9729267954826355}, {"head": 2, "memory_to_active": 0.0144127132371068, "contribution_to_active": 0.6277973055839539, "mix_mean": 0.6697925329208374, "mix_low": 0.00150299072265625, "mix_high": 0.0206451416015625, "read_cosine": -0.3761930465698242, "beta_mean": 8.31848883535713e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009545642882585526, "mix_std": 0.25778067111968994, "beta_std": 0.0003736774960998446, "beta_low": 0.999755859375, "beta_high": 0.0, "orthogonal_to_active": 0.009416859596967697, "evidence_weights": [-0.25471487641334534, -0.3085327446460724, 0.1780797839164734], "agreement_weight": -0.20933237671852112, "orthogonal_weight": -1.0618797540664673}, {"head": 3, "memory_to_active": 1.5744857788085938, "contribution_to_active": 0.25128185749053955, "mix_mean": 0.1215810775756836, "mix_low": 0.11032962799072266, "mix_high": 0.0, "read_cosine": 0.14595279097557068, "beta_mean": 0.096101313829422, "nonfinite": 0.0, "weighted_memory_to_active": 0.3823225796222687, "mix_std": 0.15136049687862396, "beta_std": 0.1488092839717865, "beta_low": 0.21795654296875, "beta_high": 0.0, "orthogonal_to_active": 0.1620131880044937, "evidence_weights": [-0.07601327449083328, -0.09828142076730728, 0.12834087014198303], "agreement_weight": 0.04784771427512169, "orthogonal_weight": -0.4973852336406708}, {"head": 4, "memory_to_active": 0.3977159857749939, "contribution_to_active": 0.587367832660675, "mix_mean": 0.3184761703014374, "mix_low": 0.2127676010131836, "mix_high": 0.0045070648193359375, "read_cosine": 0.5422786474227905, "beta_mean": 0.17996647953987122, "nonfinite": 0.0, "weighted_memory_to_active": 0.2603189945220947, "mix_std": 0.3364357650279999, "beta_std": 0.20559106767177582, "beta_low": 0.32525634765625, "beta_high": 0.0, "orthogonal_to_active": 0.11225949227809906, "evidence_weights": [-0.10847600549459457, -0.24761684238910675, -0.073140449821949], "agreement_weight": 0.030962873250246048, "orthogonal_weight": 0.6770920753479004}, {"head": 5, "memory_to_active": 0.06103896722197533, "contribution_to_active": 0.6966061592102051, "mix_mean": 0.8149443864822388, "mix_low": 0.0013484954833984375, "mix_high": 0.057051658630371094, "read_cosine": 0.15175065398216248, "beta_mean": 0.00016638431407045573, "nonfinite": 0.0, "weighted_memory_to_active": 0.04209613427519798, "mix_std": 0.21690633893013, "beta_std": 0.0008324780501425266, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.0241707731038332, "evidence_weights": [-0.29933011531829834, -0.32254043221473694, 1.805222511291504], "agreement_weight": 0.5601400136947632, "orthogonal_weight": -0.6011829972267151}]}, {"layer": 6, "memory_to_active": 0.22477920353412628, "contribution_to_active": 0.6603348255157471, "mix_mean": 0.6051633358001709, "mix_low": 0.028749942779541016, "mix_high": 0.03126939386129379, "read_cosine": 0.3176785111427307, "beta_mean": 0.05814042687416077, "nonfinite": 0.0, "weighted_memory_to_active": 0.12137623131275177, "mix_std": 0.23934169113636017, "beta_std": 0.07021068036556244, "beta_low": 0.6387838125228882, "beta_high": 0.0, "orthogonal_to_active": 0.09491607546806335, "output_influence": 0.6031580567359924, "residual_influence": 0.23499539494514465, "state_active": 0.03956855833530426, "state_memory": 0.0048626987263560295, "state_peak": 89.05208587646484, "growth_active": 1.1543501615524292, "growth_memory": 1.0941702127456665, "heads": [{"head": 0, "memory_to_active": 0.10801208019256592, "contribution_to_active": 0.791492223739624, "mix_mean": 0.8285545706748962, "mix_low": 0.0001354217529296875, "mix_high": 0.06951141357421875, "read_cosine": 0.32033097743988037, "beta_mean": 0.010454094037413597, "nonfinite": 0.0, "weighted_memory_to_active": 0.09761663526296616, "mix_std": 0.19790226221084595, "beta_std": 0.03343771770596504, "beta_low": 0.8531494140625, "beta_high": 0.0, "orthogonal_to_active": 0.06206555292010307, "evidence_weights": [-0.2892329692840576, -0.3385045826435089, 1.1618893146514893], "agreement_weight": 0.9691522121429443, "orthogonal_weight": -0.6900414228439331}, {"head": 1, "memory_to_active": 0.0031445191707462072, "contribution_to_active": 0.5267738103866577, "mix_mean": 0.6127226948738098, "mix_low": 0.000278472900390625, "mix_high": 0.019376754760742188, "read_cosine": 0.4792642593383789, "beta_mean": 6.453069090639474e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0014893098268657923, "mix_std": 0.2788130044937134, "beta_std": 1.459207669540774e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000766203214880079, "evidence_weights": [-0.18720543384552002, -0.29862552881240845, 1.09400475025177], "agreement_weight": 0.13884572684764862, "orthogonal_weight": -0.6810968518257141}, {"head": 2, "memory_to_active": 0.5910003185272217, "contribution_to_active": 0.11285862326622009, "mix_mean": 0.06552520394325256, "mix_low": 0.165252685546875, "mix_high": 1.9073486328125e-06, "read_cosine": 0.2849811017513275, "beta_mean": 0.016852717846632004, "nonfinite": 0.0, "weighted_memory_to_active": 0.06981145590543747, "mix_std": 0.09017463028430939, "beta_std": 0.01952475681900978, "beta_low": 0.41650390625, "beta_high": 0.0, "orthogonal_to_active": 0.014444191940128803, "evidence_weights": [0.05101979151368141, -0.07809289544820786, 0.3836126923561096], "agreement_weight": 0.13156284391880035, "orthogonal_weight": -0.24616456031799316}, {"head": 3, "memory_to_active": 0.12462998926639557, "contribution_to_active": 0.6911559104919434, "mix_mean": 0.6771215200424194, "mix_low": 0.0053768157958984375, "mix_high": 0.026236534118652344, "read_cosine": 0.45388007164001465, "beta_mean": 0.2046123743057251, "nonfinite": 0.0, "weighted_memory_to_active": 0.09044524282217026, "mix_std": 0.3244572579860687, "beta_std": 0.23758481442928314, "beta_low": 0.22540283203125, "beta_high": 0.0, "orthogonal_to_active": 0.05841367691755295, "evidence_weights": [-0.16161684691905975, -0.30714061856269836, 0.09439671039581299], "agreement_weight": -0.005184601526707411, "orthogonal_weight": 0.7310861945152283}, {"head": 4, "memory_to_active": 0.5035272240638733, "contribution_to_active": 1.1869487762451172, "mix_mean": 0.7363420128822327, "mix_low": 0.001354217529296875, "mix_high": 0.0628204345703125, "read_cosine": 0.08479218184947968, "beta_mean": 0.11683692783117294, "nonfinite": 0.0, "weighted_memory_to_active": 0.4552485942840576, "mix_std": 0.3104667365550995, "beta_std": 0.1304810792207718, "beta_low": 0.33770751953125, "beta_high": 0.0, "orthogonal_to_active": 0.4207247495651245, "evidence_weights": [-0.14883103966712952, -0.49233734607696533, -0.8310347199440002], "agreement_weight": -0.015642866492271423, "orthogonal_weight": 0.9448135495185852}, {"head": 5, "memory_to_active": 0.018361089751124382, "contribution_to_active": 0.6527793407440186, "mix_mean": 0.7107139825820923, "mix_low": 0.00010204315185546875, "mix_high": 0.009669303894042969, "read_cosine": 0.28282248973846436, "beta_mean": 8.000560774235055e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013646051287651062, "mix_std": 0.2342362254858017, "beta_std": 0.000221102440264076, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01308206282556057, "evidence_weights": [-0.22989678382873535, -0.3131448030471802, 0.3460114896297455], "agreement_weight": 0.24189774692058563, "orthogonal_weight": -1.0282435417175293}]}, {"layer": 7, "memory_to_active": 1.7901198863983154, "contribution_to_active": 0.278425395488739, "mix_mean": 0.1851518750190735, "mix_low": 0.20835113525390625, "mix_high": 0.0010372798424214125, "read_cosine": 0.25364863872528076, "beta_mean": 0.1325652301311493, "nonfinite": 0.0, "weighted_memory_to_active": 0.19993939995765686, "mix_std": 0.14797216653823853, "beta_std": 0.09570552408695221, "beta_low": 0.2322184294462204, "beta_high": 0.00027465823222883046, "orthogonal_to_active": 0.0878482311964035, "output_influence": 0.29962313175201416, "residual_influence": 0.07876405864953995, "state_active": 0.014517202973365784, "state_memory": 0.022820599377155304, "state_peak": 35.16802978515625, "growth_active": 1.1532924175262451, "growth_memory": 1.265985369682312, "heads": [{"head": 0, "memory_to_active": 2.807576894760132, "contribution_to_active": 0.14742232859134674, "mix_mean": 0.03839147090911865, "mix_low": 0.40659523010253906, "mix_high": 9.5367431640625e-07, "read_cosine": 0.35210466384887695, "beta_mean": 0.16926896572113037, "nonfinite": 0.0, "weighted_memory_to_active": 0.17141419649124146, "mix_std": 0.07546114921569824, "beta_std": 0.12892131507396698, "beta_low": 0.00238037109375, "beta_high": 0.00048828125, "orthogonal_to_active": 0.0007312482921406627, "evidence_weights": [0.04619744047522545, -0.09149372577667236, 0.030073143541812897], "agreement_weight": -0.029615724459290504, "orthogonal_weight": -0.004595225211232901}, {"head": 1, "memory_to_active": 0.9577652215957642, "contribution_to_active": 0.40527206659317017, "mix_mean": 0.2352830469608307, "mix_low": 0.10326385498046875, "mix_high": 0.005423545837402344, "read_cosine": 0.40164509415626526, "beta_mean": 0.08905401825904846, "nonfinite": 0.0, "weighted_memory_to_active": 0.3414958715438843, "mix_std": 0.24888134002685547, "beta_std": 0.06285203248262405, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04658032953739166, "evidence_weights": [-0.2189931422472, -0.10565508902072906, -0.31245705485343933], "agreement_weight": 0.053305722773075104, "orthogonal_weight": 0.16355839371681213}, {"head": 2, "memory_to_active": 0.5387804508209229, "contribution_to_active": 0.4082668721675873, "mix_mean": 0.2521931827068329, "mix_low": 0.011567115783691406, "mix_high": 0.00043392181396484375, "read_cosine": 0.12561412155628204, "beta_mean": 0.0273931585252285, "nonfinite": 0.0, "weighted_memory_to_active": 0.22076070308685303, "mix_std": 0.2174106240272522, "beta_std": 0.032653991132974625, "beta_low": 0.248291015625, "beta_high": 0.0, "orthogonal_to_active": 0.13294807076454163, "evidence_weights": [-0.10069629549980164, -0.1882719099521637, 0.8935021758079529], "agreement_weight": 0.010784860700368881, "orthogonal_weight": -0.6216809749603271}, {"head": 3, "memory_to_active": 4.87122917175293, "contribution_to_active": 0.09976672381162643, "mix_mean": 0.03685203567147255, "mix_low": 0.28993701934814453, "mix_high": 0.0, "read_cosine": 0.10125409066677094, "beta_mean": 0.4482869505882263, "nonfinite": 0.0, "weighted_memory_to_active": 0.36491650342941284, "mix_std": 0.050826095044612885, "beta_std": 0.2804892063140869, "beta_low": 0.001220703125, "beta_high": 0.00115966796875, "orthogonal_to_active": 0.3145354986190796, "evidence_weights": [0.0718485563993454, -0.04817657545208931, 0.7053632140159607], "agreement_weight": 0.04704669862985611, "orthogonal_weight": -0.933215856552124}, {"head": 4, "memory_to_active": 1.5343618392944336, "contribution_to_active": 0.12020948529243469, "mix_mean": 0.027826497331261635, "mix_low": 0.4374256134033203, "mix_high": 0.0, "read_cosine": 0.28034085035324097, "beta_mean": 0.061267443001270294, "nonfinite": 0.0, "weighted_memory_to_active": 0.08501806855201721, "mix_std": 0.04930290952324867, "beta_std": 0.06899481266736984, "beta_low": 0.14154052734375, "beta_high": 0.0, "orthogonal_to_active": 0.017263753339648247, "evidence_weights": [0.10659908503293991, -0.03707423433661461, 0.18684417009353638], "agreement_weight": 0.0339314267039299, "orthogonal_weight": -0.2837579846382141}, {"head": 5, "memory_to_active": 0.031005263328552246, "contribution_to_active": 0.4896148145198822, "mix_mean": 0.5203649401664734, "mix_low": 0.0013179779052734375, "mix_high": 0.00036525726318359375, "read_cosine": 0.26093292236328125, "beta_mean": 0.00012081804743502289, "nonfinite": 0.0, "weighted_memory_to_active": 0.016031015664339066, "mix_std": 0.24595080316066742, "beta_std": 0.0003217935445718467, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.01503048837184906, "evidence_weights": [-0.18838907778263092, -0.24405620992183685, 0.46907520294189453], "agreement_weight": 0.25705233216285706, "orthogonal_weight": -0.9989106059074402}]}, {"layer": 8, "memory_to_active": 0.14032408595085144, "contribution_to_active": 0.5799074769020081, "mix_mean": 0.5187132358551025, "mix_low": 0.008600871078670025, "mix_high": 0.002144336700439453, "read_cosine": 0.385757714509964, "beta_mean": 0.012405183166265488, "nonfinite": 0.0, "weighted_memory_to_active": 0.07149501889944077, "mix_std": 0.2415696680545807, "beta_std": 0.01925499550998211, "beta_low": 0.7842000722885132, "beta_high": 0.0, "orthogonal_to_active": 0.0643707886338234, "output_influence": 0.537325918674469, "residual_influence": 0.21465931832790375, "state_active": 0.041909873485565186, "state_memory": 0.00444607250392437, "state_peak": 70.68634796142578, "growth_active": 1.1133579015731812, "growth_memory": 1.0209619998931885, "heads": [{"head": 0, "memory_to_active": 0.056176893413066864, "contribution_to_active": 0.5782222151756287, "mix_mean": 0.6110615134239197, "mix_low": 0.00040149688720703125, "mix_high": 0.0014142990112304688, "read_cosine": 0.4419930875301361, "beta_mean": 0.0031220244709402323, "nonfinite": 0.0, "weighted_memory_to_active": 0.0403997041285038, "mix_std": 0.24741536378860474, "beta_std": 0.013378787785768509, "beta_low": 0.934814453125, "beta_high": 0.0, "orthogonal_to_active": 0.025280656293034554, "evidence_weights": [-0.1466798186302185, -0.3699212372303009, 0.5490529537200928], "agreement_weight": 0.2737145721912384, "orthogonal_weight": -0.7909331321716309}, {"head": 1, "memory_to_active": 0.05878005549311638, "contribution_to_active": 0.5551096200942993, "mix_mean": 0.5358072519302368, "mix_low": 0.00013065338134765625, "mix_high": 0.002117156982421875, "read_cosine": 0.10172448307275772, "beta_mean": 0.0008211919339373708, "nonfinite": 0.0, "weighted_memory_to_active": 0.02700711227953434, "mix_std": 0.23024028539657593, "beta_std": 0.0074223969131708145, "beta_low": 0.99346923828125, "beta_high": 0.0, "orthogonal_to_active": 0.02000172808766365, "evidence_weights": [-0.18933504819869995, -0.2687190771102905, 0.5541581511497498], "agreement_weight": 0.06541392207145691, "orthogonal_weight": -0.7820329070091248}, {"head": 2, "memory_to_active": 0.008459034375846386, "contribution_to_active": 0.7006366848945618, "mix_mean": 0.6953045129776001, "mix_low": 0.0002117156982421875, "mix_high": 0.004254341125488281, "read_cosine": 0.4101448357105255, "beta_mean": 9.666271216701716e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006721036974340677, "mix_std": 0.21656198799610138, "beta_std": 0.003244297346100211, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.003834725357592106, "evidence_weights": [-0.21495071053504944, -0.3411642909049988, 0.9846968054771423], "agreement_weight": 0.3657384514808655, "orthogonal_weight": -0.6516540050506592}, {"head": 3, "memory_to_active": 0.10471992939710617, "contribution_to_active": 0.6444855332374573, "mix_mean": 0.5936093330383301, "mix_low": 0.00054931640625, "mix_high": 0.0048618316650390625, "read_cosine": 0.7915744185447693, "beta_mean": 0.008019677363336086, "nonfinite": 0.0, "weighted_memory_to_active": 0.07920905947685242, "mix_std": 0.2553521394729614, "beta_std": 0.008022784255445004, "beta_low": 0.7186279296875, "beta_high": 0.0, "orthogonal_to_active": 0.024333767592906952, "evidence_weights": [-0.1628337949514389, -0.2931549847126007, 0.5159302949905396], "agreement_weight": 0.2736501693725586, "orthogonal_weight": -0.5842981934547424}, {"head": 4, "memory_to_active": 0.2752855718135834, "contribution_to_active": 0.37335914373397827, "mix_mean": 0.3931627571582794, "mix_low": 0.0011320114135742188, "mix_high": 1.1444091796875e-05, "read_cosine": 0.08291150629520416, "beta_mean": 0.0006433227681554854, "nonfinite": 0.0, "weighted_memory_to_active": 0.09981982409954071, "mix_std": 0.23645561933517456, "beta_std": 0.014112669043242931, "beta_low": 0.99468994140625, "beta_high": 0.0, "orthogonal_to_active": 0.01767907291650772, "evidence_weights": [-0.1511443555355072, -0.20778591930866241, 0.6694835424423218], "agreement_weight": 0.0317152701318264, "orthogonal_weight": -0.17886264622211456}, {"head": 5, "memory_to_active": 0.33852294087409973, "contribution_to_active": 0.6276313662528992, "mix_mean": 0.28333377838134766, "mix_low": 0.049180030822753906, "mix_high": 0.00020694732666015625, "read_cosine": 0.486197829246521, "beta_mean": 0.06172821670770645, "nonfinite": 0.0, "weighted_memory_to_active": 0.17581339180469513, "mix_std": 0.2633925974369049, "beta_std": 0.0693490207195282, "beta_low": 0.06365966796875, "beta_high": 0.0, "orthogonal_to_active": 0.2950947880744934, "evidence_weights": [-0.15041719377040863, -0.13925760984420776, -0.10769367218017578], "agreement_weight": -0.002918920014053583, "orthogonal_weight": 1.9774482250213623}]}, {"layer": 9, "memory_to_active": 0.825103759765625, "contribution_to_active": 0.7416675090789795, "mix_mean": 0.41038623452186584, "mix_low": 0.045774780213832855, "mix_high": 0.006826878059655428, "read_cosine": 0.3401068449020386, "beta_mean": 0.030181972309947014, "nonfinite": 0.0, "weighted_memory_to_active": 0.3763810992240906, "mix_std": 0.24928364157676697, "beta_std": 0.025023894384503365, "beta_low": 0.6812744736671448, "beta_high": 0.0, "orthogonal_to_active": 0.030282996594905853, "output_influence": 0.5745739340782166, "residual_influence": 0.18657243251800537, "state_active": 0.04565349966287613, "state_memory": 0.01300058327615261, "state_peak": 60.055606842041016, "growth_active": 1.206152081489563, "growth_memory": 1.2080563306808472, "heads": [{"head": 0, "memory_to_active": 0.0002192832762375474, "contribution_to_active": 0.592140793800354, "mix_mean": 0.6658980846405029, "mix_low": 0.0009145736694335938, "mix_high": 0.028153419494628906, "read_cosine": 0.33755916357040405, "beta_mean": 5.058695933257695e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.00014901884424034506, "mix_std": 0.28066104650497437, "beta_std": 1.519701186225575e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00014394140453077853, "evidence_weights": [-0.3024812638759613, -0.3074165880680084, 0.31203073263168335], "agreement_weight": 0.10828301310539246, "orthogonal_weight": -1.0240294933319092}, {"head": 1, "memory_to_active": 0.014119859784841537, "contribution_to_active": 0.728701651096344, "mix_mean": 0.5884836912155151, "mix_low": 0.006424903869628906, "mix_high": 0.0033254623413085938, "read_cosine": 0.4374467730522156, "beta_mean": 2.9956067010061815e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010871859267354012, "mix_std": 0.27541494369506836, "beta_std": 6.674140604445711e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00662476010620594, "evidence_weights": [-0.2331453561782837, -0.28907743096351624, 0.9317478537559509], "agreement_weight": 0.5807874798774719, "orthogonal_weight": -0.7578856945037842}, {"head": 2, "memory_to_active": 0.16039933264255524, "contribution_to_active": 0.6693377494812012, "mix_mean": 0.6089375019073486, "mix_low": 0.0016660690307617188, "mix_high": 0.007503509521484375, "read_cosine": 0.366462767124176, "beta_mean": 0.0005451257457025349, "nonfinite": 0.0, "weighted_memory_to_active": 0.11049256473779678, "mix_std": 0.2627700865268707, "beta_std": 0.004352148622274399, "beta_low": 0.99652099609375, "beta_high": 0.0, "orthogonal_to_active": 0.06189271807670593, "evidence_weights": [-0.19705554842948914, -0.27320265769958496, 0.7485533356666565], "agreement_weight": 0.3729182779788971, "orthogonal_weight": -0.6591587066650391}, {"head": 3, "memory_to_active": 0.6499501466751099, "contribution_to_active": 0.30287277698516846, "mix_mean": 0.19922694563865662, "mix_low": 0.06483650207519531, "mix_high": 2.574920654296875e-05, "read_cosine": 0.3783286213874817, "beta_mean": 0.02079220861196518, "nonfinite": 0.0, "weighted_memory_to_active": 0.1861068308353424, "mix_std": 0.21833489835262299, "beta_std": 0.023608768358826637, "beta_low": 0.36260986328125, "beta_high": 0.0, "orthogonal_to_active": 0.0934472531080246, "evidence_weights": [-0.05057636275887489, -0.1234385296702385, 0.5458884239196777], "agreement_weight": 0.0895862951874733, "orthogonal_weight": -0.5581773519515991}, {"head": 4, "memory_to_active": 3.098905086517334, "contribution_to_active": 1.802361249923706, "mix_mean": 0.15245676040649414, "mix_low": 0.11935997009277344, "mix_high": 0.0010519027709960938, "read_cosine": 0.09139065444469452, "beta_mean": 0.14450523257255554, "nonfinite": 0.0, "weighted_memory_to_active": 1.6819627285003662, "mix_std": 0.21133050322532654, "beta_std": 0.0817410945892334, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.003239397658035159, "evidence_weights": [-0.0780339241027832, -0.1470576971769333, -0.04558359086513519], "agreement_weight": -0.06838317215442657, "orthogonal_weight": 0.0019945919048041105}, {"head": 5, "memory_to_active": 1.0270285606384277, "contribution_to_active": 0.3545910716056824, "mix_mean": 0.24731427431106567, "mix_low": 0.08144664764404297, "mix_high": 0.0009012222290039062, "read_cosine": 0.4294531047344208, "beta_mean": 0.015218783169984818, "nonfinite": 0.0, "weighted_memory_to_active": 0.2687036395072937, "mix_std": 0.24719032645225525, "beta_std": 0.04037308320403099, "beta_low": 0.728515625, "beta_high": 0.0, "orthogonal_to_active": 0.016349907964468002, "evidence_weights": [0.024947110563516617, -0.2045905888080597, 0.14530882239341736], "agreement_weight": 0.006013317499309778, "orthogonal_weight": 0.06579352170228958}]}, {"layer": 10, "memory_to_active": 0.213758647441864, "contribution_to_active": 0.7907278537750244, "mix_mean": 0.5349133610725403, "mix_low": 0.023840587586164474, "mix_high": 0.005816618911921978, "read_cosine": 0.38354721665382385, "beta_mean": 0.0414838045835495, "nonfinite": 0.0, "weighted_memory_to_active": 0.09257324784994125, "mix_std": 0.26449403166770935, "beta_std": 0.043993182480335236, "beta_low": 0.6683858633041382, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.31619739532470703, "output_influence": 0.48928093910217285, "residual_influence": 0.18353965878486633, "state_active": 0.05333314836025238, "state_memory": 0.013253800570964813, "state_peak": 126.75067901611328, "growth_active": 0.9965280890464783, "growth_memory": 1.0043667554855347, "heads": [{"head": 0, "memory_to_active": 0.7990498542785645, "contribution_to_active": 2.3877511024475098, "mix_mean": 0.5734457969665527, "mix_low": 0.11750030517578125, "mix_high": 0.008691787719726562, "read_cosine": 0.07153549790382385, "beta_mean": 0.18174389004707336, "nonfinite": 0.0, "weighted_memory_to_active": 0.4543972313404083, "mix_std": 0.39951273798942566, "beta_std": 0.195499449968338, "beta_low": 0.1663818359375, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.82590913772583, "evidence_weights": [-0.07915913313627243, -0.3561389744281769, -2.02301287651062], "agreement_weight": 0.04514686390757561, "orthogonal_weight": 4.044003486633301}, {"head": 1, "memory_to_active": 0.004191829357296228, "contribution_to_active": 0.6396401524543762, "mix_mean": 0.6487851142883301, "mix_low": 0.0008726119995117188, "mix_high": 0.01855182647705078, "read_cosine": 0.345574289560318, "beta_mean": 2.7408614187152125e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0024950753431767225, "mix_std": 0.25832638144493103, "beta_std": 5.484371286001988e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003810396883636713, "evidence_weights": [-0.2477298080921173, -0.19026798009872437, 0.5429010391235352], "agreement_weight": 0.6871058940887451, "orthogonal_weight": -1.666517972946167}, {"head": 2, "memory_to_active": 0.01067732647061348, "contribution_to_active": 0.6253858208656311, "mix_mean": 0.6754169464111328, "mix_low": 0.0002040863037109375, "mix_high": 0.006134986877441406, "read_cosine": 0.04465286433696747, "beta_mean": 2.1716787159675732e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004742437973618507, "mix_std": 0.24594494700431824, "beta_std": 8.417133358307183e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001897827722132206, "evidence_weights": [-0.265718549489975, -0.2723842263221741, 0.7230181694030762], "agreement_weight": -0.23403537273406982, "orthogonal_weight": -0.3831019699573517}, {"head": 3, "memory_to_active": 0.3425405025482178, "contribution_to_active": 0.09107665717601776, "mix_mean": 0.2798541188240051, "mix_low": 0.022772789001464844, "mix_high": 3.0517578125e-05, "read_cosine": 0.929618239402771, "beta_mean": 0.06077640876173973, "nonfinite": 0.0, "weighted_memory_to_active": 0.03456563502550125, "mix_std": 0.22098349034786224, "beta_std": 0.05922609567642212, "beta_low": 0.008056640625, "beta_high": 0.0, "orthogonal_to_active": 0.035920869559049606, "evidence_weights": [-0.08232808858156204, -0.13913396000862122, 0.1446610689163208], "agreement_weight": 0.018874868750572205, "orthogonal_weight": 1.5048662424087524}, {"head": 4, "memory_to_active": 0.010011774487793446, "contribution_to_active": 0.583267331123352, "mix_mean": 0.608479380607605, "mix_low": 0.0002269744873046875, "mix_high": 0.0014858245849609375, "read_cosine": 0.09692380577325821, "beta_mean": 2.397152638877742e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006149479653686285, "mix_std": 0.2387169450521469, "beta_std": 5.155303006176837e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009015602990984917, "evidence_weights": [-0.18003714084625244, -0.2472921907901764, 0.9048821330070496], "agreement_weight": 0.20249053835868835, "orthogonal_weight": -1.5433317422866821}, {"head": 5, "memory_to_active": 0.116080641746521, "contribution_to_active": 0.4172457158565521, "mix_mean": 0.4234984815120697, "mix_low": 0.0014667510986328125, "mix_high": 4.76837158203125e-06, "read_cosine": 0.8129787445068359, "beta_mean": 0.006309411954134703, "nonfinite": 0.0, "weighted_memory_to_active": 0.05308960750699043, "mix_std": 0.2234797179698944, "beta_std": 0.009042958728969097, "beta_low": 0.83587646484375, "beta_high": 0.0, "orthogonal_to_active": 0.02063054032623768, "evidence_weights": [-0.13416418433189392, -0.16644786298274994, 0.2807575464248657], "agreement_weight": 0.09778357297182083, "orthogonal_weight": -0.6782587766647339}]}, {"layer": 11, "memory_to_active": 0.06937958300113678, "contribution_to_active": 0.6399160623550415, "mix_mean": 0.5347709059715271, "mix_low": 0.1130325049161911, "mix_high": 0.04923757165670395, "read_cosine": 0.3421005606651306, "beta_mean": 0.022390764206647873, "nonfinite": 0.0, "weighted_memory_to_active": 0.0387648269534111, "mix_std": 0.3223937153816223, "beta_std": 0.03361636400222778, "beta_low": 0.721405029296875, "beta_high": 0.0, "orthogonal_to_active": 0.04146607965230942, "output_influence": 0.5725663304328918, "residual_influence": 0.17405720055103302, "state_active": 0.21106639504432678, "state_memory": 0.0026321643963456154, "state_peak": 34.0030632019043, "growth_active": 1.073816180229187, "growth_memory": 1.4377784729003906, "heads": [{"head": 0, "memory_to_active": 0.006655511446297169, "contribution_to_active": 0.6192886233329773, "mix_mean": 0.6718456149101257, "mix_low": 0.0003986358642578125, "mix_high": 0.008998870849609375, "read_cosine": 0.2350960373878479, "beta_mean": 6.900572770973668e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005064975470304489, "mix_std": 0.24694426357746124, "beta_std": 0.003265364095568657, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004172537475824356, "evidence_weights": [-0.17512056231498718, -0.35265296697616577, 0.797653317451477], "agreement_weight": 0.4316712021827698, "orthogonal_weight": -0.8484678268432617}, {"head": 1, "memory_to_active": 0.10093369334936142, "contribution_to_active": 0.45574405789375305, "mix_mean": 0.48539790511131287, "mix_low": 0.003954887390136719, "mix_high": 0.0007152557373046875, "read_cosine": 0.48826029896736145, "beta_mean": 0.0014501817058771849, "nonfinite": 0.0, "weighted_memory_to_active": 0.05364512279629707, "mix_std": 0.26902130246162415, "beta_std": 0.007672699633985758, "beta_low": 0.97601318359375, "beta_high": 0.0, "orthogonal_to_active": 0.03007047437131405, "evidence_weights": [-0.12867005169391632, -0.278058797121048, 0.6002313494682312], "agreement_weight": 0.0916336327791214, "orthogonal_weight": -0.6658325791358948}, {"head": 2, "memory_to_active": 0.004404845647513866, "contribution_to_active": 0.9895509481430054, "mix_mean": 0.43765178322792053, "mix_low": 0.3281393051147461, "mix_high": 0.1847095489501953, "read_cosine": 0.1409776508808136, "beta_mean": 0.05984105169773102, "nonfinite": 0.0, "weighted_memory_to_active": 0.003114446997642517, "mix_std": 0.45347949862480164, "beta_std": 0.08282579481601715, "beta_low": 0.50738525390625, "beta_high": 0.0, "orthogonal_to_active": 0.03366231173276901, "evidence_weights": [0.04816850647330284, -0.3234279155731201, -1.3211137056350708], "agreement_weight": 0.002300422638654709, "orthogonal_weight": 11.136636734008789}, {"head": 3, "memory_to_active": 0.006451951805502176, "contribution_to_active": 0.8859891891479492, "mix_mean": 0.4168938398361206, "mix_low": 0.3385009765625, "mix_high": 0.09298038482666016, "read_cosine": 0.5816476345062256, "beta_mean": 0.04838190972805023, "nonfinite": 0.0, "weighted_memory_to_active": 0.004189063794910908, "mix_std": 0.44287198781967163, "beta_std": 0.07596997916698456, "beta_low": 0.54486083984375, "beta_high": 0.0, "orthogonal_to_active": 0.030863836407661438, "evidence_weights": [0.2266930192708969, -0.2632182240486145, -0.06385981291532516], "agreement_weight": 0.049050476402044296, "orthogonal_weight": 11.415931701660156}, {"head": 4, "memory_to_active": 0.21553529798984528, "contribution_to_active": 0.45375779271125793, "mix_mean": 0.6150652170181274, "mix_low": 0.006409645080566406, "mix_high": 0.0016984939575195312, "read_cosine": 0.006608095020055771, "beta_mean": 0.017242807894945145, "nonfinite": 0.0, "weighted_memory_to_active": 0.12469760328531265, "mix_std": 0.26746830344200134, "beta_std": 0.021003033965826035, "beta_low": 0.48968505859375, "beta_high": 0.0, "orthogonal_to_active": 0.12304901331663132, "evidence_weights": [-0.22693826258182526, -0.23485605418682098, 1.885796070098877], "agreement_weight": -0.2381230592727661, "orthogonal_weight": -0.9972390532493591}, {"head": 5, "memory_to_active": 0.08229617774486542, "contribution_to_active": 0.43516573309898376, "mix_mean": 0.5817707777023315, "mix_low": 0.0007915496826171875, "mix_high": 0.0063228607177734375, "read_cosine": 0.6000134944915771, "beta_mean": 0.007359628565609455, "nonfinite": 0.0, "weighted_memory_to_active": 0.04187772795557976, "mix_std": 0.2545769214630127, "beta_std": 0.010961301624774933, "beta_low": 0.81109619140625, "beta_high": 0.0, "orthogonal_to_active": 0.0269782692193985, "evidence_weights": [-0.18816055357456207, -0.3050576448440552, 0.6685919165611267], "agreement_weight": 0.0813475102186203, "orthogonal_weight": -0.8585057854652405}]}, {"layer": 12, "memory_to_active": 0.22045859694480896, "contribution_to_active": 0.5986796617507935, "mix_mean": 0.5562983751296997, "mix_low": 0.008375327102839947, "mix_high": 0.007303874008357525, "read_cosine": 0.21818691492080688, "beta_mean": 0.014377865940332413, "nonfinite": 0.0, "weighted_memory_to_active": 0.11182200908660889, "mix_std": 0.2557411789894104, "beta_std": 0.01581544801592827, "beta_low": 0.8280843496322632, "beta_high": 0.0, "orthogonal_to_active": 0.06753714382648468, "output_influence": 0.5308876037597656, "residual_influence": 0.17064014077186584, "state_active": 0.03468318283557892, "state_memory": 0.007120306137949228, "state_peak": 75.33641052246094, "growth_active": 1.191061019897461, "growth_memory": 1.1081141233444214, "heads": [{"head": 0, "memory_to_active": 0.03403698280453682, "contribution_to_active": 0.6397403478622437, "mix_mean": 0.5990572571754456, "mix_low": 0.000858306884765625, "mix_high": 0.0074939727783203125, "read_cosine": 0.2654920220375061, "beta_mean": 0.0005795134347863495, "nonfinite": 0.0, "weighted_memory_to_active": 0.01962178386747837, "mix_std": 0.2614242434501648, "beta_std": 0.0012469240464270115, "beta_low": 0.9969482421875, "beta_high": 0.0, "orthogonal_to_active": 0.01788024604320526, "evidence_weights": [-0.19899895787239075, -0.2684169113636017, -0.10048073530197144], "agreement_weight": 0.10326796025037766, "orthogonal_weight": -0.9761921763420105}, {"head": 1, "memory_to_active": 0.07059329748153687, "contribution_to_active": 0.5802140831947327, "mix_mean": 0.5754276514053345, "mix_low": 0.00026798248291015625, "mix_high": 0.0026788711547851562, "read_cosine": 0.39752206206321716, "beta_mean": 0.0004001006600446999, "nonfinite": 0.0, "weighted_memory_to_active": 0.04417398199439049, "mix_std": 0.2614216208457947, "beta_std": 0.007158451247960329, "beta_low": 0.9979248046875, "beta_high": 0.0, "orthogonal_to_active": 0.036142781376838684, "evidence_weights": [-0.19462400674819946, -0.2687385082244873, 1.0252443552017212], "agreement_weight": 0.29731321334838867, "orthogonal_weight": -0.9044486880302429}, {"head": 2, "memory_to_active": 0.5844991207122803, "contribution_to_active": 0.4758591651916504, "mix_mean": 0.31596577167510986, "mix_low": 0.025407791137695312, "mix_high": 1.33514404296875e-05, "read_cosine": 0.5806718468666077, "beta_mean": 0.08392558246850967, "nonfinite": 0.0, "weighted_memory_to_active": 0.21771834790706635, "mix_std": 0.23166586458683014, "beta_std": 0.0655052587389946, "beta_low": 0.00286865234375, "beta_high": 0.0, "orthogonal_to_active": 0.25485193729400635, "evidence_weights": [-0.19843333959579468, -0.06099192053079605, -0.02924313023686409], "agreement_weight": 0.026023877784609795, "orthogonal_weight": 1.4699589014053345}, {"head": 3, "memory_to_active": 0.02868473529815674, "contribution_to_active": 0.7817164063453674, "mix_mean": 0.7360778450965881, "mix_low": 9.441375732421875e-05, "mix_high": 0.022676467895507812, "read_cosine": 0.07284548878669739, "beta_mean": 0.00024049292551353574, "nonfinite": 0.0, "weighted_memory_to_active": 0.02108432725071907, "mix_std": 0.22167734801769257, "beta_std": 0.005377928726375103, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.019875550642609596, "evidence_weights": [-0.2830401360988617, -0.33016717433929443, 1.7177871465682983], "agreement_weight": 0.660273551940918, "orthogonal_weight": -0.9636929631233215}, {"head": 4, "memory_to_active": 0.5099449157714844, "contribution_to_active": 0.6090562343597412, "mix_mean": 0.4694957733154297, "mix_low": 0.016966819763183594, "mix_high": 0.0047855377197265625, "read_cosine": -0.2058449387550354, "beta_mean": 0.0006466216291300952, "nonfinite": 0.0, "weighted_memory_to_active": 0.3052823543548584, "mix_std": 0.2879444658756256, "beta_std": 0.008983202278614044, "beta_low": 0.9835205078125, "beta_high": 0.0, "orthogonal_to_active": 0.029692554846405983, "evidence_weights": [-0.1544336974620819, -0.21145567297935486, -0.15924222767353058], "agreement_weight": 0.022442661225795746, "orthogonal_weight": -0.10047242045402527}, {"head": 5, "memory_to_active": 0.094992496073246, "contribution_to_active": 0.5054917335510254, "mix_mean": 0.641765832901001, "mix_low": 0.006656646728515625, "mix_high": 0.006175041198730469, "read_cosine": 0.19843488931655884, "beta_mean": 0.0004748871724586934, "nonfinite": 0.0, "weighted_memory_to_active": 0.063051238656044, "mix_std": 0.2703133821487427, "beta_std": 0.00662092212587595, "beta_low": 0.98883056640625, "beta_high": 0.0, "orthogonal_to_active": 0.04677978530526161, "evidence_weights": [-0.21745319664478302, -0.307273268699646, 1.2245172262191772], "agreement_weight": 0.43551692366600037, "orthogonal_weight": -0.7984840273857117}]}, {"layer": 13, "memory_to_active": 0.11119117587804794, "contribution_to_active": 0.546810507774353, "mix_mean": 0.549675703048706, "mix_low": 0.006163756363093853, "mix_high": 0.009406408295035362, "read_cosine": 0.4209728240966797, "beta_mean": 0.013071427121758461, "nonfinite": 0.0, "weighted_memory_to_active": 0.047739334404468536, "mix_std": 0.25677376985549927, "beta_std": 0.014987196773290634, "beta_low": 0.8129171133041382, "beta_high": 0.0, "orthogonal_to_active": 0.051015954464673996, "output_influence": 0.5390485525131226, "residual_influence": 0.21029448509216309, "state_active": 0.12581954896450043, "state_memory": 0.038282182067632675, "state_peak": 96.301025390625, "growth_active": 0.9101741313934326, "growth_memory": 0.9738516807556152, "heads": [{"head": 0, "memory_to_active": 0.007555488497018814, "contribution_to_active": 0.619879424571991, "mix_mean": 0.6271906495094299, "mix_low": 0.0008563995361328125, "mix_high": 0.004960060119628906, "read_cosine": 0.4468514323234558, "beta_mean": 2.647052679094486e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005080764647573233, "mix_std": 0.26214826107025146, "beta_std": 5.997348125674762e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004612508229911327, "evidence_weights": [-0.18689580261707306, -0.2619171738624573, 0.717614471912384], "agreement_weight": 0.2503297030925751, "orthogonal_weight": -1.019906997680664}, {"head": 1, "memory_to_active": 0.0070113688707351685, "contribution_to_active": 0.6181493997573853, "mix_mean": 0.5835250616073608, "mix_low": 0.0013246536254882812, "mix_high": 0.004981040954589844, "read_cosine": 0.24819263815879822, "beta_mean": 2.8861957616754808e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003962882794439793, "mix_std": 0.27285122871398926, "beta_std": 6.367303285514936e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004875839222222567, "evidence_weights": [-0.17564575374126434, -0.2586599290370941, 0.5018532276153564], "agreement_weight": 0.6120452880859375, "orthogonal_weight": -1.2799147367477417}, {"head": 2, "memory_to_active": 0.06169328838586807, "contribution_to_active": 0.5063572525978088, "mix_mean": 0.4038270115852356, "mix_low": 0.0010356903076171875, "mix_high": 0.0001888275146484375, "read_cosine": 0.651590883731842, "beta_mean": 0.006049902644008398, "nonfinite": 0.0, "weighted_memory_to_active": 0.030705362558364868, "mix_std": 0.22966381907463074, "beta_std": 0.02285761572420597, "beta_low": 0.88165283203125, "beta_high": 0.0, "orthogonal_to_active": 0.016676731407642365, "evidence_weights": [-0.17262360453605652, -0.2145547866821289, 0.40901535749435425], "agreement_weight": 0.005031413398683071, "orthogonal_weight": -0.853394627571106}, {"head": 3, "memory_to_active": 0.16840773820877075, "contribution_to_active": 0.5452589988708496, "mix_mean": 0.6630377769470215, "mix_low": 0.026948928833007812, "mix_high": 0.03367424011230469, "read_cosine": -0.05280773341655731, "beta_mean": 0.0008938233950175345, "nonfinite": 0.0, "weighted_memory_to_active": 0.12177275121212006, "mix_std": 0.2914353013038635, "beta_std": 0.010722069069743156, "beta_low": 0.9952392578125, "beta_high": 0.0, "orthogonal_to_active": 0.1108773797750473, "evidence_weights": [-0.3149442970752716, -0.23788979649543762, 1.4409033060073853], "agreement_weight": 6.915136337280273, "orthogonal_weight": -0.9765410423278809}, {"head": 4, "memory_to_active": 0.38503438234329224, "contribution_to_active": 0.265301376581192, "mix_mean": 0.357977032661438, "mix_low": 0.004687309265136719, "mix_high": 0.0, "read_cosine": 0.8866050243377686, "beta_mean": 0.07126960903406143, "nonfinite": 0.0, "weighted_memory_to_active": 0.099699005484581, "mix_std": 0.21539048850536346, "beta_std": 0.05435200035572052, "beta_low": 0.001220703125, "beta_high": 0.0, "orthogonal_to_active": 0.14784999191761017, "evidence_weights": [-0.05353737249970436, -0.2187640517950058, 0.40680819749832153], "agreement_weight": 0.007215595804154873, "orthogonal_weight": 2.334702730178833}, {"head": 5, "memory_to_active": 0.03744475543498993, "contribution_to_active": 0.7259165644645691, "mix_mean": 0.6624967455863953, "mix_low": 0.0021295547485351562, "mix_high": 0.01263427734375, "read_cosine": 0.34540438652038574, "beta_mean": 0.00015989331586752087, "nonfinite": 0.0, "weighted_memory_to_active": 0.02521521970629692, "mix_std": 0.2691534757614136, "beta_std": 0.0018678480992093682, "beta_low": 0.9993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.02120327576994896, "evidence_weights": [-0.21730701625347137, -0.28140658140182495, 0.8747784495353699], "agreement_weight": 0.22413502633571625, "orthogonal_weight": -0.9285889863967896}]}, {"layer": 14, "memory_to_active": 0.2142964005470276, "contribution_to_active": 0.4160720705986023, "mix_mean": 0.4313943386077881, "mix_low": 0.027583125978708267, "mix_high": 0.006533145904541016, "read_cosine": 0.609481692314148, "beta_mean": 0.058801762759685516, "nonfinite": 0.0, "weighted_memory_to_active": 0.0865892618894577, "mix_std": 0.23663544654846191, "beta_std": 0.05773967131972313, "beta_low": 0.509735107421875, "beta_high": 0.0, "orthogonal_to_active": 0.0407387875020504, "output_influence": 0.3513181209564209, "residual_influence": 0.162570059299469, "state_active": 0.05023470148444176, "state_memory": 0.01290078554302454, "state_peak": 52.533851623535156, "growth_active": 1.0839146375656128, "growth_memory": 1.2047531604766846, "heads": [{"head": 0, "memory_to_active": 0.3625839352607727, "contribution_to_active": 0.19737601280212402, "mix_mean": 0.19519147276878357, "mix_low": 0.0177001953125, "mix_high": 0.0, "read_cosine": 0.8143209218978882, "beta_mean": 0.067204549908638, "nonfinite": 0.0, "weighted_memory_to_active": 0.0771971195936203, "mix_std": 0.17396491765975952, "beta_std": 0.06366616487503052, "beta_low": 0.01123046875, "beta_high": 0.0, "orthogonal_to_active": 0.09233975410461426, "evidence_weights": [-0.09166357666254044, -0.04504207894206047, 0.3520679175853729], "agreement_weight": 0.020414289087057114, "orthogonal_weight": 1.965279221534729}, {"head": 1, "memory_to_active": 0.4482177495956421, "contribution_to_active": 0.28480029106140137, "mix_mean": 0.19446590542793274, "mix_low": 0.13623332977294922, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8857312202453613, "beta_mean": 0.24411143362522125, "nonfinite": 0.0, "weighted_memory_to_active": 0.19484560191631317, "mix_std": 0.22494927048683167, "beta_std": 0.24141892790794373, "beta_low": 0.09625244140625, "beta_high": 0.0, "orthogonal_to_active": 0.020744619891047478, "evidence_weights": [0.017271433025598526, -0.18627944588661194, -0.10149802267551422], "agreement_weight": 0.05747130513191223, "orthogonal_weight": 0.35350358486175537}, {"head": 2, "memory_to_active": 0.11010996997356415, "contribution_to_active": 0.5893919467926025, "mix_mean": 0.5925313234329224, "mix_low": 0.00240325927734375, "mix_high": 0.005633354187011719, "read_cosine": 0.39438486099243164, "beta_mean": 0.0015185198280960321, "nonfinite": 0.0, "weighted_memory_to_active": 0.07075762748718262, "mix_std": 0.26336878538131714, "beta_std": 0.002259333385154605, "beta_low": 0.98712158203125, "beta_high": 0.0, "orthogonal_to_active": 0.05448920279741287, "evidence_weights": [-0.12355279922485352, -0.31321507692337036, 0.8969510197639465], "agreement_weight": 0.08433648198843002, "orthogonal_weight": -0.8649746775627136}, {"head": 3, "memory_to_active": 0.03814469277858734, "contribution_to_active": 0.5860480666160583, "mix_mean": 0.6831995248794556, "mix_low": 0.0037670135498046875, "mix_high": 0.02422809600830078, "read_cosine": -0.08351335674524307, "beta_mean": 0.005209904629737139, "nonfinite": 0.0, "weighted_memory_to_active": 0.024550698697566986, "mix_std": 0.2539481818675995, "beta_std": 0.008401881903409958, "beta_low": 0.85577392578125, "beta_high": 0.0, "orthogonal_to_active": 0.019153321161866188, "evidence_weights": [-0.3385636508464813, -0.3122396767139435, 0.31410959362983704], "agreement_weight": 0.06853849440813065, "orthogonal_weight": -0.8389318585395813}, {"head": 4, "memory_to_active": 0.15169072151184082, "contribution_to_active": 0.4950946569442749, "mix_mean": 0.5372895002365112, "mix_low": 0.0026178359985351562, "mix_high": 0.008286476135253906, "read_cosine": 0.9083152413368225, "beta_mean": 0.03395479917526245, "nonfinite": 0.0, "weighted_memory_to_active": 0.08839195221662521, "mix_std": 0.25631847977638245, "beta_std": 0.025305671617388725, "beta_low": 0.119140625, "beta_high": 0.0, "orthogonal_to_active": 0.023990269750356674, "evidence_weights": [-0.19940710067749023, -0.23348911106586456, 0.4534401297569275], "agreement_weight": 0.1343037188053131, "orthogonal_weight": -0.6286961436271667}, {"head": 5, "memory_to_active": 0.17503128945827484, "contribution_to_active": 0.3437214195728302, "mix_mean": 0.3856881260871887, "mix_low": 0.002777099609375, "mix_high": 0.0010480880737304688, "read_cosine": 0.7376512289047241, "beta_mean": 0.0008113789954222739, "nonfinite": 0.0, "weighted_memory_to_active": 0.06379255652427673, "mix_std": 0.24726295471191406, "beta_std": 0.0053860279731452465, "beta_low": 0.9888916015625, "beta_high": 0.0, "orthogonal_to_active": 0.03371555730700493, "evidence_weights": [-0.17122037708759308, -0.18504931032657623, 0.3792191743850708], "agreement_weight": 0.05887623131275177, "orthogonal_weight": -0.7607431411743164}]}, {"layer": 15, "memory_to_active": 0.04838164895772934, "contribution_to_active": 0.5042635202407837, "mix_mean": 0.44620805978775024, "mix_low": 0.005680720321834087, "mix_high": 0.0031549138948321342, "read_cosine": 0.6644250154495239, "beta_mean": 0.04053478315472603, "nonfinite": 0.0, "weighted_memory_to_active": 0.025053277611732483, "mix_std": 0.2818918824195862, "beta_std": 0.045013993978500366, "beta_low": 0.7029521465301514, "beta_high": 0.0, "orthogonal_to_active": 0.010534209199249744, "output_influence": 0.3293374180793762, "residual_influence": 0.20802628993988037, "state_active": 0.1710285097360611, "state_memory": 0.009619306772947311, "state_peak": 26.161317825317383, "growth_active": 1.2040003538131714, "growth_memory": 1.2097423076629639, "heads": [{"head": 0, "memory_to_active": 0.11679567396640778, "contribution_to_active": 0.48187607526779175, "mix_mean": 0.40899938344955444, "mix_low": 0.005794525146484375, "mix_high": 0.006566047668457031, "read_cosine": 0.4824634790420532, "beta_mean": 0.05253686010837555, "nonfinite": 0.0, "weighted_memory_to_active": 0.05994557589292526, "mix_std": 0.2867143452167511, "beta_std": 0.037372179329395294, "beta_low": 0.05255126953125, "beta_high": 0.0, "orthogonal_to_active": 0.04069795459508896, "evidence_weights": [-0.1793716549873352, -0.19453303515911102, 0.5577623844146729], "agreement_weight": 0.09045498073101044, "orthogonal_weight": -0.8566582798957825}, {"head": 1, "memory_to_active": 0.0743333175778389, "contribution_to_active": 0.43844684958457947, "mix_mean": 0.45557141304016113, "mix_low": 0.0005540847778320312, "mix_high": 0.007994651794433594, "read_cosine": 0.9296161532402039, "beta_mean": 0.00012432145013008267, "nonfinite": 0.0, "weighted_memory_to_active": 0.0313691645860672, "mix_std": 0.2808619737625122, "beta_std": 0.0011885337298735976, "beta_low": 0.9991455078125, "beta_high": 0.0, "orthogonal_to_active": 0.0030181675683707, "evidence_weights": [-0.16379661858081818, -0.17798179388046265, 0.15484417974948883], "agreement_weight": 0.0995873436331749, "orthogonal_weight": 0.20330114662647247}, {"head": 2, "memory_to_active": 0.007163468282669783, "contribution_to_active": 0.5286571979522705, "mix_mean": 0.3693738281726837, "mix_low": 0.009294509887695312, "mix_high": 8.869171142578125e-05, "read_cosine": 0.8155279755592346, "beta_mean": 8.317596075357869e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003982906695455313, "mix_std": 0.27368468046188354, "beta_std": 0.0002475498476997018, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.001358063891530037, "evidence_weights": [-0.19465427100658417, -0.2675391435623169, 0.12909483909606934], "agreement_weight": 0.010400076396763325, "orthogonal_weight": 0.9928410649299622}, {"head": 3, "memory_to_active": 0.0019844688940793276, "contribution_to_active": 0.3770081698894501, "mix_mean": 0.42682719230651855, "mix_low": 0.0031690597534179688, "mix_high": 0.0006084442138671875, "read_cosine": 0.6262133717536926, "beta_mean": 1.0422945706523024e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0006755177746526897, "mix_std": 0.28693723678588867, "beta_std": 2.51765814027749e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0003342100535519421, "evidence_weights": [-0.2745002508163452, -0.23982003331184387, 0.17462213337421417], "agreement_weight": 0.06462278962135315, "orthogonal_weight": -0.4944380223751068}, {"head": 4, "memory_to_active": 0.05897235870361328, "contribution_to_active": 0.6432199478149414, "mix_mean": 0.4107990562915802, "mix_low": 0.0062465667724609375, "mix_high": 0.0007619857788085938, "read_cosine": 0.7554316520690918, "beta_mean": 0.18734602630138397, "nonfinite": 0.0, "weighted_memory_to_active": 0.03305162116885185, "mix_std": 0.30778300762176514, "beta_std": 0.20976151525974274, "beta_low": 0.20025634765625, "beta_high": 0.0, "orthogonal_to_active": 0.005772086791694164, "evidence_weights": [-0.2040559947490692, -0.2510492205619812, -0.028736844658851624], "agreement_weight": 0.011164099909365177, "orthogonal_weight": 0.4083198606967926}, {"head": 5, "memory_to_active": 0.031040586531162262, "contribution_to_active": 0.5563727617263794, "mix_mean": 0.6056774854660034, "mix_low": 0.00902557373046875, "mix_high": 0.0029096603393554688, "read_cosine": 0.3772971034049988, "beta_mean": 0.003107875119894743, "nonfinite": 0.0, "weighted_memory_to_active": 0.021294869482517242, "mix_std": 0.25536999106407166, "beta_std": 0.0214889794588089, "beta_low": 0.9659423828125, "beta_high": 0.0, "orthogonal_to_active": 0.012024763971567154, "evidence_weights": [-0.2776322662830353, -0.30803319811820984, 0.5477399230003357], "agreement_weight": 0.31509461998939514, "orthogonal_weight": 0.8164213299751282}]}]}, {"step": 78000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.7821448419999797, "memory_to_active": 0.8152336478233337, "contribution_to_active": 0.5898606777191162, "mix_mean": 0.4353795647621155, "mix_low": 0.11892196536064148, "mix_high": 0.011041323654353619, "read_cosine": 0.3439645767211914, "beta_mean": 0.06803040951490402, "nonfinite": 0.0, "weighted_memory_to_active": 0.20175236463546753, "mix_std": 0.22125375270843506, "beta_std": 0.05904565006494522, "beta_low": 0.6004709005355835, "beta_high": 4.4504806282930076e-05, "orthogonal_to_active": 0.06673040986061096, "samples": 4, "scored_tokens": 899, "normal_loss": 3.005824089050293, "normal_token_accuracy": 0.4627363681793213, "state_active": 0.06475264579057693, "state_memory": 0.015680383890867233, "state_peak": 125.14424896240234, "growth_active": 1.1163554191589355, "growth_memory": 1.261785626411438, "output_influence": 0.4582974910736084, "residual_influence": 0.17387667298316956, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1285903453826904, "no_memory_loss_delta": 0.79749995470047, "scrambled_loss_delta": 0.14332489669322968, "memory_only_loss_delta": 3.9981327056884766, "no_evidence_loss_delta": 0.4558134973049164, "no_agreement_loss_delta": 0.012386652640998363, "no_orthogonal_loss_delta": 0.016697531566023827}, {"index": 94, "tokens": 256, "normal_loss": 3.087270736694336, "no_memory_loss_delta": 0.9207556247711182, "scrambled_loss_delta": 0.46883583068847656, "memory_only_loss_delta": 4.630658149719238, "no_evidence_loss_delta": 0.39230677485466003, "no_agreement_loss_delta": 0.041373156011104584, "no_orthogonal_loss_delta": 0.055397555232048035}, {"index": 188, "tokens": 256, "normal_loss": 3.3024635314941406, "no_memory_loss_delta": 1.0395017862319946, "scrambled_loss_delta": 0.126507967710495, "memory_only_loss_delta": 3.4251255989074707, "no_evidence_loss_delta": 0.5296262502670288, "no_agreement_loss_delta": 0.048721008002758026, "no_orthogonal_loss_delta": 0.044897906482219696}, {"index": 283, "tokens": 131, "normal_loss": 2.027059555053711, "no_memory_loss_delta": 0.41221460700035095, "scrambled_loss_delta": 0.06540465354919434, "memory_only_loss_delta": 2.9730448722839355, "no_evidence_loss_delta": 0.1714809238910675, "no_agreement_loss_delta": -0.0005044564022682607, "no_orthogonal_loss_delta": -0.03853220492601395}], "agreement_weight_abs_mean": 0.25830650329589844, "agreement_weight_abs_max": 6.92716121673584, "agreement_weight_mean": 0.2239161878824234, "orthogonal_weight_abs_mean": 0.9576923251152039, "orthogonal_weight_abs_max": 11.449040412902832, "orthogonal_weight_mean": -0.027168551459908485, "no_memory_loss": 3.8511922359466553, "no_memory_loss_delta": 0.8453682661056519, "no_memory_token_accuracy_gain": 0.12013348191976547, "no_memory_token_win_fraction": 0.7130144834518433, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 57.06008040800303, "scrambled_loss": 3.225698471069336, "scrambled_loss_delta": 0.2198745161294937, "scrambled_token_accuracy_gain": 0.03337041288614273, "scrambled_token_win_fraction": 0.6162402629852295, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 19.73804925540234, "memory_only_loss": 6.871532440185547, "memory_only_loss_delta": 3.8657078742980957, "memory_only_token_accuracy_gain": 0.3103448152542114, "memory_only_token_win_fraction": 0.9199110269546509, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 97.90519114260799, "no_evidence_loss": 3.423139810562134, "no_evidence_loss_delta": 0.4173160195350647, "no_evidence_token_accuracy_gain": 0.06340378522872925, "no_evidence_token_win_fraction": 0.6173526048660278, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 34.11873106363994, "no_agreement_loss": 3.034933090209961, "no_agreement_loss_delta": 0.029109014198184013, "no_agreement_token_accuracy_gain": 0.004449388012290001, "no_agreement_token_win_fraction": 0.4716351628303528, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.8689427948378525, "no_orthogonal_loss": 3.03352427482605, "no_orthogonal_loss_delta": 0.027700209990143776, "no_orthogonal_token_accuracy_gain": 0.0, "no_orthogonal_token_win_fraction": 0.5917686223983765, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.732007718007844, "short_no_memory_loss_delta": 0.8084315657615662, "long_context_memory_gain": 0.03693670034408569, "normal_context_loss_gain": 0.1259164661169052, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.734694004058838, "contribution_to_active": 1.277374029159546, "mix_mean": 0.11110709607601166, "mix_low": 0.49577969312667847, "mix_high": 0.0047656698152422905, "read_cosine": 0.4090418219566345, "beta_mean": 0.21843492984771729, "nonfinite": 0.0, "weighted_memory_to_active": 1.2930657863616943, "mix_std": 0.09191620349884033, "beta_std": 0.142114520072937, "beta_low": 0.1926473081111908, "beta_high": 0.0, "orthogonal_to_active": 0.052512772381305695, "output_influence": 0.2127007097005844, "residual_influence": 0.21705147624015808, "state_active": 0.06562136113643646, "state_memory": 0.0459725521504879, "state_peak": 46.173423767089844, "growth_active": 1.175925612449646, "growth_memory": 1.2484644651412964, "heads": [{"head": 0, "memory_to_active": 3.0941779613494873, "contribution_to_active": 0.44025927782058716, "mix_mean": 0.01844184845685959, "mix_low": 0.6266298294067383, "mix_high": 0.0, "read_cosine": 0.22342169284820557, "beta_mean": 0.2070792317390442, "nonfinite": 0.0, "weighted_memory_to_active": 0.5504747033119202, "mix_std": 0.055089469999074936, "beta_std": 0.09935259819030762, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0549885630607605, "evidence_weights": [-0.00194890599232167, -0.03803142532706261, -0.06176737695932388], "agreement_weight": -0.23783248662948608, "orthogonal_weight": -0.11685223877429962}, {"head": 1, "memory_to_active": 7.0271453857421875, "contribution_to_active": 3.209048271179199, "mix_mean": 0.01959424465894699, "mix_low": 0.6110572814941406, "mix_high": 0.0, "read_cosine": 0.43824249505996704, "beta_mean": 0.4628707766532898, "nonfinite": 0.0, "weighted_memory_to_active": 3.416654586791992, "mix_std": 0.05741652846336365, "beta_std": 0.20882590115070343, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11122000217437744, "evidence_weights": [0.024509502574801445, -0.09898058325052261, 0.674699068069458], "agreement_weight": 0.029283419251441956, "orthogonal_weight": 0.07258757203817368}, {"head": 2, "memory_to_active": 3.041821002960205, "contribution_to_active": 0.7073208689689636, "mix_mean": 0.014454668387770653, "mix_low": 0.6482086181640625, "mix_high": 0.0, "read_cosine": 0.2733038663864136, "beta_mean": 0.265735387802124, "nonfinite": 0.0, "weighted_memory_to_active": 0.8167227506637573, "mix_std": 0.031499236822128296, "beta_std": 0.21443748474121094, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.058971017599105835, "evidence_weights": [0.06298646330833435, 0.04585253819823265, 0.28257977962493896], "agreement_weight": 0.011256816796958447, "orthogonal_weight": -0.19619150459766388}, {"head": 3, "memory_to_active": 7.8742780685424805, "contribution_to_active": 2.775538921356201, "mix_mean": 0.0281657911837101, "mix_low": 0.49732303619384766, "mix_high": 0.0, "read_cosine": 0.2870294451713562, "beta_mean": 0.3382217586040497, "nonfinite": 0.0, "weighted_memory_to_active": 2.8765368461608887, "mix_std": 0.0648789331316948, "beta_std": 0.2712777554988861, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.07061976939439774, "evidence_weights": [-0.033856138586997986, -0.024117549881339073, -0.14225465059280396], "agreement_weight": -0.007020745892077684, "orthogonal_weight": -0.028918378055095673}, {"head": 4, "memory_to_active": 1.2685754299163818, "contribution_to_active": 0.06287877261638641, "mix_mean": 0.025688529014587402, "mix_low": 0.5792093276977539, "mix_high": 0.0, "read_cosine": 0.30464041233062744, "beta_mean": 0.018889974802732468, "nonfinite": 0.0, "weighted_memory_to_active": 0.043186210095882416, "mix_std": 0.06536711007356644, "beta_std": 0.03893151506781578, "beta_low": 0.65167236328125, "beta_high": 0.0, "orthogonal_to_active": 0.0010652788914740086, "evidence_weights": [0.08091474324464798, 0.04975612461566925, -0.14171451330184937], "agreement_weight": -0.21081702411174774, "orthogonal_weight": 0.02342512458562851}, {"head": 5, "memory_to_active": 0.10216477513313293, "contribution_to_active": 0.4691971242427826, "mix_mean": 0.5602974891662598, "mix_low": 0.012249946594238281, "mix_high": 0.028594017028808594, "read_cosine": 0.9276130199432373, "beta_mean": 0.01781252957880497, "nonfinite": 0.0, "weighted_memory_to_active": 0.05481975898146629, "mix_std": 0.2772459387779236, "beta_std": 0.019861843436956406, "beta_low": 0.50384521484375, "beta_high": 0.0, "orthogonal_to_active": 0.018212009221315384, "evidence_weights": [-0.1719895899295807, -0.2152405083179474, 0.3597671091556549], "agreement_weight": 0.2818422019481659, "orthogonal_weight": -0.7297476530075073}]}, {"layer": 1, "memory_to_active": 2.615083694458008, "contribution_to_active": 0.4989645183086395, "mix_mean": 0.22039008140563965, "mix_low": 0.36987242102622986, "mix_high": 0.0013395946007221937, "read_cosine": 0.29261860251426697, "beta_mean": 0.18392808735370636, "nonfinite": 0.0, "weighted_memory_to_active": 0.3395683765411377, "mix_std": 0.11238676309585571, "beta_std": 0.13206513226032257, "beta_low": 0.3548889458179474, "beta_high": 0.00015258790517691523, "orthogonal_to_active": 0.03867529332637787, "output_influence": 0.28164929151535034, "residual_influence": 0.08209669589996338, "state_active": 0.026596438139677048, "state_memory": 0.0336204394698143, "state_peak": 41.41156768798828, "growth_active": 1.0963085889816284, "growth_memory": 1.4255086183547974, "heads": [{"head": 0, "memory_to_active": 0.014203410595655441, "contribution_to_active": 0.49992501735687256, "mix_mean": 0.5557267665863037, "mix_low": 0.003753662109375, "mix_high": 0.0032587051391601562, "read_cosine": 0.02136325277388096, "beta_mean": 9.062593744602054e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00885188952088356, "mix_std": 0.2809196710586548, "beta_std": 0.00015419743431266397, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00858825258910656, "evidence_weights": [-0.21045005321502686, -0.269941508769989, 0.1813034862279892], "agreement_weight": 0.20095717906951904, "orthogonal_weight": -1.0116264820098877}, {"head": 1, "memory_to_active": 3.2166850566864014, "contribution_to_active": 0.30733513832092285, "mix_mean": 0.02009669691324234, "mix_low": 0.5101137161254883, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5138608813285828, "beta_mean": 0.2824499309062958, "nonfinite": 0.0, "weighted_memory_to_active": 0.33422696590423584, "mix_std": 0.0393136702477932, "beta_std": 0.19381918013095856, "beta_low": 0.0020751953125, "beta_high": 0.0003662109375, "orthogonal_to_active": 0.02276557870209217, "evidence_weights": [0.028852924704551697, -0.06147107481956482, 0.04825679585337639], "agreement_weight": -0.0013601597165688872, "orthogonal_weight": 0.21211740374565125}, {"head": 2, "memory_to_active": 0.03319668769836426, "contribution_to_active": 0.5496771335601807, "mix_mean": 0.6849396824836731, "mix_low": 0.00022411346435546875, "mix_high": 0.0047779083251953125, "read_cosine": 0.4490429759025574, "beta_mean": 0.0010987019632011652, "nonfinite": 0.0, "weighted_memory_to_active": 0.021983113139867783, "mix_std": 0.22434644401073456, "beta_std": 0.0033736699260771275, "beta_low": 0.982177734375, "beta_high": 0.0, "orthogonal_to_active": 0.016165705397725105, "evidence_weights": [-0.26822978258132935, -0.36509376764297485, 0.6343112587928772], "agreement_weight": 0.05302221328020096, "orthogonal_weight": -0.8407167196273804}, {"head": 3, "memory_to_active": 3.2611541748046875, "contribution_to_active": 0.25043004751205444, "mix_mean": 0.025074735283851624, "mix_low": 0.5042905807495117, "mix_high": 0.0, "read_cosine": 0.17926405370235443, "beta_mean": 0.3470517694950104, "nonfinite": 0.0, "weighted_memory_to_active": 0.32804185152053833, "mix_std": 0.06238974630832672, "beta_std": 0.2649456858634949, "beta_low": 0.0694580078125, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.11645607650279999, "evidence_weights": [0.03802910074591637, -0.020307958126068115, -0.01295147929340601], "agreement_weight": -0.02853788249194622, "orthogonal_weight": -0.4722172021865845}, {"head": 4, "memory_to_active": 4.306756019592285, "contribution_to_active": 1.2939320802688599, "mix_mean": 0.030196068808436394, "mix_low": 0.34877967834472656, "mix_high": 0.0, "read_cosine": 0.3235330283641815, "beta_mean": 0.16634608805179596, "nonfinite": 0.0, "weighted_memory_to_active": 1.2818408012390137, "mix_std": 0.04917721822857857, "beta_std": 0.1322518289089203, "beta_low": 0.043701171875, "beta_high": 0.000244140625, "orthogonal_to_active": 0.041550107300281525, "evidence_weights": [-0.008494428358972073, -0.0014504840364679694, 0.1775011569261551], "agreement_weight": -0.004642623942345381, "orthogonal_weight": 0.05154185742139816}, {"head": 5, "memory_to_active": 4.858506202697754, "contribution_to_active": 0.0924873873591423, "mix_mean": 0.006306495517492294, "mix_low": 0.8520727157592773, "mix_high": 0.0, "read_cosine": 0.2686474323272705, "beta_mean": 0.30653131008148193, "nonfinite": 0.0, "weighted_memory_to_active": 0.06246572732925415, "mix_std": 0.018173784017562866, "beta_std": 0.19784623384475708, "beta_low": 0.03192138671875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.026526043191552162, "evidence_weights": [0.1592654138803482, 0.07500890642404556, 0.08754248917102814], "agreement_weight": -0.11422543972730637, "orthogonal_weight": 0.41744565963745117}]}, {"layer": 2, "memory_to_active": 1.0567598342895508, "contribution_to_active": 0.3608332872390747, "mix_mean": 0.4164631962776184, "mix_low": 0.23753754794597626, "mix_high": 0.004446506500244141, "read_cosine": 0.2917080521583557, "beta_mean": 0.07923004776239395, "nonfinite": 0.0, "weighted_memory_to_active": 0.07067262381315231, "mix_std": 0.18321914970874786, "beta_std": 0.06684686988592148, "beta_low": 0.617462158203125, "beta_high": 0.00021362306142691523, "orthogonal_to_active": 0.0146899139508605, "output_influence": 0.41908949613571167, "residual_influence": 0.12410782277584076, "state_active": 0.040664613246917725, "state_memory": 0.012860900722444057, "state_peak": 28.596839904785156, "growth_active": 1.063591480255127, "growth_memory": 1.5398023128509521, "heads": [{"head": 0, "memory_to_active": 0.003013734705746174, "contribution_to_active": 0.3734608292579651, "mix_mean": 0.6527712941169739, "mix_low": 0.00048351287841796875, "mix_high": 0.009774208068847656, "read_cosine": 0.14255677163600922, "beta_mean": 2.5699766410980374e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0019974508322775364, "mix_std": 0.26263898611068726, "beta_std": 4.4098480429966e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020469045266509056, "evidence_weights": [-0.3710383176803589, -0.21239548921585083, 0.2133607566356659], "agreement_weight": 0.14203612506389618, "orthogonal_weight": -1.063063144683838}, {"head": 1, "memory_to_active": 3.3139944076538086, "contribution_to_active": 0.05043402314186096, "mix_mean": 0.005812034942209721, "mix_low": 0.8635387420654297, "mix_high": 0.0, "read_cosine": 0.12039237469434738, "beta_mean": 0.21532607078552246, "nonfinite": 0.0, "weighted_memory_to_active": 0.04041871801018715, "mix_std": 0.011510903015732765, "beta_std": 0.1446283459663391, "beta_low": 0.0035400390625, "beta_high": 0.0, "orthogonal_to_active": 0.01154661551117897, "evidence_weights": [0.1285630464553833, 0.099140964448452, 0.5701990127563477], "agreement_weight": 0.026321424171328545, "orthogonal_weight": 0.31723615527153015}, {"head": 2, "memory_to_active": 0.007071745116263628, "contribution_to_active": 0.4108509123325348, "mix_mean": 0.5620484352111816, "mix_low": 0.0008153915405273438, "mix_high": 0.00392913818359375, "read_cosine": 0.40704602003097534, "beta_mean": 1.9054359654546715e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002959689125418663, "mix_std": 0.2734512686729431, "beta_std": 3.786104571190663e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00023743932251818478, "evidence_weights": [-0.24481205642223358, -0.2888484597206116, 0.5652613639831543], "agreement_weight": -0.040312547236680984, "orthogonal_weight": 0.09984045475721359}, {"head": 3, "memory_to_active": 0.08801175653934479, "contribution_to_active": 0.5513023138046265, "mix_mean": 0.5927673578262329, "mix_low": 0.00016498565673828125, "mix_high": 0.005337715148925781, "read_cosine": 0.5206916928291321, "beta_mean": 0.013879918493330479, "nonfinite": 0.0, "weighted_memory_to_active": 0.0562271885573864, "mix_std": 0.24995899200439453, "beta_std": 0.02254042960703373, "beta_low": 0.64276123046875, "beta_high": 0.0, "orthogonal_to_active": 0.041345372796058655, "evidence_weights": [-0.1954541951417923, -0.32385098934173584, 0.7150022387504578], "agreement_weight": 0.05296700447797775, "orthogonal_weight": -0.8959585428237915}, {"head": 4, "memory_to_active": 2.921039342880249, "contribution_to_active": 0.24270197749137878, "mix_mean": 0.020241031423211098, "mix_low": 0.559840202331543, "mix_high": 0.0, "read_cosine": 0.18763890862464905, "beta_mean": 0.24610395729541779, "nonfinite": 0.0, "weighted_memory_to_active": 0.31825006008148193, "mix_std": 0.04689588025212288, "beta_std": 0.23379698395729065, "beta_low": 0.0584716796875, "beta_high": 0.00128173828125, "orthogonal_to_active": 0.03187600150704384, "evidence_weights": [0.02248883992433548, -0.0011759954504668713, 0.035089362412691116], "agreement_weight": 0.021192623302340508, "orthogonal_weight": -0.12072575092315674}, {"head": 5, "memory_to_active": 0.007427137345075607, "contribution_to_active": 0.5362496376037598, "mix_mean": 0.6651387810707092, "mix_low": 0.00038242340087890625, "mix_high": 0.007637977600097656, "read_cosine": 0.3719222843647003, "beta_mean": 2.5523375370539725e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00418259808793664, "mix_std": 0.2548587918281555, "beta_std": 3.348413883941248e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00108714634552598, "evidence_weights": [-0.27207672595977783, -0.2820611596107483, 0.8831228017807007], "agreement_weight": 0.293966680765152, "orthogonal_weight": -0.30538317561149597}]}, {"layer": 3, "memory_to_active": 0.40127575397491455, "contribution_to_active": 0.5036219358444214, "mix_mean": 0.5338948965072632, "mix_low": 0.05172872543334961, "mix_high": 0.01178344152867794, "read_cosine": 0.17800407111644745, "beta_mean": 0.023371098563075066, "nonfinite": 0.0, "weighted_memory_to_active": 0.1155451163649559, "mix_std": 0.2535218894481659, "beta_std": 0.023357799276709557, "beta_low": 0.777740478515625, "beta_high": 0.0, "orthogonal_to_active": 0.04764532670378685, "output_influence": 0.45609256625175476, "residual_influence": 0.12162865698337555, "state_active": 0.022714080289006233, "state_memory": 0.005940405651926994, "state_peak": 25.252763748168945, "growth_active": 1.1244301795959473, "growth_memory": 1.5747876167297363, "heads": [{"head": 0, "memory_to_active": 0.32454589009284973, "contribution_to_active": 0.630343496799469, "mix_mean": 0.6217657327651978, "mix_low": 0.0009555816650390625, "mix_high": 0.021943092346191406, "read_cosine": 0.22359448671340942, "beta_mean": 0.00871711503714323, "nonfinite": 0.0, "weighted_memory_to_active": 0.20445099472999573, "mix_std": 0.29341182112693787, "beta_std": 0.00992162898182869, "beta_low": 0.72271728515625, "beta_high": 0.0, "orthogonal_to_active": 0.15051838755607605, "evidence_weights": [-0.2127874791622162, -0.2464093118906021, 1.063713550567627], "agreement_weight": 0.11018498241901398, "orthogonal_weight": -0.7716282606124878}, {"head": 1, "memory_to_active": 0.006027693394571543, "contribution_to_active": 0.4795922338962555, "mix_mean": 0.6064493656158447, "mix_low": 0.0022783279418945312, "mix_high": 0.012320518493652344, "read_cosine": 0.17269322276115417, "beta_mean": 2.6751196855911985e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0037604172248393297, "mix_std": 0.28916263580322266, "beta_std": 6.829611083958298e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0030235876329243183, "evidence_weights": [-0.2660713195800781, -0.2149113416671753, 0.6831666231155396], "agreement_weight": 0.6158539056777954, "orthogonal_weight": -0.8720828890800476}, {"head": 2, "memory_to_active": 1.8425068855285645, "contribution_to_active": 0.27798154950141907, "mix_mean": 0.056726038455963135, "mix_low": 0.300140380859375, "mix_high": 0.0, "read_cosine": 0.22794857621192932, "beta_mean": 0.12734520435333252, "nonfinite": 0.0, "weighted_memory_to_active": 0.3114202618598938, "mix_std": 0.1084567978978157, "beta_std": 0.11928164958953857, "beta_low": 0.02581787109375, "beta_high": 0.0, "orthogonal_to_active": 0.006249414291232824, "evidence_weights": [-0.0998404249548912, -0.008240862749516964, -0.027260860428214073], "agreement_weight": -0.006719536613672972, "orthogonal_weight": 0.025901636108756065}, {"head": 3, "memory_to_active": 0.013714577071368694, "contribution_to_active": 0.5534495711326599, "mix_mean": 0.6229901313781738, "mix_low": 0.0031785964965820312, "mix_high": 0.010130882263183594, "read_cosine": 0.2774130403995514, "beta_mean": 6.0008449509041384e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009324113838374615, "mix_std": 0.27950146794319153, "beta_std": 0.00013527735427487642, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002147841965779662, "evidence_weights": [-0.26314976811408997, -0.22065789997577667, 2.354769229888916], "agreement_weight": 2.3883473873138428, "orthogonal_weight": -0.30343952775001526}, {"head": 4, "memory_to_active": 0.013939082622528076, "contribution_to_active": 0.41488003730773926, "mix_mean": 0.5566860437393188, "mix_low": 0.0015649795532226562, "mix_high": 0.0032825469970703125, "read_cosine": 0.1270826756954193, "beta_mean": 5.9938483900623396e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009508941322565079, "mix_std": 0.28620392084121704, "beta_std": 0.00010521881631575525, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008883091621100903, "evidence_weights": [-0.24550899863243103, -0.22228004038333893, 0.050791703164577484], "agreement_weight": -0.06677553057670593, "orthogonal_weight": -1.0215083360671997}, {"head": 5, "memory_to_active": 0.20692028105258942, "contribution_to_active": 0.6654845476150513, "mix_mean": 0.7387518882751465, "mix_low": 0.002254486083984375, "mix_high": 0.023023605346679688, "read_cosine": 0.0392923466861248, "beta_mean": 0.004017572849988937, "nonfinite": 0.0, "weighted_memory_to_active": 0.15480594336986542, "mix_std": 0.264394611120224, "beta_std": 0.01063472032546997, "beta_low": 0.91790771484375, "beta_high": 0.0, "orthogonal_to_active": 0.11504961550235748, "evidence_weights": [-0.25717926025390625, -0.34030619263648987, 0.5602812170982361], "agreement_weight": 0.24627119302749634, "orthogonal_weight": -0.7610284090042114}]}, {"layer": 4, "memory_to_active": 0.9966404438018799, "contribution_to_active": 0.4854666590690613, "mix_mean": 0.3792246878147125, "mix_low": 0.2188882827758789, "mix_high": 0.012114366516470909, "read_cosine": 0.2977644205093384, "beta_mean": 0.10141079872846603, "nonfinite": 0.0, "weighted_memory_to_active": 0.12807197868824005, "mix_std": 0.1616978645324707, "beta_std": 0.09343824535608292, "beta_low": 0.4020386040210724, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05732978880405426, "output_influence": 0.517543613910675, "residual_influence": 0.1753697544336319, "state_active": 0.03734143078327179, "state_memory": 0.015080108307301998, "state_peak": 42.90932083129883, "growth_active": 1.1940028667449951, "growth_memory": 1.5144413709640503, "heads": [{"head": 0, "memory_to_active": 0.13650920987129211, "contribution_to_active": 0.821173906326294, "mix_mean": 0.6937953233718872, "mix_low": 0.008671760559082031, "mix_high": 0.033431053161621094, "read_cosine": 0.42314937710762024, "beta_mean": 0.22732506692409515, "nonfinite": 0.0, "weighted_memory_to_active": 0.11775214970111847, "mix_std": 0.3450324833393097, "beta_std": 0.22475126385688782, "beta_low": 0.2607421875, "beta_high": 0.0, "orthogonal_to_active": 0.09249881654977798, "evidence_weights": [-0.23775272071361542, -0.3515373468399048, -0.24834132194519043], "agreement_weight": 0.00537131680175662, "orthogonal_weight": 0.9161237478256226}, {"head": 1, "memory_to_active": 0.039191871881484985, "contribution_to_active": 0.67059326171875, "mix_mean": 0.7121297717094421, "mix_low": 5.245208740234375e-05, "mix_high": 0.020854949951171875, "read_cosine": 0.5449032783508301, "beta_mean": 0.0008137811673805118, "nonfinite": 0.0, "weighted_memory_to_active": 0.029737256467342377, "mix_std": 0.23403798043727875, "beta_std": 0.006744138430804014, "beta_low": 0.9827880859375, "beta_high": 0.0, "orthogonal_to_active": 0.014391403645277023, "evidence_weights": [-0.23092685639858246, -0.36176085472106934, 0.5488839149475098], "agreement_weight": 0.3763653337955475, "orthogonal_weight": -0.663000762462616}, {"head": 2, "memory_to_active": 1.3799878358840942, "contribution_to_active": 0.37812086939811707, "mix_mean": 0.06410276889801025, "mix_low": 0.2699298858642578, "mix_high": 2.47955322265625e-05, "read_cosine": 0.2258998155593872, "beta_mean": 0.048968903720378876, "nonfinite": 0.0, "weighted_memory_to_active": 0.27972644567489624, "mix_std": 0.10645850747823715, "beta_std": 0.05419160798192024, "beta_low": 0.14080810546875, "beta_high": 0.0, "orthogonal_to_active": 0.11196751147508621, "evidence_weights": [0.02022288553416729, -0.07401208579540253, 0.39919525384902954], "agreement_weight": 0.12228808552026749, "orthogonal_weight": -0.5142918825149536}, {"head": 3, "memory_to_active": 1.8965721130371094, "contribution_to_active": 0.13585540652275085, "mix_mean": 0.03240334242582321, "mix_low": 0.39568519592285156, "mix_high": 0.0, "read_cosine": 0.22909358143806458, "beta_mean": 0.12918201088905334, "nonfinite": 0.0, "weighted_memory_to_active": 0.18179813027381897, "mix_std": 0.05322860926389694, "beta_std": 0.11847291886806488, "beta_low": 0.02789306640625, "beta_high": 0.0, "orthogonal_to_active": 0.0035883374512195587, "evidence_weights": [-0.023070063441991806, 0.049122389405965805, -0.2613753378391266], "agreement_weight": 0.07536166906356812, "orthogonal_weight": 0.028300222009420395}, {"head": 4, "memory_to_active": 0.006803792901337147, "contribution_to_active": 0.6604126691818237, "mix_mean": 0.7591235637664795, "mix_low": 1.049041748046875e-05, "mix_high": 0.018369674682617188, "read_cosine": 0.12891018390655518, "beta_mean": 1.4347927390190307e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005215719807893038, "mix_std": 0.20430850982666016, "beta_std": 4.6744018618483096e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003260565223172307, "evidence_weights": [-0.32320636510849, -0.3111219108104706, 0.21869945526123047], "agreement_weight": 0.3211536407470703, "orthogonal_weight": -0.6641144156455994}, {"head": 5, "memory_to_active": 2.520777702331543, "contribution_to_active": 0.2466437816619873, "mix_mean": 0.013792986050248146, "mix_low": 0.6389799118041992, "mix_high": 5.7220458984375e-06, "read_cosine": 0.23463018238544464, "beta_mean": 0.20216065645217896, "nonfinite": 0.0, "weighted_memory_to_active": 0.1542021781206131, "mix_std": 0.027121173217892647, "beta_std": 0.15642277896404266, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.1182720959186554, "evidence_weights": [0.05620158836245537, -0.018375640735030174, -0.2600834369659424], "agreement_weight": -0.09258078038692474, "orthogonal_weight": 1.133151888847351}]}, {"layer": 5, "memory_to_active": 0.3630586862564087, "contribution_to_active": 0.5520785450935364, "mix_mean": 0.5258736610412598, "mix_low": 0.055581092834472656, "mix_high": 0.020486515015363693, "read_cosine": 0.10165915638208389, "beta_mean": 0.049095313996076584, "nonfinite": 0.0, "weighted_memory_to_active": 0.12349896132946014, "mix_std": 0.24866609275341034, "beta_std": 0.06434200704097748, "beta_low": 0.69110107421875, "beta_high": 0.0, "orthogonal_to_active": 0.054445307701826096, "output_influence": 0.609162449836731, "residual_influence": 0.21844105422496796, "state_active": 0.03822000324726105, "state_memory": 0.008977600373327732, "state_peak": 60.91023254394531, "growth_active": 1.111293911933899, "growth_memory": 1.232452630996704, "heads": [{"head": 0, "memory_to_active": 0.15459546446800232, "contribution_to_active": 0.5909435749053955, "mix_mean": 0.5168707370758057, "mix_low": 0.005329132080078125, "mix_high": 0.0020847320556640625, "read_cosine": -0.10498353093862534, "beta_mean": 0.014582648873329163, "nonfinite": 0.0, "weighted_memory_to_active": 0.058812420815229416, "mix_std": 0.27449923753738403, "beta_std": 0.026550795882940292, "beta_low": 0.61077880859375, "beta_high": 0.0, "orthogonal_to_active": 0.02697184309363365, "evidence_weights": [-0.16850334405899048, -0.18692491948604584, 0.1198829784989357], "agreement_weight": -0.021232444792985916, "orthogonal_weight": -0.4801650941371918}, {"head": 1, "memory_to_active": 0.003206418827176094, "contribution_to_active": 0.5760639905929565, "mix_mean": 0.7173494100570679, "mix_low": 0.0005035400390625, "mix_high": 0.03849029541015625, "read_cosine": 0.2571384310722351, "beta_mean": 7.079527676978614e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0020700222812592983, "mix_std": 0.260689914226532, "beta_std": 1.4797471521887928e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019035697914659977, "evidence_weights": [-0.24304881691932678, -0.2823317050933838, 0.9409438371658325], "agreement_weight": 0.7484105825424194, "orthogonal_weight": -0.9734594225883484}, {"head": 2, "memory_to_active": 0.013851485215127468, "contribution_to_active": 0.6207488179206848, "mix_mean": 0.6696174740791321, "mix_low": 0.0016050338745117188, "mix_high": 0.020265579223632812, "read_cosine": -0.395672082901001, "beta_mean": 7.232523785205558e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009315533563494682, "mix_std": 0.2572421729564667, "beta_std": 0.00030156708089634776, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.00923190452158451, "evidence_weights": [-0.2537781298160553, -0.30716264247894287, 0.1793709546327591], "agreement_weight": -0.2088879942893982, "orthogonal_weight": -1.0619423389434814}, {"head": 3, "memory_to_active": 1.5440914630889893, "contribution_to_active": 0.2465890496969223, "mix_mean": 0.11956807971000671, "mix_low": 0.11184215545654297, "mix_high": 0.0, "read_cosine": 0.1492304503917694, "beta_mean": 0.09349977970123291, "nonfinite": 0.0, "weighted_memory_to_active": 0.3708849847316742, "mix_std": 0.14964623749256134, "beta_std": 0.14558176696300507, "beta_low": 0.2191162109375, "beta_high": 0.0, "orthogonal_to_active": 0.1523171365261078, "evidence_weights": [-0.07516501843929291, -0.09763853251934052, 0.13307900726795197], "agreement_weight": 0.0484791062772274, "orthogonal_weight": -0.4965844452381134}, {"head": 4, "memory_to_active": 0.401151180267334, "contribution_to_active": 0.5772942900657654, "mix_mean": 0.315594881772995, "mix_low": 0.21285629272460938, "mix_high": 0.0039196014404296875, "read_cosine": 0.5440592169761658, "beta_mean": 0.18624594807624817, "nonfinite": 0.0, "weighted_memory_to_active": 0.25699275732040405, "mix_std": 0.33438611030578613, "beta_std": 0.21280665695667267, "beta_low": 0.318115234375, "beta_high": 0.0, "orthogonal_to_active": 0.1117313876748085, "evidence_weights": [-0.10713685303926468, -0.24660071730613708, -0.08044352382421494], "agreement_weight": 0.026994239538908005, "orthogonal_weight": 0.680999755859375}, {"head": 5, "memory_to_active": 0.061456017196178436, "contribution_to_active": 0.7008312940597534, "mix_mean": 0.8162412643432617, "mix_low": 0.00135040283203125, "mix_high": 0.05815887451171875, "read_cosine": 0.16018235683441162, "beta_mean": 0.00016406772192567587, "nonfinite": 0.0, "weighted_memory_to_active": 0.04291801527142525, "mix_std": 0.2155328094959259, "beta_std": 0.0007964216638356447, "beta_low": 0.9986572265625, "beta_high": 0.0, "orthogonal_to_active": 0.02451600506901741, "evidence_weights": [-0.2987416982650757, -0.32205674052238464, 1.7991323471069336], "agreement_weight": 0.5665448904037476, "orthogonal_weight": -0.5996448397636414}]}, {"layer": 6, "memory_to_active": 0.22591999173164368, "contribution_to_active": 0.6621376872062683, "mix_mean": 0.6046299934387207, "mix_low": 0.029224872589111328, "mix_high": 0.029876234009861946, "read_cosine": 0.3062558174133301, "beta_mean": 0.060353271663188934, "nonfinite": 0.0, "weighted_memory_to_active": 0.12103931605815887, "mix_std": 0.2384321689605713, "beta_std": 0.07252249121665955, "beta_low": 0.6360067129135132, "beta_high": 0.0, "orthogonal_to_active": 0.0967567041516304, "output_influence": 0.6012736558914185, "residual_influence": 0.23642174899578094, "state_active": 0.04100220277905464, "state_memory": 0.0049116723239421844, "state_peak": 89.79046630859375, "growth_active": 1.1734602451324463, "growth_memory": 1.0997729301452637, "heads": [{"head": 0, "memory_to_active": 0.11098740994930267, "contribution_to_active": 0.791377067565918, "mix_mean": 0.8301929235458374, "mix_low": 0.0001468658447265625, "mix_high": 0.06917476654052734, "read_cosine": 0.2843671441078186, "beta_mean": 0.010683382861316204, "nonfinite": 0.0, "weighted_memory_to_active": 0.09969980269670486, "mix_std": 0.19589641690254211, "beta_std": 0.035658128559589386, "beta_low": 0.8546142578125, "beta_high": 0.0, "orthogonal_to_active": 0.06373828649520874, "evidence_weights": [-0.28808262944221497, -0.3372901678085327, 1.1604547500610352], "agreement_weight": 0.9761437773704529, "orthogonal_weight": -0.6852308511734009}, {"head": 1, "memory_to_active": 0.0028676134534180164, "contribution_to_active": 0.5313144326210022, "mix_mean": 0.6136953830718994, "mix_low": 0.000331878662109375, "mix_high": 0.01898956298828125, "read_cosine": 0.4786074161529541, "beta_mean": 6.153735739644617e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0013588480651378632, "mix_std": 0.27849847078323364, "beta_std": 1.3713277439819649e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006928087677806616, "evidence_weights": [-0.1870657354593277, -0.2984763979911804, 1.0945885181427002], "agreement_weight": 0.13983285427093506, "orthogonal_weight": -0.674640417098999}, {"head": 2, "memory_to_active": 0.5908342599868774, "contribution_to_active": 0.10919959843158722, "mix_mean": 0.06377105414867401, "mix_low": 0.1682300567626953, "mix_high": 9.5367431640625e-07, "read_cosine": 0.27727586030960083, "beta_mean": 0.01722557470202446, "nonfinite": 0.0, "weighted_memory_to_active": 0.0661497637629509, "mix_std": 0.08767589926719666, "beta_std": 0.020154669880867004, "beta_low": 0.40802001953125, "beta_high": 0.0, "orthogonal_to_active": 0.01399662159383297, "evidence_weights": [0.05138950049877167, -0.07783862948417664, 0.3863567113876343], "agreement_weight": 0.13140568137168884, "orthogonal_weight": -0.2466178983449936}, {"head": 3, "memory_to_active": 0.12415090203285217, "contribution_to_active": 0.6872119903564453, "mix_mean": 0.6750748157501221, "mix_low": 0.0052814483642578125, "mix_high": 0.024492263793945312, "read_cosine": 0.4512749910354614, "beta_mean": 0.21416932344436646, "nonfinite": 0.0, "weighted_memory_to_active": 0.08995448797941208, "mix_std": 0.32466423511505127, "beta_std": 0.24520128965377808, "beta_low": 0.21722412109375, "beta_high": 0.0, "orthogonal_to_active": 0.058576345443725586, "evidence_weights": [-0.16046646237373352, -0.306003600358963, 0.08989894390106201], "agreement_weight": -0.010141263715922832, "orthogonal_weight": 0.7366803288459778}, {"head": 4, "memory_to_active": 0.5098828673362732, "contribution_to_active": 1.1993616819381714, "mix_mean": 0.7339514493942261, "mix_low": 0.0012311935424804688, "mix_high": 0.057196617126464844, "read_cosine": 0.07115710526704788, "beta_mean": 0.11996504664421082, "nonfinite": 0.0, "weighted_memory_to_active": 0.45662546157836914, "mix_std": 0.31083646416664124, "beta_std": 0.13392657041549683, "beta_low": 0.33624267578125, "beta_high": 0.0, "orthogonal_to_active": 0.4316171109676361, "evidence_weights": [-0.14664538204669952, -0.48964664340019226, -0.8416659235954285], "agreement_weight": -0.017496010288596153, "orthogonal_weight": 0.9666109681129456}, {"head": 5, "memory_to_active": 0.016796961426734924, "contribution_to_active": 0.6543610095977783, "mix_mean": 0.7110941410064697, "mix_low": 0.0001277923583984375, "mix_high": 0.009403228759765625, "read_cosine": 0.2748524248600006, "beta_mean": 7.016541349003091e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.012447492219507694, "mix_std": 0.23302148282527924, "beta_std": 0.00018053344683721662, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.011919058859348297, "evidence_weights": [-0.22959986329078674, -0.31285804510116577, 0.34520235657691956], "agreement_weight": 0.24188357591629028, "orthogonal_weight": -1.0243875980377197}]}, {"layer": 7, "memory_to_active": 1.810910701751709, "contribution_to_active": 0.2783367335796356, "mix_mean": 0.18413636088371277, "mix_low": 0.20868651568889618, "mix_high": 0.0009264945983886719, "read_cosine": 0.25382405519485474, "beta_mean": 0.13574227690696716, "nonfinite": 0.0, "weighted_memory_to_active": 0.196543350815773, "mix_std": 0.14770233631134033, "beta_std": 0.09693305194377899, "beta_low": 0.2344563901424408, "beta_high": 0.00032552084303461015, "orthogonal_to_active": 0.08632979542016983, "output_influence": 0.3008900284767151, "residual_influence": 0.07955502718687057, "state_active": 0.014331279322504997, "state_memory": 0.022803841158747673, "state_peak": 32.8664436340332, "growth_active": 1.1584243774414062, "growth_memory": 1.2701597213745117, "heads": [{"head": 0, "memory_to_active": 2.8949904441833496, "contribution_to_active": 0.151699498295784, "mix_mean": 0.03833817318081856, "mix_low": 0.4095630645751953, "mix_high": 9.5367431640625e-07, "read_cosine": 0.35620272159576416, "beta_mean": 0.1882414072751999, "nonfinite": 0.0, "weighted_memory_to_active": 0.1752030849456787, "mix_std": 0.07567963004112244, "beta_std": 0.13511773943901062, "beta_low": 0.0008544921875, "beta_high": 0.0006103515625, "orthogonal_to_active": 0.0013019804609939456, "evidence_weights": [0.04590458422899246, -0.09207668900489807, 0.03380664810538292], "agreement_weight": -0.028347132727503777, "orthogonal_weight": -0.007821914739906788}, {"head": 1, "memory_to_active": 0.9562574625015259, "contribution_to_active": 0.3920580744743347, "mix_mean": 0.22560471296310425, "mix_low": 0.10521697998046875, "mix_high": 0.004752159118652344, "read_cosine": 0.4172831177711487, "beta_mean": 0.09249909967184067, "nonfinite": 0.0, "weighted_memory_to_active": 0.3328196406364441, "mix_std": 0.24357828497886658, "beta_std": 0.0633772611618042, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.04507888853549957, "evidence_weights": [-0.2184857577085495, -0.10472334921360016, -0.3168434202671051], "agreement_weight": 0.05285818502306938, "orthogonal_weight": 0.16343998908996582}, {"head": 2, "memory_to_active": 0.5215334892272949, "contribution_to_active": 0.41184189915657043, "mix_mean": 0.25628161430358887, "mix_low": 0.011000633239746094, "mix_high": 0.000415802001953125, "read_cosine": 0.12698626518249512, "beta_mean": 0.0279239509254694, "nonfinite": 0.0, "weighted_memory_to_active": 0.21553367376327515, "mix_std": 0.21911859512329102, "beta_std": 0.033805519342422485, "beta_low": 0.25048828125, "beta_high": 0.0, "orthogonal_to_active": 0.13046301901340485, "evidence_weights": [-0.10128434002399445, -0.18888920545578003, 0.8928105235099792], "agreement_weight": 0.010728385299444199, "orthogonal_weight": -0.6248809099197388}, {"head": 3, "memory_to_active": 4.9581804275512695, "contribution_to_active": 0.1029827892780304, "mix_mean": 0.036538735032081604, "mix_low": 0.2921733856201172, "mix_high": 0.0, "read_cosine": 0.08886292576789856, "beta_mean": 0.44742774963378906, "nonfinite": 0.0, "weighted_memory_to_active": 0.3609620928764343, "mix_std": 0.05039161443710327, "beta_std": 0.282200425863266, "beta_low": 0.0015869140625, "beta_high": 0.0013427734375, "orthogonal_to_active": 0.31165048480033875, "evidence_weights": [0.07143691182136536, -0.0487125962972641, 0.7039973139762878], "agreement_weight": 0.04722382128238678, "orthogonal_weight": -0.9339336156845093}, {"head": 4, "memory_to_active": 1.5086690187454224, "contribution_to_active": 0.11890903115272522, "mix_mean": 0.02852439507842064, "mix_low": 0.4327716827392578, "mix_high": 0.0, "read_cosine": 0.2747231423854828, "beta_mean": 0.058263469487428665, "nonfinite": 0.0, "weighted_memory_to_active": 0.08172442018985748, "mix_std": 0.05112864449620247, "beta_std": 0.06681817024946213, "beta_low": 0.1539306640625, "beta_high": 0.0, "orthogonal_to_active": 0.017326418310403824, "evidence_weights": [0.10501117259263992, -0.038511745631694794, 0.18758410215377808], "agreement_weight": 0.032146111130714417, "orthogonal_weight": -0.283892959356308}, {"head": 5, "memory_to_active": 0.02583281137049198, "contribution_to_active": 0.4925289452075958, "mix_mean": 0.5195303559303284, "mix_low": 0.0013933181762695312, "mix_high": 0.00039005279541015625, "read_cosine": 0.2588861584663391, "beta_mean": 9.796427912078798e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01301717571914196, "mix_std": 0.24631716310977936, "beta_std": 0.0002792007871903479, "beta_low": 0.99981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.01215799804776907, "evidence_weights": [-0.18807658553123474, -0.24386344850063324, 0.46793681383132935], "agreement_weight": 0.2567073106765747, "orthogonal_weight": -0.9944518208503723}]}, {"layer": 8, "memory_to_active": 0.1369885355234146, "contribution_to_active": 0.5814398527145386, "mix_mean": 0.5204923748970032, "mix_low": 0.008831501007080078, "mix_high": 0.002222220180556178, "read_cosine": 0.38299721479415894, "beta_mean": 0.012109948322176933, "nonfinite": 0.0, "weighted_memory_to_active": 0.07081686705350876, "mix_std": 0.24200934171676636, "beta_std": 0.018475433811545372, "beta_low": 0.788787841796875, "beta_high": 0.0, "orthogonal_to_active": 0.06467342376708984, "output_influence": 0.5404655933380127, "residual_influence": 0.216334730386734, "state_active": 0.04142012447118759, "state_memory": 0.004276413470506668, "state_peak": 65.08533477783203, "growth_active": 1.1279293298721313, "growth_memory": 1.0302740335464478, "heads": [{"head": 0, "memory_to_active": 0.05850400775671005, "contribution_to_active": 0.5736707448959351, "mix_mean": 0.6138229966163635, "mix_low": 0.00039958953857421875, "mix_high": 0.0014362335205078125, "read_cosine": 0.43730780482292175, "beta_mean": 0.0030209810938686132, "nonfinite": 0.0, "weighted_memory_to_active": 0.04232089966535568, "mix_std": 0.24664786458015442, "beta_std": 0.012502921745181084, "beta_low": 0.9359130859375, "beta_high": 0.0, "orthogonal_to_active": 0.026581896468997, "evidence_weights": [-0.14600354433059692, -0.3691237270832062, 0.5473822355270386], "agreement_weight": 0.2738651633262634, "orthogonal_weight": -0.7882986068725586}, {"head": 1, "memory_to_active": 0.056959860026836395, "contribution_to_active": 0.5551106333732605, "mix_mean": 0.536805272102356, "mix_low": 0.00010395050048828125, "mix_high": 0.0023756027221679688, "read_cosine": 0.08517627418041229, "beta_mean": 0.0007152833859436214, "nonfinite": 0.0, "weighted_memory_to_active": 0.02558143436908722, "mix_std": 0.2305939942598343, "beta_std": 0.006775339599698782, "beta_low": 0.9945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.019167423248291016, "evidence_weights": [-0.18825331330299377, -0.26744526624679565, 0.549274742603302], "agreement_weight": 0.06712310761213303, "orthogonal_weight": -0.784184455871582}, {"head": 2, "memory_to_active": 0.009356923401355743, "contribution_to_active": 0.7016015648841858, "mix_mean": 0.6978127956390381, "mix_low": 0.000202178955078125, "mix_high": 0.004452705383300781, "read_cosine": 0.401766300201416, "beta_mean": 0.0001000206102617085, "nonfinite": 0.0, "weighted_memory_to_active": 0.007486237213015556, "mix_std": 0.21589191257953644, "beta_std": 0.0032820249907672405, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004263649694621563, "evidence_weights": [-0.21419434249401093, -0.34060558676719666, 0.9832703471183777], "agreement_weight": 0.3694964051246643, "orthogonal_weight": -0.6476419568061829}, {"head": 3, "memory_to_active": 0.09955775737762451, "contribution_to_active": 0.646025538444519, "mix_mean": 0.5952396392822266, "mix_low": 0.0006475448608398438, "mix_high": 0.004856109619140625, "read_cosine": 0.8078387975692749, "beta_mean": 0.007568106986582279, "nonfinite": 0.0, "weighted_memory_to_active": 0.0761093869805336, "mix_std": 0.2545841336250305, "beta_std": 0.007427346892654896, "beta_low": 0.74700927734375, "beta_high": 0.0, "orthogonal_to_active": 0.022463258355855942, "evidence_weights": [-0.16106747090816498, -0.29182544350624084, 0.5135584473609924], "agreement_weight": 0.2748206853866577, "orthogonal_weight": -0.5818679332733154}, {"head": 4, "memory_to_active": 0.2633592188358307, "contribution_to_active": 0.3762470781803131, "mix_mean": 0.3919270634651184, "mix_low": 0.0010166168212890625, "mix_high": 1.33514404296875e-05, "read_cosine": 0.08511973172426224, "beta_mean": 0.0006092028343118727, "nonfinite": 0.0, "weighted_memory_to_active": 0.09686333686113358, "mix_std": 0.23657220602035522, "beta_std": 0.014056062325835228, "beta_low": 0.9949951171875, "beta_high": 0.0, "orthogonal_to_active": 0.01696535386145115, "evidence_weights": [-0.1499091237783432, -0.20654194056987762, 0.6663543581962585], "agreement_weight": 0.02888163924217224, "orthogonal_weight": -0.17711962759494781}, {"head": 5, "memory_to_active": 0.3341934084892273, "contribution_to_active": 0.6359837055206299, "mix_mean": 0.2873465418815613, "mix_low": 0.05061912536621094, "mix_high": 0.00019931793212890625, "read_cosine": 0.48077425360679626, "beta_mean": 0.060646094381809235, "nonfinite": 0.0, "weighted_memory_to_active": 0.17653991281986237, "mix_std": 0.26776593923568726, "beta_std": 0.06680889427661896, "beta_low": 0.06036376953125, "beta_high": 0.0, "orthogonal_to_active": 0.2985989451408386, "evidence_weights": [-0.15122753381729126, -0.14001065492630005, -0.11018780618906021], "agreement_weight": -0.006510485429316759, "orthogonal_weight": 1.9829764366149902}]}, {"layer": 9, "memory_to_active": 0.8340560793876648, "contribution_to_active": 0.7373048663139343, "mix_mean": 0.41190361976623535, "mix_low": 0.04430612176656723, "mix_high": 0.006545384880155325, "read_cosine": 0.3553212285041809, "beta_mean": 0.03316032141447067, "nonfinite": 0.0, "weighted_memory_to_active": 0.374194860458374, "mix_std": 0.2482551783323288, "beta_std": 0.026530753821134567, "beta_low": 0.6685791015625, "beta_high": 0.0, "orthogonal_to_active": 0.030215328559279442, "output_influence": 0.5802881717681885, "residual_influence": 0.1897766888141632, "state_active": 0.04606332257390022, "state_memory": 0.013291759416460991, "state_peak": 61.79045104980469, "growth_active": 1.191798448562622, "growth_memory": 1.2115346193313599, "heads": [{"head": 0, "memory_to_active": 0.0002224401687271893, "contribution_to_active": 0.5946047306060791, "mix_mean": 0.6696118116378784, "mix_low": 0.0007276535034179688, "mix_high": 0.02705860137939453, "read_cosine": 0.35317522287368774, "beta_mean": 4.896219820693659e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.00015132964472286403, "mix_std": 0.2783661484718323, "beta_std": 1.4739099469807115e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001456248137401417, "evidence_weights": [-0.30229347944259644, -0.30686432123184204, 0.3058703541755676], "agreement_weight": 0.10716494917869568, "orthogonal_weight": -1.0212053060531616}, {"head": 1, "memory_to_active": 0.013640714809298515, "contribution_to_active": 0.7331158518791199, "mix_mean": 0.5901588201522827, "mix_low": 0.006592750549316406, "mix_high": 0.0032672882080078125, "read_cosine": 0.45658600330352783, "beta_mean": 2.7970825613010675e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010448482818901539, "mix_std": 0.27378758788108826, "beta_std": 6.58641365589574e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006259080488234758, "evidence_weights": [-0.2327306866645813, -0.2887139618396759, 0.9254291653633118], "agreement_weight": 0.5830561518669128, "orthogonal_weight": -0.7523202896118164}, {"head": 2, "memory_to_active": 0.15718427300453186, "contribution_to_active": 0.6686109900474548, "mix_mean": 0.608720064163208, "mix_low": 0.0016040802001953125, "mix_high": 0.006958961486816406, "read_cosine": 0.38664281368255615, "beta_mean": 0.0005239129532128572, "nonfinite": 0.0, "weighted_memory_to_active": 0.10927288979291916, "mix_std": 0.26111575961112976, "beta_std": 0.004243867471814156, "beta_low": 0.9967041015625, "beta_high": 0.0, "orthogonal_to_active": 0.0602232925593853, "evidence_weights": [-0.19612623751163483, -0.272581547498703, 0.749946117401123], "agreement_weight": 0.37618133425712585, "orthogonal_weight": -0.6557044982910156}, {"head": 3, "memory_to_active": 0.6817868947982788, "contribution_to_active": 0.3080214560031891, "mix_mean": 0.2033866047859192, "mix_low": 0.061974525451660156, "mix_high": 5.14984130859375e-05, "read_cosine": 0.3740900158882141, "beta_mean": 0.0255613774061203, "nonfinite": 0.0, "weighted_memory_to_active": 0.19683298468589783, "mix_std": 0.21981079876422882, "beta_std": 0.028047330677509308, "beta_low": 0.289794921875, "beta_high": 0.0, "orthogonal_to_active": 0.09786643087863922, "evidence_weights": [-0.05234606936573982, -0.12559440732002258, 0.5526050925254822], "agreement_weight": 0.08890588581562042, "orthogonal_weight": -0.5560837984085083}, {"head": 4, "memory_to_active": 3.1454763412475586, "contribution_to_active": 1.7679811716079712, "mix_mean": 0.15143287181854248, "mix_low": 0.1162118911743164, "mix_high": 0.0011796951293945312, "read_cosine": 0.11167319118976593, "beta_mean": 0.15635159611701965, "nonfinite": 0.0, "weighted_memory_to_active": 1.653354525566101, "mix_std": 0.2101948857307434, "beta_std": 0.0838998332619667, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001541231613373384, "evidence_weights": [-0.07809720188379288, -0.14703711867332458, -0.04618683084845543], "agreement_weight": -0.07031642645597458, "orthogonal_weight": 9.618711919756606e-05}, {"head": 5, "memory_to_active": 1.0060253143310547, "contribution_to_active": 0.3514947295188904, "mix_mean": 0.2481115460395813, "mix_low": 0.07872581481933594, "mix_high": 0.0007562637329101562, "read_cosine": 0.44975996017456055, "beta_mean": 0.0164965707808733, "nonfinite": 0.0, "weighted_memory_to_active": 0.2751089334487915, "mix_std": 0.24625585973262787, "beta_std": 0.04292614758014679, "beta_low": 0.7249755859375, "beta_high": 0.0, "orthogonal_to_active": 0.01664341427385807, "evidence_weights": [0.024512629956007004, -0.20507505536079407, 0.14563851058483124], "agreement_weight": 0.008535094559192657, "orthogonal_weight": 0.06734716147184372}]}, {"layer": 10, "memory_to_active": 0.21610823273658752, "contribution_to_active": 0.7921994924545288, "mix_mean": 0.5364923477172852, "mix_low": 0.02372312732040882, "mix_high": 0.005742550361901522, "read_cosine": 0.3825243413448334, "beta_mean": 0.042096130549907684, "nonfinite": 0.0, "weighted_memory_to_active": 0.09170354902744293, "mix_std": 0.263952374458313, "beta_std": 0.04375498741865158, "beta_low": 0.6683452129364014, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.31622904539108276, "output_influence": 0.49122604727745056, "residual_influence": 0.18523000180721283, "state_active": 0.05254397913813591, "state_memory": 0.013280494138598442, "state_peak": 125.14424896240234, "growth_active": 1.0060290098190308, "growth_memory": 1.0018550157546997, "heads": [{"head": 0, "memory_to_active": 0.8041386604309082, "contribution_to_active": 2.386270523071289, "mix_mean": 0.5713174939155579, "mix_low": 0.11948013305664062, "mix_high": 0.008316993713378906, "read_cosine": 0.0666738897562027, "beta_mean": 0.18372756242752075, "nonfinite": 0.0, "weighted_memory_to_active": 0.4518166184425354, "mix_std": 0.3986831307411194, "beta_std": 0.1947924792766571, "beta_low": 0.168701171875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.8260928392410278, "evidence_weights": [-0.07899736613035202, -0.3559877574443817, -2.0282227993011475], "agreement_weight": 0.04748687893152237, "orthogonal_weight": 4.067201614379883}, {"head": 1, "memory_to_active": 0.004216628149151802, "contribution_to_active": 0.6471632122993469, "mix_mean": 0.6513609886169434, "mix_low": 0.0007696151733398438, "mix_high": 0.018548011779785156, "read_cosine": 0.3411455750465393, "beta_mean": 2.6601388526614755e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002553265541791916, "mix_std": 0.25693464279174805, "beta_std": 5.077597961644642e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00390344625338912, "evidence_weights": [-0.24734686315059662, -0.1898612231016159, 0.541950523853302], "agreement_weight": 0.6888446807861328, "orthogonal_weight": -1.6670918464660645}, {"head": 2, "memory_to_active": 0.009320177137851715, "contribution_to_active": 0.62105393409729, "mix_mean": 0.6749159097671509, "mix_low": 0.00016307830810546875, "mix_high": 0.006009101867675781, "read_cosine": 0.0356399342417717, "beta_mean": 2.0401244910317473e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004150493536144495, "mix_std": 0.24578483402729034, "beta_std": 6.661058432655409e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.001655325759202242, "evidence_weights": [-0.26520487666130066, -0.2714439034461975, 0.7238901257514954], "agreement_weight": -0.23738768696784973, "orthogonal_weight": -0.38375788927078247}, {"head": 3, "memory_to_active": 0.36218637228012085, "contribution_to_active": 0.09650105237960815, "mix_mean": 0.28601133823394775, "mix_low": 0.020447731018066406, "mix_high": 2.574920654296875e-05, "read_cosine": 0.9329478740692139, "beta_mean": 0.0624716691672802, "nonfinite": 0.0, "weighted_memory_to_active": 0.03718038275837898, "mix_std": 0.22257444262504578, "beta_std": 0.05887088179588318, "beta_low": 0.006103515625, "beta_high": 0.0, "orthogonal_to_active": 0.0391756072640419, "evidence_weights": [-0.08212528377771378, -0.13959890604019165, 0.14406436681747437], "agreement_weight": 0.017922399565577507, "orthogonal_weight": 1.5119938850402832}, {"head": 4, "memory_to_active": 0.008187615312635899, "contribution_to_active": 0.5862878561019897, "mix_mean": 0.6106952428817749, "mix_low": 0.0001964569091796875, "mix_high": 0.0015516281127929688, "read_cosine": 0.09802137315273285, "beta_mean": 1.9259314285591245e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005020690616220236, "mix_std": 0.23715128004550934, "beta_std": 3.882728924509138e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007351425942033529, "evidence_weights": [-0.17951925098896027, -0.2466920018196106, 0.9002825021743774], "agreement_weight": 0.2041328102350235, "orthogonal_weight": -1.5429025888442993}, {"head": 5, "memory_to_active": 0.1085999459028244, "contribution_to_active": 0.4159201383590698, "mix_mean": 0.4246531128883362, "mix_low": 0.00128173828125, "mix_high": 3.814697265625e-06, "read_cosine": 0.8207173347473145, "beta_mean": 0.006311281584203243, "nonfinite": 0.0, "weighted_memory_to_active": 0.04949980974197388, "mix_std": 0.22258594632148743, "beta_std": 0.008710341528058052, "beta_low": 0.83526611328125, "beta_high": 0.0, "orthogonal_to_active": 0.01919533871114254, "evidence_weights": [-0.13329941034317017, -0.16612103581428528, 0.2806686758995056], "agreement_weight": 0.09805788099765778, "orthogonal_weight": -0.6784864664077759}]}, {"layer": 11, "memory_to_active": 0.0702807605266571, "contribution_to_active": 0.6397751569747925, "mix_mean": 0.5365164279937744, "mix_low": 0.11178398132324219, "mix_high": 0.05079937353730202, "read_cosine": 0.3340058922767639, "beta_mean": 0.023331671953201294, "nonfinite": 0.0, "weighted_memory_to_active": 0.039537571370601654, "mix_std": 0.32168856263160706, "beta_std": 0.034849099814891815, "beta_low": 0.7231547236442566, "beta_high": 0.0, "orthogonal_to_active": 0.04132936894893646, "output_influence": 0.5707722902297974, "residual_influence": 0.17660745978355408, "state_active": 0.2237195372581482, "state_memory": 0.002668858040124178, "state_peak": 33.72714614868164, "growth_active": 1.0943573713302612, "growth_memory": 1.4314780235290527, "heads": [{"head": 0, "memory_to_active": 0.008133519440889359, "contribution_to_active": 0.6132424473762512, "mix_mean": 0.6752288341522217, "mix_low": 0.0003566741943359375, "mix_high": 0.00893402099609375, "read_cosine": 0.21870100498199463, "beta_mean": 8.207491191569716e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.006195222493261099, "mix_std": 0.2446339726448059, "beta_std": 0.0035281095188111067, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.005083558615297079, "evidence_weights": [-0.17501942813396454, -0.3520565330982208, 0.7984906435012817], "agreement_weight": 0.4363057017326355, "orthogonal_weight": -0.850034236907959}, {"head": 1, "memory_to_active": 0.10472066700458527, "contribution_to_active": 0.4618203639984131, "mix_mean": 0.48858118057250977, "mix_low": 0.00408172607421875, "mix_high": 0.0006837844848632812, "read_cosine": 0.48772814869880676, "beta_mean": 0.0014552779030054808, "nonfinite": 0.0, "weighted_memory_to_active": 0.0550515353679657, "mix_std": 0.267747163772583, "beta_std": 0.007709639146924019, "beta_low": 0.9757080078125, "beta_high": 0.0, "orthogonal_to_active": 0.030598558485507965, "evidence_weights": [-0.12730051577091217, -0.2772562503814697, 0.6005190014839172], "agreement_weight": 0.09158632904291153, "orthogonal_weight": -0.6639782190322876}, {"head": 2, "memory_to_active": 0.004130153451114893, "contribution_to_active": 0.9900505542755127, "mix_mean": 0.440735399723053, "mix_low": 0.32427310943603516, "mix_high": 0.19266700744628906, "read_cosine": 0.13744503259658813, "beta_mean": 0.0632980465888977, "nonfinite": 0.0, "weighted_memory_to_active": 0.0029276800341904163, "mix_std": 0.4541986286640167, "beta_std": 0.08683470636606216, "beta_low": 0.51190185546875, "beta_high": 0.0, "orthogonal_to_active": 0.03176020085811615, "evidence_weights": [0.047553520649671555, -0.32242751121520996, -1.324198842048645], "agreement_weight": -8.284255454782397e-05, "orthogonal_weight": 11.16897201538086}, {"head": 3, "memory_to_active": 0.006166106089949608, "contribution_to_active": 0.8944603800773621, "mix_mean": 0.4187169075012207, "mix_low": 0.33508872985839844, "mix_high": 0.09406185150146484, "read_cosine": 0.5527951121330261, "beta_mean": 0.05039330944418907, "nonfinite": 0.0, "weighted_memory_to_active": 0.004121874459087849, "mix_std": 0.4432799518108368, "beta_std": 0.07852175086736679, "beta_low": 0.5498046875, "beta_high": 0.0, "orthogonal_to_active": 0.02889067679643631, "evidence_weights": [0.22785983979701996, -0.2612425982952118, -0.06007222831249237], "agreement_weight": 0.049291159957647324, "orthogonal_weight": 11.449040412902832}, {"head": 4, "memory_to_active": 0.21817420423030853, "contribution_to_active": 0.44429486989974976, "mix_mean": 0.6148037910461426, "mix_low": 0.006195068359375, "mix_high": 0.0016193389892578125, "read_cosine": 0.016917653381824493, "beta_mean": 0.017461981624364853, "nonfinite": 0.0, "weighted_memory_to_active": 0.12722501158714294, "mix_std": 0.2667376697063446, "beta_std": 0.021061941981315613, "beta_low": 0.4794921875, "beta_high": 0.0, "orthogonal_to_active": 0.12517064809799194, "evidence_weights": [-0.2255658656358719, -0.23341277241706848, 1.8857195377349854], "agreement_weight": -0.2395067662000656, "orthogonal_weight": -0.9950974583625793}, {"head": 5, "memory_to_active": 0.08035994321107864, "contribution_to_active": 0.4347822070121765, "mix_mean": 0.5810322761535645, "mix_low": 0.0007085800170898438, "mix_high": 0.0068302154541015625, "read_cosine": 0.5904483795166016, "beta_mean": 0.0072993328794837, "nonfinite": 0.0, "weighted_memory_to_active": 0.041704099625349045, "mix_std": 0.25353384017944336, "beta_std": 0.011438461020588875, "beta_low": 0.82269287109375, "beta_high": 0.0, "orthogonal_to_active": 0.026472555473446846, "evidence_weights": [-0.1863129436969757, -0.30383214354515076, 0.6697787642478943], "agreement_weight": 0.08154859393835068, "orthogonal_weight": -0.8590346574783325}]}, {"layer": 12, "memory_to_active": 0.212684765458107, "contribution_to_active": 0.5982488393783569, "mix_mean": 0.5595461130142212, "mix_low": 0.0079015102237463, "mix_high": 0.0073175434954464436, "read_cosine": 0.2278667390346527, "beta_mean": 0.014851873740553856, "nonfinite": 0.0, "weighted_memory_to_active": 0.10701194405555725, "mix_std": 0.25521472096443176, "beta_std": 0.015063149854540825, "beta_low": 0.8293660879135132, "beta_high": 0.0, "orthogonal_to_active": 0.06662151962518692, "output_influence": 0.5332942605018616, "residual_influence": 0.1716686636209488, "state_active": 0.034474942833185196, "state_memory": 0.007028099615126848, "state_peak": 79.41300201416016, "growth_active": 1.1749176979064941, "growth_memory": 1.1387062072753906, "heads": [{"head": 0, "memory_to_active": 0.03046242892742157, "contribution_to_active": 0.6429015398025513, "mix_mean": 0.6018908023834229, "mix_low": 0.000957489013671875, "mix_high": 0.007378578186035156, "read_cosine": 0.28050267696380615, "beta_mean": 0.00048131850780919194, "nonfinite": 0.0, "weighted_memory_to_active": 0.01740771345794201, "mix_std": 0.26057299971580505, "beta_std": 0.0010061103384941816, "beta_low": 0.99853515625, "beta_high": 0.0, "orthogonal_to_active": 0.015676476061344147, "evidence_weights": [-0.19847995042800903, -0.2680750787258148, -0.10340236872434616], "agreement_weight": 0.10248054563999176, "orthogonal_weight": -0.9712250232696533}, {"head": 1, "memory_to_active": 0.07117536664009094, "contribution_to_active": 0.5844941139221191, "mix_mean": 0.5794272422790527, "mix_low": 0.00022125244140625, "mix_high": 0.002758026123046875, "read_cosine": 0.4134121537208557, "beta_mean": 0.0003975997387897223, "nonfinite": 0.0, "weighted_memory_to_active": 0.044122014194726944, "mix_std": 0.26010194420814514, "beta_std": 0.007043448742479086, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.03589126467704773, "evidence_weights": [-0.19423885643482208, -0.2684679925441742, 1.0232943296432495], "agreement_weight": 0.3041066825389862, "orthogonal_weight": -0.9048560857772827}, {"head": 2, "memory_to_active": 0.5926148295402527, "contribution_to_active": 0.4762987792491913, "mix_mean": 0.323638916015625, "mix_low": 0.023682594299316406, "mix_high": 3.24249267578125e-05, "read_cosine": 0.5972959995269775, "beta_mean": 0.08705225586891174, "nonfinite": 0.0, "weighted_memory_to_active": 0.22256620228290558, "mix_std": 0.23264257609844208, "beta_std": 0.06375223398208618, "beta_low": 0.0018310546875, "beta_high": 0.0, "orthogonal_to_active": 0.2572806477546692, "evidence_weights": [-0.1984245777130127, -0.061530131846666336, -0.029034029692411423], "agreement_weight": 0.02722172439098358, "orthogonal_weight": 1.4761658906936646}, {"head": 3, "memory_to_active": 0.0344788022339344, "contribution_to_active": 0.7813158631324768, "mix_mean": 0.735515832901001, "mix_low": 8.296966552734375e-05, "mix_high": 0.02254486083984375, "read_cosine": 0.07887773215770721, "beta_mean": 0.0002998393902089447, "nonfinite": 0.0, "weighted_memory_to_active": 0.025694847106933594, "mix_std": 0.22078119218349457, "beta_std": 0.006428604479879141, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.024264466017484665, "evidence_weights": [-0.2818848490715027, -0.32896357774734497, 1.7285810708999634], "agreement_weight": 0.667792797088623, "orthogonal_weight": -0.9640498757362366}, {"head": 4, "memory_to_active": 0.46653297543525696, "contribution_to_active": 0.5963187217712402, "mix_mean": 0.47405052185058594, "mix_low": 0.01617431640625, "mix_high": 0.004902839660644531, "read_cosine": -0.21324467658996582, "beta_mean": 0.0005226465873420238, "nonfinite": 0.0, "weighted_memory_to_active": 0.27851295471191406, "mix_std": 0.28803545236587524, "beta_std": 0.0074648503214120865, "beta_low": 0.987060546875, "beta_high": 0.0, "orthogonal_to_active": 0.026812465861439705, "evidence_weights": [-0.15425051748752594, -0.21127159893512726, -0.1592661589384079], "agreement_weight": 0.024240490049123764, "orthogonal_weight": -0.09947183728218079}, {"head": 5, "memory_to_active": 0.08084416389465332, "contribution_to_active": 0.5081639885902405, "mix_mean": 0.6427533030509949, "mix_low": 0.006290435791015625, "mix_high": 0.0062885284423828125, "read_cosine": 0.2103564739227295, "beta_mean": 0.00035757431760430336, "nonfinite": 0.0, "weighted_memory_to_active": 0.053767964243888855, "mix_std": 0.26915407180786133, "beta_std": 0.0046836514957249165, "beta_low": 0.992919921875, "beta_high": 0.0, "orthogonal_to_active": 0.03980374336242676, "evidence_weights": [-0.21780459582805634, -0.30723729729652405, 1.2198556661605835], "agreement_weight": 0.43717077374458313, "orthogonal_weight": -0.7968792915344238}]}, {"layer": 13, "memory_to_active": 0.10825113207101822, "contribution_to_active": 0.5470837354660034, "mix_mean": 0.5512082576751709, "mix_low": 0.005937099456787109, "mix_high": 0.009099802002310753, "read_cosine": 0.42032933235168457, "beta_mean": 0.013423432596027851, "nonfinite": 0.0, "weighted_memory_to_active": 0.0460672490298748, "mix_std": 0.25463980436325073, "beta_std": 0.01457037404179573, "beta_low": 0.8139750361442566, "beta_high": 0.0, "orthogonal_to_active": 0.048416730016469955, "output_influence": 0.5374084711074829, "residual_influence": 0.21033704280853271, "state_active": 0.1280003935098648, "state_memory": 0.03780779987573624, "state_peak": 96.90975952148438, "growth_active": 0.9100212454795837, "growth_memory": 0.9648786783218384, "heads": [{"head": 0, "memory_to_active": 0.006906844675540924, "contribution_to_active": 0.6190188527107239, "mix_mean": 0.6265959739685059, "mix_low": 0.0008134841918945312, "mix_high": 0.004253387451171875, "read_cosine": 0.4464753568172455, "beta_mean": 2.4825309083098546e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0045940992422401905, "mix_std": 0.26051366329193115, "beta_std": 6.011496589053422e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00416978495195508, "evidence_weights": [-0.1864594668149948, -0.26152023673057556, 0.7148621082305908], "agreement_weight": 0.25200772285461426, "orthogonal_weight": -1.0198906660079956}, {"head": 1, "memory_to_active": 0.00754073541611433, "contribution_to_active": 0.6228302717208862, "mix_mean": 0.5844337940216064, "mix_low": 0.00127410888671875, "mix_high": 0.0045623779296875, "read_cosine": 0.23747272789478302, "beta_mean": 3.030810876225587e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004327381029725075, "mix_std": 0.2710988521575928, "beta_std": 6.383272557286546e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005297801923006773, "evidence_weights": [-0.1754363775253296, -0.258699506521225, 0.4979124367237091], "agreement_weight": 0.6154472231864929, "orthogonal_weight": -1.2753260135650635}, {"head": 2, "memory_to_active": 0.06340306252241135, "contribution_to_active": 0.5103169679641724, "mix_mean": 0.4110102653503418, "mix_low": 0.000911712646484375, "mix_high": 0.000148773193359375, "read_cosine": 0.6506962180137634, "beta_mean": 0.005749932490289211, "nonfinite": 0.0, "weighted_memory_to_active": 0.032322511076927185, "mix_std": 0.22978204488754272, "beta_std": 0.022111346945166588, "beta_low": 0.8878173828125, "beta_high": 0.0, "orthogonal_to_active": 0.017650822177529335, "evidence_weights": [-0.17303656041622162, -0.2154897004365921, 0.4104483127593994], "agreement_weight": 0.005145602859556675, "orthogonal_weight": -0.8531783223152161}, {"head": 3, "memory_to_active": 0.16394542157649994, "contribution_to_active": 0.5474043488502502, "mix_mean": 0.6703990697860718, "mix_low": 0.026297569274902344, "mix_high": 0.033542633056640625, "read_cosine": -0.058749906718730927, "beta_mean": 0.0008488129824399948, "nonfinite": 0.0, "weighted_memory_to_active": 0.11990021914243698, "mix_std": 0.2873293161392212, "beta_std": 0.009972905740141869, "beta_low": 0.99566650390625, "beta_high": 0.0, "orthogonal_to_active": 0.1096118688583374, "evidence_weights": [-0.3160141706466675, -0.2385432869195938, 1.4273970127105713], "agreement_weight": 6.92716121673584, "orthogonal_weight": -0.9789636731147766}, {"head": 4, "memory_to_active": 0.3762684464454651, "contribution_to_active": 0.2513200044631958, "mix_mean": 0.34939849376678467, "mix_low": 0.004344940185546875, "mix_high": 0.0, "read_cosine": 0.895389199256897, "beta_mean": 0.0737585723400116, "nonfinite": 0.0, "weighted_memory_to_active": 0.09395076334476471, "mix_std": 0.21171995997428894, "beta_std": 0.05371217802166939, "beta_low": 0.0008544921875, "beta_high": 0.0, "orthogonal_to_active": 0.13594429194927216, "evidence_weights": [-0.05266552418470383, -0.21818529069423676, 0.40497833490371704], "agreement_weight": 0.007896119728684425, "orthogonal_weight": 2.3400309085845947}, {"head": 5, "memory_to_active": 0.03144225850701332, "contribution_to_active": 0.731611967086792, "mix_mean": 0.6654119491577148, "mix_low": 0.0019807815551757812, "mix_high": 0.012091636657714844, "read_cosine": 0.3506922125816345, "beta_mean": 0.00012814342335332185, "nonfinite": 0.0, "weighted_memory_to_active": 0.021308530122041702, "mix_std": 0.2673949599266052, "beta_std": 0.0015018677804619074, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.01782580465078354, "evidence_weights": [-0.21726852655410767, -0.2809597849845886, 0.8786985874176025], "agreement_weight": 0.22613872587680817, "orthogonal_weight": -0.9263082146644592}]}, {"layer": 14, "memory_to_active": 0.21383927762508392, "contribution_to_active": 0.4158651828765869, "mix_mean": 0.43106043338775635, "mix_low": 0.02731546014547348, "mix_high": 0.006174882873892784, "read_cosine": 0.6136932373046875, "beta_mean": 0.06011199206113815, "nonfinite": 0.0, "weighted_memory_to_active": 0.08596386760473251, "mix_std": 0.2354820966720581, "beta_std": 0.05761027708649635, "beta_low": 0.5054830312728882, "beta_high": 0.0, "orthogonal_to_active": 0.04145614057779312, "output_influence": 0.3532119393348694, "residual_influence": 0.1651228964328766, "state_active": 0.0488855354487896, "state_memory": 0.012704720720648766, "state_peak": 54.1541862487793, "growth_active": 1.085676908493042, "growth_memory": 1.2403923273086548, "heads": [{"head": 0, "memory_to_active": 0.34712621569633484, "contribution_to_active": 0.19827035069465637, "mix_mean": 0.19752025604248047, "mix_low": 0.015437126159667969, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8128303289413452, "beta_mean": 0.0676935538649559, "nonfinite": 0.0, "weighted_memory_to_active": 0.07564801722764969, "mix_std": 0.17574450373649597, "beta_std": 0.06066933274269104, "beta_low": 0.01068115234375, "beta_high": 0.0, "orthogonal_to_active": 0.09113793075084686, "evidence_weights": [-0.09112230688333511, -0.04481504112482071, 0.3511195182800293], "agreement_weight": 0.019167423248291016, "orthogonal_weight": 1.972759485244751}, {"head": 1, "memory_to_active": 0.44371068477630615, "contribution_to_active": 0.282137930393219, "mix_mean": 0.19097639620304108, "mix_low": 0.1376943588256836, "mix_high": 0.0, "read_cosine": 0.8776713609695435, "beta_mean": 0.24661682546138763, "nonfinite": 0.0, "weighted_memory_to_active": 0.18325193226337433, "mix_std": 0.2216523140668869, "beta_std": 0.24135226011276245, "beta_low": 0.08941650390625, "beta_high": 0.0, "orthogonal_to_active": 0.020762603729963303, "evidence_weights": [0.018758734688162804, -0.18491123616695404, -0.1035289317369461], "agreement_weight": 0.056408874690532684, "orthogonal_weight": 0.35854658484458923}, {"head": 2, "memory_to_active": 0.11861367523670197, "contribution_to_active": 0.5899205207824707, "mix_mean": 0.5932810306549072, "mix_low": 0.0023622512817382812, "mix_high": 0.0050716400146484375, "read_cosine": 0.40788814425468445, "beta_mean": 0.0016241185367107391, "nonfinite": 0.0, "weighted_memory_to_active": 0.07644765079021454, "mix_std": 0.26145991683006287, "beta_std": 0.0022532278671860695, "beta_low": 0.98779296875, "beta_high": 0.0, "orthogonal_to_active": 0.058353256434202194, "evidence_weights": [-0.12338614463806152, -0.3132549226284027, 0.8973816633224487], "agreement_weight": 0.08531181514263153, "orthogonal_weight": -0.8627239465713501}, {"head": 3, "memory_to_active": 0.036615729331970215, "contribution_to_active": 0.5900077819824219, "mix_mean": 0.6842423677444458, "mix_low": 0.003322601318359375, "mix_high": 0.023319244384765625, "read_cosine": -0.05905384570360184, "beta_mean": 0.004985880106687546, "nonfinite": 0.0, "weighted_memory_to_active": 0.023584268987178802, "mix_std": 0.2520688772201538, "beta_std": 0.007898603565990925, "beta_low": 0.86407470703125, "beta_high": 0.0, "orthogonal_to_active": 0.01836073398590088, "evidence_weights": [-0.33767369389533997, -0.311535120010376, 0.31227636337280273], "agreement_weight": 0.06906319409608841, "orthogonal_weight": -0.8425267338752747}, {"head": 4, "memory_to_active": 0.15860185027122498, "contribution_to_active": 0.49479708075523376, "mix_mean": 0.5369507670402527, "mix_low": 0.0024652481079101562, "mix_high": 0.0076885223388671875, "read_cosine": 0.9031181931495667, "beta_mean": 0.03894122689962387, "nonfinite": 0.0, "weighted_memory_to_active": 0.09282435476779938, "mix_std": 0.25577205419540405, "beta_std": 0.02830355241894722, "beta_low": 0.09283447265625, "beta_high": 0.0, "orthogonal_to_active": 0.025898657739162445, "evidence_weights": [-0.1984136402606964, -0.2327611893415451, 0.45473435521125793], "agreement_weight": 0.13391155004501343, "orthogonal_weight": -0.6301501393318176}, {"head": 5, "memory_to_active": 0.17836755514144897, "contribution_to_active": 0.34005704522132874, "mix_mean": 0.38339167833328247, "mix_low": 0.0026111602783203125, "mix_high": 0.0009670257568359375, "read_cosine": 0.7397052049636841, "beta_mean": 0.0008103298605419695, "nonfinite": 0.0, "weighted_memory_to_active": 0.06402700394392014, "mix_std": 0.24619479477405548, "beta_std": 0.005184671841561794, "beta_low": 0.98809814453125, "beta_high": 0.0, "orthogonal_to_active": 0.03422366827726364, "evidence_weights": [-0.17034178972244263, -0.18472954630851746, 0.38039857149124146], "agreement_weight": 0.05753501132130623, "orthogonal_weight": -0.7580848336219788}]}, {"layer": 15, "memory_to_active": 0.047186799347400665, "contribution_to_active": 0.507039487361908, "mix_mean": 0.4431331157684326, "mix_low": 0.005653381813317537, "mix_high": 0.003020604606717825, "read_cosine": 0.655819296836853, "beta_mean": 0.037835314869880676, "nonfinite": 0.0, "weighted_memory_to_active": 0.02473653294146061, "mix_std": 0.28127551078796387, "beta_std": 0.0422561950981617, "beta_low": 0.7035014033317566, "beta_high": 0.0, "orthogonal_to_active": 0.01036016270518303, "output_influence": 0.3276914060115814, "residual_influence": 0.212277352809906, "state_active": 0.1744430810213089, "state_memory": 0.009660466574132442, "state_peak": 25.18922996520996, "growth_active": 1.173520803451538, "growth_memory": 1.264062523841858, "heads": [{"head": 0, "memory_to_active": 0.11612987518310547, "contribution_to_active": 0.4815964996814728, "mix_mean": 0.40885260701179504, "mix_low": 0.0059986114501953125, "mix_high": 0.006564140319824219, "read_cosine": 0.4914310574531555, "beta_mean": 0.052481018006801605, "nonfinite": 0.0, "weighted_memory_to_active": 0.061937328428030014, "mix_std": 0.2858562469482422, "beta_std": 0.035962361842393875, "beta_low": 0.0477294921875, "beta_high": 0.0, "orthogonal_to_active": 0.04081563651561737, "evidence_weights": [-0.17752128839492798, -0.19329868257045746, 0.5608067512512207], "agreement_weight": 0.08863864839076996, "orthogonal_weight": -0.8587377667427063}, {"head": 1, "memory_to_active": 0.07454202324151993, "contribution_to_active": 0.4371713399887085, "mix_mean": 0.4551089406013489, "mix_low": 0.0005092620849609375, "mix_high": 0.007511138916015625, "read_cosine": 0.9303693771362305, "beta_mean": 0.00012652503210119903, "nonfinite": 0.0, "weighted_memory_to_active": 0.031519051641225815, "mix_std": 0.27940070629119873, "beta_std": 0.0012024356983602047, "beta_low": 0.99920654296875, "beta_high": 0.0, "orthogonal_to_active": 0.002985081635415554, "evidence_weights": [-0.16436149179935455, -0.17891709506511688, 0.1533442586660385], "agreement_weight": 0.09697705507278442, "orthogonal_weight": 0.20167450606822968}, {"head": 2, "memory_to_active": 0.005744338966906071, "contribution_to_active": 0.530756413936615, "mix_mean": 0.365874320268631, "mix_low": 0.009015083312988281, "mix_high": 7.82012939453125e-05, "read_cosine": 0.8025379776954651, "beta_mean": 7.044003723422065e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0031736588571220636, "mix_std": 0.27343225479125977, "beta_std": 0.00018717406783252954, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.0010622591944411397, "evidence_weights": [-0.19384120404720306, -0.2671442925930023, 0.12923553586006165], "agreement_weight": 0.008395839482545853, "orthogonal_weight": 0.9997608661651611}, {"head": 3, "memory_to_active": 0.0014831260778009892, "contribution_to_active": 0.3789655566215515, "mix_mean": 0.4202808439731598, "mix_low": 0.003406524658203125, "mix_high": 0.0005159378051757812, "read_cosine": 0.6131011247634888, "beta_mean": 8.069663635978941e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0005174186662770808, "mix_std": 0.28749218583106995, "beta_std": 1.9726550817722455e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000245200761128217, "evidence_weights": [-0.2730855345726013, -0.23899440467357635, 0.17035706341266632], "agreement_weight": 0.0608723983168602, "orthogonal_weight": -0.48598989844322205}, {"head": 4, "memory_to_active": 0.05517040193080902, "contribution_to_active": 0.6539161205291748, "mix_mean": 0.4082522988319397, "mix_low": 0.005938529968261719, "mix_high": 0.0006818771362304688, "read_cosine": 0.7294796705245972, "beta_mean": 0.1716482788324356, "nonfinite": 0.0, "weighted_memory_to_active": 0.030963335186243057, "mix_std": 0.30693045258522034, "beta_std": 0.1980290561914444, "beta_low": 0.2069091796875, "beta_high": 0.0, "orthogonal_to_active": 0.0055916644632816315, "evidence_weights": [-0.20320509374141693, -0.25078365206718445, -0.03006081096827984], "agreement_weight": 0.010652212426066399, "orthogonal_weight": 0.40885576605796814}, {"head": 5, "memory_to_active": 0.030051015317440033, "contribution_to_active": 0.559830904006958, "mix_mean": 0.6004294753074646, "mix_low": 0.009052276611328125, "mix_high": 0.0027723312377929688, "read_cosine": 0.3679964542388916, "beta_mean": 0.002677545417100191, "nonfinite": 0.0, "weighted_memory_to_active": 0.02030840888619423, "mix_std": 0.2545410394668579, "beta_std": 0.018136411905288696, "beta_low": 0.96722412109375, "beta_high": 0.0, "orthogonal_to_active": 0.01146113220602274, "evidence_weights": [-0.27654653787612915, -0.3076213300228119, 0.5382147431373596], "agreement_weight": 0.31633883714675903, "orthogonal_weight": 0.8143156170845032}]}]}, {"step": 79000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8341394230010337, "memory_to_active": 0.821843147277832, "contribution_to_active": 0.5919958353042603, "mix_mean": 0.43654394149780273, "mix_low": 0.11884865164756775, "mix_high": 0.011088421568274498, "read_cosine": 0.34381282329559326, "beta_mean": 0.06817431002855301, "nonfinite": 0.0, "weighted_memory_to_active": 0.2039128988981247, "mix_std": 0.2210678607225418, "beta_std": 0.05913610756397247, "beta_low": 0.6009795069694519, "beta_high": 4.38690185546875e-05, "orthogonal_to_active": 0.0661095455288887, "samples": 4, "scored_tokens": 899, "normal_loss": 2.981510877609253, "normal_token_accuracy": 0.46384871006011963, "state_active": 0.06517928838729858, "state_memory": 0.015569524839520454, "state_peak": 124.6712875366211, "growth_active": 1.1195613145828247, "growth_memory": 1.2605278491973877, "output_influence": 0.46058663725852966, "residual_influence": 0.17548933625221252, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.1078927516937256, "no_memory_loss_delta": 0.8561471700668335, "scrambled_loss_delta": 0.14766910672187805, "memory_only_loss_delta": 4.226161003112793, "no_evidence_loss_delta": 0.4904395043849945, "no_agreement_loss_delta": 0.01847720518708229, "no_orthogonal_loss_delta": 0.02079949527978897}, {"index": 94, "tokens": 256, "normal_loss": 3.0665600299835205, "no_memory_loss_delta": 0.8774802684783936, "scrambled_loss_delta": 0.47693830728530884, "memory_only_loss_delta": 4.945252895355225, "no_evidence_loss_delta": 0.37855860590934753, "no_agreement_loss_delta": 0.037567026913166046, "no_orthogonal_loss_delta": 0.048195190727710724}, {"index": 188, "tokens": 256, "normal_loss": 3.2827341556549072, "no_memory_loss_delta": 1.0943609476089478, "scrambled_loss_delta": 0.130612313747406, "memory_only_loss_delta": 3.4126522541046143, "no_evidence_loss_delta": 0.5544167160987854, "no_agreement_loss_delta": 0.04835888743400574, "no_orthogonal_loss_delta": 0.039742935448884964}, {"index": 283, "tokens": 131, "normal_loss": 1.9796797037124634, "no_memory_loss_delta": 0.4421288073062897, "scrambled_loss_delta": 0.1200324147939682, "memory_only_loss_delta": 3.0746936798095703, "no_evidence_loss_delta": 0.18806876242160797, "no_agreement_loss_delta": -0.011105842888355255, "no_orthogonal_loss_delta": -0.04509305581450462}], "agreement_weight_abs_mean": 0.2591446340084076, "agreement_weight_abs_max": 6.934495449066162, "agreement_weight_mean": 0.2245805263519287, "orthogonal_weight_abs_mean": 0.9582980275154114, "orthogonal_weight_abs_max": 11.471078872680664, "orthogonal_weight_mean": -0.025330722332000732, "no_memory_loss": 3.8512368202209473, "no_memory_loss_delta": 0.8697262406349182, "no_memory_token_accuracy_gain": 0.12235817313194275, "no_memory_token_win_fraction": 0.7119021415710449, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 58.0933743141814, "scrambled_loss": 3.2140583992004395, "scrambled_loss_delta": 0.2325478196144104, "scrambled_token_accuracy_gain": 0.027808675542473793, "scrambled_token_win_fraction": 0.6106785535812378, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.748814917048325, "memory_only_loss": 7.012996196746826, "memory_only_loss_delta": 4.031485557556152, "memory_only_token_accuracy_gain": 0.3192436099052429, "memory_only_token_win_fraction": 0.9221357107162476, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 98.22520552332993, "no_evidence_loss": 3.41424822807312, "no_evidence_loss_delta": 0.43273770809173584, "no_evidence_token_accuracy_gain": 0.05228031054139137, "no_evidence_token_win_fraction": 0.6162402629852295, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 35.126937371453025, "no_agreement_loss": 3.00962233543396, "no_agreement_loss_delta": 0.02811160497367382, "no_agreement_token_accuracy_gain": 0.0033370412420481443, "no_agreement_token_win_fraction": 0.48164626955986023, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.7720150521181597, "no_orthogonal_loss": 3.005903720855713, "no_orthogonal_loss_delta": 0.024393372237682343, "no_orthogonal_token_accuracy_gain": 0.0022246940061450005, "no_orthogonal_token_win_fraction": 0.5817574858665466, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.409825841019538, "short_no_memory_loss_delta": 0.834079921245575, "long_context_memory_gain": 0.03564631938934326, "normal_context_loss_gain": 0.13327112793922424, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.770279884338379, "contribution_to_active": 1.2879077196121216, "mix_mean": 0.1111612543463707, "mix_low": 0.49708449840545654, "mix_high": 0.004655997268855572, "read_cosine": 0.4059736132621765, "beta_mean": 0.22041922807693481, "nonfinite": 0.0, "weighted_memory_to_active": 1.3037335872650146, "mix_std": 0.09167756140232086, "beta_std": 0.14159631729125977, "beta_low": 0.19281005859375, "beta_high": 0.0, "orthogonal_to_active": 0.054373037070035934, "output_influence": 0.21248222887516022, "residual_influence": 0.21456077694892883, "state_active": 0.06582885980606079, "state_memory": 0.04626310616731644, "state_peak": 46.57040023803711, "growth_active": 1.1629148721694946, "growth_memory": 1.2516505718231201, "heads": [{"head": 0, "memory_to_active": 3.1197729110717773, "contribution_to_active": 0.4354795217514038, "mix_mean": 0.018378879874944687, "mix_low": 0.6303272247314453, "mix_high": 0.0, "read_cosine": 0.22254081070423126, "beta_mean": 0.20745091140270233, "nonfinite": 0.0, "weighted_memory_to_active": 0.5495451092720032, "mix_std": 0.05450645834207535, "beta_std": 0.10059069097042084, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.054443374276161194, "evidence_weights": [-0.0022873973939567804, -0.038261886686086655, -0.06173713132739067], "agreement_weight": -0.23828163743019104, "orthogonal_weight": -0.1167735755443573}, {"head": 1, "memory_to_active": 6.944421291351318, "contribution_to_active": 3.17056941986084, "mix_mean": 0.019717087969183922, "mix_low": 0.609379768371582, "mix_high": 0.0, "read_cosine": 0.4391024708747864, "beta_mean": 0.47420865297317505, "nonfinite": 0.0, "weighted_memory_to_active": 3.3724427223205566, "mix_std": 0.05796881765127182, "beta_std": 0.21417076885700226, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11654764413833618, "evidence_weights": [0.023842548951506615, -0.09955532848834991, 0.6749514937400818], "agreement_weight": 0.02886113151907921, "orthogonal_weight": 0.07595345377922058}, {"head": 2, "memory_to_active": 3.0448999404907227, "contribution_to_active": 0.7266594767570496, "mix_mean": 0.014099824242293835, "mix_low": 0.6521568298339844, "mix_high": 0.0, "read_cosine": 0.26351502537727356, "beta_mean": 0.2681719660758972, "nonfinite": 0.0, "weighted_memory_to_active": 0.8380441069602966, "mix_std": 0.031470198184251785, "beta_std": 0.21048516035079956, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.061524614691734314, "evidence_weights": [0.06351248919963837, 0.0459817498922348, 0.286165326833725], "agreement_weight": 0.011035793460905552, "orthogonal_weight": -0.19404157996177673}, {"head": 3, "memory_to_active": 8.187032699584961, "contribution_to_active": 2.8646492958068848, "mix_mean": 0.027908753603696823, "mix_low": 0.4946107864379883, "mix_high": 0.0, "read_cosine": 0.27343037724494934, "beta_mean": 0.33831673860549927, "nonfinite": 0.0, "weighted_memory_to_active": 2.965848207473755, "mix_std": 0.0636400580406189, "beta_std": 0.27133533358573914, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.07442931085824966, "evidence_weights": [-0.03331954777240753, -0.023387523368000984, -0.1422438770532608], "agreement_weight": -0.0066738976165652275, "orthogonal_weight": -0.02921980805695057}, {"head": 4, "memory_to_active": 1.2239124774932861, "contribution_to_active": 0.06193678081035614, "mix_mean": 0.0256972499191761, "mix_low": 0.5841236114501953, "mix_high": 0.0, "read_cosine": 0.308184951543808, "beta_mean": 0.016827348619699478, "nonfinite": 0.0, "weighted_memory_to_active": 0.04204411059617996, "mix_std": 0.06627152860164642, "beta_std": 0.033112362027168274, "beta_low": 0.65167236328125, "beta_high": 0.0, "orthogonal_to_active": 0.0010472654830664396, "evidence_weights": [0.0811968743801117, 0.05029943212866783, -0.1444188952445984], "agreement_weight": -0.21440283954143524, "orthogonal_weight": 0.023680025711655617}, {"head": 5, "memory_to_active": 0.10163839161396027, "contribution_to_active": 0.4681510627269745, "mix_mean": 0.5611656904220581, "mix_low": 0.011908531188964844, "mix_high": 0.02793598175048828, "read_cosine": 0.929067850112915, "beta_mean": 0.017539629712700844, "nonfinite": 0.0, "weighted_memory_to_active": 0.054476916790008545, "mix_std": 0.2762082815170288, "beta_std": 0.01988353207707405, "beta_low": 0.50482177734375, "beta_high": 0.0, "orthogonal_to_active": 0.018245980143547058, "evidence_weights": [-0.17237894237041473, -0.21536105871200562, 0.35830628871917725], "agreement_weight": 0.2816495895385742, "orthogonal_weight": -0.7325509786605835}]}, {"layer": 1, "memory_to_active": 2.670801877975464, "contribution_to_active": 0.515916109085083, "mix_mean": 0.21985474228858948, "mix_low": 0.37340039014816284, "mix_high": 0.0013165473937988281, "read_cosine": 0.2944986820220947, "beta_mean": 0.1847001612186432, "nonfinite": 0.0, "weighted_memory_to_active": 0.3563650846481323, "mix_std": 0.11179161071777344, "beta_std": 0.13285793364048004, "beta_low": 0.3540547788143158, "beta_high": 0.00016276042151730508, "orthogonal_to_active": 0.03793472424149513, "output_influence": 0.2859865427017212, "residual_influence": 0.08331770449876785, "state_active": 0.027110926806926727, "state_memory": 0.03373216092586517, "state_peak": 42.24124526977539, "growth_active": 1.1058130264282227, "growth_memory": 1.42575204372406, "heads": [{"head": 0, "memory_to_active": 0.0153726851567626, "contribution_to_active": 0.4979879856109619, "mix_mean": 0.5551316142082214, "mix_low": 0.0038309097290039062, "mix_high": 0.00327301025390625, "read_cosine": 0.01221533864736557, "beta_mean": 9.481364395469427e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009582570753991604, "mix_std": 0.28070068359375, "beta_std": 0.0001748977811075747, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.009290274232625961, "evidence_weights": [-0.2096676528453827, -0.2692088782787323, 0.18370284140110016], "agreement_weight": 0.20515739917755127, "orthogonal_weight": -1.0112838745117188}, {"head": 1, "memory_to_active": 3.200852870941162, "contribution_to_active": 0.30843690037727356, "mix_mean": 0.019540978595614433, "mix_low": 0.5206012725830078, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5298134088516235, "beta_mean": 0.27805840969085693, "nonfinite": 0.0, "weighted_memory_to_active": 0.338093638420105, "mix_std": 0.03878619149327278, "beta_std": 0.19706523418426514, "beta_low": 0.00238037109375, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.021809879690408707, "evidence_weights": [0.030130689963698387, -0.060390278697013855, 0.0472557507455349], "agreement_weight": -0.0019965586252510548, "orthogonal_weight": 0.21112893521785736}, {"head": 2, "memory_to_active": 0.031331147998571396, "contribution_to_active": 0.5492132902145386, "mix_mean": 0.6838746070861816, "mix_low": 0.0002346038818359375, "mix_high": 0.0046253204345703125, "read_cosine": 0.4539429247379303, "beta_mean": 0.0010468546533957124, "nonfinite": 0.0, "weighted_memory_to_active": 0.02062639221549034, "mix_std": 0.22365260124206543, "beta_std": 0.0032761225011199713, "beta_low": 0.982666015625, "beta_high": 0.0, "orthogonal_to_active": 0.015203837305307388, "evidence_weights": [-0.26676684617996216, -0.36363685131073, 0.6340780854225159], "agreement_weight": 0.05346498265862465, "orthogonal_weight": -0.8412162065505981}, {"head": 3, "memory_to_active": 3.285066843032837, "contribution_to_active": 0.2518185079097748, "mix_mean": 0.024929232895374298, "mix_low": 0.5070075988769531, "mix_high": 0.0, "read_cosine": 0.1805354505777359, "beta_mean": 0.3513483703136444, "nonfinite": 0.0, "weighted_memory_to_active": 0.3275027573108673, "mix_std": 0.061636220663785934, "beta_std": 0.266044557094574, "beta_low": 0.06939697265625, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.10890160501003265, "evidence_weights": [0.037567153573036194, -0.020829536020755768, -0.011654662899672985], "agreement_weight": -0.02745918743312359, "orthogonal_weight": -0.4699661433696747}, {"head": 4, "memory_to_active": 4.509462356567383, "contribution_to_active": 1.3880302906036377, "mix_mean": 0.029439248144626617, "mix_low": 0.35415172576904297, "mix_high": 0.0, "read_cosine": 0.3191719949245453, "beta_mean": 0.1709449291229248, "nonfinite": 0.0, "weighted_memory_to_active": 1.3747177124023438, "mix_std": 0.04843834787607193, "beta_std": 0.13496454060077667, "beta_low": 0.037109375, "beta_high": 0.000244140625, "orthogonal_to_active": 0.043460115790367126, "evidence_weights": [-0.007164805196225643, -0.0006797327660024166, 0.17292961478233337], "agreement_weight": -0.0053639961406588554, "orthogonal_weight": 0.0507829487323761}, {"head": 5, "memory_to_active": 4.982725620269775, "contribution_to_active": 0.10000932216644287, "mix_mean": 0.006212668027728796, "mix_low": 0.8545761108398438, "mix_high": 0.0, "read_cosine": 0.27131277322769165, "beta_mean": 0.30670762062072754, "nonfinite": 0.0, "weighted_memory_to_active": 0.06766737997531891, "mix_std": 0.017535563558340073, "beta_std": 0.19562214612960815, "beta_low": 0.03277587890625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.028942618519067764, "evidence_weights": [0.1592901051044464, 0.0748348981142044, 0.09223616123199463], "agreement_weight": -0.11310803145170212, "orthogonal_weight": 0.41659849882125854}]}, {"layer": 2, "memory_to_active": 1.0443309545516968, "contribution_to_active": 0.35931891202926636, "mix_mean": 0.4182615876197815, "mix_low": 0.23781491816043854, "mix_high": 0.004478295799344778, "read_cosine": 0.28779083490371704, "beta_mean": 0.07765967398881912, "nonfinite": 0.0, "weighted_memory_to_active": 0.06963835656642914, "mix_std": 0.18281716108322144, "beta_std": 0.06672678887844086, "beta_low": 0.619049072265625, "beta_high": 0.000213623046875, "orthogonal_to_active": 0.014482397586107254, "output_influence": 0.42016828060150146, "residual_influence": 0.12535163760185242, "state_active": 0.04118042811751366, "state_memory": 0.012649428099393845, "state_peak": 27.952316284179688, "growth_active": 1.06498122215271, "growth_memory": 1.5492225885391235, "heads": [{"head": 0, "memory_to_active": 0.0028087953105568886, "contribution_to_active": 0.3614961802959442, "mix_mean": 0.6565678119659424, "mix_low": 0.00041675567626953125, "mix_high": 0.009705543518066406, "read_cosine": 0.13917537033557892, "beta_mean": 2.501964809198398e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0018786246655508876, "mix_std": 0.2614390552043915, "beta_std": 4.455481393961236e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019213648047298193, "evidence_weights": [-0.37023407220840454, -0.21185027062892914, 0.2121497541666031], "agreement_weight": 0.1436278074979782, "orthogonal_weight": -1.0582802295684814}, {"head": 1, "memory_to_active": 3.266836404800415, "contribution_to_active": 0.048559799790382385, "mix_mean": 0.005648150108754635, "mix_low": 0.8695497512817383, "mix_high": 0.0, "read_cosine": 0.11603611707687378, "beta_mean": 0.211710125207901, "nonfinite": 0.0, "weighted_memory_to_active": 0.03896332532167435, "mix_std": 0.011289180256426334, "beta_std": 0.1425221860408783, "beta_low": 0.0037841796875, "beta_high": 0.0, "orthogonal_to_active": 0.010939221829175949, "evidence_weights": [0.130384624004364, 0.10099232941865921, 0.570485532283783], "agreement_weight": 0.02608601190149784, "orthogonal_weight": 0.31296733021736145}, {"head": 2, "memory_to_active": 0.006975548341870308, "contribution_to_active": 0.42068547010421753, "mix_mean": 0.5637314319610596, "mix_low": 0.0007791519165039062, "mix_high": 0.003936767578125, "read_cosine": 0.39911043643951416, "beta_mean": 1.867433456936851e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0030248600523918867, "mix_std": 0.27340230345726013, "beta_std": 3.6571931559592485e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00024574872804805636, "evidence_weights": [-0.2441873550415039, -0.28793272376060486, 0.558707594871521], "agreement_weight": -0.039127279072999954, "orthogonal_weight": 0.10284815728664398}, {"head": 3, "memory_to_active": 0.08432315289974213, "contribution_to_active": 0.5496342182159424, "mix_mean": 0.5947543382644653, "mix_low": 0.0001659393310546875, "mix_high": 0.005476951599121094, "read_cosine": 0.5043120384216309, "beta_mean": 0.013908951543271542, "nonfinite": 0.0, "weighted_memory_to_active": 0.05408860370516777, "mix_std": 0.2490200251340866, "beta_std": 0.023807581514120102, "beta_low": 0.65155029296875, "beta_high": 0.0, "orthogonal_to_active": 0.04055434465408325, "evidence_weights": [-0.19560028612613678, -0.3242146670818329, 0.7148388624191284], "agreement_weight": 0.053886499255895615, "orthogonal_weight": -0.8973168134689331}, {"head": 4, "memory_to_active": 2.8979835510253906, "contribution_to_active": 0.2458096742630005, "mix_mean": 0.02049368992447853, "mix_low": 0.5556230545043945, "mix_high": 0.0, "read_cosine": 0.1860721856355667, "beta_mean": 0.24027171730995178, "nonfinite": 0.0, "weighted_memory_to_active": 0.3159836530685425, "mix_std": 0.047634899616241455, "beta_std": 0.23391933739185333, "beta_low": 0.0589599609375, "beta_high": 0.00128173828125, "orthogonal_to_active": 0.03223668411374092, "evidence_weights": [0.022946396842598915, -0.0006958780577406287, 0.03356432542204857], "agreement_weight": 0.020571144297719002, "orthogonal_weight": -0.12007244676351547}, {"head": 5, "memory_to_active": 0.0070577943697571754, "contribution_to_active": 0.5297278165817261, "mix_mean": 0.6683739423751831, "mix_low": 0.000354766845703125, "mix_high": 0.007750511169433594, "read_cosine": 0.3820387125015259, "beta_mean": 2.3549251636723056e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038910165894776583, "mix_std": 0.25411751866340637, "beta_std": 3.0470262572634965e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.000997011549770832, "evidence_weights": [-0.271514356136322, -0.28163883090019226, 0.8841853141784668], "agreement_weight": 0.29267534613609314, "orthogonal_weight": -0.3015424907207489}]}, {"layer": 3, "memory_to_active": 0.4035379886627197, "contribution_to_active": 0.5059559941291809, "mix_mean": 0.5350492596626282, "mix_low": 0.0507502555847168, "mix_high": 0.012049516662955284, "read_cosine": 0.17967841029167175, "beta_mean": 0.023201819509267807, "nonfinite": 0.0, "weighted_memory_to_active": 0.11687162518501282, "mix_std": 0.2532179355621338, "beta_std": 0.023444583639502525, "beta_low": 0.7776693105697632, "beta_high": 0.0, "orthogonal_to_active": 0.04844582825899124, "output_influence": 0.46055135130882263, "residual_influence": 0.12374544888734818, "state_active": 0.022937754169106483, "state_memory": 0.005880963057279587, "state_peak": 25.141420364379883, "growth_active": 1.1337506771087646, "growth_memory": 1.591002345085144, "heads": [{"head": 0, "memory_to_active": 0.335502564907074, "contribution_to_active": 0.6290931105613708, "mix_mean": 0.6211888194084167, "mix_low": 0.00093841552734375, "mix_high": 0.02234649658203125, "read_cosine": 0.2250525951385498, "beta_mean": 0.008570647798478603, "nonfinite": 0.0, "weighted_memory_to_active": 0.2105063945055008, "mix_std": 0.29285115003585815, "beta_std": 0.009889115579426289, "beta_low": 0.7314453125, "beta_high": 0.0, "orthogonal_to_active": 0.15470348298549652, "evidence_weights": [-0.2119051069021225, -0.24542136490345, 1.064745545387268], "agreement_weight": 0.11193497478961945, "orthogonal_weight": -0.7715005874633789}, {"head": 1, "memory_to_active": 0.006606356240808964, "contribution_to_active": 0.4765807092189789, "mix_mean": 0.6064481735229492, "mix_low": 0.0022869110107421875, "mix_high": 0.012146949768066406, "read_cosine": 0.17268416285514832, "beta_mean": 3.028936953342054e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004144665319472551, "mix_std": 0.288865327835083, "beta_std": 8.048956806305796e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0033163970801979303, "evidence_weights": [-0.2660578787326813, -0.21486933529376984, 0.682852566242218], "agreement_weight": 0.6202294230461121, "orthogonal_weight": -0.8688589930534363}, {"head": 2, "memory_to_active": 1.8447781801223755, "contribution_to_active": 0.2771698236465454, "mix_mean": 0.05867675319314003, "mix_low": 0.2948112487792969, "mix_high": 0.0, "read_cosine": 0.22721651196479797, "beta_mean": 0.126007080078125, "nonfinite": 0.0, "weighted_memory_to_active": 0.31175029277801514, "mix_std": 0.11074423789978027, "beta_std": 0.11839107424020767, "beta_low": 0.0247802734375, "beta_high": 0.0, "orthogonal_to_active": 0.006363907363265753, "evidence_weights": [-0.10020017623901367, -0.00854992400854826, -0.027552148327231407], "agreement_weight": -0.007603522855788469, "orthogonal_weight": 0.026533491909503937}, {"head": 3, "memory_to_active": 0.014867730438709259, "contribution_to_active": 0.5627830028533936, "mix_mean": 0.6254594922065735, "mix_low": 0.0029582977294921875, "mix_high": 0.010549545288085938, "read_cosine": 0.27573680877685547, "beta_mean": 6.409800698747858e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01013594027608633, "mix_std": 0.2788102924823761, "beta_std": 0.000152083215652965, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.002329094335436821, "evidence_weights": [-0.26322507858276367, -0.22061406075954437, 2.3530521392822266], "agreement_weight": 2.39363956451416, "orthogonal_weight": -0.3011366128921509}, {"head": 4, "memory_to_active": 0.011992423795163631, "contribution_to_active": 0.42238596081733704, "mix_mean": 0.5562038421630859, "mix_low": 0.001483917236328125, "mix_high": 0.0031309127807617188, "read_cosine": 0.12964877486228943, "beta_mean": 5.6716788094490767e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00814911164343357, "mix_std": 0.2855752408504486, "beta_std": 0.00010319064313080162, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007692858576774597, "evidence_weights": [-0.24492858350276947, -0.2214478850364685, 0.05069604143500328], "agreement_weight": -0.06797602027654648, "orthogonal_weight": -1.0249955654144287}, {"head": 5, "memory_to_active": 0.20748060941696167, "contribution_to_active": 0.6677231788635254, "mix_mean": 0.7423182129859924, "mix_low": 0.0020227432250976562, "mix_high": 0.024123191833496094, "read_cosine": 0.04773149639368057, "beta_mean": 0.004482085816562176, "nonfinite": 0.0, "weighted_memory_to_active": 0.1565432995557785, "mix_std": 0.26246124505996704, "beta_std": 0.012051551602780819, "beta_low": 0.9097900390625, "beta_high": 0.0, "orthogonal_to_active": 0.11626920849084854, "evidence_weights": [-0.258008748292923, -0.34104564785957336, 0.5616148114204407], "agreement_weight": 0.24809472262859344, "orthogonal_weight": -0.761736273765564}]}, {"layer": 4, "memory_to_active": 1.026984453201294, "contribution_to_active": 0.495734304189682, "mix_mean": 0.38013899326324463, "mix_low": 0.2161092758178711, "mix_high": 0.012121042236685753, "read_cosine": 0.29629236459732056, "beta_mean": 0.10162512958049774, "nonfinite": 0.0, "weighted_memory_to_active": 0.13508480787277222, "mix_std": 0.16168510913848877, "beta_std": 0.09363238513469696, "beta_low": 0.4021708369255066, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.060922037810087204, "output_influence": 0.5191028118133545, "residual_influence": 0.1767996847629547, "state_active": 0.037403546273708344, "state_memory": 0.015281829982995987, "state_peak": 44.806312561035156, "growth_active": 1.1979141235351562, "growth_memory": 1.5155389308929443, "heads": [{"head": 0, "memory_to_active": 0.13559119403362274, "contribution_to_active": 0.8244171142578125, "mix_mean": 0.6950570344924927, "mix_low": 0.008306503295898438, "mix_high": 0.03348350524902344, "read_cosine": 0.4245077669620514, "beta_mean": 0.2313814014196396, "nonfinite": 0.0, "weighted_memory_to_active": 0.11784939467906952, "mix_std": 0.34446755051612854, "beta_std": 0.22740955650806427, "beta_low": 0.26434326171875, "beta_high": 0.0, "orthogonal_to_active": 0.09264077991247177, "evidence_weights": [-0.23656979203224182, -0.35065093636512756, -0.253267765045166], "agreement_weight": 0.00393615011125803, "orthogonal_weight": 0.9203612208366394}, {"head": 1, "memory_to_active": 0.04057503491640091, "contribution_to_active": 0.6707988381385803, "mix_mean": 0.7145918607711792, "mix_low": 4.00543212890625e-05, "mix_high": 0.020763397216796875, "read_cosine": 0.5453546047210693, "beta_mean": 0.0008454727940261364, "nonfinite": 0.0, "weighted_memory_to_active": 0.03079039976000786, "mix_std": 0.23287568986415863, "beta_std": 0.006911744363605976, "beta_low": 0.98138427734375, "beta_high": 0.0, "orthogonal_to_active": 0.014917625114321709, "evidence_weights": [-0.23072241246700287, -0.36168450117111206, 0.547630250453949], "agreement_weight": 0.3791735768318176, "orthogonal_weight": -0.6607219576835632}, {"head": 2, "memory_to_active": 1.3868026733398438, "contribution_to_active": 0.38230040669441223, "mix_mean": 0.06417088955640793, "mix_low": 0.2680244445800781, "mix_high": 2.384185791015625e-05, "read_cosine": 0.2286844104528427, "beta_mean": 0.04983532428741455, "nonfinite": 0.0, "weighted_memory_to_active": 0.2811473608016968, "mix_std": 0.10590887814760208, "beta_std": 0.055109765380620956, "beta_low": 0.13824462890625, "beta_high": 0.0, "orthogonal_to_active": 0.11163200438022614, "evidence_weights": [0.020555511116981506, -0.07377084344625473, 0.40124812722206116], "agreement_weight": 0.12052259594202042, "orthogonal_weight": -0.5153692960739136}, {"head": 3, "memory_to_active": 1.957749366760254, "contribution_to_active": 0.1502564251422882, "mix_mean": 0.03358178585767746, "mix_low": 0.3855743408203125, "mix_high": 0.0, "read_cosine": 0.2179296761751175, "beta_mean": 0.12865857779979706, "nonfinite": 0.0, "weighted_memory_to_active": 0.19725871086120605, "mix_std": 0.05533111095428467, "beta_std": 0.1194382756948471, "beta_low": 0.029052734375, "beta_high": 0.0, "orthogonal_to_active": 0.004382695537060499, "evidence_weights": [-0.025131234899163246, 0.04728775471448898, -0.26149803400039673], "agreement_weight": 0.07720337808132172, "orthogonal_weight": 0.03137253597378731}, {"head": 4, "memory_to_active": 0.007552847731858492, "contribution_to_active": 0.6557211875915527, "mix_mean": 0.7594500780105591, "mix_low": 9.5367431640625e-06, "mix_high": 0.018449783325195312, "read_cosine": 0.12423292547464371, "beta_mean": 1.571932079968974e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005682840012013912, "mix_std": 0.20417696237564087, "beta_std": 5.140883877174929e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0035381366033107042, "evidence_weights": [-0.3222677707672119, -0.31012022495269775, 0.21624289453029633], "agreement_weight": 0.3218103051185608, "orthogonal_weight": -0.6606406569480896}, {"head": 5, "memory_to_active": 2.6336350440979004, "contribution_to_active": 0.2909115254878998, "mix_mean": 0.01398205105215311, "mix_low": 0.6347007751464844, "mix_high": 5.7220458984375e-06, "read_cosine": 0.23704469203948975, "beta_mean": 0.19901424646377563, "nonfinite": 0.0, "weighted_memory_to_active": 0.1777801513671875, "mix_std": 0.027350423857569695, "beta_std": 0.15287354588508606, "beta_low": 0.0, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.138420969247818, "evidence_weights": [0.05598962679505348, -0.018711190670728683, -0.25883880257606506], "agreement_weight": -0.09252183884382248, "orthogonal_weight": 1.1345832347869873}]}, {"layer": 5, "memory_to_active": 0.3561163544654846, "contribution_to_active": 0.5524395704269409, "mix_mean": 0.526688277721405, "mix_low": 0.055535636842250824, "mix_high": 0.02008708380162716, "read_cosine": 0.09957706183195114, "beta_mean": 0.04951415956020355, "nonfinite": 0.0, "weighted_memory_to_active": 0.1255722939968109, "mix_std": 0.2486327737569809, "beta_std": 0.06438461691141129, "beta_low": 0.6923624873161316, "beta_high": 0.0, "orthogonal_to_active": 0.055083733052015305, "output_influence": 0.6082274913787842, "residual_influence": 0.2192724347114563, "state_active": 0.03808344155550003, "state_memory": 0.008894694037735462, "state_peak": 63.749610900878906, "growth_active": 1.1084468364715576, "growth_memory": 1.2333999872207642, "heads": [{"head": 0, "memory_to_active": 0.15151047706604004, "contribution_to_active": 0.5901255011558533, "mix_mean": 0.5193586349487305, "mix_low": 0.0050144195556640625, "mix_high": 0.00200653076171875, "read_cosine": -0.10503101348876953, "beta_mean": 0.013239104300737381, "nonfinite": 0.0, "weighted_memory_to_active": 0.054858721792697906, "mix_std": 0.2739368975162506, "beta_std": 0.023108292371034622, "beta_low": 0.63006591796875, "beta_high": 0.0, "orthogonal_to_active": 0.025026194751262665, "evidence_weights": [-0.16864019632339478, -0.18692244589328766, 0.12254117429256439], "agreement_weight": -0.021925073117017746, "orthogonal_weight": -0.4818600118160248}, {"head": 1, "memory_to_active": 0.003449240932241082, "contribution_to_active": 0.5772411227226257, "mix_mean": 0.7200806140899658, "mix_low": 0.0005006790161132812, "mix_high": 0.03896903991699219, "read_cosine": 0.26025956869125366, "beta_mean": 7.815020580892451e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0022389721125364304, "mix_std": 0.2590419054031372, "beta_std": 1.7191308870678768e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0020557940006256104, "evidence_weights": [-0.24313592910766602, -0.28239673376083374, 0.9412111043930054], "agreement_weight": 0.7513718605041504, "orthogonal_weight": -0.9729800820350647}, {"head": 2, "memory_to_active": 0.013155411928892136, "contribution_to_active": 0.6222249269485474, "mix_mean": 0.6674817800521851, "mix_low": 0.0016412734985351562, "mix_high": 0.01916790008544922, "read_cosine": -0.3919466733932495, "beta_mean": 7.403065683320165e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008638516999781132, "mix_std": 0.2571399211883545, "beta_std": 0.00032414583256468177, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.008469311520457268, "evidence_weights": [-0.25302913784980774, -0.30612432956695557, 0.1800040751695633], "agreement_weight": -0.20997445285320282, "orthogonal_weight": -1.0616061687469482}, {"head": 3, "memory_to_active": 1.500497579574585, "contribution_to_active": 0.24729213118553162, "mix_mean": 0.11918297410011292, "mix_low": 0.11296749114990234, "mix_high": 0.0, "read_cosine": 0.1504671275615692, "beta_mean": 0.09594499319791794, "nonfinite": 0.0, "weighted_memory_to_active": 0.379245400428772, "mix_std": 0.1492781639099121, "beta_std": 0.1486133635044098, "beta_low": 0.20892333984375, "beta_high": 0.0, "orthogonal_to_active": 0.15560676157474518, "evidence_weights": [-0.07430668920278549, -0.09697646647691727, 0.1357157826423645], "agreement_weight": 0.04978104680776596, "orthogonal_weight": -0.497083455324173}, {"head": 4, "memory_to_active": 0.4079926609992981, "contribution_to_active": 0.5798081159591675, "mix_mean": 0.31899377703666687, "mix_low": 0.2117624282836914, "mix_high": 0.004052162170410156, "read_cosine": 0.5406396389007568, "beta_mean": 0.18764972686767578, "nonfinite": 0.0, "weighted_memory_to_active": 0.26679474115371704, "mix_std": 0.3360705077648163, "beta_std": 0.21338650584220886, "beta_low": 0.3172607421875, "beta_high": 0.0, "orthogonal_to_active": 0.11551383882761002, "evidence_weights": [-0.10697608441114426, -0.24663668870925903, -0.08055557310581207], "agreement_weight": 0.030275601893663406, "orthogonal_weight": 0.6848315000534058}, {"head": 5, "memory_to_active": 0.06009264662861824, "contribution_to_active": 0.6979451179504395, "mix_mean": 0.8150319457054138, "mix_low": 0.0013275146484375, "mix_high": 0.056326866149902344, "read_cosine": 0.1430736482143402, "beta_mean": 0.00016927775868680328, "nonfinite": 0.0, "weighted_memory_to_active": 0.04165731370449066, "mix_std": 0.21632926166057587, "beta_std": 0.0008582025184296072, "beta_low": 0.998046875, "beta_high": 0.0, "orthogonal_to_active": 0.023830480873584747, "evidence_weights": [-0.2980365753173828, -0.3211723864078522, 1.795464277267456], "agreement_weight": 0.5663058161735535, "orthogonal_weight": -0.5961384177207947}]}, {"layer": 6, "memory_to_active": 0.22045649588108063, "contribution_to_active": 0.6585797667503357, "mix_mean": 0.6047417521476746, "mix_low": 0.02839994616806507, "mix_high": 0.029813610017299652, "read_cosine": 0.31173205375671387, "beta_mean": 0.06006667762994766, "nonfinite": 0.0, "weighted_memory_to_active": 0.11704026162624359, "mix_std": 0.2391803115606308, "beta_std": 0.07224269211292267, "beta_low": 0.6366170644760132, "beta_high": 0.0, "orthogonal_to_active": 0.09354208409786224, "output_influence": 0.6038902401924133, "residual_influence": 0.23974598944187164, "state_active": 0.04063308238983154, "state_memory": 0.004831385798752308, "state_peak": 86.9854965209961, "growth_active": 1.1796354055404663, "growth_memory": 1.1198288202285767, "heads": [{"head": 0, "memory_to_active": 0.10718695819377899, "contribution_to_active": 0.7949912548065186, "mix_mean": 0.8304149508476257, "mix_low": 0.00012493133544921875, "mix_high": 0.06942558288574219, "read_cosine": 0.30860579013824463, "beta_mean": 0.01113144587725401, "nonfinite": 0.0, "weighted_memory_to_active": 0.09660721570253372, "mix_std": 0.1973406970500946, "beta_std": 0.036689624190330505, "beta_low": 0.85498046875, "beta_high": 0.0, "orthogonal_to_active": 0.06094387173652649, "evidence_weights": [-0.28717830777168274, -0.33638039231300354, 1.1600940227508545], "agreement_weight": 0.9808452725410461, "orthogonal_weight": -0.6821480393409729}, {"head": 1, "memory_to_active": 0.0027066245675086975, "contribution_to_active": 0.5321817994117737, "mix_mean": 0.6132292151451111, "mix_low": 0.00029087066650390625, "mix_high": 0.01866436004638672, "read_cosine": 0.4784952998161316, "beta_mean": 5.997154858050635e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.00128867756575346, "mix_std": 0.2781907916069031, "beta_std": 1.3883286555937957e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006580345798283815, "evidence_weights": [-0.18602515757083893, -0.2975948452949524, 1.0955848693847656], "agreement_weight": 0.14041565358638763, "orthogonal_weight": -0.6721099615097046}, {"head": 2, "memory_to_active": 0.5885108113288879, "contribution_to_active": 0.11332523822784424, "mix_mean": 0.06528035551309586, "mix_low": 0.16368484497070312, "mix_high": 1.9073486328125e-06, "read_cosine": 0.28143101930618286, "beta_mean": 0.017798390239477158, "nonfinite": 0.0, "weighted_memory_to_active": 0.07079477608203888, "mix_std": 0.08942190557718277, "beta_std": 0.021655742079019547, "beta_low": 0.4051513671875, "beta_high": 0.0, "orthogonal_to_active": 0.014864107593894005, "evidence_weights": [0.05000081658363342, -0.07926002889871597, 0.3876069486141205], "agreement_weight": 0.13220299780368805, "orthogonal_weight": -0.24651888012886047}, {"head": 3, "memory_to_active": 0.12390150129795074, "contribution_to_active": 0.6925195455551147, "mix_mean": 0.6753915548324585, "mix_low": 0.004895210266113281, "mix_high": 0.023166656494140625, "read_cosine": 0.4529061019420624, "beta_mean": 0.21365587413311005, "nonfinite": 0.0, "weighted_memory_to_active": 0.0896802768111229, "mix_std": 0.3244597315788269, "beta_std": 0.24339137971401215, "beta_low": 0.21514892578125, "beta_high": 0.0, "orthogonal_to_active": 0.05860929563641548, "evidence_weights": [-0.1596584916114807, -0.30491796135902405, 0.08716809749603271], "agreement_weight": -0.008531169034540653, "orthogonal_weight": 0.7398713231086731}, {"head": 4, "memory_to_active": 0.4826289713382721, "contribution_to_active": 1.1643527746200562, "mix_mean": 0.7332570552825928, "mix_low": 0.0012941360473632812, "mix_high": 0.057860374450683594, "read_cosine": 0.07752647995948792, "beta_mean": 0.11773276329040527, "nonfinite": 0.0, "weighted_memory_to_active": 0.43070274591445923, "mix_std": 0.3120717406272888, "beta_std": 0.13150346279144287, "beta_low": 0.344482421875, "beta_high": 0.0, "orthogonal_to_active": 0.4135679602622986, "evidence_weights": [-0.1447526067495346, -0.48793911933898926, -0.8493901491165161], "agreement_weight": -0.01602490246295929, "orthogonal_weight": 0.9798787832260132}, {"head": 5, "memory_to_active": 0.017804063856601715, "contribution_to_active": 0.6541077494621277, "mix_mean": 0.7108773589134216, "mix_low": 0.00010967254638671875, "mix_high": 0.009762763977050781, "read_cosine": 0.27142736315727234, "beta_mean": 7.557213393738493e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01316783856600523, "mix_std": 0.23359695076942444, "beta_std": 0.00020205644250381738, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.012609182856976986, "evidence_weights": [-0.22910140454769135, -0.312406450510025, 0.3435896635055542], "agreement_weight": 0.2435114085674286, "orthogonal_weight": -1.0232386589050293}]}, {"layer": 7, "memory_to_active": 1.8286035060882568, "contribution_to_active": 0.27913007140159607, "mix_mean": 0.18505077064037323, "mix_low": 0.20612001419067383, "mix_high": 0.0009129842510446906, "read_cosine": 0.2544291317462921, "beta_mean": 0.1365526020526886, "nonfinite": 0.0, "weighted_memory_to_active": 0.19975946843624115, "mix_std": 0.1474127322435379, "beta_std": 0.09661969542503357, "beta_low": 0.2343648374080658, "beta_high": 0.00030517578125, "orthogonal_to_active": 0.08886371552944183, "output_influence": 0.3060972988605499, "residual_influence": 0.08139978349208832, "state_active": 0.01417938806116581, "state_memory": 0.022613294422626495, "state_peak": 34.22159194946289, "growth_active": 1.1624374389648438, "growth_memory": 1.2716857194900513, "heads": [{"head": 0, "memory_to_active": 2.903325080871582, "contribution_to_active": 0.15556876361370087, "mix_mean": 0.03827831521630287, "mix_low": 0.40868663787841797, "mix_high": 9.5367431640625e-07, "read_cosine": 0.3569498062133789, "beta_mean": 0.18736013770103455, "nonfinite": 0.0, "weighted_memory_to_active": 0.1751268208026886, "mix_std": 0.07502029836177826, "beta_std": 0.13258694112300873, "beta_low": 0.0006103515625, "beta_high": 0.0006103515625, "orthogonal_to_active": 0.0010735007235780358, "evidence_weights": [0.04604165256023407, -0.09237832576036453, 0.03211330994963646], "agreement_weight": -0.028688762336969376, "orthogonal_weight": -0.006461597513407469}, {"head": 1, "memory_to_active": 0.9642583131790161, "contribution_to_active": 0.39355775713920593, "mix_mean": 0.22760269045829773, "mix_low": 0.10157299041748047, "mix_high": 0.0046482086181640625, "read_cosine": 0.4116422235965729, "beta_mean": 0.09427912533283234, "nonfinite": 0.0, "weighted_memory_to_active": 0.33527055382728577, "mix_std": 0.24324461817741394, "beta_std": 0.06316420435905457, "beta_low": 6.103515625e-05, "beta_high": 0.0, "orthogonal_to_active": 0.044980667531490326, "evidence_weights": [-0.21793314814567566, -0.10401152074337006, -0.3207796812057495], "agreement_weight": 0.05674039572477341, "orthogonal_weight": 0.16230961680412292}, {"head": 2, "memory_to_active": 0.5232300162315369, "contribution_to_active": 0.41700655221939087, "mix_mean": 0.25646865367889404, "mix_low": 0.010951995849609375, "mix_high": 0.0004291534423828125, "read_cosine": 0.13051310181617737, "beta_mean": 0.02797076292335987, "nonfinite": 0.0, "weighted_memory_to_active": 0.2204522341489792, "mix_std": 0.2189061939716339, "beta_std": 0.033537495881319046, "beta_low": 0.24468994140625, "beta_high": 0.0, "orthogonal_to_active": 0.13315968215465546, "evidence_weights": [-0.10139907896518707, -0.18891556560993195, 0.8950908780097961], "agreement_weight": 0.01040367130190134, "orthogonal_weight": -0.6240566372871399}, {"head": 3, "memory_to_active": 5.019926071166992, "contribution_to_active": 0.10349389910697937, "mix_mean": 0.0370401106774807, "mix_low": 0.28723716735839844, "mix_high": 0.0, "read_cosine": 0.09204067289829254, "beta_mean": 0.45047640800476074, "nonfinite": 0.0, "weighted_memory_to_active": 0.3748232424259186, "mix_std": 0.05089518055319786, "beta_std": 0.28268080949783325, "beta_low": 0.00128173828125, "beta_high": 0.001220703125, "orthogonal_to_active": 0.32455870509147644, "evidence_weights": [0.07069205492734909, -0.04969798028469086, 0.7043362855911255], "agreement_weight": 0.04594578593969345, "orthogonal_weight": -0.933231770992279}, {"head": 4, "memory_to_active": 1.5346354246139526, "contribution_to_active": 0.11532524973154068, "mix_mean": 0.02873946726322174, "mix_low": 0.42685413360595703, "mix_high": 0.0, "read_cosine": 0.279055655002594, "beta_mean": 0.05913124233484268, "nonfinite": 0.0, "weighted_memory_to_active": 0.07956258207559586, "mix_std": 0.05095088481903076, "beta_std": 0.06747744977474213, "beta_low": 0.15966796875, "beta_high": 0.0, "orthogonal_to_active": 0.01701054349541664, "evidence_weights": [0.10307107120752335, -0.03978568688035011, 0.1866370439529419], "agreement_weight": 0.032781101763248444, "orthogonal_weight": -0.2811543047428131}, {"head": 5, "memory_to_active": 0.026246115565299988, "contribution_to_active": 0.48982828855514526, "mix_mean": 0.5221753120422363, "mix_low": 0.0014171600341796875, "mix_high": 0.00039958953857421875, "read_cosine": 0.2563732862472534, "beta_mean": 9.797491657081991e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.013321388512849808, "mix_std": 0.2454591989517212, "beta_std": 0.0002712092245928943, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.012399168685078621, "evidence_weights": [-0.1884966641664505, -0.24430561065673828, 0.4678652286529541], "agreement_weight": 0.25769710540771484, "orthogonal_weight": -0.9911525845527649}]}, {"layer": 8, "memory_to_active": 0.14063842594623566, "contribution_to_active": 0.5825958847999573, "mix_mean": 0.5213015675544739, "mix_low": 0.008658727630972862, "mix_high": 0.0021936101838946342, "read_cosine": 0.3862455487251282, "beta_mean": 0.012608937919139862, "nonfinite": 0.0, "weighted_memory_to_active": 0.07254000008106232, "mix_std": 0.2412155270576477, "beta_std": 0.018880261108279228, "beta_low": 0.783233642578125, "beta_high": 0.0, "orthogonal_to_active": 0.06431209295988083, "output_influence": 0.5437343716621399, "residual_influence": 0.21915262937545776, "state_active": 0.04056204482913017, "state_memory": 0.00433207955211401, "state_peak": 68.19194030761719, "growth_active": 1.1118751764297485, "growth_memory": 1.029179573059082, "heads": [{"head": 0, "memory_to_active": 0.05698121339082718, "contribution_to_active": 0.5757620334625244, "mix_mean": 0.6124981641769409, "mix_low": 0.00037670135498046875, "mix_high": 0.0013065338134765625, "read_cosine": 0.4410106837749481, "beta_mean": 0.0030972794629633427, "nonfinite": 0.0, "weighted_memory_to_active": 0.04105515405535698, "mix_std": 0.24635006487369537, "beta_std": 0.013141759671270847, "beta_low": 0.9356689453125, "beta_high": 0.0, "orthogonal_to_active": 0.025638382881879807, "evidence_weights": [-0.14476892352104187, -0.36790788173675537, 0.5455490350723267], "agreement_weight": 0.27377620339393616, "orthogonal_weight": -0.7877495884895325}, {"head": 1, "memory_to_active": 0.060405433177948, "contribution_to_active": 0.5540074706077576, "mix_mean": 0.5353673696517944, "mix_low": 0.00012969970703125, "mix_high": 0.0023288726806640625, "read_cosine": 0.08993346244096756, "beta_mean": 0.0008202474564313889, "nonfinite": 0.0, "weighted_memory_to_active": 0.027249911800026894, "mix_std": 0.22998970746994019, "beta_std": 0.0069752465933561325, "beta_low": 0.99273681640625, "beta_high": 0.0, "orthogonal_to_active": 0.020387519150972366, "evidence_weights": [-0.18741191923618317, -0.26649555563926697, 0.54327791929245], "agreement_weight": 0.06856764107942581, "orthogonal_weight": -0.7837880849838257}, {"head": 2, "memory_to_active": 0.009102323092520237, "contribution_to_active": 0.7044770121574402, "mix_mean": 0.6991339921951294, "mix_low": 0.0001811981201171875, "mix_high": 0.004208564758300781, "read_cosine": 0.41062289476394653, "beta_mean": 9.862527076620609e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.007336970418691635, "mix_std": 0.2143009454011917, "beta_std": 0.003195504890754819, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.004155448172241449, "evidence_weights": [-0.21356530487537384, -0.3399812579154968, 0.985368549823761], "agreement_weight": 0.3721054196357727, "orthogonal_weight": -0.647449791431427}, {"head": 3, "memory_to_active": 0.10455317795276642, "contribution_to_active": 0.6527220010757446, "mix_mean": 0.5982409715652466, "mix_low": 0.0007114410400390625, "mix_high": 0.005131721496582031, "read_cosine": 0.7969187498092651, "beta_mean": 0.00790642760694027, "nonfinite": 0.0, "weighted_memory_to_active": 0.0797366201877594, "mix_std": 0.25382667779922485, "beta_std": 0.008012323640286922, "beta_low": 0.72607421875, "beta_high": 0.0, "orthogonal_to_active": 0.024050891399383545, "evidence_weights": [-0.1608096808195114, -0.29191917181015015, 0.5134140849113464], "agreement_weight": 0.27557483315467834, "orthogonal_weight": -0.5814352035522461}, {"head": 4, "memory_to_active": 0.281100869178772, "contribution_to_active": 0.37976205348968506, "mix_mean": 0.3961249589920044, "mix_low": 0.001018524169921875, "mix_high": 1.33514404296875e-05, "read_cosine": 0.0867331251502037, "beta_mean": 0.0006029570940881968, "nonfinite": 0.0, "weighted_memory_to_active": 0.10527005046606064, "mix_std": 0.2366960644721985, "beta_std": 0.014402351342141628, "beta_low": 0.99578857421875, "beta_high": 0.0, "orthogonal_to_active": 0.01826297864317894, "evidence_weights": [-0.15003599226474762, -0.2067740559577942, 0.6680504679679871], "agreement_weight": 0.03060287982225418, "orthogonal_weight": -0.1754675954580307}, {"head": 5, "memory_to_active": 0.3316875100135803, "contribution_to_active": 0.628844678401947, "mix_mean": 0.28644394874572754, "mix_low": 0.04953479766845703, "mix_high": 0.00017261505126953125, "read_cosine": 0.4922541379928589, "beta_mean": 0.06312809139490128, "nonfinite": 0.0, "weighted_memory_to_active": 0.1745912879705429, "mix_std": 0.26612961292266846, "beta_std": 0.06755437701940536, "beta_low": 0.0491943359375, "beta_high": 0.0, "orthogonal_to_active": 0.2933773398399353, "evidence_weights": [-0.15071457624435425, -0.13947880268096924, -0.110568568110466], "agreement_weight": -0.004315308295190334, "orthogonal_weight": 1.986352801322937}]}, {"layer": 9, "memory_to_active": 0.8251050710678101, "contribution_to_active": 0.7442845106124878, "mix_mean": 0.4129985570907593, "mix_low": 0.04359229654073715, "mix_high": 0.00671259593218565, "read_cosine": 0.35155969858169556, "beta_mean": 0.03187163919210434, "nonfinite": 0.0, "weighted_memory_to_active": 0.3745002746582031, "mix_std": 0.24815818667411804, "beta_std": 0.02610412985086441, "beta_low": 0.6825460195541382, "beta_high": 0.0, "orthogonal_to_active": 0.02897869050502777, "output_influence": 0.5827394723892212, "residual_influence": 0.19207817316055298, "state_active": 0.046637751162052155, "state_memory": 0.01296495646238327, "state_peak": 61.18037414550781, "growth_active": 1.2125067710876465, "growth_memory": 1.20639169216156, "heads": [{"head": 0, "memory_to_active": 0.00021464351448230445, "contribution_to_active": 0.5954815149307251, "mix_mean": 0.6714406609535217, "mix_low": 0.0006685256958007812, "mix_high": 0.027980804443359375, "read_cosine": 0.35935524106025696, "beta_mean": 4.839686198465643e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.00014623886090703309, "mix_std": 0.27722495794296265, "beta_std": 1.45996352785005e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00014002859825268388, "evidence_weights": [-0.30257555842399597, -0.30713051557540894, 0.30512264370918274], "agreement_weight": 0.10892809927463531, "orthogonal_weight": -1.018192172050476}, {"head": 1, "memory_to_active": 0.014740373939275742, "contribution_to_active": 0.7309039235115051, "mix_mean": 0.5892834663391113, "mix_low": 0.006470680236816406, "mix_high": 0.0032711029052734375, "read_cosine": 0.44484299421310425, "beta_mean": 3.068431033170782e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011267205700278282, "mix_std": 0.2738364338874817, "beta_std": 7.534472388215363e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006809726357460022, "evidence_weights": [-0.23196370899677277, -0.28783807158470154, 0.9245750308036804], "agreement_weight": 0.5868009328842163, "orthogonal_weight": -0.7494365572929382}, {"head": 2, "memory_to_active": 0.15745487809181213, "contribution_to_active": 0.6711993217468262, "mix_mean": 0.6100647449493408, "mix_low": 0.0015163421630859375, "mix_high": 0.0069828033447265625, "read_cosine": 0.3791022300720215, "beta_mean": 0.0005088745965622365, "nonfinite": 0.0, "weighted_memory_to_active": 0.10784784704446793, "mix_std": 0.26086780428886414, "beta_std": 0.004329617600888014, "beta_low": 0.99664306640625, "beta_high": 0.0, "orthogonal_to_active": 0.0593789778649807, "evidence_weights": [-0.1959228217601776, -0.27257752418518066, 0.7495655417442322], "agreement_weight": 0.37760859727859497, "orthogonal_weight": -0.652930498123169}, {"head": 3, "memory_to_active": 0.6367443799972534, "contribution_to_active": 0.3026329278945923, "mix_mean": 0.20466195046901703, "mix_low": 0.06030750274658203, "mix_high": 2.86102294921875e-05, "read_cosine": 0.38702982664108276, "beta_mean": 0.019883302971720695, "nonfinite": 0.0, "weighted_memory_to_active": 0.17576953768730164, "mix_std": 0.21914881467819214, "beta_std": 0.02179241180419922, "beta_low": 0.37066650390625, "beta_high": 0.0, "orthogonal_to_active": 0.08927066624164581, "evidence_weights": [-0.05294620618224144, -0.12642902135849, 0.5547592639923096], "agreement_weight": 0.08761352300643921, "orthogonal_weight": -0.5605283975601196}, {"head": 4, "memory_to_active": 3.1422557830810547, "contribution_to_active": 1.8181653022766113, "mix_mean": 0.15257593989372253, "mix_low": 0.11533355712890625, "mix_high": 0.001194000244140625, "read_cosine": 0.09973949939012527, "beta_mean": 0.15357482433319092, "nonfinite": 0.0, "weighted_memory_to_active": 1.6913546323776245, "mix_std": 0.21135751903057098, "beta_std": 0.08264749497175217, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0024283649399876595, "evidence_weights": [-0.07824712991714478, -0.1470765322446823, -0.0442117415368557], "agreement_weight": -0.06976456195116043, "orthogonal_weight": 0.0014831137377768755}, {"head": 5, "memory_to_active": 0.9992204904556274, "contribution_to_active": 0.3473241925239563, "mix_mean": 0.24996453523635864, "mix_low": 0.07725715637207031, "mix_high": 0.0008182525634765625, "read_cosine": 0.4392881691455841, "beta_mean": 0.017231661826372147, "nonfinite": 0.0, "weighted_memory_to_active": 0.260616272687912, "mix_std": 0.24651354551315308, "beta_std": 0.04777844622731209, "beta_low": 0.72796630859375, "beta_high": 0.0, "orthogonal_to_active": 0.015844371169805527, "evidence_weights": [0.024135904386639595, -0.20530545711517334, 0.1452687531709671], "agreement_weight": 0.0064992113038897514, "orthogonal_weight": 0.0665643960237503}]}, {"layer": 10, "memory_to_active": 0.21172483265399933, "contribution_to_active": 0.7749963998794556, "mix_mean": 0.5362107753753662, "mix_low": 0.024553140625357628, "mix_high": 0.005462487693876028, "read_cosine": 0.38229191303253174, "beta_mean": 0.041696641594171524, "nonfinite": 0.0, "weighted_memory_to_active": 0.08805230259895325, "mix_std": 0.26320379972457886, "beta_std": 0.043475404381752014, "beta_low": 0.6682332754135132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.3026028871536255, "output_influence": 0.49213308095932007, "residual_influence": 0.18654128909111023, "state_active": 0.05294495075941086, "state_memory": 0.013092848472297192, "state_peak": 124.6712875366211, "growth_active": 1.008098840713501, "growth_memory": 1.0063875913619995, "heads": [{"head": 0, "memory_to_active": 0.7790298461914062, "contribution_to_active": 2.2873358726501465, "mix_mean": 0.5675781965255737, "mix_low": 0.1251964569091797, "mix_high": 0.006618499755859375, "read_cosine": 0.07073330879211426, "beta_mean": 0.18000438809394836, "nonfinite": 0.0, "weighted_memory_to_active": 0.4304478168487549, "mix_std": 0.3986775577068329, "beta_std": 0.1928044557571411, "beta_low": 0.172607421875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.745276927947998, "evidence_weights": [-0.07798843830823898, -0.35429537296295166, -2.035611152648926], "agreement_weight": 0.046822067350149155, "orthogonal_weight": 4.081365585327148}, {"head": 1, "memory_to_active": 0.004175299312919378, "contribution_to_active": 0.6451997756958008, "mix_mean": 0.6509902477264404, "mix_low": 0.0008058547973632812, "mix_high": 0.018430709838867188, "read_cosine": 0.34906649589538574, "beta_mean": 2.5513352738926187e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002513628453016281, "mix_std": 0.2563595771789551, "beta_std": 4.993578477296978e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003835903713479638, "evidence_weights": [-0.24721650779247284, -0.1896718591451645, 0.5393045544624329], "agreement_weight": 0.6905186772346497, "orthogonal_weight": -1.6629201173782349}, {"head": 2, "memory_to_active": 0.010840415954589844, "contribution_to_active": 0.6186832785606384, "mix_mean": 0.677093505859375, "mix_low": 0.00015163421630859375, "mix_high": 0.0060634613037109375, "read_cosine": 0.03140377625823021, "beta_mean": 2.2873464331496507e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004842843860387802, "mix_std": 0.24468819797039032, "beta_std": 8.26428658911027e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019423651974648237, "evidence_weights": [-0.2648933231830597, -0.2708432376384735, 0.7232404947280884], "agreement_weight": -0.24151939153671265, "orthogonal_weight": -0.384758859872818}, {"head": 3, "memory_to_active": 0.3594856262207031, "contribution_to_active": 0.09474626183509827, "mix_mean": 0.2853876054286957, "mix_low": 0.019697189331054688, "mix_high": 1.621246337890625e-05, "read_cosine": 0.9330646991729736, "beta_mean": 0.06367059797048569, "nonfinite": 0.0, "weighted_memory_to_active": 0.03607197478413582, "mix_std": 0.22016194462776184, "beta_std": 0.05884777009487152, "beta_low": 0.00469970703125, "beta_high": 0.0, "orthogonal_to_active": 0.038074031472206116, "evidence_weights": [-0.08151614665985107, -0.139459028840065, 0.14292095601558685], "agreement_weight": 0.01792413741350174, "orthogonal_weight": 1.515080213546753}, {"head": 4, "memory_to_active": 0.00786829274147749, "contribution_to_active": 0.5879488587379456, "mix_mean": 0.6118904948234558, "mix_low": 0.000209808349609375, "mix_high": 0.0016431808471679688, "read_cosine": 0.09728700667619705, "beta_mean": 1.8520280718803406e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00483265146613121, "mix_std": 0.23749366402626038, "beta_std": 3.782222847803496e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007069321349263191, "evidence_weights": [-0.17912821471691132, -0.24639436602592468, 0.8984858989715576], "agreement_weight": 0.20789749920368195, "orthogonal_weight": -1.541853666305542}, {"head": 5, "memory_to_active": 0.10894954204559326, "contribution_to_active": 0.4160640239715576, "mix_mean": 0.42432427406311035, "mix_low": 0.0012578964233398438, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8121962547302246, "beta_mean": 0.006437946110963821, "nonfinite": 0.0, "weighted_memory_to_active": 0.0496048778295517, "mix_std": 0.22184182703495026, "beta_std": 0.009029804728925228, "beta_low": 0.83209228515625, "beta_high": 0.0, "orthogonal_to_active": 0.019418857991695404, "evidence_weights": [-0.13262149691581726, -0.1657911241054535, 0.2808980345726013], "agreement_weight": 0.09794910252094269, "orthogonal_weight": -0.6788536310195923}]}, {"layer": 11, "memory_to_active": 0.06943763792514801, "contribution_to_active": 0.6411678791046143, "mix_mean": 0.5370215773582458, "mix_low": 0.11280982196331024, "mix_high": 0.05120722949504852, "read_cosine": 0.3416476547718048, "beta_mean": 0.02350729890167713, "nonfinite": 0.0, "weighted_memory_to_active": 0.03897354379296303, "mix_std": 0.32180535793304443, "beta_std": 0.035029709339141846, "beta_low": 0.72174072265625, "beta_high": 0.0, "orthogonal_to_active": 0.040677592158317566, "output_influence": 0.5741699934005737, "residual_influence": 0.1794499307870865, "state_active": 0.22707056999206543, "state_memory": 0.002621967112645507, "state_peak": 34.937599182128906, "growth_active": 1.1044337749481201, "growth_memory": 1.4195902347564697, "heads": [{"head": 0, "memory_to_active": 0.006613908335566521, "contribution_to_active": 0.6173049211502075, "mix_mean": 0.6753222942352295, "mix_low": 0.000335693359375, "mix_high": 0.00881195068359375, "read_cosine": 0.23765669763088226, "beta_mean": 6.585827213712037e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.005079630296677351, "mix_std": 0.24453520774841309, "beta_std": 0.002947141882032156, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.004185252822935581, "evidence_weights": [-0.17446182668209076, -0.3512604832649231, 0.7981043457984924], "agreement_weight": 0.43747246265411377, "orthogonal_weight": -0.8515815734863281}, {"head": 1, "memory_to_active": 0.10395336151123047, "contribution_to_active": 0.45533493161201477, "mix_mean": 0.487516313791275, "mix_low": 0.004172325134277344, "mix_high": 0.0007610321044921875, "read_cosine": 0.49891048669815063, "beta_mean": 0.0016126984264701605, "nonfinite": 0.0, "weighted_memory_to_active": 0.054409027099609375, "mix_std": 0.26832500100135803, "beta_std": 0.008528131991624832, "beta_low": 0.9708251953125, "beta_high": 0.0, "orthogonal_to_active": 0.029997512698173523, "evidence_weights": [-0.12590304017066956, -0.27644801139831543, 0.6003211140632629], "agreement_weight": 0.09179527312517166, "orthogonal_weight": -0.6633535027503967}, {"head": 2, "memory_to_active": 0.00404729600995779, "contribution_to_active": 0.9899722337722778, "mix_mean": 0.4410252571105957, "mix_low": 0.32645702362060547, "mix_high": 0.1956920623779297, "read_cosine": 0.14300185441970825, "beta_mean": 0.06419174373149872, "nonfinite": 0.0, "weighted_memory_to_active": 0.002862358931452036, "mix_std": 0.4547955095767975, "beta_std": 0.08726714551448822, "beta_low": 0.50238037109375, "beta_high": 0.0, "orthogonal_to_active": 0.03096800297498703, "evidence_weights": [0.04596148058772087, -0.3225463330745697, -1.322911024093628], "agreement_weight": -0.0027864922303706408, "orthogonal_weight": 11.191657066345215}, {"head": 3, "memory_to_active": 0.006050917785614729, "contribution_to_active": 0.8906251192092896, "mix_mean": 0.41842061281204224, "mix_low": 0.3386507034301758, "mix_high": 0.09398651123046875, "read_cosine": 0.5977525115013123, "beta_mean": 0.05068014934659004, "nonfinite": 0.0, "weighted_memory_to_active": 0.003994654398411512, "mix_std": 0.44337236881256104, "beta_std": 0.07861161231994629, "beta_low": 0.54327392578125, "beta_high": 0.0, "orthogonal_to_active": 0.028279339894652367, "evidence_weights": [0.22953222692012787, -0.2593326270580292, -0.0575714185833931], "agreement_weight": 0.04952879622578621, "orthogonal_weight": 11.471078872680664}, {"head": 4, "memory_to_active": 0.21336700022220612, "contribution_to_active": 0.4474707841873169, "mix_mean": 0.6165922284126282, "mix_low": 0.00640106201171875, "mix_high": 0.0015163421630859375, "read_cosine": 0.011862657032907009, "beta_mean": 0.016839737072587013, "nonfinite": 0.0, "weighted_memory_to_active": 0.12393419444561005, "mix_std": 0.266517698764801, "beta_std": 0.020535167306661606, "beta_low": 0.49493408203125, "beta_high": 0.0, "orthogonal_to_active": 0.12221003323793411, "evidence_weights": [-0.22547604143619537, -0.23310595750808716, 1.8850570917129517], "agreement_weight": -0.2410327047109604, "orthogonal_weight": -0.9957382678985596}, {"head": 5, "memory_to_active": 0.08259332180023193, "contribution_to_active": 0.4462992250919342, "mix_mean": 0.5832524299621582, "mix_low": 0.0008420944213867188, "mix_high": 0.0064754486083984375, "read_cosine": 0.5607016086578369, "beta_mean": 0.007653598673641682, "nonfinite": 0.0, "weighted_memory_to_active": 0.04356140270829201, "mix_std": 0.2532861828804016, "beta_std": 0.012289058417081833, "beta_low": 0.819580078125, "beta_high": 0.0, "orthogonal_to_active": 0.02842540293931961, "evidence_weights": [-0.1852855086326599, -0.30313700437545776, 0.6702393293380737], "agreement_weight": 0.08250035345554352, "orthogonal_weight": -0.8577987551689148}]}, {"layer": 12, "memory_to_active": 0.21303154528141022, "contribution_to_active": 0.5986841320991516, "mix_mean": 0.5622988939285278, "mix_low": 0.0078798932954669, "mix_high": 0.007503668777644634, "read_cosine": 0.22163334488868713, "beta_mean": 0.015201108530163765, "nonfinite": 0.0, "weighted_memory_to_active": 0.10758711397647858, "mix_std": 0.25499022006988525, "beta_std": 0.01558796875178814, "beta_low": 0.8289693593978882, "beta_high": 0.0, "orthogonal_to_active": 0.06719540059566498, "output_influence": 0.5353248119354248, "residual_influence": 0.17391090095043182, "state_active": 0.033995699137449265, "state_memory": 0.007002905011177063, "state_peak": 79.897705078125, "growth_active": 1.1703662872314453, "growth_memory": 1.1194078922271729, "heads": [{"head": 0, "memory_to_active": 0.030039310455322266, "contribution_to_active": 0.6465322971343994, "mix_mean": 0.6046949625015259, "mix_low": 0.000926971435546875, "mix_high": 0.007979393005371094, "read_cosine": 0.27166399359703064, "beta_mean": 0.00048725842498242855, "nonfinite": 0.0, "weighted_memory_to_active": 0.017393097281455994, "mix_std": 0.2601630687713623, "beta_std": 0.0010646667797118425, "beta_low": 0.9981689453125, "beta_high": 0.0, "orthogonal_to_active": 0.015614494681358337, "evidence_weights": [-0.19804342091083527, -0.2676926851272583, -0.10740590840578079], "agreement_weight": 0.10087659955024719, "orthogonal_weight": -0.9663335084915161}, {"head": 1, "memory_to_active": 0.07752339541912079, "contribution_to_active": 0.5853918194770813, "mix_mean": 0.5814406871795654, "mix_low": 0.00021648406982421875, "mix_high": 0.00274658203125, "read_cosine": 0.3967243432998657, "beta_mean": 0.0003931907704100013, "nonfinite": 0.0, "weighted_memory_to_active": 0.04868823289871216, "mix_std": 0.25966039299964905, "beta_std": 0.0073498147539794445, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.03980433940887451, "evidence_weights": [-0.19356702268123627, -0.26784220337867737, 1.0218133926391602], "agreement_weight": 0.30658599734306335, "orthogonal_weight": -0.904606819152832}, {"head": 2, "memory_to_active": 0.5849562883377075, "contribution_to_active": 0.4733290672302246, "mix_mean": 0.3270183801651001, "mix_low": 0.023363113403320312, "mix_high": 1.239776611328125e-05, "read_cosine": 0.6094386577606201, "beta_mean": 0.08906778693199158, "nonfinite": 0.0, "weighted_memory_to_active": 0.21874023973941803, "mix_std": 0.2334652990102768, "beta_std": 0.06488659977912903, "beta_low": 0.001708984375, "beta_high": 0.0, "orthogonal_to_active": 0.25424662232398987, "evidence_weights": [-0.19793327152729034, -0.06139528006315231, -0.028312498703598976], "agreement_weight": 0.027135007083415985, "orthogonal_weight": 1.477443814277649}, {"head": 3, "memory_to_active": 0.03373495116829872, "contribution_to_active": 0.783879280090332, "mix_mean": 0.7394397258758545, "mix_low": 6.580352783203125e-05, "mix_high": 0.023057937622070312, "read_cosine": 0.08167317509651184, "beta_mean": 0.0002889223978854716, "nonfinite": 0.0, "weighted_memory_to_active": 0.02537066861987114, "mix_std": 0.21965940296649933, "beta_std": 0.006438193842768669, "beta_low": 0.99774169921875, "beta_high": 0.0, "orthogonal_to_active": 0.023953620344400406, "evidence_weights": [-0.2812739312648773, -0.3283173143863678, 1.7330753803253174], "agreement_weight": 0.6721412539482117, "orthogonal_weight": -0.964644193649292}, {"head": 4, "memory_to_active": 0.464519739151001, "contribution_to_active": 0.5964296460151672, "mix_mean": 0.47813093662261963, "mix_low": 0.01649951934814453, "mix_high": 0.0049686431884765625, "read_cosine": -0.22288063168525696, "beta_mean": 0.0005643799086101353, "nonfinite": 0.0, "weighted_memory_to_active": 0.27727147936820984, "mix_std": 0.28799760341644287, "beta_std": 0.008054569363594055, "beta_low": 0.98651123046875, "beta_high": 0.0, "orthogonal_to_active": 0.026495955884456635, "evidence_weights": [-0.15451925992965698, -0.21139341592788696, -0.1591389924287796], "agreement_weight": 0.02439028210937977, "orthogonal_weight": -0.0986156091094017}, {"head": 5, "memory_to_active": 0.08741558343172073, "contribution_to_active": 0.5065426826477051, "mix_mean": 0.643068790435791, "mix_low": 0.006207466125488281, "mix_high": 0.006257057189941406, "read_cosine": 0.19318050146102905, "beta_mean": 0.0004051181022077799, "nonfinite": 0.0, "weighted_memory_to_active": 0.05805891007184982, "mix_std": 0.26899564266204834, "beta_std": 0.005733967293053865, "beta_low": 0.9915771484375, "beta_high": 0.0, "orthogonal_to_active": 0.04305732995271683, "evidence_weights": [-0.21683435142040253, -0.3063052296638489, 1.2195155620574951], "agreement_weight": 0.4379707872867584, "orthogonal_weight": -0.7958110570907593}]}, {"layer": 13, "memory_to_active": 0.10859037935733795, "contribution_to_active": 0.5485439300537109, "mix_mean": 0.5545172691345215, "mix_low": 0.005859851837158203, "mix_high": 0.00943613052368164, "read_cosine": 0.4196198582649231, "beta_mean": 0.013420147821307182, "nonfinite": 0.0, "weighted_memory_to_active": 0.04634060710668564, "mix_std": 0.25472062826156616, "beta_std": 0.014932224527001381, "beta_low": 0.8130798935890198, "beta_high": 0.0, "orthogonal_to_active": 0.04895523563027382, "output_influence": 0.5411875247955322, "residual_influence": 0.21239909529685974, "state_active": 0.12842097878456116, "state_memory": 0.037010032683610916, "state_peak": 98.41849517822266, "growth_active": 0.9054444432258606, "growth_memory": 0.9618417620658875, "heads": [{"head": 0, "memory_to_active": 0.006070987321436405, "contribution_to_active": 0.6246094703674316, "mix_mean": 0.6316579580307007, "mix_low": 0.000759124755859375, "mix_high": 0.0045642852783203125, "read_cosine": 0.4595763683319092, "beta_mean": 2.0590145140886307e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004065559711307287, "mix_std": 0.259647011756897, "beta_std": 4.6648739953525364e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0036363275721669197, "evidence_weights": [-0.18660421669483185, -0.26150891184806824, 0.7135615348815918], "agreement_weight": 0.25269952416419983, "orthogonal_weight": -1.0124467611312866}, {"head": 1, "memory_to_active": 0.007669471204280853, "contribution_to_active": 0.6245808005332947, "mix_mean": 0.5881803035736084, "mix_low": 0.001255035400390625, "mix_high": 0.0047855377197265625, "read_cosine": 0.23611024022102356, "beta_mean": 2.9780941986246035e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00433754175901413, "mix_std": 0.2707170248031616, "beta_std": 6.726148421876132e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.005285733379423618, "evidence_weights": [-0.1753360778093338, -0.2585392892360687, 0.495792031288147], "agreement_weight": 0.6181736588478088, "orthogonal_weight": -1.268503189086914}, {"head": 2, "memory_to_active": 0.05830549821257591, "contribution_to_active": 0.5139480233192444, "mix_mean": 0.4083971381187439, "mix_low": 0.000957489013671875, "mix_high": 0.00017642974853515625, "read_cosine": 0.634617805480957, "beta_mean": 0.006246885284781456, "nonfinite": 0.0, "weighted_memory_to_active": 0.02814164012670517, "mix_std": 0.2290533035993576, "beta_std": 0.023496156558394432, "beta_low": 0.88275146484375, "beta_high": 0.0, "orthogonal_to_active": 0.015292449854314327, "evidence_weights": [-0.1718292236328125, -0.21465249359607697, 0.4097277522087097], "agreement_weight": 0.005064216908067465, "orthogonal_weight": -0.8537432551383972}, {"head": 3, "memory_to_active": 0.17258697748184204, "contribution_to_active": 0.5448868274688721, "mix_mean": 0.6736450791358948, "mix_low": 0.02647113800048828, "mix_high": 0.034401893615722656, "read_cosine": -0.04989447444677353, "beta_mean": 0.0008632786921225488, "nonfinite": 0.0, "weighted_memory_to_active": 0.12615476548671722, "mix_std": 0.2873024046421051, "beta_std": 0.010582995600998402, "beta_low": 0.9951171875, "beta_high": 0.0, "orthogonal_to_active": 0.116144098341465, "evidence_weights": [-0.3162928521633148, -0.23848403990268707, 1.42543363571167], "agreement_weight": 6.934495449066162, "orthogonal_weight": -0.9818163514137268}, {"head": 4, "memory_to_active": 0.37272316217422485, "contribution_to_active": 0.2516586482524872, "mix_mean": 0.3580697774887085, "mix_low": 0.0037813186645507812, "mix_high": 0.0, "read_cosine": 0.8996415734291077, "beta_mean": 0.07322688400745392, "nonfinite": 0.0, "weighted_memory_to_active": 0.09257276356220245, "mix_std": 0.21488872170448303, "beta_std": 0.053567901253700256, "beta_low": 0.0010986328125, "beta_high": 0.0, "orthogonal_to_active": 0.1341550052165985, "evidence_weights": [-0.05243131145834923, -0.21817858517169952, 0.40368208289146423], "agreement_weight": 0.0068262130953371525, "orthogonal_weight": 2.3427329063415527}, {"head": 5, "memory_to_active": 0.034186143428087234, "contribution_to_active": 0.7315798997879028, "mix_mean": 0.667153000831604, "mix_low": 0.0019350051879882812, "mix_high": 0.012688636779785156, "read_cosine": 0.3376675844192505, "beta_mean": 0.000133467314299196, "nonfinite": 0.0, "weighted_memory_to_active": 0.022771352902054787, "mix_std": 0.26671528816223145, "beta_std": 0.001832383219152689, "beta_low": 0.99951171875, "beta_high": 0.0, "orthogonal_to_active": 0.01921777054667473, "evidence_weights": [-0.21673768758773804, -0.2800173759460449, 0.8778508305549622], "agreement_weight": 0.2273276150226593, "orthogonal_weight": -0.9268612265586853}]}, {"layer": 14, "memory_to_active": 0.2135644406080246, "contribution_to_active": 0.4147602915763855, "mix_mean": 0.4307992160320282, "mix_low": 0.02784077450633049, "mix_high": 0.006278038490563631, "read_cosine": 0.6136126518249512, "beta_mean": 0.05984872207045555, "nonfinite": 0.0, "weighted_memory_to_active": 0.08610861748456955, "mix_std": 0.23500531911849976, "beta_std": 0.057611867785453796, "beta_low": 0.5069783926010132, "beta_high": 0.0, "orthogonal_to_active": 0.04117104783654213, "output_influence": 0.3515161871910095, "residual_influence": 0.1661168336868286, "state_active": 0.049475692212581635, "state_memory": 0.012398181483149529, "state_peak": 55.722774505615234, "growth_active": 1.0959546566009521, "growth_memory": 1.2265766859054565, "heads": [{"head": 0, "memory_to_active": 0.34354323148727417, "contribution_to_active": 0.19668981432914734, "mix_mean": 0.1968725621700287, "mix_low": 0.015848159790039062, "mix_high": 7.62939453125e-06, "read_cosine": 0.8139024972915649, "beta_mean": 0.0661107748746872, "nonfinite": 0.0, "weighted_memory_to_active": 0.07401110231876373, "mix_std": 0.17548689246177673, "beta_std": 0.05955033749341965, "beta_low": 0.010009765625, "beta_high": 0.0, "orthogonal_to_active": 0.08922693878412247, "evidence_weights": [-0.09105809032917023, -0.04484356567263603, 0.3502323627471924], "agreement_weight": 0.018854312598705292, "orthogonal_weight": 1.9791254997253418}, {"head": 1, "memory_to_active": 0.45153215527534485, "contribution_to_active": 0.27444514632225037, "mix_mean": 0.1887100338935852, "mix_low": 0.14038753509521484, "mix_high": 0.0, "read_cosine": 0.8847724199295044, "beta_mean": 0.24813026189804077, "nonfinite": 0.0, "weighted_memory_to_active": 0.18782256543636322, "mix_std": 0.22019296884536743, "beta_std": 0.2433943897485733, "beta_low": 0.08929443359375, "beta_high": 0.0, "orthogonal_to_active": 0.020756976678967476, "evidence_weights": [0.01932218112051487, -0.1843200922012329, -0.10513994842767715], "agreement_weight": 0.05587129667401314, "orthogonal_weight": 0.3620036840438843}, {"head": 2, "memory_to_active": 0.1211029589176178, "contribution_to_active": 0.5888246893882751, "mix_mean": 0.593603253364563, "mix_low": 0.0023241043090820312, "mix_high": 0.005311012268066406, "read_cosine": 0.41191786527633667, "beta_mean": 0.0015575569123029709, "nonfinite": 0.0, "weighted_memory_to_active": 0.07837492227554321, "mix_std": 0.26133814454078674, "beta_std": 0.002128555905073881, "beta_low": 0.9896240234375, "beta_high": 0.0, "orthogonal_to_active": 0.0595167875289917, "evidence_weights": [-0.1230735331773758, -0.3131885230541229, 0.8959912061691284], "agreement_weight": 0.08525793999433517, "orthogonal_weight": -0.8619420528411865}, {"head": 3, "memory_to_active": 0.03629017621278763, "contribution_to_active": 0.5970426797866821, "mix_mean": 0.6847288012504578, "mix_low": 0.0034275054931640625, "mix_high": 0.023276329040527344, "read_cosine": -0.06806755065917969, "beta_mean": 0.005034942179918289, "nonfinite": 0.0, "weighted_memory_to_active": 0.02379383146762848, "mix_std": 0.25208234786987305, "beta_std": 0.008077510632574558, "beta_low": 0.86297607421875, "beta_high": 0.0, "orthogonal_to_active": 0.01853914000093937, "evidence_weights": [-0.3372182548046112, -0.31109619140625, 0.3129693865776062], "agreement_weight": 0.0695897787809372, "orthogonal_weight": -0.844756543636322}, {"head": 4, "memory_to_active": 0.1550137996673584, "contribution_to_active": 0.489698588848114, "mix_mean": 0.5378692746162415, "mix_low": 0.0024557113647460938, "mix_high": 0.008069992065429688, "read_cosine": 0.9017763733863831, "beta_mean": 0.03744897618889809, "nonfinite": 0.0, "weighted_memory_to_active": 0.09019109606742859, "mix_std": 0.2549213171005249, "beta_std": 0.02755449339747429, "beta_low": 0.102294921875, "beta_high": 0.0, "orthogonal_to_active": 0.025528771802783012, "evidence_weights": [-0.1976558268070221, -0.23233553767204285, 0.4551597833633423], "agreement_weight": 0.13355213403701782, "orthogonal_weight": -0.6361611485481262}, {"head": 5, "memory_to_active": 0.17390432953834534, "contribution_to_active": 0.34186074137687683, "mix_mean": 0.3830113410949707, "mix_low": 0.00260162353515625, "mix_high": 0.001003265380859375, "read_cosine": 0.7373743057250977, "beta_mean": 0.0008097962127067149, "nonfinite": 0.0, "weighted_memory_to_active": 0.06245820224285126, "mix_std": 0.24601025879383087, "beta_std": 0.004965893924236298, "beta_low": 0.9876708984375, "beta_high": 0.0, "orthogonal_to_active": 0.0334576740860939, "evidence_weights": [-0.17011040449142456, -0.18468886613845825, 0.3810010254383087], "agreement_weight": 0.05694011226296425, "orthogonal_weight": -0.7574175596237183}]}, {"layer": 15, "memory_to_active": 0.04628656059503555, "contribution_to_active": 0.5119184851646423, "mix_mean": 0.44860899448394775, "mix_low": 0.005168914794921875, "mix_high": 0.0031859080772846937, "read_cosine": 0.6544222831726074, "beta_mean": 0.038894932717084885, "nonfinite": 0.0, "weighted_memory_to_active": 0.024438437074422836, "mix_std": 0.281571626663208, "beta_std": 0.043051160871982574, "beta_low": 0.7017924189567566, "beta_high": 0.0, "orthogonal_to_active": 0.010212256573140621, "output_influence": 0.332074373960495, "residual_influence": 0.2139870822429657, "state_active": 0.1764034628868103, "state_memory": 0.009542569518089294, "state_peak": 25.12162971496582, "growth_active": 1.1884078979492188, "growth_memory": 1.2409889698028564, "heads": [{"head": 0, "memory_to_active": 0.11077176034450531, "contribution_to_active": 0.4808751046657562, "mix_mean": 0.4110134243965149, "mix_low": 0.005329132080078125, "mix_high": 0.006671905517578125, "read_cosine": 0.4655963182449341, "beta_mean": 0.05428018420934677, "nonfinite": 0.0, "weighted_memory_to_active": 0.05918304622173309, "mix_std": 0.28590089082717896, "beta_std": 0.037433821707963943, "beta_low": 0.04290771484375, "beta_high": 0.0, "orthogonal_to_active": 0.03981233015656471, "evidence_weights": [-0.17697177827358246, -0.19313274323940277, 0.5618574619293213], "agreement_weight": 0.08893724530935287, "orthogonal_weight": -0.8604612350463867}, {"head": 1, "memory_to_active": 0.07572665810585022, "contribution_to_active": 0.43885472416877747, "mix_mean": 0.4578903913497925, "mix_low": 0.0005102157592773438, "mix_high": 0.007937431335449219, "read_cosine": 0.9297523498535156, "beta_mean": 0.00012273983156774193, "nonfinite": 0.0, "weighted_memory_to_active": 0.032145652920007706, "mix_std": 0.2794935703277588, "beta_std": 0.0013197563821449876, "beta_low": 0.99932861328125, "beta_high": 0.0, "orthogonal_to_active": 0.003085668897256255, "evidence_weights": [-0.16455842554569244, -0.17933601140975952, 0.1525583565235138], "agreement_weight": 0.09557479619979858, "orthogonal_weight": 0.20358353853225708}, {"head": 2, "memory_to_active": 0.005981104448437691, "contribution_to_active": 0.5365283489227295, "mix_mean": 0.36982399225234985, "mix_low": 0.008051872253417969, "mix_high": 7.534027099609375e-05, "read_cosine": 0.7864199280738831, "beta_mean": 7.21719698049128e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.003338769543915987, "mix_std": 0.2741292715072632, "beta_std": 0.00020481145475059748, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0010875897714868188, "evidence_weights": [-0.19370856881141663, -0.26765555143356323, 0.1284790188074112], "agreement_weight": 0.009156158193945885, "orthogonal_weight": 1.0015748739242554}, {"head": 3, "memory_to_active": 0.0014088128227740526, "contribution_to_active": 0.3830687701702118, "mix_mean": 0.4284008741378784, "mix_low": 0.0032215118408203125, "mix_high": 0.0006341934204101562, "read_cosine": 0.6197268962860107, "beta_mean": 7.026064849924296e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0004869020776823163, "mix_std": 0.28779512643814087, "beta_std": 1.6526169929420575e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00023084355052560568, "evidence_weights": [-0.2732253968715668, -0.23943744599819183, 0.17178869247436523], "agreement_weight": 0.06315454840660095, "orthogonal_weight": -0.4792581796646118}, {"head": 4, "memory_to_active": 0.05304093658924103, "contribution_to_active": 0.6625303626060486, "mix_mean": 0.41704612970352173, "mix_low": 0.005129814147949219, "mix_high": 0.0009832382202148438, "read_cosine": 0.7332241535186768, "beta_mean": 0.17613190412521362, "nonfinite": 0.0, "weighted_memory_to_active": 0.030145004391670227, "mix_std": 0.3087271749973297, "beta_std": 0.20016062259674072, "beta_low": 0.200927734375, "beta_high": 0.0, "orthogonal_to_active": 0.005392916034907103, "evidence_weights": [-0.20483337342739105, -0.2522575557231903, -0.02720162272453308], "agreement_weight": 0.010900445282459259, "orthogonal_weight": 0.41200724244117737}, {"head": 5, "memory_to_active": 0.030790090560913086, "contribution_to_active": 0.5696533918380737, "mix_mean": 0.6074788570404053, "mix_low": 0.008770942687988281, "mix_high": 0.0028133392333984375, "read_cosine": 0.3918137550354004, "beta_mean": 0.0027555685956031084, "nonfinite": 0.0, "weighted_memory_to_active": 0.021331239491701126, "mix_std": 0.25338369607925415, "beta_std": 0.019171422347426414, "beta_low": 0.96771240234375, "beta_high": 0.0, "orthogonal_to_active": 0.011664191260933876, "evidence_weights": [-0.2760564982891083, -0.30781662464141846, 0.5329281687736511], "agreement_weight": 0.3150089383125305, "orthogonal_weight": 0.816677987575531}]}]}, {"step": 80000, "recipe": "linear-50m-gdn2-delta-memory-read-mix-v5", "data": [["HuggingFaceTB/dclm-edu", null, 0.20000000298023224], ["karpathy/climbmix-400b-shuffle", null, 0.44999998807907104], ["IFM/TxT360-v2", "web-high-medium", 0.25], ["Yxanul/cc-math-finest", null, 0.10000000149011612]], "validation_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "tokenizer_revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "seconds": 1.8341804629999388, "memory_to_active": 0.8211885690689087, "contribution_to_active": 0.5930289030075073, "mix_mean": 0.4366197884082794, "mix_low": 0.11890584230422974, "mix_high": 0.011126529425382614, "read_cosine": 0.3443656861782074, "beta_mean": 0.06767690926790237, "nonfinite": 0.0, "weighted_memory_to_active": 0.2036864161491394, "mix_std": 0.22098661959171295, "beta_std": 0.05874311923980713, "beta_low": 0.601385772228241, "beta_high": 4.259745401213877e-05, "orthogonal_to_active": 0.06660555303096771, "samples": 4, "scored_tokens": 899, "normal_loss": 2.9746792316436768, "normal_token_accuracy": 0.4627363681793213, "state_active": 0.06487378478050232, "state_memory": 0.015510840341448784, "state_peak": 124.06892395019531, "growth_active": 1.1180408000946045, "growth_memory": 1.2601583003997803, "output_influence": 0.46033117175102234, "residual_influence": 0.17585976421833038, "probes": [{"index": 0, "tokens": 256, "normal_loss": 3.0916554927825928, "no_memory_loss_delta": 0.8532938957214355, "scrambled_loss_delta": 0.14014378190040588, "memory_only_loss_delta": 4.233811378479004, "no_evidence_loss_delta": 0.48315319418907166, "no_agreement_loss_delta": 0.017483090981841087, "no_orthogonal_loss_delta": 0.01799044944345951}, {"index": 94, "tokens": 256, "normal_loss": 3.0618557929992676, "no_memory_loss_delta": 0.8999823927879333, "scrambled_loss_delta": 0.48925837874412537, "memory_only_loss_delta": 4.925253868103027, "no_evidence_loss_delta": 0.38527557253837585, "no_agreement_loss_delta": 0.03892563283443451, "no_orthogonal_loss_delta": 0.04820066690444946}, {"index": 188, "tokens": 256, "normal_loss": 3.2667369842529297, "no_memory_loss_delta": 1.1136914491653442, "scrambled_loss_delta": 0.1275671124458313, "memory_only_loss_delta": 3.4752774238586426, "no_evidence_loss_delta": 0.5494494438171387, "no_agreement_loss_delta": 0.049992289394140244, "no_orthogonal_loss_delta": 0.04265208542346954}, {"index": 283, "tokens": 131, "normal_loss": 2.00498628616333, "no_memory_loss_delta": 0.3853759169578552, "scrambled_loss_delta": 0.07785139977931976, "memory_only_loss_delta": 3.141810655593872, "no_evidence_loss_delta": 0.18761566281318665, "no_agreement_loss_delta": -0.011200208216905594, "no_orthogonal_loss_delta": -0.05979868769645691}], "agreement_weight_abs_mean": 0.2593419849872589, "agreement_weight_abs_max": 6.937392711639404, "agreement_weight_mean": 0.224739670753479, "orthogonal_weight_abs_mean": 0.9585723876953125, "orthogonal_weight_abs_max": 11.479480743408203, "orthogonal_weight_mean": -0.024615028873085976, "no_memory_loss": 3.847235918045044, "no_memory_loss_delta": 0.8725560903549194, "no_memory_token_accuracy_gain": 0.12124583125114441, "no_memory_token_win_fraction": 0.7141268253326416, "no_memory_probe_win_fraction": 1.0, "no_memory_ppl_reduction_pct": 58.211796130145935, "scrambled_loss": 3.2015793323516846, "scrambled_loss_delta": 0.22689951956272125, "scrambled_token_accuracy_gain": 0.030033370479941368, "scrambled_token_win_fraction": 0.6140155792236328, "scrambled_probe_win_fraction": 1.0, "scrambled_ppl_reduction_pct": 20.29991387380616, "memory_only_loss": 7.030261993408203, "memory_only_loss_delta": 4.0555830001831055, "memory_only_token_accuracy_gain": 0.32035595178604126, "memory_only_token_win_fraction": 0.9210233688354492, "memory_only_probe_win_fraction": 1.0, "memory_only_ppl_reduction_pct": 98.26746234591562, "no_evidence_loss": 3.4057745933532715, "no_evidence_loss_delta": 0.4310951232910156, "no_evidence_token_accuracy_gain": 0.05228031054139137, "no_evidence_token_win_fraction": 0.6206896305084229, "no_evidence_probe_win_fraction": 1.0, "no_evidence_ppl_reduction_pct": 35.02029030035276, "no_agreement_loss": 3.0033462047576904, "no_agreement_loss_delta": 0.028666777536273003, "no_agreement_token_accuracy_gain": -0.0022246940061450005, "no_agreement_token_win_fraction": 0.4805339276790619, "no_agreement_probe_win_fraction": 0.75, "no_agreement_ppl_reduction_pct": 2.8259783808122587, "no_orthogonal_loss": 2.996960163116455, "no_orthogonal_loss_delta": 0.02228056825697422, "no_orthogonal_token_accuracy_gain": -0.0033370412420481443, "no_orthogonal_token_win_fraction": 0.5728587508201599, "no_orthogonal_probe_win_fraction": 0.75, "no_orthogonal_ppl_reduction_pct": 2.203418960721609, "short_no_memory_loss_delta": 0.833667516708374, "long_context_memory_gain": 0.03888857364654541, "normal_context_loss_gain": 0.13488759100437164, "full_context": 4096, "short_context": 512, "scored_tail": 256, "layers": [{"layer": 0, "memory_to_active": 3.7995312213897705, "contribution_to_active": 1.298820972442627, "mix_mean": 0.11121522635221481, "mix_low": 0.49731385707855225, "mix_high": 0.004696369171142578, "read_cosine": 0.4057970643043518, "beta_mean": 0.22095544636249542, "nonfinite": 0.0, "weighted_memory_to_active": 1.3152000904083252, "mix_std": 0.09159769117832184, "beta_std": 0.14231786131858826, "beta_low": 0.1947530210018158, "beta_high": 0.0, "orthogonal_to_active": 0.05478530377149582, "output_influence": 0.21277689933776855, "residual_influence": 0.21417838335037231, "state_active": 0.066605344414711, "state_memory": 0.046341121196746826, "state_peak": 46.0934944152832, "growth_active": 1.166988492012024, "growth_memory": 1.2470990419387817, "heads": [{"head": 0, "memory_to_active": 3.1284894943237305, "contribution_to_active": 0.4365971088409424, "mix_mean": 0.018281113356351852, "mix_low": 0.6322154998779297, "mix_high": 0.0, "read_cosine": 0.22324150800704956, "beta_mean": 0.20822285115718842, "nonfinite": 0.0, "weighted_memory_to_active": 0.5529214143753052, "mix_std": 0.05447234958410263, "beta_std": 0.10120918601751328, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.054520152509212494, "evidence_weights": [-0.0021979508455842733, -0.03812672197818756, -0.061400946229696274], "agreement_weight": -0.2375219762325287, "orthogonal_weight": -0.11678395420312881}, {"head": 1, "memory_to_active": 7.119095802307129, "contribution_to_active": 3.2470126152038574, "mix_mean": 0.01954500749707222, "mix_low": 0.6110591888427734, "mix_high": 0.0, "read_cosine": 0.4385891556739807, "beta_mean": 0.4726454019546509, "nonfinite": 0.0, "weighted_memory_to_active": 3.4507622718811035, "mix_std": 0.057754646986722946, "beta_std": 0.21132540702819824, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.11910209059715271, "evidence_weights": [0.024172654375433922, -0.09928779304027557, 0.6747355461120605], "agreement_weight": 0.028910310938954353, "orthogonal_weight": 0.07641718536615372}, {"head": 2, "memory_to_active": 3.104689598083496, "contribution_to_active": 0.7279616594314575, "mix_mean": 0.01405645627528429, "mix_low": 0.6531753540039062, "mix_high": 0.0, "read_cosine": 0.2598040997982025, "beta_mean": 0.27454379200935364, "nonfinite": 0.0, "weighted_memory_to_active": 0.8381592035293579, "mix_std": 0.031041808426380157, "beta_std": 0.2161484956741333, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.06133953481912613, "evidence_weights": [0.06364818662405014, 0.045997656881809235, 0.28581756353378296], "agreement_weight": 0.011057068593800068, "orthogonal_weight": -0.1946548968553543}, {"head": 3, "memory_to_active": 8.116400718688965, "contribution_to_active": 2.8511576652526855, "mix_mean": 0.027709592133760452, "mix_low": 0.49524784088134766, "mix_high": 0.0, "read_cosine": 0.27470529079437256, "beta_mean": 0.3362966477870941, "nonfinite": 0.0, "weighted_memory_to_active": 2.953522205352783, "mix_std": 0.06346842646598816, "beta_std": 0.27237680554389954, "beta_low": 0.0003662109375, "beta_high": 0.0, "orthogonal_to_active": 0.07448242604732513, "evidence_weights": [-0.033060427755117416, -0.023126734420657158, -0.14249640703201294], "agreement_weight": -0.006003331393003464, "orthogonal_weight": -0.029505984857678413}, {"head": 4, "memory_to_active": 1.2279136180877686, "contribution_to_active": 0.0615435391664505, "mix_mean": 0.02598184160888195, "mix_low": 0.5806169509887695, "mix_high": 0.0, "read_cosine": 0.309704452753067, "beta_mean": 0.016690440475940704, "nonfinite": 0.0, "weighted_memory_to_active": 0.041774049401283264, "mix_std": 0.06677598506212234, "beta_std": 0.03304293006658554, "beta_low": 0.65777587890625, "beta_high": 0.0, "orthogonal_to_active": 0.0011019675293937325, "evidence_weights": [0.08024482429027557, 0.04944014176726341, -0.14446847140789032], "agreement_weight": -0.21525244414806366, "orthogonal_weight": 0.025081705302000046}, {"head": 5, "memory_to_active": 0.10059723258018494, "contribution_to_active": 0.46865230798721313, "mix_mean": 0.5617172718048096, "mix_low": 0.011568069458007812, "mix_high": 0.02817821502685547, "read_cosine": 0.9287375807762146, "beta_mean": 0.01733352057635784, "nonfinite": 0.0, "weighted_memory_to_active": 0.05406107008457184, "mix_std": 0.2760729193687439, "beta_std": 0.019804274663329124, "beta_low": 0.5103759765625, "beta_high": 0.0, "orthogonal_to_active": 0.018165618181228638, "evidence_weights": [-0.1723969578742981, -0.2151540219783783, 0.3576032519340515], "agreement_weight": 0.28164923191070557, "orthogonal_weight": -0.7338878512382507}]}, {"layer": 1, "memory_to_active": 2.6621921062469482, "contribution_to_active": 0.5113076567649841, "mix_mean": 0.2206680178642273, "mix_low": 0.3731228709220886, "mix_high": 0.0013128917198628187, "read_cosine": 0.29616883397102356, "beta_mean": 0.18183153867721558, "nonfinite": 0.0, "weighted_memory_to_active": 0.350071519613266, "mix_std": 0.11169076710939407, "beta_std": 0.13124042749404907, "beta_low": 0.35455322265625, "beta_high": 0.000152587890625, "orthogonal_to_active": 0.03761535882949829, "output_influence": 0.2851184010505676, "residual_influence": 0.08305372297763824, "state_active": 0.02676115185022354, "state_memory": 0.033534303307533264, "state_peak": 42.16162109375, "growth_active": 1.0903816223144531, "growth_memory": 1.4308137893676758, "heads": [{"head": 0, "memory_to_active": 0.014812659472227097, "contribution_to_active": 0.5000437498092651, "mix_mean": 0.5565390586853027, "mix_low": 0.0035724639892578125, "mix_high": 0.0032339096069335938, "read_cosine": 0.025115052238106728, "beta_mean": 9.240738290827721e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.009210407733917236, "mix_std": 0.28057461977005005, "beta_std": 0.0001642952993279323, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.008936119265854359, "evidence_weights": [-0.20977245271205902, -0.26932623982429504, 0.18254481256008148], "agreement_weight": 0.20470812916755676, "orthogonal_weight": -1.011125922203064}, {"head": 1, "memory_to_active": 3.2024104595184326, "contribution_to_active": 0.31440043449401855, "mix_mean": 0.019584979861974716, "mix_low": 0.5208244323730469, "mix_high": 9.5367431640625e-07, "read_cosine": 0.5295946002006531, "beta_mean": 0.27698031067848206, "nonfinite": 0.0, "weighted_memory_to_active": 0.34390151500701904, "mix_std": 0.038939666002988815, "beta_std": 0.19511888921260834, "beta_low": 0.002197265625, "beta_high": 0.00042724609375, "orthogonal_to_active": 0.022168558090925217, "evidence_weights": [0.029888227581977844, -0.0606817752122879, 0.04843838885426521], "agreement_weight": -0.001540495315566659, "orthogonal_weight": 0.21130062639713287}, {"head": 2, "memory_to_active": 0.03119991533458233, "contribution_to_active": 0.5488633513450623, "mix_mean": 0.6870929598808289, "mix_low": 0.00020694732666015625, "mix_high": 0.004642486572265625, "read_cosine": 0.455652117729187, "beta_mean": 0.00101938855368644, "nonfinite": 0.0, "weighted_memory_to_active": 0.020548736676573753, "mix_std": 0.22254261374473572, "beta_std": 0.0031902482733130455, "beta_low": 0.98321533203125, "beta_high": 0.0, "orthogonal_to_active": 0.01518609095364809, "evidence_weights": [-0.26704686880111694, -0.3640233874320984, 0.6345897912979126], "agreement_weight": 0.05348530411720276, "orthogonal_weight": -0.8423277139663696}, {"head": 3, "memory_to_active": 3.285146951675415, "contribution_to_active": 0.23909585177898407, "mix_mean": 0.02490001916885376, "mix_low": 0.5057296752929688, "mix_high": 0.0, "read_cosine": 0.17298416793346405, "beta_mean": 0.34595760703086853, "nonfinite": 0.0, "weighted_memory_to_active": 0.30578547716140747, "mix_std": 0.06168238818645477, "beta_std": 0.2638041377067566, "beta_low": 0.0697021484375, "beta_high": 0.0001220703125, "orthogonal_to_active": 0.10799122601747513, "evidence_weights": [0.0374581515789032, -0.02086988091468811, -0.01245027408003807], "agreement_weight": -0.026849709451198578, "orthogonal_weight": -0.46948474645614624}, {"head": 4, "memory_to_active": 4.502762794494629, "contribution_to_active": 1.3684498071670532, "mix_mean": 0.02968888357281685, "mix_low": 0.35260772705078125, "mix_high": 0.0, "read_cosine": 0.32239830493927, "beta_mean": 0.16785398125648499, "nonfinite": 0.0, "weighted_memory_to_active": 1.3550891876220703, "mix_std": 0.04862895607948303, "beta_std": 0.13202066719532013, "beta_low": 0.03814697265625, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.043593160808086395, "evidence_weights": [-0.00777466269209981, -0.00131860317196697, 0.17382243275642395], "agreement_weight": -0.004347975365817547, "orthogonal_weight": 0.051764119416475296}, {"head": 5, "memory_to_active": 4.936819553375244, "contribution_to_active": 0.09699269384145737, "mix_mean": 0.006202138960361481, "mix_low": 0.8557958602905273, "mix_high": 0.0, "read_cosine": 0.27126872539520264, "beta_mean": 0.29908543825149536, "nonfinite": 0.0, "weighted_memory_to_active": 0.06589381396770477, "mix_std": 0.017776338383555412, "beta_std": 0.193144291639328, "beta_low": 0.0340576171875, "beta_high": 0.00018310546875, "orthogonal_to_active": 0.027816995978355408, "evidence_weights": [0.1593714952468872, 0.07496102899312973, 0.09286469221115112], "agreement_weight": -0.11387273669242859, "orthogonal_weight": 0.41669273376464844}]}, {"layer": 2, "memory_to_active": 1.048250436782837, "contribution_to_active": 0.3612172603607178, "mix_mean": 0.41982579231262207, "mix_low": 0.23840491473674774, "mix_high": 0.004610697738826275, "read_cosine": 0.2919278144836426, "beta_mean": 0.07813084870576859, "nonfinite": 0.0, "weighted_memory_to_active": 0.07039451599121094, "mix_std": 0.18240635097026825, "beta_std": 0.0666879415512085, "beta_low": 0.6189168691635132, "beta_high": 0.000213623046875, "orthogonal_to_active": 0.014622174203395844, "output_influence": 0.42165011167526245, "residual_influence": 0.1258881539106369, "state_active": 0.041170135140419006, "state_memory": 0.012660427950322628, "state_peak": 27.890989303588867, "growth_active": 1.060643196105957, "growth_memory": 1.5456609725952148, "heads": [{"head": 0, "memory_to_active": 0.0026805170346051455, "contribution_to_active": 0.36364734172821045, "mix_mean": 0.657141923904419, "mix_low": 0.00041294097900390625, "mix_high": 0.00980377197265625, "read_cosine": 0.14176617562770844, "beta_mean": 2.331608993699774e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0017928170273080468, "mix_std": 0.2612949013710022, "beta_std": 4.093549432582222e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0018310685409232974, "evidence_weights": [-0.36985936760902405, -0.21149949729442596, 0.210784912109375], "agreement_weight": 0.14247651398181915, "orthogonal_weight": -1.0580271482467651}, {"head": 1, "memory_to_active": 3.30216646194458, "contribution_to_active": 0.04913279414176941, "mix_mean": 0.005556492134928703, "mix_low": 0.8722305297851562, "mix_high": 0.0, "read_cosine": 0.1188393384218216, "beta_mean": 0.21470578014850616, "nonfinite": 0.0, "weighted_memory_to_active": 0.03929821401834488, "mix_std": 0.011009041219949722, "beta_std": 0.14286518096923828, "beta_low": 0.00372314453125, "beta_high": 0.0, "orthogonal_to_active": 0.01109488494694233, "evidence_weights": [0.13094279170036316, 0.10157240927219391, 0.5707160830497742], "agreement_weight": 0.025583596900105476, "orthogonal_weight": 0.3126668334007263}, {"head": 2, "memory_to_active": 0.006654240190982819, "contribution_to_active": 0.41969993710517883, "mix_mean": 0.5663543939590454, "mix_low": 0.000751495361328125, "mix_high": 0.004012107849121094, "read_cosine": 0.395864874124527, "beta_mean": 1.8033710148301907e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0028634897898882627, "mix_std": 0.2732436954975128, "beta_std": 3.486379864625633e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00023735806462354958, "evidence_weights": [-0.24472877383232117, -0.2884843349456787, 0.5587171316146851], "agreement_weight": -0.03909992054104805, "orthogonal_weight": 0.10451164841651917}, {"head": 3, "memory_to_active": 0.0864967554807663, "contribution_to_active": 0.5533472895622253, "mix_mean": 0.5985211730003357, "mix_low": 0.00015354156494140625, "mix_high": 0.005833625793457031, "read_cosine": 0.5240917801856995, "beta_mean": 0.013902805745601654, "nonfinite": 0.0, "weighted_memory_to_active": 0.05629456788301468, "mix_std": 0.2484472692012787, "beta_std": 0.023435091599822044, "beta_low": 0.65045166015625, "beta_high": 0.0, "orthogonal_to_active": 0.041541606187820435, "evidence_weights": [-0.1953064501285553, -0.3239766061306, 0.714514434337616], "agreement_weight": 0.05330738052725792, "orthogonal_weight": -0.8966618180274963}, {"head": 4, "memory_to_active": 2.884469509124756, "contribution_to_active": 0.24548962712287903, "mix_mean": 0.020388033241033554, "mix_low": 0.5565519332885742, "mix_high": 0.0, "read_cosine": 0.188289612531662, "beta_mean": 0.24011188745498657, "nonfinite": 0.0, "weighted_memory_to_active": 0.3181881010532379, "mix_std": 0.04704810678958893, "beta_std": 0.23372143507003784, "beta_low": 0.059326171875, "beta_high": 0.00128173828125, "orthogonal_to_active": 0.032029036432504654, "evidence_weights": [0.022663922980427742, -0.0010081259533762932, 0.033877983689308167], "agreement_weight": 0.020997725427150726, "orthogonal_weight": -0.11956554651260376}, {"head": 5, "memory_to_active": 0.007034847512841225, "contribution_to_active": 0.5359864234924316, "mix_mean": 0.6709924936294556, "mix_low": 0.00032901763916015625, "mix_high": 0.008014678955078125, "read_cosine": 0.3827151656150818, "beta_mean": 2.3270404199138284e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0039299107156693935, "mix_std": 0.25339511036872864, "beta_std": 3.016670780198183e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0009990937542170286, "evidence_weights": [-0.27174922823905945, -0.28189000487327576, 0.883765697479248], "agreement_weight": 0.2937981188297272, "orthogonal_weight": -0.3006657361984253}]}, {"layer": 3, "memory_to_active": 0.40696975588798523, "contribution_to_active": 0.5048639178276062, "mix_mean": 0.5359988212585449, "mix_low": 0.051020145416259766, "mix_high": 0.012190818786621094, "read_cosine": 0.17959998548030853, "beta_mean": 0.0233956016600132, "nonfinite": 0.0, "weighted_memory_to_active": 0.11639908701181412, "mix_std": 0.25257670879364014, "beta_std": 0.023508552461862564, "beta_low": 0.7780762314796448, "beta_high": 0.0, "orthogonal_to_active": 0.04884658008813858, "output_influence": 0.4597763419151306, "residual_influence": 0.12419840693473816, "state_active": 0.022631917148828506, "state_memory": 0.005860829260200262, "state_peak": 25.113758087158203, "growth_active": 1.1189237833023071, "growth_memory": 1.583366870880127, "heads": [{"head": 0, "memory_to_active": 0.33721792697906494, "contribution_to_active": 0.6304738521575928, "mix_mean": 0.622501015663147, "mix_low": 0.0008907318115234375, "mix_high": 0.022423744201660156, "read_cosine": 0.22288545966148376, "beta_mean": 0.008475858718156815, "nonfinite": 0.0, "weighted_memory_to_active": 0.21031609177589417, "mix_std": 0.29244667291641235, "beta_std": 0.009701725095510483, "beta_low": 0.73284912109375, "beta_high": 0.0, "orthogonal_to_active": 0.154868483543396, "evidence_weights": [-0.21198368072509766, -0.24540969729423523, 1.0652796030044556], "agreement_weight": 0.11256220191717148, "orthogonal_weight": -0.7714829444885254}, {"head": 1, "memory_to_active": 0.006058180704712868, "contribution_to_active": 0.47858405113220215, "mix_mean": 0.6092847585678101, "mix_low": 0.0021724700927734375, "mix_high": 0.012848854064941406, "read_cosine": 0.17345860600471497, "beta_mean": 2.7720936486730352e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0038115959614515305, "mix_std": 0.2883680462837219, "beta_std": 7.474477752111852e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00304493959993124, "evidence_weights": [-0.26632747054100037, -0.21520067751407623, 0.6815682053565979], "agreement_weight": 0.6189420819282532, "orthogonal_weight": -0.868217408657074}, {"head": 2, "memory_to_active": 1.8610057830810547, "contribution_to_active": 0.2731172442436218, "mix_mean": 0.05746610462665558, "mix_low": 0.29659175872802734, "mix_high": 0.0, "read_cosine": 0.2262207269668579, "beta_mean": 0.12727466225624084, "nonfinite": 0.0, "weighted_memory_to_active": 0.30600154399871826, "mix_std": 0.10857536643743515, "beta_std": 0.11865851283073425, "beta_low": 0.02471923828125, "beta_high": 0.0, "orthogonal_to_active": 0.006430267821997404, "evidence_weights": [-0.09994170069694519, -0.008441734127700329, -0.02624339424073696], "agreement_weight": -0.007889721542596817, "orthogonal_weight": 0.02718733623623848}, {"head": 3, "memory_to_active": 0.014826938509941101, "contribution_to_active": 0.5625059604644775, "mix_mean": 0.6267409324645996, "mix_low": 0.0029315948486328125, "mix_high": 0.010636329650878906, "read_cosine": 0.2788125276565552, "beta_mean": 6.352191849146038e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.010142751969397068, "mix_std": 0.27819716930389404, "beta_std": 0.00014911071048118174, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023095032665878534, "evidence_weights": [-0.26328930258750916, -0.22066166996955872, 2.354083299636841], "agreement_weight": 2.3953444957733154, "orthogonal_weight": -0.3002708852291107}, {"head": 4, "memory_to_active": 0.012517141178250313, "contribution_to_active": 0.41926854848861694, "mix_mean": 0.5572159290313721, "mix_low": 0.0014886856079101562, "mix_high": 0.0031366348266601562, "read_cosine": 0.12374471127986908, "beta_mean": 5.818865975015797e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008486869744956493, "mix_std": 0.2852804660797119, "beta_std": 0.00010695058881537989, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.007989512756466866, "evidence_weights": [-0.24492962658405304, -0.22142958641052246, 0.04902031645178795], "agreement_weight": -0.06895393878221512, "orthogonal_weight": -1.0240134000778198}, {"head": 5, "memory_to_active": 0.2101925015449524, "contribution_to_active": 0.6652337312698364, "mix_mean": 0.7427841424942017, "mix_low": 0.0020456314086914062, "mix_high": 0.024099349975585938, "read_cosine": 0.05247777700424194, "beta_mean": 0.004473656415939331, "nonfinite": 0.0, "weighted_memory_to_active": 0.15963563323020935, "mix_std": 0.2625924050807953, "beta_std": 0.012360271997749805, "beta_low": 0.910888671875, "beta_high": 0.0, "orthogonal_to_active": 0.1184367686510086, "evidence_weights": [-0.2577812075614929, -0.34086740016937256, 0.5612977743148804], "agreement_weight": 0.2488863170146942, "orthogonal_weight": -0.7612296342849731}]}, {"layer": 4, "memory_to_active": 1.0124882459640503, "contribution_to_active": 0.49421626329421997, "mix_mean": 0.38087770342826843, "mix_low": 0.21575325727462769, "mix_high": 0.01242574118077755, "read_cosine": 0.29576802253723145, "beta_mean": 0.09911233186721802, "nonfinite": 0.0, "weighted_memory_to_active": 0.13371425867080688, "mix_std": 0.16154348850250244, "beta_std": 0.09173445403575897, "beta_low": 0.4042562246322632, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.05945068225264549, "output_influence": 0.5193994641304016, "residual_influence": 0.17733453214168549, "state_active": 0.036920152604579926, "state_memory": 0.015069338493049145, "state_peak": 44.28224563598633, "growth_active": 1.193621277809143, "growth_memory": 1.5212419033050537, "heads": [{"head": 0, "memory_to_active": 0.13460493087768555, "contribution_to_active": 0.8274820446968079, "mix_mean": 0.6981891989707947, "mix_low": 0.008162498474121094, "mix_high": 0.03500652313232422, "read_cosine": 0.42157143354415894, "beta_mean": 0.22363382577896118, "nonfinite": 0.0, "weighted_memory_to_active": 0.11735379695892334, "mix_std": 0.34389591217041016, "beta_std": 0.22068531811237335, "beta_low": 0.267822265625, "beta_high": 0.0, "orthogonal_to_active": 0.0925474762916565, "evidence_weights": [-0.23655514419078827, -0.3507092595100403, -0.253598690032959], "agreement_weight": 0.003982156980782747, "orthogonal_weight": 0.922006368637085}, {"head": 1, "memory_to_active": 0.04036113619804382, "contribution_to_active": 0.6702954769134521, "mix_mean": 0.7145464420318604, "mix_low": 4.76837158203125e-05, "mix_high": 0.02068042755126953, "read_cosine": 0.5399823784828186, "beta_mean": 0.000850105716381222, "nonfinite": 0.0, "weighted_memory_to_active": 0.030638352036476135, "mix_std": 0.232656329870224, "beta_std": 0.006913162302225828, "beta_low": 0.98138427734375, "beta_high": 0.0, "orthogonal_to_active": 0.014839624054729939, "evidence_weights": [-0.23030273616313934, -0.36135411262512207, 0.5469638109207153], "agreement_weight": 0.37994658946990967, "orthogonal_weight": -0.6598744988441467}, {"head": 2, "memory_to_active": 1.36257004737854, "contribution_to_active": 0.3868439197540283, "mix_mean": 0.06543146073818207, "mix_low": 0.2631826400756836, "mix_high": 2.47955322265625e-05, "read_cosine": 0.2274007499217987, "beta_mean": 0.04794372618198395, "nonfinite": 0.0, "weighted_memory_to_active": 0.2802925109863281, "mix_std": 0.10751432925462723, "beta_std": 0.05344049632549286, "beta_low": 0.14703369140625, "beta_high": 0.0, "orthogonal_to_active": 0.11199173331260681, "evidence_weights": [0.019239412620663643, -0.07501417398452759, 0.402265727519989], "agreement_weight": 0.12085194140672684, "orthogonal_weight": -0.5154896974563599}, {"head": 3, "memory_to_active": 1.9299802780151367, "contribution_to_active": 0.14735059440135956, "mix_mean": 0.033034853637218475, "mix_low": 0.3876514434814453, "mix_high": 0.0, "read_cosine": 0.2207309603691101, "beta_mean": 0.12691576778888702, "nonfinite": 0.0, "weighted_memory_to_active": 0.1966409683227539, "mix_std": 0.05402379482984543, "beta_std": 0.11881428211927414, "beta_low": 0.02923583984375, "beta_high": 0.0, "orthogonal_to_active": 0.004253344144672155, "evidence_weights": [-0.02456648088991642, 0.047990258783102036, -0.26269277930259705], "agreement_weight": 0.07661920040845871, "orthogonal_weight": 0.03109513781964779}, {"head": 4, "memory_to_active": 0.007820912636816502, "contribution_to_active": 0.6588878035545349, "mix_mean": 0.7601654529571533, "mix_low": 7.62939453125e-06, "mix_high": 0.01883697509765625, "read_cosine": 0.12352286279201508, "beta_mean": 1.568979314470198e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00590023398399353, "mix_std": 0.2039225697517395, "beta_std": 5.408223660197109e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0036723450757563114, "evidence_weights": [-0.3221597671508789, -0.3099764585494995, 0.21592842042446136], "agreement_weight": 0.3228185176849365, "orthogonal_weight": -0.6605190634727478}, {"head": 5, "memory_to_active": 2.5995912551879883, "contribution_to_active": 0.2744377851486206, "mix_mean": 0.013898862525820732, "mix_low": 0.635467529296875, "mix_high": 5.7220458984375e-06, "read_cosine": 0.2413996309041977, "beta_mean": 0.19531480967998505, "nonfinite": 0.0, "weighted_memory_to_active": 0.17145967483520508, "mix_std": 0.02724788896739483, "beta_std": 0.1504993885755539, "beta_low": 6.103515625e-05, "beta_high": 6.103515625e-05, "orthogonal_to_active": 0.12939955294132233, "evidence_weights": [0.0561276450753212, -0.018660856410861015, -0.2589262127876282], "agreement_weight": -0.09281713515520096, "orthogonal_weight": 1.1354182958602905}]}, {"layer": 5, "memory_to_active": 0.3512963652610779, "contribution_to_active": 0.5532194972038269, "mix_mean": 0.5274474620819092, "mix_low": 0.05564069747924805, "mix_high": 0.02055438421666622, "read_cosine": 0.1009877547621727, "beta_mean": 0.048459358513355255, "nonfinite": 0.0, "weighted_memory_to_active": 0.12304691970348358, "mix_std": 0.24845759570598602, "beta_std": 0.06350967288017273, "beta_low": 0.6929830312728882, "beta_high": 0.0, "orthogonal_to_active": 0.05422437563538551, "output_influence": 0.6103641986846924, "residual_influence": 0.22071166336536407, "state_active": 0.03822684288024902, "state_memory": 0.008764762431383133, "state_peak": 62.352577209472656, "growth_active": 1.1166051626205444, "growth_memory": 1.231619119644165, "heads": [{"head": 0, "memory_to_active": 0.15389400720596313, "contribution_to_active": 0.5959539413452148, "mix_mean": 0.521480917930603, "mix_low": 0.005062103271484375, "mix_high": 0.0021295547485351562, "read_cosine": -0.1007746160030365, "beta_mean": 0.013836417347192764, "nonfinite": 0.0, "weighted_memory_to_active": 0.05555444210767746, "mix_std": 0.2738673686981201, "beta_std": 0.02423156052827835, "beta_low": 0.617919921875, "beta_high": 0.0, "orthogonal_to_active": 0.0252983495593071, "evidence_weights": [-0.16881202161312103, -0.1870691180229187, 0.12265397608280182], "agreement_weight": -0.022567452862858772, "orthogonal_weight": -0.48074936866760254}, {"head": 1, "memory_to_active": 0.0032071718014776707, "contribution_to_active": 0.5805826187133789, "mix_mean": 0.7215280532836914, "mix_low": 0.00048160552978515625, "mix_high": 0.03985404968261719, "read_cosine": 0.26555293798446655, "beta_mean": 7.063982593535911e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.002085227519273758, "mix_std": 0.25898420810699463, "beta_std": 1.5640445781173185e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0019119924400001764, "evidence_weights": [-0.24267376959323883, -0.28191494941711426, 0.9414057731628418], "agreement_weight": 0.7511116862297058, "orthogonal_weight": -0.9714680314064026}, {"head": 2, "memory_to_active": 0.012280466966331005, "contribution_to_active": 0.614091694355011, "mix_mean": 0.6680092811584473, "mix_low": 0.001560211181640625, "mix_high": 0.019845962524414062, "read_cosine": -0.3929762840270996, "beta_mean": 6.927668437128887e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.008094807155430317, "mix_std": 0.2571205198764801, "beta_std": 0.0002959395351354033, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.007983189076185226, "evidence_weights": [-0.2530316114425659, -0.30604270100593567, 0.18053945899009705], "agreement_weight": -0.20924846827983856, "orthogonal_weight": -1.0615971088409424}, {"head": 3, "memory_to_active": 1.4782130718231201, "contribution_to_active": 0.24605681002140045, "mix_mean": 0.11939692497253418, "mix_low": 0.1121377944946289, "mix_high": 0.0, "read_cosine": 0.15081200003623962, "beta_mean": 0.09360380470752716, "nonfinite": 0.0, "weighted_memory_to_active": 0.36913105845451355, "mix_std": 0.14890113472938538, "beta_std": 0.14756925404071808, "beta_low": 0.218994140625, "beta_high": 0.0, "orthogonal_to_active": 0.15231215953826904, "evidence_weights": [-0.07475131005048752, -0.09740129113197327, 0.13646423816680908], "agreement_weight": 0.04899801313877106, "orthogonal_weight": -0.4962770342826843}, {"head": 4, "memory_to_active": 0.4028685390949249, "contribution_to_active": 0.5846315026283264, "mix_mean": 0.3186070919036865, "mix_low": 0.2133340835571289, "mix_high": 0.0040435791015625, "read_cosine": 0.5388428568840027, "beta_mean": 0.183084636926651, "nonfinite": 0.0, "weighted_memory_to_active": 0.26372620463371277, "mix_std": 0.3360198736190796, "beta_std": 0.2081494778394699, "beta_low": 0.3226318359375, "beta_high": 0.0, "orthogonal_to_active": 0.11515293270349503, "evidence_weights": [-0.10592754185199738, -0.24566040933132172, -0.08222368359565735], "agreement_weight": 0.030701249837875366, "orthogonal_weight": 0.6855152249336243}, {"head": 5, "memory_to_active": 0.05731469765305519, "contribution_to_active": 0.6980000734329224, "mix_mean": 0.8156624436378479, "mix_low": 0.0012683868408203125, "mix_high": 0.057453155517578125, "read_cosine": 0.14446957409381866, "beta_mean": 0.000154926921823062, "nonfinite": 0.0, "weighted_memory_to_active": 0.039689742028713226, "mix_std": 0.21585236489772797, "beta_std": 0.000796144362539053, "beta_low": 0.9984130859375, "beta_high": 0.0, "orthogonal_to_active": 0.022687647491693497, "evidence_weights": [-0.2975427508354187, -0.3206501007080078, 1.7946981191635132], "agreement_weight": 0.5666333436965942, "orthogonal_weight": -0.5957244038581848}]}, {"layer": 6, "memory_to_active": 0.22311297059059143, "contribution_to_active": 0.6611379981040955, "mix_mean": 0.605424702167511, "mix_low": 0.028183937072753906, "mix_high": 0.030457179993391037, "read_cosine": 0.3120928704738617, "beta_mean": 0.05981754884123802, "nonfinite": 0.0, "weighted_memory_to_active": 0.11948682367801666, "mix_std": 0.23926769196987152, "beta_std": 0.0717412680387497, "beta_low": 0.6357320547103882, "beta_high": 0.0, "orthogonal_to_active": 0.09623469412326813, "output_influence": 0.6030833721160889, "residual_influence": 0.2403029352426529, "state_active": 0.04031240940093994, "state_memory": 0.004848190117627382, "state_peak": 88.0126953125, "growth_active": 1.1751070022583008, "growth_memory": 1.1152459383010864, "heads": [{"head": 0, "memory_to_active": 0.10847125947475433, "contribution_to_active": 0.7917388081550598, "mix_mean": 0.8295701146125793, "mix_low": 0.0001220703125, "mix_high": 0.07060623168945312, "read_cosine": 0.2998115122318268, "beta_mean": 0.01044054701924324, "nonfinite": 0.0, "weighted_memory_to_active": 0.09812473505735397, "mix_std": 0.198592409491539, "beta_std": 0.03456422686576843, "beta_low": 0.859619140625, "beta_high": 0.0, "orthogonal_to_active": 0.06215561926364899, "evidence_weights": [-0.28665706515312195, -0.33584776520729065, 1.1597429513931274], "agreement_weight": 0.9825983047485352, "orthogonal_weight": -0.6818684935569763}, {"head": 1, "memory_to_active": 0.002636716002598405, "contribution_to_active": 0.5331891775131226, "mix_mean": 0.6136729717254639, "mix_low": 0.00026988983154296875, "mix_high": 0.01890087127685547, "read_cosine": 0.48198312520980835, "beta_mean": 5.693507318937918e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0012588286772370338, "mix_std": 0.2782396078109741, "beta_std": 1.2783844795194454e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0006352848722599447, "evidence_weights": [-0.1857270747423172, -0.2972763478755951, 1.0947729349136353], "agreement_weight": 0.14027804136276245, "orthogonal_weight": -0.6695235371589661}, {"head": 2, "memory_to_active": 0.588952362537384, "contribution_to_active": 0.11231403797864914, "mix_mean": 0.0656767264008522, "mix_low": 0.1621856689453125, "mix_high": 2.86102294921875e-06, "read_cosine": 0.28354179859161377, "beta_mean": 0.018080782145261765, "nonfinite": 0.0, "weighted_memory_to_active": 0.07067719101905823, "mix_std": 0.08981605619192123, "beta_std": 0.022212672978639603, "beta_low": 0.396484375, "beta_high": 0.0, "orthogonal_to_active": 0.014926495030522346, "evidence_weights": [0.04992850124835968, -0.07928965240716934, 0.3872929811477661], "agreement_weight": 0.13183262944221497, "orthogonal_weight": -0.2468567043542862}, {"head": 3, "memory_to_active": 0.12336141616106033, "contribution_to_active": 0.6878961324691772, "mix_mean": 0.6763038635253906, "mix_low": 0.0051403045654296875, "mix_high": 0.02367687225341797, "read_cosine": 0.45299020409584045, "beta_mean": 0.21333885192871094, "nonfinite": 0.0, "weighted_memory_to_active": 0.08919984847307205, "mix_std": 0.32452550530433655, "beta_std": 0.24315758049488068, "beta_low": 0.21429443359375, "beta_high": 0.0, "orthogonal_to_active": 0.05830053240060806, "evidence_weights": [-0.15970191359519958, -0.30492106080055237, 0.08685673773288727], "agreement_weight": -0.008189585991203785, "orthogonal_weight": 0.7410923838615417}, {"head": 4, "memory_to_active": 0.49633005261421204, "contribution_to_active": 1.1858291625976562, "mix_mean": 0.7346540689468384, "mix_low": 0.001270294189453125, "mix_high": 0.059576988220214844, "read_cosine": 0.0767410397529602, "beta_mean": 0.11696112155914307, "nonfinite": 0.0, "weighted_memory_to_active": 0.4435599744319916, "mix_std": 0.3117148280143738, "beta_std": 0.13028484582901, "beta_low": 0.34405517578125, "beta_high": 0.0, "orthogonal_to_active": 0.4279325604438782, "evidence_weights": [-0.1444563865661621, -0.48753124475479126, -0.8514115810394287], "agreement_weight": -0.015858598053455353, "orthogonal_weight": 0.9848229885101318}, {"head": 5, "memory_to_active": 0.018926016986370087, "contribution_to_active": 0.6558606624603271, "mix_mean": 0.7126705050468445, "mix_low": 0.00011539459228515625, "mix_high": 0.009979248046875, "read_cosine": 0.27748939394950867, "beta_mean": 7.82708011684008e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.014100336469709873, "mix_std": 0.23271776735782623, "beta_std": 0.00021542370086535811, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.01345764845609665, "evidence_weights": [-0.2291341871023178, -0.31251823902130127, 0.3431081175804138], "agreement_weight": 0.24457083642482758, "orthogonal_weight": -1.0224357843399048}]}, {"layer": 7, "memory_to_active": 1.8152174949645996, "contribution_to_active": 0.2782996892929077, "mix_mean": 0.18485449254512787, "mix_low": 0.2058045119047165, "mix_high": 0.0009120305767282844, "read_cosine": 0.2550317347049713, "beta_mean": 0.13627533614635468, "nonfinite": 0.0, "weighted_memory_to_active": 0.19703027606010437, "mix_std": 0.14739184081554413, "beta_std": 0.09667818248271942, "beta_low": 0.2337951809167862, "beta_high": 0.00029500326490961015, "orthogonal_to_active": 0.08704571425914764, "output_influence": 0.3049970269203186, "residual_influence": 0.08139430731534958, "state_active": 0.014464612118899822, "state_memory": 0.022675763815641403, "state_peak": 34.353981018066406, "growth_active": 1.1570428609848022, "growth_memory": 1.2687723636627197, "heads": [{"head": 0, "memory_to_active": 2.8695383071899414, "contribution_to_active": 0.15182964503765106, "mix_mean": 0.03819293528795242, "mix_low": 0.4086618423461914, "mix_high": 9.5367431640625e-07, "read_cosine": 0.35755839943885803, "beta_mean": 0.1865110695362091, "nonfinite": 0.0, "weighted_memory_to_active": 0.17230206727981567, "mix_std": 0.0750240683555603, "beta_std": 0.13284927606582642, "beta_low": 0.00079345703125, "beta_high": 0.00054931640625, "orthogonal_to_active": 0.0009377964306622744, "evidence_weights": [0.046023979783058167, -0.09240692108869553, 0.030826125293970108], "agreement_weight": -0.02846134826540947, "orthogonal_weight": -0.005773012991994619}, {"head": 1, "memory_to_active": 0.9542831182479858, "contribution_to_active": 0.3894973397254944, "mix_mean": 0.22633565962314606, "mix_low": 0.10204029083251953, "mix_high": 0.004619598388671875, "read_cosine": 0.4194405674934387, "beta_mean": 0.09461917728185654, "nonfinite": 0.0, "weighted_memory_to_active": 0.33291593194007874, "mix_std": 0.24257305264472961, "beta_std": 0.06306365877389908, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.04438989609479904, "evidence_weights": [-0.2177789807319641, -0.10382742434740067, -0.32233935594558716], "agreement_weight": 0.055451348423957825, "orthogonal_weight": 0.16257356107234955}, {"head": 2, "memory_to_active": 0.5210288763046265, "contribution_to_active": 0.4170859754085541, "mix_mean": 0.2564907371997833, "mix_low": 0.010816574096679688, "mix_high": 0.00042438507080078125, "read_cosine": 0.12767481803894043, "beta_mean": 0.027769140899181366, "nonfinite": 0.0, "weighted_memory_to_active": 0.2179919332265854, "mix_std": 0.2191031128168106, "beta_std": 0.03301302343606949, "beta_low": 0.24700927734375, "beta_high": 0.0, "orthogonal_to_active": 0.1317162662744522, "evidence_weights": [-0.10127653926610947, -0.18875744938850403, 0.8949324488639832], "agreement_weight": 0.010110974311828613, "orthogonal_weight": -0.6244621872901917}, {"head": 3, "memory_to_active": 5.002871990203857, "contribution_to_active": 0.10229325294494629, "mix_mean": 0.03688646852970123, "mix_low": 0.2875852584838867, "mix_high": 0.0, "read_cosine": 0.09228210896253586, "beta_mean": 0.44789183139801025, "nonfinite": 0.0, "weighted_memory_to_active": 0.3663748800754547, "mix_std": 0.05047321692109108, "beta_std": 0.28200459480285645, "beta_low": 0.00140380859375, "beta_high": 0.001220703125, "orthogonal_to_active": 0.3170265555381775, "evidence_weights": [0.07131414115428925, -0.04911526292562485, 0.7050465941429138], "agreement_weight": 0.04648945853114128, "orthogonal_weight": -0.9333919882774353}, {"head": 4, "memory_to_active": 1.519895315170288, "contribution_to_active": 0.11688366532325745, "mix_mean": 0.029017889872193336, "mix_low": 0.42435741424560547, "mix_high": 0.0, "read_cosine": 0.2709251940250397, "beta_mean": 0.06077020987868309, "nonfinite": 0.0, "weighted_memory_to_active": 0.08059904724359512, "mix_std": 0.05147566646337509, "beta_std": 0.06889022141695023, "beta_low": 0.15362548828125, "beta_high": 0.0, "orthogonal_to_active": 0.01708381250500679, "evidence_weights": [0.10283498466014862, -0.040176209062337875, 0.18604542315006256], "agreement_weight": 0.03231571242213249, "orthogonal_weight": -0.28136420249938965}, {"head": 5, "memory_to_active": 0.02368757501244545, "contribution_to_active": 0.49220821261405945, "mix_mean": 0.5222032070159912, "mix_low": 0.00136566162109375, "mix_high": 0.00042724609375, "read_cosine": 0.26230934262275696, "beta_mean": 9.057970601134002e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.011997836641967297, "mix_std": 0.2457018792629242, "beta_std": 0.0002483040443621576, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.011119935661554337, "evidence_weights": [-0.1881372481584549, -0.24389763176441193, 0.46634599566459656], "agreement_weight": 0.25783464312553406, "orthogonal_weight": -0.9882254600524902}]}, {"layer": 8, "memory_to_active": 0.13855531811714172, "contribution_to_active": 0.5813162326812744, "mix_mean": 0.5213104486465454, "mix_low": 0.008678119629621506, "mix_high": 0.0023058257065713406, "read_cosine": 0.3872188627719879, "beta_mean": 0.012556774541735649, "nonfinite": 0.0, "weighted_memory_to_active": 0.07137356698513031, "mix_std": 0.2417076677083969, "beta_std": 0.018869180232286453, "beta_low": 0.783416748046875, "beta_high": 0.0, "orthogonal_to_active": 0.0640949159860611, "output_influence": 0.5413606762886047, "residual_influence": 0.21826069056987762, "state_active": 0.04080861434340477, "state_memory": 0.0042716809548437595, "state_peak": 67.62996673583984, "growth_active": 1.1218900680541992, "growth_memory": 1.0313677787780762, "heads": [{"head": 0, "memory_to_active": 0.058503158390522, "contribution_to_active": 0.574316143989563, "mix_mean": 0.6124730110168457, "mix_low": 0.00038242340087890625, "mix_high": 0.0013103485107421875, "read_cosine": 0.4384051561355591, "beta_mean": 0.003138383384793997, "nonfinite": 0.0, "weighted_memory_to_active": 0.042088158428668976, "mix_std": 0.2466633915901184, "beta_std": 0.013464314863085747, "beta_low": 0.93536376953125, "beta_high": 0.0, "orthogonal_to_active": 0.026343073695898056, "evidence_weights": [-0.1443292647600174, -0.3675396740436554, 0.5446600914001465], "agreement_weight": 0.2734971046447754, "orthogonal_weight": -0.7879189848899841}, {"head": 1, "memory_to_active": 0.0595950223505497, "contribution_to_active": 0.5498860478401184, "mix_mean": 0.5341951847076416, "mix_low": 0.00012969970703125, "mix_high": 0.0024042129516601562, "read_cosine": 0.09714360535144806, "beta_mean": 0.000857825274579227, "nonfinite": 0.0, "weighted_memory_to_active": 0.026962516829371452, "mix_std": 0.23016265034675598, "beta_std": 0.007440455257892609, "beta_low": 0.99212646484375, "beta_high": 0.0, "orthogonal_to_active": 0.02013561688363552, "evidence_weights": [-0.18694427609443665, -0.26597025990486145, 0.5424489378929138], "agreement_weight": 0.0676044225692749, "orthogonal_weight": -0.7836998701095581}, {"head": 2, "memory_to_active": 0.008320489898324013, "contribution_to_active": 0.7048130035400391, "mix_mean": 0.6995686888694763, "mix_low": 0.00018310546875, "mix_high": 0.0043487548828125, "read_cosine": 0.4062148928642273, "beta_mean": 9.103854245040566e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0067130569368600845, "mix_std": 0.21452946960926056, "beta_std": 0.003000171622261405, "beta_low": 0.99993896484375, "beta_high": 0.0, "orthogonal_to_active": 0.003811071626842022, "evidence_weights": [-0.21354156732559204, -0.3399772047996521, 0.9857389330863953], "agreement_weight": 0.3736118972301483, "orthogonal_weight": -0.6471865773200989}, {"head": 3, "memory_to_active": 0.10319490730762482, "contribution_to_active": 0.6500439643859863, "mix_mean": 0.5984286069869995, "mix_low": 0.0006113052368164062, "mix_high": 0.005551338195800781, "read_cosine": 0.8018563985824585, "beta_mean": 0.007869837805628777, "nonfinite": 0.0, "weighted_memory_to_active": 0.0787210762500763, "mix_std": 0.2543785572052002, "beta_std": 0.007954101078212261, "beta_low": 0.72723388671875, "beta_high": 0.0, "orthogonal_to_active": 0.023449590429663658, "evidence_weights": [-0.16051658987998962, -0.29179248213768005, 0.5131677389144897], "agreement_weight": 0.2769775688648224, "orthogonal_weight": -0.58107990026474}, {"head": 4, "memory_to_active": 0.2721443474292755, "contribution_to_active": 0.3774373233318329, "mix_mean": 0.3948048949241638, "mix_low": 0.0011119842529296875, "mix_high": 1.239776611328125e-05, "read_cosine": 0.0880771279335022, "beta_mean": 0.0005770378047600389, "nonfinite": 0.0, "weighted_memory_to_active": 0.09915515780448914, "mix_std": 0.2368508130311966, "beta_std": 0.014294895343482494, "beta_low": 0.99615478515625, "beta_high": 0.0, "orthogonal_to_active": 0.01726275123655796, "evidence_weights": [-0.1493254154920578, -0.2061055302619934, 0.6668699979782104], "agreement_weight": 0.030758826062083244, "orthogonal_weight": -0.1759130358695984}, {"head": 5, "memory_to_active": 0.3295738697052002, "contribution_to_active": 0.6314004063606262, "mix_mean": 0.288392037153244, "mix_low": 0.04965019226074219, "mix_high": 0.0002079010009765625, "read_cosine": 0.491615891456604, "beta_mean": 0.06280652433633804, "nonfinite": 0.0, "weighted_memory_to_active": 0.17460140585899353, "mix_std": 0.2676611542701721, "beta_std": 0.06706114113330841, "beta_low": 0.0496826171875, "beta_high": 0.0, "orthogonal_to_active": 0.29356735944747925, "evidence_weights": [-0.1510135382413864, -0.13975898921489716, -0.11070851981639862], "agreement_weight": -0.005881324876099825, "orthogonal_weight": 1.9873602390289307}]}, {"layer": 9, "memory_to_active": 0.8170264363288879, "contribution_to_active": 0.7398245334625244, "mix_mean": 0.4129495620727539, "mix_low": 0.04337676614522934, "mix_high": 0.006567478179931641, "read_cosine": 0.3503875434398651, "beta_mean": 0.031806908547878265, "nonfinite": 0.0, "weighted_memory_to_active": 0.37113046646118164, "mix_std": 0.24824807047843933, "beta_std": 0.025680795311927795, "beta_low": 0.6775411367416382, "beta_high": 0.0, "orthogonal_to_active": 0.029306724667549133, "output_influence": 0.5819255709648132, "residual_influence": 0.19269847869873047, "state_active": 0.04708893597126007, "state_memory": 0.012944639660418034, "state_peak": 61.03725814819336, "growth_active": 1.2067736387252808, "growth_memory": 1.2117735147476196, "heads": [{"head": 0, "memory_to_active": 0.00020983452850487083, "contribution_to_active": 0.5928897857666016, "mix_mean": 0.6706706285476685, "mix_low": 0.0007038116455078125, "mix_high": 0.027395248413085938, "read_cosine": 0.3558730483055115, "beta_mean": 4.6874563963683613e-07, "nonfinite": 0.0, "weighted_memory_to_active": 0.0001435105368727818, "mix_std": 0.2771171033382416, "beta_std": 1.3916984471507021e-06, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0001371897233184427, "evidence_weights": [-0.3021148443222046, -0.3067350387573242, 0.3041132688522339], "agreement_weight": 0.10896354913711548, "orthogonal_weight": -1.0167092084884644}, {"head": 1, "memory_to_active": 0.014638944528996944, "contribution_to_active": 0.7325367331504822, "mix_mean": 0.590369701385498, "mix_low": 0.0063762664794921875, "mix_high": 0.0032110214233398438, "read_cosine": 0.4486530125141144, "beta_mean": 2.996130751853343e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.01124940812587738, "mix_std": 0.27337515354156494, "beta_std": 7.096870103850961e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0067530241794884205, "evidence_weights": [-0.23173250257968903, -0.2875470519065857, 0.9231098294258118], "agreement_weight": 0.5870639681816101, "orthogonal_weight": -0.7479307055473328}, {"head": 2, "memory_to_active": 0.1541181355714798, "contribution_to_active": 0.6691910028457642, "mix_mean": 0.6077700853347778, "mix_low": 0.0015878677368164062, "mix_high": 0.006824493408203125, "read_cosine": 0.37943586707115173, "beta_mean": 0.00048821012023836374, "nonfinite": 0.0, "weighted_memory_to_active": 0.10567313432693481, "mix_std": 0.2615272104740143, "beta_std": 0.004194147884845734, "beta_low": 0.9970703125, "beta_high": 0.0, "orthogonal_to_active": 0.05818586423993111, "evidence_weights": [-0.1954326629638672, -0.27204492688179016, 0.7492389678955078], "agreement_weight": 0.3784095346927643, "orthogonal_weight": -0.6524624824523926}, {"head": 3, "memory_to_active": 0.6557692289352417, "contribution_to_active": 0.30344700813293457, "mix_mean": 0.2053602784872055, "mix_low": 0.05970001220703125, "mix_high": 2.384185791015625e-05, "read_cosine": 0.3871942162513733, "beta_mean": 0.021495549008250237, "nonfinite": 0.0, "weighted_memory_to_active": 0.18394264578819275, "mix_std": 0.21932391822338104, "beta_std": 0.023350592702627182, "beta_low": 0.3370361328125, "beta_high": 0.0, "orthogonal_to_active": 0.0928802564740181, "evidence_weights": [-0.05305588245391846, -0.126612588763237, 0.556409478187561], "agreement_weight": 0.08894454687833786, "orthogonal_weight": -0.560279905796051}, {"head": 4, "memory_to_active": 3.1074233055114746, "contribution_to_active": 1.7972966432571411, "mix_mean": 0.15254977345466614, "mix_low": 0.11490821838378906, "mix_high": 0.0011243820190429688, "read_cosine": 0.0927324965596199, "beta_mean": 0.1527917981147766, "nonfinite": 0.0, "weighted_memory_to_active": 1.6698973178863525, "mix_std": 0.21100863814353943, "beta_std": 0.08250492811203003, "beta_low": 0.0, "beta_high": 0.0, "orthogonal_to_active": 0.0023555615916848183, "evidence_weights": [-0.0779702290892601, -0.1467897742986679, -0.044594574719667435], "agreement_weight": -0.07057245075702667, "orthogonal_weight": 0.0014556554378941655}, {"head": 5, "memory_to_active": 0.9699990153312683, "contribution_to_active": 0.3435859978199005, "mix_mean": 0.2509770095348358, "mix_low": 0.07698440551757812, "mix_high": 0.0008258819580078125, "read_cosine": 0.43843668699264526, "beta_mean": 0.016035450622439384, "nonfinite": 0.0, "weighted_memory_to_active": 0.2558766007423401, "mix_std": 0.24713638424873352, "beta_std": 0.043962735682725906, "beta_low": 0.73114013671875, "beta_high": 0.0, "orthogonal_to_active": 0.015528428368270397, "evidence_weights": [0.02376074716448784, -0.2055942416191101, 0.14478617906570435], "agreement_weight": 0.006075665354728699, "orthogonal_weight": 0.06642422080039978}]}, {"layer": 10, "memory_to_active": 0.21814437210559845, "contribution_to_active": 0.7892870903015137, "mix_mean": 0.5366412401199341, "mix_low": 0.02417357824742794, "mix_high": 0.005649725906550884, "read_cosine": 0.380914568901062, "beta_mean": 0.041422970592975616, "nonfinite": 0.0, "weighted_memory_to_active": 0.0905904471874237, "mix_std": 0.263372540473938, "beta_std": 0.04334259405732155, "beta_low": 0.6692098379135132, "beta_high": 1.0172526344831567e-05, "orthogonal_to_active": 0.31426727771759033, "output_influence": 0.49139752984046936, "residual_influence": 0.18599775433540344, "state_active": 0.052159346640110016, "state_memory": 0.013154301792383194, "state_peak": 124.06892395019531, "growth_active": 1.0095328092575073, "growth_memory": 1.0009313821792603, "heads": [{"head": 0, "memory_to_active": 0.8176342248916626, "contribution_to_active": 2.3709981441497803, "mix_mean": 0.5701853036880493, "mix_low": 0.12267494201660156, "mix_high": 0.007224082946777344, "read_cosine": 0.06596622616052628, "beta_mean": 0.17850778996944427, "nonfinite": 0.0, "weighted_memory_to_active": 0.4469742178916931, "mix_std": 0.39928126335144043, "beta_std": 0.191926509141922, "beta_low": 0.1776123046875, "beta_high": 6.103515625e-05, "orthogonal_to_active": 1.8155584335327148, "evidence_weights": [-0.07824815064668655, -0.35438215732574463, -2.0375924110412598], "agreement_weight": 0.04769885540008545, "orthogonal_weight": 4.086847305297852}, {"head": 1, "memory_to_active": 0.004061487503349781, "contribution_to_active": 0.6474447250366211, "mix_mean": 0.6524730324745178, "mix_low": 0.0007925033569335938, "mix_high": 0.01872730255126953, "read_cosine": 0.3490471839904785, "beta_mean": 2.5149258362944238e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.002460974734276533, "mix_std": 0.2561313807964325, "beta_std": 4.9987207603408024e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.003746969625353813, "evidence_weights": [-0.24720707535743713, -0.18962424993515015, 0.539438009262085], "agreement_weight": 0.691053032875061, "orthogonal_weight": -1.6615735292434692}, {"head": 2, "memory_to_active": 0.010028159245848656, "contribution_to_active": 0.6204636693000793, "mix_mean": 0.6757615804672241, "mix_low": 0.000118255615234375, "mix_high": 0.0062274932861328125, "read_cosine": 0.03826969116926193, "beta_mean": 2.1679210476577282e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004474112298339605, "mix_std": 0.24522337317466736, "beta_std": 7.504360837629065e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0017917598597705364, "evidence_weights": [-0.26452165842056274, -0.27040794491767883, 0.7239469289779663], "agreement_weight": -0.2414747029542923, "orthogonal_weight": -0.38472437858581543}, {"head": 3, "memory_to_active": 0.3625485599040985, "contribution_to_active": 0.09590782225131989, "mix_mean": 0.28673824667930603, "mix_low": 0.020000457763671875, "mix_high": 2.6702880859375e-05, "read_cosine": 0.933759331703186, "beta_mean": 0.06351543217897415, "nonfinite": 0.0, "weighted_memory_to_active": 0.03685762360692024, "mix_std": 0.2208290696144104, "beta_std": 0.05892474949359894, "beta_low": 0.0052490234375, "beta_high": 0.0, "orthogonal_to_active": 0.038593731820583344, "evidence_weights": [-0.08155164122581482, -0.13966751098632812, 0.1428002566099167], "agreement_weight": 0.018335111439228058, "orthogonal_weight": 1.5168238878250122}, {"head": 4, "memory_to_active": 0.00773976044729352, "contribution_to_active": 0.5874086618423462, "mix_mean": 0.6112346053123474, "mix_low": 0.000202178955078125, "mix_high": 0.001689910888671875, "read_cosine": 0.08471561968326569, "beta_mean": 1.7945978470379487e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0047555724158883095, "mix_std": 0.23735041916370392, "beta_std": 3.635664688772522e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.006954867392778397, "evidence_weights": [-0.17887796461582184, -0.2461026906967163, 0.8974105715751648], "agreement_weight": 0.2082958221435547, "orthogonal_weight": -1.5420199632644653}, {"head": 5, "memory_to_active": 0.10685393959283829, "contribution_to_active": 0.4134993255138397, "mix_mean": 0.4234544038772583, "mix_low": 0.0012531280517578125, "mix_high": 2.86102294921875e-06, "read_cosine": 0.8137294054031372, "beta_mean": 0.006449807435274124, "nonfinite": 0.0, "weighted_memory_to_active": 0.04802018031477928, "mix_std": 0.22141963243484497, "beta_std": 0.009042908437550068, "beta_low": 0.8323974609375, "beta_high": 0.0, "orthogonal_to_active": 0.01895778626203537, "evidence_weights": [-0.13242502510547638, -0.16580736637115479, 0.281306654214859], "agreement_weight": 0.09795677661895752, "orthogonal_weight": -0.6788455247879028}]}, {"layer": 11, "memory_to_active": 0.06845585256814957, "contribution_to_active": 0.6407747864723206, "mix_mean": 0.5365368723869324, "mix_low": 0.11367019265890121, "mix_high": 0.050227485597133636, "read_cosine": 0.34159648418426514, "beta_mean": 0.02294575609266758, "nonfinite": 0.0, "weighted_memory_to_active": 0.03824944421648979, "mix_std": 0.32157766819000244, "beta_std": 0.034357793629169464, "beta_low": 0.7265828847885132, "beta_high": 0.0, "orthogonal_to_active": 0.03996756300330162, "output_influence": 0.5733586549758911, "residual_influence": 0.17942354083061218, "state_active": 0.22720128297805786, "state_memory": 0.002581091597676277, "state_peak": 35.500057220458984, "growth_active": 1.1052801609039307, "growth_memory": 1.42193603515625, "heads": [{"head": 0, "memory_to_active": 0.006631175056099892, "contribution_to_active": 0.6162533164024353, "mix_mean": 0.6756977438926697, "mix_low": 0.00032711029052734375, "mix_high": 0.008874893188476562, "read_cosine": 0.23311570286750793, "beta_mean": 6.430082430597395e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.00509207509458065, "mix_std": 0.24410611391067505, "beta_std": 0.0028141222428530455, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.004198287148028612, "evidence_weights": [-0.17439979314804077, -0.35109108686447144, 0.7971579432487488], "agreement_weight": 0.43904998898506165, "orthogonal_weight": -0.8522441983222961}, {"head": 1, "memory_to_active": 0.10327835381031036, "contribution_to_active": 0.4556199610233307, "mix_mean": 0.48711568117141724, "mix_low": 0.003935813903808594, "mix_high": 0.0007448196411132812, "read_cosine": 0.49912163615226746, "beta_mean": 0.0013932321453467011, "nonfinite": 0.0, "weighted_memory_to_active": 0.05394691973924637, "mix_std": 0.2678268253803253, "beta_std": 0.0075678592547774315, "beta_low": 0.9766845703125, "beta_high": 0.0, "orthogonal_to_active": 0.02984086237847805, "evidence_weights": [-0.12536010146141052, -0.2760576009750366, 0.6003903746604919], "agreement_weight": 0.09125557541847229, "orthogonal_weight": -0.6631219983100891}, {"head": 2, "memory_to_active": 0.004057922400534153, "contribution_to_active": 0.9897316098213196, "mix_mean": 0.43912672996520996, "mix_low": 0.3298187255859375, "mix_high": 0.1911029815673828, "read_cosine": 0.1411750167608261, "beta_mean": 0.06249856948852539, "nonfinite": 0.0, "weighted_memory_to_active": 0.0028655403293669224, "mix_std": 0.45477724075317383, "beta_std": 0.08559076488018036, "beta_low": 0.5081787109375, "beta_high": 0.0, "orthogonal_to_active": 0.03104514814913273, "evidence_weights": [0.04670022055506706, -0.32144927978515625, -1.3222007751464844], "agreement_weight": -0.0027533655520528555, "orthogonal_weight": 11.198851585388184}, {"head": 3, "memory_to_active": 0.006003126502037048, "contribution_to_active": 0.8950445652008057, "mix_mean": 0.41741669178009033, "mix_low": 0.3407011032104492, "mix_high": 0.0924386978149414, "read_cosine": 0.5869500637054443, "beta_mean": 0.04995901510119438, "nonfinite": 0.0, "weighted_memory_to_active": 0.0040000611916184425, "mix_std": 0.44346746802330017, "beta_std": 0.07834326475858688, "beta_low": 0.54754638671875, "beta_high": 0.0, "orthogonal_to_active": 0.027974527329206467, "evidence_weights": [0.22920173406600952, -0.2593495547771454, -0.05656671151518822], "agreement_weight": 0.04931362345814705, "orthogonal_weight": 11.479480743408203}, {"head": 4, "memory_to_active": 0.20808306336402893, "contribution_to_active": 0.4441075026988983, "mix_mean": 0.6163699626922607, "mix_low": 0.006485939025878906, "mix_high": 0.0015859603881835938, "read_cosine": 0.015201233327388763, "beta_mean": 0.016194261610507965, "nonfinite": 0.0, "weighted_memory_to_active": 0.1201661005616188, "mix_std": 0.2666817307472229, "beta_std": 0.01983078569173813, "beta_low": 0.50836181640625, "beta_high": 0.0, "orthogonal_to_active": 0.11853041499853134, "evidence_weights": [-0.2251487821340561, -0.2326842099428177, 1.884482741355896], "agreement_weight": -0.2417556792497635, "orthogonal_weight": -0.995103120803833}, {"head": 5, "memory_to_active": 0.08268145471811295, "contribution_to_active": 0.44389182329177856, "mix_mean": 0.5834943056106567, "mix_low": 0.0007524490356445312, "mix_high": 0.006617546081542969, "read_cosine": 0.5740151405334473, "beta_mean": 0.007565133273601532, "nonfinite": 0.0, "weighted_memory_to_active": 0.04342596232891083, "mix_std": 0.25260648131370544, "beta_std": 0.01199997216463089, "beta_low": 0.81927490234375, "beta_high": 0.0, "orthogonal_to_active": 0.02821614034473896, "evidence_weights": [-0.18490414321422577, -0.3028881847858429, 0.6703182458877563], "agreement_weight": 0.08282354474067688, "orthogonal_weight": -0.8571265935897827}]}, {"layer": 12, "memory_to_active": 0.2100038230419159, "contribution_to_active": 0.5986232161521912, "mix_mean": 0.561603307723999, "mix_low": 0.007871150970458984, "mix_high": 0.007579803932458162, "read_cosine": 0.22129687666893005, "beta_mean": 0.015016147866845131, "nonfinite": 0.0, "weighted_memory_to_active": 0.10551419854164124, "mix_std": 0.25480151176452637, "beta_std": 0.015262860804796219, "beta_low": 0.8288981318473816, "beta_high": 0.0, "orthogonal_to_active": 0.06560958176851273, "output_influence": 0.5361288785934448, "residual_influence": 0.17512376606464386, "state_active": 0.03404802083969116, "state_memory": 0.006919455248862505, "state_peak": 80.75962829589844, "growth_active": 1.1799694299697876, "growth_memory": 1.1210240125656128, "heads": [{"head": 0, "memory_to_active": 0.02805030159652233, "contribution_to_active": 0.6461982727050781, "mix_mean": 0.6038817167282104, "mix_low": 0.0009775161743164062, "mix_high": 0.007899284362792969, "read_cosine": 0.270033597946167, "beta_mean": 0.0004633720964193344, "nonfinite": 0.0, "weighted_memory_to_active": 0.016250919550657272, "mix_std": 0.26028093695640564, "beta_std": 0.0010170161258429289, "beta_low": 0.99847412109375, "beta_high": 0.0, "orthogonal_to_active": 0.014581316150724888, "evidence_weights": [-0.19784456491470337, -0.2674655020236969, -0.10821700841188431], "agreement_weight": 0.1012798622250557, "orthogonal_weight": -0.9662251472473145}, {"head": 1, "memory_to_active": 0.07030334323644638, "contribution_to_active": 0.5860065221786499, "mix_mean": 0.5816001296043396, "mix_low": 0.000202178955078125, "mix_high": 0.002719879150390625, "read_cosine": 0.4085831046104431, "beta_mean": 0.00037525693187490106, "nonfinite": 0.0, "weighted_memory_to_active": 0.04450249671936035, "mix_std": 0.2592925429344177, "beta_std": 0.006849815137684345, "beta_low": 0.998291015625, "beta_high": 0.0, "orthogonal_to_active": 0.03621676564216614, "evidence_weights": [-0.19356688857078552, -0.2678981423377991, 1.0217353105545044], "agreement_weight": 0.3076845705509186, "orthogonal_weight": -0.9039061069488525}, {"head": 2, "memory_to_active": 0.5831867456436157, "contribution_to_active": 0.4700869917869568, "mix_mean": 0.3241361975669861, "mix_low": 0.02353954315185547, "mix_high": 2.002716064453125e-05, "read_cosine": 0.6065865159034729, "beta_mean": 0.08803287893533707, "nonfinite": 0.0, "weighted_memory_to_active": 0.2175421118736267, "mix_std": 0.2328287661075592, "beta_std": 0.0647571012377739, "beta_low": 0.00213623046875, "beta_high": 0.0, "orthogonal_to_active": 0.2523910403251648, "evidence_weights": [-0.19766855239868164, -0.06122923642396927, -0.028431078419089317], "agreement_weight": 0.027778249233961105, "orthogonal_weight": 1.4783592224121094}, {"head": 3, "memory_to_active": 0.03199264779686928, "contribution_to_active": 0.7846221327781677, "mix_mean": 0.7391624450683594, "mix_low": 5.91278076171875e-05, "mix_high": 0.023565292358398438, "read_cosine": 0.0773254856467247, "beta_mean": 0.00026895813061855733, "nonfinite": 0.0, "weighted_memory_to_active": 0.024151068180799484, "mix_std": 0.21984116733074188, "beta_std": 0.0061943428590893745, "beta_low": 0.99810791015625, "beta_high": 0.0, "orthogonal_to_active": 0.022764571011066437, "evidence_weights": [-0.2811923027038574, -0.328253835439682, 1.7338323593139648], "agreement_weight": 0.6732929944992065, "orthogonal_weight": -0.9631509780883789}, {"head": 4, "memory_to_active": 0.46188801527023315, "contribution_to_active": 0.5977029800415039, "mix_mean": 0.47764408588409424, "mix_low": 0.016193389892578125, "mix_high": 0.004986763000488281, "read_cosine": -0.22587180137634277, "beta_mean": 0.0005548819317482412, "nonfinite": 0.0, "weighted_memory_to_active": 0.2745826840400696, "mix_std": 0.28777801990509033, "beta_std": 0.007538728881627321, "beta_low": 0.98529052734375, "beta_high": 0.0, "orthogonal_to_active": 0.02616827003657818, "evidence_weights": [-0.15426762402057648, -0.21114251017570496, -0.1593998670578003], "agreement_weight": 0.024867704138159752, "orthogonal_weight": -0.09841370582580566}, {"head": 5, "memory_to_active": 0.08460193872451782, "contribution_to_active": 0.5071223974227905, "mix_mean": 0.643195390701294, "mix_low": 0.006255149841308594, "mix_high": 0.006287574768066406, "read_cosine": 0.1911243498325348, "beta_mean": 0.0004015386220999062, "nonfinite": 0.0, "weighted_memory_to_active": 0.05605590343475342, "mix_std": 0.2687874734401703, "beta_std": 0.005220157094299793, "beta_low": 0.9910888671875, "beta_high": 0.0, "orthogonal_to_active": 0.04153553023934364, "evidence_weights": [-0.2168206423521042, -0.30631232261657715, 1.2186946868896484], "agreement_weight": 0.4377491772174835, "orthogonal_weight": -0.7954943180084229}]}, {"layer": 13, "memory_to_active": 0.10874523222446442, "contribution_to_active": 0.5477502346038818, "mix_mean": 0.5532950758934021, "mix_low": 0.005912621971219778, "mix_high": 0.009135246276855469, "read_cosine": 0.42053887248039246, "beta_mean": 0.013194939121603966, "nonfinite": 0.0, "weighted_memory_to_active": 0.04588984325528145, "mix_std": 0.25456947088241577, "beta_std": 0.014733136631548405, "beta_low": 0.8133545517921448, "beta_high": 0.0, "orthogonal_to_active": 0.04834410548210144, "output_influence": 0.5419423580169678, "residual_influence": 0.21345609426498413, "state_active": 0.12617608904838562, "state_memory": 0.036721400916576385, "state_peak": 98.7492446899414, "growth_active": 0.8982714414596558, "growth_memory": 0.9597798585891724, "heads": [{"head": 0, "memory_to_active": 0.005315668415278196, "contribution_to_active": 0.6248157024383545, "mix_mean": 0.6317243576049805, "mix_low": 0.0007801055908203125, "mix_high": 0.004509925842285156, "read_cosine": 0.45747488737106323, "beta_mean": 1.835393049987033e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0035550540778785944, "mix_std": 0.25980421900749207, "beta_std": 4.146150968153961e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.0031821634620428085, "evidence_weights": [-0.18639340996742249, -0.26129430532455444, 0.7127842903137207], "agreement_weight": 0.25243884325027466, "orthogonal_weight": -1.011368751525879}, {"head": 1, "memory_to_active": 0.007112941239029169, "contribution_to_active": 0.6237893104553223, "mix_mean": 0.5879089832305908, "mix_low": 0.0012722015380859375, "mix_high": 0.00472259521484375, "read_cosine": 0.23710566759109497, "beta_mean": 2.7682412110152654e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.004016024526208639, "mix_std": 0.2707502245903015, "beta_std": 6.23899104539305e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.004888057243078947, "evidence_weights": [-0.17520467936992645, -0.25845685601234436, 0.49501773715019226], "agreement_weight": 0.6183592677116394, "orthogonal_weight": -1.2661316394805908}, {"head": 2, "memory_to_active": 0.05972392112016678, "contribution_to_active": 0.515792965888977, "mix_mean": 0.40825921297073364, "mix_low": 0.0009489059448242188, "mix_high": 0.0001678466796875, "read_cosine": 0.647323489189148, "beta_mean": 0.0061294944025576115, "nonfinite": 0.0, "weighted_memory_to_active": 0.029505206272006035, "mix_std": 0.22898809611797333, "beta_std": 0.022981982678174973, "beta_low": 0.8834228515625, "beta_high": 0.0, "orthogonal_to_active": 0.015618490055203438, "evidence_weights": [-0.1716202199459076, -0.21465598046779633, 0.4106012284755707], "agreement_weight": 0.005257309414446354, "orthogonal_weight": -0.853558361530304}, {"head": 3, "memory_to_active": 0.16959887742996216, "contribution_to_active": 0.5383725762367249, "mix_mean": 0.6722206473350525, "mix_low": 0.026421546936035156, "mix_high": 0.03283214569091797, "read_cosine": -0.058171164244413376, "beta_mean": 0.0008533411892130971, "nonfinite": 0.0, "weighted_memory_to_active": 0.12316909432411194, "mix_std": 0.28701090812683105, "beta_std": 0.010529407300055027, "beta_low": 0.99578857421875, "beta_high": 0.0, "orthogonal_to_active": 0.11330316960811615, "evidence_weights": [-0.3159446120262146, -0.2380707859992981, 1.4245339632034302], "agreement_weight": 6.937392711639404, "orthogonal_weight": -0.9826270937919617}, {"head": 4, "memory_to_active": 0.3777104318141937, "contribution_to_active": 0.25167426466941833, "mix_mean": 0.3522677719593048, "mix_low": 0.004155158996582031, "mix_high": 0.0, "read_cosine": 0.8985648155212402, "beta_mean": 0.07201392948627472, "nonfinite": 0.0, "weighted_memory_to_active": 0.09312979131937027, "mix_std": 0.21439263224601746, "beta_std": 0.05312168598175049, "beta_low": 0.00128173828125, "beta_high": 0.0, "orthogonal_to_active": 0.13452614843845367, "evidence_weights": [-0.051945485174655914, -0.21779009699821472, 0.4029071033000946], "agreement_weight": 0.0070669460110366344, "orthogonal_weight": 2.3439266681671143}, {"head": 5, "memory_to_active": 0.03300957381725311, "contribution_to_active": 0.7320565581321716, "mix_mean": 0.6673895120620728, "mix_low": 0.0018978118896484375, "mix_high": 0.012578964233398438, "read_cosine": 0.3409353494644165, "beta_mean": 0.00012682797387242317, "nonfinite": 0.0, "weighted_memory_to_active": 0.02196386083960533, "mix_std": 0.26647067070007324, "beta_std": 0.001661890302784741, "beta_low": 0.9996337890625, "beta_high": 0.0, "orthogonal_to_active": 0.01854660175740719, "evidence_weights": [-0.2166299670934677, -0.27979883551597595, 0.8782114386558533], "agreement_weight": 0.22948448359966278, "orthogonal_weight": -0.9291233420372009}]}, {"layer": 14, "memory_to_active": 0.21309302747249603, "contribution_to_active": 0.4146574139595032, "mix_mean": 0.4302665591239929, "mix_low": 0.028265317901968956, "mix_high": 0.006270250305533409, "read_cosine": 0.6128231287002563, "beta_mean": 0.05941727012395859, "nonfinite": 0.0, "weighted_memory_to_active": 0.08650107681751251, "mix_std": 0.23542027175426483, "beta_std": 0.057389263063669205, "beta_low": 0.5080668330192566, "beta_high": 0.0, "orthogonal_to_active": 0.04091867059469223, "output_influence": 0.34928062558174133, "residual_influence": 0.16645923256874084, "state_active": 0.04906604811549187, "state_memory": 0.012319008819758892, "state_peak": 55.86249542236328, "growth_active": 1.0949221849441528, "growth_memory": 1.228307843208313, "heads": [{"head": 0, "memory_to_active": 0.34379541873931885, "contribution_to_active": 0.19592925906181335, "mix_mean": 0.1962401419878006, "mix_low": 0.017333984375, "mix_high": 1.9073486328125e-06, "read_cosine": 0.8155729174613953, "beta_mean": 0.066094771027565, "nonfinite": 0.0, "weighted_memory_to_active": 0.0737089067697525, "mix_std": 0.17647024989128113, "beta_std": 0.05978729575872421, "beta_low": 0.01165771484375, "beta_high": 0.0, "orthogonal_to_active": 0.0887889415025711, "evidence_weights": [-0.09075948596000671, -0.044560763984918594, 0.3499706983566284], "agreement_weight": 0.018446974456310272, "orthogonal_weight": 1.9813731908798218}, {"head": 1, "memory_to_active": 0.4504406452178955, "contribution_to_active": 0.27825698256492615, "mix_mean": 0.19016095995903015, "mix_low": 0.14139938354492188, "mix_high": 9.5367431640625e-07, "read_cosine": 0.8862746953964233, "beta_mean": 0.24660181999206543, "nonfinite": 0.0, "weighted_memory_to_active": 0.19217240810394287, "mix_std": 0.22120246291160583, "beta_std": 0.2427508682012558, "beta_low": 0.0887451171875, "beta_high": 0.0, "orthogonal_to_active": 0.020881306380033493, "evidence_weights": [0.01892828941345215, -0.1847573220729828, -0.1046542301774025], "agreement_weight": 0.05632314831018448, "orthogonal_weight": 0.364146888256073}, {"head": 2, "memory_to_active": 0.12011410295963287, "contribution_to_active": 0.5892377495765686, "mix_mean": 0.5913418531417847, "mix_low": 0.0023450851440429688, "mix_high": 0.005307197570800781, "read_cosine": 0.40512049198150635, "beta_mean": 0.0015740245580673218, "nonfinite": 0.0, "weighted_memory_to_active": 0.07739738374948502, "mix_std": 0.26153817772865295, "beta_std": 0.0021633110009133816, "beta_low": 0.98907470703125, "beta_high": 0.0, "orthogonal_to_active": 0.0590008869767189, "evidence_weights": [-0.12279745936393738, -0.3129822909832001, 0.8949101567268372], "agreement_weight": 0.08500713855028152, "orthogonal_weight": -0.861509382724762}, {"head": 3, "memory_to_active": 0.03596293181180954, "contribution_to_active": 0.5901101231575012, "mix_mean": 0.6835771203041077, "mix_low": 0.0034084320068359375, "mix_high": 0.023201942443847656, "read_cosine": -0.07163682579994202, "beta_mean": 0.004974620416760445, "nonfinite": 0.0, "weighted_memory_to_active": 0.023374253883957863, "mix_std": 0.25233617424964905, "beta_std": 0.007888062857091427, "beta_low": 0.86376953125, "beta_high": 0.0, "orthogonal_to_active": 0.018216049298644066, "evidence_weights": [-0.33675724267959595, -0.3107340931892395, 0.3127900958061218], "agreement_weight": 0.06961391866207123, "orthogonal_weight": -0.8456810116767883}, {"head": 4, "memory_to_active": 0.1549806445837021, "contribution_to_active": 0.4915195107460022, "mix_mean": 0.537926435470581, "mix_low": 0.0025644302368164062, "mix_high": 0.008090019226074219, "read_cosine": 0.9045324325561523, "beta_mean": 0.03645854443311691, "nonfinite": 0.0, "weighted_memory_to_active": 0.08993616700172424, "mix_std": 0.2552174925804138, "beta_std": 0.026685690507292747, "beta_low": 0.10723876953125, "beta_high": 0.0, "orthogonal_to_active": 0.025240030139684677, "evidence_weights": [-0.1975213587284088, -0.2323259562253952, 0.45544567704200745], "agreement_weight": 0.13345710933208466, "orthogonal_weight": -0.63704514503479}, {"head": 5, "memory_to_active": 0.17326436936855316, "contribution_to_active": 0.3428907096385956, "mix_mean": 0.38235288858413696, "mix_low": 0.00254058837890625, "mix_high": 0.0010194778442382812, "read_cosine": 0.7370747923851013, "beta_mean": 0.0007998296059668064, "nonfinite": 0.0, "weighted_memory_to_active": 0.06241729110479355, "mix_std": 0.24575701355934143, "beta_std": 0.005060337949544191, "beta_low": 0.9879150390625, "beta_high": 0.0, "orthogonal_to_active": 0.03338480740785599, "evidence_weights": [-0.17004184424877167, -0.18474403023719788, 0.3812662661075592], "agreement_weight": 0.05684323608875275, "orthogonal_weight": -0.757031261920929}]}, {"layer": 15, "memory_to_active": 0.045934323221445084, "contribution_to_active": 0.5131459832191467, "mix_mean": 0.44700121879577637, "mix_low": 0.005301475524902344, "mix_high": 0.0031285288278013468, "read_cosine": 0.6577003002166748, "beta_mean": 0.03849181532859802, "nonfinite": 0.0, "weighted_memory_to_active": 0.024390362203121185, "mix_std": 0.2811565101146698, "beta_std": 0.04283590614795685, "beta_low": 0.7020366191864014, "beta_high": 0.0, "orthogonal_to_active": 0.010355140082538128, "output_influence": 0.3327387869358063, "residual_influence": 0.21527479588985443, "state_active": 0.17433962225914001, "state_memory": 0.009507106617093086, "state_peak": 25.01085662841797, "growth_active": 1.192699909210205, "growth_memory": 1.2435911893844604, "heads": [{"head": 0, "memory_to_active": 0.11271308362483978, "contribution_to_active": 0.4865127205848694, "mix_mean": 0.41200608015060425, "mix_low": 0.005359649658203125, "mix_high": 0.0066547393798828125, "read_cosine": 0.47225242853164673, "beta_mean": 0.05455857142806053, "nonfinite": 0.0, "weighted_memory_to_active": 0.06063966825604439, "mix_std": 0.2856648564338684, "beta_std": 0.03772306814789772, "beta_low": 0.04345703125, "beta_high": 0.0, "orthogonal_to_active": 0.04083796590566635, "evidence_weights": [-0.17663468420505524, -0.19305475056171417, 0.5626001954078674], "agreement_weight": 0.08859021961688995, "orthogonal_weight": -0.8627030849456787}, {"head": 1, "memory_to_active": 0.07155384868383408, "contribution_to_active": 0.4402855336666107, "mix_mean": 0.4569060206413269, "mix_low": 0.0004911422729492188, "mix_high": 0.007970809936523438, "read_cosine": 0.9303094744682312, "beta_mean": 0.00011775612074416131, "nonfinite": 0.0, "weighted_memory_to_active": 0.030415397137403488, "mix_std": 0.2791994512081146, "beta_std": 0.0011890612076967955, "beta_low": 0.99945068359375, "beta_high": 0.0, "orthogonal_to_active": 0.002914943266659975, "evidence_weights": [-0.16483844816684723, -0.17971171438694, 0.15245629847049713], "agreement_weight": 0.09491179138422012, "orthogonal_weight": 0.2042374163866043}, {"head": 2, "memory_to_active": 0.006460440345108509, "contribution_to_active": 0.5377333164215088, "mix_mean": 0.36959585547447205, "mix_low": 0.008274078369140625, "mix_high": 6.67572021484375e-05, "read_cosine": 0.7919231653213501, "beta_mean": 7.641985575901344e-05, "nonfinite": 0.0, "weighted_memory_to_active": 0.0036307922564446926, "mix_std": 0.27395308017730713, "beta_std": 0.00021497136913239956, "beta_low": 0.9998779296875, "beta_high": 0.0, "orthogonal_to_active": 0.0012232378358021379, "evidence_weights": [-0.19380508363246918, -0.2679530382156372, 0.12829820811748505], "agreement_weight": 0.00861046276986599, "orthogonal_weight": 1.0037490129470825}, {"head": 3, "memory_to_active": 0.001435799291357398, "contribution_to_active": 0.38224685192108154, "mix_mean": 0.4248886704444885, "mix_low": 0.003345489501953125, "mix_high": 0.00055694580078125, "read_cosine": 0.6269109845161438, "beta_mean": 7.308363819902297e-06, "nonfinite": 0.0, "weighted_memory_to_active": 0.0004921596264466643, "mix_std": 0.2878856360912323, "beta_std": 1.691028410277795e-05, "beta_low": 1.0, "beta_high": 0.0, "orthogonal_to_active": 0.00023262934701051563, "evidence_weights": [-0.2729249894618988, -0.23935341835021973, 0.17211994528770447], "agreement_weight": 0.06306026875972748, "orthogonal_weight": -0.47758179903030396}, {"head": 4, "memory_to_active": 0.052871257066726685, "contribution_to_active": 0.6615841388702393, "mix_mean": 0.41216257214546204, "mix_low": 0.005809783935546875, "mix_high": 0.0007419586181640625, "read_cosine": 0.7347708940505981, "beta_mean": 0.1734192967414856, "nonfinite": 0.0, "weighted_memory_to_active": 0.03005855157971382, "mix_std": 0.30764883756637573, "beta_std": 0.19882461428642273, "beta_low": 0.20220947265625, "beta_high": 0.0, "orthogonal_to_active": 0.0053621744737029076, "evidence_weights": [-0.2040935754776001, -0.2515939176082611, -0.027613939717411995], "agreement_weight": 0.010787377133965492, "orthogonal_weight": 0.4120810329914093}, {"head": 5, "memory_to_active": 0.030571505427360535, "contribution_to_active": 0.570513129234314, "mix_mean": 0.6064481139183044, "mix_low": 0.008528709411621094, "mix_high": 0.0027799606323242188, "read_cosine": 0.3900347352027893, "beta_mean": 0.0027715214528143406, "nonfinite": 0.0, "weighted_memory_to_active": 0.021105609834194183, "mix_std": 0.2525871694087982, "beta_std": 0.01904681883752346, "beta_low": 0.96722412109375, "beta_high": 0.0, "orthogonal_to_active": 0.01155989058315754, "evidence_weights": [-0.27595391869544983, -0.3080306053161621, 0.5296410322189331], "agreement_weight": 0.31507280468940735, "orthogonal_weight": 0.8166937232017517}]}]}]}