{ "schema_version": 1, "title": "Five alphaBell model experiments", "generated_at": "2026-10-23T11:17:02.577204+00:00", "methodology": { "summary": "Real measured model runs, retained in full for this fixed case set. The public interface replays these results.", "limitations": [ "Each condition has one greedy generation. Small demonstrations do not establish general capabilities or population preferences.", "Qwen captures use the alphaBell fine-tuned derivative, not the untouched Qwen parent.", "Native experiment source and build recipe accompany the measurements." ], "reproduce": [ "Download the prompts and results.", "Use the pinned engine build and experiment runner in experiments/reproduce.", "Match checkpoint SHA-256 values and recorded settings before comparing outputs." ] }, "provenance": { "engine": "llama.cpp 74686f084c53506a7afa471d857db02e5c912543", "device": "CPU", "threads": 8, "nice": 10, "context": 2048, "flash_attention": false, "kv_cache": "default F16 attention KV; model-native recurrent state", "sampling": { "temperature": 0, "seed": 42, "max_tokens": 64, "reasoning": false }, "intervention": { "tensor": "ffn_out-12", "layer_indexing": "zero-based", "applies_to": "all prompt and generated positions on each forward pass", "operation": "scale computed FFN output before residual addition", "scales": [ 1, 0.5, 0 ], "weights_modified": false }, "memory_metric": "ru_maxrss process high-water mark, includes model loading and warm-up; KiB on Linux", "latency_metric": "native monotonic request elapsed plus client wall clock; first-token latency includes prompt prefill", "created_at": "2026-10-23T10:57:11.012459+00:00", "source_sha256": "4a29fca44df4174dc7db51a70d7bdc0d0f27ba06d18ed4338c9da8db81e4bda9", "binary_sha256": "4b733ebd42d2b88ecfdaf527915327e17734fad04bc7f35846fc5c9ea8f6e55d", "checkpoints": [ { "model": "flint", "quantization": "Q8_0", "file_bytes": 42350359392, "architecture_sha256": "1dcc4eb75752f7847b49a6cab17e377c0282050b81bb0603d33a5271112bd058", "sha256": "f97ff2fc77564a05919357685ba8e1892bec279cf733fdb6e3ef0ea8c8e45543", "gguf_metadata": { "general.architecture": "flint", "general.type": "model", "general.sampling.top_k": 20, "general.sampling.top_p": 0.949999988079071, "general.sampling.temp": 1.0, "general.name": "Flint Bf16", "general.size_label": "256x2.1B", "general.quantization_version": 2, "general.file_type": 7 }, "tensor_count": 779 }, { "model": "flint", "quantization": "Q4_1", "file_bytes": 24955171680, "architecture_sha256": "1dcc4eb75752f7847b49a6cab17e377c0282050b81bb0603d33a5271112bd058", "sha256": "1d8ea8d2fd133baa54a7412d56d39d20499a9efc080b45486f23853e3f072143", "gguf_metadata": { "general.architecture": "flint", "general.type": "model", "general.sampling.top_k": 20, "general.sampling.top_p": 0.949999988079071, "general.sampling.temp": 1.0, "general.name": "Flint Bf16", "general.size_label": "256x2.1B", "general.quantization_version": 2, "general.file_type": 3 }, "tensor_count": 779 }, { "model": "qwen", "quantization": "Q8_0", "file_bytes": 28595763712, "architecture_sha256": "2b58cd217c64118083a3d622cdf9c9b7488adb7bb12bc114762322062bb365c3", "sha256": "b2a3172342ff220a8f9a06fc2fd252d1e84615ce8dfc00b2a66941610aa8f43c", "gguf_metadata": { "general.architecture": "qwen35", "general.type": "model", "general.sampling.top_k": 20, "general.sampling.top_p": 0.949999988079071, "general.sampling.temp": 1.0, "general.name": "Qwen3.8 27B Open", "general.finetune": "open", "general.basename": "Qwen3.8", "general.size_label": "27B", "general.quantization_version": 2, "general.file_type": 7 }, "tensor_count": 851 }, { "model": "qwen", "quantization": "Q4_1", "file_bytes": 16829921792, "architecture_sha256": "2b58cd217c64118083a3d622cdf9c9b7488adb7bb12bc114762322062bb365c3", "sha256": "7c0814054c69120ff0e5b3ac3f625f98a9f86185fc39f5d71eae02cb54eb5825", "gguf_metadata": { "general.architecture": "qwen35", "general.type": "model", "general.sampling.top_k": 20, "general.sampling.top_p": 0.949999988079071, "general.sampling.temp": 1.0, "general.name": "Qwen3.8 27B Open", "general.finetune": "open", "general.basename": "Qwen3.8", "general.size_label": "27B", "general.quantization_version": 2, "general.file_type": 3 }, "tensor_count": 851 } ], "completed_at": "2026-10-23T11:00:04.436046+00:00", "source_tree": "83e9d4c276f84753cce063502a2a9e2325f15ce3", "public_base_commit": "0a1ef309cdbb9f2024f58a38fb4c42352c5db569", "source_patch_sha256": "87a1aca766c7584c433093e82e6fbfe21caa4578e8f1c1faca5b8e2385ee664d", "header_privacy_audit": { "passed": true, "metadata_entries_scanned_per_file": 41, "excluded_keys": [ "tokenizer.ggml.tokens" ], "source_urls_found": 0, "identity_matches": 0 }, "flint_capture_compatibility": { "tensor_payload_equal": true, "compared_bytes": 42344889472, "original_offset": 5469920, "matched_offset": 5469920, "original_container_sha256": "147f02266f809e7bb4b28fdeef9606520d9693b601f8bc0d817caf11a2ed1e5b", "matched_container_sha256": "f97ff2fc77564a05919357685ba8e1892bec279cf733fdb6e3ef0ea8c8e45543", "tokenizer_equal": true, "only_metadata_differences": [ { "key": "general.name", "original": "Recursive Flint", "matched": "Flint Bf16" }, { "key": "general.size_label", "original": "40B", "matched": "256x2.1B" } ] }, "hardware": "AMD EPYC 7B13 CPU. Native quantization/intervention runner: 8 threads. Full topology captures: 16 threads. CPU-only.", "model_sources": [ { "name": "Recursive Flint, source BF16", "url": "https://huggingface.co/RecursiveSI/flint", "revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" }, { "name": "alphaBell Qwen Open, related public FP8 sibling", "url": "https://huggingface.co/alphabell/alphaBell-Qwen3.8-27B-Open-FP8", "revision": "38595c77e73f8d04afa003510dadd35c750590ef" }, { "name": "Qwen parent architecture configuration", "url": "https://huggingface.co/Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" } ], "qwen_checkpoint_origin": "alphaBell fine-tuned Qwen3.8-27B, converted from the lab BF16 checkpoint to GGUF; the public FP8 release is a sibling format.", "topology_capture_checkpoints": { "flint": { "quantization": "Q8_0", "container_sha256": "147f02266f809e7bb4b28fdeef9606520d9693b601f8bc0d817caf11a2ed1e5b", "matched_tensor_payload_identical": true, "proof": "https://huggingface.co/spaces/alphabell/Model-Observatory/resolve/main/experiments/reproduce/flint-capture-equivalence.json" }, "qwen": { "quantization": "Q8_0", "container_sha256": "b2a3172342ff220a8f9a06fc2fd252d1e84615ce8dfc00b2a66941610aa8f43c" } } }, "experiments": { "reroute": { "title": "One word changed.", "summary": "Three prompt pairs, two architectures, twelve complete captures. Inspect the first output prediction before continuations diverge.", "method": "Change one word with temperature 0, seed 42 and reasoning disabled. Compare the same model and precision. The displayed Flint block is the lowest block with a changed first-frame expert set.", "limitations": [ "Later output positions are ordinal, not semantic alignments. Routing changes do not localize concepts or establish a causal explanation of an answer." ], "cases": [ { "id": "json-color-flint", "title": "One word changes the JSON value · Flint", "prompts": [ { "id": "a", "label": "Prompt A", "text": "Return only JSON with one key named color and value blue." }, { "id": "b", "label": "Prompt B", "text": "Return only JSON with one key named color and value red." } ], "conditions": [ { "id": "a", "label": "Prompt A · Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Return only JSON with one key named color and value blue.", "output": "{\"color\":\"blue\"}", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=reroute-json-color-flint-a&token=1", "metrics": { "generated_tokens": 5, "prompt_tokens": 28, "elapsed_seconds": 1.124171917, "termination": "eos", "displayed_block": 3 }, "routing": { "layer": 3, "expert_count": 256, "description": "Measured selected experts in zero-based block 3.", "tokens": [ { "token": "{\"", "experts": [ 117, 87, 35, 5, 168, 84 ], "weights": [ 0.32084164023399353, 0.2539198398590088, 0.14681749045848846, 0.12157176434993744, 0.10437823086977005, 0.052470993250608444 ] }, { "token": "color", "experts": [ 39, 59, 143, 13, 34, 250 ], "weights": [ 0.33017292618751526, 0.30293041467666626, 0.1303112655878067, 0.10549050569534302, 0.09110484272241592, 0.03999003767967224 ] }, { "token": "\":\"", "experts": [ 115, 224, 186, 216, 15, 29 ], "weights": [ 0.6117783784866333, 0.09889795631170273, 0.09107732027769089, 0.07742965966463089, 0.06268590688705444, 0.05813080072402954 ] }, { "token": "blue", "experts": [ 39, 111, 197, 137, 119, 28 ], "weights": [ 0.4875814616680145, 0.17773117125034332, 0.10596126317977905, 0.09250641614198685, 0.08542586863040924, 0.05079379305243492 ] }, { "token": "\"}", "experts": [ 216, 1, 152, 102, 182, 15 ], "weights": [ 0.38251131772994995, 0.3607138693332672, 0.07649347931146622, 0.06965707242488861, 0.06540820747613907, 0.04521603509783745 ] } ] } }, { "id": "b", "label": "Prompt B · Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Return only JSON with one key named color and value red.", "output": "{\"color\":\"red\"}", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=reroute-json-color-flint-b&token=1", "metrics": { "generated_tokens": 5, "prompt_tokens": 28, "elapsed_seconds": 1.116444861, "termination": "eos", "displayed_block": 3 }, "routing": { "layer": 3, "expert_count": 256, "description": "Measured selected experts in zero-based block 3.", "tokens": [ { "token": "{\"", "experts": [ 117, 87, 35, 5, 168, 33 ], "weights": [ 0.3200564980506897, 0.25333839654922485, 0.1516059786081314, 0.11339818686246872, 0.10805189609527588, 0.05354905501008034 ] }, { "token": "color", "experts": [ 39, 59, 143, 13, 34, 250 ], "weights": [ 0.3271667957305908, 0.29962509870529175, 0.1341659426689148, 0.1078995019197464, 0.0922822579741478, 0.03886038064956665 ] }, { "token": "\":\"", "experts": [ 115, 224, 186, 216, 15, 29 ], "weights": [ 0.6083868145942688, 0.101140096783638, 0.0891016498208046, 0.07671405375003815, 0.06320527195930481, 0.06145205348730087 ] }, { "token": "red", "experts": [ 39, 111, 197, 137, 119, 28 ], "weights": [ 0.48268941044807434, 0.1775527149438858, 0.10889095813035965, 0.09247402846813202, 0.08805633336305618, 0.05033653602004051 ] }, { "token": "\"}", "experts": [ 1, 6, 199, 216, 48, 187 ], "weights": [ 0.3797229826450348, 0.31145796179771423, 0.13263653218746185, 0.08601834625005722, 0.04618379473686218, 0.043980374932289124 ] } ] } } ], "comparison": { "comparison_frame": "First output prediction, before generated continuations diverge.", "prompt_words_changed": 1, "prompt_token_counts": [ 28, 28 ], "routed_blocks_compared": 47, "blocks_with_different_expert_sets": 18, "expert_selections_in_each_first_frame": 282, "shared_expert_selections": 263, "displayed_zero_based_block": 3, "display_rule": "Lowest routed block with a different first-frame expert set; full captures contain every block.", "first_frame_head_rms_relative_l2": 0.10240989178622689 } }, { "id": "json-color-qwen", "title": "One word changes the JSON value · Qwen Open", "prompts": [ { "id": "a", "label": "Prompt A", "text": "Return only JSON with one key named color and value blue." }, { "id": "b", "label": "Prompt B", "text": "Return only JSON with one key named color and value red." } ], "conditions": [ { "id": "a", "label": "Prompt A · alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Return only JSON with one key named color and value blue.", "output": "{\"color\":\"blue\"}", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=reroute-json-color-qwen-a&token=1", "metrics": { "generated_tokens": 5, "prompt_tokens": 42, "elapsed_seconds": 5.535615226, "termination": "eos", "displayed_block": 0 } }, { "id": "b", "label": "Prompt B · alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Return only JSON with one key named color and value red.", "output": "{\"color\":\"red\"}", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=reroute-json-color-qwen-b&token=1", "metrics": { "generated_tokens": 5, "prompt_tokens": 42, "elapsed_seconds": 5.440314309, "termination": "eos", "displayed_block": 0 } } ], "comparison": { "comparison_frame": "First output prediction, before generated continuations diverge.", "prompt_words_changed": 1, "prompt_token_counts": [ 42, 42 ], "first_frame_head_rms_relative_l2": 0.02018521900284211 } }, { "id": "opposite-flint", "title": "One word changes the opposite · Flint", "prompts": [ { "id": "a", "label": "Prompt A", "text": "Answer with one word: name the opposite of hot." }, { "id": "b", "label": "Prompt B", "text": "Answer with one word: name the opposite of cold." } ], "conditions": [ { "id": "a", "label": "Prompt A · Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Answer with one word: name the opposite of hot.", "output": "Cold", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=reroute-opposite-flint-a&token=1", "metrics": { "generated_tokens": 1, "prompt_tokens": 27, "elapsed_seconds": 0.695859422, "termination": "eos", "displayed_block": 2 }, "routing": { "layer": 2, "expert_count": 256, "description": "Measured selected experts in zero-based block 2.", "tokens": [ { "token": "Cold", "experts": [ 88, 213, 186, 46, 162, 237 ], "weights": [ 0.5531415939331055, 0.178231343626976, 0.09052453190088272, 0.08529653400182724, 0.04749872162938118, 0.045307233929634094 ] } ] } }, { "id": "b", "label": "Prompt B · Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Answer with one word: name the opposite of cold.", "output": "Hot", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=reroute-opposite-flint-b&token=1", "metrics": { "generated_tokens": 1, "prompt_tokens": 27, "elapsed_seconds": 0.644852095, "termination": "eos", "displayed_block": 2 }, "routing": { "layer": 2, "expert_count": 256, "description": "Measured selected experts in zero-based block 2.", "tokens": [ { "token": "Hot", "experts": [ 88, 213, 186, 46, 162, 1 ], "weights": [ 0.5620047450065613, 0.17373046278953552, 0.09409793466329575, 0.07710109651088715, 0.047040797770023346, 0.046024974435567856 ] } ] } } ], "comparison": { "comparison_frame": "First output prediction, before generated continuations diverge.", "prompt_words_changed": 1, "prompt_token_counts": [ 27, 27 ], "routed_blocks_compared": 47, "blocks_with_different_expert_sets": 25, "expert_selections_in_each_first_frame": 282, "shared_expert_selections": 243, "displayed_zero_based_block": 2, "display_rule": "Lowest routed block with a different first-frame expert set; full captures contain every block.", "first_frame_head_rms_relative_l2": 0.25766445665876486 } }, { "id": "opposite-qwen", "title": "One word changes the opposite · Qwen Open", "prompts": [ { "id": "a", "label": "Prompt A", "text": "Answer with one word: name the opposite of hot." }, { "id": "b", "label": "Prompt B", "text": "Answer with one word: name the opposite of cold." } ], "conditions": [ { "id": "a", "label": "Prompt A · alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Answer with one word: name the opposite of hot.", "output": "Cold", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=reroute-opposite-qwen-a&token=1", "metrics": { "generated_tokens": 1, "prompt_tokens": 41, "elapsed_seconds": 3.133467638, "termination": "eos", "displayed_block": 0 } }, { "id": "b", "label": "Prompt B · alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Answer with one word: name the opposite of cold.", "output": "Hot", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=reroute-opposite-qwen-b&token=1", "metrics": { "generated_tokens": 1, "prompt_tokens": 41, "elapsed_seconds": 3.118574647, "termination": "eos", "displayed_block": 0 } } ], "comparison": { "comparison_frame": "First output prediction, before generated continuations diverge.", "prompt_words_changed": 1, "prompt_token_counts": [ 41, 41 ], "first_frame_head_rms_relative_l2": 0.18418391836536593 } }, { "id": "python-flint", "title": "One word changes the function · Flint", "prompts": [ { "id": "a", "label": "Prompt A", "text": "Write only a Python expression that selects the maximum of x and y." }, { "id": "b", "label": "Prompt B", "text": "Write only a Python expression that selects the minimum of x and y." } ], "conditions": [ { "id": "a", "label": "Prompt A · Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Write only a Python expression that selects the maximum of x and y.", "output": "max(x, y)", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=reroute-python-flint-a&token=1", "metrics": { "generated_tokens": 5, "prompt_tokens": 30, "elapsed_seconds": 1.156945927, "termination": "eos", "displayed_block": 3 }, "routing": { "layer": 3, "expert_count": 256, "description": "Measured selected experts in zero-based block 3.", "tokens": [ { "token": "max", "experts": [ 117, 87, 35, 5, 168, 57 ], "weights": [ 0.4010581374168396, 0.21458370983600616, 0.1538792848587036, 0.10982739180326462, 0.07339246571063995, 0.047259047627449036 ] }, { "token": "(x", "experts": [ 219, 149, 69, 243, 106, 168 ], "weights": [ 0.356155127286911, 0.28351446986198425, 0.15893621742725372, 0.0888138860464096, 0.08155017346143723, 0.031030092388391495 ] }, { "token": ",", "experts": [ 124, 25, 177, 158, 255, 224 ], "weights": [ 0.5517688393592834, 0.12534935772418976, 0.1014571487903595, 0.09167846292257309, 0.07228443771600723, 0.057461753487586975 ] }, { "token": " y", "experts": [ 19, 77, 14, 203, 96, 121 ], "weights": [ 0.4720388352870941, 0.34413912892341614, 0.0540812686085701, 0.04676710069179535, 0.04236053302884102, 0.040613144636154175 ] }, { "token": ")", "experts": [ 124, 25, 21, 66, 235, 28 ], "weights": [ 0.4778953790664673, 0.24744132161140442, 0.10221158713102341, 0.06340343505144119, 0.05989007651805878, 0.04915820062160492 ] } ] } }, { "id": "b", "label": "Prompt B · Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Write only a Python expression that selects the minimum of x and y.", "output": "min(x, y)", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=reroute-python-flint-b&token=1", "metrics": { "generated_tokens": 5, "prompt_tokens": 30, "elapsed_seconds": 1.128900005, "termination": "eos", "displayed_block": 3 }, "routing": { "layer": 3, "expert_count": 256, "description": "Measured selected experts in zero-based block 3.", "tokens": [ { "token": "min", "experts": [ 117, 87, 35, 5, 168, 7 ], "weights": [ 0.4022853970527649, 0.23592405021190643, 0.1466451734304428, 0.09799264371395111, 0.06999532133340836, 0.04715738818049431 ] }, { "token": "(x", "experts": [ 149, 219, 189, 69, 243, 106 ], "weights": [ 0.4026070237159729, 0.15572281181812286, 0.1464761197566986, 0.1269456148147583, 0.09798307716846466, 0.07026540488004684 ] }, { "token": ",", "experts": [ 124, 25, 177, 255, 158, 74 ], "weights": [ 0.5216768383979797, 0.13582339882850647, 0.10417815297842026, 0.09238538146018982, 0.07687641680240631, 0.0690598264336586 ] }, { "token": " y", "experts": [ 19, 77, 14, 203, 81, 121 ], "weights": [ 0.46713194251060486, 0.34391525387763977, 0.056301526725292206, 0.04728133976459503, 0.04271211847662926, 0.042657796293497086 ] }, { "token": ")", "experts": [ 124, 25, 21, 66, 235, 28 ], "weights": [ 0.4766605496406555, 0.2452375292778015, 0.11082471162080765, 0.062690868973732, 0.05392761155962944, 0.050658706575632095 ] } ] } } ], "comparison": { "comparison_frame": "First output prediction, before generated continuations diverge.", "prompt_words_changed": 1, "prompt_token_counts": [ 30, 30 ], "routed_blocks_compared": 47, "blocks_with_different_expert_sets": 20, "expert_selections_in_each_first_frame": 282, "shared_expert_selections": 255, "displayed_zero_based_block": 3, "display_rule": "Lowest routed block with a different first-frame expert set; full captures contain every block.", "first_frame_head_rms_relative_l2": 0.1035147140665932 } }, { "id": "python-qwen", "title": "One word changes the function · Qwen Open", "prompts": [ { "id": "a", "label": "Prompt A", "text": "Write only a Python expression that selects the maximum of x and y." }, { "id": "b", "label": "Prompt B", "text": "Write only a Python expression that selects the minimum of x and y." } ], "conditions": [ { "id": "a", "label": "Prompt A · alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Write only a Python expression that selects the maximum of x and y.", "output": "max(x, y)", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=reroute-python-qwen-a&token=1", "metrics": { "generated_tokens": 5, "prompt_tokens": 44, "elapsed_seconds": 5.603033013, "termination": "eos", "displayed_block": 0 } }, { "id": "b", "label": "Prompt B · alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Write only a Python expression that selects the minimum of x and y.", "output": "min(x, y)", "settings": { "max_tokens": 48, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=reroute-python-qwen-b&token=1", "metrics": { "generated_tokens": 5, "prompt_tokens": 44, "elapsed_seconds": 5.566880933, "termination": "eos", "displayed_block": 0 } } ], "comparison": { "comparison_frame": "First output prediction, before generated continuations diverge.", "prompt_words_changed": 1, "prompt_token_counts": [ 44, 44 ], "first_frame_head_rms_relative_l2": 0.1294852695128142 } } ] }, "quant": { "title": "Can you spot the 4-bit model?", "summary": "Twelve blind pairs from six fixed tasks on Flint and alphaBell Qwen Open. Eleven pairs produced identical text; one arithmetic pair differed and both answers were wrong.", "method": "Matched BF16-derived Q4_1 and Q8_0 files; identical prompts and decoding. Four separate CPU model processes, one warm-up per process. All cases retained.", "limitations": [ "Six tasks per model, one generation per condition. This is not a general quality or speed ranking. Process peak RSS is a high-water mark including loading and warm-up, not a per-request increment. Timings include tracing overhead and are single observations." ], "cases": [ { "id": "flint-arithmetic", "title": "Exact arithmetic", "prompt": "Calculate (17 * 23) - 19. Reply with the integer only.", "checks": [ "Rule: integer", "Expected: 372", "Q8_0: failed", "Q4_1: failed" ], "conditions": [ { "id": "flint-q8_0", "label": "Q8_0", "model": "Recursive Flint", "quant": "Q8_0", "output": "374", "metrics": { "generated_tokens": 1, "prompt_tokens": 33, "elapsed_seconds": 0.738173075, "request_wall_seconds": 0.7542800180381164, "time_to_first_token_seconds": 0.689367342973128, "decode_tokens_per_second": null, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.0042319691359885515, "block12_after_rms": 0.0042319691359885515 }, "check": { "passed": false, "rule": "integer", "expected": "372", "character_count": 3, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 9056, "input_position": 32, "layers": [ { "after_rms": 0.004216483101124048, "before_rms": 0.004216483101124048, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.0042319691359885515, "before_rms": 0.0042319691359885515, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.005525138486063405, "before_rms": 0.005525138486063405, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.014221511290226446, "before_rms": 0.014221511290226446, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 270336 } ] } ] }, { "id": "flint-q4_1", "label": "Q4_1", "model": "Recursive Flint", "quant": "Q4_1", "output": "352", "metrics": { "generated_tokens": 1, "prompt_tokens": 33, "elapsed_seconds": 0.781993782, "request_wall_seconds": 0.798555999994278, "time_to_first_token_seconds": 0.7473802480380982, "decode_tokens_per_second": null, "process_peak_rss_kib": 24811756, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 24955171680, "block12_before_rms": 0.00415956677122455, "block12_after_rms": 0.00415956677122455 }, "check": { "passed": false, "rule": "integer", "expected": "372", "character_count": 3, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 11510, "input_position": 32, "layers": [ { "after_rms": 0.004206818881751983, "before_rms": 0.004206818881751983, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.00415956677122455, "before_rms": 0.00415956677122455, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.005537144078691912, "before_rms": 0.005537144078691912, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.014134159625799036, "before_rms": 0.014134159625799036, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 270336 } ] } ] } ], "model": "flint" }, { "id": "flint-json", "title": "Strict JSON format", "prompt": "Return exactly this JSON object and no other text: {\"status\":\"ok\",\"count\":3}", "checks": [ "Rule: json", "Expected: {'status': 'ok', 'count': 3}", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "flint-q8_0", "label": "Q8_0", "model": "Recursive Flint", "quant": "Q8_0", "output": "{\"status\":\"ok\",\"count\":3}", "metrics": { "generated_tokens": 9, "prompt_tokens": 35, "elapsed_seconds": 1.259416502, "request_wall_seconds": 1.2757591230329126, "time_to_first_token_seconds": 0.7007529039401561, "decode_tokens_per_second": 13.911468772577386, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004073247840262152, "block12_after_rms": 0.004073247840262152 }, "check": { "passed": true, "rule": "json", "expected": { "status": "ok", "count": 3 }, "character_count": 25, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 5482, "input_position": 34, "layers": [ { "after_rms": 0.00442951382677711, "before_rms": 0.00442951382677711, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.004073247840262152, "before_rms": 0.004073247840262152, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.00524975286610561, "before_rms": 0.00524975286610561, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.014444385872118682, "before_rms": 0.014444385872118682, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 286720 } ] }, { "step": 1, "token_id": 5462, "input_position": 35, "layers": [ { "after_rms": 0.005793034803083275, "before_rms": 0.005793034803083275, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0046820736039610645, "before_rms": 0.0046820736039610645, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00649377540352318, "before_rms": 0.00649377540352318, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019726933354943656, "before_rms": 0.019726933354943656, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 4890, "input_position": 36, "layers": [ { "after_rms": 0.005294017353787343, "before_rms": 0.005294017353787343, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0034695353362188502, "before_rms": 0.0034695353362188502, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00738829527350583, "before_rms": 0.00738829527350583, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019360584231844193, "before_rms": 0.019360584231844193, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 544, "input_position": 37, "layers": [ { "after_rms": 0.00491148399515983, "before_rms": 0.00491148399515983, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00380345983046727, "before_rms": 0.00380345983046727, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005149091746140284, "before_rms": 0.005149091746140284, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016276301961648348, "before_rms": 0.016276301961648348, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1147, "input_position": 38, "layers": [ { "after_rms": 0.006222679356489009, "before_rms": 0.006222679356489009, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0036829646504058975, "before_rms": 0.0036829646504058975, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006655483694052688, "before_rms": 0.006655483694052688, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019208434114849963, "before_rms": 0.019208434114849963, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 2816, "input_position": 39, "layers": [ { "after_rms": 0.005760162518562755, "before_rms": 0.005760162518562755, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003275269655906223, "before_rms": 0.003275269655906223, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004388659397945476, "before_rms": 0.004388659397945476, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015988995629268844, "before_rms": 0.015988995629268844, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 1129, "input_position": 40, "layers": [ { "after_rms": 0.004863272662823825, "before_rms": 0.004863272662823825, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0038314286271215804, "before_rms": 0.0038314286271215804, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007286998970545343, "before_rms": 0.007286998970545343, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018843493317539196, "before_rms": 0.018843493317539196, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 18, "input_position": 41, "layers": [ { "after_rms": 0.004603991231053795, "before_rms": 0.004603991231053795, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0030208238582806433, "before_rms": 0.0030208238582806433, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004703706397060763, "before_rms": 0.004703706397060763, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.013840299536615416, "before_rms": 0.013840299536615416, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 92, "input_position": 42, "layers": [ { "after_rms": 0.0039039619301298022, "before_rms": 0.0039039619301298022, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004057874942645444, "before_rms": 0.004057874942645444, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006582297878179331, "before_rms": 0.006582297878179331, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016272639100438333, "before_rms": 0.016272639100438333, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "flint-q4_1", "label": "Q4_1", "model": "Recursive Flint", "quant": "Q4_1", "output": "{\"status\":\"ok\",\"count\":3}", "metrics": { "generated_tokens": 9, "prompt_tokens": 35, "elapsed_seconds": 1.251735163, "request_wall_seconds": 1.2701693959534168, "time_to_first_token_seconds": 0.7594751949654892, "decode_tokens_per_second": 15.663206261728522, "process_peak_rss_kib": 24811756, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 24955171680, "block12_before_rms": 0.004019226021822854, "block12_after_rms": 0.004019226021822854 }, "check": { "passed": true, "rule": "json", "expected": { "status": "ok", "count": 3 }, "character_count": 25, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 5482, "input_position": 34, "layers": [ { "after_rms": 0.004480802624651158, "before_rms": 0.004480802624651158, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.004019226021822854, "before_rms": 0.004019226021822854, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.005142506994361342, "before_rms": 0.005142506994361342, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.014475124728967394, "before_rms": 0.014475124728967394, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 286720 } ] }, { "step": 1, "token_id": 5462, "input_position": 35, "layers": [ { "after_rms": 0.005884499521933463, "before_rms": 0.005884499521933463, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004714484449236878, "before_rms": 0.004714484449236878, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006689554817134999, "before_rms": 0.006689554817134999, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020091649545950115, "before_rms": 0.020091649545950115, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 4890, "input_position": 36, "layers": [ { "after_rms": 0.005296309235185447, "before_rms": 0.005296309235185447, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0034924733951837345, "before_rms": 0.0034924733951837345, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00761576937963421, "before_rms": 0.00761576937963421, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01972855739429675, "before_rms": 0.01972855739429675, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 544, "input_position": 37, "layers": [ { "after_rms": 0.005061610553787093, "before_rms": 0.005061610553787093, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003724008220189829, "before_rms": 0.003724008220189829, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004973375331562022, "before_rms": 0.004973375331562022, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016514705424796036, "before_rms": 0.016514705424796036, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1147, "input_position": 38, "layers": [ { "after_rms": 0.006274683971296353, "before_rms": 0.006274683971296353, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0037093873591057375, "before_rms": 0.0037093873591057375, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006668038059028412, "before_rms": 0.006668038059028412, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019377753581390548, "before_rms": 0.019377753581390548, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 2816, "input_position": 39, "layers": [ { "after_rms": 0.005775051951262021, "before_rms": 0.005775051951262021, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0033710668003589326, "before_rms": 0.0033710668003589326, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004385254089389432, "before_rms": 0.004385254089389432, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015951241921845414, "before_rms": 0.015951241921845414, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 1129, "input_position": 40, "layers": [ { "after_rms": 0.00490185922327077, "before_rms": 0.00490185922327077, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003771268940693942, "before_rms": 0.003771268940693942, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007377846105883496, "before_rms": 0.007377846105883496, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019096257942254946, "before_rms": 0.019096257942254946, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 18, "input_position": 41, "layers": [ { "after_rms": 0.004675410678881592, "before_rms": 0.004675410678881592, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0029219677425947276, "before_rms": 0.0029219677425947276, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00468134851761768, "before_rms": 0.00468134851761768, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.013991205065508688, "before_rms": 0.013991205065508688, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 92, "input_position": 42, "layers": [ { "after_rms": 0.0039219998776194335, "before_rms": 0.0039219998776194335, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004008549607569253, "before_rms": 0.004008549607569253, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006525887799501659, "before_rms": 0.006525887799501659, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016431090183615604, "before_rms": 0.016431090183615604, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] } ], "model": "flint" }, { "id": "flint-fact", "title": "Short factual recall", "prompt": "What is the chemical symbol for gold? Reply with the symbol only.", "checks": [ "Rule: exact", "Expected: Au", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "flint-q8_0", "label": "Q8_0", "model": "Recursive Flint", "quant": "Q8_0", "output": "Au", "metrics": { "generated_tokens": 1, "prompt_tokens": 30, "elapsed_seconds": 0.650667801, "request_wall_seconds": 0.6673325290903449, "time_to_first_token_seconds": 0.6034285150235519, "decode_tokens_per_second": null, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004120685912594556, "block12_after_rms": 0.004120685912594556 }, "check": { "passed": true, "rule": "exact", "expected": "Au", "character_count": 2, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 22911, "input_position": 29, "layers": [ { "after_rms": 0.0042503533024456225, "before_rms": 0.0042503533024456225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.004120685912594556, "before_rms": 0.004120685912594556, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.005313107066375563, "before_rms": 0.005313107066375563, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.01490764588916619, "before_rms": 0.01490764588916619, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 245760 } ] } ] }, { "id": "flint-q4_1", "label": "Q4_1", "model": "Recursive Flint", "quant": "Q4_1", "output": "Au", "metrics": { "generated_tokens": 1, "prompt_tokens": 30, "elapsed_seconds": 0.713642926, "request_wall_seconds": 0.7302313540130854, "time_to_first_token_seconds": 0.6695935919415206, "decode_tokens_per_second": null, "process_peak_rss_kib": 24811756, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 24955171680, "block12_before_rms": 0.0040212076224896524, "block12_after_rms": 0.0040212076224896524 }, "check": { "passed": true, "rule": "exact", "expected": "Au", "character_count": 2, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 22911, "input_position": 29, "layers": [ { "after_rms": 0.004262134108703956, "before_rms": 0.004262134108703956, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.0040212076224896524, "before_rms": 0.0040212076224896524, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.005205860826013204, "before_rms": 0.005205860826013204, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.014788299051992781, "before_rms": 0.014788299051992781, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 245760 } ] } ] } ], "model": "flint" }, { "id": "flint-correction", "title": "Use the corrected fact", "prompt": "The access color was blue. Correction: the access color is now amber. What is the current access color? Reply with one word only.", "checks": [ "Rule: exact", "Expected: amber", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "flint-q8_0", "label": "Q8_0", "model": "Recursive Flint", "quant": "Q8_0", "output": "amber", "metrics": { "generated_tokens": 1, "prompt_tokens": 44, "elapsed_seconds": 0.974556193, "request_wall_seconds": 0.9955227529862896, "time_to_first_token_seconds": 0.9343198160640895, "decode_tokens_per_second": null, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.00441159816545625, "block12_after_rms": 0.00441159816545625 }, "check": { "passed": true, "rule": "exact", "expected": "amber", "character_count": 5, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 13732, "input_position": 43, "layers": [ { "after_rms": 0.004252703885422568, "before_rms": 0.004252703885422568, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.00441159816545625, "before_rms": 0.00441159816545625, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.005657736200656421, "before_rms": 0.005657736200656421, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.014876767054565872, "before_rms": 0.014876767054565872, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 360448 } ] } ] }, { "id": "flint-q4_1", "label": "Q4_1", "model": "Recursive Flint", "quant": "Q4_1", "output": "amber", "metrics": { "generated_tokens": 1, "prompt_tokens": 44, "elapsed_seconds": 1.028451992, "request_wall_seconds": 1.0458013030001894, "time_to_first_token_seconds": 0.9873899329686537, "decode_tokens_per_second": null, "process_peak_rss_kib": 24813804, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 24955171680, "block12_before_rms": 0.004339712902236177, "block12_after_rms": 0.004339712902236177 }, "check": { "passed": true, "rule": "exact", "expected": "amber", "character_count": 5, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 13732, "input_position": 43, "layers": [ { "after_rms": 0.004209592099785874, "before_rms": 0.004209592099785874, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.004339712902236177, "before_rms": 0.004339712902236177, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.005707988547356796, "before_rms": 0.005707988547356796, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.014914561282848718, "before_rms": 0.014914561282848718, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 360448 } ] } ] } ], "model": "flint" }, { "id": "flint-code", "title": "Python expression repair", "prompt": "Complete the function with one Python expression only, no markdown: def is_even(n): return ___ . The function must return True exactly when integer n is even.", "checks": [ "Rule: expression", "Expected: n % 2 == 0", "Q8_0: failed", "Q4_1: failed" ], "conditions": [ { "id": "flint-q8_0", "label": "Q8_0", "model": "Recursive Flint", "quant": "Q8_0", "output": "return n % 2 == 0", "metrics": { "generated_tokens": 8, "prompt_tokens": 48, "elapsed_seconds": 1.522581758, "request_wall_seconds": 1.5477090129861608, "time_to_first_token_seconds": 0.9909320520237088, "decode_tokens_per_second": 12.570319015535855, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004202357680490731, "block12_after_rms": 0.004202357680490731 }, "check": { "passed": false, "rule": "expression", "expected": "n % 2 == 0", "character_count": 17, "word_count": 6 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 963, "input_position": 47, "layers": [ { "after_rms": 0.004194344649342437, "before_rms": 0.004194344649342437, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.004202357680490731, "before_rms": 0.004202357680490731, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.0052131473290487635, "before_rms": 0.0052131473290487635, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.014781441026154475, "before_rms": 0.014781441026154475, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 393216 } ] }, { "step": 1, "token_id": 307, "input_position": 48, "layers": [ { "after_rms": 0.005277509304535801, "before_rms": 0.005277509304535801, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00609774510971192, "before_rms": 0.00609774510971192, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005608145566049825, "before_rms": 0.005608145566049825, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018110784621771632, "before_rms": 0.018110784621771632, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 1152, "input_position": 49, "layers": [ { "after_rms": 0.004954062696681225, "before_rms": 0.004954062696681225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004947633813264256, "before_rms": 0.004947633813264256, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006281951776179792, "before_rms": 0.006281951776179792, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01793926095231872, "before_rms": 0.01793926095231872, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 220, "input_position": 50, "layers": [ { "after_rms": 0.005725603347206012, "before_rms": 0.005725603347206012, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007613934635856348, "before_rms": 0.007613934635856348, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007067586106815484, "before_rms": 0.007067586106815484, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02065595811031347, "before_rms": 0.02065595811031347, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 17, "input_position": 51, "layers": [ { "after_rms": 0.004384558665618408, "before_rms": 0.004384558665618408, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002694924917014614, "before_rms": 0.002694924917014614, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003985706515046729, "before_rms": 0.003985706515046729, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.012855823082554165, "before_rms": 0.012855823082554165, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 823, "input_position": 52, "layers": [ { "after_rms": 0.00695481022115809, "before_rms": 0.00695481022115809, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005148284890432175, "before_rms": 0.005148284890432175, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00860817356635732, "before_rms": 0.00860817356635732, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019382560123796404, "before_rms": 0.019382560123796404, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 220, "input_position": 53, "layers": [ { "after_rms": 0.008360185779230117, "before_rms": 0.008360185779230117, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005331645591182674, "before_rms": 0.005331645591182674, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.011988298999222614, "before_rms": 0.011988298999222614, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02324778120292584, "before_rms": 0.02324778120292584, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 15, "input_position": 54, "layers": [ { "after_rms": 0.004389820760472024, "before_rms": 0.004389820760472024, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002823151948738866, "before_rms": 0.002823151948738866, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004278350453083995, "before_rms": 0.004278350453083995, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.013667174465740126, "before_rms": 0.013667174465740126, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "flint-q4_1", "label": "Q4_1", "model": "Recursive Flint", "quant": "Q4_1", "output": "return n % 2 == 0", "metrics": { "generated_tokens": 8, "prompt_tokens": 48, "elapsed_seconds": 1.428655518, "request_wall_seconds": 1.4456852129660547, "time_to_first_token_seconds": 1.0364669429836795, "decode_tokens_per_second": 17.098353812752265, "process_peak_rss_kib": 24813804, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 24955171680, "block12_before_rms": 0.004124282082456951, "block12_after_rms": 0.004124282082456951 }, "check": { "passed": false, "rule": "expression", "expected": "n % 2 == 0", "character_count": 17, "word_count": 6 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 963, "input_position": 47, "layers": [ { "after_rms": 0.004197426447709307, "before_rms": 0.004197426447709307, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.004124282082456951, "before_rms": 0.004124282082456951, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.005129178661979122, "before_rms": 0.005129178661979122, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.014727759161136935, "before_rms": 0.014727759161136935, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 393216 } ] }, { "step": 1, "token_id": 307, "input_position": 48, "layers": [ { "after_rms": 0.00529162387726611, "before_rms": 0.00529162387726611, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006311261171671491, "before_rms": 0.006311261171671491, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0055793974325739105, "before_rms": 0.0055793974325739105, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018152769626134782, "before_rms": 0.018152769626134782, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 1152, "input_position": 49, "layers": [ { "after_rms": 0.004977992102677547, "before_rms": 0.004977992102677547, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005044167186539479, "before_rms": 0.005044167186539479, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00618730909234251, "before_rms": 0.00618730909234251, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017977599413659836, "before_rms": 0.017977599413659836, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 220, "input_position": 50, "layers": [ { "after_rms": 0.005692971542768754, "before_rms": 0.005692971542768754, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007667436190392576, "before_rms": 0.007667436190392576, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00715658608387803, "before_rms": 0.00715658608387803, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02069278331850969, "before_rms": 0.02069278331850969, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 17, "input_position": 51, "layers": [ { "after_rms": 0.0043834519236579, "before_rms": 0.0043834519236579, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0027110249785654086, "before_rms": 0.0027110249785654086, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003933361905406773, "before_rms": 0.003933361905406773, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01299392506244475, "before_rms": 0.01299392506244475, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 823, "input_position": 52, "layers": [ { "after_rms": 0.007086739036977866, "before_rms": 0.007086739036977866, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005068799612035294, "before_rms": 0.005068799612035294, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008616946732804713, "before_rms": 0.008616946732804713, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019539045407508844, "before_rms": 0.019539045407508844, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 220, "input_position": 53, "layers": [ { "after_rms": 0.008330422942053637, "before_rms": 0.008330422942053637, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0052894895354913506, "before_rms": 0.0052894895354913506, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.012248586143235164, "before_rms": 0.012248586143235164, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.023511864970171558, "before_rms": 0.023511864970171558, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 15, "input_position": 54, "layers": [ { "after_rms": 0.0044348412653756105, "before_rms": 0.0044348412653756105, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0027982348810609656, "before_rms": 0.0027982348810609656, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004289321679896335, "before_rms": 0.004289321679896335, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.013781103726701278, "before_rms": 0.013781103726701278, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] } ], "model": "flint" }, { "id": "flint-brief", "title": "One sentence explanation", "prompt": "Explain photosynthesis in exactly one sentence containing the words sunlight and glucose.", "checks": [ "Rule: sentence", "Expected: ['sunlight', 'glucose']", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "flint-q8_0", "label": "Q8_0", "model": "Recursive Flint", "quant": "Q8_0", "output": "Photosynthesis is the process by which plants use sunlight to create glucose, storing solar energy in chemical form.", "metrics": { "generated_tokens": 21, "prompt_tokens": 29, "elapsed_seconds": 1.985051171, "request_wall_seconds": 2.0436750209191814, "time_to_first_token_seconds": 0.6549666819628328, "decode_tokens_per_second": 14.402335769662937, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.0044633950081139125, "block12_after_rms": 0.0044633950081139125 }, "check": { "passed": true, "rule": "sentence", "expected": [ "sunlight", "glucose" ], "character_count": 116, "word_count": 18 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 49106, "input_position": 28, "layers": [ { "after_rms": 0.004260129892907466, "before_rms": 0.004260129892907466, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.0044633950081139125, "before_rms": 0.0044633950081139125, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.005860773055104808, "before_rms": 0.005860773055104808, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.015381996005573538, "before_rms": 0.015381996005573538, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 237568 } ] }, { "step": 1, "token_id": 25424, "input_position": 29, "layers": [ { "after_rms": 0.0056475146712665295, "before_rms": 0.0056475146712665295, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008613622622386482, "before_rms": 0.008613622622386482, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006688744656356641, "before_rms": 0.006688744656356641, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.022823769881939016, "before_rms": 0.022823769881939016, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 332, "input_position": 30, "layers": [ { "after_rms": 0.0064819400327529395, "before_rms": 0.0064819400327529395, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0066762037147183205, "before_rms": 0.0066762037147183205, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008751997501954894, "before_rms": 0.008751997501954894, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.027229842448171573, "before_rms": 0.027229842448171573, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 269, "input_position": 31, "layers": [ { "after_rms": 0.0045512880200197945, "before_rms": 0.0045512880200197945, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005065420297108188, "before_rms": 0.005065420297108188, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005762789431004507, "before_rms": 0.005762789431004507, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017768401646140505, "before_rms": 0.017768401646140505, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1260, "input_position": 32, "layers": [ { "after_rms": 0.004939956954683621, "before_rms": 0.004939956954683621, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0038142763724026384, "before_rms": 0.0038142763724026384, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005913005531372122, "before_rms": 0.005913005531372122, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01761890031507447, "before_rms": 0.01761890031507447, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 491, "input_position": 33, "layers": [ { "after_rms": 0.00603825473034925, "before_rms": 0.00603825473034925, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005399106143511229, "before_rms": 0.005399106143511229, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008736535756150335, "before_rms": 0.008736535756150335, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.023288508898713978, "before_rms": 0.023288508898713978, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 687, "input_position": 34, "layers": [ { "after_rms": 0.004825344575484225, "before_rms": 0.004825344575484225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0035722557633475504, "before_rms": 0.0035722557633475504, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004528394446715205, "before_rms": 0.004528394446715205, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015558510785105506, "before_rms": 0.015558510785105506, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 7370, "input_position": 35, "layers": [ { "after_rms": 0.00599941128253067, "before_rms": 0.00599941128253067, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00554379069270193, "before_rms": 0.00554379069270193, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00600245072054164, "before_rms": 0.00600245072054164, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021459060251264164, "before_rms": 0.021459060251264164, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 896, "input_position": 36, "layers": [ { "after_rms": 0.0051950504054689125, "before_rms": 0.0051950504054689125, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006141893723174439, "before_rms": 0.006141893723174439, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009373710872705807, "before_rms": 0.009373710872705807, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024138161560965755, "before_rms": 0.024138161560965755, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 9, "token_id": 24044, "input_position": 37, "layers": [ { "after_rms": 0.008031516167890848, "before_rms": 0.008031516167890848, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005368163828714329, "before_rms": 0.005368163828714329, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005839425814527791, "before_rms": 0.005839425814527791, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019011043273201134, "before_rms": 0.019011043273201134, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 10, "token_id": 295, "input_position": 38, "layers": [ { "after_rms": 0.009314847401164192, "before_rms": 0.009314847401164192, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00741898116192189, "before_rms": 0.00741898116192189, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00969530415712232, "before_rms": 0.00969530415712232, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02606507768278446, "before_rms": 0.02606507768278446, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 11, "token_id": 1815, "input_position": 39, "layers": [ { "after_rms": 0.0051360181463417694, "before_rms": 0.0051360181463417694, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005146656362487788, "before_rms": 0.005146656362487788, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005882866866859857, "before_rms": 0.005882866866859857, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017485786719118993, "before_rms": 0.017485786719118993, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 12, "token_id": 18175, "input_position": 40, "layers": [ { "after_rms": 0.004435782376176152, "before_rms": 0.004435782376176152, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007327068248162281, "before_rms": 0.007327068248162281, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006088986661020862, "before_rms": 0.006088986661020862, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020237327919821312, "before_rms": 0.020237327919821312, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 13, "token_id": 11, "input_position": 41, "layers": [ { "after_rms": 0.010509592928354338, "before_rms": 0.010509592928354338, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0072267658458743745, "before_rms": 0.0072267658458743745, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0072706489196035584, "before_rms": 0.0072706489196035584, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.026778168430377582, "before_rms": 0.026778168430377582, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 14, "token_id": 20827, "input_position": 42, "layers": [ { "after_rms": 0.0036465031366290737, "before_rms": 0.0036465031366290737, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003456715901650377, "before_rms": 0.003456715901650377, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005546723538161339, "before_rms": 0.005546723538161339, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.014684442530549374, "before_rms": 0.014684442530549374, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 15, "token_id": 8437, "input_position": 43, "layers": [ { "after_rms": 0.00628374835469256, "before_rms": 0.00628374835469256, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007316398861469263, "before_rms": 0.007316398861469263, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008040298007985535, "before_rms": 0.008040298007985535, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021912364803827578, "before_rms": 0.021912364803827578, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 16, "token_id": 2488, "input_position": 44, "layers": [ { "after_rms": 0.008596546950231054, "before_rms": 0.008596546950231054, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005564570157863948, "before_rms": 0.005564570157863948, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007757301988850941, "before_rms": 0.007757301988850941, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020669327020220094, "before_rms": 0.020669327020220094, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 17, "token_id": 293, "input_position": 45, "layers": [ { "after_rms": 0.011432653860628939, "before_rms": 0.011432653860628939, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005583320073300356, "before_rms": 0.005583320073300356, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009875868656773605, "before_rms": 0.009875868656773605, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024941505866581348, "before_rms": 0.024941505866581348, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 18, "token_id": 6999, "input_position": 46, "layers": [ { "after_rms": 0.0053301379768005384, "before_rms": 0.0053301379768005384, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005000846356165002, "before_rms": 0.005000846356165002, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005456690270258401, "before_rms": 0.005456690270258401, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016748723982626335, "before_rms": 0.016748723982626335, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 19, "token_id": 981, "input_position": 47, "layers": [ { "after_rms": 0.007946814942115127, "before_rms": 0.007946814942115127, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005158417494632955, "before_rms": 0.005158417494632955, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007460860282340885, "before_rms": 0.007460860282340885, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020411307476897476, "before_rms": 0.020411307476897476, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 20, "token_id": 13, "input_position": 48, "layers": [ { "after_rms": 0.005514051374901345, "before_rms": 0.005514051374901345, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0070394921880856, "before_rms": 0.0070394921880856, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0066148112542871755, "before_rms": 0.0066148112542871755, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020639906847932543, "before_rms": 0.020639906847932543, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "flint-q4_1", "label": "Q4_1", "model": "Recursive Flint", "quant": "Q4_1", "output": "Photosynthesis is the process by which plants use sunlight to create glucose, storing solar energy in chemical form.", "metrics": { "generated_tokens": 21, "prompt_tokens": 29, "elapsed_seconds": 1.833752451, "request_wall_seconds": 1.854664323036559, "time_to_first_token_seconds": 0.6819874360226095, "decode_tokens_per_second": 17.05229557764334, "process_peak_rss_kib": 24813804, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 24955171680, "block12_before_rms": 0.004311247559165804, "block12_after_rms": 0.004311247559165804 }, "check": { "passed": true, "rule": "sentence", "expected": [ "sunlight", "glucose" ], "character_count": 116, "word_count": 18 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 49106, "input_position": 28, "layers": [ { "after_rms": 0.0042605978840198275, "before_rms": 0.0042605978840198275, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.004311247559165804, "before_rms": 0.004311247559165804, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.00589800417416362, "before_rms": 0.00589800417416362, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.015296940228987576, "before_rms": 0.015296940228987576, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 237568 } ] }, { "step": 1, "token_id": 25424, "input_position": 29, "layers": [ { "after_rms": 0.005977093382486065, "before_rms": 0.005977093382486065, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008193012211801604, "before_rms": 0.008193012211801604, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00683992608315464, "before_rms": 0.00683992608315464, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.022575839889733613, "before_rms": 0.022575839889733613, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 332, "input_position": 30, "layers": [ { "after_rms": 0.0066737027281631835, "before_rms": 0.0066737027281631835, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006813913551885524, "before_rms": 0.006813913551885524, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00890446060152158, "before_rms": 0.00890446060152158, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.027505878131413395, "before_rms": 0.027505878131413395, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 269, "input_position": 31, "layers": [ { "after_rms": 0.0044293615064545915, "before_rms": 0.0044293615064545915, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0051075544884603705, "before_rms": 0.0051075544884603705, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00575102074067097, "before_rms": 0.00575102074067097, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017673491619069545, "before_rms": 0.017673491619069545, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1260, "input_position": 32, "layers": [ { "after_rms": 0.005036949590614196, "before_rms": 0.005036949590614196, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003912458062000313, "before_rms": 0.003912458062000313, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005885844487977124, "before_rms": 0.005885844487977124, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017499873717491128, "before_rms": 0.017499873717491128, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 491, "input_position": 33, "layers": [ { "after_rms": 0.0059711445350189634, "before_rms": 0.0059711445350189634, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005691547282022589, "before_rms": 0.005691547282022589, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008772311292878003, "before_rms": 0.008772311292878003, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.023367520048642263, "before_rms": 0.023367520048642263, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 687, "input_position": 34, "layers": [ { "after_rms": 0.00468986110627818, "before_rms": 0.00468986110627818, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003597916846153238, "before_rms": 0.003597916846153238, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00449814071189118, "before_rms": 0.00449814071189118, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015331526438640039, "before_rms": 0.015331526438640039, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 7370, "input_position": 35, "layers": [ { "after_rms": 0.006125063657130967, "before_rms": 0.006125063657130967, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005508366787700634, "before_rms": 0.005508366787700634, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00612145615591054, "before_rms": 0.00612145615591054, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021503676321074378, "before_rms": 0.021503676321074378, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 896, "input_position": 36, "layers": [ { "after_rms": 0.005212994705122076, "before_rms": 0.005212994705122076, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006187058312930796, "before_rms": 0.006187058312930796, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009534886956672891, "before_rms": 0.009534886956672891, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024496713090488783, "before_rms": 0.024496713090488783, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 9, "token_id": 24044, "input_position": 37, "layers": [ { "after_rms": 0.008439496163631783, "before_rms": 0.008439496163631783, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005501932939207207, "before_rms": 0.005501932939207207, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006071353726939792, "before_rms": 0.006071353726939792, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01955441476435441, "before_rms": 0.01955441476435441, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 10, "token_id": 295, "input_position": 38, "layers": [ { "after_rms": 0.009453040298008773, "before_rms": 0.009453040298008773, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007468227249751736, "before_rms": 0.007468227249751736, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009923443718920184, "before_rms": 0.009923443718920184, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.026507614891845922, "before_rms": 0.026507614891845922, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 11, "token_id": 1815, "input_position": 39, "layers": [ { "after_rms": 0.005096602651817111, "before_rms": 0.005096602651817111, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00521502443061508, "before_rms": 0.00521502443061508, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005931677256268344, "before_rms": 0.005931677256268344, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017564191438152, "before_rms": 0.017564191438152, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 12, "token_id": 18175, "input_position": 40, "layers": [ { "after_rms": 0.004618430168805286, "before_rms": 0.004618430168805286, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007451683719624791, "before_rms": 0.007451683719624791, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006378351819867548, "before_rms": 0.006378351819867548, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02045012871478634, "before_rms": 0.02045012871478634, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 13, "token_id": 11, "input_position": 41, "layers": [ { "after_rms": 0.010203299320963541, "before_rms": 0.010203299320963541, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0074903907754410755, "before_rms": 0.0074903907754410755, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0074366114309835155, "before_rms": 0.0074366114309835155, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02642472904218289, "before_rms": 0.02642472904218289, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 14, "token_id": 20827, "input_position": 42, "layers": [ { "after_rms": 0.003682021169442006, "before_rms": 0.003682021169442006, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003522532163645522, "before_rms": 0.003522532163645522, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005599300959517324, "before_rms": 0.005599300959517324, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.014870400340011209, "before_rms": 0.014870400340011209, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 15, "token_id": 8437, "input_position": 43, "layers": [ { "after_rms": 0.006406295163748658, "before_rms": 0.006406295163748658, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007487654139117973, "before_rms": 0.007487654139117973, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008307528862641168, "before_rms": 0.008307528862641168, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02216700022530452, "before_rms": 0.02216700022530452, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 16, "token_id": 2488, "input_position": 44, "layers": [ { "after_rms": 0.009338543902993778, "before_rms": 0.009338543902993778, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005618921543998528, "before_rms": 0.005618921543998528, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007794415755047646, "before_rms": 0.007794415755047646, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021055979028721958, "before_rms": 0.021055979028721958, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 17, "token_id": 293, "input_position": 45, "layers": [ { "after_rms": 0.012090981045666482, "before_rms": 0.012090981045666482, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005692148533663447, "before_rms": 0.005692148533663447, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009866038408325634, "before_rms": 0.009866038408325634, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02533462806548674, "before_rms": 0.02533462806548674, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 18, "token_id": 6999, "input_position": 46, "layers": [ { "after_rms": 0.0053002691790724555, "before_rms": 0.0053002691790724555, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005049569138548446, "before_rms": 0.005049569138548446, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0056289488702639424, "before_rms": 0.0056289488702639424, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016855422564220696, "before_rms": 0.016855422564220696, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 19, "token_id": 981, "input_position": 47, "layers": [ { "after_rms": 0.007835640918744254, "before_rms": 0.007835640918744254, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005132931328426137, "before_rms": 0.005132931328426137, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008011038840819223, "before_rms": 0.008011038840819223, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02066205167657649, "before_rms": 0.02066205167657649, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 20, "token_id": 13, "input_position": 48, "layers": [ { "after_rms": 0.005366822874419904, "before_rms": 0.005366822874419904, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0069233652002111, "before_rms": 0.0069233652002111, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006798509328318403, "before_rms": 0.006798509328318403, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020444238619442973, "before_rms": 0.020444238619442973, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] } ], "model": "flint" }, { "id": "qwen-arithmetic", "title": "Exact arithmetic", "prompt": "Calculate (17 * 23) - 19. Reply with the integer only.", "checks": [ "Rule: integer", "Expected: 372", "Q8_0: failed", "Q4_1: failed" ], "conditions": [ { "id": "qwen-q8_0", "label": "Q8_0", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "output": "362", "metrics": { "generated_tokens": 3, "prompt_tokens": 50, "elapsed_seconds": 6.370927408, "request_wall_seconds": 6.398233153973706, "time_to_first_token_seconds": 5.195338613004424, "decode_tokens_per_second": 1.6625734938534744, "process_peak_rss_kib": 28517200, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 28595763712, "block12_before_rms": 0.060774775703729224, "block12_after_rms": 0.060774775703729224 }, "check": { "passed": false, "rule": "integer", "expected": "372", "character_count": 3, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 18, "input_position": 49, "layers": [ { "after_rms": 0.051227998654021174, "before_rms": 0.051227998654021174, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1024000 }, { "after_rms": 0.060774775703729224, "before_rms": 0.060774775703729224, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1024000 }, { "after_rms": 0.05656805709715899, "before_rms": 0.05656805709715899, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1024000 }, { "after_rms": 0.6482024672223894, "before_rms": 0.6482024672223894, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 1024000 } ] }, { "step": 1, "token_id": 21, "input_position": 50, "layers": [ { "after_rms": 0.04010414688700296, "before_rms": 0.04010414688700296, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.047943996526761117, "before_rms": 0.047943996526761117, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.059109413227162184, "before_rms": 0.059109413227162184, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6682861418633491, "before_rms": 0.6682861418633491, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 2, "token_id": 17, "input_position": 51, "layers": [ { "after_rms": 0.041057583688967295, "before_rms": 0.041057583688967295, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04121733317261862, "before_rms": 0.04121733317261862, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.047359839621201105, "before_rms": 0.047359839621201105, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8032033925827468, "before_rms": 0.8032033925827468, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] }, { "id": "qwen-q4_1", "label": "Q4_1", "model": "alphaBell Qwen Open 27B", "quant": "Q4_1", "output": "362", "metrics": { "generated_tokens": 3, "prompt_tokens": 50, "elapsed_seconds": 9.60077816, "request_wall_seconds": 9.647026313003153, "time_to_first_token_seconds": 8.771988685010001, "decode_tokens_per_second": 2.285481485264205, "process_peak_rss_kib": 17027080, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 16829921792, "block12_before_rms": 0.0627527559695309, "block12_after_rms": 0.0627527559695309 }, "check": { "passed": false, "rule": "integer", "expected": "372", "character_count": 3, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 18, "input_position": 49, "layers": [ { "after_rms": 0.05048796369798099, "before_rms": 0.05048796369798099, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1024000 }, { "after_rms": 0.0627527559695309, "before_rms": 0.0627527559695309, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1024000 }, { "after_rms": 0.05658700984894215, "before_rms": 0.05658700984894215, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1024000 }, { "after_rms": 0.6455786601758424, "before_rms": 0.6455786601758424, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 1024000 } ] }, { "step": 1, "token_id": 21, "input_position": 50, "layers": [ { "after_rms": 0.03930109211337971, "before_rms": 0.03930109211337971, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.049347960526798046, "before_rms": 0.049347960526798046, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06041073044793531, "before_rms": 0.06041073044793531, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6694457398914584, "before_rms": 0.6694457398914584, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 2, "token_id": 17, "input_position": 51, "layers": [ { "after_rms": 0.039943780185542364, "before_rms": 0.039943780185542364, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.040380369555943325, "before_rms": 0.040380369555943325, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04655456958881338, "before_rms": 0.04655456958881338, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.806759676095322, "before_rms": 0.806759676095322, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] } ], "model": "qwen" }, { "id": "qwen-json", "title": "Strict JSON format", "prompt": "Return exactly this JSON object and no other text: {\"status\":\"ok\",\"count\":3}", "checks": [ "Rule: json", "Expected: {'status': 'ok', 'count': 3}", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "qwen-q8_0", "label": "Q8_0", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "output": "{\"status\":\"ok\",\"count\":3}", "metrics": { "generated_tokens": 9, "prompt_tokens": 49, "elapsed_seconds": 8.587899932, "request_wall_seconds": 8.640126045094803, "time_to_first_token_seconds": 5.242650785017759, "decode_tokens_per_second": 2.3546587753888293, "process_peak_rss_kib": 28517200, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 28595763712, "block12_before_rms": 0.06654119101326558, "block12_after_rms": 0.06654119101326558 }, "check": { "passed": true, "rule": "json", "expected": { "status": "ok", "count": 3 }, "character_count": 25, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 4754, "input_position": 48, "layers": [ { "after_rms": 0.052291073879200234, "before_rms": 0.052291073879200234, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1003520 }, { "after_rms": 0.06654119101326558, "before_rms": 0.06654119101326558, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1003520 }, { "after_rms": 0.05664182123474859, "before_rms": 0.05664182123474859, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1003520 }, { "after_rms": 0.6845949822068217, "before_rms": 0.6845949822068217, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 1003520 } ] }, { "step": 1, "token_id": 2738, "input_position": 49, "layers": [ { "after_rms": 0.03855443169110616, "before_rms": 0.03855443169110616, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.047620845341347987, "before_rms": 0.047620845341347987, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04753896437018176, "before_rms": 0.04753896437018176, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7466266976354985, "before_rms": 0.7466266976354985, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 2, "token_id": 3147, "input_position": 50, "layers": [ { "after_rms": 0.038951229722960494, "before_rms": 0.038951229722960494, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03903383968014138, "before_rms": 0.03903383968014138, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.040843553885173405, "before_rms": 0.040843553885173405, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9274036435751518, "before_rms": 0.9274036435751518, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 3, "token_id": 547, "input_position": 51, "layers": [ { "after_rms": 0.028340486221480752, "before_rms": 0.028340486221480752, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.031217358447974707, "before_rms": 0.031217358447974707, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03351828683553298, "before_rms": 0.03351828683553298, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.696227792238104, "before_rms": 0.696227792238104, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 4, "token_id": 2129, "input_position": 52, "layers": [ { "after_rms": 0.039811305925765116, "before_rms": 0.039811305925765116, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04377612280244084, "before_rms": 0.04377612280244084, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04746907424551777, "before_rms": 0.04746907424551777, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8157278441144455, "before_rms": 0.8157278441144455, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 5, "token_id": 1767, "input_position": 53, "layers": [ { "after_rms": 0.032082000343200995, "before_rms": 0.032082000343200995, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03318282436223156, "before_rms": 0.03318282436223156, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.033619616830206873, "before_rms": 0.033619616830206873, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6970969978843475, "before_rms": 0.6970969978843475, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 6, "token_id": 763, "input_position": 54, "layers": [ { "after_rms": 0.03466834851881882, "before_rms": 0.03466834851881882, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03830034553467172, "before_rms": 0.03830034553467172, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03505207487460683, "before_rms": 0.03505207487460683, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7499139970375372, "before_rms": 0.7499139970375372, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 7, "token_id": 18, "input_position": 55, "layers": [ { "after_rms": 0.02138759797992662, "before_rms": 0.02138759797992662, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.024774094253474845, "before_rms": 0.024774094253474845, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.02468320711787904, "before_rms": 0.02468320711787904, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7000268392851855, "before_rms": 0.7000268392851855, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 8, "token_id": 92, "input_position": 56, "layers": [ { "after_rms": 0.03105064736729475, "before_rms": 0.03105064736729475, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03308346918008162, "before_rms": 0.03308346918008162, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0373820159022643, "before_rms": 0.0373820159022643, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6878214874370955, "before_rms": 0.6878214874370955, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] }, { "id": "qwen-q4_1", "label": "Q4_1", "model": "alphaBell Qwen Open 27B", "quant": "Q4_1", "output": "{\"status\":\"ok\",\"count\":3}", "metrics": { "generated_tokens": 9, "prompt_tokens": 49, "elapsed_seconds": 11.27109005, "request_wall_seconds": 11.301758665940724, "time_to_first_token_seconds": 8.48897207097616, "decode_tokens_per_second": 2.8440503495699967, "process_peak_rss_kib": 17027080, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 16829921792, "block12_before_rms": 0.06706882773636201, "block12_after_rms": 0.06706882773636201 }, "check": { "passed": true, "rule": "json", "expected": { "status": "ok", "count": 3 }, "character_count": 25, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 4754, "input_position": 48, "layers": [ { "after_rms": 0.05224029844078737, "before_rms": 0.05224029844078737, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1003520 }, { "after_rms": 0.06706882773636201, "before_rms": 0.06706882773636201, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1003520 }, { "after_rms": 0.056588501864807024, "before_rms": 0.056588501864807024, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1003520 }, { "after_rms": 0.6822709962112733, "before_rms": 0.6822709962112733, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 1003520 } ] }, { "step": 1, "token_id": 2738, "input_position": 49, "layers": [ { "after_rms": 0.03752622482634915, "before_rms": 0.03752622482634915, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.046903273237478865, "before_rms": 0.046903273237478865, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04722570539635123, "before_rms": 0.04722570539635123, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7377205249987369, "before_rms": 0.7377205249987369, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 2, "token_id": 3147, "input_position": 50, "layers": [ { "after_rms": 0.0383644658257016, "before_rms": 0.0383644658257016, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03938200710595041, "before_rms": 0.03938200710595041, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.040526837904161464, "before_rms": 0.040526837904161464, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9237316979852525, "before_rms": 0.9237316979852525, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 3, "token_id": 547, "input_position": 51, "layers": [ { "after_rms": 0.02778849024723728, "before_rms": 0.02778849024723728, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03214516611442832, "before_rms": 0.03214516611442832, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03363793762611696, "before_rms": 0.03363793762611696, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6944275380038829, "before_rms": 0.6944275380038829, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 4, "token_id": 2129, "input_position": 52, "layers": [ { "after_rms": 0.039622563175138806, "before_rms": 0.039622563175138806, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.044796938258385224, "before_rms": 0.044796938258385224, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.048279328936419154, "before_rms": 0.048279328936419154, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8158488049148617, "before_rms": 0.8158488049148617, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 5, "token_id": 1767, "input_position": 53, "layers": [ { "after_rms": 0.03299336016454038, "before_rms": 0.03299336016454038, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.035306595030275154, "before_rms": 0.035306595030275154, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.034968922773907535, "before_rms": 0.034968922773907535, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7012426405133863, "before_rms": 0.7012426405133863, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 6, "token_id": 763, "input_position": 54, "layers": [ { "after_rms": 0.03539100194372584, "before_rms": 0.03539100194372584, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03888318460173007, "before_rms": 0.03888318460173007, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.036111392321474096, "before_rms": 0.036111392321474096, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7510159984696372, "before_rms": 0.7510159984696372, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 7, "token_id": 18, "input_position": 55, "layers": [ { "after_rms": 0.021069990443326445, "before_rms": 0.021069990443326445, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.02628960155652075, "before_rms": 0.02628960155652075, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.02576443542713521, "before_rms": 0.02576443542713521, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6989044785673804, "before_rms": 0.6989044785673804, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 8, "token_id": 92, "input_position": 56, "layers": [ { "after_rms": 0.03149593114335555, "before_rms": 0.03149593114335555, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.033202658246637436, "before_rms": 0.033202658246637436, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.037501340133310815, "before_rms": 0.037501340133310815, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6955924601927745, "before_rms": 0.6955924601927745, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] } ], "model": "qwen" }, { "id": "qwen-fact", "title": "Short factual recall", "prompt": "What is the chemical symbol for gold? Reply with the symbol only.", "checks": [ "Rule: exact", "Expected: Au", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "qwen-q8_0", "label": "Q8_0", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "output": "Au", "metrics": { "generated_tokens": 1, "prompt_tokens": 44, "elapsed_seconds": 4.642115172, "request_wall_seconds": 4.671560399001464, "time_to_first_token_seconds": 4.299511207034811, "decode_tokens_per_second": null, "process_peak_rss_kib": 28517200, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 28595763712, "block12_before_rms": 0.07549879978282885, "block12_after_rms": 0.07549879978282885 }, "check": { "passed": true, "rule": "exact", "expected": "Au", "character_count": 2, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 63126, "input_position": 43, "layers": [ { "after_rms": 0.0577547722697427, "before_rms": 0.0577547722697427, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 901120 }, { "after_rms": 0.07549879978282885, "before_rms": 0.07549879978282885, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 901120 }, { "after_rms": 0.0707093987239072, "before_rms": 0.0707093987239072, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 901120 }, { "after_rms": 0.6240880098652177, "before_rms": 0.6240880098652177, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 901120 } ] } ] }, { "id": "qwen-q4_1", "label": "Q4_1", "model": "alphaBell Qwen Open 27B", "quant": "Q4_1", "output": "Au", "metrics": { "generated_tokens": 1, "prompt_tokens": 44, "elapsed_seconds": 7.763876279, "request_wall_seconds": 7.807731770095415, "time_to_first_token_seconds": 7.462882330059074, "decode_tokens_per_second": null, "process_peak_rss_kib": 17027080, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 16829921792, "block12_before_rms": 0.07777038091284097, "block12_after_rms": 0.07777038091284097 }, "check": { "passed": true, "rule": "exact", "expected": "Au", "character_count": 2, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 63126, "input_position": 43, "layers": [ { "after_rms": 0.058337225299247154, "before_rms": 0.058337225299247154, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 901120 }, { "after_rms": 0.07777038091284097, "before_rms": 0.07777038091284097, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 901120 }, { "after_rms": 0.07119122784735792, "before_rms": 0.07119122784735792, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 901120 }, { "after_rms": 0.627043092250394, "before_rms": 0.627043092250394, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 901120 } ] } ] } ], "model": "qwen" }, { "id": "qwen-correction", "title": "Use the corrected fact", "prompt": "The access color was blue. Correction: the access color is now amber. What is the current access color? Reply with one word only.", "checks": [ "Rule: exact", "Expected: amber", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "qwen-q8_0", "label": "Q8_0", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "output": "Amber", "metrics": { "generated_tokens": 2, "prompt_tokens": 58, "elapsed_seconds": 6.922008574, "request_wall_seconds": 6.929079352994449, "time_to_first_token_seconds": 6.135061308043078, "decode_tokens_per_second": 1.2593577760195043, "process_peak_rss_kib": 28517200, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 28595763712, "block12_before_rms": 0.08439747530644362, "block12_after_rms": 0.08439747530644362 }, "check": { "passed": true, "rule": "exact", "expected": "amber", "character_count": 5, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 5901, "input_position": 57, "layers": [ { "after_rms": 0.05835882673381115, "before_rms": 0.05835882673381115, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1187840 }, { "after_rms": 0.08439747530644362, "before_rms": 0.08439747530644362, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1187840 }, { "after_rms": 0.07025392596556453, "before_rms": 0.07025392596556453, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1187840 }, { "after_rms": 0.6934357017487144, "before_rms": 0.6934357017487144, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 1187840 } ] }, { "step": 1, "token_id": 634, "input_position": 58, "layers": [ { "after_rms": 0.059411132201425514, "before_rms": 0.059411132201425514, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07671841412821269, "before_rms": 0.07671841412821269, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08880618504780236, "before_rms": 0.08880618504780236, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8700114799969301, "before_rms": 0.8700114799969301, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] }, { "id": "qwen-q4_1", "label": "Q4_1", "model": "alphaBell Qwen Open 27B", "quant": "Q4_1", "output": "Amber", "metrics": { "generated_tokens": 2, "prompt_tokens": 58, "elapsed_seconds": 10.122600283, "request_wall_seconds": 10.232366479933262, "time_to_first_token_seconds": 9.576763171004131, "decode_tokens_per_second": 1.5252099559031584, "process_peak_rss_kib": 17029128, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 16829921792, "block12_before_rms": 0.08882206964440575, "block12_after_rms": 0.08882206964440575 }, "check": { "passed": true, "rule": "exact", "expected": "amber", "character_count": 5, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 5901, "input_position": 57, "layers": [ { "after_rms": 0.05771725253557467, "before_rms": 0.05771725253557467, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1187840 }, { "after_rms": 0.08882206964440575, "before_rms": 0.08882206964440575, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1187840 }, { "after_rms": 0.07251657251443638, "before_rms": 0.07251657251443638, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1187840 }, { "after_rms": 0.694176497002211, "before_rms": 0.694176497002211, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 1187840 } ] }, { "step": 1, "token_id": 634, "input_position": 58, "layers": [ { "after_rms": 0.06058345247387532, "before_rms": 0.06058345247387532, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07852469801096601, "before_rms": 0.07852469801096601, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0889624477213204, "before_rms": 0.0889624477213204, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8659660560094563, "before_rms": 0.8659660560094563, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] } ], "model": "qwen" }, { "id": "qwen-code", "title": "Python expression repair", "prompt": "Complete the function with one Python expression only, no markdown: def is_even(n): return ___ . The function must return True exactly when integer n is even.", "checks": [ "Rule: expression", "Expected: n % 2 == 0", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "qwen-q8_0", "label": "Q8_0", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "output": "n % 2 == 0", "metrics": { "generated_tokens": 7, "prompt_tokens": 62, "elapsed_seconds": 9.254872529, "request_wall_seconds": 9.284579382976517, "time_to_first_token_seconds": 6.607404271955602, "decode_tokens_per_second": 2.2411405168871537, "process_peak_rss_kib": 28517200, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 28595763712, "block12_before_rms": 0.05339986668971442, "block12_after_rms": 0.05339986668971442 }, "check": { "passed": true, "rule": "expression", "expected": "n % 2 == 0", "character_count": 10, "word_count": 5 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 77, "input_position": 61, "layers": [ { "after_rms": 0.050335302931553125, "before_rms": 0.050335302931553125, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1269760 }, { "after_rms": 0.05339986668971442, "before_rms": 0.05339986668971442, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1269760 }, { "after_rms": 0.050829254488739616, "before_rms": 0.050829254488739616, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1269760 }, { "after_rms": 0.6219835311460259, "before_rms": 0.6219835311460259, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 1269760 } ] }, { "step": 1, "token_id": 984, "input_position": 62, "layers": [ { "after_rms": 0.053686741784584366, "before_rms": 0.053686741784584366, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.060073512071431336, "before_rms": 0.060073512071431336, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07386188586159291, "before_rms": 0.07386188586159291, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7373913972139406, "before_rms": 0.7373913972139406, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 2, "token_id": 220, "input_position": 63, "layers": [ { "after_rms": 0.0757632347354875, "before_rms": 0.0757632347354875, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08447271653317262, "before_rms": 0.08447271653317262, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08083366248253042, "before_rms": 0.08083366248253042, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 1.0242087065193417, "before_rms": 1.0242087065193417, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 3, "token_id": 17, "input_position": 64, "layers": [ { "after_rms": 0.031847120231657235, "before_rms": 0.031847120231657235, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.034744065101999524, "before_rms": 0.034744065101999524, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0356676406493493, "before_rms": 0.0356676406493493, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7133838975226968, "before_rms": 0.7133838975226968, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 4, "token_id": 606, "input_position": 65, "layers": [ { "after_rms": 0.06511419656269898, "before_rms": 0.06511419656269898, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06654091916956285, "before_rms": 0.06654091916956285, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0625665999000719, "before_rms": 0.0625665999000719, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9783984307722204, "before_rms": 0.9783984307722204, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 5, "token_id": 220, "input_position": 66, "layers": [ { "after_rms": 0.0645209284738379, "before_rms": 0.0645209284738379, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06279313344255302, "before_rms": 0.06279313344255302, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07065568848215707, "before_rms": 0.07065568848215707, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8914932839543546, "before_rms": 0.8914932839543546, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 6, "token_id": 15, "input_position": 67, "layers": [ { "after_rms": 0.01992336177071922, "before_rms": 0.01992336177071922, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.01935371501039362, "before_rms": 0.01935371501039362, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.02030963950506069, "before_rms": 0.02030963950506069, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.5854232238481374, "before_rms": 0.5854232238481374, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] }, { "id": "qwen-q4_1", "label": "Q4_1", "model": "alphaBell Qwen Open 27B", "quant": "Q4_1", "output": "n % 2 == 0", "metrics": { "generated_tokens": 7, "prompt_tokens": 62, "elapsed_seconds": 12.254965777, "request_wall_seconds": 12.296226173988543, "time_to_first_token_seconds": 10.16412831703201, "decode_tokens_per_second": 2.8140651531387326, "process_peak_rss_kib": 17029128, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 16829921792, "block12_before_rms": 0.05348751558536074, "block12_after_rms": 0.05348751558536074 }, "check": { "passed": true, "rule": "expression", "expected": "n % 2 == 0", "character_count": 10, "word_count": 5 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 77, "input_position": 61, "layers": [ { "after_rms": 0.05019944799383319, "before_rms": 0.05019944799383319, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1269760 }, { "after_rms": 0.05348751558536074, "before_rms": 0.05348751558536074, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1269760 }, { "after_rms": 0.05175455397380355, "before_rms": 0.05175455397380355, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 1269760 }, { "after_rms": 0.6209295945416631, "before_rms": 0.6209295945416631, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 1269760 } ] }, { "step": 1, "token_id": 984, "input_position": 62, "layers": [ { "after_rms": 0.05431832143619491, "before_rms": 0.05431832143619491, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0596135618336448, "before_rms": 0.0596135618336448, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07570250992978683, "before_rms": 0.07570250992978683, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7405282278687231, "before_rms": 0.7405282278687231, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 2, "token_id": 220, "input_position": 63, "layers": [ { "after_rms": 0.07608705520961619, "before_rms": 0.07608705520961619, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08613149135695485, "before_rms": 0.08613149135695485, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08167221964365984, "before_rms": 0.08167221964365984, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 1.0218643697263319, "before_rms": 1.0218643697263319, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 3, "token_id": 17, "input_position": 64, "layers": [ { "after_rms": 0.03293924682560806, "before_rms": 0.03293924682560806, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03573770767293814, "before_rms": 0.03573770767293814, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.03788883557000408, "before_rms": 0.03788883557000408, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7037175496589, "before_rms": 0.7037175496589, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 4, "token_id": 606, "input_position": 65, "layers": [ { "after_rms": 0.06427408980785317, "before_rms": 0.06427408980785317, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0662650177596904, "before_rms": 0.0662650177596904, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06341284549994454, "before_rms": 0.06341284549994454, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9659503607467934, "before_rms": 0.9659503607467934, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 5, "token_id": 220, "input_position": 66, "layers": [ { "after_rms": 0.06468317950504401, "before_rms": 0.06468317950504401, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06446495337470849, "before_rms": 0.06446495337470849, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07169159170100954, "before_rms": 0.07169159170100954, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8967088457352965, "before_rms": 0.8967088457352965, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 6, "token_id": 15, "input_position": 67, "layers": [ { "after_rms": 0.01985784365149535, "before_rms": 0.01985784365149535, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.020029877274204144, "before_rms": 0.020029877274204144, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.020914043695738526, "before_rms": 0.020914043695738526, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.5841152795565343, "before_rms": 0.5841152795565343, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] } ], "model": "qwen" }, { "id": "qwen-brief", "title": "One sentence explanation", "prompt": "Explain photosynthesis in exactly one sentence containing the words sunlight and glucose.", "checks": [ "Rule: sentence", "Expected: ['sunlight', 'glucose']", "Q8_0: passed", "Q4_1: passed" ], "conditions": [ { "id": "qwen-q8_0", "label": "Q8_0", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "output": "Photosynthesis is the process by which plants use sunlight to convert carbon dioxide and water into glucose and oxygen.", "metrics": { "generated_tokens": 21, "prompt_tokens": 45, "elapsed_seconds": 12.680948171, "request_wall_seconds": 12.710689318017103, "time_to_first_token_seconds": 4.477205132017843, "decode_tokens_per_second": 2.4290935490085785, "process_peak_rss_kib": 28517200, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 28595763712, "block12_before_rms": 0.08826706557870591, "block12_after_rms": 0.08826706557870591 }, "check": { "passed": true, "rule": "sentence", "expected": [ "sunlight", "glucose" ], "character_count": 119, "word_count": 19 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 30729, "input_position": 44, "layers": [ { "after_rms": 0.058797602262198566, "before_rms": 0.058797602262198566, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 921600 }, { "after_rms": 0.08826706557870591, "before_rms": 0.08826706557870591, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 921600 }, { "after_rms": 0.07744769492165124, "before_rms": 0.07744769492165124, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 921600 }, { "after_rms": 0.6493664827145925, "before_rms": 0.6493664827145925, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 921600 } ] }, { "step": 1, "token_id": 71163, "input_position": 45, "layers": [ { "after_rms": 0.090079163721148, "before_rms": 0.090079163721148, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.09129523578154757, "before_rms": 0.09129523578154757, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0829387283770895, "before_rms": 0.0829387283770895, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9874844483140827, "before_rms": 0.9874844483140827, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 2, "token_id": 369, "input_position": 46, "layers": [ { "after_rms": 0.10315392514524502, "before_rms": 0.10315392514524502, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.09600644261008953, "before_rms": 0.09600644261008953, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0857152751972423, "before_rms": 0.0857152751972423, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 1.1279006781817724, "before_rms": 1.1279006781817724, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 3, "token_id": 279, "input_position": 47, "layers": [ { "after_rms": 0.0703105570552835, "before_rms": 0.0703105570552835, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08696065973360559, "before_rms": 0.08696065973360559, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08599798105761434, "before_rms": 0.08599798105761434, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7075713993722996, "before_rms": 0.7075713993722996, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 4, "token_id": 1817, "input_position": 48, "layers": [ { "after_rms": 0.04824121607819701, "before_rms": 0.04824121607819701, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.053480497826960974, "before_rms": 0.053480497826960974, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06288983601150724, "before_rms": 0.06288983601150724, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6498592562479432, "before_rms": 0.6498592562479432, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 5, "token_id": 539, "input_position": 49, "layers": [ { "after_rms": 0.0857769249129497, "before_rms": 0.0857769249129497, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.09649908523834096, "before_rms": 0.09649908523834096, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.10047606095155863, "before_rms": 0.10047606095155863, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8915207377991599, "before_rms": 0.8915207377991599, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 6, "token_id": 864, "input_position": 50, "layers": [ { "after_rms": 0.08697706879734413, "before_rms": 0.08697706879734413, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07727010638748308, "before_rms": 0.07727010638748308, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.09060006959943864, "before_rms": 0.09060006959943864, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7683805849623969, "before_rms": 0.7683805849623969, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 7, "token_id": 10462, "input_position": 51, "layers": [ { "after_rms": 0.05259421797317353, "before_rms": 0.05259421797317353, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.05075043909793059, "before_rms": 0.05075043909793059, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.05619568725069516, "before_rms": 0.05619568725069516, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.682036783205715, "before_rms": 0.682036783205715, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 8, "token_id": 958, "input_position": 52, "layers": [ { "after_rms": 0.08697357786711173, "before_rms": 0.08697357786711173, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.10030394478249688, "before_rms": 0.10030394478249688, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08877069591266067, "before_rms": 0.08877069591266067, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9646197687285322, "before_rms": 0.9646197687285322, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 9, "token_id": 37728, "input_position": 53, "layers": [ { "after_rms": 0.07099648705814063, "before_rms": 0.07099648705814063, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06559096588235168, "before_rms": 0.06559096588235168, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07033321001871201, "before_rms": 0.07033321001871201, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7932168808370963, "before_rms": 0.7932168808370963, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 10, "token_id": 310, "input_position": 54, "layers": [ { "after_rms": 0.08567177205503797, "before_rms": 0.08567177205503797, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08384291426005887, "before_rms": 0.08384291426005887, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08019719454528439, "before_rms": 0.08019719454528439, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9847601168939447, "before_rms": 0.9847601168939447, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 11, "token_id": 5335, "input_position": 55, "layers": [ { "after_rms": 0.043123201419361115, "before_rms": 0.043123201419361115, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04955096633139602, "before_rms": 0.04955096633139602, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.053935569085920204, "before_rms": 0.053935569085920204, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7026575509832893, "before_rms": 0.7026575509832893, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 12, "token_id": 12141, "input_position": 56, "layers": [ { "after_rms": 0.07978349390889998, "before_rms": 0.07978349390889998, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07617391418880347, "before_rms": 0.07617391418880347, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07734026858151206, "before_rms": 0.07734026858151206, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8642118185671083, "before_rms": 0.8642118185671083, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 13, "token_id": 38182, "input_position": 57, "layers": [ { "after_rms": 0.0852385224689407, "before_rms": 0.0852385224689407, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07231971038626593, "before_rms": 0.07231971038626593, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06564426590463994, "before_rms": 0.06564426590463994, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9157673634228397, "before_rms": 0.9157673634228397, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 14, "token_id": 321, "input_position": 58, "layers": [ { "after_rms": 0.08898192682211213, "before_rms": 0.08898192682211213, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0894750809473178, "before_rms": 0.0894750809473178, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08616370851207737, "before_rms": 0.08616370851207737, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 1.0029595110641643, "before_rms": 1.0029595110641643, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 15, "token_id": 2919, "input_position": 59, "layers": [ { "after_rms": 0.03762191629076314, "before_rms": 0.03762191629076314, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.049029101729874604, "before_rms": 0.049029101729874604, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.05884392489297117, "before_rms": 0.05884392489297117, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6250386128828833, "before_rms": 0.6250386128828833, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 16, "token_id": 1083, "input_position": 60, "layers": [ { "after_rms": 0.06327166050315482, "before_rms": 0.06327166050315482, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06857836855297815, "before_rms": 0.06857836855297815, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06961224727562185, "before_rms": 0.06961224727562185, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8674029708685703, "before_rms": 0.8674029708685703, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 17, "token_id": 32120, "input_position": 61, "layers": [ { "after_rms": 0.0530895463732545, "before_rms": 0.0530895463732545, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06423360109914558, "before_rms": 0.06423360109914558, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.052746699956901094, "before_rms": 0.052746699956901094, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7559156680420246, "before_rms": 0.7559156680420246, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 18, "token_id": 321, "input_position": 62, "layers": [ { "after_rms": 0.07834838045994791, "before_rms": 0.07834838045994791, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07525093120253516, "before_rms": 0.07525093120253516, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07091507308763366, "before_rms": 0.07091507308763366, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.954111191683783, "before_rms": 0.954111191683783, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 19, "token_id": 22817, "input_position": 63, "layers": [ { "after_rms": 0.0353542432718733, "before_rms": 0.0353542432718733, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04198844081291262, "before_rms": 0.04198844081291262, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.054421407751238354, "before_rms": 0.054421407751238354, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.629625149193875, "before_rms": 0.629625149193875, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 20, "token_id": 13, "input_position": 64, "layers": [ { "after_rms": 0.07315881109901357, "before_rms": 0.07315881109901357, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0687687379925814, "before_rms": 0.0687687379925814, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06567824611420153, "before_rms": 0.06567824611420153, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8984019945181603, "before_rms": 0.8984019945181603, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] }, { "id": "qwen-q4_1", "label": "Q4_1", "model": "alphaBell Qwen Open 27B", "quant": "Q4_1", "output": "Photosynthesis is the process by which plants use sunlight to convert carbon dioxide and water into glucose and oxygen.", "metrics": { "generated_tokens": 21, "prompt_tokens": 45, "elapsed_seconds": 13.409067066, "request_wall_seconds": 13.439180701971054, "time_to_first_token_seconds": 7.273399753961712, "decode_tokens_per_second": 3.243670129869688, "process_peak_rss_kib": 17029128, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 16829921792, "block12_before_rms": 0.09098383032221528, "block12_after_rms": 0.09098383032221528 }, "check": { "passed": true, "rule": "sentence", "expected": [ "sunlight", "glucose" ], "character_count": 119, "word_count": 19 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 30729, "input_position": 44, "layers": [ { "after_rms": 0.059582062101886914, "before_rms": 0.059582062101886914, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 921600 }, { "after_rms": 0.09098383032221528, "before_rms": 0.09098383032221528, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 921600 }, { "after_rms": 0.08016257842928678, "before_rms": 0.08016257842928678, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 921600 }, { "after_rms": 0.6536671790903104, "before_rms": 0.6536671790903104, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 921600 } ] }, { "step": 1, "token_id": 71163, "input_position": 45, "layers": [ { "after_rms": 0.09190790143581268, "before_rms": 0.09190790143581268, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.09553453447497137, "before_rms": 0.09553453447497137, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08593551701844118, "before_rms": 0.08593551701844118, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9966251527987956, "before_rms": 0.9966251527987956, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 2, "token_id": 369, "input_position": 46, "layers": [ { "after_rms": 0.10115326211799784, "before_rms": 0.10115326211799784, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0958071133672348, "before_rms": 0.0958071133672348, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0854668936412898, "before_rms": 0.0854668936412898, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 1.1255902406232912, "before_rms": 1.1255902406232912, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 3, "token_id": 279, "input_position": 47, "layers": [ { "after_rms": 0.07451807797268321, "before_rms": 0.07451807797268321, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0966776949197356, "before_rms": 0.0966776949197356, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.09200739468581592, "before_rms": 0.09200739468581592, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7275052188652554, "before_rms": 0.7275052188652554, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 4, "token_id": 1817, "input_position": 48, "layers": [ { "after_rms": 0.0478125046485284, "before_rms": 0.0478125046485284, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0548085725164451, "before_rms": 0.0548085725164451, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06471972775053976, "before_rms": 0.06471972775053976, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6538473476210616, "before_rms": 0.6538473476210616, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 5, "token_id": 539, "input_position": 49, "layers": [ { "after_rms": 0.08518049302904517, "before_rms": 0.08518049302904517, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.09818462339752433, "before_rms": 0.09818462339752433, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.10130589343276135, "before_rms": 0.10130589343276135, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8866838924389607, "before_rms": 0.8866838924389607, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 6, "token_id": 864, "input_position": 50, "layers": [ { "after_rms": 0.08680030692428589, "before_rms": 0.08680030692428589, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08012581611715355, "before_rms": 0.08012581611715355, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.09241248361191251, "before_rms": 0.09241248361191251, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7686299073659799, "before_rms": 0.7686299073659799, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 7, "token_id": 10462, "input_position": 51, "layers": [ { "after_rms": 0.05017937165905816, "before_rms": 0.05017937165905816, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.050057021551545955, "before_rms": 0.050057021551545955, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.05497654538007085, "before_rms": 0.05497654538007085, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6670578115164874, "before_rms": 0.6670578115164874, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 8, "token_id": 958, "input_position": 52, "layers": [ { "after_rms": 0.08568044433943381, "before_rms": 0.08568044433943381, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.10045397748397368, "before_rms": 0.10045397748397368, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08773616457482147, "before_rms": 0.08773616457482147, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9600428235856283, "before_rms": 0.9600428235856283, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 9, "token_id": 37728, "input_position": 53, "layers": [ { "after_rms": 0.06924941047054128, "before_rms": 0.06924941047054128, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0655311740748727, "before_rms": 0.0655311740748727, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07066158387714026, "before_rms": 0.07066158387714026, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7935303713786936, "before_rms": 0.7935303713786936, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 10, "token_id": 310, "input_position": 54, "layers": [ { "after_rms": 0.08433748009746503, "before_rms": 0.08433748009746503, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08393242733321232, "before_rms": 0.08393242733321232, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.0819084026323373, "before_rms": 0.0819084026323373, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9796541806488255, "before_rms": 0.9796541806488255, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 11, "token_id": 5335, "input_position": 55, "layers": [ { "after_rms": 0.039579585893422226, "before_rms": 0.039579585893422226, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.046979610302819906, "before_rms": 0.046979610302819906, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.05344053164079346, "before_rms": 0.05344053164079346, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6960761835563655, "before_rms": 0.6960761835563655, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 12, "token_id": 12141, "input_position": 56, "layers": [ { "after_rms": 0.07903697242878281, "before_rms": 0.07903697242878281, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07614783932603204, "before_rms": 0.07614783932603204, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07944953913293375, "before_rms": 0.07944953913293375, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8567541215882547, "before_rms": 0.8567541215882547, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 13, "token_id": 38182, "input_position": 57, "layers": [ { "after_rms": 0.0864315395888373, "before_rms": 0.0864315395888373, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07366431154681498, "before_rms": 0.07366431154681498, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06620859996023702, "before_rms": 0.06620859996023702, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9160313052637309, "before_rms": 0.9160313052637309, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 14, "token_id": 321, "input_position": 58, "layers": [ { "after_rms": 0.08880976995639013, "before_rms": 0.08880976995639013, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08999483103099812, "before_rms": 0.08999483103099812, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.08782409614422529, "before_rms": 0.08782409614422529, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9846578342137169, "before_rms": 0.9846578342137169, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 15, "token_id": 2919, "input_position": 59, "layers": [ { "after_rms": 0.03571734510932391, "before_rms": 0.03571734510932391, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.047244981585028045, "before_rms": 0.047244981585028045, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.05755773777600419, "before_rms": 0.05755773777600419, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6213354826118737, "before_rms": 0.6213354826118737, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 16, "token_id": 1083, "input_position": 60, "layers": [ { "after_rms": 0.0626217871949745, "before_rms": 0.0626217871949745, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06839292559199892, "before_rms": 0.06839292559199892, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07067343367902718, "before_rms": 0.07067343367902718, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.857509349072362, "before_rms": 0.857509349072362, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 17, "token_id": 32120, "input_position": 61, "layers": [ { "after_rms": 0.05464629387267075, "before_rms": 0.05464629387267075, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06755006814151779, "before_rms": 0.06755006814151779, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.05537102787014218, "before_rms": 0.05537102787014218, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.7564307778216667, "before_rms": 0.7564307778216667, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 18, "token_id": 321, "input_position": 62, "layers": [ { "after_rms": 0.07894008680252157, "before_rms": 0.07894008680252157, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07594297305618296, "before_rms": 0.07594297305618296, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07215577303867654, "before_rms": 0.07215577303867654, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.9455045304946262, "before_rms": 0.9455045304946262, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 19, "token_id": 22817, "input_position": 63, "layers": [ { "after_rms": 0.03382951016881296, "before_rms": 0.03382951016881296, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.04193784117869559, "before_rms": 0.04193784117869559, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.054778189331242615, "before_rms": 0.054778189331242615, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.6269059013903844, "before_rms": 0.6269059013903844, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] }, { "step": 20, "token_id": 13, "input_position": 64, "layers": [ { "after_rms": 0.07374082621976509, "before_rms": 0.07374082621976509, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.07036659145204018, "before_rms": 0.07036659145204018, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.06651851708741562, "before_rms": 0.06651851708741562, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 20480 }, { "after_rms": 0.8993101756034922, "before_rms": 0.8993101756034922, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 20480 } ] } ] } ], "model": "qwen" } ] }, "surgery": { "title": "Model surgery: what else changed?", "summary": "Scale one Flint feed-forward branch to 1, 0.5 or 0. Outputs change on some cases; the same four of six objective checks pass at every scale.", "method": "Modify the computed ffn_out-12 tensor before residual addition at every prompt and decode position. Measure pre/post RMS and neighboring blocks. No checkpoint weights are rewritten.", "limitations": [ "One block, six tasks and one greedy run per condition. No improvement claim. Later outputs can differ in context as well as intervention. Only the first-frame upstream equality is a direct matched-input control." ], "cases": [ { "id": "arithmetic", "title": "Exact arithmetic", "prompt": "Calculate (17 * 23) - 19. Reply with the integer only.", "checks": [ "Rule: integer", "Expected: 372", "Baseline, scale 1: failed", "Half FFN, scale 0.5: failed", "Zero FFN, scale 0: failed" ], "intervention": { "tensor": "ffn_out-12", "layer_indexing": "zero-based", "applies_to": "all prompt and generated positions on each forward pass", "operation": "scale computed FFN output before residual addition", "scales": [ 1, 0.5, 0 ], "weights_modified": false, "kind": "activation_ablation", "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." }, "conditions": [ { "id": "baseline", "label": "Baseline, scale 1", "model": "Recursive Flint", "quant": "Q8_0", "output": "374", "metrics": { "generated_tokens": 1, "prompt_tokens": 33, "elapsed_seconds": 0.738173075, "request_wall_seconds": 0.7542800180381164, "time_to_first_token_seconds": 0.689367342973128, "decode_tokens_per_second": null, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.0042319691359885515, "block12_after_rms": 0.0042319691359885515 }, "check": { "passed": false, "rule": "integer", "expected": "372", "character_count": 3, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 9056, "input_position": 32, "layers": [ { "after_rms": 0.004216483101124048, "before_rms": 0.004216483101124048, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.0042319691359885515, "before_rms": 0.0042319691359885515, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.005525138486063405, "before_rms": 0.005525138486063405, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.014221511290226446, "before_rms": 0.014221511290226446, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 270336 } ] } ] }, { "id": "half", "label": "Half FFN, scale 0.5", "model": "Recursive Flint", "quant": "Q8_0", "output": "352", "metrics": { "generated_tokens": 1, "prompt_tokens": 33, "elapsed_seconds": 0.719981441, "request_wall_seconds": 0.7371821940178052, "time_to_first_token_seconds": 0.6725124060176313, "decode_tokens_per_second": null, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.0042319691359885515, "block12_after_rms": 0.0021159845679942758 }, "check": { "passed": false, "rule": "integer", "expected": "372", "character_count": 3, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0.5 }, "measurements": [ { "step": 0, "token_id": 11510, "input_position": 32, "layers": [ { "after_rms": 0.004216483101124048, "before_rms": 0.004216483101124048, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.0021159845679942758, "before_rms": 0.0042319691359885515, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.005548826881067284, "before_rms": 0.005548826881067284, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.013772509153955564, "before_rms": 0.013772509153955564, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 270336 } ] } ] }, { "id": "zero", "label": "Zero FFN, scale 0", "model": "Recursive Flint", "quant": "Q8_0", "output": "340", "metrics": { "generated_tokens": 1, "prompt_tokens": 33, "elapsed_seconds": 0.721295279, "request_wall_seconds": 0.7335850830422714, "time_to_first_token_seconds": 0.6692783629987389, "decode_tokens_per_second": null, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.0042319691359885515, "block12_after_rms": 0.0 }, "check": { "passed": false, "rule": "integer", "expected": "372", "character_count": 3, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0 }, "measurements": [ { "step": 0, "token_id": 10333, "input_position": 32, "layers": [ { "after_rms": 0.004216483101124048, "before_rms": 0.004216483101124048, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.0, "before_rms": 0.0042319691359885515, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.0056156565381101035, "before_rms": 0.0056156565381101035, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 270336 }, { "after_rms": 0.013872404589524597, "before_rms": 0.013872404589524597, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 270336 } ] } ] } ], "comparison": { "untargeted_first_frame_block11_identical": true, "downstream_first_frame_block13_changed": true, "weights_modified": false, "all_saved_frames_available_in_results": true } }, { "id": "json", "title": "Strict JSON format", "prompt": "Return exactly this JSON object and no other text: {\"status\":\"ok\",\"count\":3}", "checks": [ "Rule: json", "Expected: {'status': 'ok', 'count': 3}", "Baseline, scale 1: passed", "Half FFN, scale 0.5: passed", "Zero FFN, scale 0: passed" ], "intervention": { "tensor": "ffn_out-12", "layer_indexing": "zero-based", "applies_to": "all prompt and generated positions on each forward pass", "operation": "scale computed FFN output before residual addition", "scales": [ 1, 0.5, 0 ], "weights_modified": false, "kind": "activation_ablation", "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." }, "conditions": [ { "id": "baseline", "label": "Baseline, scale 1", "model": "Recursive Flint", "quant": "Q8_0", "output": "{\"status\":\"ok\",\"count\":3}", "metrics": { "generated_tokens": 9, "prompt_tokens": 35, "elapsed_seconds": 1.259416502, "request_wall_seconds": 1.2757591230329126, "time_to_first_token_seconds": 0.7007529039401561, "decode_tokens_per_second": 13.911468772577386, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004073247840262152, "block12_after_rms": 0.004073247840262152 }, "check": { "passed": true, "rule": "json", "expected": { "status": "ok", "count": 3 }, "character_count": 25, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 5482, "input_position": 34, "layers": [ { "after_rms": 0.00442951382677711, "before_rms": 0.00442951382677711, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.004073247840262152, "before_rms": 0.004073247840262152, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.00524975286610561, "before_rms": 0.00524975286610561, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.014444385872118682, "before_rms": 0.014444385872118682, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 286720 } ] }, { "step": 1, "token_id": 5462, "input_position": 35, "layers": [ { "after_rms": 0.005793034803083275, "before_rms": 0.005793034803083275, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0046820736039610645, "before_rms": 0.0046820736039610645, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00649377540352318, "before_rms": 0.00649377540352318, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019726933354943656, "before_rms": 0.019726933354943656, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 4890, "input_position": 36, "layers": [ { "after_rms": 0.005294017353787343, "before_rms": 0.005294017353787343, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0034695353362188502, "before_rms": 0.0034695353362188502, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00738829527350583, "before_rms": 0.00738829527350583, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019360584231844193, "before_rms": 0.019360584231844193, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 544, "input_position": 37, "layers": [ { "after_rms": 0.00491148399515983, "before_rms": 0.00491148399515983, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00380345983046727, "before_rms": 0.00380345983046727, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005149091746140284, "before_rms": 0.005149091746140284, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016276301961648348, "before_rms": 0.016276301961648348, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1147, "input_position": 38, "layers": [ { "after_rms": 0.006222679356489009, "before_rms": 0.006222679356489009, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0036829646504058975, "before_rms": 0.0036829646504058975, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006655483694052688, "before_rms": 0.006655483694052688, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019208434114849963, "before_rms": 0.019208434114849963, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 2816, "input_position": 39, "layers": [ { "after_rms": 0.005760162518562755, "before_rms": 0.005760162518562755, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003275269655906223, "before_rms": 0.003275269655906223, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004388659397945476, "before_rms": 0.004388659397945476, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015988995629268844, "before_rms": 0.015988995629268844, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 1129, "input_position": 40, "layers": [ { "after_rms": 0.004863272662823825, "before_rms": 0.004863272662823825, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0038314286271215804, "before_rms": 0.0038314286271215804, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007286998970545343, "before_rms": 0.007286998970545343, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018843493317539196, "before_rms": 0.018843493317539196, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 18, "input_position": 41, "layers": [ { "after_rms": 0.004603991231053795, "before_rms": 0.004603991231053795, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0030208238582806433, "before_rms": 0.0030208238582806433, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004703706397060763, "before_rms": 0.004703706397060763, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.013840299536615416, "before_rms": 0.013840299536615416, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 92, "input_position": 42, "layers": [ { "after_rms": 0.0039039619301298022, "before_rms": 0.0039039619301298022, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004057874942645444, "before_rms": 0.004057874942645444, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006582297878179331, "before_rms": 0.006582297878179331, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016272639100438333, "before_rms": 0.016272639100438333, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "half", "label": "Half FFN, scale 0.5", "model": "Recursive Flint", "quant": "Q8_0", "output": "{\"status\":\"ok\",\"count\":3}", "metrics": { "generated_tokens": 9, "prompt_tokens": 35, "elapsed_seconds": 1.246835879, "request_wall_seconds": 1.262888611992821, "time_to_first_token_seconds": 0.6985334729542956, "decode_tokens_per_second": 14.17425932571783, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004073247840262152, "block12_after_rms": 0.002036623920131076 }, "check": { "passed": true, "rule": "json", "expected": { "status": "ok", "count": 3 }, "character_count": 25, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0.5 }, "measurements": [ { "step": 0, "token_id": 5482, "input_position": 34, "layers": [ { "after_rms": 0.00442951382677711, "before_rms": 0.00442951382677711, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.002036623920131076, "before_rms": 0.004073247840262152, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.00511203630467351, "before_rms": 0.00511203630467351, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.014008885307881237, "before_rms": 0.014008885307881237, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 286720 } ] }, { "step": 1, "token_id": 5462, "input_position": 35, "layers": [ { "after_rms": 0.005793034803083275, "before_rms": 0.005793034803083275, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0023410368019805322, "before_rms": 0.0046820736039610645, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006683535999525596, "before_rms": 0.006683535999525596, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01886032206135178, "before_rms": 0.01886032206135178, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 4890, "input_position": 36, "layers": [ { "after_rms": 0.005294017353787343, "before_rms": 0.005294017353787343, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0017347676681094251, "before_rms": 0.0034695353362188502, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0076820001587381715, "before_rms": 0.0076820001587381715, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019039892595830377, "before_rms": 0.019039892595830377, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 544, "input_position": 37, "layers": [ { "after_rms": 0.00491148399515983, "before_rms": 0.00491148399515983, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.001901729915233635, "before_rms": 0.00380345983046727, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005344106452152085, "before_rms": 0.005344106452152085, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01594381241067434, "before_rms": 0.01594381241067434, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1147, "input_position": 38, "layers": [ { "after_rms": 0.006222679356489009, "before_rms": 0.006222679356489009, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0018414823252029487, "before_rms": 0.0036829646504058975, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00664047642037073, "before_rms": 0.00664047642037073, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01916710645697236, "before_rms": 0.01916710645697236, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 2816, "input_position": 39, "layers": [ { "after_rms": 0.005760162518562755, "before_rms": 0.005760162518562755, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0016376348279531115, "before_rms": 0.003275269655906223, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0046118119033721995, "before_rms": 0.0046118119033721995, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015733109219504087, "before_rms": 0.015733109219504087, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 1129, "input_position": 40, "layers": [ { "after_rms": 0.004863272662823825, "before_rms": 0.004863272662823825, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0019157143135607902, "before_rms": 0.0038314286271215804, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007354879141339851, "before_rms": 0.007354879141339851, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0188168576455134, "before_rms": 0.0188168576455134, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 18, "input_position": 41, "layers": [ { "after_rms": 0.004603991231053795, "before_rms": 0.004603991231053795, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0015104119291403217, "before_rms": 0.0030208238582806433, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004847850483184061, "before_rms": 0.004847850483184061, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.013329843619400705, "before_rms": 0.013329843619400705, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 92, "input_position": 42, "layers": [ { "after_rms": 0.0039039619301298022, "before_rms": 0.0039039619301298022, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002028937471322722, "before_rms": 0.004057874942645444, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006716603826791902, "before_rms": 0.006716603826791902, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015738018767318247, "before_rms": 0.015738018767318247, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "zero", "label": "Zero FFN, scale 0", "model": "Recursive Flint", "quant": "Q8_0", "output": "{\"status\":\"ok\",\"count\":3}", "metrics": { "generated_tokens": 9, "prompt_tokens": 35, "elapsed_seconds": 1.252187772, "request_wall_seconds": 1.2679177080281079, "time_to_first_token_seconds": 0.7051889860304072, "decode_tokens_per_second": 14.213787672176204, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004073247840262152, "block12_after_rms": 0.0 }, "check": { "passed": true, "rule": "json", "expected": { "status": "ok", "count": 3 }, "character_count": 25, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0 }, "measurements": [ { "step": 0, "token_id": 5482, "input_position": 34, "layers": [ { "after_rms": 0.00442951382677711, "before_rms": 0.00442951382677711, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.0, "before_rms": 0.004073247840262152, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.005232460518540055, "before_rms": 0.005232460518540055, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 286720 }, { "after_rms": 0.014132856084779098, "before_rms": 0.014132856084779098, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 286720 } ] }, { "step": 1, "token_id": 5462, "input_position": 35, "layers": [ { "after_rms": 0.005793034803083275, "before_rms": 0.005793034803083275, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0046820736039610645, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006932431350240155, "before_rms": 0.006932431350240155, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018253630668038652, "before_rms": 0.018253630668038652, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 4890, "input_position": 36, "layers": [ { "after_rms": 0.005294017353787343, "before_rms": 0.005294017353787343, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0034695353362188502, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007877159840683485, "before_rms": 0.007877159840683485, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018856526179738382, "before_rms": 0.018856526179738382, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 544, "input_position": 37, "layers": [ { "after_rms": 0.00491148399515983, "before_rms": 0.00491148399515983, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.00380345983046727, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005610946438388422, "before_rms": 0.005610946438388422, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016040718140881722, "before_rms": 0.016040718140881722, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1147, "input_position": 38, "layers": [ { "after_rms": 0.006222679356489009, "before_rms": 0.006222679356489009, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0036829646504058975, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0065644993447098255, "before_rms": 0.0065644993447098255, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01935782943457066, "before_rms": 0.01935782943457066, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 2816, "input_position": 39, "layers": [ { "after_rms": 0.005760162518562755, "before_rms": 0.005760162518562755, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.003275269655906223, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0050584932701879986, "before_rms": 0.0050584932701879986, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015686042844506247, "before_rms": 0.015686042844506247, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 1129, "input_position": 40, "layers": [ { "after_rms": 0.004863272662823825, "before_rms": 0.004863272662823825, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0038314286271215804, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007195615412259338, "before_rms": 0.007195615412259338, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018477358488448153, "before_rms": 0.018477358488448153, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 18, "input_position": 41, "layers": [ { "after_rms": 0.004603991231053795, "before_rms": 0.004603991231053795, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0030208238582806433, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005075615167638164, "before_rms": 0.005075615167638164, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.013303633799015144, "before_rms": 0.013303633799015144, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 92, "input_position": 42, "layers": [ { "after_rms": 0.0039039619301298022, "before_rms": 0.0039039619301298022, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.004057874942645444, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0069263167645750995, "before_rms": 0.0069263167645750995, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015314707984930225, "before_rms": 0.015314707984930225, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] } ], "comparison": { "untargeted_first_frame_block11_identical": true, "downstream_first_frame_block13_changed": true, "weights_modified": false, "all_saved_frames_available_in_results": true } }, { "id": "fact", "title": "Short factual recall", "prompt": "What is the chemical symbol for gold? Reply with the symbol only.", "checks": [ "Rule: exact", "Expected: Au", "Baseline, scale 1: passed", "Half FFN, scale 0.5: passed", "Zero FFN, scale 0: passed" ], "intervention": { "tensor": "ffn_out-12", "layer_indexing": "zero-based", "applies_to": "all prompt and generated positions on each forward pass", "operation": "scale computed FFN output before residual addition", "scales": [ 1, 0.5, 0 ], "weights_modified": false, "kind": "activation_ablation", "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." }, "conditions": [ { "id": "baseline", "label": "Baseline, scale 1", "model": "Recursive Flint", "quant": "Q8_0", "output": "Au", "metrics": { "generated_tokens": 1, "prompt_tokens": 30, "elapsed_seconds": 0.650667801, "request_wall_seconds": 0.6673325290903449, "time_to_first_token_seconds": 0.6034285150235519, "decode_tokens_per_second": null, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004120685912594556, "block12_after_rms": 0.004120685912594556 }, "check": { "passed": true, "rule": "exact", "expected": "Au", "character_count": 2, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 22911, "input_position": 29, "layers": [ { "after_rms": 0.0042503533024456225, "before_rms": 0.0042503533024456225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.004120685912594556, "before_rms": 0.004120685912594556, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.005313107066375563, "before_rms": 0.005313107066375563, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.01490764588916619, "before_rms": 0.01490764588916619, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 245760 } ] } ] }, { "id": "half", "label": "Half FFN, scale 0.5", "model": "Recursive Flint", "quant": "Q8_0", "output": "Au", "metrics": { "generated_tokens": 1, "prompt_tokens": 30, "elapsed_seconds": 0.751291221, "request_wall_seconds": 0.7725467890268192, "time_to_first_token_seconds": 0.6922730850055814, "decode_tokens_per_second": null, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004120685912594556, "block12_after_rms": 0.002060342956297278 }, "check": { "passed": true, "rule": "exact", "expected": "Au", "character_count": 2, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0.5 }, "measurements": [ { "step": 0, "token_id": 22911, "input_position": 29, "layers": [ { "after_rms": 0.0042503533024456225, "before_rms": 0.0042503533024456225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.002060342956297278, "before_rms": 0.004120685912594556, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.005343264427809506, "before_rms": 0.005343264427809506, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.014565723365685252, "before_rms": 0.014565723365685252, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 245760 } ] } ] }, { "id": "zero", "label": "Zero FFN, scale 0", "model": "Recursive Flint", "quant": "Q8_0", "output": "Au", "metrics": { "generated_tokens": 1, "prompt_tokens": 30, "elapsed_seconds": 0.764433258, "request_wall_seconds": 0.7862005030037835, "time_to_first_token_seconds": 0.7070632589748129, "decode_tokens_per_second": null, "process_peak_rss_kib": 41799256, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004120685912594556, "block12_after_rms": 0.0 }, "check": { "passed": true, "rule": "exact", "expected": "Au", "character_count": 2, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0 }, "measurements": [ { "step": 0, "token_id": 22911, "input_position": 29, "layers": [ { "after_rms": 0.0042503533024456225, "before_rms": 0.0042503533024456225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.0, "before_rms": 0.004120685912594556, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.005448940453841594, "before_rms": 0.005448940453841594, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 245760 }, { "after_rms": 0.01475624234457033, "before_rms": 0.01475624234457033, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 245760 } ] } ] } ], "comparison": { "untargeted_first_frame_block11_identical": true, "downstream_first_frame_block13_changed": true, "weights_modified": false, "all_saved_frames_available_in_results": true } }, { "id": "correction", "title": "Use the corrected fact", "prompt": "The access color was blue. Correction: the access color is now amber. What is the current access color? Reply with one word only.", "checks": [ "Rule: exact", "Expected: amber", "Baseline, scale 1: passed", "Half FFN, scale 0.5: passed", "Zero FFN, scale 0: passed" ], "intervention": { "tensor": "ffn_out-12", "layer_indexing": "zero-based", "applies_to": "all prompt and generated positions on each forward pass", "operation": "scale computed FFN output before residual addition", "scales": [ 1, 0.5, 0 ], "weights_modified": false, "kind": "activation_ablation", "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." }, "conditions": [ { "id": "baseline", "label": "Baseline, scale 1", "model": "Recursive Flint", "quant": "Q8_0", "output": "amber", "metrics": { "generated_tokens": 1, "prompt_tokens": 44, "elapsed_seconds": 0.974556193, "request_wall_seconds": 0.9955227529862896, "time_to_first_token_seconds": 0.9343198160640895, "decode_tokens_per_second": null, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.00441159816545625, "block12_after_rms": 0.00441159816545625 }, "check": { "passed": true, "rule": "exact", "expected": "amber", "character_count": 5, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 13732, "input_position": 43, "layers": [ { "after_rms": 0.004252703885422568, "before_rms": 0.004252703885422568, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.00441159816545625, "before_rms": 0.00441159816545625, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.005657736200656421, "before_rms": 0.005657736200656421, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.014876767054565872, "before_rms": 0.014876767054565872, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 360448 } ] } ] }, { "id": "half", "label": "Half FFN, scale 0.5", "model": "Recursive Flint", "quant": "Q8_0", "output": "amber", "metrics": { "generated_tokens": 1, "prompt_tokens": 44, "elapsed_seconds": 0.845619764, "request_wall_seconds": 0.8612813500221819, "time_to_first_token_seconds": 0.8017731390427798, "decode_tokens_per_second": null, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.00441159816545625, "block12_after_rms": 0.002205799082728125 }, "check": { "passed": true, "rule": "exact", "expected": "amber", "character_count": 5, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0.5 }, "measurements": [ { "step": 0, "token_id": 13732, "input_position": 43, "layers": [ { "after_rms": 0.004252703885422568, "before_rms": 0.004252703885422568, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.002205799082728125, "before_rms": 0.00441159816545625, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.005628658790824781, "before_rms": 0.005628658790824781, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.014474070728207288, "before_rms": 0.014474070728207288, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 360448 } ] } ] }, { "id": "zero", "label": "Zero FFN, scale 0", "model": "Recursive Flint", "quant": "Q8_0", "output": "amber", "metrics": { "generated_tokens": 1, "prompt_tokens": 44, "elapsed_seconds": 1.003642273, "request_wall_seconds": 1.0181882489705458, "time_to_first_token_seconds": 0.9386005689157173, "decode_tokens_per_second": null, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.00441159816545625, "block12_after_rms": 0.0 }, "check": { "passed": true, "rule": "exact", "expected": "amber", "character_count": 5, "word_count": 1 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0 }, "measurements": [ { "step": 0, "token_id": 13732, "input_position": 43, "layers": [ { "after_rms": 0.004252703885422568, "before_rms": 0.004252703885422568, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.0, "before_rms": 0.00441159816545625, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.005651444761795099, "before_rms": 0.005651444761795099, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 360448 }, { "after_rms": 0.01456451414024136, "before_rms": 0.01456451414024136, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 360448 } ] } ] } ], "comparison": { "untargeted_first_frame_block11_identical": true, "downstream_first_frame_block13_changed": true, "weights_modified": false, "all_saved_frames_available_in_results": true } }, { "id": "code", "title": "Python expression repair", "prompt": "Complete the function with one Python expression only, no markdown: def is_even(n): return ___ . The function must return True exactly when integer n is even.", "checks": [ "Rule: expression", "Expected: n % 2 == 0", "Baseline, scale 1: failed", "Half FFN, scale 0.5: failed", "Zero FFN, scale 0: failed" ], "intervention": { "tensor": "ffn_out-12", "layer_indexing": "zero-based", "applies_to": "all prompt and generated positions on each forward pass", "operation": "scale computed FFN output before residual addition", "scales": [ 1, 0.5, 0 ], "weights_modified": false, "kind": "activation_ablation", "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." }, "conditions": [ { "id": "baseline", "label": "Baseline, scale 1", "model": "Recursive Flint", "quant": "Q8_0", "output": "return n % 2 == 0", "metrics": { "generated_tokens": 8, "prompt_tokens": 48, "elapsed_seconds": 1.522581758, "request_wall_seconds": 1.5477090129861608, "time_to_first_token_seconds": 0.9909320520237088, "decode_tokens_per_second": 12.570319015535855, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004202357680490731, "block12_after_rms": 0.004202357680490731 }, "check": { "passed": false, "rule": "expression", "expected": "n % 2 == 0", "character_count": 17, "word_count": 6 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 963, "input_position": 47, "layers": [ { "after_rms": 0.004194344649342437, "before_rms": 0.004194344649342437, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.004202357680490731, "before_rms": 0.004202357680490731, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.0052131473290487635, "before_rms": 0.0052131473290487635, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.014781441026154475, "before_rms": 0.014781441026154475, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 393216 } ] }, { "step": 1, "token_id": 307, "input_position": 48, "layers": [ { "after_rms": 0.005277509304535801, "before_rms": 0.005277509304535801, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00609774510971192, "before_rms": 0.00609774510971192, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005608145566049825, "before_rms": 0.005608145566049825, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018110784621771632, "before_rms": 0.018110784621771632, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 1152, "input_position": 49, "layers": [ { "after_rms": 0.004954062696681225, "before_rms": 0.004954062696681225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004947633813264256, "before_rms": 0.004947633813264256, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006281951776179792, "before_rms": 0.006281951776179792, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01793926095231872, "before_rms": 0.01793926095231872, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 220, "input_position": 50, "layers": [ { "after_rms": 0.005725603347206012, "before_rms": 0.005725603347206012, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007613934635856348, "before_rms": 0.007613934635856348, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007067586106815484, "before_rms": 0.007067586106815484, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02065595811031347, "before_rms": 0.02065595811031347, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 17, "input_position": 51, "layers": [ { "after_rms": 0.004384558665618408, "before_rms": 0.004384558665618408, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002694924917014614, "before_rms": 0.002694924917014614, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003985706515046729, "before_rms": 0.003985706515046729, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.012855823082554165, "before_rms": 0.012855823082554165, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 823, "input_position": 52, "layers": [ { "after_rms": 0.00695481022115809, "before_rms": 0.00695481022115809, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005148284890432175, "before_rms": 0.005148284890432175, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00860817356635732, "before_rms": 0.00860817356635732, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019382560123796404, "before_rms": 0.019382560123796404, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 220, "input_position": 53, "layers": [ { "after_rms": 0.008360185779230117, "before_rms": 0.008360185779230117, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005331645591182674, "before_rms": 0.005331645591182674, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.011988298999222614, "before_rms": 0.011988298999222614, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02324778120292584, "before_rms": 0.02324778120292584, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 15, "input_position": 54, "layers": [ { "after_rms": 0.004389820760472024, "before_rms": 0.004389820760472024, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002823151948738866, "before_rms": 0.002823151948738866, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004278350453083995, "before_rms": 0.004278350453083995, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.013667174465740126, "before_rms": 0.013667174465740126, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "half", "label": "Half FFN, scale 0.5", "model": "Recursive Flint", "quant": "Q8_0", "output": "return n % 2 == 0", "metrics": { "generated_tokens": 8, "prompt_tokens": 48, "elapsed_seconds": 1.451376005, "request_wall_seconds": 1.5098812960786745, "time_to_first_token_seconds": 0.9736877980176359, "decode_tokens_per_second": 13.053039182374336, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004202357680490731, "block12_after_rms": 0.0021011788402453655 }, "check": { "passed": false, "rule": "expression", "expected": "n % 2 == 0", "character_count": 17, "word_count": 6 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0.5 }, "measurements": [ { "step": 0, "token_id": 963, "input_position": 47, "layers": [ { "after_rms": 0.004194344649342437, "before_rms": 0.004194344649342437, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.0021011788402453655, "before_rms": 0.004202357680490731, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.005090052356663985, "before_rms": 0.005090052356663985, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.014359352110858759, "before_rms": 0.014359352110858759, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 393216 } ] }, { "step": 1, "token_id": 307, "input_position": 48, "layers": [ { "after_rms": 0.005277509304535801, "before_rms": 0.005277509304535801, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00304887255485596, "before_rms": 0.00609774510971192, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006014683540483773, "before_rms": 0.006014683540483773, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017053751245143085, "before_rms": 0.017053751245143085, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 1152, "input_position": 49, "layers": [ { "after_rms": 0.004954062696681225, "before_rms": 0.004954062696681225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002473816906632128, "before_rms": 0.004947633813264256, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006547528545526086, "before_rms": 0.006547528545526086, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017097207015519773, "before_rms": 0.017097207015519773, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 220, "input_position": 50, "layers": [ { "after_rms": 0.005725603347206012, "before_rms": 0.005725603347206012, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003806967317928174, "before_rms": 0.007613934635856348, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007202666322056073, "before_rms": 0.007202666322056073, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019556659993829776, "before_rms": 0.019556659993829776, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 17, "input_position": 51, "layers": [ { "after_rms": 0.004384558665618408, "before_rms": 0.004384558665618408, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.001347462458507307, "before_rms": 0.002694924917014614, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004306118830807208, "before_rms": 0.004306118830807208, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.012367683992419722, "before_rms": 0.012367683992419722, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 823, "input_position": 52, "layers": [ { "after_rms": 0.00695481022115809, "before_rms": 0.00695481022115809, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0025741424452160877, "before_rms": 0.005148284890432175, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0085343766880633, "before_rms": 0.0085343766880633, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018658682254849446, "before_rms": 0.018658682254849446, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 220, "input_position": 53, "layers": [ { "after_rms": 0.008360185779230117, "before_rms": 0.008360185779230117, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002665822795591337, "before_rms": 0.005331645591182674, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.012100347801828254, "before_rms": 0.012100347801828254, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02291563837710362, "before_rms": 0.02291563837710362, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 15, "input_position": 54, "layers": [ { "after_rms": 0.004389820760472024, "before_rms": 0.004389820760472024, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.001411575974369433, "before_rms": 0.002823151948738866, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00448153573004611, "before_rms": 0.00448153573004611, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.012590223970782119, "before_rms": 0.012590223970782119, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "zero", "label": "Zero FFN, scale 0", "model": "Recursive Flint", "quant": "Q8_0", "output": "def is_even(n): return n % 2 == 0", "metrics": { "generated_tokens": 13, "prompt_tokens": 48, "elapsed_seconds": 1.766926317, "request_wall_seconds": 1.8241781580727547, "time_to_first_token_seconds": 0.9580211520660669, "decode_tokens_per_second": 13.853450095998406, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": false, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.004202357680490731, "block12_after_rms": 0.0 }, "check": { "passed": false, "rule": "expression", "expected": "n % 2 == 0", "character_count": 33, "word_count": 8 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0 }, "measurements": [ { "step": 0, "token_id": 1182, "input_position": 47, "layers": [ { "after_rms": 0.004194344649342437, "before_rms": 0.004194344649342437, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.0, "before_rms": 0.004202357680490731, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.005070940113459419, "before_rms": 0.005070940113459419, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 393216 }, { "after_rms": 0.014431911821101652, "before_rms": 0.014431911821101652, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 393216 } ] }, { "step": 1, "token_id": 332, "input_position": 48, "layers": [ { "after_rms": 0.00480106946797479, "before_rms": 0.00480106946797479, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0055027957166948055, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0056345791848020155, "before_rms": 0.0056345791848020155, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017990489860241182, "before_rms": 0.017990489860241182, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 75154, "input_position": 49, "layers": [ { "after_rms": 0.005708045099524496, "before_rms": 0.005708045099524496, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.004207955013596103, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0075556508119026065, "before_rms": 0.0075556508119026065, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017833711235606925, "before_rms": 0.017833711235606925, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 1249, "input_position": 50, "layers": [ { "after_rms": 0.0058045473429944065, "before_rms": 0.0058045473429944065, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005650785619108264, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007835386583253242, "before_rms": 0.007835386583253242, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018218131354517456, "before_rms": 0.018218131354517456, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1682, "input_position": 51, "layers": [ { "after_rms": 0.005459167018013679, "before_rms": 0.005459167018013679, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005458814458024571, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006723062471136803, "before_rms": 0.006723062471136803, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018003037284926235, "before_rms": 0.018003037284926235, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 597, "input_position": 52, "layers": [ { "after_rms": 0.004881552841111804, "before_rms": 0.004881552841111804, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.004638313561400321, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006991441598900912, "before_rms": 0.006991441598900912, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016203231535129245, "before_rms": 0.016203231535129245, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 307, "input_position": 53, "layers": [ { "after_rms": 0.005143842728252836, "before_rms": 0.005143842728252836, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005127108596991448, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008170598422530883, "before_rms": 0.008170598422530883, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01798307359808869, "before_rms": 0.01798307359808869, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 1152, "input_position": 54, "layers": [ { "after_rms": 0.0049824502938795555, "before_rms": 0.0049824502938795555, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005199642872045386, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006848563763133479, "before_rms": 0.006848563763133479, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016730393035201727, "before_rms": 0.016730393035201727, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 220, "input_position": 55, "layers": [ { "after_rms": 0.005708276925466525, "before_rms": 0.005708276925466525, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.008022970417094337, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007115667659550918, "before_rms": 0.007115667659550918, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019026613242409732, "before_rms": 0.019026613242409732, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 9, "token_id": 17, "input_position": 56, "layers": [ { "after_rms": 0.0042886941345147015, "before_rms": 0.0042886941345147015, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.002710862488091774, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004265423359902893, "before_rms": 0.004265423359902893, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01199161845757347, "before_rms": 0.01199161845757347, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 10, "token_id": 823, "input_position": 57, "layers": [ { "after_rms": 0.007280227848737238, "before_rms": 0.007280227848737238, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005472617971061407, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008876968970410364, "before_rms": 0.008876968970410364, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.018901080384531817, "before_rms": 0.018901080384531817, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 11, "token_id": 220, "input_position": 58, "layers": [ { "after_rms": 0.008286967782600638, "before_rms": 0.008286967782600638, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005098250235859169, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01039179921955715, "before_rms": 0.01039179921955715, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021144833704014477, "before_rms": 0.021144833704014477, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 12, "token_id": 15, "input_position": 59, "layers": [ { "after_rms": 0.004301028624991628, "before_rms": 0.004301028624991628, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.002916336479821133, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004546469810197431, "before_rms": 0.004546469810197431, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01185733829801787, "before_rms": 0.01185733829801787, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] } ], "comparison": { "untargeted_first_frame_block11_identical": true, "downstream_first_frame_block13_changed": true, "weights_modified": false, "all_saved_frames_available_in_results": true } }, { "id": "brief", "title": "One sentence explanation", "prompt": "Explain photosynthesis in exactly one sentence containing the words sunlight and glucose.", "checks": [ "Rule: sentence", "Expected: ['sunlight', 'glucose']", "Baseline, scale 1: passed", "Half FFN, scale 0.5: passed", "Zero FFN, scale 0: passed" ], "intervention": { "tensor": "ffn_out-12", "layer_indexing": "zero-based", "applies_to": "all prompt and generated positions on each forward pass", "operation": "scale computed FFN output before residual addition", "scales": [ 1, 0.5, 0 ], "weights_modified": false, "kind": "activation_ablation", "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." }, "conditions": [ { "id": "baseline", "label": "Baseline, scale 1", "model": "Recursive Flint", "quant": "Q8_0", "output": "Photosynthesis is the process by which plants use sunlight to create glucose, storing solar energy in chemical form.", "metrics": { "generated_tokens": 21, "prompt_tokens": 29, "elapsed_seconds": 1.985051171, "request_wall_seconds": 2.0436750209191814, "time_to_first_token_seconds": 0.6549666819628328, "decode_tokens_per_second": 14.402335769662937, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.0044633950081139125, "block12_after_rms": 0.0044633950081139125 }, "check": { "passed": true, "rule": "sentence", "expected": [ "sunlight", "glucose" ], "character_count": 116, "word_count": 18 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 1 }, "measurements": [ { "step": 0, "token_id": 49106, "input_position": 28, "layers": [ { "after_rms": 0.004260129892907466, "before_rms": 0.004260129892907466, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.0044633950081139125, "before_rms": 0.0044633950081139125, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.005860773055104808, "before_rms": 0.005860773055104808, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.015381996005573538, "before_rms": 0.015381996005573538, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 237568 } ] }, { "step": 1, "token_id": 25424, "input_position": 29, "layers": [ { "after_rms": 0.0056475146712665295, "before_rms": 0.0056475146712665295, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008613622622386482, "before_rms": 0.008613622622386482, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006688744656356641, "before_rms": 0.006688744656356641, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.022823769881939016, "before_rms": 0.022823769881939016, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 332, "input_position": 30, "layers": [ { "after_rms": 0.0064819400327529395, "before_rms": 0.0064819400327529395, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0066762037147183205, "before_rms": 0.0066762037147183205, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008751997501954894, "before_rms": 0.008751997501954894, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.027229842448171573, "before_rms": 0.027229842448171573, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 269, "input_position": 31, "layers": [ { "after_rms": 0.0045512880200197945, "before_rms": 0.0045512880200197945, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005065420297108188, "before_rms": 0.005065420297108188, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005762789431004507, "before_rms": 0.005762789431004507, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017768401646140505, "before_rms": 0.017768401646140505, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1260, "input_position": 32, "layers": [ { "after_rms": 0.004939956954683621, "before_rms": 0.004939956954683621, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0038142763724026384, "before_rms": 0.0038142763724026384, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005913005531372122, "before_rms": 0.005913005531372122, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01761890031507447, "before_rms": 0.01761890031507447, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 491, "input_position": 33, "layers": [ { "after_rms": 0.00603825473034925, "before_rms": 0.00603825473034925, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005399106143511229, "before_rms": 0.005399106143511229, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008736535756150335, "before_rms": 0.008736535756150335, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.023288508898713978, "before_rms": 0.023288508898713978, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 687, "input_position": 34, "layers": [ { "after_rms": 0.004825344575484225, "before_rms": 0.004825344575484225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0035722557633475504, "before_rms": 0.0035722557633475504, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004528394446715205, "before_rms": 0.004528394446715205, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015558510785105506, "before_rms": 0.015558510785105506, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 7370, "input_position": 35, "layers": [ { "after_rms": 0.00599941128253067, "before_rms": 0.00599941128253067, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00554379069270193, "before_rms": 0.00554379069270193, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00600245072054164, "before_rms": 0.00600245072054164, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021459060251264164, "before_rms": 0.021459060251264164, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 896, "input_position": 36, "layers": [ { "after_rms": 0.0051950504054689125, "before_rms": 0.0051950504054689125, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006141893723174439, "before_rms": 0.006141893723174439, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009373710872705807, "before_rms": 0.009373710872705807, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024138161560965755, "before_rms": 0.024138161560965755, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 9, "token_id": 24044, "input_position": 37, "layers": [ { "after_rms": 0.008031516167890848, "before_rms": 0.008031516167890848, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005368163828714329, "before_rms": 0.005368163828714329, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005839425814527791, "before_rms": 0.005839425814527791, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019011043273201134, "before_rms": 0.019011043273201134, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 10, "token_id": 295, "input_position": 38, "layers": [ { "after_rms": 0.009314847401164192, "before_rms": 0.009314847401164192, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00741898116192189, "before_rms": 0.00741898116192189, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00969530415712232, "before_rms": 0.00969530415712232, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02606507768278446, "before_rms": 0.02606507768278446, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 11, "token_id": 1815, "input_position": 39, "layers": [ { "after_rms": 0.0051360181463417694, "before_rms": 0.0051360181463417694, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005146656362487788, "before_rms": 0.005146656362487788, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005882866866859857, "before_rms": 0.005882866866859857, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017485786719118993, "before_rms": 0.017485786719118993, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 12, "token_id": 18175, "input_position": 40, "layers": [ { "after_rms": 0.004435782376176152, "before_rms": 0.004435782376176152, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007327068248162281, "before_rms": 0.007327068248162281, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006088986661020862, "before_rms": 0.006088986661020862, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020237327919821312, "before_rms": 0.020237327919821312, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 13, "token_id": 11, "input_position": 41, "layers": [ { "after_rms": 0.010509592928354338, "before_rms": 0.010509592928354338, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0072267658458743745, "before_rms": 0.0072267658458743745, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0072706489196035584, "before_rms": 0.0072706489196035584, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.026778168430377582, "before_rms": 0.026778168430377582, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 14, "token_id": 20827, "input_position": 42, "layers": [ { "after_rms": 0.0036465031366290737, "before_rms": 0.0036465031366290737, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003456715901650377, "before_rms": 0.003456715901650377, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005546723538161339, "before_rms": 0.005546723538161339, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.014684442530549374, "before_rms": 0.014684442530549374, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 15, "token_id": 8437, "input_position": 43, "layers": [ { "after_rms": 0.00628374835469256, "before_rms": 0.00628374835469256, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007316398861469263, "before_rms": 0.007316398861469263, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008040298007985535, "before_rms": 0.008040298007985535, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021912364803827578, "before_rms": 0.021912364803827578, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 16, "token_id": 2488, "input_position": 44, "layers": [ { "after_rms": 0.008596546950231054, "before_rms": 0.008596546950231054, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005564570157863948, "before_rms": 0.005564570157863948, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007757301988850941, "before_rms": 0.007757301988850941, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020669327020220094, "before_rms": 0.020669327020220094, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 17, "token_id": 293, "input_position": 45, "layers": [ { "after_rms": 0.011432653860628939, "before_rms": 0.011432653860628939, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005583320073300356, "before_rms": 0.005583320073300356, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009875868656773605, "before_rms": 0.009875868656773605, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024941505866581348, "before_rms": 0.024941505866581348, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 18, "token_id": 6999, "input_position": 46, "layers": [ { "after_rms": 0.0053301379768005384, "before_rms": 0.0053301379768005384, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005000846356165002, "before_rms": 0.005000846356165002, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005456690270258401, "before_rms": 0.005456690270258401, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016748723982626335, "before_rms": 0.016748723982626335, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 19, "token_id": 981, "input_position": 47, "layers": [ { "after_rms": 0.007946814942115127, "before_rms": 0.007946814942115127, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005158417494632955, "before_rms": 0.005158417494632955, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007460860282340885, "before_rms": 0.007460860282340885, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020411307476897476, "before_rms": 0.020411307476897476, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 20, "token_id": 13, "input_position": 48, "layers": [ { "after_rms": 0.005514051374901345, "before_rms": 0.005514051374901345, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0070394921880856, "before_rms": 0.0070394921880856, "layer": 12, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0066148112542871755, "before_rms": 0.0066148112542871755, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020639906847932543, "before_rms": 0.020639906847932543, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "half", "label": "Half FFN, scale 0.5", "model": "Recursive Flint", "quant": "Q8_0", "output": "Photosynthesis is the process by which plants use sunlight to create glucose, storing solar energy in chemical form.", "metrics": { "generated_tokens": 21, "prompt_tokens": 29, "elapsed_seconds": 2.160672568, "request_wall_seconds": 2.2178376889787614, "time_to_first_token_seconds": 0.6635361380176619, "decode_tokens_per_second": 12.866885314377939, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.0044633950081139125, "block12_after_rms": 0.0022316975040569563 }, "check": { "passed": true, "rule": "sentence", "expected": [ "sunlight", "glucose" ], "character_count": 116, "word_count": 18 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0.5 }, "measurements": [ { "step": 0, "token_id": 49106, "input_position": 28, "layers": [ { "after_rms": 0.004260129892907466, "before_rms": 0.004260129892907466, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.0022316975040569563, "before_rms": 0.0044633950081139125, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.005735151133850015, "before_rms": 0.005735151133850015, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.01486159838115774, "before_rms": 0.01486159838115774, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 237568 } ] }, { "step": 1, "token_id": 25424, "input_position": 29, "layers": [ { "after_rms": 0.0056475146712665295, "before_rms": 0.0056475146712665295, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004306811311193241, "before_rms": 0.008613622622386482, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007436545118137283, "before_rms": 0.007436545118137283, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020810310578413545, "before_rms": 0.020810310578413545, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 332, "input_position": 30, "layers": [ { "after_rms": 0.0064819400327529395, "before_rms": 0.0064819400327529395, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0033381018573591602, "before_rms": 0.0066762037147183205, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009007999001296582, "before_rms": 0.009007999001296582, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.026166335899804105, "before_rms": 0.026166335899804105, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 269, "input_position": 31, "layers": [ { "after_rms": 0.0045512880200197945, "before_rms": 0.0045512880200197945, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002532710148554094, "before_rms": 0.005065420297108188, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005911283538375086, "before_rms": 0.005911283538375086, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016813015849254234, "before_rms": 0.016813015849254234, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1260, "input_position": 32, "layers": [ { "after_rms": 0.004939956954683621, "before_rms": 0.004939956954683621, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0019071381862013192, "before_rms": 0.0038142763724026384, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00595557221350112, "before_rms": 0.00595557221350112, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017146784199957084, "before_rms": 0.017146784199957084, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 491, "input_position": 33, "layers": [ { "after_rms": 0.00603825473034925, "before_rms": 0.00603825473034925, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0026995530717556144, "before_rms": 0.005399106143511229, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008767531752803006, "before_rms": 0.008767531752803006, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.022466743651620685, "before_rms": 0.022466743651620685, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 687, "input_position": 34, "layers": [ { "after_rms": 0.004825344575484225, "before_rms": 0.004825344575484225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0017861278816737752, "before_rms": 0.0035722557633475504, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.004811283125730314, "before_rms": 0.004811283125730314, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.014872505126769303, "before_rms": 0.014872505126769303, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 7370, "input_position": 35, "layers": [ { "after_rms": 0.00599941128253067, "before_rms": 0.00599941128253067, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002771895346350965, "before_rms": 0.00554379069270193, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006169826908450906, "before_rms": 0.006169826908450906, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02094382532383083, "before_rms": 0.02094382532383083, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 896, "input_position": 36, "layers": [ { "after_rms": 0.0051950504054689125, "before_rms": 0.0051950504054689125, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0030709468615872193, "before_rms": 0.006141893723174439, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009589415026818995, "before_rms": 0.009589415026818995, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.023335435813653795, "before_rms": 0.023335435813653795, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 9, "token_id": 24044, "input_position": 37, "layers": [ { "after_rms": 0.008031516167890848, "before_rms": 0.008031516167890848, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0026840819143571647, "before_rms": 0.005368163828714329, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005801143981718462, "before_rms": 0.005801143981718462, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01859743623390119, "before_rms": 0.01859743623390119, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 10, "token_id": 295, "input_position": 38, "layers": [ { "after_rms": 0.009314847401164192, "before_rms": 0.009314847401164192, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.003709490580960945, "before_rms": 0.00741898116192189, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00977155465584999, "before_rms": 0.00977155465584999, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02491673002871025, "before_rms": 0.02491673002871025, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 11, "token_id": 1815, "input_position": 39, "layers": [ { "after_rms": 0.0051360181463417694, "before_rms": 0.0051360181463417694, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002573328181243894, "before_rms": 0.005146656362487788, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005799911492815732, "before_rms": 0.005799911492815732, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016933047785703814, "before_rms": 0.016933047785703814, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 12, "token_id": 18175, "input_position": 40, "layers": [ { "after_rms": 0.004435782376176152, "before_rms": 0.004435782376176152, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0036635341240811403, "before_rms": 0.007327068248162281, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00613417633614374, "before_rms": 0.00613417633614374, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019118337346322598, "before_rms": 0.019118337346322598, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 13, "token_id": 11, "input_position": 41, "layers": [ { "after_rms": 0.010509592928354338, "before_rms": 0.010509592928354338, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0036133829229371872, "before_rms": 0.0072267658458743745, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007475141941789141, "before_rms": 0.007475141941789141, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.025868505159618332, "before_rms": 0.025868505159618332, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 14, "token_id": 20827, "input_position": 42, "layers": [ { "after_rms": 0.0036465031366290737, "before_rms": 0.0036465031366290737, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0017283579508251886, "before_rms": 0.003456715901650377, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00533638225204982, "before_rms": 0.00533638225204982, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.014436190642373524, "before_rms": 0.014436190642373524, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 15, "token_id": 8437, "input_position": 43, "layers": [ { "after_rms": 0.00628374835469256, "before_rms": 0.00628374835469256, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0036581994307346316, "before_rms": 0.007316398861469263, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008102150834431271, "before_rms": 0.008102150834431271, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020984366848385224, "before_rms": 0.020984366848385224, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 16, "token_id": 2488, "input_position": 44, "layers": [ { "after_rms": 0.008596546950231054, "before_rms": 0.008596546950231054, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002782285078931974, "before_rms": 0.005564570157863948, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007806100403261486, "before_rms": 0.007806100403261486, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020226905469513747, "before_rms": 0.020226905469513747, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 17, "token_id": 293, "input_position": 45, "layers": [ { "after_rms": 0.011432653860628939, "before_rms": 0.011432653860628939, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002791660036650178, "before_rms": 0.005583320073300356, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009973486185624623, "before_rms": 0.009973486185624623, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024628251420628085, "before_rms": 0.024628251420628085, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 18, "token_id": 6999, "input_position": 46, "layers": [ { "after_rms": 0.0053301379768005384, "before_rms": 0.0053301379768005384, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.002500423178082501, "before_rms": 0.005000846356165002, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0053801173686054875, "before_rms": 0.0053801173686054875, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016217885602163445, "before_rms": 0.016217885602163445, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 19, "token_id": 981, "input_position": 47, "layers": [ { "after_rms": 0.007946814942115127, "before_rms": 0.007946814942115127, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0025792087473164773, "before_rms": 0.005158417494632955, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007857079141286923, "before_rms": 0.007857079141286923, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020573162868035226, "before_rms": 0.020573162868035226, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 20, "token_id": 13, "input_position": 48, "layers": [ { "after_rms": 0.005514051374901345, "before_rms": 0.005514051374901345, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0035197460940428, "before_rms": 0.0070394921880856, "layer": 12, "scale": 0.5, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006700234982153566, "before_rms": 0.006700234982153566, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019504194134602675, "before_rms": 0.019504194134602675, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] }, { "id": "zero", "label": "Zero FFN, scale 0", "model": "Recursive Flint", "quant": "Q8_0", "output": "Photosynthesis is the process by which plants use sunlight to convert carbon dioxide and water into glucose, releasing oxygen as a byproduct.", "metrics": { "generated_tokens": 25, "prompt_tokens": 29, "elapsed_seconds": 2.391091285, "request_wall_seconds": 2.459595420048572, "time_to_first_token_seconds": 0.7541828820249066, "decode_tokens_per_second": 14.071746536403907, "process_peak_rss_kib": 41801304, "termination": "eos", "complete": true, "objective_check_passed": true, "checkpoint_bytes": 42350359392, "block12_before_rms": 0.0044633950081139125, "block12_after_rms": 0.0 }, "check": { "passed": true, "rule": "sentence", "expected": [ "sunlight", "glucose" ], "character_count": 141, "word_count": 22 }, "settings": { "max_tokens": 64, "temperature": 0, "seed": 42, "reasoning": false, "intervention_layer": 12, "intervention_scale": 0 }, "measurements": [ { "step": 0, "token_id": 49106, "input_position": 28, "layers": [ { "after_rms": 0.004260129892907466, "before_rms": 0.004260129892907466, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.0, "before_rms": 0.0044633950081139125, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.005661502074284029, "before_rms": 0.005661502074284029, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 237568 }, { "after_rms": 0.014899536247766184, "before_rms": 0.014899536247766184, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 237568 } ] }, { "step": 1, "token_id": 25424, "input_position": 29, "layers": [ { "after_rms": 0.0056475146712665295, "before_rms": 0.0056475146712665295, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.008613622622386482, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008525602935404572, "before_rms": 0.008525602935404572, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019733810862655715, "before_rms": 0.019733810862655715, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 2, "token_id": 332, "input_position": 30, "layers": [ { "after_rms": 0.0064819400327529395, "before_rms": 0.0064819400327529395, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0066762037147183205, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009135572197523601, "before_rms": 0.009135572197523601, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02557741137246633, "before_rms": 0.02557741137246633, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 3, "token_id": 269, "input_position": 31, "layers": [ { "after_rms": 0.0045512880200197945, "before_rms": 0.0045512880200197945, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005065420297108188, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006092569880549685, "before_rms": 0.006092569880549685, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.016503642792897465, "before_rms": 0.016503642792897465, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 4, "token_id": 1260, "input_position": 32, "layers": [ { "after_rms": 0.004939956954683621, "before_rms": 0.004939956954683621, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0038142763724026384, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006157849949139536, "before_rms": 0.006157849949139536, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017203518557001942, "before_rms": 0.017203518557001942, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 5, "token_id": 491, "input_position": 33, "layers": [ { "after_rms": 0.00603825473034925, "before_rms": 0.00603825473034925, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005399106143511229, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00904802642504343, "before_rms": 0.00904802642504343, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02225766191781135, "before_rms": 0.02225766191781135, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 6, "token_id": 687, "input_position": 34, "layers": [ { "after_rms": 0.004825344575484225, "before_rms": 0.004825344575484225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.0035722557633475504, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005533683753255847, "before_rms": 0.005533683753255847, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.014873267868110489, "before_rms": 0.014873267868110489, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 7, "token_id": 7370, "input_position": 35, "layers": [ { "after_rms": 0.00599941128253067, "before_rms": 0.00599941128253067, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.00554379069270193, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006390727713863449, "before_rms": 0.006390727713863449, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02091061618332828, "before_rms": 0.02091061618332828, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 8, "token_id": 896, "input_position": 36, "layers": [ { "after_rms": 0.0051950504054689125, "before_rms": 0.0051950504054689125, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.006141893723174439, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009947159109400337, "before_rms": 0.009947159109400337, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.023151455733089174, "before_rms": 0.023151455733089174, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 9, "token_id": 24044, "input_position": 37, "layers": [ { "after_rms": 0.008031516167890848, "before_rms": 0.008031516167890848, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005368163828714329, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006423750612719223, "before_rms": 0.006423750612719223, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.019295778586505254, "before_rms": 0.019295778586505254, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 10, "token_id": 295, "input_position": 38, "layers": [ { "after_rms": 0.009314847401164192, "before_rms": 0.009314847401164192, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.00741898116192189, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.009693459099318338, "before_rms": 0.009693459099318338, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024237413637575162, "before_rms": 0.024237413637575162, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 11, "token_id": 6766, "input_position": 39, "layers": [ { "after_rms": 0.0051360181463417694, "before_rms": 0.0051360181463417694, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005146656362487788, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005767883251008539, "before_rms": 0.005767883251008539, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.01708687035677014, "before_rms": 0.01708687035677014, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 12, "token_id": 7185, "input_position": 40, "layers": [ { "after_rms": 0.0073586153911031, "before_rms": 0.0073586153911031, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.006591188015659504, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0070010157973419635, "before_rms": 0.0070010157973419635, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.022579139951650742, "before_rms": 0.022579139951650742, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 13, "token_id": 23776, "input_position": 41, "layers": [ { "after_rms": 0.008194359117077644, "before_rms": 0.008194359117077644, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.007596196041918696, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0075531294632927015, "before_rms": 0.0075531294632927015, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021887967030610505, "before_rms": 0.021887967030610505, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 14, "token_id": 299, "input_position": 42, "layers": [ { "after_rms": 0.007857027218294529, "before_rms": 0.007857027218294529, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.008314784468177716, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.008147101435200647, "before_rms": 0.008147101435200647, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.02264674988179593, "before_rms": 0.02264674988179593, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 15, "token_id": 1952, "input_position": 43, "layers": [ { "after_rms": 0.004688345103889225, "before_rms": 0.004688345103889225, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.004946909109298921, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006079777076870683, "before_rms": 0.006079777076870683, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015899963838293046, "before_rms": 0.015899963838293046, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 16, "token_id": 831, "input_position": 44, "layers": [ { "after_rms": 0.007450920463116443, "before_rms": 0.007450920463116443, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.006978048184271669, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006659637268169784, "before_rms": 0.006659637268169784, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020155346635423232, "before_rms": 0.020155346635423232, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 17, "token_id": 18175, "input_position": 45, "layers": [ { "after_rms": 0.007443990344351715, "before_rms": 0.007443990344351715, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005543317793659353, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.00888447540290056, "before_rms": 0.00888447540290056, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.020591210203462766, "before_rms": 0.020591210203462766, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 18, "token_id": 11, "input_position": 46, "layers": [ { "after_rms": 0.009927458171845966, "before_rms": 0.009927458171845966, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.006904034655866483, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007498876939890208, "before_rms": 0.007498876939890208, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024867085464211167, "before_rms": 0.024867085464211167, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 19, "token_id": 27762, "input_position": 47, "layers": [ { "after_rms": 0.003583892367306353, "before_rms": 0.003583892367306353, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.003674512887912858, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.005366533667651147, "before_rms": 0.005366533667651147, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.015057416354372646, "before_rms": 0.015057416354372646, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 20, "token_id": 11472, "input_position": 48, "layers": [ { "after_rms": 0.0071015945130851466, "before_rms": 0.0071015945130851466, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.006051436756781676, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006862596596600148, "before_rms": 0.006862596596600148, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021336925675778752, "before_rms": 0.021336925675778752, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 21, "token_id": 389, "input_position": 49, "layers": [ { "after_rms": 0.007580029165171899, "before_rms": 0.007580029165171899, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.006902773819282416, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007204155372984763, "before_rms": 0.007204155372984763, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.021924978969430717, "before_rms": 0.021924978969430717, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 22, "token_id": 260, "input_position": 50, "layers": [ { "after_rms": 0.00546253050622268, "before_rms": 0.00546253050622268, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.005206685222875008, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.006133790801518977, "before_rms": 0.006133790801518977, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.017978821140417742, "before_rms": 0.017978821140417742, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 23, "token_id": 93485, "input_position": 51, "layers": [ { "after_rms": 0.00462712758092533, "before_rms": 0.00462712758092533, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.004455167000851032, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0069830694392977915, "before_rms": 0.0069830694392977915, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0157750242951739, "before_rms": 0.0157750242951739, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] }, { "step": 24, "token_id": 13, "input_position": 52, "layers": [ { "after_rms": 0.010873441338987835, "before_rms": 0.010873441338987835, "layer": 11, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.0, "before_rms": 0.006011455301735444, "layer": 12, "scale": 0.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.007365985817524404, "before_rms": 0.007365985817524404, "layer": 13, "scale": 1.0, "tensor": "ffn_out", "tensor_bytes": 8192 }, { "after_rms": 0.024886336584602996, "before_rms": 0.024886336584602996, "layer": 13, "scale": 1.0, "tensor": "l_out", "tensor_bytes": 8192 } ] } ] } ], "comparison": { "untargeted_first_frame_block11_identical": true, "downstream_first_frame_block13_changed": true, "weights_modified": false, "all_saved_frames_available_in_results": true } } ] }, "memory": { "title": "Remembered, but outdated?", "summary": "Six short fictional logs test current facts and original facts. Flint and alphaBell Qwen Open answered all six correctly. Predict the answer, then inspect each recorded result.", "method": "Three log structures, each queried for the current and original access code. Exact four-digit answer scoring. Same decoding settings for both models.", "limitations": [ "These prompts are at most 144 tokens. They test short in-context correction and recall, not long-context performance or permanent memory." ], "cases": [ { "id": "distractors-current", "title": "A correction among distractors: current fact", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", "question": "What is the current museum access code?", "ground_truth": "8624", "choices": [ "4319", "8624", "5972" ], "kind": "current", "conditions": [ { "id": "flint", "label": "Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", "output": "8624", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=memory-distractors-current-flint&token=1", "metrics": { "generated_tokens": 2, "prompt_tokens": 126, "elapsed_seconds": 1.662128068, "termination": "eos", "exact_answer_passed": true } }, { "id": "qwen", "label": "alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", "output": "8624", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=memory-distractors-current-qwen&token=1", "metrics": { "generated_tokens": 4, "prompt_tokens": 144, "elapsed_seconds": 10.655367113, "termination": "eos", "exact_answer_passed": true } } ], "checks": [ "Expected four-digit answer: 8624", "Recursive Flint: passed", "alphaBell Qwen Open 27B: passed" ] }, { "id": "distractors-original", "title": "A correction among distractors: original fact", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", "question": "What is the original museum access code?", "ground_truth": "4319", "choices": [ "4319", "8624", "5972" ], "kind": "original", "conditions": [ { "id": "flint", "label": "Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", "output": "4319", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=memory-distractors-original-flint&token=1", "metrics": { "generated_tokens": 2, "prompt_tokens": 126, "elapsed_seconds": 1.715333822, "termination": "eos", "exact_answer_passed": true } }, { "id": "qwen", "label": "alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", "output": "4319", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=memory-distractors-original-qwen&token=1", "metrics": { "generated_tokens": 4, "prompt_tokens": 144, "elapsed_seconds": 10.655514738, "termination": "eos", "exact_answer_passed": true } } ], "checks": [ "Expected four-digit answer: 4319", "Recursive Flint: passed", "alphaBell Qwen Open 27B: passed" ] }, { "id": "short-current", "title": "One correction: current fact", "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", "question": "What is the current museum access code?", "ground_truth": "8624", "choices": [ "4319", "8624", "5972" ], "kind": "current", "conditions": [ { "id": "flint", "label": "Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", "output": "8624", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=memory-short-current-flint&token=1", "metrics": { "generated_tokens": 2, "prompt_tokens": 67, "elapsed_seconds": 1.14846904, "termination": "eos", "exact_answer_passed": true } }, { "id": "qwen", "label": "alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", "output": "8624", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=memory-short-current-qwen&token=1", "metrics": { "generated_tokens": 4, "prompt_tokens": 85, "elapsed_seconds": 7.286205131, "termination": "eos", "exact_answer_passed": true } } ], "checks": [ "Expected four-digit answer: 8624", "Recursive Flint: passed", "alphaBell Qwen Open 27B: passed" ] }, { "id": "short-original", "title": "One correction: original fact", "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", "question": "What is the original museum access code?", "ground_truth": "4319", "choices": [ "4319", "8624", "5972" ], "kind": "original", "conditions": [ { "id": "flint", "label": "Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", "output": "4319", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=memory-short-original-flint&token=1", "metrics": { "generated_tokens": 2, "prompt_tokens": 67, "elapsed_seconds": 1.1675653, "termination": "eos", "exact_answer_passed": true } }, { "id": "qwen", "label": "alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", "output": "4319", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=memory-short-original-qwen&token=1", "metrics": { "generated_tokens": 4, "prompt_tokens": 85, "elapsed_seconds": 7.234881181, "termination": "eos", "exact_answer_passed": true } } ], "checks": [ "Expected four-digit answer: 4319", "Recursive Flint: passed", "alphaBell Qwen Open 27B: passed" ] }, { "id": "twice-current", "title": "Two successive corrections: current fact", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the current museum access code? Answer with only the four digits.", "question": "What is the current museum access code?", "ground_truth": "5972", "choices": [ "4319", "8624", "5972" ], "kind": "current", "conditions": [ { "id": "flint", "label": "Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the current museum access code? Answer with only the four digits.", "output": "5972", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=memory-twice-current-flint&token=1", "metrics": { "generated_tokens": 2, "prompt_tokens": 105, "elapsed_seconds": 1.501386008, "termination": "eos", "exact_answer_passed": true } }, { "id": "qwen", "label": "alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the current museum access code? Answer with only the four digits.", "output": "5972", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=memory-twice-current-qwen&token=1", "metrics": { "generated_tokens": 4, "prompt_tokens": 125, "elapsed_seconds": 9.111306724, "termination": "eos", "exact_answer_passed": true } } ], "checks": [ "Expected four-digit answer: 5972", "Recursive Flint: passed", "alphaBell Qwen Open 27B: passed" ] }, { "id": "twice-original", "title": "Two successive corrections: original fact", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the original museum access code? Answer with only the four digits.", "question": "What is the original museum access code?", "ground_truth": "4319", "choices": [ "4319", "8624", "5972" ], "kind": "original", "conditions": [ { "id": "flint", "label": "Recursive Flint", "model": "Recursive Flint", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the original museum access code? Answer with only the four digits.", "output": "4319", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=flint&capture=memory-twice-original-flint&token=1", "metrics": { "generated_tokens": 2, "prompt_tokens": 105, "elapsed_seconds": 1.648641691, "termination": "eos", "exact_answer_passed": true } }, { "id": "qwen", "label": "alphaBell Qwen Open 27B", "model": "alphaBell Qwen Open 27B", "quant": "Q8_0", "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the original museum access code? Answer with only the four digits.", "output": "4319", "settings": { "max_tokens": 16, "temperature": 0, "seed": 42, "reasoning": false }, "trace_url": "../index.html?model=qwen&capture=memory-twice-original-qwen&token=1", "metrics": { "generated_tokens": 4, "prompt_tokens": 125, "elapsed_seconds": 9.107945102, "termination": "eos", "exact_answer_passed": true } } ], "checks": [ "Expected four-digit answer: 4319", "Recursive Flint: passed", "alphaBell Qwen Open 27B: passed" ] } ] } } }