diff --git "a/experiments/data/results.json" "b/experiments/data/results.json" new file mode 100644--- /dev/null +++ "b/experiments/data/results.json" @@ -0,0 +1,15279 @@ +{ + "schema_version": 1, + "title": "Five alphaBell model experiments", + "generated_at": "2026-10-23T11:17:02.577204+00:00", + "methodology": { + "summary": "Real measured model runs, retained in full for this fixed case set. The public interface replays these results.", + "limitations": [ + "Each condition has one greedy generation. Small demonstrations do not establish general capabilities or population preferences.", + "Qwen captures use the alphaBell fine-tuned derivative, not the untouched Qwen parent.", + "Native experiment source and build recipe accompany the measurements." + ], + "reproduce": [ + "Download the prompts and results.", + "Use the pinned engine build and experiment runner in experiments/reproduce.", + "Match checkpoint SHA-256 values and recorded settings before comparing outputs." + ] + }, + "provenance": { + "engine": "llama.cpp 74686f084c53506a7afa471d857db02e5c912543", + "device": "CPU", + "threads": 8, + "nice": 10, + "context": 2048, + "flash_attention": false, + "kv_cache": "default F16 attention KV; model-native recurrent state", + "sampling": { + "temperature": 0, + "seed": 42, + "max_tokens": 64, + "reasoning": false + }, + "intervention": { + "tensor": "ffn_out-12", + "layer_indexing": "zero-based", + "applies_to": "all prompt and generated positions on each forward pass", + "operation": "scale computed FFN output before residual addition", + "scales": [ + 1, + 0.5, + 0 + ], + "weights_modified": false + }, + "memory_metric": "ru_maxrss process high-water mark, includes model loading and warm-up; KiB on Linux", + "latency_metric": "native monotonic request elapsed plus client wall clock; first-token latency includes prompt prefill", + "created_at": "2026-10-23T10:57:11.012459+00:00", + "source_sha256": "4a29fca44df4174dc7db51a70d7bdc0d0f27ba06d18ed4338c9da8db81e4bda9", + "binary_sha256": "4b733ebd42d2b88ecfdaf527915327e17734fad04bc7f35846fc5c9ea8f6e55d", + "checkpoints": [ + { + "model": "flint", + "quantization": "Q8_0", + "file_bytes": 42350359392, + "architecture_sha256": "1dcc4eb75752f7847b49a6cab17e377c0282050b81bb0603d33a5271112bd058", + "sha256": "f97ff2fc77564a05919357685ba8e1892bec279cf733fdb6e3ef0ea8c8e45543", + "gguf_metadata": { + "general.architecture": "flint", + "general.type": "model", + "general.sampling.top_k": 20, + "general.sampling.top_p": 0.949999988079071, + "general.sampling.temp": 1.0, + "general.name": "Flint Bf16", + "general.size_label": "256x2.1B", + "general.quantization_version": 2, + "general.file_type": 7 + }, + "tensor_count": 779 + }, + { + "model": "flint", + "quantization": "Q4_1", + "file_bytes": 24955171680, + "architecture_sha256": "1dcc4eb75752f7847b49a6cab17e377c0282050b81bb0603d33a5271112bd058", + "sha256": "1d8ea8d2fd133baa54a7412d56d39d20499a9efc080b45486f23853e3f072143", + "gguf_metadata": { + "general.architecture": "flint", + "general.type": "model", + "general.sampling.top_k": 20, + "general.sampling.top_p": 0.949999988079071, + "general.sampling.temp": 1.0, + "general.name": "Flint Bf16", + "general.size_label": "256x2.1B", + "general.quantization_version": 2, + "general.file_type": 3 + }, + "tensor_count": 779 + }, + { + "model": "qwen", + "quantization": "Q8_0", + "file_bytes": 28595763712, + "architecture_sha256": "2b58cd217c64118083a3d622cdf9c9b7488adb7bb12bc114762322062bb365c3", + "sha256": "b2a3172342ff220a8f9a06fc2fd252d1e84615ce8dfc00b2a66941610aa8f43c", + "gguf_metadata": { + "general.architecture": "qwen35", + "general.type": "model", + "general.sampling.top_k": 20, + "general.sampling.top_p": 0.949999988079071, + "general.sampling.temp": 1.0, + "general.name": "Qwen3.8 27B Open", + "general.finetune": "open", + "general.basename": "Qwen3.8", + "general.size_label": "27B", + "general.quantization_version": 2, + "general.file_type": 7 + }, + "tensor_count": 851 + }, + { + "model": "qwen", + "quantization": "Q4_1", + "file_bytes": 16829921792, + "architecture_sha256": "2b58cd217c64118083a3d622cdf9c9b7488adb7bb12bc114762322062bb365c3", + "sha256": "7c0814054c69120ff0e5b3ac3f625f98a9f86185fc39f5d71eae02cb54eb5825", + "gguf_metadata": { + "general.architecture": "qwen35", + "general.type": "model", + "general.sampling.top_k": 20, + "general.sampling.top_p": 0.949999988079071, + "general.sampling.temp": 1.0, + "general.name": "Qwen3.8 27B Open", + "general.finetune": "open", + "general.basename": "Qwen3.8", + "general.size_label": "27B", + "general.quantization_version": 2, + "general.file_type": 3 + }, + "tensor_count": 851 + } + ], + "completed_at": "2026-10-23T11:00:04.436046+00:00", + "source_tree": "83e9d4c276f84753cce063502a2a9e2325f15ce3", + "public_base_commit": "0a1ef309cdbb9f2024f58a38fb4c42352c5db569", + "source_patch_sha256": "87a1aca766c7584c433093e82e6fbfe21caa4578e8f1c1faca5b8e2385ee664d", + "header_privacy_audit": { + "passed": true, + "metadata_entries_scanned_per_file": 41, + "excluded_keys": [ + "tokenizer.ggml.tokens" + ], + "source_urls_found": 0, + "identity_matches": 0 + }, + "flint_capture_compatibility": { + "tensor_payload_equal": true, + "compared_bytes": 42344889472, + "original_offset": 5469920, + "matched_offset": 5469920, + "original_container_sha256": "147f02266f809e7bb4b28fdeef9606520d9693b601f8bc0d817caf11a2ed1e5b", + "matched_container_sha256": "f97ff2fc77564a05919357685ba8e1892bec279cf733fdb6e3ef0ea8c8e45543", + "tokenizer_equal": true, + "only_metadata_differences": [ + { + "key": "general.name", + "original": "Recursive Flint", + "matched": "Flint Bf16" + }, + { + "key": "general.size_label", + "original": "40B", + "matched": "256x2.1B" + } + ] + }, + "hardware": "AMD EPYC 7B13 CPU. Native quantization/intervention runner: 8 threads. Full topology captures: 16 threads. CPU-only.", + "model_sources": [ + { + "name": "Recursive Flint, source BF16", + "url": "https://huggingface.co/RecursiveSI/flint", + "revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" + }, + { + "name": "alphaBell Qwen Open, related public FP8 sibling", + "url": "https://huggingface.co/alphabell/alphaBell-Qwen3.8-27B-Open-FP8", + "revision": "38595c77e73f8d04afa003510dadd35c750590ef" + }, + { + "name": "Qwen parent architecture configuration", + "url": "https://huggingface.co/Qwen/Qwen3.8-27B", + "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" + } + ], + "qwen_checkpoint_origin": "alphaBell fine-tuned Qwen3.8-27B, converted from the lab BF16 checkpoint to GGUF; the public FP8 release is a sibling format.", + "topology_capture_checkpoints": { + "flint": { + "quantization": "Q8_0", + "container_sha256": "147f02266f809e7bb4b28fdeef9606520d9693b601f8bc0d817caf11a2ed1e5b", + "matched_tensor_payload_identical": true, + "proof": "https://huggingface.co/spaces/alphabell/Model-Observatory/resolve/main/experiments/reproduce/flint-capture-equivalence.json" + }, + "qwen": { + "quantization": "Q8_0", + "container_sha256": "b2a3172342ff220a8f9a06fc2fd252d1e84615ce8dfc00b2a66941610aa8f43c" + } + } + }, + "experiments": { + "reroute": { + "title": "One word changed.", + "summary": "Three prompt pairs, two architectures, twelve complete captures. Inspect the first output prediction before continuations diverge.", + "method": "Change one word with temperature 0, seed 42 and reasoning disabled. Compare the same model and precision. The displayed Flint block is the lowest block with a changed first-frame expert set.", + "limitations": [ + "Later output positions are ordinal, not semantic alignments. Routing changes do not localize concepts or establish a causal explanation of an answer." + ], + "cases": [ + { + "id": "json-color-flint", + "title": "One word changes the JSON value · Flint", + "prompts": [ + { + "id": "a", + "label": "Prompt A", + "text": "Return only JSON with one key named color and value blue." + }, + { + "id": "b", + "label": "Prompt B", + "text": "Return only JSON with one key named color and value red." + } + ], + "conditions": [ + { + "id": "a", + "label": "Prompt A · Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Return only JSON with one key named color and value blue.", + "output": "{\"color\":\"blue\"}", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=reroute-json-color-flint-a&token=1", + "metrics": { + "generated_tokens": 5, + "prompt_tokens": 28, + "elapsed_seconds": 1.124171917, + "termination": "eos", + "displayed_block": 3 + }, + "routing": { + "layer": 3, + "expert_count": 256, + "description": "Measured selected experts in zero-based block 3.", + "tokens": [ + { + "token": "{\"", + "experts": [ + 117, + 87, + 35, + 5, + 168, + 84 + ], + "weights": [ + 0.32084164023399353, + 0.2539198398590088, + 0.14681749045848846, + 0.12157176434993744, + 0.10437823086977005, + 0.052470993250608444 + ] + }, + { + "token": "color", + "experts": [ + 39, + 59, + 143, + 13, + 34, + 250 + ], + "weights": [ + 0.33017292618751526, + 0.30293041467666626, + 0.1303112655878067, + 0.10549050569534302, + 0.09110484272241592, + 0.03999003767967224 + ] + }, + { + "token": "\":\"", + "experts": [ + 115, + 224, + 186, + 216, + 15, + 29 + ], + "weights": [ + 0.6117783784866333, + 0.09889795631170273, + 0.09107732027769089, + 0.07742965966463089, + 0.06268590688705444, + 0.05813080072402954 + ] + }, + { + "token": "blue", + "experts": [ + 39, + 111, + 197, + 137, + 119, + 28 + ], + "weights": [ + 0.4875814616680145, + 0.17773117125034332, + 0.10596126317977905, + 0.09250641614198685, + 0.08542586863040924, + 0.05079379305243492 + ] + }, + { + "token": "\"}", + "experts": [ + 216, + 1, + 152, + 102, + 182, + 15 + ], + "weights": [ + 0.38251131772994995, + 0.3607138693332672, + 0.07649347931146622, + 0.06965707242488861, + 0.06540820747613907, + 0.04521603509783745 + ] + } + ] + } + }, + { + "id": "b", + "label": "Prompt B · Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Return only JSON with one key named color and value red.", + "output": "{\"color\":\"red\"}", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=reroute-json-color-flint-b&token=1", + "metrics": { + "generated_tokens": 5, + "prompt_tokens": 28, + "elapsed_seconds": 1.116444861, + "termination": "eos", + "displayed_block": 3 + }, + "routing": { + "layer": 3, + "expert_count": 256, + "description": "Measured selected experts in zero-based block 3.", + "tokens": [ + { + "token": "{\"", + "experts": [ + 117, + 87, + 35, + 5, + 168, + 33 + ], + "weights": [ + 0.3200564980506897, + 0.25333839654922485, + 0.1516059786081314, + 0.11339818686246872, + 0.10805189609527588, + 0.05354905501008034 + ] + }, + { + "token": "color", + "experts": [ + 39, + 59, + 143, + 13, + 34, + 250 + ], + "weights": [ + 0.3271667957305908, + 0.29962509870529175, + 0.1341659426689148, + 0.1078995019197464, + 0.0922822579741478, + 0.03886038064956665 + ] + }, + { + "token": "\":\"", + "experts": [ + 115, + 224, + 186, + 216, + 15, + 29 + ], + "weights": [ + 0.6083868145942688, + 0.101140096783638, + 0.0891016498208046, + 0.07671405375003815, + 0.06320527195930481, + 0.06145205348730087 + ] + }, + { + "token": "red", + "experts": [ + 39, + 111, + 197, + 137, + 119, + 28 + ], + "weights": [ + 0.48268941044807434, + 0.1775527149438858, + 0.10889095813035965, + 0.09247402846813202, + 0.08805633336305618, + 0.05033653602004051 + ] + }, + { + "token": "\"}", + "experts": [ + 1, + 6, + 199, + 216, + 48, + 187 + ], + "weights": [ + 0.3797229826450348, + 0.31145796179771423, + 0.13263653218746185, + 0.08601834625005722, + 0.04618379473686218, + 0.043980374932289124 + ] + } + ] + } + } + ], + "comparison": { + "comparison_frame": "First output prediction, before generated continuations diverge.", + "prompt_words_changed": 1, + "prompt_token_counts": [ + 28, + 28 + ], + "routed_blocks_compared": 47, + "blocks_with_different_expert_sets": 18, + "expert_selections_in_each_first_frame": 282, + "shared_expert_selections": 263, + "displayed_zero_based_block": 3, + "display_rule": "Lowest routed block with a different first-frame expert set; full captures contain every block.", + "first_frame_head_rms_relative_l2": 0.10240989178622689 + } + }, + { + "id": "json-color-qwen", + "title": "One word changes the JSON value · Qwen Open", + "prompts": [ + { + "id": "a", + "label": "Prompt A", + "text": "Return only JSON with one key named color and value blue." + }, + { + "id": "b", + "label": "Prompt B", + "text": "Return only JSON with one key named color and value red." + } + ], + "conditions": [ + { + "id": "a", + "label": "Prompt A · alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Return only JSON with one key named color and value blue.", + "output": "{\"color\":\"blue\"}", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=reroute-json-color-qwen-a&token=1", + "metrics": { + "generated_tokens": 5, + "prompt_tokens": 42, + "elapsed_seconds": 5.535615226, + "termination": "eos", + "displayed_block": 0 + } + }, + { + "id": "b", + "label": "Prompt B · alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Return only JSON with one key named color and value red.", + "output": "{\"color\":\"red\"}", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=reroute-json-color-qwen-b&token=1", + "metrics": { + "generated_tokens": 5, + "prompt_tokens": 42, + "elapsed_seconds": 5.440314309, + "termination": "eos", + "displayed_block": 0 + } + } + ], + "comparison": { + "comparison_frame": "First output prediction, before generated continuations diverge.", + "prompt_words_changed": 1, + "prompt_token_counts": [ + 42, + 42 + ], + "first_frame_head_rms_relative_l2": 0.02018521900284211 + } + }, + { + "id": "opposite-flint", + "title": "One word changes the opposite · Flint", + "prompts": [ + { + "id": "a", + "label": "Prompt A", + "text": "Answer with one word: name the opposite of hot." + }, + { + "id": "b", + "label": "Prompt B", + "text": "Answer with one word: name the opposite of cold." + } + ], + "conditions": [ + { + "id": "a", + "label": "Prompt A · Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Answer with one word: name the opposite of hot.", + "output": "Cold", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=reroute-opposite-flint-a&token=1", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 27, + "elapsed_seconds": 0.695859422, + "termination": "eos", + "displayed_block": 2 + }, + "routing": { + "layer": 2, + "expert_count": 256, + "description": "Measured selected experts in zero-based block 2.", + "tokens": [ + { + "token": "Cold", + "experts": [ + 88, + 213, + 186, + 46, + 162, + 237 + ], + "weights": [ + 0.5531415939331055, + 0.178231343626976, + 0.09052453190088272, + 0.08529653400182724, + 0.04749872162938118, + 0.045307233929634094 + ] + } + ] + } + }, + { + "id": "b", + "label": "Prompt B · Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Answer with one word: name the opposite of cold.", + "output": "Hot", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=reroute-opposite-flint-b&token=1", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 27, + "elapsed_seconds": 0.644852095, + "termination": "eos", + "displayed_block": 2 + }, + "routing": { + "layer": 2, + "expert_count": 256, + "description": "Measured selected experts in zero-based block 2.", + "tokens": [ + { + "token": "Hot", + "experts": [ + 88, + 213, + 186, + 46, + 162, + 1 + ], + "weights": [ + 0.5620047450065613, + 0.17373046278953552, + 0.09409793466329575, + 0.07710109651088715, + 0.047040797770023346, + 0.046024974435567856 + ] + } + ] + } + } + ], + "comparison": { + "comparison_frame": "First output prediction, before generated continuations diverge.", + "prompt_words_changed": 1, + "prompt_token_counts": [ + 27, + 27 + ], + "routed_blocks_compared": 47, + "blocks_with_different_expert_sets": 25, + "expert_selections_in_each_first_frame": 282, + "shared_expert_selections": 243, + "displayed_zero_based_block": 2, + "display_rule": "Lowest routed block with a different first-frame expert set; full captures contain every block.", + "first_frame_head_rms_relative_l2": 0.25766445665876486 + } + }, + { + "id": "opposite-qwen", + "title": "One word changes the opposite · Qwen Open", + "prompts": [ + { + "id": "a", + "label": "Prompt A", + "text": "Answer with one word: name the opposite of hot." + }, + { + "id": "b", + "label": "Prompt B", + "text": "Answer with one word: name the opposite of cold." + } + ], + "conditions": [ + { + "id": "a", + "label": "Prompt A · alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Answer with one word: name the opposite of hot.", + "output": "Cold", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=reroute-opposite-qwen-a&token=1", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 41, + "elapsed_seconds": 3.133467638, + "termination": "eos", + "displayed_block": 0 + } + }, + { + "id": "b", + "label": "Prompt B · alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Answer with one word: name the opposite of cold.", + "output": "Hot", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=reroute-opposite-qwen-b&token=1", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 41, + "elapsed_seconds": 3.118574647, + "termination": "eos", + "displayed_block": 0 + } + } + ], + "comparison": { + "comparison_frame": "First output prediction, before generated continuations diverge.", + "prompt_words_changed": 1, + "prompt_token_counts": [ + 41, + 41 + ], + "first_frame_head_rms_relative_l2": 0.18418391836536593 + } + }, + { + "id": "python-flint", + "title": "One word changes the function · Flint", + "prompts": [ + { + "id": "a", + "label": "Prompt A", + "text": "Write only a Python expression that selects the maximum of x and y." + }, + { + "id": "b", + "label": "Prompt B", + "text": "Write only a Python expression that selects the minimum of x and y." + } + ], + "conditions": [ + { + "id": "a", + "label": "Prompt A · Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Write only a Python expression that selects the maximum of x and y.", + "output": "max(x, y)", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=reroute-python-flint-a&token=1", + "metrics": { + "generated_tokens": 5, + "prompt_tokens": 30, + "elapsed_seconds": 1.156945927, + "termination": "eos", + "displayed_block": 3 + }, + "routing": { + "layer": 3, + "expert_count": 256, + "description": "Measured selected experts in zero-based block 3.", + "tokens": [ + { + "token": "max", + "experts": [ + 117, + 87, + 35, + 5, + 168, + 57 + ], + "weights": [ + 0.4010581374168396, + 0.21458370983600616, + 0.1538792848587036, + 0.10982739180326462, + 0.07339246571063995, + 0.047259047627449036 + ] + }, + { + "token": "(x", + "experts": [ + 219, + 149, + 69, + 243, + 106, + 168 + ], + "weights": [ + 0.356155127286911, + 0.28351446986198425, + 0.15893621742725372, + 0.0888138860464096, + 0.08155017346143723, + 0.031030092388391495 + ] + }, + { + "token": ",", + "experts": [ + 124, + 25, + 177, + 158, + 255, + 224 + ], + "weights": [ + 0.5517688393592834, + 0.12534935772418976, + 0.1014571487903595, + 0.09167846292257309, + 0.07228443771600723, + 0.057461753487586975 + ] + }, + { + "token": " y", + "experts": [ + 19, + 77, + 14, + 203, + 96, + 121 + ], + "weights": [ + 0.4720388352870941, + 0.34413912892341614, + 0.0540812686085701, + 0.04676710069179535, + 0.04236053302884102, + 0.040613144636154175 + ] + }, + { + "token": ")", + "experts": [ + 124, + 25, + 21, + 66, + 235, + 28 + ], + "weights": [ + 0.4778953790664673, + 0.24744132161140442, + 0.10221158713102341, + 0.06340343505144119, + 0.05989007651805878, + 0.04915820062160492 + ] + } + ] + } + }, + { + "id": "b", + "label": "Prompt B · Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Write only a Python expression that selects the minimum of x and y.", + "output": "min(x, y)", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=reroute-python-flint-b&token=1", + "metrics": { + "generated_tokens": 5, + "prompt_tokens": 30, + "elapsed_seconds": 1.128900005, + "termination": "eos", + "displayed_block": 3 + }, + "routing": { + "layer": 3, + "expert_count": 256, + "description": "Measured selected experts in zero-based block 3.", + "tokens": [ + { + "token": "min", + "experts": [ + 117, + 87, + 35, + 5, + 168, + 7 + ], + "weights": [ + 0.4022853970527649, + 0.23592405021190643, + 0.1466451734304428, + 0.09799264371395111, + 0.06999532133340836, + 0.04715738818049431 + ] + }, + { + "token": "(x", + "experts": [ + 149, + 219, + 189, + 69, + 243, + 106 + ], + "weights": [ + 0.4026070237159729, + 0.15572281181812286, + 0.1464761197566986, + 0.1269456148147583, + 0.09798307716846466, + 0.07026540488004684 + ] + }, + { + "token": ",", + "experts": [ + 124, + 25, + 177, + 255, + 158, + 74 + ], + "weights": [ + 0.5216768383979797, + 0.13582339882850647, + 0.10417815297842026, + 0.09238538146018982, + 0.07687641680240631, + 0.0690598264336586 + ] + }, + { + "token": " y", + "experts": [ + 19, + 77, + 14, + 203, + 81, + 121 + ], + "weights": [ + 0.46713194251060486, + 0.34391525387763977, + 0.056301526725292206, + 0.04728133976459503, + 0.04271211847662926, + 0.042657796293497086 + ] + }, + { + "token": ")", + "experts": [ + 124, + 25, + 21, + 66, + 235, + 28 + ], + "weights": [ + 0.4766605496406555, + 0.2452375292778015, + 0.11082471162080765, + 0.062690868973732, + 0.05392761155962944, + 0.050658706575632095 + ] + } + ] + } + } + ], + "comparison": { + "comparison_frame": "First output prediction, before generated continuations diverge.", + "prompt_words_changed": 1, + "prompt_token_counts": [ + 30, + 30 + ], + "routed_blocks_compared": 47, + "blocks_with_different_expert_sets": 20, + "expert_selections_in_each_first_frame": 282, + "shared_expert_selections": 255, + "displayed_zero_based_block": 3, + "display_rule": "Lowest routed block with a different first-frame expert set; full captures contain every block.", + "first_frame_head_rms_relative_l2": 0.1035147140665932 + } + }, + { + "id": "python-qwen", + "title": "One word changes the function · Qwen Open", + "prompts": [ + { + "id": "a", + "label": "Prompt A", + "text": "Write only a Python expression that selects the maximum of x and y." + }, + { + "id": "b", + "label": "Prompt B", + "text": "Write only a Python expression that selects the minimum of x and y." + } + ], + "conditions": [ + { + "id": "a", + "label": "Prompt A · alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Write only a Python expression that selects the maximum of x and y.", + "output": "max(x, y)", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=reroute-python-qwen-a&token=1", + "metrics": { + "generated_tokens": 5, + "prompt_tokens": 44, + "elapsed_seconds": 5.603033013, + "termination": "eos", + "displayed_block": 0 + } + }, + { + "id": "b", + "label": "Prompt B · alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Write only a Python expression that selects the minimum of x and y.", + "output": "min(x, y)", + "settings": { + "max_tokens": 48, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=reroute-python-qwen-b&token=1", + "metrics": { + "generated_tokens": 5, + "prompt_tokens": 44, + "elapsed_seconds": 5.566880933, + "termination": "eos", + "displayed_block": 0 + } + } + ], + "comparison": { + "comparison_frame": "First output prediction, before generated continuations diverge.", + "prompt_words_changed": 1, + "prompt_token_counts": [ + 44, + 44 + ], + "first_frame_head_rms_relative_l2": 0.1294852695128142 + } + } + ] + }, + "quant": { + "title": "Can you spot the 4-bit model?", + "summary": "Twelve blind pairs from six fixed tasks on Flint and alphaBell Qwen Open. Eleven pairs produced identical text; one arithmetic pair differed and both answers were wrong.", + "method": "Matched BF16-derived Q4_1 and Q8_0 files; identical prompts and decoding. Four separate CPU model processes, one warm-up per process. All cases retained.", + "limitations": [ + "Six tasks per model, one generation per condition. This is not a general quality or speed ranking. Process peak RSS is a high-water mark including loading and warm-up, not a per-request increment. Timings include tracing overhead and are single observations." + ], + "cases": [ + { + "id": "flint-arithmetic", + "title": "Exact arithmetic", + "prompt": "Calculate (17 * 23) - 19. Reply with the integer only.", + "checks": [ + "Rule: integer", + "Expected: 372", + "Q8_0: failed", + "Q4_1: failed" + ], + "conditions": [ + { + "id": "flint-q8_0", + "label": "Q8_0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "374", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 33, + "elapsed_seconds": 0.738173075, + "request_wall_seconds": 0.7542800180381164, + "time_to_first_token_seconds": 0.689367342973128, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.0042319691359885515, + "block12_after_rms": 0.0042319691359885515 + }, + "check": { + "passed": false, + "rule": "integer", + "expected": "372", + "character_count": 3, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 9056, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004216483101124048, + "before_rms": 0.004216483101124048, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.0042319691359885515, + "before_rms": 0.0042319691359885515, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.005525138486063405, + "before_rms": 0.005525138486063405, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.014221511290226446, + "before_rms": 0.014221511290226446, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 270336 + } + ] + } + ] + }, + { + "id": "flint-q4_1", + "label": "Q4_1", + "model": "Recursive Flint", + "quant": "Q4_1", + "output": "352", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 33, + "elapsed_seconds": 0.781993782, + "request_wall_seconds": 0.798555999994278, + "time_to_first_token_seconds": 0.7473802480380982, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 24811756, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 24955171680, + "block12_before_rms": 0.00415956677122455, + "block12_after_rms": 0.00415956677122455 + }, + "check": { + "passed": false, + "rule": "integer", + "expected": "372", + "character_count": 3, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 11510, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004206818881751983, + "before_rms": 0.004206818881751983, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.00415956677122455, + "before_rms": 0.00415956677122455, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.005537144078691912, + "before_rms": 0.005537144078691912, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.014134159625799036, + "before_rms": 0.014134159625799036, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 270336 + } + ] + } + ] + } + ], + "model": "flint" + }, + { + "id": "flint-json", + "title": "Strict JSON format", + "prompt": "Return exactly this JSON object and no other text: {\"status\":\"ok\",\"count\":3}", + "checks": [ + "Rule: json", + "Expected: {'status': 'ok', 'count': 3}", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "flint-q8_0", + "label": "Q8_0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "{\"status\":\"ok\",\"count\":3}", + "metrics": { + "generated_tokens": 9, + "prompt_tokens": 35, + "elapsed_seconds": 1.259416502, + "request_wall_seconds": 1.2757591230329126, + "time_to_first_token_seconds": 0.7007529039401561, + "decode_tokens_per_second": 13.911468772577386, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004073247840262152, + "block12_after_rms": 0.004073247840262152 + }, + "check": { + "passed": true, + "rule": "json", + "expected": { + "status": "ok", + "count": 3 + }, + "character_count": 25, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 5482, + "input_position": 34, + "layers": [ + { + "after_rms": 0.00442951382677711, + "before_rms": 0.00442951382677711, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.004073247840262152, + "before_rms": 0.004073247840262152, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.00524975286610561, + "before_rms": 0.00524975286610561, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.014444385872118682, + "before_rms": 0.014444385872118682, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 286720 + } + ] + }, + { + "step": 1, + "token_id": 5462, + "input_position": 35, + "layers": [ + { + "after_rms": 0.005793034803083275, + "before_rms": 0.005793034803083275, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0046820736039610645, + "before_rms": 0.0046820736039610645, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00649377540352318, + "before_rms": 0.00649377540352318, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019726933354943656, + "before_rms": 0.019726933354943656, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 4890, + "input_position": 36, + "layers": [ + { + "after_rms": 0.005294017353787343, + "before_rms": 0.005294017353787343, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0034695353362188502, + "before_rms": 0.0034695353362188502, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00738829527350583, + "before_rms": 0.00738829527350583, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019360584231844193, + "before_rms": 0.019360584231844193, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 544, + "input_position": 37, + "layers": [ + { + "after_rms": 0.00491148399515983, + "before_rms": 0.00491148399515983, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00380345983046727, + "before_rms": 0.00380345983046727, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005149091746140284, + "before_rms": 0.005149091746140284, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016276301961648348, + "before_rms": 0.016276301961648348, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1147, + "input_position": 38, + "layers": [ + { + "after_rms": 0.006222679356489009, + "before_rms": 0.006222679356489009, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0036829646504058975, + "before_rms": 0.0036829646504058975, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006655483694052688, + "before_rms": 0.006655483694052688, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019208434114849963, + "before_rms": 0.019208434114849963, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 2816, + "input_position": 39, + "layers": [ + { + "after_rms": 0.005760162518562755, + "before_rms": 0.005760162518562755, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003275269655906223, + "before_rms": 0.003275269655906223, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004388659397945476, + "before_rms": 0.004388659397945476, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015988995629268844, + "before_rms": 0.015988995629268844, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 1129, + "input_position": 40, + "layers": [ + { + "after_rms": 0.004863272662823825, + "before_rms": 0.004863272662823825, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0038314286271215804, + "before_rms": 0.0038314286271215804, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007286998970545343, + "before_rms": 0.007286998970545343, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018843493317539196, + "before_rms": 0.018843493317539196, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 18, + "input_position": 41, + "layers": [ + { + "after_rms": 0.004603991231053795, + "before_rms": 0.004603991231053795, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0030208238582806433, + "before_rms": 0.0030208238582806433, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004703706397060763, + "before_rms": 0.004703706397060763, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.013840299536615416, + "before_rms": 0.013840299536615416, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 92, + "input_position": 42, + "layers": [ + { + "after_rms": 0.0039039619301298022, + "before_rms": 0.0039039619301298022, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004057874942645444, + "before_rms": 0.004057874942645444, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006582297878179331, + "before_rms": 0.006582297878179331, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016272639100438333, + "before_rms": 0.016272639100438333, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "flint-q4_1", + "label": "Q4_1", + "model": "Recursive Flint", + "quant": "Q4_1", + "output": "{\"status\":\"ok\",\"count\":3}", + "metrics": { + "generated_tokens": 9, + "prompt_tokens": 35, + "elapsed_seconds": 1.251735163, + "request_wall_seconds": 1.2701693959534168, + "time_to_first_token_seconds": 0.7594751949654892, + "decode_tokens_per_second": 15.663206261728522, + "process_peak_rss_kib": 24811756, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 24955171680, + "block12_before_rms": 0.004019226021822854, + "block12_after_rms": 0.004019226021822854 + }, + "check": { + "passed": true, + "rule": "json", + "expected": { + "status": "ok", + "count": 3 + }, + "character_count": 25, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 5482, + "input_position": 34, + "layers": [ + { + "after_rms": 0.004480802624651158, + "before_rms": 0.004480802624651158, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.004019226021822854, + "before_rms": 0.004019226021822854, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.005142506994361342, + "before_rms": 0.005142506994361342, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.014475124728967394, + "before_rms": 0.014475124728967394, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 286720 + } + ] + }, + { + "step": 1, + "token_id": 5462, + "input_position": 35, + "layers": [ + { + "after_rms": 0.005884499521933463, + "before_rms": 0.005884499521933463, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004714484449236878, + "before_rms": 0.004714484449236878, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006689554817134999, + "before_rms": 0.006689554817134999, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020091649545950115, + "before_rms": 0.020091649545950115, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 4890, + "input_position": 36, + "layers": [ + { + "after_rms": 0.005296309235185447, + "before_rms": 0.005296309235185447, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0034924733951837345, + "before_rms": 0.0034924733951837345, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00761576937963421, + "before_rms": 0.00761576937963421, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01972855739429675, + "before_rms": 0.01972855739429675, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 544, + "input_position": 37, + "layers": [ + { + "after_rms": 0.005061610553787093, + "before_rms": 0.005061610553787093, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003724008220189829, + "before_rms": 0.003724008220189829, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004973375331562022, + "before_rms": 0.004973375331562022, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016514705424796036, + "before_rms": 0.016514705424796036, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1147, + "input_position": 38, + "layers": [ + { + "after_rms": 0.006274683971296353, + "before_rms": 0.006274683971296353, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0037093873591057375, + "before_rms": 0.0037093873591057375, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006668038059028412, + "before_rms": 0.006668038059028412, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019377753581390548, + "before_rms": 0.019377753581390548, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 2816, + "input_position": 39, + "layers": [ + { + "after_rms": 0.005775051951262021, + "before_rms": 0.005775051951262021, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0033710668003589326, + "before_rms": 0.0033710668003589326, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004385254089389432, + "before_rms": 0.004385254089389432, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015951241921845414, + "before_rms": 0.015951241921845414, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 1129, + "input_position": 40, + "layers": [ + { + "after_rms": 0.00490185922327077, + "before_rms": 0.00490185922327077, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003771268940693942, + "before_rms": 0.003771268940693942, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007377846105883496, + "before_rms": 0.007377846105883496, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019096257942254946, + "before_rms": 0.019096257942254946, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 18, + "input_position": 41, + "layers": [ + { + "after_rms": 0.004675410678881592, + "before_rms": 0.004675410678881592, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0029219677425947276, + "before_rms": 0.0029219677425947276, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00468134851761768, + "before_rms": 0.00468134851761768, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.013991205065508688, + "before_rms": 0.013991205065508688, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 92, + "input_position": 42, + "layers": [ + { + "after_rms": 0.0039219998776194335, + "before_rms": 0.0039219998776194335, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004008549607569253, + "before_rms": 0.004008549607569253, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006525887799501659, + "before_rms": 0.006525887799501659, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016431090183615604, + "before_rms": 0.016431090183615604, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + } + ], + "model": "flint" + }, + { + "id": "flint-fact", + "title": "Short factual recall", + "prompt": "What is the chemical symbol for gold? Reply with the symbol only.", + "checks": [ + "Rule: exact", + "Expected: Au", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "flint-q8_0", + "label": "Q8_0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "Au", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 30, + "elapsed_seconds": 0.650667801, + "request_wall_seconds": 0.6673325290903449, + "time_to_first_token_seconds": 0.6034285150235519, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004120685912594556, + "block12_after_rms": 0.004120685912594556 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "Au", + "character_count": 2, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 22911, + "input_position": 29, + "layers": [ + { + "after_rms": 0.0042503533024456225, + "before_rms": 0.0042503533024456225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.004120685912594556, + "before_rms": 0.004120685912594556, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.005313107066375563, + "before_rms": 0.005313107066375563, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.01490764588916619, + "before_rms": 0.01490764588916619, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 245760 + } + ] + } + ] + }, + { + "id": "flint-q4_1", + "label": "Q4_1", + "model": "Recursive Flint", + "quant": "Q4_1", + "output": "Au", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 30, + "elapsed_seconds": 0.713642926, + "request_wall_seconds": 0.7302313540130854, + "time_to_first_token_seconds": 0.6695935919415206, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 24811756, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 24955171680, + "block12_before_rms": 0.0040212076224896524, + "block12_after_rms": 0.0040212076224896524 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "Au", + "character_count": 2, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 22911, + "input_position": 29, + "layers": [ + { + "after_rms": 0.004262134108703956, + "before_rms": 0.004262134108703956, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.0040212076224896524, + "before_rms": 0.0040212076224896524, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.005205860826013204, + "before_rms": 0.005205860826013204, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.014788299051992781, + "before_rms": 0.014788299051992781, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 245760 + } + ] + } + ] + } + ], + "model": "flint" + }, + { + "id": "flint-correction", + "title": "Use the corrected fact", + "prompt": "The access color was blue. Correction: the access color is now amber. What is the current access color? Reply with one word only.", + "checks": [ + "Rule: exact", + "Expected: amber", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "flint-q8_0", + "label": "Q8_0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "amber", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 44, + "elapsed_seconds": 0.974556193, + "request_wall_seconds": 0.9955227529862896, + "time_to_first_token_seconds": 0.9343198160640895, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.00441159816545625, + "block12_after_rms": 0.00441159816545625 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "amber", + "character_count": 5, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 13732, + "input_position": 43, + "layers": [ + { + "after_rms": 0.004252703885422568, + "before_rms": 0.004252703885422568, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.00441159816545625, + "before_rms": 0.00441159816545625, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.005657736200656421, + "before_rms": 0.005657736200656421, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.014876767054565872, + "before_rms": 0.014876767054565872, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 360448 + } + ] + } + ] + }, + { + "id": "flint-q4_1", + "label": "Q4_1", + "model": "Recursive Flint", + "quant": "Q4_1", + "output": "amber", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 44, + "elapsed_seconds": 1.028451992, + "request_wall_seconds": 1.0458013030001894, + "time_to_first_token_seconds": 0.9873899329686537, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 24813804, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 24955171680, + "block12_before_rms": 0.004339712902236177, + "block12_after_rms": 0.004339712902236177 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "amber", + "character_count": 5, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 13732, + "input_position": 43, + "layers": [ + { + "after_rms": 0.004209592099785874, + "before_rms": 0.004209592099785874, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.004339712902236177, + "before_rms": 0.004339712902236177, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.005707988547356796, + "before_rms": 0.005707988547356796, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.014914561282848718, + "before_rms": 0.014914561282848718, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 360448 + } + ] + } + ] + } + ], + "model": "flint" + }, + { + "id": "flint-code", + "title": "Python expression repair", + "prompt": "Complete the function with one Python expression only, no markdown: def is_even(n): return ___ . The function must return True exactly when integer n is even.", + "checks": [ + "Rule: expression", + "Expected: n % 2 == 0", + "Q8_0: failed", + "Q4_1: failed" + ], + "conditions": [ + { + "id": "flint-q8_0", + "label": "Q8_0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "return n % 2 == 0", + "metrics": { + "generated_tokens": 8, + "prompt_tokens": 48, + "elapsed_seconds": 1.522581758, + "request_wall_seconds": 1.5477090129861608, + "time_to_first_token_seconds": 0.9909320520237088, + "decode_tokens_per_second": 12.570319015535855, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004202357680490731, + "block12_after_rms": 0.004202357680490731 + }, + "check": { + "passed": false, + "rule": "expression", + "expected": "n % 2 == 0", + "character_count": 17, + "word_count": 6 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 963, + "input_position": 47, + "layers": [ + { + "after_rms": 0.004194344649342437, + "before_rms": 0.004194344649342437, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.004202357680490731, + "before_rms": 0.004202357680490731, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.0052131473290487635, + "before_rms": 0.0052131473290487635, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.014781441026154475, + "before_rms": 0.014781441026154475, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 393216 + } + ] + }, + { + "step": 1, + "token_id": 307, + "input_position": 48, + "layers": [ + { + "after_rms": 0.005277509304535801, + "before_rms": 0.005277509304535801, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00609774510971192, + "before_rms": 0.00609774510971192, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005608145566049825, + "before_rms": 0.005608145566049825, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018110784621771632, + "before_rms": 0.018110784621771632, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 1152, + "input_position": 49, + "layers": [ + { + "after_rms": 0.004954062696681225, + "before_rms": 0.004954062696681225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004947633813264256, + "before_rms": 0.004947633813264256, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006281951776179792, + "before_rms": 0.006281951776179792, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01793926095231872, + "before_rms": 0.01793926095231872, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 220, + "input_position": 50, + "layers": [ + { + "after_rms": 0.005725603347206012, + "before_rms": 0.005725603347206012, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007613934635856348, + "before_rms": 0.007613934635856348, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007067586106815484, + "before_rms": 0.007067586106815484, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02065595811031347, + "before_rms": 0.02065595811031347, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 17, + "input_position": 51, + "layers": [ + { + "after_rms": 0.004384558665618408, + "before_rms": 0.004384558665618408, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002694924917014614, + "before_rms": 0.002694924917014614, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003985706515046729, + "before_rms": 0.003985706515046729, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.012855823082554165, + "before_rms": 0.012855823082554165, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 823, + "input_position": 52, + "layers": [ + { + "after_rms": 0.00695481022115809, + "before_rms": 0.00695481022115809, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005148284890432175, + "before_rms": 0.005148284890432175, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00860817356635732, + "before_rms": 0.00860817356635732, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019382560123796404, + "before_rms": 0.019382560123796404, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 220, + "input_position": 53, + "layers": [ + { + "after_rms": 0.008360185779230117, + "before_rms": 0.008360185779230117, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005331645591182674, + "before_rms": 0.005331645591182674, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.011988298999222614, + "before_rms": 0.011988298999222614, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02324778120292584, + "before_rms": 0.02324778120292584, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 15, + "input_position": 54, + "layers": [ + { + "after_rms": 0.004389820760472024, + "before_rms": 0.004389820760472024, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002823151948738866, + "before_rms": 0.002823151948738866, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004278350453083995, + "before_rms": 0.004278350453083995, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.013667174465740126, + "before_rms": 0.013667174465740126, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "flint-q4_1", + "label": "Q4_1", + "model": "Recursive Flint", + "quant": "Q4_1", + "output": "return n % 2 == 0", + "metrics": { + "generated_tokens": 8, + "prompt_tokens": 48, + "elapsed_seconds": 1.428655518, + "request_wall_seconds": 1.4456852129660547, + "time_to_first_token_seconds": 1.0364669429836795, + "decode_tokens_per_second": 17.098353812752265, + "process_peak_rss_kib": 24813804, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 24955171680, + "block12_before_rms": 0.004124282082456951, + "block12_after_rms": 0.004124282082456951 + }, + "check": { + "passed": false, + "rule": "expression", + "expected": "n % 2 == 0", + "character_count": 17, + "word_count": 6 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 963, + "input_position": 47, + "layers": [ + { + "after_rms": 0.004197426447709307, + "before_rms": 0.004197426447709307, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.004124282082456951, + "before_rms": 0.004124282082456951, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.005129178661979122, + "before_rms": 0.005129178661979122, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.014727759161136935, + "before_rms": 0.014727759161136935, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 393216 + } + ] + }, + { + "step": 1, + "token_id": 307, + "input_position": 48, + "layers": [ + { + "after_rms": 0.00529162387726611, + "before_rms": 0.00529162387726611, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006311261171671491, + "before_rms": 0.006311261171671491, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0055793974325739105, + "before_rms": 0.0055793974325739105, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018152769626134782, + "before_rms": 0.018152769626134782, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 1152, + "input_position": 49, + "layers": [ + { + "after_rms": 0.004977992102677547, + "before_rms": 0.004977992102677547, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005044167186539479, + "before_rms": 0.005044167186539479, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00618730909234251, + "before_rms": 0.00618730909234251, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017977599413659836, + "before_rms": 0.017977599413659836, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 220, + "input_position": 50, + "layers": [ + { + "after_rms": 0.005692971542768754, + "before_rms": 0.005692971542768754, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007667436190392576, + "before_rms": 0.007667436190392576, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00715658608387803, + "before_rms": 0.00715658608387803, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02069278331850969, + "before_rms": 0.02069278331850969, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 17, + "input_position": 51, + "layers": [ + { + "after_rms": 0.0043834519236579, + "before_rms": 0.0043834519236579, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0027110249785654086, + "before_rms": 0.0027110249785654086, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003933361905406773, + "before_rms": 0.003933361905406773, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01299392506244475, + "before_rms": 0.01299392506244475, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 823, + "input_position": 52, + "layers": [ + { + "after_rms": 0.007086739036977866, + "before_rms": 0.007086739036977866, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005068799612035294, + "before_rms": 0.005068799612035294, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008616946732804713, + "before_rms": 0.008616946732804713, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019539045407508844, + "before_rms": 0.019539045407508844, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 220, + "input_position": 53, + "layers": [ + { + "after_rms": 0.008330422942053637, + "before_rms": 0.008330422942053637, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0052894895354913506, + "before_rms": 0.0052894895354913506, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.012248586143235164, + "before_rms": 0.012248586143235164, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.023511864970171558, + "before_rms": 0.023511864970171558, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 15, + "input_position": 54, + "layers": [ + { + "after_rms": 0.0044348412653756105, + "before_rms": 0.0044348412653756105, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0027982348810609656, + "before_rms": 0.0027982348810609656, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004289321679896335, + "before_rms": 0.004289321679896335, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.013781103726701278, + "before_rms": 0.013781103726701278, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + } + ], + "model": "flint" + }, + { + "id": "flint-brief", + "title": "One sentence explanation", + "prompt": "Explain photosynthesis in exactly one sentence containing the words sunlight and glucose.", + "checks": [ + "Rule: sentence", + "Expected: ['sunlight', 'glucose']", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "flint-q8_0", + "label": "Q8_0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "Photosynthesis is the process by which plants use sunlight to create glucose, storing solar energy in chemical form.", + "metrics": { + "generated_tokens": 21, + "prompt_tokens": 29, + "elapsed_seconds": 1.985051171, + "request_wall_seconds": 2.0436750209191814, + "time_to_first_token_seconds": 0.6549666819628328, + "decode_tokens_per_second": 14.402335769662937, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.0044633950081139125, + "block12_after_rms": 0.0044633950081139125 + }, + "check": { + "passed": true, + "rule": "sentence", + "expected": [ + "sunlight", + "glucose" + ], + "character_count": 116, + "word_count": 18 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 49106, + "input_position": 28, + "layers": [ + { + "after_rms": 0.004260129892907466, + "before_rms": 0.004260129892907466, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.0044633950081139125, + "before_rms": 0.0044633950081139125, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.005860773055104808, + "before_rms": 0.005860773055104808, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.015381996005573538, + "before_rms": 0.015381996005573538, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 237568 + } + ] + }, + { + "step": 1, + "token_id": 25424, + "input_position": 29, + "layers": [ + { + "after_rms": 0.0056475146712665295, + "before_rms": 0.0056475146712665295, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008613622622386482, + "before_rms": 0.008613622622386482, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006688744656356641, + "before_rms": 0.006688744656356641, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.022823769881939016, + "before_rms": 0.022823769881939016, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 332, + "input_position": 30, + "layers": [ + { + "after_rms": 0.0064819400327529395, + "before_rms": 0.0064819400327529395, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0066762037147183205, + "before_rms": 0.0066762037147183205, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008751997501954894, + "before_rms": 0.008751997501954894, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.027229842448171573, + "before_rms": 0.027229842448171573, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 269, + "input_position": 31, + "layers": [ + { + "after_rms": 0.0045512880200197945, + "before_rms": 0.0045512880200197945, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005065420297108188, + "before_rms": 0.005065420297108188, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005762789431004507, + "before_rms": 0.005762789431004507, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017768401646140505, + "before_rms": 0.017768401646140505, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1260, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004939956954683621, + "before_rms": 0.004939956954683621, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0038142763724026384, + "before_rms": 0.0038142763724026384, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005913005531372122, + "before_rms": 0.005913005531372122, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01761890031507447, + "before_rms": 0.01761890031507447, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 491, + "input_position": 33, + "layers": [ + { + "after_rms": 0.00603825473034925, + "before_rms": 0.00603825473034925, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005399106143511229, + "before_rms": 0.005399106143511229, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008736535756150335, + "before_rms": 0.008736535756150335, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.023288508898713978, + "before_rms": 0.023288508898713978, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 687, + "input_position": 34, + "layers": [ + { + "after_rms": 0.004825344575484225, + "before_rms": 0.004825344575484225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0035722557633475504, + "before_rms": 0.0035722557633475504, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004528394446715205, + "before_rms": 0.004528394446715205, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015558510785105506, + "before_rms": 0.015558510785105506, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 7370, + "input_position": 35, + "layers": [ + { + "after_rms": 0.00599941128253067, + "before_rms": 0.00599941128253067, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00554379069270193, + "before_rms": 0.00554379069270193, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00600245072054164, + "before_rms": 0.00600245072054164, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021459060251264164, + "before_rms": 0.021459060251264164, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 896, + "input_position": 36, + "layers": [ + { + "after_rms": 0.0051950504054689125, + "before_rms": 0.0051950504054689125, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006141893723174439, + "before_rms": 0.006141893723174439, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009373710872705807, + "before_rms": 0.009373710872705807, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024138161560965755, + "before_rms": 0.024138161560965755, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 9, + "token_id": 24044, + "input_position": 37, + "layers": [ + { + "after_rms": 0.008031516167890848, + "before_rms": 0.008031516167890848, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005368163828714329, + "before_rms": 0.005368163828714329, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005839425814527791, + "before_rms": 0.005839425814527791, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019011043273201134, + "before_rms": 0.019011043273201134, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 10, + "token_id": 295, + "input_position": 38, + "layers": [ + { + "after_rms": 0.009314847401164192, + "before_rms": 0.009314847401164192, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00741898116192189, + "before_rms": 0.00741898116192189, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00969530415712232, + "before_rms": 0.00969530415712232, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02606507768278446, + "before_rms": 0.02606507768278446, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 11, + "token_id": 1815, + "input_position": 39, + "layers": [ + { + "after_rms": 0.0051360181463417694, + "before_rms": 0.0051360181463417694, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005146656362487788, + "before_rms": 0.005146656362487788, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005882866866859857, + "before_rms": 0.005882866866859857, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017485786719118993, + "before_rms": 0.017485786719118993, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 12, + "token_id": 18175, + "input_position": 40, + "layers": [ + { + "after_rms": 0.004435782376176152, + "before_rms": 0.004435782376176152, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007327068248162281, + "before_rms": 0.007327068248162281, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006088986661020862, + "before_rms": 0.006088986661020862, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020237327919821312, + "before_rms": 0.020237327919821312, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 13, + "token_id": 11, + "input_position": 41, + "layers": [ + { + "after_rms": 0.010509592928354338, + "before_rms": 0.010509592928354338, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0072267658458743745, + "before_rms": 0.0072267658458743745, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0072706489196035584, + "before_rms": 0.0072706489196035584, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.026778168430377582, + "before_rms": 0.026778168430377582, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 14, + "token_id": 20827, + "input_position": 42, + "layers": [ + { + "after_rms": 0.0036465031366290737, + "before_rms": 0.0036465031366290737, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003456715901650377, + "before_rms": 0.003456715901650377, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005546723538161339, + "before_rms": 0.005546723538161339, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.014684442530549374, + "before_rms": 0.014684442530549374, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 15, + "token_id": 8437, + "input_position": 43, + "layers": [ + { + "after_rms": 0.00628374835469256, + "before_rms": 0.00628374835469256, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007316398861469263, + "before_rms": 0.007316398861469263, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008040298007985535, + "before_rms": 0.008040298007985535, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021912364803827578, + "before_rms": 0.021912364803827578, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 16, + "token_id": 2488, + "input_position": 44, + "layers": [ + { + "after_rms": 0.008596546950231054, + "before_rms": 0.008596546950231054, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005564570157863948, + "before_rms": 0.005564570157863948, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007757301988850941, + "before_rms": 0.007757301988850941, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020669327020220094, + "before_rms": 0.020669327020220094, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 17, + "token_id": 293, + "input_position": 45, + "layers": [ + { + "after_rms": 0.011432653860628939, + "before_rms": 0.011432653860628939, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005583320073300356, + "before_rms": 0.005583320073300356, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009875868656773605, + "before_rms": 0.009875868656773605, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024941505866581348, + "before_rms": 0.024941505866581348, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 18, + "token_id": 6999, + "input_position": 46, + "layers": [ + { + "after_rms": 0.0053301379768005384, + "before_rms": 0.0053301379768005384, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005000846356165002, + "before_rms": 0.005000846356165002, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005456690270258401, + "before_rms": 0.005456690270258401, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016748723982626335, + "before_rms": 0.016748723982626335, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 19, + "token_id": 981, + "input_position": 47, + "layers": [ + { + "after_rms": 0.007946814942115127, + "before_rms": 0.007946814942115127, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005158417494632955, + "before_rms": 0.005158417494632955, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007460860282340885, + "before_rms": 0.007460860282340885, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020411307476897476, + "before_rms": 0.020411307476897476, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 20, + "token_id": 13, + "input_position": 48, + "layers": [ + { + "after_rms": 0.005514051374901345, + "before_rms": 0.005514051374901345, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0070394921880856, + "before_rms": 0.0070394921880856, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0066148112542871755, + "before_rms": 0.0066148112542871755, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020639906847932543, + "before_rms": 0.020639906847932543, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "flint-q4_1", + "label": "Q4_1", + "model": "Recursive Flint", + "quant": "Q4_1", + "output": "Photosynthesis is the process by which plants use sunlight to create glucose, storing solar energy in chemical form.", + "metrics": { + "generated_tokens": 21, + "prompt_tokens": 29, + "elapsed_seconds": 1.833752451, + "request_wall_seconds": 1.854664323036559, + "time_to_first_token_seconds": 0.6819874360226095, + "decode_tokens_per_second": 17.05229557764334, + "process_peak_rss_kib": 24813804, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 24955171680, + "block12_before_rms": 0.004311247559165804, + "block12_after_rms": 0.004311247559165804 + }, + "check": { + "passed": true, + "rule": "sentence", + "expected": [ + "sunlight", + "glucose" + ], + "character_count": 116, + "word_count": 18 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 49106, + "input_position": 28, + "layers": [ + { + "after_rms": 0.0042605978840198275, + "before_rms": 0.0042605978840198275, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.004311247559165804, + "before_rms": 0.004311247559165804, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.00589800417416362, + "before_rms": 0.00589800417416362, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.015296940228987576, + "before_rms": 0.015296940228987576, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 237568 + } + ] + }, + { + "step": 1, + "token_id": 25424, + "input_position": 29, + "layers": [ + { + "after_rms": 0.005977093382486065, + "before_rms": 0.005977093382486065, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008193012211801604, + "before_rms": 0.008193012211801604, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00683992608315464, + "before_rms": 0.00683992608315464, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.022575839889733613, + "before_rms": 0.022575839889733613, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 332, + "input_position": 30, + "layers": [ + { + "after_rms": 0.0066737027281631835, + "before_rms": 0.0066737027281631835, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006813913551885524, + "before_rms": 0.006813913551885524, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00890446060152158, + "before_rms": 0.00890446060152158, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.027505878131413395, + "before_rms": 0.027505878131413395, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 269, + "input_position": 31, + "layers": [ + { + "after_rms": 0.0044293615064545915, + "before_rms": 0.0044293615064545915, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0051075544884603705, + "before_rms": 0.0051075544884603705, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00575102074067097, + "before_rms": 0.00575102074067097, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017673491619069545, + "before_rms": 0.017673491619069545, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1260, + "input_position": 32, + "layers": [ + { + "after_rms": 0.005036949590614196, + "before_rms": 0.005036949590614196, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003912458062000313, + "before_rms": 0.003912458062000313, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005885844487977124, + "before_rms": 0.005885844487977124, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017499873717491128, + "before_rms": 0.017499873717491128, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 491, + "input_position": 33, + "layers": [ + { + "after_rms": 0.0059711445350189634, + "before_rms": 0.0059711445350189634, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005691547282022589, + "before_rms": 0.005691547282022589, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008772311292878003, + "before_rms": 0.008772311292878003, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.023367520048642263, + "before_rms": 0.023367520048642263, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 687, + "input_position": 34, + "layers": [ + { + "after_rms": 0.00468986110627818, + "before_rms": 0.00468986110627818, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003597916846153238, + "before_rms": 0.003597916846153238, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00449814071189118, + "before_rms": 0.00449814071189118, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015331526438640039, + "before_rms": 0.015331526438640039, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 7370, + "input_position": 35, + "layers": [ + { + "after_rms": 0.006125063657130967, + "before_rms": 0.006125063657130967, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005508366787700634, + "before_rms": 0.005508366787700634, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00612145615591054, + "before_rms": 0.00612145615591054, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021503676321074378, + "before_rms": 0.021503676321074378, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 896, + "input_position": 36, + "layers": [ + { + "after_rms": 0.005212994705122076, + "before_rms": 0.005212994705122076, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006187058312930796, + "before_rms": 0.006187058312930796, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009534886956672891, + "before_rms": 0.009534886956672891, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024496713090488783, + "before_rms": 0.024496713090488783, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 9, + "token_id": 24044, + "input_position": 37, + "layers": [ + { + "after_rms": 0.008439496163631783, + "before_rms": 0.008439496163631783, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005501932939207207, + "before_rms": 0.005501932939207207, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006071353726939792, + "before_rms": 0.006071353726939792, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01955441476435441, + "before_rms": 0.01955441476435441, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 10, + "token_id": 295, + "input_position": 38, + "layers": [ + { + "after_rms": 0.009453040298008773, + "before_rms": 0.009453040298008773, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007468227249751736, + "before_rms": 0.007468227249751736, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009923443718920184, + "before_rms": 0.009923443718920184, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.026507614891845922, + "before_rms": 0.026507614891845922, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 11, + "token_id": 1815, + "input_position": 39, + "layers": [ + { + "after_rms": 0.005096602651817111, + "before_rms": 0.005096602651817111, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00521502443061508, + "before_rms": 0.00521502443061508, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005931677256268344, + "before_rms": 0.005931677256268344, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017564191438152, + "before_rms": 0.017564191438152, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 12, + "token_id": 18175, + "input_position": 40, + "layers": [ + { + "after_rms": 0.004618430168805286, + "before_rms": 0.004618430168805286, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007451683719624791, + "before_rms": 0.007451683719624791, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006378351819867548, + "before_rms": 0.006378351819867548, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02045012871478634, + "before_rms": 0.02045012871478634, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 13, + "token_id": 11, + "input_position": 41, + "layers": [ + { + "after_rms": 0.010203299320963541, + "before_rms": 0.010203299320963541, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0074903907754410755, + "before_rms": 0.0074903907754410755, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0074366114309835155, + "before_rms": 0.0074366114309835155, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02642472904218289, + "before_rms": 0.02642472904218289, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 14, + "token_id": 20827, + "input_position": 42, + "layers": [ + { + "after_rms": 0.003682021169442006, + "before_rms": 0.003682021169442006, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003522532163645522, + "before_rms": 0.003522532163645522, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005599300959517324, + "before_rms": 0.005599300959517324, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.014870400340011209, + "before_rms": 0.014870400340011209, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 15, + "token_id": 8437, + "input_position": 43, + "layers": [ + { + "after_rms": 0.006406295163748658, + "before_rms": 0.006406295163748658, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007487654139117973, + "before_rms": 0.007487654139117973, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008307528862641168, + "before_rms": 0.008307528862641168, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02216700022530452, + "before_rms": 0.02216700022530452, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 16, + "token_id": 2488, + "input_position": 44, + "layers": [ + { + "after_rms": 0.009338543902993778, + "before_rms": 0.009338543902993778, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005618921543998528, + "before_rms": 0.005618921543998528, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007794415755047646, + "before_rms": 0.007794415755047646, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021055979028721958, + "before_rms": 0.021055979028721958, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 17, + "token_id": 293, + "input_position": 45, + "layers": [ + { + "after_rms": 0.012090981045666482, + "before_rms": 0.012090981045666482, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005692148533663447, + "before_rms": 0.005692148533663447, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009866038408325634, + "before_rms": 0.009866038408325634, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02533462806548674, + "before_rms": 0.02533462806548674, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 18, + "token_id": 6999, + "input_position": 46, + "layers": [ + { + "after_rms": 0.0053002691790724555, + "before_rms": 0.0053002691790724555, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005049569138548446, + "before_rms": 0.005049569138548446, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0056289488702639424, + "before_rms": 0.0056289488702639424, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016855422564220696, + "before_rms": 0.016855422564220696, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 19, + "token_id": 981, + "input_position": 47, + "layers": [ + { + "after_rms": 0.007835640918744254, + "before_rms": 0.007835640918744254, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005132931328426137, + "before_rms": 0.005132931328426137, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008011038840819223, + "before_rms": 0.008011038840819223, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02066205167657649, + "before_rms": 0.02066205167657649, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 20, + "token_id": 13, + "input_position": 48, + "layers": [ + { + "after_rms": 0.005366822874419904, + "before_rms": 0.005366822874419904, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0069233652002111, + "before_rms": 0.0069233652002111, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006798509328318403, + "before_rms": 0.006798509328318403, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020444238619442973, + "before_rms": 0.020444238619442973, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + } + ], + "model": "flint" + }, + { + "id": "qwen-arithmetic", + "title": "Exact arithmetic", + "prompt": "Calculate (17 * 23) - 19. Reply with the integer only.", + "checks": [ + "Rule: integer", + "Expected: 372", + "Q8_0: failed", + "Q4_1: failed" + ], + "conditions": [ + { + "id": "qwen-q8_0", + "label": "Q8_0", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "output": "362", + "metrics": { + "generated_tokens": 3, + "prompt_tokens": 50, + "elapsed_seconds": 6.370927408, + "request_wall_seconds": 6.398233153973706, + "time_to_first_token_seconds": 5.195338613004424, + "decode_tokens_per_second": 1.6625734938534744, + "process_peak_rss_kib": 28517200, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 28595763712, + "block12_before_rms": 0.060774775703729224, + "block12_after_rms": 0.060774775703729224 + }, + "check": { + "passed": false, + "rule": "integer", + "expected": "372", + "character_count": 3, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 18, + "input_position": 49, + "layers": [ + { + "after_rms": 0.051227998654021174, + "before_rms": 0.051227998654021174, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1024000 + }, + { + "after_rms": 0.060774775703729224, + "before_rms": 0.060774775703729224, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1024000 + }, + { + "after_rms": 0.05656805709715899, + "before_rms": 0.05656805709715899, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1024000 + }, + { + "after_rms": 0.6482024672223894, + "before_rms": 0.6482024672223894, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 1024000 + } + ] + }, + { + "step": 1, + "token_id": 21, + "input_position": 50, + "layers": [ + { + "after_rms": 0.04010414688700296, + "before_rms": 0.04010414688700296, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.047943996526761117, + "before_rms": 0.047943996526761117, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.059109413227162184, + "before_rms": 0.059109413227162184, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6682861418633491, + "before_rms": 0.6682861418633491, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 2, + "token_id": 17, + "input_position": 51, + "layers": [ + { + "after_rms": 0.041057583688967295, + "before_rms": 0.041057583688967295, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04121733317261862, + "before_rms": 0.04121733317261862, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.047359839621201105, + "before_rms": 0.047359839621201105, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8032033925827468, + "before_rms": 0.8032033925827468, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + }, + { + "id": "qwen-q4_1", + "label": "Q4_1", + "model": "alphaBell Qwen Open 27B", + "quant": "Q4_1", + "output": "362", + "metrics": { + "generated_tokens": 3, + "prompt_tokens": 50, + "elapsed_seconds": 9.60077816, + "request_wall_seconds": 9.647026313003153, + "time_to_first_token_seconds": 8.771988685010001, + "decode_tokens_per_second": 2.285481485264205, + "process_peak_rss_kib": 17027080, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 16829921792, + "block12_before_rms": 0.0627527559695309, + "block12_after_rms": 0.0627527559695309 + }, + "check": { + "passed": false, + "rule": "integer", + "expected": "372", + "character_count": 3, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 18, + "input_position": 49, + "layers": [ + { + "after_rms": 0.05048796369798099, + "before_rms": 0.05048796369798099, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1024000 + }, + { + "after_rms": 0.0627527559695309, + "before_rms": 0.0627527559695309, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1024000 + }, + { + "after_rms": 0.05658700984894215, + "before_rms": 0.05658700984894215, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1024000 + }, + { + "after_rms": 0.6455786601758424, + "before_rms": 0.6455786601758424, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 1024000 + } + ] + }, + { + "step": 1, + "token_id": 21, + "input_position": 50, + "layers": [ + { + "after_rms": 0.03930109211337971, + "before_rms": 0.03930109211337971, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.049347960526798046, + "before_rms": 0.049347960526798046, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06041073044793531, + "before_rms": 0.06041073044793531, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6694457398914584, + "before_rms": 0.6694457398914584, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 2, + "token_id": 17, + "input_position": 51, + "layers": [ + { + "after_rms": 0.039943780185542364, + "before_rms": 0.039943780185542364, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.040380369555943325, + "before_rms": 0.040380369555943325, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04655456958881338, + "before_rms": 0.04655456958881338, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.806759676095322, + "before_rms": 0.806759676095322, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + } + ], + "model": "qwen" + }, + { + "id": "qwen-json", + "title": "Strict JSON format", + "prompt": "Return exactly this JSON object and no other text: {\"status\":\"ok\",\"count\":3}", + "checks": [ + "Rule: json", + "Expected: {'status': 'ok', 'count': 3}", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "qwen-q8_0", + "label": "Q8_0", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "output": "{\"status\":\"ok\",\"count\":3}", + "metrics": { + "generated_tokens": 9, + "prompt_tokens": 49, + "elapsed_seconds": 8.587899932, + "request_wall_seconds": 8.640126045094803, + "time_to_first_token_seconds": 5.242650785017759, + "decode_tokens_per_second": 2.3546587753888293, + "process_peak_rss_kib": 28517200, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 28595763712, + "block12_before_rms": 0.06654119101326558, + "block12_after_rms": 0.06654119101326558 + }, + "check": { + "passed": true, + "rule": "json", + "expected": { + "status": "ok", + "count": 3 + }, + "character_count": 25, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 4754, + "input_position": 48, + "layers": [ + { + "after_rms": 0.052291073879200234, + "before_rms": 0.052291073879200234, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1003520 + }, + { + "after_rms": 0.06654119101326558, + "before_rms": 0.06654119101326558, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1003520 + }, + { + "after_rms": 0.05664182123474859, + "before_rms": 0.05664182123474859, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1003520 + }, + { + "after_rms": 0.6845949822068217, + "before_rms": 0.6845949822068217, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 1003520 + } + ] + }, + { + "step": 1, + "token_id": 2738, + "input_position": 49, + "layers": [ + { + "after_rms": 0.03855443169110616, + "before_rms": 0.03855443169110616, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.047620845341347987, + "before_rms": 0.047620845341347987, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04753896437018176, + "before_rms": 0.04753896437018176, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7466266976354985, + "before_rms": 0.7466266976354985, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 2, + "token_id": 3147, + "input_position": 50, + "layers": [ + { + "after_rms": 0.038951229722960494, + "before_rms": 0.038951229722960494, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03903383968014138, + "before_rms": 0.03903383968014138, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.040843553885173405, + "before_rms": 0.040843553885173405, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9274036435751518, + "before_rms": 0.9274036435751518, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 3, + "token_id": 547, + "input_position": 51, + "layers": [ + { + "after_rms": 0.028340486221480752, + "before_rms": 0.028340486221480752, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.031217358447974707, + "before_rms": 0.031217358447974707, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03351828683553298, + "before_rms": 0.03351828683553298, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.696227792238104, + "before_rms": 0.696227792238104, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 4, + "token_id": 2129, + "input_position": 52, + "layers": [ + { + "after_rms": 0.039811305925765116, + "before_rms": 0.039811305925765116, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04377612280244084, + "before_rms": 0.04377612280244084, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04746907424551777, + "before_rms": 0.04746907424551777, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8157278441144455, + "before_rms": 0.8157278441144455, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 5, + "token_id": 1767, + "input_position": 53, + "layers": [ + { + "after_rms": 0.032082000343200995, + "before_rms": 0.032082000343200995, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03318282436223156, + "before_rms": 0.03318282436223156, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.033619616830206873, + "before_rms": 0.033619616830206873, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6970969978843475, + "before_rms": 0.6970969978843475, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 6, + "token_id": 763, + "input_position": 54, + "layers": [ + { + "after_rms": 0.03466834851881882, + "before_rms": 0.03466834851881882, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03830034553467172, + "before_rms": 0.03830034553467172, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03505207487460683, + "before_rms": 0.03505207487460683, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7499139970375372, + "before_rms": 0.7499139970375372, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 7, + "token_id": 18, + "input_position": 55, + "layers": [ + { + "after_rms": 0.02138759797992662, + "before_rms": 0.02138759797992662, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.024774094253474845, + "before_rms": 0.024774094253474845, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.02468320711787904, + "before_rms": 0.02468320711787904, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7000268392851855, + "before_rms": 0.7000268392851855, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 8, + "token_id": 92, + "input_position": 56, + "layers": [ + { + "after_rms": 0.03105064736729475, + "before_rms": 0.03105064736729475, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03308346918008162, + "before_rms": 0.03308346918008162, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0373820159022643, + "before_rms": 0.0373820159022643, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6878214874370955, + "before_rms": 0.6878214874370955, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + }, + { + "id": "qwen-q4_1", + "label": "Q4_1", + "model": "alphaBell Qwen Open 27B", + "quant": "Q4_1", + "output": "{\"status\":\"ok\",\"count\":3}", + "metrics": { + "generated_tokens": 9, + "prompt_tokens": 49, + "elapsed_seconds": 11.27109005, + "request_wall_seconds": 11.301758665940724, + "time_to_first_token_seconds": 8.48897207097616, + "decode_tokens_per_second": 2.8440503495699967, + "process_peak_rss_kib": 17027080, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 16829921792, + "block12_before_rms": 0.06706882773636201, + "block12_after_rms": 0.06706882773636201 + }, + "check": { + "passed": true, + "rule": "json", + "expected": { + "status": "ok", + "count": 3 + }, + "character_count": 25, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 4754, + "input_position": 48, + "layers": [ + { + "after_rms": 0.05224029844078737, + "before_rms": 0.05224029844078737, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1003520 + }, + { + "after_rms": 0.06706882773636201, + "before_rms": 0.06706882773636201, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1003520 + }, + { + "after_rms": 0.056588501864807024, + "before_rms": 0.056588501864807024, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1003520 + }, + { + "after_rms": 0.6822709962112733, + "before_rms": 0.6822709962112733, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 1003520 + } + ] + }, + { + "step": 1, + "token_id": 2738, + "input_position": 49, + "layers": [ + { + "after_rms": 0.03752622482634915, + "before_rms": 0.03752622482634915, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.046903273237478865, + "before_rms": 0.046903273237478865, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04722570539635123, + "before_rms": 0.04722570539635123, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7377205249987369, + "before_rms": 0.7377205249987369, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 2, + "token_id": 3147, + "input_position": 50, + "layers": [ + { + "after_rms": 0.0383644658257016, + "before_rms": 0.0383644658257016, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03938200710595041, + "before_rms": 0.03938200710595041, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.040526837904161464, + "before_rms": 0.040526837904161464, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9237316979852525, + "before_rms": 0.9237316979852525, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 3, + "token_id": 547, + "input_position": 51, + "layers": [ + { + "after_rms": 0.02778849024723728, + "before_rms": 0.02778849024723728, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03214516611442832, + "before_rms": 0.03214516611442832, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03363793762611696, + "before_rms": 0.03363793762611696, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6944275380038829, + "before_rms": 0.6944275380038829, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 4, + "token_id": 2129, + "input_position": 52, + "layers": [ + { + "after_rms": 0.039622563175138806, + "before_rms": 0.039622563175138806, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.044796938258385224, + "before_rms": 0.044796938258385224, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.048279328936419154, + "before_rms": 0.048279328936419154, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8158488049148617, + "before_rms": 0.8158488049148617, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 5, + "token_id": 1767, + "input_position": 53, + "layers": [ + { + "after_rms": 0.03299336016454038, + "before_rms": 0.03299336016454038, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.035306595030275154, + "before_rms": 0.035306595030275154, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.034968922773907535, + "before_rms": 0.034968922773907535, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7012426405133863, + "before_rms": 0.7012426405133863, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 6, + "token_id": 763, + "input_position": 54, + "layers": [ + { + "after_rms": 0.03539100194372584, + "before_rms": 0.03539100194372584, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03888318460173007, + "before_rms": 0.03888318460173007, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.036111392321474096, + "before_rms": 0.036111392321474096, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7510159984696372, + "before_rms": 0.7510159984696372, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 7, + "token_id": 18, + "input_position": 55, + "layers": [ + { + "after_rms": 0.021069990443326445, + "before_rms": 0.021069990443326445, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.02628960155652075, + "before_rms": 0.02628960155652075, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.02576443542713521, + "before_rms": 0.02576443542713521, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6989044785673804, + "before_rms": 0.6989044785673804, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 8, + "token_id": 92, + "input_position": 56, + "layers": [ + { + "after_rms": 0.03149593114335555, + "before_rms": 0.03149593114335555, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.033202658246637436, + "before_rms": 0.033202658246637436, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.037501340133310815, + "before_rms": 0.037501340133310815, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6955924601927745, + "before_rms": 0.6955924601927745, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + } + ], + "model": "qwen" + }, + { + "id": "qwen-fact", + "title": "Short factual recall", + "prompt": "What is the chemical symbol for gold? Reply with the symbol only.", + "checks": [ + "Rule: exact", + "Expected: Au", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "qwen-q8_0", + "label": "Q8_0", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "output": "Au", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 44, + "elapsed_seconds": 4.642115172, + "request_wall_seconds": 4.671560399001464, + "time_to_first_token_seconds": 4.299511207034811, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 28517200, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 28595763712, + "block12_before_rms": 0.07549879978282885, + "block12_after_rms": 0.07549879978282885 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "Au", + "character_count": 2, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 63126, + "input_position": 43, + "layers": [ + { + "after_rms": 0.0577547722697427, + "before_rms": 0.0577547722697427, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 901120 + }, + { + "after_rms": 0.07549879978282885, + "before_rms": 0.07549879978282885, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 901120 + }, + { + "after_rms": 0.0707093987239072, + "before_rms": 0.0707093987239072, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 901120 + }, + { + "after_rms": 0.6240880098652177, + "before_rms": 0.6240880098652177, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 901120 + } + ] + } + ] + }, + { + "id": "qwen-q4_1", + "label": "Q4_1", + "model": "alphaBell Qwen Open 27B", + "quant": "Q4_1", + "output": "Au", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 44, + "elapsed_seconds": 7.763876279, + "request_wall_seconds": 7.807731770095415, + "time_to_first_token_seconds": 7.462882330059074, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 17027080, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 16829921792, + "block12_before_rms": 0.07777038091284097, + "block12_after_rms": 0.07777038091284097 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "Au", + "character_count": 2, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 63126, + "input_position": 43, + "layers": [ + { + "after_rms": 0.058337225299247154, + "before_rms": 0.058337225299247154, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 901120 + }, + { + "after_rms": 0.07777038091284097, + "before_rms": 0.07777038091284097, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 901120 + }, + { + "after_rms": 0.07119122784735792, + "before_rms": 0.07119122784735792, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 901120 + }, + { + "after_rms": 0.627043092250394, + "before_rms": 0.627043092250394, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 901120 + } + ] + } + ] + } + ], + "model": "qwen" + }, + { + "id": "qwen-correction", + "title": "Use the corrected fact", + "prompt": "The access color was blue. Correction: the access color is now amber. What is the current access color? Reply with one word only.", + "checks": [ + "Rule: exact", + "Expected: amber", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "qwen-q8_0", + "label": "Q8_0", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "output": "Amber", + "metrics": { + "generated_tokens": 2, + "prompt_tokens": 58, + "elapsed_seconds": 6.922008574, + "request_wall_seconds": 6.929079352994449, + "time_to_first_token_seconds": 6.135061308043078, + "decode_tokens_per_second": 1.2593577760195043, + "process_peak_rss_kib": 28517200, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 28595763712, + "block12_before_rms": 0.08439747530644362, + "block12_after_rms": 0.08439747530644362 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "amber", + "character_count": 5, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 5901, + "input_position": 57, + "layers": [ + { + "after_rms": 0.05835882673381115, + "before_rms": 0.05835882673381115, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1187840 + }, + { + "after_rms": 0.08439747530644362, + "before_rms": 0.08439747530644362, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1187840 + }, + { + "after_rms": 0.07025392596556453, + "before_rms": 0.07025392596556453, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1187840 + }, + { + "after_rms": 0.6934357017487144, + "before_rms": 0.6934357017487144, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 1187840 + } + ] + }, + { + "step": 1, + "token_id": 634, + "input_position": 58, + "layers": [ + { + "after_rms": 0.059411132201425514, + "before_rms": 0.059411132201425514, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07671841412821269, + "before_rms": 0.07671841412821269, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08880618504780236, + "before_rms": 0.08880618504780236, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8700114799969301, + "before_rms": 0.8700114799969301, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + }, + { + "id": "qwen-q4_1", + "label": "Q4_1", + "model": "alphaBell Qwen Open 27B", + "quant": "Q4_1", + "output": "Amber", + "metrics": { + "generated_tokens": 2, + "prompt_tokens": 58, + "elapsed_seconds": 10.122600283, + "request_wall_seconds": 10.232366479933262, + "time_to_first_token_seconds": 9.576763171004131, + "decode_tokens_per_second": 1.5252099559031584, + "process_peak_rss_kib": 17029128, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 16829921792, + "block12_before_rms": 0.08882206964440575, + "block12_after_rms": 0.08882206964440575 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "amber", + "character_count": 5, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 5901, + "input_position": 57, + "layers": [ + { + "after_rms": 0.05771725253557467, + "before_rms": 0.05771725253557467, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1187840 + }, + { + "after_rms": 0.08882206964440575, + "before_rms": 0.08882206964440575, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1187840 + }, + { + "after_rms": 0.07251657251443638, + "before_rms": 0.07251657251443638, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1187840 + }, + { + "after_rms": 0.694176497002211, + "before_rms": 0.694176497002211, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 1187840 + } + ] + }, + { + "step": 1, + "token_id": 634, + "input_position": 58, + "layers": [ + { + "after_rms": 0.06058345247387532, + "before_rms": 0.06058345247387532, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07852469801096601, + "before_rms": 0.07852469801096601, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0889624477213204, + "before_rms": 0.0889624477213204, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8659660560094563, + "before_rms": 0.8659660560094563, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + } + ], + "model": "qwen" + }, + { + "id": "qwen-code", + "title": "Python expression repair", + "prompt": "Complete the function with one Python expression only, no markdown: def is_even(n): return ___ . The function must return True exactly when integer n is even.", + "checks": [ + "Rule: expression", + "Expected: n % 2 == 0", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "qwen-q8_0", + "label": "Q8_0", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "output": "n % 2 == 0", + "metrics": { + "generated_tokens": 7, + "prompt_tokens": 62, + "elapsed_seconds": 9.254872529, + "request_wall_seconds": 9.284579382976517, + "time_to_first_token_seconds": 6.607404271955602, + "decode_tokens_per_second": 2.2411405168871537, + "process_peak_rss_kib": 28517200, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 28595763712, + "block12_before_rms": 0.05339986668971442, + "block12_after_rms": 0.05339986668971442 + }, + "check": { + "passed": true, + "rule": "expression", + "expected": "n % 2 == 0", + "character_count": 10, + "word_count": 5 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 77, + "input_position": 61, + "layers": [ + { + "after_rms": 0.050335302931553125, + "before_rms": 0.050335302931553125, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1269760 + }, + { + "after_rms": 0.05339986668971442, + "before_rms": 0.05339986668971442, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1269760 + }, + { + "after_rms": 0.050829254488739616, + "before_rms": 0.050829254488739616, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1269760 + }, + { + "after_rms": 0.6219835311460259, + "before_rms": 0.6219835311460259, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 1269760 + } + ] + }, + { + "step": 1, + "token_id": 984, + "input_position": 62, + "layers": [ + { + "after_rms": 0.053686741784584366, + "before_rms": 0.053686741784584366, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.060073512071431336, + "before_rms": 0.060073512071431336, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07386188586159291, + "before_rms": 0.07386188586159291, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7373913972139406, + "before_rms": 0.7373913972139406, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 2, + "token_id": 220, + "input_position": 63, + "layers": [ + { + "after_rms": 0.0757632347354875, + "before_rms": 0.0757632347354875, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08447271653317262, + "before_rms": 0.08447271653317262, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08083366248253042, + "before_rms": 0.08083366248253042, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 1.0242087065193417, + "before_rms": 1.0242087065193417, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 3, + "token_id": 17, + "input_position": 64, + "layers": [ + { + "after_rms": 0.031847120231657235, + "before_rms": 0.031847120231657235, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.034744065101999524, + "before_rms": 0.034744065101999524, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0356676406493493, + "before_rms": 0.0356676406493493, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7133838975226968, + "before_rms": 0.7133838975226968, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 4, + "token_id": 606, + "input_position": 65, + "layers": [ + { + "after_rms": 0.06511419656269898, + "before_rms": 0.06511419656269898, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06654091916956285, + "before_rms": 0.06654091916956285, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0625665999000719, + "before_rms": 0.0625665999000719, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9783984307722204, + "before_rms": 0.9783984307722204, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 5, + "token_id": 220, + "input_position": 66, + "layers": [ + { + "after_rms": 0.0645209284738379, + "before_rms": 0.0645209284738379, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06279313344255302, + "before_rms": 0.06279313344255302, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07065568848215707, + "before_rms": 0.07065568848215707, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8914932839543546, + "before_rms": 0.8914932839543546, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 6, + "token_id": 15, + "input_position": 67, + "layers": [ + { + "after_rms": 0.01992336177071922, + "before_rms": 0.01992336177071922, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.01935371501039362, + "before_rms": 0.01935371501039362, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.02030963950506069, + "before_rms": 0.02030963950506069, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.5854232238481374, + "before_rms": 0.5854232238481374, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + }, + { + "id": "qwen-q4_1", + "label": "Q4_1", + "model": "alphaBell Qwen Open 27B", + "quant": "Q4_1", + "output": "n % 2 == 0", + "metrics": { + "generated_tokens": 7, + "prompt_tokens": 62, + "elapsed_seconds": 12.254965777, + "request_wall_seconds": 12.296226173988543, + "time_to_first_token_seconds": 10.16412831703201, + "decode_tokens_per_second": 2.8140651531387326, + "process_peak_rss_kib": 17029128, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 16829921792, + "block12_before_rms": 0.05348751558536074, + "block12_after_rms": 0.05348751558536074 + }, + "check": { + "passed": true, + "rule": "expression", + "expected": "n % 2 == 0", + "character_count": 10, + "word_count": 5 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 77, + "input_position": 61, + "layers": [ + { + "after_rms": 0.05019944799383319, + "before_rms": 0.05019944799383319, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1269760 + }, + { + "after_rms": 0.05348751558536074, + "before_rms": 0.05348751558536074, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1269760 + }, + { + "after_rms": 0.05175455397380355, + "before_rms": 0.05175455397380355, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 1269760 + }, + { + "after_rms": 0.6209295945416631, + "before_rms": 0.6209295945416631, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 1269760 + } + ] + }, + { + "step": 1, + "token_id": 984, + "input_position": 62, + "layers": [ + { + "after_rms": 0.05431832143619491, + "before_rms": 0.05431832143619491, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0596135618336448, + "before_rms": 0.0596135618336448, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07570250992978683, + "before_rms": 0.07570250992978683, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7405282278687231, + "before_rms": 0.7405282278687231, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 2, + "token_id": 220, + "input_position": 63, + "layers": [ + { + "after_rms": 0.07608705520961619, + "before_rms": 0.07608705520961619, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08613149135695485, + "before_rms": 0.08613149135695485, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08167221964365984, + "before_rms": 0.08167221964365984, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 1.0218643697263319, + "before_rms": 1.0218643697263319, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 3, + "token_id": 17, + "input_position": 64, + "layers": [ + { + "after_rms": 0.03293924682560806, + "before_rms": 0.03293924682560806, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03573770767293814, + "before_rms": 0.03573770767293814, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.03788883557000408, + "before_rms": 0.03788883557000408, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7037175496589, + "before_rms": 0.7037175496589, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 4, + "token_id": 606, + "input_position": 65, + "layers": [ + { + "after_rms": 0.06427408980785317, + "before_rms": 0.06427408980785317, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0662650177596904, + "before_rms": 0.0662650177596904, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06341284549994454, + "before_rms": 0.06341284549994454, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9659503607467934, + "before_rms": 0.9659503607467934, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 5, + "token_id": 220, + "input_position": 66, + "layers": [ + { + "after_rms": 0.06468317950504401, + "before_rms": 0.06468317950504401, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06446495337470849, + "before_rms": 0.06446495337470849, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07169159170100954, + "before_rms": 0.07169159170100954, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8967088457352965, + "before_rms": 0.8967088457352965, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 6, + "token_id": 15, + "input_position": 67, + "layers": [ + { + "after_rms": 0.01985784365149535, + "before_rms": 0.01985784365149535, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.020029877274204144, + "before_rms": 0.020029877274204144, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.020914043695738526, + "before_rms": 0.020914043695738526, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.5841152795565343, + "before_rms": 0.5841152795565343, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + } + ], + "model": "qwen" + }, + { + "id": "qwen-brief", + "title": "One sentence explanation", + "prompt": "Explain photosynthesis in exactly one sentence containing the words sunlight and glucose.", + "checks": [ + "Rule: sentence", + "Expected: ['sunlight', 'glucose']", + "Q8_0: passed", + "Q4_1: passed" + ], + "conditions": [ + { + "id": "qwen-q8_0", + "label": "Q8_0", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "output": "Photosynthesis is the process by which plants use sunlight to convert carbon dioxide and water into glucose and oxygen.", + "metrics": { + "generated_tokens": 21, + "prompt_tokens": 45, + "elapsed_seconds": 12.680948171, + "request_wall_seconds": 12.710689318017103, + "time_to_first_token_seconds": 4.477205132017843, + "decode_tokens_per_second": 2.4290935490085785, + "process_peak_rss_kib": 28517200, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 28595763712, + "block12_before_rms": 0.08826706557870591, + "block12_after_rms": 0.08826706557870591 + }, + "check": { + "passed": true, + "rule": "sentence", + "expected": [ + "sunlight", + "glucose" + ], + "character_count": 119, + "word_count": 19 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 30729, + "input_position": 44, + "layers": [ + { + "after_rms": 0.058797602262198566, + "before_rms": 0.058797602262198566, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 921600 + }, + { + "after_rms": 0.08826706557870591, + "before_rms": 0.08826706557870591, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 921600 + }, + { + "after_rms": 0.07744769492165124, + "before_rms": 0.07744769492165124, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 921600 + }, + { + "after_rms": 0.6493664827145925, + "before_rms": 0.6493664827145925, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 921600 + } + ] + }, + { + "step": 1, + "token_id": 71163, + "input_position": 45, + "layers": [ + { + "after_rms": 0.090079163721148, + "before_rms": 0.090079163721148, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.09129523578154757, + "before_rms": 0.09129523578154757, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0829387283770895, + "before_rms": 0.0829387283770895, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9874844483140827, + "before_rms": 0.9874844483140827, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 2, + "token_id": 369, + "input_position": 46, + "layers": [ + { + "after_rms": 0.10315392514524502, + "before_rms": 0.10315392514524502, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.09600644261008953, + "before_rms": 0.09600644261008953, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0857152751972423, + "before_rms": 0.0857152751972423, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 1.1279006781817724, + "before_rms": 1.1279006781817724, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 3, + "token_id": 279, + "input_position": 47, + "layers": [ + { + "after_rms": 0.0703105570552835, + "before_rms": 0.0703105570552835, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08696065973360559, + "before_rms": 0.08696065973360559, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08599798105761434, + "before_rms": 0.08599798105761434, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7075713993722996, + "before_rms": 0.7075713993722996, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 4, + "token_id": 1817, + "input_position": 48, + "layers": [ + { + "after_rms": 0.04824121607819701, + "before_rms": 0.04824121607819701, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.053480497826960974, + "before_rms": 0.053480497826960974, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06288983601150724, + "before_rms": 0.06288983601150724, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6498592562479432, + "before_rms": 0.6498592562479432, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 5, + "token_id": 539, + "input_position": 49, + "layers": [ + { + "after_rms": 0.0857769249129497, + "before_rms": 0.0857769249129497, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.09649908523834096, + "before_rms": 0.09649908523834096, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.10047606095155863, + "before_rms": 0.10047606095155863, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8915207377991599, + "before_rms": 0.8915207377991599, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 6, + "token_id": 864, + "input_position": 50, + "layers": [ + { + "after_rms": 0.08697706879734413, + "before_rms": 0.08697706879734413, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07727010638748308, + "before_rms": 0.07727010638748308, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.09060006959943864, + "before_rms": 0.09060006959943864, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7683805849623969, + "before_rms": 0.7683805849623969, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 7, + "token_id": 10462, + "input_position": 51, + "layers": [ + { + "after_rms": 0.05259421797317353, + "before_rms": 0.05259421797317353, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.05075043909793059, + "before_rms": 0.05075043909793059, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.05619568725069516, + "before_rms": 0.05619568725069516, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.682036783205715, + "before_rms": 0.682036783205715, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 8, + "token_id": 958, + "input_position": 52, + "layers": [ + { + "after_rms": 0.08697357786711173, + "before_rms": 0.08697357786711173, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.10030394478249688, + "before_rms": 0.10030394478249688, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08877069591266067, + "before_rms": 0.08877069591266067, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9646197687285322, + "before_rms": 0.9646197687285322, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 9, + "token_id": 37728, + "input_position": 53, + "layers": [ + { + "after_rms": 0.07099648705814063, + "before_rms": 0.07099648705814063, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06559096588235168, + "before_rms": 0.06559096588235168, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07033321001871201, + "before_rms": 0.07033321001871201, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7932168808370963, + "before_rms": 0.7932168808370963, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 10, + "token_id": 310, + "input_position": 54, + "layers": [ + { + "after_rms": 0.08567177205503797, + "before_rms": 0.08567177205503797, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08384291426005887, + "before_rms": 0.08384291426005887, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08019719454528439, + "before_rms": 0.08019719454528439, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9847601168939447, + "before_rms": 0.9847601168939447, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 11, + "token_id": 5335, + "input_position": 55, + "layers": [ + { + "after_rms": 0.043123201419361115, + "before_rms": 0.043123201419361115, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04955096633139602, + "before_rms": 0.04955096633139602, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.053935569085920204, + "before_rms": 0.053935569085920204, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7026575509832893, + "before_rms": 0.7026575509832893, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 12, + "token_id": 12141, + "input_position": 56, + "layers": [ + { + "after_rms": 0.07978349390889998, + "before_rms": 0.07978349390889998, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07617391418880347, + "before_rms": 0.07617391418880347, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07734026858151206, + "before_rms": 0.07734026858151206, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8642118185671083, + "before_rms": 0.8642118185671083, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 13, + "token_id": 38182, + "input_position": 57, + "layers": [ + { + "after_rms": 0.0852385224689407, + "before_rms": 0.0852385224689407, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07231971038626593, + "before_rms": 0.07231971038626593, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06564426590463994, + "before_rms": 0.06564426590463994, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9157673634228397, + "before_rms": 0.9157673634228397, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 14, + "token_id": 321, + "input_position": 58, + "layers": [ + { + "after_rms": 0.08898192682211213, + "before_rms": 0.08898192682211213, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0894750809473178, + "before_rms": 0.0894750809473178, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08616370851207737, + "before_rms": 0.08616370851207737, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 1.0029595110641643, + "before_rms": 1.0029595110641643, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 15, + "token_id": 2919, + "input_position": 59, + "layers": [ + { + "after_rms": 0.03762191629076314, + "before_rms": 0.03762191629076314, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.049029101729874604, + "before_rms": 0.049029101729874604, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.05884392489297117, + "before_rms": 0.05884392489297117, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6250386128828833, + "before_rms": 0.6250386128828833, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 16, + "token_id": 1083, + "input_position": 60, + "layers": [ + { + "after_rms": 0.06327166050315482, + "before_rms": 0.06327166050315482, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06857836855297815, + "before_rms": 0.06857836855297815, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06961224727562185, + "before_rms": 0.06961224727562185, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8674029708685703, + "before_rms": 0.8674029708685703, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 17, + "token_id": 32120, + "input_position": 61, + "layers": [ + { + "after_rms": 0.0530895463732545, + "before_rms": 0.0530895463732545, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06423360109914558, + "before_rms": 0.06423360109914558, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.052746699956901094, + "before_rms": 0.052746699956901094, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7559156680420246, + "before_rms": 0.7559156680420246, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 18, + "token_id": 321, + "input_position": 62, + "layers": [ + { + "after_rms": 0.07834838045994791, + "before_rms": 0.07834838045994791, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07525093120253516, + "before_rms": 0.07525093120253516, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07091507308763366, + "before_rms": 0.07091507308763366, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.954111191683783, + "before_rms": 0.954111191683783, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 19, + "token_id": 22817, + "input_position": 63, + "layers": [ + { + "after_rms": 0.0353542432718733, + "before_rms": 0.0353542432718733, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04198844081291262, + "before_rms": 0.04198844081291262, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.054421407751238354, + "before_rms": 0.054421407751238354, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.629625149193875, + "before_rms": 0.629625149193875, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 20, + "token_id": 13, + "input_position": 64, + "layers": [ + { + "after_rms": 0.07315881109901357, + "before_rms": 0.07315881109901357, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0687687379925814, + "before_rms": 0.0687687379925814, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06567824611420153, + "before_rms": 0.06567824611420153, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8984019945181603, + "before_rms": 0.8984019945181603, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + }, + { + "id": "qwen-q4_1", + "label": "Q4_1", + "model": "alphaBell Qwen Open 27B", + "quant": "Q4_1", + "output": "Photosynthesis is the process by which plants use sunlight to convert carbon dioxide and water into glucose and oxygen.", + "metrics": { + "generated_tokens": 21, + "prompt_tokens": 45, + "elapsed_seconds": 13.409067066, + "request_wall_seconds": 13.439180701971054, + "time_to_first_token_seconds": 7.273399753961712, + "decode_tokens_per_second": 3.243670129869688, + "process_peak_rss_kib": 17029128, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 16829921792, + "block12_before_rms": 0.09098383032221528, + "block12_after_rms": 0.09098383032221528 + }, + "check": { + "passed": true, + "rule": "sentence", + "expected": [ + "sunlight", + "glucose" + ], + "character_count": 119, + "word_count": 19 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 30729, + "input_position": 44, + "layers": [ + { + "after_rms": 0.059582062101886914, + "before_rms": 0.059582062101886914, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 921600 + }, + { + "after_rms": 0.09098383032221528, + "before_rms": 0.09098383032221528, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 921600 + }, + { + "after_rms": 0.08016257842928678, + "before_rms": 0.08016257842928678, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 921600 + }, + { + "after_rms": 0.6536671790903104, + "before_rms": 0.6536671790903104, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 921600 + } + ] + }, + { + "step": 1, + "token_id": 71163, + "input_position": 45, + "layers": [ + { + "after_rms": 0.09190790143581268, + "before_rms": 0.09190790143581268, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.09553453447497137, + "before_rms": 0.09553453447497137, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08593551701844118, + "before_rms": 0.08593551701844118, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9966251527987956, + "before_rms": 0.9966251527987956, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 2, + "token_id": 369, + "input_position": 46, + "layers": [ + { + "after_rms": 0.10115326211799784, + "before_rms": 0.10115326211799784, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0958071133672348, + "before_rms": 0.0958071133672348, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0854668936412898, + "before_rms": 0.0854668936412898, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 1.1255902406232912, + "before_rms": 1.1255902406232912, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 3, + "token_id": 279, + "input_position": 47, + "layers": [ + { + "after_rms": 0.07451807797268321, + "before_rms": 0.07451807797268321, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0966776949197356, + "before_rms": 0.0966776949197356, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.09200739468581592, + "before_rms": 0.09200739468581592, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7275052188652554, + "before_rms": 0.7275052188652554, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 4, + "token_id": 1817, + "input_position": 48, + "layers": [ + { + "after_rms": 0.0478125046485284, + "before_rms": 0.0478125046485284, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0548085725164451, + "before_rms": 0.0548085725164451, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06471972775053976, + "before_rms": 0.06471972775053976, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6538473476210616, + "before_rms": 0.6538473476210616, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 5, + "token_id": 539, + "input_position": 49, + "layers": [ + { + "after_rms": 0.08518049302904517, + "before_rms": 0.08518049302904517, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.09818462339752433, + "before_rms": 0.09818462339752433, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.10130589343276135, + "before_rms": 0.10130589343276135, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8866838924389607, + "before_rms": 0.8866838924389607, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 6, + "token_id": 864, + "input_position": 50, + "layers": [ + { + "after_rms": 0.08680030692428589, + "before_rms": 0.08680030692428589, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08012581611715355, + "before_rms": 0.08012581611715355, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.09241248361191251, + "before_rms": 0.09241248361191251, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7686299073659799, + "before_rms": 0.7686299073659799, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 7, + "token_id": 10462, + "input_position": 51, + "layers": [ + { + "after_rms": 0.05017937165905816, + "before_rms": 0.05017937165905816, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.050057021551545955, + "before_rms": 0.050057021551545955, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.05497654538007085, + "before_rms": 0.05497654538007085, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6670578115164874, + "before_rms": 0.6670578115164874, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 8, + "token_id": 958, + "input_position": 52, + "layers": [ + { + "after_rms": 0.08568044433943381, + "before_rms": 0.08568044433943381, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.10045397748397368, + "before_rms": 0.10045397748397368, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08773616457482147, + "before_rms": 0.08773616457482147, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9600428235856283, + "before_rms": 0.9600428235856283, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 9, + "token_id": 37728, + "input_position": 53, + "layers": [ + { + "after_rms": 0.06924941047054128, + "before_rms": 0.06924941047054128, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0655311740748727, + "before_rms": 0.0655311740748727, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07066158387714026, + "before_rms": 0.07066158387714026, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7935303713786936, + "before_rms": 0.7935303713786936, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 10, + "token_id": 310, + "input_position": 54, + "layers": [ + { + "after_rms": 0.08433748009746503, + "before_rms": 0.08433748009746503, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08393242733321232, + "before_rms": 0.08393242733321232, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.0819084026323373, + "before_rms": 0.0819084026323373, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9796541806488255, + "before_rms": 0.9796541806488255, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 11, + "token_id": 5335, + "input_position": 55, + "layers": [ + { + "after_rms": 0.039579585893422226, + "before_rms": 0.039579585893422226, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.046979610302819906, + "before_rms": 0.046979610302819906, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.05344053164079346, + "before_rms": 0.05344053164079346, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6960761835563655, + "before_rms": 0.6960761835563655, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 12, + "token_id": 12141, + "input_position": 56, + "layers": [ + { + "after_rms": 0.07903697242878281, + "before_rms": 0.07903697242878281, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07614783932603204, + "before_rms": 0.07614783932603204, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07944953913293375, + "before_rms": 0.07944953913293375, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8567541215882547, + "before_rms": 0.8567541215882547, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 13, + "token_id": 38182, + "input_position": 57, + "layers": [ + { + "after_rms": 0.0864315395888373, + "before_rms": 0.0864315395888373, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07366431154681498, + "before_rms": 0.07366431154681498, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06620859996023702, + "before_rms": 0.06620859996023702, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9160313052637309, + "before_rms": 0.9160313052637309, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 14, + "token_id": 321, + "input_position": 58, + "layers": [ + { + "after_rms": 0.08880976995639013, + "before_rms": 0.08880976995639013, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08999483103099812, + "before_rms": 0.08999483103099812, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.08782409614422529, + "before_rms": 0.08782409614422529, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9846578342137169, + "before_rms": 0.9846578342137169, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 15, + "token_id": 2919, + "input_position": 59, + "layers": [ + { + "after_rms": 0.03571734510932391, + "before_rms": 0.03571734510932391, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.047244981585028045, + "before_rms": 0.047244981585028045, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.05755773777600419, + "before_rms": 0.05755773777600419, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6213354826118737, + "before_rms": 0.6213354826118737, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 16, + "token_id": 1083, + "input_position": 60, + "layers": [ + { + "after_rms": 0.0626217871949745, + "before_rms": 0.0626217871949745, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06839292559199892, + "before_rms": 0.06839292559199892, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07067343367902718, + "before_rms": 0.07067343367902718, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.857509349072362, + "before_rms": 0.857509349072362, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 17, + "token_id": 32120, + "input_position": 61, + "layers": [ + { + "after_rms": 0.05464629387267075, + "before_rms": 0.05464629387267075, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06755006814151779, + "before_rms": 0.06755006814151779, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.05537102787014218, + "before_rms": 0.05537102787014218, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.7564307778216667, + "before_rms": 0.7564307778216667, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 18, + "token_id": 321, + "input_position": 62, + "layers": [ + { + "after_rms": 0.07894008680252157, + "before_rms": 0.07894008680252157, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07594297305618296, + "before_rms": 0.07594297305618296, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07215577303867654, + "before_rms": 0.07215577303867654, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.9455045304946262, + "before_rms": 0.9455045304946262, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 19, + "token_id": 22817, + "input_position": 63, + "layers": [ + { + "after_rms": 0.03382951016881296, + "before_rms": 0.03382951016881296, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.04193784117869559, + "before_rms": 0.04193784117869559, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.054778189331242615, + "before_rms": 0.054778189331242615, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.6269059013903844, + "before_rms": 0.6269059013903844, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + }, + { + "step": 20, + "token_id": 13, + "input_position": 64, + "layers": [ + { + "after_rms": 0.07374082621976509, + "before_rms": 0.07374082621976509, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.07036659145204018, + "before_rms": 0.07036659145204018, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.06651851708741562, + "before_rms": 0.06651851708741562, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 20480 + }, + { + "after_rms": 0.8993101756034922, + "before_rms": 0.8993101756034922, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 20480 + } + ] + } + ] + } + ], + "model": "qwen" + } + ] + }, + "surgery": { + "title": "Model surgery: what else changed?", + "summary": "Scale one Flint feed-forward branch to 1, 0.5 or 0. Outputs change on some cases; the same four of six objective checks pass at every scale.", + "method": "Modify the computed ffn_out-12 tensor before residual addition at every prompt and decode position. Measure pre/post RMS and neighboring blocks. No checkpoint weights are rewritten.", + "limitations": [ + "One block, six tasks and one greedy run per condition. No improvement claim. Later outputs can differ in context as well as intervention. Only the first-frame upstream equality is a direct matched-input control." + ], + "cases": [ + { + "id": "arithmetic", + "title": "Exact arithmetic", + "prompt": "Calculate (17 * 23) - 19. Reply with the integer only.", + "checks": [ + "Rule: integer", + "Expected: 372", + "Baseline, scale 1: failed", + "Half FFN, scale 0.5: failed", + "Zero FFN, scale 0: failed" + ], + "intervention": { + "tensor": "ffn_out-12", + "layer_indexing": "zero-based", + "applies_to": "all prompt and generated positions on each forward pass", + "operation": "scale computed FFN output before residual addition", + "scales": [ + 1, + 0.5, + 0 + ], + "weights_modified": false, + "kind": "activation_ablation", + "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." + }, + "conditions": [ + { + "id": "baseline", + "label": "Baseline, scale 1", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "374", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 33, + "elapsed_seconds": 0.738173075, + "request_wall_seconds": 0.7542800180381164, + "time_to_first_token_seconds": 0.689367342973128, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.0042319691359885515, + "block12_after_rms": 0.0042319691359885515 + }, + "check": { + "passed": false, + "rule": "integer", + "expected": "372", + "character_count": 3, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 9056, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004216483101124048, + "before_rms": 0.004216483101124048, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.0042319691359885515, + "before_rms": 0.0042319691359885515, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.005525138486063405, + "before_rms": 0.005525138486063405, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.014221511290226446, + "before_rms": 0.014221511290226446, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 270336 + } + ] + } + ] + }, + { + "id": "half", + "label": "Half FFN, scale 0.5", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "352", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 33, + "elapsed_seconds": 0.719981441, + "request_wall_seconds": 0.7371821940178052, + "time_to_first_token_seconds": 0.6725124060176313, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.0042319691359885515, + "block12_after_rms": 0.0021159845679942758 + }, + "check": { + "passed": false, + "rule": "integer", + "expected": "372", + "character_count": 3, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0.5 + }, + "measurements": [ + { + "step": 0, + "token_id": 11510, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004216483101124048, + "before_rms": 0.004216483101124048, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.0021159845679942758, + "before_rms": 0.0042319691359885515, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.005548826881067284, + "before_rms": 0.005548826881067284, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.013772509153955564, + "before_rms": 0.013772509153955564, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 270336 + } + ] + } + ] + }, + { + "id": "zero", + "label": "Zero FFN, scale 0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "340", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 33, + "elapsed_seconds": 0.721295279, + "request_wall_seconds": 0.7335850830422714, + "time_to_first_token_seconds": 0.6692783629987389, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.0042319691359885515, + "block12_after_rms": 0.0 + }, + "check": { + "passed": false, + "rule": "integer", + "expected": "372", + "character_count": 3, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0 + }, + "measurements": [ + { + "step": 0, + "token_id": 10333, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004216483101124048, + "before_rms": 0.004216483101124048, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.0, + "before_rms": 0.0042319691359885515, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.0056156565381101035, + "before_rms": 0.0056156565381101035, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 270336 + }, + { + "after_rms": 0.013872404589524597, + "before_rms": 0.013872404589524597, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 270336 + } + ] + } + ] + } + ], + "comparison": { + "untargeted_first_frame_block11_identical": true, + "downstream_first_frame_block13_changed": true, + "weights_modified": false, + "all_saved_frames_available_in_results": true + } + }, + { + "id": "json", + "title": "Strict JSON format", + "prompt": "Return exactly this JSON object and no other text: {\"status\":\"ok\",\"count\":3}", + "checks": [ + "Rule: json", + "Expected: {'status': 'ok', 'count': 3}", + "Baseline, scale 1: passed", + "Half FFN, scale 0.5: passed", + "Zero FFN, scale 0: passed" + ], + "intervention": { + "tensor": "ffn_out-12", + "layer_indexing": "zero-based", + "applies_to": "all prompt and generated positions on each forward pass", + "operation": "scale computed FFN output before residual addition", + "scales": [ + 1, + 0.5, + 0 + ], + "weights_modified": false, + "kind": "activation_ablation", + "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." + }, + "conditions": [ + { + "id": "baseline", + "label": "Baseline, scale 1", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "{\"status\":\"ok\",\"count\":3}", + "metrics": { + "generated_tokens": 9, + "prompt_tokens": 35, + "elapsed_seconds": 1.259416502, + "request_wall_seconds": 1.2757591230329126, + "time_to_first_token_seconds": 0.7007529039401561, + "decode_tokens_per_second": 13.911468772577386, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004073247840262152, + "block12_after_rms": 0.004073247840262152 + }, + "check": { + "passed": true, + "rule": "json", + "expected": { + "status": "ok", + "count": 3 + }, + "character_count": 25, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 5482, + "input_position": 34, + "layers": [ + { + "after_rms": 0.00442951382677711, + "before_rms": 0.00442951382677711, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.004073247840262152, + "before_rms": 0.004073247840262152, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.00524975286610561, + "before_rms": 0.00524975286610561, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.014444385872118682, + "before_rms": 0.014444385872118682, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 286720 + } + ] + }, + { + "step": 1, + "token_id": 5462, + "input_position": 35, + "layers": [ + { + "after_rms": 0.005793034803083275, + "before_rms": 0.005793034803083275, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0046820736039610645, + "before_rms": 0.0046820736039610645, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00649377540352318, + "before_rms": 0.00649377540352318, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019726933354943656, + "before_rms": 0.019726933354943656, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 4890, + "input_position": 36, + "layers": [ + { + "after_rms": 0.005294017353787343, + "before_rms": 0.005294017353787343, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0034695353362188502, + "before_rms": 0.0034695353362188502, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00738829527350583, + "before_rms": 0.00738829527350583, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019360584231844193, + "before_rms": 0.019360584231844193, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 544, + "input_position": 37, + "layers": [ + { + "after_rms": 0.00491148399515983, + "before_rms": 0.00491148399515983, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00380345983046727, + "before_rms": 0.00380345983046727, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005149091746140284, + "before_rms": 0.005149091746140284, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016276301961648348, + "before_rms": 0.016276301961648348, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1147, + "input_position": 38, + "layers": [ + { + "after_rms": 0.006222679356489009, + "before_rms": 0.006222679356489009, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0036829646504058975, + "before_rms": 0.0036829646504058975, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006655483694052688, + "before_rms": 0.006655483694052688, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019208434114849963, + "before_rms": 0.019208434114849963, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 2816, + "input_position": 39, + "layers": [ + { + "after_rms": 0.005760162518562755, + "before_rms": 0.005760162518562755, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003275269655906223, + "before_rms": 0.003275269655906223, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004388659397945476, + "before_rms": 0.004388659397945476, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015988995629268844, + "before_rms": 0.015988995629268844, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 1129, + "input_position": 40, + "layers": [ + { + "after_rms": 0.004863272662823825, + "before_rms": 0.004863272662823825, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0038314286271215804, + "before_rms": 0.0038314286271215804, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007286998970545343, + "before_rms": 0.007286998970545343, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018843493317539196, + "before_rms": 0.018843493317539196, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 18, + "input_position": 41, + "layers": [ + { + "after_rms": 0.004603991231053795, + "before_rms": 0.004603991231053795, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0030208238582806433, + "before_rms": 0.0030208238582806433, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004703706397060763, + "before_rms": 0.004703706397060763, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.013840299536615416, + "before_rms": 0.013840299536615416, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 92, + "input_position": 42, + "layers": [ + { + "after_rms": 0.0039039619301298022, + "before_rms": 0.0039039619301298022, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004057874942645444, + "before_rms": 0.004057874942645444, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006582297878179331, + "before_rms": 0.006582297878179331, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016272639100438333, + "before_rms": 0.016272639100438333, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "half", + "label": "Half FFN, scale 0.5", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "{\"status\":\"ok\",\"count\":3}", + "metrics": { + "generated_tokens": 9, + "prompt_tokens": 35, + "elapsed_seconds": 1.246835879, + "request_wall_seconds": 1.262888611992821, + "time_to_first_token_seconds": 0.6985334729542956, + "decode_tokens_per_second": 14.17425932571783, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004073247840262152, + "block12_after_rms": 0.002036623920131076 + }, + "check": { + "passed": true, + "rule": "json", + "expected": { + "status": "ok", + "count": 3 + }, + "character_count": 25, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0.5 + }, + "measurements": [ + { + "step": 0, + "token_id": 5482, + "input_position": 34, + "layers": [ + { + "after_rms": 0.00442951382677711, + "before_rms": 0.00442951382677711, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.002036623920131076, + "before_rms": 0.004073247840262152, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.00511203630467351, + "before_rms": 0.00511203630467351, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.014008885307881237, + "before_rms": 0.014008885307881237, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 286720 + } + ] + }, + { + "step": 1, + "token_id": 5462, + "input_position": 35, + "layers": [ + { + "after_rms": 0.005793034803083275, + "before_rms": 0.005793034803083275, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0023410368019805322, + "before_rms": 0.0046820736039610645, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006683535999525596, + "before_rms": 0.006683535999525596, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01886032206135178, + "before_rms": 0.01886032206135178, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 4890, + "input_position": 36, + "layers": [ + { + "after_rms": 0.005294017353787343, + "before_rms": 0.005294017353787343, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0017347676681094251, + "before_rms": 0.0034695353362188502, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0076820001587381715, + "before_rms": 0.0076820001587381715, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019039892595830377, + "before_rms": 0.019039892595830377, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 544, + "input_position": 37, + "layers": [ + { + "after_rms": 0.00491148399515983, + "before_rms": 0.00491148399515983, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.001901729915233635, + "before_rms": 0.00380345983046727, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005344106452152085, + "before_rms": 0.005344106452152085, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01594381241067434, + "before_rms": 0.01594381241067434, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1147, + "input_position": 38, + "layers": [ + { + "after_rms": 0.006222679356489009, + "before_rms": 0.006222679356489009, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0018414823252029487, + "before_rms": 0.0036829646504058975, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00664047642037073, + "before_rms": 0.00664047642037073, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01916710645697236, + "before_rms": 0.01916710645697236, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 2816, + "input_position": 39, + "layers": [ + { + "after_rms": 0.005760162518562755, + "before_rms": 0.005760162518562755, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0016376348279531115, + "before_rms": 0.003275269655906223, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0046118119033721995, + "before_rms": 0.0046118119033721995, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015733109219504087, + "before_rms": 0.015733109219504087, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 1129, + "input_position": 40, + "layers": [ + { + "after_rms": 0.004863272662823825, + "before_rms": 0.004863272662823825, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0019157143135607902, + "before_rms": 0.0038314286271215804, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007354879141339851, + "before_rms": 0.007354879141339851, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0188168576455134, + "before_rms": 0.0188168576455134, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 18, + "input_position": 41, + "layers": [ + { + "after_rms": 0.004603991231053795, + "before_rms": 0.004603991231053795, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0015104119291403217, + "before_rms": 0.0030208238582806433, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004847850483184061, + "before_rms": 0.004847850483184061, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.013329843619400705, + "before_rms": 0.013329843619400705, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 92, + "input_position": 42, + "layers": [ + { + "after_rms": 0.0039039619301298022, + "before_rms": 0.0039039619301298022, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002028937471322722, + "before_rms": 0.004057874942645444, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006716603826791902, + "before_rms": 0.006716603826791902, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015738018767318247, + "before_rms": 0.015738018767318247, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "zero", + "label": "Zero FFN, scale 0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "{\"status\":\"ok\",\"count\":3}", + "metrics": { + "generated_tokens": 9, + "prompt_tokens": 35, + "elapsed_seconds": 1.252187772, + "request_wall_seconds": 1.2679177080281079, + "time_to_first_token_seconds": 0.7051889860304072, + "decode_tokens_per_second": 14.213787672176204, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004073247840262152, + "block12_after_rms": 0.0 + }, + "check": { + "passed": true, + "rule": "json", + "expected": { + "status": "ok", + "count": 3 + }, + "character_count": 25, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0 + }, + "measurements": [ + { + "step": 0, + "token_id": 5482, + "input_position": 34, + "layers": [ + { + "after_rms": 0.00442951382677711, + "before_rms": 0.00442951382677711, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.0, + "before_rms": 0.004073247840262152, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.005232460518540055, + "before_rms": 0.005232460518540055, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 286720 + }, + { + "after_rms": 0.014132856084779098, + "before_rms": 0.014132856084779098, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 286720 + } + ] + }, + { + "step": 1, + "token_id": 5462, + "input_position": 35, + "layers": [ + { + "after_rms": 0.005793034803083275, + "before_rms": 0.005793034803083275, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0046820736039610645, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006932431350240155, + "before_rms": 0.006932431350240155, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018253630668038652, + "before_rms": 0.018253630668038652, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 4890, + "input_position": 36, + "layers": [ + { + "after_rms": 0.005294017353787343, + "before_rms": 0.005294017353787343, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0034695353362188502, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007877159840683485, + "before_rms": 0.007877159840683485, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018856526179738382, + "before_rms": 0.018856526179738382, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 544, + "input_position": 37, + "layers": [ + { + "after_rms": 0.00491148399515983, + "before_rms": 0.00491148399515983, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.00380345983046727, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005610946438388422, + "before_rms": 0.005610946438388422, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016040718140881722, + "before_rms": 0.016040718140881722, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1147, + "input_position": 38, + "layers": [ + { + "after_rms": 0.006222679356489009, + "before_rms": 0.006222679356489009, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0036829646504058975, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0065644993447098255, + "before_rms": 0.0065644993447098255, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01935782943457066, + "before_rms": 0.01935782943457066, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 2816, + "input_position": 39, + "layers": [ + { + "after_rms": 0.005760162518562755, + "before_rms": 0.005760162518562755, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.003275269655906223, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0050584932701879986, + "before_rms": 0.0050584932701879986, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015686042844506247, + "before_rms": 0.015686042844506247, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 1129, + "input_position": 40, + "layers": [ + { + "after_rms": 0.004863272662823825, + "before_rms": 0.004863272662823825, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0038314286271215804, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007195615412259338, + "before_rms": 0.007195615412259338, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018477358488448153, + "before_rms": 0.018477358488448153, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 18, + "input_position": 41, + "layers": [ + { + "after_rms": 0.004603991231053795, + "before_rms": 0.004603991231053795, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0030208238582806433, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005075615167638164, + "before_rms": 0.005075615167638164, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.013303633799015144, + "before_rms": 0.013303633799015144, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 92, + "input_position": 42, + "layers": [ + { + "after_rms": 0.0039039619301298022, + "before_rms": 0.0039039619301298022, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.004057874942645444, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0069263167645750995, + "before_rms": 0.0069263167645750995, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015314707984930225, + "before_rms": 0.015314707984930225, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + } + ], + "comparison": { + "untargeted_first_frame_block11_identical": true, + "downstream_first_frame_block13_changed": true, + "weights_modified": false, + "all_saved_frames_available_in_results": true + } + }, + { + "id": "fact", + "title": "Short factual recall", + "prompt": "What is the chemical symbol for gold? Reply with the symbol only.", + "checks": [ + "Rule: exact", + "Expected: Au", + "Baseline, scale 1: passed", + "Half FFN, scale 0.5: passed", + "Zero FFN, scale 0: passed" + ], + "intervention": { + "tensor": "ffn_out-12", + "layer_indexing": "zero-based", + "applies_to": "all prompt and generated positions on each forward pass", + "operation": "scale computed FFN output before residual addition", + "scales": [ + 1, + 0.5, + 0 + ], + "weights_modified": false, + "kind": "activation_ablation", + "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." + }, + "conditions": [ + { + "id": "baseline", + "label": "Baseline, scale 1", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "Au", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 30, + "elapsed_seconds": 0.650667801, + "request_wall_seconds": 0.6673325290903449, + "time_to_first_token_seconds": 0.6034285150235519, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004120685912594556, + "block12_after_rms": 0.004120685912594556 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "Au", + "character_count": 2, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 22911, + "input_position": 29, + "layers": [ + { + "after_rms": 0.0042503533024456225, + "before_rms": 0.0042503533024456225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.004120685912594556, + "before_rms": 0.004120685912594556, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.005313107066375563, + "before_rms": 0.005313107066375563, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.01490764588916619, + "before_rms": 0.01490764588916619, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 245760 + } + ] + } + ] + }, + { + "id": "half", + "label": "Half FFN, scale 0.5", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "Au", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 30, + "elapsed_seconds": 0.751291221, + "request_wall_seconds": 0.7725467890268192, + "time_to_first_token_seconds": 0.6922730850055814, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004120685912594556, + "block12_after_rms": 0.002060342956297278 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "Au", + "character_count": 2, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0.5 + }, + "measurements": [ + { + "step": 0, + "token_id": 22911, + "input_position": 29, + "layers": [ + { + "after_rms": 0.0042503533024456225, + "before_rms": 0.0042503533024456225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.002060342956297278, + "before_rms": 0.004120685912594556, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.005343264427809506, + "before_rms": 0.005343264427809506, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.014565723365685252, + "before_rms": 0.014565723365685252, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 245760 + } + ] + } + ] + }, + { + "id": "zero", + "label": "Zero FFN, scale 0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "Au", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 30, + "elapsed_seconds": 0.764433258, + "request_wall_seconds": 0.7862005030037835, + "time_to_first_token_seconds": 0.7070632589748129, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41799256, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004120685912594556, + "block12_after_rms": 0.0 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "Au", + "character_count": 2, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0 + }, + "measurements": [ + { + "step": 0, + "token_id": 22911, + "input_position": 29, + "layers": [ + { + "after_rms": 0.0042503533024456225, + "before_rms": 0.0042503533024456225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.0, + "before_rms": 0.004120685912594556, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.005448940453841594, + "before_rms": 0.005448940453841594, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 245760 + }, + { + "after_rms": 0.01475624234457033, + "before_rms": 0.01475624234457033, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 245760 + } + ] + } + ] + } + ], + "comparison": { + "untargeted_first_frame_block11_identical": true, + "downstream_first_frame_block13_changed": true, + "weights_modified": false, + "all_saved_frames_available_in_results": true + } + }, + { + "id": "correction", + "title": "Use the corrected fact", + "prompt": "The access color was blue. Correction: the access color is now amber. What is the current access color? Reply with one word only.", + "checks": [ + "Rule: exact", + "Expected: amber", + "Baseline, scale 1: passed", + "Half FFN, scale 0.5: passed", + "Zero FFN, scale 0: passed" + ], + "intervention": { + "tensor": "ffn_out-12", + "layer_indexing": "zero-based", + "applies_to": "all prompt and generated positions on each forward pass", + "operation": "scale computed FFN output before residual addition", + "scales": [ + 1, + 0.5, + 0 + ], + "weights_modified": false, + "kind": "activation_ablation", + "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." + }, + "conditions": [ + { + "id": "baseline", + "label": "Baseline, scale 1", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "amber", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 44, + "elapsed_seconds": 0.974556193, + "request_wall_seconds": 0.9955227529862896, + "time_to_first_token_seconds": 0.9343198160640895, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.00441159816545625, + "block12_after_rms": 0.00441159816545625 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "amber", + "character_count": 5, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 13732, + "input_position": 43, + "layers": [ + { + "after_rms": 0.004252703885422568, + "before_rms": 0.004252703885422568, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.00441159816545625, + "before_rms": 0.00441159816545625, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.005657736200656421, + "before_rms": 0.005657736200656421, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.014876767054565872, + "before_rms": 0.014876767054565872, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 360448 + } + ] + } + ] + }, + { + "id": "half", + "label": "Half FFN, scale 0.5", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "amber", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 44, + "elapsed_seconds": 0.845619764, + "request_wall_seconds": 0.8612813500221819, + "time_to_first_token_seconds": 0.8017731390427798, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.00441159816545625, + "block12_after_rms": 0.002205799082728125 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "amber", + "character_count": 5, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0.5 + }, + "measurements": [ + { + "step": 0, + "token_id": 13732, + "input_position": 43, + "layers": [ + { + "after_rms": 0.004252703885422568, + "before_rms": 0.004252703885422568, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.002205799082728125, + "before_rms": 0.00441159816545625, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.005628658790824781, + "before_rms": 0.005628658790824781, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.014474070728207288, + "before_rms": 0.014474070728207288, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 360448 + } + ] + } + ] + }, + { + "id": "zero", + "label": "Zero FFN, scale 0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "amber", + "metrics": { + "generated_tokens": 1, + "prompt_tokens": 44, + "elapsed_seconds": 1.003642273, + "request_wall_seconds": 1.0181882489705458, + "time_to_first_token_seconds": 0.9386005689157173, + "decode_tokens_per_second": null, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.00441159816545625, + "block12_after_rms": 0.0 + }, + "check": { + "passed": true, + "rule": "exact", + "expected": "amber", + "character_count": 5, + "word_count": 1 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0 + }, + "measurements": [ + { + "step": 0, + "token_id": 13732, + "input_position": 43, + "layers": [ + { + "after_rms": 0.004252703885422568, + "before_rms": 0.004252703885422568, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.0, + "before_rms": 0.00441159816545625, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.005651444761795099, + "before_rms": 0.005651444761795099, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 360448 + }, + { + "after_rms": 0.01456451414024136, + "before_rms": 0.01456451414024136, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 360448 + } + ] + } + ] + } + ], + "comparison": { + "untargeted_first_frame_block11_identical": true, + "downstream_first_frame_block13_changed": true, + "weights_modified": false, + "all_saved_frames_available_in_results": true + } + }, + { + "id": "code", + "title": "Python expression repair", + "prompt": "Complete the function with one Python expression only, no markdown: def is_even(n): return ___ . The function must return True exactly when integer n is even.", + "checks": [ + "Rule: expression", + "Expected: n % 2 == 0", + "Baseline, scale 1: failed", + "Half FFN, scale 0.5: failed", + "Zero FFN, scale 0: failed" + ], + "intervention": { + "tensor": "ffn_out-12", + "layer_indexing": "zero-based", + "applies_to": "all prompt and generated positions on each forward pass", + "operation": "scale computed FFN output before residual addition", + "scales": [ + 1, + 0.5, + 0 + ], + "weights_modified": false, + "kind": "activation_ablation", + "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." + }, + "conditions": [ + { + "id": "baseline", + "label": "Baseline, scale 1", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "return n % 2 == 0", + "metrics": { + "generated_tokens": 8, + "prompt_tokens": 48, + "elapsed_seconds": 1.522581758, + "request_wall_seconds": 1.5477090129861608, + "time_to_first_token_seconds": 0.9909320520237088, + "decode_tokens_per_second": 12.570319015535855, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004202357680490731, + "block12_after_rms": 0.004202357680490731 + }, + "check": { + "passed": false, + "rule": "expression", + "expected": "n % 2 == 0", + "character_count": 17, + "word_count": 6 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 963, + "input_position": 47, + "layers": [ + { + "after_rms": 0.004194344649342437, + "before_rms": 0.004194344649342437, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.004202357680490731, + "before_rms": 0.004202357680490731, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.0052131473290487635, + "before_rms": 0.0052131473290487635, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.014781441026154475, + "before_rms": 0.014781441026154475, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 393216 + } + ] + }, + { + "step": 1, + "token_id": 307, + "input_position": 48, + "layers": [ + { + "after_rms": 0.005277509304535801, + "before_rms": 0.005277509304535801, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00609774510971192, + "before_rms": 0.00609774510971192, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005608145566049825, + "before_rms": 0.005608145566049825, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018110784621771632, + "before_rms": 0.018110784621771632, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 1152, + "input_position": 49, + "layers": [ + { + "after_rms": 0.004954062696681225, + "before_rms": 0.004954062696681225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004947633813264256, + "before_rms": 0.004947633813264256, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006281951776179792, + "before_rms": 0.006281951776179792, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01793926095231872, + "before_rms": 0.01793926095231872, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 220, + "input_position": 50, + "layers": [ + { + "after_rms": 0.005725603347206012, + "before_rms": 0.005725603347206012, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007613934635856348, + "before_rms": 0.007613934635856348, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007067586106815484, + "before_rms": 0.007067586106815484, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02065595811031347, + "before_rms": 0.02065595811031347, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 17, + "input_position": 51, + "layers": [ + { + "after_rms": 0.004384558665618408, + "before_rms": 0.004384558665618408, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002694924917014614, + "before_rms": 0.002694924917014614, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003985706515046729, + "before_rms": 0.003985706515046729, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.012855823082554165, + "before_rms": 0.012855823082554165, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 823, + "input_position": 52, + "layers": [ + { + "after_rms": 0.00695481022115809, + "before_rms": 0.00695481022115809, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005148284890432175, + "before_rms": 0.005148284890432175, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00860817356635732, + "before_rms": 0.00860817356635732, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019382560123796404, + "before_rms": 0.019382560123796404, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 220, + "input_position": 53, + "layers": [ + { + "after_rms": 0.008360185779230117, + "before_rms": 0.008360185779230117, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005331645591182674, + "before_rms": 0.005331645591182674, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.011988298999222614, + "before_rms": 0.011988298999222614, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02324778120292584, + "before_rms": 0.02324778120292584, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 15, + "input_position": 54, + "layers": [ + { + "after_rms": 0.004389820760472024, + "before_rms": 0.004389820760472024, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002823151948738866, + "before_rms": 0.002823151948738866, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004278350453083995, + "before_rms": 0.004278350453083995, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.013667174465740126, + "before_rms": 0.013667174465740126, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "half", + "label": "Half FFN, scale 0.5", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "return n % 2 == 0", + "metrics": { + "generated_tokens": 8, + "prompt_tokens": 48, + "elapsed_seconds": 1.451376005, + "request_wall_seconds": 1.5098812960786745, + "time_to_first_token_seconds": 0.9736877980176359, + "decode_tokens_per_second": 13.053039182374336, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004202357680490731, + "block12_after_rms": 0.0021011788402453655 + }, + "check": { + "passed": false, + "rule": "expression", + "expected": "n % 2 == 0", + "character_count": 17, + "word_count": 6 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0.5 + }, + "measurements": [ + { + "step": 0, + "token_id": 963, + "input_position": 47, + "layers": [ + { + "after_rms": 0.004194344649342437, + "before_rms": 0.004194344649342437, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.0021011788402453655, + "before_rms": 0.004202357680490731, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.005090052356663985, + "before_rms": 0.005090052356663985, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.014359352110858759, + "before_rms": 0.014359352110858759, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 393216 + } + ] + }, + { + "step": 1, + "token_id": 307, + "input_position": 48, + "layers": [ + { + "after_rms": 0.005277509304535801, + "before_rms": 0.005277509304535801, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00304887255485596, + "before_rms": 0.00609774510971192, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006014683540483773, + "before_rms": 0.006014683540483773, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017053751245143085, + "before_rms": 0.017053751245143085, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 1152, + "input_position": 49, + "layers": [ + { + "after_rms": 0.004954062696681225, + "before_rms": 0.004954062696681225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002473816906632128, + "before_rms": 0.004947633813264256, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006547528545526086, + "before_rms": 0.006547528545526086, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017097207015519773, + "before_rms": 0.017097207015519773, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 220, + "input_position": 50, + "layers": [ + { + "after_rms": 0.005725603347206012, + "before_rms": 0.005725603347206012, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003806967317928174, + "before_rms": 0.007613934635856348, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007202666322056073, + "before_rms": 0.007202666322056073, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019556659993829776, + "before_rms": 0.019556659993829776, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 17, + "input_position": 51, + "layers": [ + { + "after_rms": 0.004384558665618408, + "before_rms": 0.004384558665618408, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.001347462458507307, + "before_rms": 0.002694924917014614, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004306118830807208, + "before_rms": 0.004306118830807208, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.012367683992419722, + "before_rms": 0.012367683992419722, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 823, + "input_position": 52, + "layers": [ + { + "after_rms": 0.00695481022115809, + "before_rms": 0.00695481022115809, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0025741424452160877, + "before_rms": 0.005148284890432175, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0085343766880633, + "before_rms": 0.0085343766880633, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018658682254849446, + "before_rms": 0.018658682254849446, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 220, + "input_position": 53, + "layers": [ + { + "after_rms": 0.008360185779230117, + "before_rms": 0.008360185779230117, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002665822795591337, + "before_rms": 0.005331645591182674, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.012100347801828254, + "before_rms": 0.012100347801828254, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02291563837710362, + "before_rms": 0.02291563837710362, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 15, + "input_position": 54, + "layers": [ + { + "after_rms": 0.004389820760472024, + "before_rms": 0.004389820760472024, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.001411575974369433, + "before_rms": 0.002823151948738866, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00448153573004611, + "before_rms": 0.00448153573004611, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.012590223970782119, + "before_rms": 0.012590223970782119, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "zero", + "label": "Zero FFN, scale 0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "def is_even(n): return n % 2 == 0", + "metrics": { + "generated_tokens": 13, + "prompt_tokens": 48, + "elapsed_seconds": 1.766926317, + "request_wall_seconds": 1.8241781580727547, + "time_to_first_token_seconds": 0.9580211520660669, + "decode_tokens_per_second": 13.853450095998406, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": false, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.004202357680490731, + "block12_after_rms": 0.0 + }, + "check": { + "passed": false, + "rule": "expression", + "expected": "n % 2 == 0", + "character_count": 33, + "word_count": 8 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0 + }, + "measurements": [ + { + "step": 0, + "token_id": 1182, + "input_position": 47, + "layers": [ + { + "after_rms": 0.004194344649342437, + "before_rms": 0.004194344649342437, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.0, + "before_rms": 0.004202357680490731, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.005070940113459419, + "before_rms": 0.005070940113459419, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 393216 + }, + { + "after_rms": 0.014431911821101652, + "before_rms": 0.014431911821101652, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 393216 + } + ] + }, + { + "step": 1, + "token_id": 332, + "input_position": 48, + "layers": [ + { + "after_rms": 0.00480106946797479, + "before_rms": 0.00480106946797479, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0055027957166948055, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0056345791848020155, + "before_rms": 0.0056345791848020155, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017990489860241182, + "before_rms": 0.017990489860241182, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 75154, + "input_position": 49, + "layers": [ + { + "after_rms": 0.005708045099524496, + "before_rms": 0.005708045099524496, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.004207955013596103, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0075556508119026065, + "before_rms": 0.0075556508119026065, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017833711235606925, + "before_rms": 0.017833711235606925, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 1249, + "input_position": 50, + "layers": [ + { + "after_rms": 0.0058045473429944065, + "before_rms": 0.0058045473429944065, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005650785619108264, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007835386583253242, + "before_rms": 0.007835386583253242, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018218131354517456, + "before_rms": 0.018218131354517456, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1682, + "input_position": 51, + "layers": [ + { + "after_rms": 0.005459167018013679, + "before_rms": 0.005459167018013679, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005458814458024571, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006723062471136803, + "before_rms": 0.006723062471136803, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018003037284926235, + "before_rms": 0.018003037284926235, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 597, + "input_position": 52, + "layers": [ + { + "after_rms": 0.004881552841111804, + "before_rms": 0.004881552841111804, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.004638313561400321, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006991441598900912, + "before_rms": 0.006991441598900912, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016203231535129245, + "before_rms": 0.016203231535129245, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 307, + "input_position": 53, + "layers": [ + { + "after_rms": 0.005143842728252836, + "before_rms": 0.005143842728252836, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005127108596991448, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008170598422530883, + "before_rms": 0.008170598422530883, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01798307359808869, + "before_rms": 0.01798307359808869, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 1152, + "input_position": 54, + "layers": [ + { + "after_rms": 0.0049824502938795555, + "before_rms": 0.0049824502938795555, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005199642872045386, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006848563763133479, + "before_rms": 0.006848563763133479, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016730393035201727, + "before_rms": 0.016730393035201727, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 220, + "input_position": 55, + "layers": [ + { + "after_rms": 0.005708276925466525, + "before_rms": 0.005708276925466525, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.008022970417094337, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007115667659550918, + "before_rms": 0.007115667659550918, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019026613242409732, + "before_rms": 0.019026613242409732, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 9, + "token_id": 17, + "input_position": 56, + "layers": [ + { + "after_rms": 0.0042886941345147015, + "before_rms": 0.0042886941345147015, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.002710862488091774, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004265423359902893, + "before_rms": 0.004265423359902893, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01199161845757347, + "before_rms": 0.01199161845757347, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 10, + "token_id": 823, + "input_position": 57, + "layers": [ + { + "after_rms": 0.007280227848737238, + "before_rms": 0.007280227848737238, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005472617971061407, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008876968970410364, + "before_rms": 0.008876968970410364, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.018901080384531817, + "before_rms": 0.018901080384531817, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 11, + "token_id": 220, + "input_position": 58, + "layers": [ + { + "after_rms": 0.008286967782600638, + "before_rms": 0.008286967782600638, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005098250235859169, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01039179921955715, + "before_rms": 0.01039179921955715, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021144833704014477, + "before_rms": 0.021144833704014477, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 12, + "token_id": 15, + "input_position": 59, + "layers": [ + { + "after_rms": 0.004301028624991628, + "before_rms": 0.004301028624991628, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.002916336479821133, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004546469810197431, + "before_rms": 0.004546469810197431, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01185733829801787, + "before_rms": 0.01185733829801787, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + } + ], + "comparison": { + "untargeted_first_frame_block11_identical": true, + "downstream_first_frame_block13_changed": true, + "weights_modified": false, + "all_saved_frames_available_in_results": true + } + }, + { + "id": "brief", + "title": "One sentence explanation", + "prompt": "Explain photosynthesis in exactly one sentence containing the words sunlight and glucose.", + "checks": [ + "Rule: sentence", + "Expected: ['sunlight', 'glucose']", + "Baseline, scale 1: passed", + "Half FFN, scale 0.5: passed", + "Zero FFN, scale 0: passed" + ], + "intervention": { + "tensor": "ffn_out-12", + "layer_indexing": "zero-based", + "applies_to": "all prompt and generated positions on each forward pass", + "operation": "scale computed FFN output before residual addition", + "scales": [ + 1, + 0.5, + 0 + ], + "weights_modified": false, + "kind": "activation_ablation", + "description": "Scale Flint block 12 feed-forward output before residual addition at every prompt and decode position. Compare unchanged (1), half (0.5) and zero (0). Checkpoint files are unchanged." + }, + "conditions": [ + { + "id": "baseline", + "label": "Baseline, scale 1", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "Photosynthesis is the process by which plants use sunlight to create glucose, storing solar energy in chemical form.", + "metrics": { + "generated_tokens": 21, + "prompt_tokens": 29, + "elapsed_seconds": 1.985051171, + "request_wall_seconds": 2.0436750209191814, + "time_to_first_token_seconds": 0.6549666819628328, + "decode_tokens_per_second": 14.402335769662937, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.0044633950081139125, + "block12_after_rms": 0.0044633950081139125 + }, + "check": { + "passed": true, + "rule": "sentence", + "expected": [ + "sunlight", + "glucose" + ], + "character_count": 116, + "word_count": 18 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 1 + }, + "measurements": [ + { + "step": 0, + "token_id": 49106, + "input_position": 28, + "layers": [ + { + "after_rms": 0.004260129892907466, + "before_rms": 0.004260129892907466, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.0044633950081139125, + "before_rms": 0.0044633950081139125, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.005860773055104808, + "before_rms": 0.005860773055104808, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.015381996005573538, + "before_rms": 0.015381996005573538, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 237568 + } + ] + }, + { + "step": 1, + "token_id": 25424, + "input_position": 29, + "layers": [ + { + "after_rms": 0.0056475146712665295, + "before_rms": 0.0056475146712665295, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008613622622386482, + "before_rms": 0.008613622622386482, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006688744656356641, + "before_rms": 0.006688744656356641, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.022823769881939016, + "before_rms": 0.022823769881939016, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 332, + "input_position": 30, + "layers": [ + { + "after_rms": 0.0064819400327529395, + "before_rms": 0.0064819400327529395, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0066762037147183205, + "before_rms": 0.0066762037147183205, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008751997501954894, + "before_rms": 0.008751997501954894, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.027229842448171573, + "before_rms": 0.027229842448171573, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 269, + "input_position": 31, + "layers": [ + { + "after_rms": 0.0045512880200197945, + "before_rms": 0.0045512880200197945, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005065420297108188, + "before_rms": 0.005065420297108188, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005762789431004507, + "before_rms": 0.005762789431004507, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017768401646140505, + "before_rms": 0.017768401646140505, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1260, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004939956954683621, + "before_rms": 0.004939956954683621, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0038142763724026384, + "before_rms": 0.0038142763724026384, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005913005531372122, + "before_rms": 0.005913005531372122, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01761890031507447, + "before_rms": 0.01761890031507447, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 491, + "input_position": 33, + "layers": [ + { + "after_rms": 0.00603825473034925, + "before_rms": 0.00603825473034925, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005399106143511229, + "before_rms": 0.005399106143511229, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008736535756150335, + "before_rms": 0.008736535756150335, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.023288508898713978, + "before_rms": 0.023288508898713978, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 687, + "input_position": 34, + "layers": [ + { + "after_rms": 0.004825344575484225, + "before_rms": 0.004825344575484225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0035722557633475504, + "before_rms": 0.0035722557633475504, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004528394446715205, + "before_rms": 0.004528394446715205, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015558510785105506, + "before_rms": 0.015558510785105506, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 7370, + "input_position": 35, + "layers": [ + { + "after_rms": 0.00599941128253067, + "before_rms": 0.00599941128253067, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00554379069270193, + "before_rms": 0.00554379069270193, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00600245072054164, + "before_rms": 0.00600245072054164, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021459060251264164, + "before_rms": 0.021459060251264164, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 896, + "input_position": 36, + "layers": [ + { + "after_rms": 0.0051950504054689125, + "before_rms": 0.0051950504054689125, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006141893723174439, + "before_rms": 0.006141893723174439, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009373710872705807, + "before_rms": 0.009373710872705807, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024138161560965755, + "before_rms": 0.024138161560965755, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 9, + "token_id": 24044, + "input_position": 37, + "layers": [ + { + "after_rms": 0.008031516167890848, + "before_rms": 0.008031516167890848, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005368163828714329, + "before_rms": 0.005368163828714329, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005839425814527791, + "before_rms": 0.005839425814527791, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019011043273201134, + "before_rms": 0.019011043273201134, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 10, + "token_id": 295, + "input_position": 38, + "layers": [ + { + "after_rms": 0.009314847401164192, + "before_rms": 0.009314847401164192, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00741898116192189, + "before_rms": 0.00741898116192189, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00969530415712232, + "before_rms": 0.00969530415712232, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02606507768278446, + "before_rms": 0.02606507768278446, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 11, + "token_id": 1815, + "input_position": 39, + "layers": [ + { + "after_rms": 0.0051360181463417694, + "before_rms": 0.0051360181463417694, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005146656362487788, + "before_rms": 0.005146656362487788, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005882866866859857, + "before_rms": 0.005882866866859857, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017485786719118993, + "before_rms": 0.017485786719118993, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 12, + "token_id": 18175, + "input_position": 40, + "layers": [ + { + "after_rms": 0.004435782376176152, + "before_rms": 0.004435782376176152, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007327068248162281, + "before_rms": 0.007327068248162281, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006088986661020862, + "before_rms": 0.006088986661020862, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020237327919821312, + "before_rms": 0.020237327919821312, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 13, + "token_id": 11, + "input_position": 41, + "layers": [ + { + "after_rms": 0.010509592928354338, + "before_rms": 0.010509592928354338, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0072267658458743745, + "before_rms": 0.0072267658458743745, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0072706489196035584, + "before_rms": 0.0072706489196035584, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.026778168430377582, + "before_rms": 0.026778168430377582, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 14, + "token_id": 20827, + "input_position": 42, + "layers": [ + { + "after_rms": 0.0036465031366290737, + "before_rms": 0.0036465031366290737, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003456715901650377, + "before_rms": 0.003456715901650377, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005546723538161339, + "before_rms": 0.005546723538161339, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.014684442530549374, + "before_rms": 0.014684442530549374, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 15, + "token_id": 8437, + "input_position": 43, + "layers": [ + { + "after_rms": 0.00628374835469256, + "before_rms": 0.00628374835469256, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007316398861469263, + "before_rms": 0.007316398861469263, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008040298007985535, + "before_rms": 0.008040298007985535, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021912364803827578, + "before_rms": 0.021912364803827578, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 16, + "token_id": 2488, + "input_position": 44, + "layers": [ + { + "after_rms": 0.008596546950231054, + "before_rms": 0.008596546950231054, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005564570157863948, + "before_rms": 0.005564570157863948, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007757301988850941, + "before_rms": 0.007757301988850941, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020669327020220094, + "before_rms": 0.020669327020220094, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 17, + "token_id": 293, + "input_position": 45, + "layers": [ + { + "after_rms": 0.011432653860628939, + "before_rms": 0.011432653860628939, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005583320073300356, + "before_rms": 0.005583320073300356, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009875868656773605, + "before_rms": 0.009875868656773605, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024941505866581348, + "before_rms": 0.024941505866581348, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 18, + "token_id": 6999, + "input_position": 46, + "layers": [ + { + "after_rms": 0.0053301379768005384, + "before_rms": 0.0053301379768005384, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005000846356165002, + "before_rms": 0.005000846356165002, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005456690270258401, + "before_rms": 0.005456690270258401, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016748723982626335, + "before_rms": 0.016748723982626335, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 19, + "token_id": 981, + "input_position": 47, + "layers": [ + { + "after_rms": 0.007946814942115127, + "before_rms": 0.007946814942115127, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005158417494632955, + "before_rms": 0.005158417494632955, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007460860282340885, + "before_rms": 0.007460860282340885, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020411307476897476, + "before_rms": 0.020411307476897476, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 20, + "token_id": 13, + "input_position": 48, + "layers": [ + { + "after_rms": 0.005514051374901345, + "before_rms": 0.005514051374901345, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0070394921880856, + "before_rms": 0.0070394921880856, + "layer": 12, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0066148112542871755, + "before_rms": 0.0066148112542871755, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020639906847932543, + "before_rms": 0.020639906847932543, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "half", + "label": "Half FFN, scale 0.5", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "Photosynthesis is the process by which plants use sunlight to create glucose, storing solar energy in chemical form.", + "metrics": { + "generated_tokens": 21, + "prompt_tokens": 29, + "elapsed_seconds": 2.160672568, + "request_wall_seconds": 2.2178376889787614, + "time_to_first_token_seconds": 0.6635361380176619, + "decode_tokens_per_second": 12.866885314377939, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.0044633950081139125, + "block12_after_rms": 0.0022316975040569563 + }, + "check": { + "passed": true, + "rule": "sentence", + "expected": [ + "sunlight", + "glucose" + ], + "character_count": 116, + "word_count": 18 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0.5 + }, + "measurements": [ + { + "step": 0, + "token_id": 49106, + "input_position": 28, + "layers": [ + { + "after_rms": 0.004260129892907466, + "before_rms": 0.004260129892907466, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.0022316975040569563, + "before_rms": 0.0044633950081139125, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.005735151133850015, + "before_rms": 0.005735151133850015, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.01486159838115774, + "before_rms": 0.01486159838115774, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 237568 + } + ] + }, + { + "step": 1, + "token_id": 25424, + "input_position": 29, + "layers": [ + { + "after_rms": 0.0056475146712665295, + "before_rms": 0.0056475146712665295, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004306811311193241, + "before_rms": 0.008613622622386482, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007436545118137283, + "before_rms": 0.007436545118137283, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020810310578413545, + "before_rms": 0.020810310578413545, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 332, + "input_position": 30, + "layers": [ + { + "after_rms": 0.0064819400327529395, + "before_rms": 0.0064819400327529395, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0033381018573591602, + "before_rms": 0.0066762037147183205, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009007999001296582, + "before_rms": 0.009007999001296582, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.026166335899804105, + "before_rms": 0.026166335899804105, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 269, + "input_position": 31, + "layers": [ + { + "after_rms": 0.0045512880200197945, + "before_rms": 0.0045512880200197945, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002532710148554094, + "before_rms": 0.005065420297108188, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005911283538375086, + "before_rms": 0.005911283538375086, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016813015849254234, + "before_rms": 0.016813015849254234, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1260, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004939956954683621, + "before_rms": 0.004939956954683621, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0019071381862013192, + "before_rms": 0.0038142763724026384, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00595557221350112, + "before_rms": 0.00595557221350112, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017146784199957084, + "before_rms": 0.017146784199957084, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 491, + "input_position": 33, + "layers": [ + { + "after_rms": 0.00603825473034925, + "before_rms": 0.00603825473034925, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0026995530717556144, + "before_rms": 0.005399106143511229, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008767531752803006, + "before_rms": 0.008767531752803006, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.022466743651620685, + "before_rms": 0.022466743651620685, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 687, + "input_position": 34, + "layers": [ + { + "after_rms": 0.004825344575484225, + "before_rms": 0.004825344575484225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0017861278816737752, + "before_rms": 0.0035722557633475504, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.004811283125730314, + "before_rms": 0.004811283125730314, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.014872505126769303, + "before_rms": 0.014872505126769303, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 7370, + "input_position": 35, + "layers": [ + { + "after_rms": 0.00599941128253067, + "before_rms": 0.00599941128253067, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002771895346350965, + "before_rms": 0.00554379069270193, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006169826908450906, + "before_rms": 0.006169826908450906, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02094382532383083, + "before_rms": 0.02094382532383083, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 896, + "input_position": 36, + "layers": [ + { + "after_rms": 0.0051950504054689125, + "before_rms": 0.0051950504054689125, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0030709468615872193, + "before_rms": 0.006141893723174439, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009589415026818995, + "before_rms": 0.009589415026818995, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.023335435813653795, + "before_rms": 0.023335435813653795, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 9, + "token_id": 24044, + "input_position": 37, + "layers": [ + { + "after_rms": 0.008031516167890848, + "before_rms": 0.008031516167890848, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0026840819143571647, + "before_rms": 0.005368163828714329, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005801143981718462, + "before_rms": 0.005801143981718462, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01859743623390119, + "before_rms": 0.01859743623390119, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 10, + "token_id": 295, + "input_position": 38, + "layers": [ + { + "after_rms": 0.009314847401164192, + "before_rms": 0.009314847401164192, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.003709490580960945, + "before_rms": 0.00741898116192189, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00977155465584999, + "before_rms": 0.00977155465584999, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02491673002871025, + "before_rms": 0.02491673002871025, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 11, + "token_id": 1815, + "input_position": 39, + "layers": [ + { + "after_rms": 0.0051360181463417694, + "before_rms": 0.0051360181463417694, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002573328181243894, + "before_rms": 0.005146656362487788, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005799911492815732, + "before_rms": 0.005799911492815732, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016933047785703814, + "before_rms": 0.016933047785703814, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 12, + "token_id": 18175, + "input_position": 40, + "layers": [ + { + "after_rms": 0.004435782376176152, + "before_rms": 0.004435782376176152, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0036635341240811403, + "before_rms": 0.007327068248162281, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00613417633614374, + "before_rms": 0.00613417633614374, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019118337346322598, + "before_rms": 0.019118337346322598, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 13, + "token_id": 11, + "input_position": 41, + "layers": [ + { + "after_rms": 0.010509592928354338, + "before_rms": 0.010509592928354338, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0036133829229371872, + "before_rms": 0.0072267658458743745, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007475141941789141, + "before_rms": 0.007475141941789141, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.025868505159618332, + "before_rms": 0.025868505159618332, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 14, + "token_id": 20827, + "input_position": 42, + "layers": [ + { + "after_rms": 0.0036465031366290737, + "before_rms": 0.0036465031366290737, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0017283579508251886, + "before_rms": 0.003456715901650377, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00533638225204982, + "before_rms": 0.00533638225204982, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.014436190642373524, + "before_rms": 0.014436190642373524, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 15, + "token_id": 8437, + "input_position": 43, + "layers": [ + { + "after_rms": 0.00628374835469256, + "before_rms": 0.00628374835469256, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0036581994307346316, + "before_rms": 0.007316398861469263, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008102150834431271, + "before_rms": 0.008102150834431271, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020984366848385224, + "before_rms": 0.020984366848385224, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 16, + "token_id": 2488, + "input_position": 44, + "layers": [ + { + "after_rms": 0.008596546950231054, + "before_rms": 0.008596546950231054, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002782285078931974, + "before_rms": 0.005564570157863948, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007806100403261486, + "before_rms": 0.007806100403261486, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020226905469513747, + "before_rms": 0.020226905469513747, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 17, + "token_id": 293, + "input_position": 45, + "layers": [ + { + "after_rms": 0.011432653860628939, + "before_rms": 0.011432653860628939, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002791660036650178, + "before_rms": 0.005583320073300356, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009973486185624623, + "before_rms": 0.009973486185624623, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024628251420628085, + "before_rms": 0.024628251420628085, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 18, + "token_id": 6999, + "input_position": 46, + "layers": [ + { + "after_rms": 0.0053301379768005384, + "before_rms": 0.0053301379768005384, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.002500423178082501, + "before_rms": 0.005000846356165002, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0053801173686054875, + "before_rms": 0.0053801173686054875, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016217885602163445, + "before_rms": 0.016217885602163445, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 19, + "token_id": 981, + "input_position": 47, + "layers": [ + { + "after_rms": 0.007946814942115127, + "before_rms": 0.007946814942115127, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0025792087473164773, + "before_rms": 0.005158417494632955, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007857079141286923, + "before_rms": 0.007857079141286923, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020573162868035226, + "before_rms": 0.020573162868035226, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 20, + "token_id": 13, + "input_position": 48, + "layers": [ + { + "after_rms": 0.005514051374901345, + "before_rms": 0.005514051374901345, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0035197460940428, + "before_rms": 0.0070394921880856, + "layer": 12, + "scale": 0.5, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006700234982153566, + "before_rms": 0.006700234982153566, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019504194134602675, + "before_rms": 0.019504194134602675, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + }, + { + "id": "zero", + "label": "Zero FFN, scale 0", + "model": "Recursive Flint", + "quant": "Q8_0", + "output": "Photosynthesis is the process by which plants use sunlight to convert carbon dioxide and water into glucose, releasing oxygen as a byproduct.", + "metrics": { + "generated_tokens": 25, + "prompt_tokens": 29, + "elapsed_seconds": 2.391091285, + "request_wall_seconds": 2.459595420048572, + "time_to_first_token_seconds": 0.7541828820249066, + "decode_tokens_per_second": 14.071746536403907, + "process_peak_rss_kib": 41801304, + "termination": "eos", + "complete": true, + "objective_check_passed": true, + "checkpoint_bytes": 42350359392, + "block12_before_rms": 0.0044633950081139125, + "block12_after_rms": 0.0 + }, + "check": { + "passed": true, + "rule": "sentence", + "expected": [ + "sunlight", + "glucose" + ], + "character_count": 141, + "word_count": 22 + }, + "settings": { + "max_tokens": 64, + "temperature": 0, + "seed": 42, + "reasoning": false, + "intervention_layer": 12, + "intervention_scale": 0 + }, + "measurements": [ + { + "step": 0, + "token_id": 49106, + "input_position": 28, + "layers": [ + { + "after_rms": 0.004260129892907466, + "before_rms": 0.004260129892907466, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.0, + "before_rms": 0.0044633950081139125, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.005661502074284029, + "before_rms": 0.005661502074284029, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 237568 + }, + { + "after_rms": 0.014899536247766184, + "before_rms": 0.014899536247766184, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 237568 + } + ] + }, + { + "step": 1, + "token_id": 25424, + "input_position": 29, + "layers": [ + { + "after_rms": 0.0056475146712665295, + "before_rms": 0.0056475146712665295, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.008613622622386482, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008525602935404572, + "before_rms": 0.008525602935404572, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019733810862655715, + "before_rms": 0.019733810862655715, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 2, + "token_id": 332, + "input_position": 30, + "layers": [ + { + "after_rms": 0.0064819400327529395, + "before_rms": 0.0064819400327529395, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0066762037147183205, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009135572197523601, + "before_rms": 0.009135572197523601, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02557741137246633, + "before_rms": 0.02557741137246633, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 3, + "token_id": 269, + "input_position": 31, + "layers": [ + { + "after_rms": 0.0045512880200197945, + "before_rms": 0.0045512880200197945, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005065420297108188, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006092569880549685, + "before_rms": 0.006092569880549685, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.016503642792897465, + "before_rms": 0.016503642792897465, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 4, + "token_id": 1260, + "input_position": 32, + "layers": [ + { + "after_rms": 0.004939956954683621, + "before_rms": 0.004939956954683621, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0038142763724026384, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006157849949139536, + "before_rms": 0.006157849949139536, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017203518557001942, + "before_rms": 0.017203518557001942, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 5, + "token_id": 491, + "input_position": 33, + "layers": [ + { + "after_rms": 0.00603825473034925, + "before_rms": 0.00603825473034925, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005399106143511229, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00904802642504343, + "before_rms": 0.00904802642504343, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02225766191781135, + "before_rms": 0.02225766191781135, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 6, + "token_id": 687, + "input_position": 34, + "layers": [ + { + "after_rms": 0.004825344575484225, + "before_rms": 0.004825344575484225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.0035722557633475504, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005533683753255847, + "before_rms": 0.005533683753255847, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.014873267868110489, + "before_rms": 0.014873267868110489, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 7, + "token_id": 7370, + "input_position": 35, + "layers": [ + { + "after_rms": 0.00599941128253067, + "before_rms": 0.00599941128253067, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.00554379069270193, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006390727713863449, + "before_rms": 0.006390727713863449, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02091061618332828, + "before_rms": 0.02091061618332828, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 8, + "token_id": 896, + "input_position": 36, + "layers": [ + { + "after_rms": 0.0051950504054689125, + "before_rms": 0.0051950504054689125, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.006141893723174439, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009947159109400337, + "before_rms": 0.009947159109400337, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.023151455733089174, + "before_rms": 0.023151455733089174, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 9, + "token_id": 24044, + "input_position": 37, + "layers": [ + { + "after_rms": 0.008031516167890848, + "before_rms": 0.008031516167890848, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005368163828714329, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006423750612719223, + "before_rms": 0.006423750612719223, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.019295778586505254, + "before_rms": 0.019295778586505254, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 10, + "token_id": 295, + "input_position": 38, + "layers": [ + { + "after_rms": 0.009314847401164192, + "before_rms": 0.009314847401164192, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.00741898116192189, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.009693459099318338, + "before_rms": 0.009693459099318338, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024237413637575162, + "before_rms": 0.024237413637575162, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 11, + "token_id": 6766, + "input_position": 39, + "layers": [ + { + "after_rms": 0.0051360181463417694, + "before_rms": 0.0051360181463417694, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005146656362487788, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005767883251008539, + "before_rms": 0.005767883251008539, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.01708687035677014, + "before_rms": 0.01708687035677014, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 12, + "token_id": 7185, + "input_position": 40, + "layers": [ + { + "after_rms": 0.0073586153911031, + "before_rms": 0.0073586153911031, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.006591188015659504, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0070010157973419635, + "before_rms": 0.0070010157973419635, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.022579139951650742, + "before_rms": 0.022579139951650742, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 13, + "token_id": 23776, + "input_position": 41, + "layers": [ + { + "after_rms": 0.008194359117077644, + "before_rms": 0.008194359117077644, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.007596196041918696, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0075531294632927015, + "before_rms": 0.0075531294632927015, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021887967030610505, + "before_rms": 0.021887967030610505, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 14, + "token_id": 299, + "input_position": 42, + "layers": [ + { + "after_rms": 0.007857027218294529, + "before_rms": 0.007857027218294529, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.008314784468177716, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.008147101435200647, + "before_rms": 0.008147101435200647, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.02264674988179593, + "before_rms": 0.02264674988179593, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 15, + "token_id": 1952, + "input_position": 43, + "layers": [ + { + "after_rms": 0.004688345103889225, + "before_rms": 0.004688345103889225, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.004946909109298921, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006079777076870683, + "before_rms": 0.006079777076870683, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015899963838293046, + "before_rms": 0.015899963838293046, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 16, + "token_id": 831, + "input_position": 44, + "layers": [ + { + "after_rms": 0.007450920463116443, + "before_rms": 0.007450920463116443, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.006978048184271669, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006659637268169784, + "before_rms": 0.006659637268169784, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020155346635423232, + "before_rms": 0.020155346635423232, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 17, + "token_id": 18175, + "input_position": 45, + "layers": [ + { + "after_rms": 0.007443990344351715, + "before_rms": 0.007443990344351715, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005543317793659353, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.00888447540290056, + "before_rms": 0.00888447540290056, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.020591210203462766, + "before_rms": 0.020591210203462766, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 18, + "token_id": 11, + "input_position": 46, + "layers": [ + { + "after_rms": 0.009927458171845966, + "before_rms": 0.009927458171845966, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.006904034655866483, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007498876939890208, + "before_rms": 0.007498876939890208, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024867085464211167, + "before_rms": 0.024867085464211167, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 19, + "token_id": 27762, + "input_position": 47, + "layers": [ + { + "after_rms": 0.003583892367306353, + "before_rms": 0.003583892367306353, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.003674512887912858, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.005366533667651147, + "before_rms": 0.005366533667651147, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.015057416354372646, + "before_rms": 0.015057416354372646, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 20, + "token_id": 11472, + "input_position": 48, + "layers": [ + { + "after_rms": 0.0071015945130851466, + "before_rms": 0.0071015945130851466, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.006051436756781676, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006862596596600148, + "before_rms": 0.006862596596600148, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021336925675778752, + "before_rms": 0.021336925675778752, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 21, + "token_id": 389, + "input_position": 49, + "layers": [ + { + "after_rms": 0.007580029165171899, + "before_rms": 0.007580029165171899, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.006902773819282416, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007204155372984763, + "before_rms": 0.007204155372984763, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.021924978969430717, + "before_rms": 0.021924978969430717, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 22, + "token_id": 260, + "input_position": 50, + "layers": [ + { + "after_rms": 0.00546253050622268, + "before_rms": 0.00546253050622268, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.005206685222875008, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.006133790801518977, + "before_rms": 0.006133790801518977, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.017978821140417742, + "before_rms": 0.017978821140417742, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 23, + "token_id": 93485, + "input_position": 51, + "layers": [ + { + "after_rms": 0.00462712758092533, + "before_rms": 0.00462712758092533, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.004455167000851032, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0069830694392977915, + "before_rms": 0.0069830694392977915, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0157750242951739, + "before_rms": 0.0157750242951739, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + }, + { + "step": 24, + "token_id": 13, + "input_position": 52, + "layers": [ + { + "after_rms": 0.010873441338987835, + "before_rms": 0.010873441338987835, + "layer": 11, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.0, + "before_rms": 0.006011455301735444, + "layer": 12, + "scale": 0.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.007365985817524404, + "before_rms": 0.007365985817524404, + "layer": 13, + "scale": 1.0, + "tensor": "ffn_out", + "tensor_bytes": 8192 + }, + { + "after_rms": 0.024886336584602996, + "before_rms": 0.024886336584602996, + "layer": 13, + "scale": 1.0, + "tensor": "l_out", + "tensor_bytes": 8192 + } + ] + } + ] + } + ], + "comparison": { + "untargeted_first_frame_block11_identical": true, + "downstream_first_frame_block13_changed": true, + "weights_modified": false, + "all_saved_frames_available_in_results": true + } + } + ] + }, + "memory": { + "title": "Remembered, but outdated?", + "summary": "Six short fictional logs test current facts and original facts. Flint and alphaBell Qwen Open answered all six correctly. Predict the answer, then inspect each recorded result.", + "method": "Three log structures, each queried for the current and original access code. Exact four-digit answer scoring. Same decoding settings for both models.", + "limitations": [ + "These prompts are at most 144 tokens. They test short in-context correction and recall, not long-context performance or permanent memory." + ], + "cases": [ + { + "id": "distractors-current", + "title": "A correction among distractors: current fact", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "question": "What is the current museum access code?", + "ground_truth": "8624", + "choices": [ + "4319", + "8624", + "5972" + ], + "kind": "current", + "conditions": [ + { + "id": "flint", + "label": "Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "output": "8624", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=memory-distractors-current-flint&token=1", + "metrics": { + "generated_tokens": 2, + "prompt_tokens": 126, + "elapsed_seconds": 1.662128068, + "termination": "eos", + "exact_answer_passed": true + } + }, + { + "id": "qwen", + "label": "alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "output": "8624", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=memory-distractors-current-qwen&token=1", + "metrics": { + "generated_tokens": 4, + "prompt_tokens": 144, + "elapsed_seconds": 10.655367113, + "termination": "eos", + "exact_answer_passed": true + } + } + ], + "checks": [ + "Expected four-digit answer: 8624", + "Recursive Flint: passed", + "alphaBell Qwen Open 27B: passed" + ] + }, + { + "id": "distractors-original", + "title": "A correction among distractors: original fact", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "question": "What is the original museum access code?", + "ground_truth": "4319", + "choices": [ + "4319", + "8624", + "5972" + ], + "kind": "original", + "conditions": [ + { + "id": "flint", + "label": "Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "output": "4319", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=memory-distractors-original-flint&token=1", + "metrics": { + "generated_tokens": 2, + "prompt_tokens": 126, + "elapsed_seconds": 1.715333822, + "termination": "eos", + "exact_answer_passed": true + } + }, + { + "id": "qwen", + "label": "alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "output": "4319", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=memory-distractors-original-qwen&token=1", + "metrics": { + "generated_tokens": 4, + "prompt_tokens": 144, + "elapsed_seconds": 10.655514738, + "termination": "eos", + "exact_answer_passed": true + } + } + ], + "checks": [ + "Expected four-digit answer: 4319", + "Recursive Flint: passed", + "alphaBell Qwen Open 27B: passed" + ] + }, + { + "id": "short-current", + "title": "One correction: current fact", + "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "question": "What is the current museum access code?", + "ground_truth": "8624", + "choices": [ + "4319", + "8624", + "5972" + ], + "kind": "current", + "conditions": [ + { + "id": "flint", + "label": "Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "output": "8624", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=memory-short-current-flint&token=1", + "metrics": { + "generated_tokens": 2, + "prompt_tokens": 67, + "elapsed_seconds": 1.14846904, + "termination": "eos", + "exact_answer_passed": true + } + }, + { + "id": "qwen", + "label": "alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "output": "8624", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=memory-short-current-qwen&token=1", + "metrics": { + "generated_tokens": 4, + "prompt_tokens": 85, + "elapsed_seconds": 7.286205131, + "termination": "eos", + "exact_answer_passed": true + } + } + ], + "checks": [ + "Expected four-digit answer: 8624", + "Recursive Flint: passed", + "alphaBell Qwen Open 27B: passed" + ] + }, + { + "id": "short-original", + "title": "One correction: original fact", + "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "question": "What is the original museum access code?", + "ground_truth": "4319", + "choices": [ + "4319", + "8624", + "5972" + ], + "kind": "original", + "conditions": [ + { + "id": "flint", + "label": "Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "output": "4319", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=memory-short-original-flint&token=1", + "metrics": { + "generated_tokens": 2, + "prompt_tokens": 67, + "elapsed_seconds": 1.1675653, + "termination": "eos", + "exact_answer_passed": true + } + }, + { + "id": "qwen", + "label": "alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. The museum access code is now 8624. The old code is no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "output": "4319", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=memory-short-original-qwen&token=1", + "metrics": { + "generated_tokens": 4, + "prompt_tokens": 85, + "elapsed_seconds": 7.234881181, + "termination": "eos", + "exact_answer_passed": true + } + } + ], + "checks": [ + "Expected four-digit answer: 4319", + "Recursive Flint: passed", + "alphaBell Qwen Open 27B: passed" + ] + }, + { + "id": "twice-current", + "title": "Two successive corrections: current fact", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "question": "What is the current museum access code?", + "ground_truth": "5972", + "choices": [ + "4319", + "8624", + "5972" + ], + "kind": "current", + "conditions": [ + { + "id": "flint", + "label": "Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "output": "5972", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=memory-twice-current-flint&token=1", + "metrics": { + "generated_tokens": 2, + "prompt_tokens": 105, + "elapsed_seconds": 1.501386008, + "termination": "eos", + "exact_answer_passed": true + } + }, + { + "id": "qwen", + "label": "alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the current museum access code? Answer with only the four digits.", + "output": "5972", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=memory-twice-current-qwen&token=1", + "metrics": { + "generated_tokens": 4, + "prompt_tokens": 125, + "elapsed_seconds": 9.111306724, + "termination": "eos", + "exact_answer_passed": true + } + } + ], + "checks": [ + "Expected four-digit answer: 5972", + "Recursive Flint: passed", + "alphaBell Qwen Open 27B: passed" + ] + }, + { + "id": "twice-original", + "title": "Two successive corrections: original fact", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "question": "What is the original museum access code?", + "ground_truth": "4319", + "choices": [ + "4319", + "8624", + "5972" + ], + "kind": "original", + "conditions": [ + { + "id": "flint", + "label": "Recursive Flint", + "model": "Recursive Flint", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "output": "4319", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=flint&capture=memory-twice-original-flint&token=1", + "metrics": { + "generated_tokens": 2, + "prompt_tokens": 105, + "elapsed_seconds": 1.648641691, + "termination": "eos", + "exact_answer_passed": true + } + }, + { + "id": "qwen", + "label": "alphaBell Qwen Open 27B", + "model": "alphaBell Qwen Open 27B", + "quant": "Q8_0", + "prompt": "Read this fictional log. The original museum access code is 4319. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. A crate is sealed. A lamp is on. The museum access code changes to 8624. Later it changes again to 5972. The earlier codes are no longer current. Question: What is the original museum access code? Answer with only the four digits.", + "output": "4319", + "settings": { + "max_tokens": 16, + "temperature": 0, + "seed": 42, + "reasoning": false + }, + "trace_url": "../index.html?model=qwen&capture=memory-twice-original-qwen&token=1", + "metrics": { + "generated_tokens": 4, + "prompt_tokens": 125, + "elapsed_seconds": 9.107945102, + "termination": "eos", + "exact_answer_passed": true + } + } + ], + "checks": [ + "Expected four-digit answer: 4319", + "Recursive Flint: passed", + "alphaBell Qwen Open 27B: passed" + ] + } + ] + } + } +}