| { |
| "schema_version": 1, |
| "model": { |
| "name": "3-digit-basic-calc", |
| "parameters": 1596160, |
| "source_checkpoint": "ckp/scratchpad_v3_256x2x8_bs32.pt", |
| "representation": "scratchpad-v3", |
| "decoding": "greedy" |
| }, |
| "conversion_validation": { |
| "date": "2026-07-24", |
| "dataset_rows_tokenized": 108000, |
| "tokenizer_id_mismatches": 0, |
| "learned_tensors_compared": 24, |
| "learned_tensor_mismatches": 0, |
| "export_tensor_count": 26, |
| "derived_export_tensors": [ |
| "decoder_layers.0.attn.rope.inv_freq", |
| "decoder_layers.1.attn.rope.inv_freq" |
| ], |
| "prompt_logit_max_abs_difference": 0.0, |
| "complete_162_token_logit_max_abs_difference": 0.0, |
| "argmax_mismatches": 0 |
| }, |
| "validation": { |
| "split": "val", |
| "rows": 4000, |
| "implementation": "exported Hugging Face architecture", |
| "device": "cpu", |
| "elapsed_seconds": 1513.414, |
| "answer": { |
| "+": {"correct": 999, "total": 1000, "accuracy": 0.999}, |
| "-": {"correct": 997, "total": 1000, "accuracy": 0.997}, |
| "*": {"correct": 1000, "total": 1000, "accuracy": 1.0}, |
| "/": {"correct": 968, "total": 1000, "accuracy": 0.968}, |
| "overall": {"correct": 3964, "total": 4000, "accuracy": 0.991} |
| }, |
| "trace_exact": { |
| "+": {"correct": 999, "total": 1000, "accuracy": 0.999}, |
| "-": {"correct": 997, "total": 1000, "accuracy": 0.997}, |
| "*": {"correct": 1000, "total": 1000, "accuracy": 1.0}, |
| "/": {"correct": 949, "total": 1000, "accuracy": 0.949}, |
| "overall": {"correct": 3945, "total": 4000, "accuracy": 0.98625} |
| }, |
| "parse_success": {"correct": 4000, "total": 4000, "accuracy": 1.0} |
| }, |
| "locked_test": { |
| "split": "test", |
| "rows": 4000, |
| "device": "mps", |
| "source_report": "sample/scratchpad_v3_test_eval.json", |
| "answer": { |
| "+": {"correct": 999, "total": 1000, "accuracy": 0.999}, |
| "-": {"correct": 999, "total": 1000, "accuracy": 0.999}, |
| "*": {"correct": 1000, "total": 1000, "accuracy": 1.0}, |
| "/": {"correct": 952, "total": 1000, "accuracy": 0.952}, |
| "overall": {"correct": 3950, "total": 4000, "accuracy": 0.9875} |
| }, |
| "sequence_exact": {"correct": 3925, "total": 4000, "accuracy": 0.98125}, |
| "parse_success": {"correct": 4000, "total": 4000, "accuracy": 1.0} |
| }, |
| "fresh_prompt_disjoint": { |
| "seed": 20260723, |
| "rows": 3000, |
| "sampling": "uniform random operands and operation-balanced, excluding all 108000 dataset prompts", |
| "implementation": "native checkpoint after exact exported-logit parity was established", |
| "answer": { |
| "+": {"correct": 747, "total": 750, "accuracy": 0.996}, |
| "-": {"correct": 748, "total": 750, "accuracy": 0.9973333333}, |
| "*": {"correct": 750, "total": 750, "accuracy": 1.0}, |
| "/": {"correct": 698, "total": 750, "accuracy": 0.9306666667}, |
| "overall": {"correct": 2943, "total": 3000, "accuracy": 0.981} |
| }, |
| "trace_exact": { |
| "+": {"correct": 747, "total": 750, "accuracy": 0.996}, |
| "-": {"correct": 748, "total": 750, "accuracy": 0.9973333333}, |
| "*": {"correct": 750, "total": 750, "accuracy": 1.0}, |
| "/": {"correct": 669, "total": 750, "accuracy": 0.892} |
| }, |
| "elapsed_seconds": 1227.1992 |
| }, |
| "division_failure_diagnosis": { |
| "source": "locked_test", |
| "answer_failures": 48, |
| "first_wrong_component": { |
| "quotient_digit_or_product": 47, |
| "remainder": 1 |
| }, |
| "first_wrong_position": { |
| "1": 4, |
| "2": 11, |
| "3": 17, |
| "4": 10, |
| "5": 4, |
| "6": 2 |
| } |
| }, |
| "grpo": { |
| "baseline_division_accuracy": 0.968, |
| "trial_1_division_accuracy": 0.968, |
| "trial_2_division_accuracy": 0.966, |
| "selected_checkpoint": "supervised baseline", |
| "reason": "neither trial improved gated greedy validation" |
| } |
| } |
|
|
