fix §4.4 interpretation to match data; aligned in §6
Browse files- make_report.py +12 -3
make_report.py
CHANGED
|
@@ -735,9 +735,15 @@ def report(evals, rows):
|
|
| 735 |
"",
|
| 736 |
*ss_lines,
|
| 737 |
"",
|
| 738 |
-
"**Observation:** accuracy degrades
|
| 739 |
-
"
|
| 740 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 741 |
"",
|
| 742 |
"### 4.5 Training dynamics",
|
| 743 |
"",
|
|
@@ -844,6 +850,9 @@ def report(evals, rows):
|
|
| 844 |
"- The rec-1x-dynamic family trains at ~5 average block-",
|
| 845 |
" applications per step (R ~ U{2..8}), slightly above the matched",
|
| 846 |
" 4 - flagged rather than hidden.",
|
|
|
|
|
|
|
|
|
|
| 847 |
"- Two duplicate jobs created by interrupted tool calls were",
|
| 848 |
" cancelled (~$0.30 wasted).",
|
| 849 |
"- Dependency pin: trackio 0.42.0 requires huggingface-hub<2.0;",
|
|
|
|
| 735 |
"",
|
| 736 |
*ss_lines,
|
| 737 |
"",
|
| 738 |
+
"**Observation:** accuracy degrades sharply and non-monotonically:",
|
| 739 |
+
"fixed-4 drops from 0.48 at S'=64 to 0.12 at S'=32 and 0.22 at",
|
| 740 |
+
"S'=16; rec-1x4 from 0.76 to 0.10 and 0.22. Relative to chance",
|
| 741 |
+
"(1/S'), smaller state spaces are far harder (roughly 3-4x chance,",
|
| 742 |
+
"vs ~30x chance at S'=64), so the learned solution is strongly",
|
| 743 |
+
"tied to the trained state space - it is not a clean, vocabulary-",
|
| 744 |
+
"independent algorithm. (The S'=16 > S'=32 inversion is unexplained;",
|
| 745 |
+
"random functions on smaller domains enter cycles sooner, which may",
|
| 746 |
+
"change the task's structure.)",
|
| 747 |
"",
|
| 748 |
"### 4.5 Training dynamics",
|
| 749 |
"",
|
|
|
|
| 850 |
"- The rec-1x-dynamic family trains at ~5 average block-",
|
| 851 |
" applications per step (R ~ U{2..8}), slightly above the matched",
|
| 852 |
" 4 - flagged rather than hidden.",
|
| 853 |
+
"- The aligned-unroll variant (R=k) is a negative result at this",
|
| 854 |
+
" budget: ~0.10 in-distribution across 3 seeds at 25k steps, and a",
|
| 855 |
+
" 100k-step probe still only reaches 0.24 - see 4.7.",
|
| 856 |
"- Two duplicate jobs created by interrupted tool calls were",
|
| 857 |
" cancelled (~$0.30 wasted).",
|
| 858 |
"- Dependency pin: trackio 0.42.0 requires huggingface-hub<2.0;",
|