Duoia commited on
Commit
2c7b2f9
·
verified ·
1 Parent(s): c5dab95

duotactic full package: checkpoints, tokenizer, config, code, docs (part 3)

Browse files
data/minif2f/numbertheory_2pownm1prime_nprime.lean ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_2pownm1prime_nprime
7
+ (n : ℕ)
8
+ (h₀ : 0 < n)
9
+ (h₁ : Nat.Prime (2^n - 1)) :
10
+ Nat.Prime n := sorry
data/minif2f/numbertheory_3pow2pownm1mod2pownp3eq2pownp2.lean ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_3pow2pownm1mod2pownp3eq2pownp2
7
+ (n : ℕ)
8
+ (h₀ : 0 < n) :
9
+ (3^(2^n) - 1) % (2^(n + 3)) = 2^(n + 2) := sorry
data/minif2f/numbertheory_4x3m7y3neq2003.lean ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_4x3m7y3neq2003
7
+ (x y : ℤ) :
8
+ 4 * x^3 - 7 * y^3 ≠ 2003 := sorry
data/minif2f/numbertheory_aneqprodakp4_anmsqrtanp1eq2.lean ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_aneqprodakp4_anmsqrtanp1eq2
7
+ (a : ℕ → ℝ)
8
+ (h₀ : a 0 = 1)
9
+ (h₁ : ∀ n, a (n + 1) = (∏ k ∈ Finset.range (n + 1), (a k)) + 4) :
10
+ ∀ n ≥ 1, a n - Real.sqrt (a (n + 1)) = 2 := sorry
data/minif2f/numbertheory_aoddbdiv4asqpbsqmod8eq1.lean ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_aoddbdiv4asqpbsqmod8eq1
7
+ (a : ℤ)
8
+ (b : ℤ)
9
+ (h₀ : Odd a)
10
+ (h₁ : 4 ∣ b)
11
+ (h₂ : b >= 0) :
12
+ (a^2 + b^2) % 8 = 1 := sorry
data/minif2f/numbertheory_exk2powkeqapb2mulbpa2_aeq1.lean ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_exk2powkeqapb2mulbpa2_aeq1
7
+ (a b : ℕ)
8
+ (h₀ : 0 < a ∧ 0 < b)
9
+ (h₁ : ∃ k > 0, 2^k = (a + b^2) * (b + a^2)) :
10
+ a = 1 := sorry
data/minif2f/numbertheory_fxeq4powxp6powxp9powx_f2powmdvdf2pown.lean ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_fxeq4powxp6powxp9powx_f2powmdvdf2pown
7
+ (m n : ℕ)
8
+ (f : ℕ → ℕ)
9
+ (h₀ : ∀ x, f x = 4^x + 6^x + 9^x)
10
+ (h₁ : 0 < m ∧ 0 < n)
11
+ (h₂ : m ≤ n) :
12
+ f (2^m)∣f (2^n) := sorry
data/minif2f/numbertheory_nckeqnm1ckpnm1ckm1.lean ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_nckeqnm1ckpnm1ckm1
7
+ (n k : ℕ)
8
+ (h₀ : 0 < n ∧ 0 < k)
9
+ (h₁ : k ≤ n) :
10
+ Nat.choose n k = Nat.choose (n - 1) k + Nat.choose (n - 1) (k - 1) := sorry
data/minif2f/numbertheory_notequiv2i2jasqbsqdiv8.lean ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_notEquiv2i2jasqbsqdiv8 :
7
+ ¬ (∀ a b : ℤ, (∃ i j, a = 2*i ∧ b=2*j) ↔ (∃ k, a^2 + b^2 = 8*k)) := sorry
data/minif2f/numbertheory_prmdvsneqnsqmodpeq0.lean ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_prmdvsneqnsqmodpeq0
7
+ (n : ℤ)
8
+ (p : ℕ)
9
+ (h₀ : Nat.Prime p) :
10
+ ↑p ∣ n ↔ (n^2) % p = 0 := sorry
data/minif2f/numbertheory_sqmod3in01d.lean ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_sqmod3in01d
7
+ (a : ℤ) :
8
+ (a^2) % 3 = 0 ∨ (a^2) % 3 = 1 := sorry
data/minif2f/numbertheory_sqmod4in01d.lean ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_sqmod4in01d
7
+ (a : ℤ) :
8
+ (a^2 % 4) = 0 ∨ (a^2 % 4) = 1 := sorry
data/minif2f/numbertheory_sumkmulnckeqnmul2pownm1.lean ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_sumkmulnckeqnmul2pownm1
7
+ (n : ℕ)
8
+ (h₀ : 0 < n) :
9
+ ∑ k ∈ Finset.Icc 1 n, (k * Nat.choose n k) = n * 2^(n - 1) := sorry
data/minif2f/numbertheory_x5neqy2p4.lean ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_x5neqy2p4
7
+ (x y : ℤ) :
8
+ x^5 ≠ y^2 + 4 := sorry
data/minif2f/numbertheory_xsqpysqintdenomeq.lean ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import Mathlib
2
+ open BigOperators
3
+ open Real
4
+ open Nat
5
+ open Topology
6
+ theorem numbertheory_xsqpysqintdenomeq
7
+ (x y : ℚ)
8
+ (h₀ : (x^2 + y^2).den = 1) :
9
+ x.den = y.den := sorry
data/minif2f/test.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
data/minif2f/valid.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
data/whitelist/README.md ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # whitelist seed
2
+
3
+ The packaged search harness (`code/eval_minif2f.py` -> `leanoar/search.py`) builds its
4
+ first-token whitelist from `<steps_root>/train_ids.npy` + `train_mask_start.npy`.
5
+ Shipping the 479 MB training arrays just for that would be silly, so these two small
6
+ files are a *seed*: `train_ids.npy` is `(N, 2)` int64 whose **column 1** holds the
7
+ first tactic token of a training sample (repeated in proportion to its frequency in
8
+ the real train split) and `train_mask_start.npy` is all ones, so the derived histogram
9
+ reproduces the real top-64 first tokens in the real order.
10
+
11
+ Top-64 entries cover 97.6% of first-token occurrences; see
12
+ `../first_token_whitelist.json` for the same list with shares and token text.
data/whitelist/train_ids.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d43e31c1c50c0b71f67f89e5073d56f97a3bcb9c5482a829e59a2e7499531b77
3
+ size 800128
data/whitelist/train_mask_start.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0c4c298c32c224a8ea8dfe760e68fde773e416c8fc78c9823b3daae3bbc9faa
3
+ size 400128
docs/USAGE.md ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Usage guide (English)
2
+
3
+ This package is **DuoTactic** — a Lean 4 proof-search harness — together with the 39.58 M-parameter
4
+ tactic model (`leanoar-schemec-40m`) that it can use as an optional candidate generator.
5
+
6
+ * To **run the harness** (sections 3 onward) you need Lean `4.35.0-rc2` and a mathlib4 checkout with a
7
+ prebuilt `.lake` cache (~10 GB). This is what produces miniF2F `test` 32.8 % / `valid` 34.8 %, at
8
+ 45 s per problem, with every accepted proof re-compiled by the kernel.
9
+ * To **load or sample the model** (sections 0–2) you need no Lean. Note that `SchemeC` is **not** a
10
+ standard `transformers` model: it is defined in `code/leanoar/model.py`, its weights are plain
11
+ `lit_model.pth` / `model.safetensors` files, and it emits a *single Lean tactic* per call. There is
12
+ no `AutoModelForCausalLM.from_pretrained`. Everything below is copy-paste runnable from the package
13
+ root.
14
+
15
+ ```
16
+ duotactic/
17
+ ├── code/eval_minif2f.py DuoTactic itself: the end-to-end search harness
18
+ ├── code/leanoar/ search + batched verifier + the 40M model
19
+ ├── checkpoints/e6/ model weights (highest dev top-1)
20
+ ├── checkpoints/stage3-e3/ model weights, end-to-end best variant (same pass@1, 16% faster)
21
+ ├── data/dev/ dev protocol arrays (1024 rows)
22
+ ├── data/minif2f/ miniF2F valid/test records + .lean files
23
+ ├── data/whitelist/ seed for the search's first-token whitelist
24
+ ├── examples/dev_sample.jsonl 16 real dev states + the real mathlib tactic
25
+ ├── config.json metrics.json tokenizer_v1.json special_tokens_v1.json SHA256SUMS
26
+ └── README.md README.zh.md docs/USAGE.md docs/USAGE.zh.md
27
+ ```
28
+
29
+ ## 0. Install and verify
30
+
31
+ ```bash
32
+ pip install -r code/requirements.txt # torch>=2.4, tokenizers>=0.19, numpy>=1.26, safetensors>=0.4
33
+ python -c "import torch; print(torch.__version__, torch.cuda.is_available())"
34
+
35
+ python code/verify_package.py # full check, 1024 dev rows (≈1 min on GPU)
36
+ python code/verify_package.py --quick # 128 rows, CPU-friendly
37
+ ```
38
+
39
+ `verify_package.py` prints four sections and ends with `PASS` / `FAIL`:
40
+
41
+ 1. **parameter count** — rebuilds `SchemeC` from `config.json`, must be 39,582,595 for both variants;
42
+ 2. **dev protocol** — recomputes top-1/top-5/CE on the packaged 1024 rows and compares with
43
+ `metrics.json` (tolerance 5e-4). This is the check that makes the published numbers auditable;
44
+ 3. **inference wiring** — runs the packaged example state through the sampler (it also shows whether
45
+ the model's preferred first token matches mathlib's; a miss is normal and does not fail the check);
46
+ 4. **checksums** — every file against `SHA256SUMS`.
47
+
48
+ Exit code is 0 only if 1, 2 and 4 pass. Use `--skip-sums` if you only have the weights.
49
+
50
+ ## 1. Standalone inference (no Lean)
51
+
52
+ ```bash
53
+ python code/generate.py # packaged dev example #0
54
+ python code/generate.py --example 3 --k 8 # another real dev state
55
+ python code/generate.py --state-file state.txt # your own state (format below)
56
+ python code/generate.py --ckpt checkpoints/stage3-e3 # the other variant
57
+ ```
58
+
59
+ A **state** is the text Lean prints for a goal, i.e. hypotheses then `⊢ goal`:
60
+
61
+ ```
62
+ n m : ℕ
63
+ h : n < m
64
+ ⊢ n ≤ m
65
+ ```
66
+
67
+ You can obtain one from Lean with `trace_state` inside a proof, or from an `unsolved goals` error
68
+ message. In your own code:
69
+
70
+ ```python
71
+ import json, os, sys, torch
72
+ sys.path.insert(0, 'code')
73
+ from common import load_model, load_tokenizer, encode_state, specials, whitelist
74
+ from generate import propose
75
+
76
+ net, cfg, device = load_model('checkpoints/e6') # or 'checkpoints/stage3-e3'
77
+ tok, wl = load_tokenizer(), whitelist()
78
+ for tactic, avg_logprob in propose(net, tok, wl, state, k=16, device=device):
79
+ ... # send to Lean for verification
80
+ ```
81
+
82
+ Four rules that matter:
83
+
84
+ * **Restrict the first token** to `first_token_whitelist.json` (64 ids, 97.6 % of training first
85
+ tokens). `propose()` already does this. Without it, most candidates start with meaningless tokens
86
+ and the type-check rate collapses.
87
+ * **One candidate is one tactic.** The model separates two intended steps by a double space and may
88
+ end a truncated tactic with a dangling connective; `propose()` cuts both.
89
+ * **Truncation**: states longer than `ctx − 4 − max_new_tokens` tokens keep the head 60 % and the
90
+ tail 40 % (the `⊢ goal` line is last). Never feed more than `context_length` tokens total.
91
+ * **Only the Lean kernel decides correctness.** Ask for many candidates and verify them; do not trust
92
+ any single output. Whole-tactic exact match is 0/200, and a "valid" (type-checking) candidate is
93
+ still not a proof.
94
+
95
+ ## 2. Reproduce the dev metric
96
+
97
+ ```bash
98
+ python code/eval_dev.py --all --compare # both variants, 1024 rows, compare with metrics.json
99
+ python code/eval_dev.py --ckpt checkpoints/e6 --rows 256
100
+ ```
101
+
102
+ Expected (tolerance 5e-4):
103
+
104
+ | variant | top-1 | top-5 | CE |
105
+ |---|---|---|---|
106
+ | `checkpoints/e6` | 0.3076 | 0.7090 | 2.5494 |
107
+ | `checkpoints/stage3-e3` | 0.2891 | 0.7246 | 2.4971 |
108
+
109
+ The protocol: take the first 1024 rows of `data/dev/dev_ids.npy` whose second token is `<|FORWARD|>`,
110
+ read the logits at `mask_start − 1` and score the token at `mask_start`. **Do not compare these with
111
+ the numbers a trainer prints** — those come from a length-bucketed reshuffle of the dev split.
112
+
113
+ ## 3. End-to-end proof search on miniF2F (needs Lean + mathlib)
114
+
115
+ ### 3.1 Prerequisites
116
+
117
+ * A **Lean 4.35.0-rc2** toolchain (e.g. `elan toolchain install leanprover/lean4:v4.35.0-rc2`).
118
+ Note: this is a *release candidate*; newer Lean versions will re-elaborate mathlib differently.
119
+ * A **mathlib4 checkout with a prebuilt `.lake` cache** (this project used the Sept 2026 mathlib
120
+ snapshot; the exact commit is not pinned in the release, so state text may drift slightly on other
121
+ revisions). Build the cache with `lake exe cache get` inside the checkout — a full local build takes
122
+ hours, the olean cache takes minutes and ~7 GB.
123
+ * ≥ 16 GB RAM if you use `--lean-parallel 6` (each `lean --json` process loads mathlib; ~0.4 s per
124
+ candidate, 6 in parallel measured at 0.034 s/candidate).
125
+
126
+ ```bash
127
+ export LEANOAR_LEAN_BIN=/path/to/lean-4.35.0-rc2-linux/bin/lean
128
+ export LEANOAR_MATHLIB=/path/to/mathlib4
129
+ $LEANOAR_LEAN_BIN --version # sanity check
130
+ ```
131
+
132
+ ### 3.2 The run that produced the published number
133
+
134
+ ```bash
135
+ python code/eval_minif2f.py --split valid --n 89 --start 0 \
136
+ --k 16 --beam 4 --depth 6 --budget 45 --round-budget 30 \
137
+ --force --force-set v2 --force-depth 99 --lean-parallel 6 \
138
+ --ckpt checkpoints/stage3-e3 --tag mine
139
+ ```
140
+
141
+ Expected: **30/89 = 33.7 %**, ≈52 min on an RTX 4060 Laptop (the `e6` variant gives the same 30
142
+ problems in ≈60 min). The full `valid` split was run end-to-end with this exact configuration:
143
+ **85/244 = 34.8 %** in ≈2.3 h (segments 33.7 % / 40.4 % / 28.8 %; 78 of the 85 solutions are one
144
+ tactic). Longer runs are easier as `--lean-parallel` rises: the fixed cost of every `lean --json`
145
+ call is ≈6.6 s of `import Mathlib` (vs ≈0.03 s per candidate), so parallelism and import trimming
146
+ are where the wall clock goes.
147
+
148
+ What the flags do — this is where the score comes from:
149
+
150
+ | Flag | Meaning |
151
+ |---|---|
152
+ | `--k 16` | candidates per state (top-16 whitelisted first tokens, greedy continuation) |
153
+ | `--beam 4` | how many open states are expanded per round |
154
+ | `--depth 6` | max proof length in tactics (measured: 25 of 30 solutions are depth 1) |
155
+ | `--budget 45` | wall-clock seconds per problem; `--round-budget 30` caps a single round |
156
+ | `--force --force-set v2` | inject the exported automation table (`rfl`, `ring`, `simp_all`, `omega`, …, `simp_all ; ring`, `gcongr`) at every depth. **This is the single biggest lever**: with it the first-89 result is 30/89, without it the whole 244 gives 8/244. |
157
+ | `--lean-parallel 6` | verify 6 chunks of candidates concurrently (pure engineering: 2.07× wall clock) |
158
+ | `--narrow-imports` | per problem, replace the full Mathlib import by `#min_imports` + `Mathlib.Tactic` (measured 3.41 s → 1.74 s per `lean --json` call). A candidate that fails with "unknown identifier/tactic" is automatically re-checked with the full import, so narrowing cannot lose a proof (verified: 85/85 accepted valid proofs still compile) |
159
+ | `--no-model` | ablation switch: verify only the injected table, no model proposals. On valid problems 0-9 this reproduces the model's result exactly (2/10, same problems, same proof terms) in half the wall clock |
160
+
161
+ Before quoting any pass rate, read **“Where this score comes from”** in `README.md`: these are
162
+ *system* numbers (injected automation table + search + kernel verification), and the shipped model
163
+ supplies 0 of the accepted proofs.
164
+
165
+ ### 3.3 Reading and resuming results
166
+
167
+ Results are appended to `reports/search_eval_valid_mine.jsonl` (one JSON object per problem:
168
+ `solved`, `depth`, `seconds`, `verified`, `proof`, `confirmed`, `reason`) and every run prints
169
+ cumulative progress with an ETA. Because the file is append-only, an interrupted run resumes with
170
+ `--start <rows already written>` — or simply re-run the same command.
171
+
172
+ ### 3.4 Troubleshooting
173
+
174
+ * `initial-state-failed` for a problem: its statement does not even compile against your mathlib
175
+ revision. Two miniF2F problems fail on mathlib master for everyone
176
+ (`mathd_numbertheory_711`, `imosl_2007_algebra_p6`).
177
+ * `No such file or directory: .../bin/lean`: set `LEANOAR_LEAN_BIN`.
178
+ * Lean errors about missing oleans / `unknown module prefix 'Mathlib'`: set `LEANOAR_MATHLIB` to a
179
+ checkout with a **built** `.lake/build/lib/lean` (and `.lake/packages/*`); the verifier assembles
180
+ `LEAN_PATH` from those paths.
181
+ * Very low `verified` counts: check that the first-token whitelist is being applied and that the
182
+ tokenizer is `tokenizer_v1.json`.
183
+ * A state longer than the prompt budget used to overflow the context by one token in old builds; the
184
+ shipped `search.py` budgets `ctx − 4 − max_new_tokens` and hard-stops at `ctx`. If you edit the
185
+ sampler, keep that invariant.
186
+
187
+ ## 4. Extending
188
+
189
+ * Swap in your own weights: put `lit_model.pth` (+ `model.safetensors`) in a directory and pass
190
+ `--ckpt <dir>` to `generate.py` / `eval_dev.py` / `eval_minif2f.py`. `config.json` must match the
191
+ architecture; the vocabulary and context length are fixed.
192
+ * Add automation tactics to `FORCE_SETS` in `code/eval_minif2f.py` — measured gains in this project
193
+ came almost exclusively from that table and from the search/verifier side, not from the model.
194
+ * The value head is shipped but unusable (AUC 0.5201, below the policy's own confidence). Do not
195
+ build a reranker on it.
196
+ * `code/leanoar/data.py` is included because the search code imports `load_specials` from it; the
197
+ training pipeline itself is not part of this package.
docs/USAGE.zh.md ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 使用说明(中文)
2
+
3
+ 本包是 **DuoTactic** —— 一个 Lean 4 证明搜索 harness —— 连同它可选的候选生成器:39.58 M 参数的
4
+ tactic 模型(`leanoar-schemec-40m`)。
5
+
6
+ * **跑 harness**(第 3 节起)需要 Lean `4.35.0-rc2` 与带预编译 `.lake` 缓存的 mathlib4(约 10 GB)。
7
+ miniF2F `test` 32.8% / `valid` 34.8% 就是它跑出来的:每题 45 s,每条被接受的证明都经内核重编译。
8
+ * **加载/采样模型**(第 0–2 节)不需要 Lean。注意 `SchemeC` **不是**标准 `transformers` 模型:它定义在
9
+ `code/leanoar/model.py`,权重是普通的 `lit_model.pth` / `model.safetensors`,每次调用只产**一条
10
+ Lean tactic**,没有 `AutoModelForCausalLM.from_pretrained` 这种入口。下面所有命令都可以在包根目录
11
+ 直接复制运行。
12
+
13
+ ```
14
+ duotactic/
15
+ ├── code/eval_minif2f.py DuoTactic 本体:端到端搜索 harness
16
+ ├── code/leanoar/ 搜索 + 批量验证器 + 40M 模型
17
+ ├── checkpoints/e6/ 模型权重(dev top-1 最高)
18
+ ├── checkpoints/stage3-e3/ 模型权重,端到端最佳变体(同样通过率,快 16%)
19
+ ├── data/dev/ dev 协议数组(1024 行)
20
+ ├── data/minif2f/ miniF2F valid/test 题目 + .lean 文件
21
+ ├── data/whitelist/ 搜索用首 token 白名单的种子文件
22
+ ├── examples/dev_sample.jsonl 16 条真实 dev 状态 + mathlib 真实用过的 tactic
23
+ ├── config.json metrics.json tokenizer_v1.json special_tokens_v1.json SHA256SUMS
24
+ └── README.md README.zh.md docs/USAGE.md docs/USAGE.zh.md
25
+ ```
26
+
27
+ ## 0. 安装与验证
28
+
29
+ ```bash
30
+ pip install -r code/requirements.txt # torch>=2.4, tokenizers>=0.19, numpy>=1.26, safetensors>=0.4
31
+ python -c "import torch; print(torch.__version__, torch.cuda.is_available())"
32
+
33
+ python code/verify_package.py # 完整校验,1024 行(GPU 约 1 分钟)
34
+ python code/verify_package.py --quick # 128 行,CPU 也能跑
35
+ ```
36
+
37
+ `verify_package.py` 打印四段,最后给出 `PASS` / `FAIL`:
38
+
39
+ 1. **参数数量**——从 `config.json` 重建 `SchemeC`,两个变体都必须是 39,582,595;
40
+ 2. **dev 协议**——在包内 1024 行上复算 top-1/top-5/CE 并与 `metrics.json` 对比(容差 5e-4)。
41
+ 这一步让公开数字是可审计的;
42
+ 3. **推理连通性**——拿包内示例状态跑一遍采样(同时显示模型首选首 token 是否与 mathlib 一致;
43
+ 不一致是正常的,不影响判定);
44
+ 4. **校验和**——逐文件对照 `SHA256SUMS`。
45
+
46
+ 只有 1、2、4 全过才返回 0。只有权重时可用 `--skip-sums`。
47
+
48
+ ## 1. 独立推理(不需要 Lean)
49
+
50
+ ```bash
51
+ python code/generate.py # 包内真实 dev 样本 #0
52
+ python code/generate.py --example 3 --k 8 # 另一条真实 dev 状态
53
+ python code/generate.py --state-file state.txt # 你自己的状态(格式见下)
54
+ python code/generate.py --ckpt checkpoints/stage3-e3 # 换变体
55
+ ```
56
+
57
+ **状态**就是 Lean 打印目标时的那段文本——先是假设,然后 `⊢ 目标`:
58
+
59
+ ```
60
+ n m : ℕ
61
+ h : n < m
62
+ ⊢ n ≤ m
63
+ ```
64
+
65
+ 可以在证明里用 `trace_state` 拿到,或从 `unsolved goals` 报错里抠出来。在自己的代码里调用:
66
+
67
+ ```python
68
+ import json, os, sys, torch
69
+ sys.path.insert(0, 'code')
70
+ from common import load_model, load_tokenizer, encode_state, specials, whitelist
71
+ from generate import propose
72
+
73
+ net, cfg, device = load_model('checkpoints/e6') # 或 'checkpoints/stage3-e3'
74
+ tok, wl = load_tokenizer(), whitelist()
75
+ for tactic, avg_logprob in propose(net, tok, wl, state, k=16, device=device):
76
+ ... # 交给 Lean 验证
77
+ ```
78
+
79
+ **四条必须遵守的规则**:
80
+
81
+ * **首个 token 必须限制**在 `first_token_whitelist.json`(64 个 id,覆盖训练集 97.6% 的首 token)。
82
+ `propose()` 已经做了这件事;不做的话多数候选会以无意义 token 开头,编译通过率会崩。
83
+ * **一条候选 = 一条 tactic**。模型用双空格分隔它想表达的两步;被截断的 tactic 可能以悬挂连接词结尾,
84
+ `propose()` 两种都会切掉。
85
+ * **截断规则**:超过 `ctx − 4 − max_new_tokens` 个 token 的状态,会保留头部 60% + 尾部 40%
86
+ (`⊢ 目标` 在最后)。总输入绝不能超过 `context_length`。
87
+ * **只有 Lean 内核能判对错**。请多要候选 + 逐个验证,不要相信任何单条输出。整条 tactic 的精确匹配率是
88
+ 0/200,而且「通过编译(valid)」也不等于「证明成立」。
89
+
90
+ ## 2. 复算 dev 指标
91
+
92
+ ```bash
93
+ python code/eval_dev.py --all --compare # 两个变体 × 1024 行,并与 metrics.json 对比
94
+ python code/eval_dev.py --ckpt checkpoints/e6 --rows 256
95
+ ```
96
+
97
+ 预期(容差 5e-4):
98
+
99
+ | 变体 | top-1 | top-5 | CE |
100
+ |---|---|---|---|
101
+ | `checkpoints/e6` | 0.3076 | 0.7090 | 2.5494 |
102
+ | `checkpoints/stage3-e3` | 0.2891 | 0.7246 | 2.4971 |
103
+
104
+ 协议:取 `data/dev/dev_ids.npy` ��第二个 token 为 `<|FORWARD|>` 的前 1024 行,读 `mask_start − 1`
105
+ 位置的 logits,对 `mask_start` 处的 token 打分。**不要拿它跟训练器打印的 dev 数字比**——后者按长度
106
+ 分桶重排过,不是同一批样本。
107
+
108
+ ## 3. miniF2F 端到端搜索(需要 Lean + mathlib)
109
+
110
+ ### 3.1 前置条件
111
+
112
+ * **Lean 4.35.0-rc2** 工具链(例如 `elan toolchain install leanprover/lean4:v4.35.0-rc2`)。
113
+ 注意这是 release candidate;换 Lean 版本后 mathlib 的展开方式与状态文本都会变。
114
+ * **带预编译 `.lake` 缓存的 mathlib4 检出**(本项目用 2026-09 的快照;发布包里没有钉死 commit,
115
+ 所以换 revision 时状态文本可能有轻微漂移)。在检出目录里执行 `lake exe cache get` 拿 olean 缓存——
116
+ 本地全量编译要数小时,缓存只要几分钟、约 7 GB。
117
+ * 用 `--lean-parallel 6` 时建议 ≥16 GB 内存(每个 `lean --json` 进程都要加载 mathlib;
118
+ 实测单候选 0.4 s、6 路并行后 0.034 s/候选)。
119
+
120
+ ```bash
121
+ export LEANOAR_LEAN_BIN=/path/to/lean-4.35.0-rc2-linux/bin/lean
122
+ export LEANOAR_MATHLIB=/path/to/mathlib4
123
+ $LEANOAR_LEAN_BIN --version # 自检
124
+ ```
125
+
126
+ ### 3.2 复现公开成绩的那条命令
127
+
128
+ ```bash
129
+ python code/eval_minif2f.py --split valid --n 89 --start 0 \
130
+ --k 16 --beam 4 --depth 6 --budget 45 --round-budget 30 \
131
+ --force --force-set v2 --force-depth 99 --lean-parallel 6 \
132
+ --ckpt checkpoints/stage3-e3 --tag mine
133
+ ```
134
+
135
+ 预期:**30/89 = 33.7%**,RTX 4060 Laptop 上约 52 分钟(换成 `e6` 变体也是同样 30 题,约 60 分钟)。
136
+ 全量 `valid` 已用这份完全相同的配置跑完:**85/244 = 34.8%**,约 2.3 小时(分段 33.7% / 40.4% / 28.8%;
137
+ 85 条解出里 78 条只有一条 tactic)。想再快就动两处:每次 `lean --json` 调用的固定成本约 **6.6 s**
138
+ 的 `import Mathlib`(而每条候选只要约 0.03 s),所以墙钟全花在并行度和导入裁剪上。
139
+
140
+ 各参数的含义——成绩其实来自这里:
141
+
142
+ | 参数 | 含义 |
143
+ |---|---|
144
+ | `--k 16` | 每个状态提 16 条候选(白名单内 top-16 首 token + 贪心续写) |
145
+ | `--beam 4` | 每轮扩展 4 个开放状态 |
146
+ | `--depth 6` | 证明长度上限(以 tactic 计)。实测:30 条解出里有 25 条是 depth 1 |
147
+ | `--budget 45` | 每题墙钟预算 45 秒;`--round-budget 30` 限制单轮 |
148
+ | `--force --force-set v2` | 在每一层注入导出的自动化表(`rfl`、`ring`、`simp_all`、`omega`、…、`simp_all ; ring`、`gcongr`)。**这是最大的杠杆**:带它前 89 题是 30/89;不带它的全量 244 题只有 8/244 |
149
+ | `--lean-parallel 6` | 6 路并发验证候选(纯工程收益:墙钟 2.07×) |
150
+ | `--narrow-imports` | 每题把全量 Mathlib 导入换成 `#min_imports` + `Mathlib.Tactic`(实测每次 `lean --json` 从 3.41 s 降到 1.74 s)。命中「未知标识符/tactic」的候选会自动用全量导入重验,所以窄导入**不可能丢证明**(已验:85/85 条已验证证明仍可编译) |
151
+ | `--no-model` | 对照开关:只验证注入表、不提模型候选。在 valid 前 10 题上它与带模型的结果完全一致(2/10、同题、同证明项),而墙钟减半 |
152
+
153
+ 引用任何通过率之前,请先读 `README.zh.md` 的「**这个成绩是怎么来的**」:这些是**系统**成绩
154
+ (注入的自动化表 + 搜索 + 内核验证),随包模型的贡献是 0 条。
155
+
156
+ ### 3.3 看结果与续跑
157
+
158
+ 结果按行追加到 `reports/search_eval_valid_mine.jsonl`(每个题目一个 JSON:`solved`、`depth`、
159
+ `seconds`、`verified`、`proof`、`confirmed`、`reason`),同时实时打印累计进度与 ETA。因为是追加写入,
160
+ 中断后用 `--start <已写入行数>` 续跑即可,或者直接重跑同一条命令。
161
+
162
+ ### 3.4 疑难排查
163
+
164
+ * 某题报 `initial-state-failed`:它的陈述在你的 mathlib revision 上根本编译不过。有 2 道 miniF2F
165
+ 题对所有人都不通过(`mathd_numbertheory_711`、`imosl_2007_algebra_p6`)。
166
+ * 报 `No such file or directory: .../bin/lean`:设 `LEANOAR_LEAN_BIN`。
167
+ * Lean 报缺 olean / `unknown module prefix 'Mathlib'`:把 `LEANOAR_MATHLIB` 指向**已编译**
168
+ (`.lake/build/lib/lean` 与 `.lake/packages/*` 齐备)的检出目录,验证器会据此拼 `LEAN_PATH`。
169
+ * `verified` 数特别低:确认首 token 白名单生效、切词器用的是 `tokenizer_v1.json`。
170
+ * 旧版本在超长状态上会让上下文溢出 1 个 token;本包内的 `search.py` 已按
171
+ `ctx − 4 − max_new_tokens` 计算预算并在 `ctx` 处硬停。如果你改采样器,请保持这个不变式。
172
+
173
+ ## 4. 扩展
174
+
175
+ * 换自己的权重:把 `lit_model.pth`(和 `model.safetensors`)放进一个目录,给
176
+ `generate.py` / `eval_dev.py` / `eval_minif2f.py` 传 `--ckpt <目录>`。`config.json` 必须与架构一致;
177
+ 词表与 ctx 是固定的。
178
+ * 往 `code/eval_minif2f.py` 的 `FORCE_SETS` 里加自动化 tactic——本项目所有实测收益几乎都来自这张表
179
+ 与搜索/验���器侧,而不是模型。
180
+ * Value 头虽然随包发出但不可用(AUC 0.5201,低于策略自身置信度)。不要拿它做重排。
181
+ * `code/leanoar/data.py` 只是因为搜索代码要用里面的 `load_specials` 才包含进来;训练流水线不在本包内。
examples/dev_sample.jsonl ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"dev_row": 0, "state": "L : Language\nL' : Language\nM : Type w\nN : Type u_1\nP : Type ?u\ninst : Structure L M\ninst : Structure L N\ninst : Structure L P\nα : Type u'\nβ : Type v'\nn : ℕ\nT : Theory L\nh : M ≅[L] N\n⊢ M ⊨ T ↔ N ⊨ T", "true_first_token": "rw", "true_tactic": "rw [Theory.model_iff_subset_completeTheory, Theory.model_iff_subset_completeTheory,\n h.completeTheory_eq]"}
2
+ {"dev_row": 68, "state": "case codisjoint\nR : Type u_1\ninst : Ring R\nE : Type u_2\ninst : AddCommGroup E\ninst : Module R E\nF : Type ?u\ninst : AddCommGroup F\ninst : Module R F\nG : Type ?u\ninst : AddCommGroup G\ninst : Module R G\np q : Submodule R E\nS : Type ?u\ninst : Semiring S\nM : Type ?u\ninst : AddCommMonoid M\ninst : Module S M\nm : Submodule S M\nf : E →ₗ[R] { x // x ∈ p }\nhf : ∀ (x : { x // x ∈ p }), ↑f ↑x = x\nx : E\na : x ∈ ⊤\n⊢ ∃ y z, ↑y + ↑z = x", "true_first_token": "refine", "true_tactic": "refine' ⟨f x, ⟨x - f x, _⟩, add_sub_cancel'_right _ _⟩"}
3
+ {"dev_row": 136, "state": "case inr.mp\nE : ℕ → Type u_1\nx y : (n : ℕ) → E n\nn : ℕ\nhne : y ≠ x\nhy : ∀ (i : ℕ), i < n → y i = x i\n⊢ n ≤ firstDiff y x", "true_first_token": "by", "true_tactic": "by_contra' H"}
4
+ {"dev_row": 214, "state": "E : Type u_1\nX : Type u_2\ninst : TopologicalSpace E\ninst : TopologicalSpace X\nf : E → X\ns : Set X\nhf : IsCoveringMapOn f s\n⊢ IsLocallyHomeomorphOn f (f ⁻¹' s)", "true_first_token": "refine", "true_tactic": "refine' IsLocallyHomeomorphOn.mk f (f ⁻¹' s) fun x hx => _"}
5
+ {"dev_row": 285, "state": "case neg.mp\nz : ℂ\nh₀ : ¬z = 0\nh : arg z = π\n⊢ (↑(↑abs z) * (cos ↑π + sin ↑π * I)).re < 0 ∧ (↑(↑abs z) * (cos ↑π + sin ↑π * I)).im = 0", "true_first_token": "simp", "true_tactic": "simp [h₀]"}
6
+ {"dev_row": 359, "state": "case this\nα : Type u\nβ : Type v\nX : Type ?u\ninst : PseudoEMetricSpace α\nx y z : α\nε ε₁ ε₂ : ℝ≥0∞\ns t : Set α\nhs : ∀ (ε : ℝ≥0∞), ε > 0 → ∃ t, Set.Countable t ∧ (⋃ (x : α) (_ : x ∈ t), closedBall x ε) = univ\n⊢ ∀ (ε : ℝ≥0∞), ε > 0 → ∃ t, Set.Countable t ∧ univ ⊆ ⋃ (x : α) (_ : x ∈ t), closedBall x ε", "true_first_token": "simpa", "true_tactic": "simpa only [univ_subset_iff] using hs"}
7
+ {"dev_row": 432, "state": "V : Type u_1\nP : Type u_2\ninst : NormedAddCommGroup V\ninst : InnerProductSpace ℝ V\ninst : MetricSpace P\ninst : NormedAddTorsor V P\np₁ p₂ p₃ : P\nh : ∠ p₁ p₂ p₃ = π / 2\nh0 : p₁ = p₂ ∨ p₃ ≠ p₂\n⊢ dist p₃ p₂ / Real.cos (∠ p₂ p₃ p₁) = dist p₁ p₃", "true_first_token": "rw", "true_tactic": "rw [angle, ← inner_eq_zero_iff_angle_eq_pi_div_two, real_inner_comm, ← neg_eq_zero, ←\n inner_neg_left, neg_vsub_eq_vsub_rev] at h"}
8
+ {"dev_row": 503, "state": "R : Type u\nS : Type v\nk : Type y\nA : Type z\na b : R\nn : ℕ\ninst : Field R\np q : R[X]\nhp0 : IsUnit (leadingCoeff p)\n⊢ Monic (↑normalize p)", "true_first_token": "rw", "true_tactic": "rw [Monic, leadingCoeff_normalize, normalize_eq_one]"}
9
+ {"dev_row": 578, "state": "V : Type u_1\nP : Type u_2\ninst : NormedAddCommGroup V\ninst : InnerProductSpace ℝ V\ninst : MetricSpace P\ninst : NormedAddTorsor V P\ns : AffineSubspace ℝ P\ninst : FiniteDimensional ℝ { x // x ∈ direction s }\nhd : finrank ℝ { x // x ∈ direction s } = 2\nc₁ c₂ p₁ p₂ p : P\nhc₁s : c₁ ∈ s\nhc₂s : c₂ ∈ s\nhp₁s : p₁ ∈ s\nhp₂s : p₂ ∈ s\nhps : p ∈ s\nr₁ r₂ : ℝ\nhc : c₁ ≠ c₂\nhp : p₁ ≠ p₂\nhp₁c₁ : dist p₁ c₁ = r₁\nhp₂c₁ : dist p₂ c₁ = r₁\nhpc₁ : dist p c₁ = r₁\nhp₁c₂ : dist p₁ c₂ = r₂\nhp₂c₂ : dist p₂ c₂ = r₂\nhpc₂ : dist p c₂ = r₂\n⊢ p = p₁ ∨ p = p₂", "true_first_token": "have", "true_tactic": "have ho : ⟪c₂ -ᵥ c₁, p₂ -ᵥ p₁⟫ = 0 :=\n inner_vsub_vsub_of_dist_eq_of_dist_eq (hp₁c₁.trans hp₂c₁.symm) (hp₁c₂.trans hp₂c₂.symm)"}
10
+ {"dev_row": 648, "state": "ι : Type u_1\nα : Type u_2\nβ : Type ?u\nr : α → α → Prop\ns : Finset ι\nf : ι → Set α\n⊢ Set.PartiallyWellOrderedOn (sup ∅ f) r ↔ ∀ (i : ι), i ∈ ∅ → Set.PartiallyWellOrderedOn (f i) r", "true_first_token": "simp", "true_tactic": "simp"}
11
+ {"dev_row": 721, "state": "α : Type ?u\nβ : Type ?u\nγ : Type ?u\nr : α → α → Prop\ns : β → β → Prop\nt : γ → γ → Prop\nα : Type u_1\nr : α → α → Prop\nwo : IsWellOrder α r\nh : ∀ (a : Ordinal), a < type r → succ a < type r\nx : α\n⊢ ∃ y, r x y", "true_first_token": "use", "true_tactic": "use enum r (succ (typein r x)) (h _ (typein_lt_type r x))"}
12
+ {"dev_row": 793, "state": "R : Type u_1\ninst : CommRing R\nn : ℕ\nx : MvPolynomial Bool R := MvPolynomial.X true\ny : MvPolynomial Bool R := MvPolynomial.X false\npderiv_true_x : ↑(pderiv true) x = 1\npderiv_true_y : ↑(pderiv true) y = 0\ne : Bool → R[X] := fun i => bif i then X else 1 - X\n⊢ ↑(MvPolynomial.aeval e) (↑(pderiv true) (↑(pderiv true) ((x + y) ^ n))) * X ^ 2 = (n * (n - 1)) • X ^ 2", "true_first_token": "simp", "true_tactic": "simp only [pderiv_one, pderiv_mul, (pderiv _).leibniz_pow, (pderiv _).map_coe_nat,\n (pderiv true).map_add, pderiv_true_x, pderiv_true_y, Algebra.id.smul_eq_mul, add_zero,\n mul_one, Derivation.map_smul_of_tower, map_nsmul, map_pow, map_add, Bool.cond_true,\n Bool.cond_false, MvPolynomial.aeval_X, add_sub_cancel'_right, one_pow, smul_smul,\n smul_one_mul]"}
13
+ {"dev_row": 865, "state": "case neg.intro\nα : Type u\nβ : Type v\nγ : Type w\ninst : TopologicalSpace α\ninst : TopologicalSpace β\ninst : LinearOrder α\ninst : LinearOrder β\ninst : OrderTopology α\ninst : OrderTopology β\nt : Set α\nx : α\ninst : IsCountablyGenerated (𝓝 x)\nhtx : IsLUB t x\nht : Set.Nonempty t\nh : ¬x ∈ t\nu : ℕ → α\nhu : StrictMono u ∧ (∀ (n : ℕ), u n < x) ∧ Tendsto u atTop (𝓝 x) ∧ ∀ (n : ℕ), u n ∈ t\n⊢ ∃ u, Monotone u ∧ (∀ (n : ℕ), u n ≤ x) ∧ Tendsto u atTop (𝓝 x) ∧ ∀ (n : ℕ), u n ∈ t", "true_first_token": "exact", "true_tactic": "exact ⟨u, hu.1.monotone, fun n => (hu.2.1 n).le, hu.2.2⟩"}
14
+ {"dev_row": 934, "state": "case neg\nR : Type u_1\ninst : AddMonoidWithOne R\nn k : ℕ\nhk : ∀ (m : ℕ), m < succ k → Nat.binCast m = ↑m\nh : ¬(k + 1) % 2 = 0\n⊢ ↑(succ k / 2 + succ k / 2) + 1 = ↑(succ k % 2 + 2 * (succ k / 2))", "true_first_token": "have", "true_tactic": "have h1 := Or.resolve_left (Nat.mod_two_eq_zero_or_one (succ k)) h"}
15
+ {"dev_row": 1010, "state": "case neg\nα : Type u_1\nE : Type u_2\nF : Type ?u\nG : Type ?u\nm m0 : MeasurableSpace α\np : ℝ≥0∞\nq : ℝ\nμ ν : Measure α\ninst : NormedAddCommGroup E\ninst : NormedAddCommGroup F\ninst : NormedAddCommGroup G\np q : ℝ≥0∞\nhpq : p ≤ q\nf : α → E\nhf : AEStronglyMeasurable f μ\nhp0 : p ≠ 0\n⊢ snorm f p μ ≤ snorm f q μ * ↑↑μ Set.univ ^ (1 / ENNReal.toReal p - 1 / ENNReal.toReal q)", "true_first_token": "have", "true_tactic": "have hp0_lt : 0 < p := lt_of_le_of_ne (zero_le _) hp0.symm"}
16
+ {"dev_row": 1080, "state": "case inl\nn : Type u_2\ninst : Fintype n\ninst : DecidableEq n\nR : Type u_1\ninst : Field R\nA : Matrix n n R\nhAps : Splits (RingHom.id R) (charpoly A)\nh : IsEmpty n\n⊢ trace A = Multiset.sum (roots (charpoly A))", "true_first_token": "rw", "true_tactic": "rw [Matrix.trace, Fintype.sum_empty, Matrix.charpoly,\n det_eq_one_of_card_eq_zero (Fintype.card_eq_zero_iff.2 h), Polynomial.roots_one,\n Multiset.empty_eq_zero, Multiset.sum_zero]"}
first_token_whitelist.json ADDED
@@ -0,0 +1,322 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "id": 782,
4
+ "token": "rw",
5
+ "share": 0.1764
6
+ },
7
+ {
8
+ "id": 559,
9
+ "token": "simp",
10
+ "share": 0.11225
11
+ },
12
+ {
13
+ "id": 1180,
14
+ "token": "exact",
15
+ "share": 0.08363
16
+ },
17
+ {
18
+ "id": 1704,
19
+ "token": "have",
20
+ "share": 0.07999
21
+ },
22
+ {
23
+ "id": 1240,
24
+ "token": "refine",
25
+ "share": 0.06312
26
+ },
27
+ {
28
+ "id": 1924,
29
+ "token": "·",
30
+ "share": 0.05229
31
+ },
32
+ {
33
+ "id": 686,
34
+ "token": "apply",
35
+ "share": 0.04007
36
+ },
37
+ {
38
+ "id": 3231,
39
+ "token": "obtain",
40
+ "share": 0.03498
41
+ },
42
+ {
43
+ "id": 450,
44
+ "token": "intro",
45
+ "share": 0.03398
46
+ },
47
+ {
48
+ "id": 829,
49
+ "token": "ext",
50
+ "share": 0.02623
51
+ },
52
+ {
53
+ "id": 3249,
54
+ "token": "rcases",
55
+ "share": 0.02152
56
+ },
57
+ {
58
+ "id": 2970,
59
+ "token": "let",
60
+ "share": 0.01925
61
+ },
62
+ {
63
+ "id": 3267,
64
+ "token": "rintro",
65
+ "share": 0.01759
66
+ },
67
+ {
68
+ "id": 2965,
69
+ "token": "simpa",
70
+ "share": 0.01715
71
+ },
72
+ {
73
+ "id": 965,
74
+ "token": "cases",
75
+ "share": 0.01655
76
+ },
77
+ {
78
+ "id": 909,
79
+ "token": "by",
80
+ "share": 0.01543
81
+ },
82
+ {
83
+ "id": 2048,
84
+ "token": "induction",
85
+ "share": 0.01204
86
+ },
87
+ {
88
+ "id": 1088,
89
+ "token": "const",
90
+ "share": 0.01068
91
+ },
92
+ {
93
+ "id": 1438,
94
+ "token": "rfl",
95
+ "share": 0.01055
96
+ },
97
+ {
98
+ "id": 580,
99
+ "token": "con",
100
+ "share": 0.00963
101
+ },
102
+ {
103
+ "id": 75,
104
+ "token": "d",
105
+ "share": 0.00959
106
+ },
107
+ {
108
+ "id": 319,
109
+ "token": "su",
110
+ "share": 0.0072
111
+ },
112
+ {
113
+ "id": 1121,
114
+ "token": "congr",
115
+ "share": 0.00628
116
+ },
117
+ {
118
+ "id": 2156,
119
+ "token": "use",
120
+ "share": 0.0055
121
+ },
122
+ {
123
+ "id": 15,
124
+ "token": "(",
125
+ "share": 0.00534
126
+ },
127
+ {
128
+ "id": 296,
129
+ "token": "re",
130
+ "share": 0.00532
131
+ },
132
+ {
133
+ "id": 306,
134
+ "token": "--",
135
+ "share": 0.00506
136
+ },
137
+ {
138
+ "id": 74,
139
+ "token": "c",
140
+ "share": 0.00491
141
+ },
142
+ {
143
+ "id": 3037,
144
+ "token": "split",
145
+ "share": 0.00434
146
+ },
147
+ {
148
+ "id": 982,
149
+ "token": "cal",
150
+ "share": 0.00417
151
+ },
152
+ {
153
+ "id": 70,
154
+ "token": "_",
155
+ "share": 0.00409
156
+ },
157
+ {
158
+ "id": 612,
159
+ "token": "ch",
160
+ "share": 0.00406
161
+ },
162
+ {
163
+ "id": 583,
164
+ "token": "sub",
165
+ "share": 0.00363
166
+ },
167
+ {
168
+ "id": 1331,
169
+ "token": "cont",
170
+ "share": 0.00355
171
+ },
172
+ {
173
+ "id": 1489,
174
+ "token": "filter",
175
+ "share": 0.00337
176
+ },
177
+ {
178
+ "id": 3701,
179
+ "token": "conv",
180
+ "share": 0.00334
181
+ },
182
+ {
183
+ "id": 860,
184
+ "token": "norm",
185
+ "share": 0.00324
186
+ },
187
+ {
188
+ "id": 301,
189
+ "token": "un",
190
+ "share": 0.00312
191
+ },
192
+ {
193
+ "id": 416,
194
+ "token": "set",
195
+ "share": 0.00296
196
+ },
197
+ {
198
+ "id": 276,
199
+ "token": "er",
200
+ "share": 0.00222
201
+ },
202
+ {
203
+ "id": 1880,
204
+ "token": "ring",
205
+ "share": 0.00216
206
+ },
207
+ {
208
+ "id": 78,
209
+ "token": "g",
210
+ "share": 0.00213
211
+ },
212
+ {
213
+ "id": 1606,
214
+ "token": "choose",
215
+ "share": 0.00189
216
+ },
217
+ {
218
+ "id": 443,
219
+ "token": "case",
220
+ "share": 0.00182
221
+ },
222
+ {
223
+ "id": 449,
224
+ "token": "fun",
225
+ "share": 0.0018
226
+ },
227
+ {
228
+ "id": 2323,
229
+ "token": "lin",
230
+ "share": 0.00174
231
+ },
232
+ {
233
+ "id": 1141,
234
+ "token": "lift",
235
+ "share": 0.00163
236
+ },
237
+ {
238
+ "id": 87,
239
+ "token": "p",
240
+ "share": 0.00145
241
+ },
242
+ {
243
+ "id": 85,
244
+ "token": "n",
245
+ "share": 0.00132
246
+ },
247
+ {
248
+ "id": 21,
249
+ "token": ".",
250
+ "share": 0.00131
251
+ },
252
+ {
253
+ "id": 2472,
254
+ "token": "spec",
255
+ "share": 0.00127
256
+ },
257
+ {
258
+ "id": 1173,
259
+ "token": "cle",
260
+ "share": 0.00115
261
+ },
262
+ {
263
+ "id": 2816,
264
+ "token": "tfae",
265
+ "share": 0.00111
266
+ },
267
+ {
268
+ "id": 2607,
269
+ "token": "nth",
270
+ "share": 0.0011
271
+ },
272
+ {
273
+ "id": 706,
274
+ "token": "←",
275
+ "share": 0.00107
276
+ },
277
+ {
278
+ "id": 90,
279
+ "token": "s",
280
+ "share": 0.00104
281
+ },
282
+ {
283
+ "id": 2044,
284
+ "token": "swap",
285
+ "share": 0.001
286
+ },
287
+ {
288
+ "id": 376,
289
+ "token": "as",
290
+ "share": 0.00095
291
+ },
292
+ {
293
+ "id": 662,
294
+ "token": "symm",
295
+ "share": 0.00092
296
+ },
297
+ {
298
+ "id": 3412,
299
+ "token": "show",
300
+ "share": 0.00092
301
+ },
302
+ {
303
+ "id": 2101,
304
+ "token": "dec",
305
+ "share": 0.00087
306
+ },
307
+ {
308
+ "id": 99,
309
+ "token": "|",
310
+ "share": 0.00083
311
+ },
312
+ {
313
+ "id": 1372,
314
+ "token": "inf",
315
+ "share": 0.00081
316
+ },
317
+ {
318
+ "id": 2980,
319
+ "token": "field",
320
+ "share": 0.00079
321
+ }
322
+ ]
metrics.json ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "system": "DuoTactic - a Lean 4 proof-search harness (the main artifact of this package: code/eval_minif2f.py + code/leanoar/{search,lean_batch}.py). The 39.58M model documented below is an optional candidate generator for it and is measured to contribute 0 of the 165 accepted proofs; see the where_the_score_comes_from block.",
3
+ "_warning": "dev numbers from the training log use a length-bucketed reshuffle and are NOT comparable across datasets; only the protocol below is.",
4
+ "dev_protocol": "first 1024 rows with dev_ids[k,1]==<|FORWARD|>, logits at position (mask_start-1) predicting the token at mask_start; ctx=768",
5
+ "dev": {
6
+ "stage4_e6_full_best": {
7
+ "top1": 0.3076,
8
+ "top5": 0.709,
9
+ "ce": 2.5494,
10
+ "step": 5000,
11
+ "train_data": "steps_v2 (317,492 samples)"
12
+ },
13
+ "stage3_e3_best": {
14
+ "top1": 0.2891,
15
+ "top5": 0.7246,
16
+ "ce": 2.4971,
17
+ "step": 2800,
18
+ "train_data": "steps (188,501 samples)"
19
+ },
20
+ "trivial_baseline": {
21
+ "top1": 0.2171,
22
+ "top5": 0.6079
23
+ }
24
+ },
25
+ "end_to_end_minif2f_valid_first89": {
26
+ "search_config": "--force --force-set v2 --force-depth 99 --lean-parallel 6 --k 16 --beam 4 --depth 6 --budget 45 --round-budget 30",
27
+ "stage3_e3_best": {
28
+ "passed": 30,
29
+ "of": 89,
30
+ "rate": 0.337,
31
+ "wall_min": 51.7
32
+ },
33
+ "stage4_e6_full_best": {
34
+ "passed": 30,
35
+ "of": 89,
36
+ "rate": 0.337,
37
+ "wall_min": 59.8,
38
+ "note": "same 30 problems at the same depths: zero new, zero lost"
39
+ },
40
+ "no_neural_proposals_244": {
41
+ "passed": 8,
42
+ "of": 244,
43
+ "rate": 0.033,
44
+ "note": "pure automation-tactic injection, no model output"
45
+ }
46
+ },
47
+ "end_to_end_minif2f_valid_full244": {
48
+ "search_config": "--force --force-set v2 --force-depth 99 --lean-parallel 6 --k 16 --beam 4 --depth 6 --budget 45 --round-budget 30",
49
+ "checkpoint": "checkpoints/stage3-e3 (lit_model.pth)",
50
+ "passed": 85,
51
+ "of": 244,
52
+ "rate": 0.348,
53
+ "wall_min": 137,
54
+ "wall_note": "sum of the recorded per-problem seconds: 63.2 min for problems 0-106 + 73.6 min for 107-243; solved problems average 13 s, unsolved ones 45 s (they run to the 45 s budget)",
55
+ "segments": {"0-89": 0.337, "89-178": 0.404, "178-244": 0.288},
56
+ "solution_depths": {"1": 78, "2": 2, "3": 2, "4": 3},
57
+ "all_confirmed": true,
58
+ "per_problem_seconds_median": 40,
59
+ "lean_calls_second_slice": 2125,
60
+ "lean_seconds_second_slice": 11975,
61
+ "note": "one run per problem, one attempt: pass@1 with the search harness (not pass@k). Problems 0-106 ran with a prompt budget of 741 tokens, 107-243 with 740 (off-by-one fix), which only affects states longer than 740 tokens."
62
+ },
63
+ "end_to_end_minif2f_test_full244": {
64
+ "search_config": "--force --force-set v2 --force-depth 99 --lean-parallel 10 --verify-chunk 64 --narrow-imports --k 16 --beam 4 --depth 6 --budget 45 --round-budget 30",
65
+ "checkpoint": "checkpoints/stage3-e3 (lit_model.pth)",
66
+ "passed": 80,
67
+ "of": 244,
68
+ "rate": 0.328,
69
+ "wall_min": 148.5,
70
+ "segments": {"0-89": 0.382, "89-178": 0.270, "178-244": 0.333},
71
+ "solution_depths": {"1": 76, "2": 2, "4": 2},
72
+ "all_confirmed": true,
73
+ "lean_calls": 2921,
74
+ "lean_seconds": 9078,
75
+ "note": "same search as the valid run; the run was additionally accelerated (narrow imports, chunk 64, 10 parallel), which changes the per-call cost (5.6 s/call -> 3.1 s/call), not the verdicts."
76
+ },
77
+ "where_the_score_comes_from": {
78
+ "final_tactic_of_each_accepted_proof": {
79
+ "valid_85_solved": {"injected_table_single_step": 78, "injected_table_multi_step_chain": 7, "model_proposed": 0},
80
+ "test_80_solved": {"injected_table_single_step": 76, "injected_table_multi_step_chain": 4, "model_proposed": 0}
81
+ },
82
+ "injected_table": ["rfl", "ring", "omega", "simp_all", "linarith", "trivial", "nlinarith", "aesop", "simp", "norm_num", "simp_all ; ring", "simp_all ; norm_num", "simp_all ; omega", "simp_all ; linarith", "gcongr", "field_simp ; ring"],
83
+ "ablation_no_model": {
84
+ "slice": "valid problems 0-9",
85
+ "with_model": {"passed": 2, "of": 10, "wall_min": 6.5},
86
+ "without_model": {"passed": 2, "of": 10, "wall_min": 3.3},
87
+ "note": "same problems, same depths, same proof terms; the model's candidates doubled the wall clock and added no solution"
88
+ },
89
+ "note": "The published rates are SYSTEM numbers: injected public automation tactics (16 entries, injected at every frontier node and every depth, so their compositions are searched too) + depth-batched best-first search + kernel-checked verification. The shipped 40M model supplies 0 of the accepted proofs. Do not cite these as neural tactic-generation results, and do not compare them with model-only pass@1 figures that have no automation table (this harness would score 0 without one)."
90
+ },
91
+ "candidate_level_stage3_e3": {
92
+ "valid_rate": 0.079,
93
+ "ge1_valid_per_problem": 0.566,
94
+ "ge1_direct_closer_per_problem": 0.063
95
+ },
96
+ "whole_tactic_greedy_exact_match": {
97
+ "matched": 0,
98
+ "of": 200,
99
+ "note": "a full tactic must be copied token-exactly"
100
+ }
101
+ }
special_tokens_v1.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|pad|>": 0,
3
+ "<|bos|>": 1,
4
+ "<|eos|>": 2,
5
+ "<|endoftext|>": 3,
6
+ "<|FORWARD|>": 4,
7
+ "<|REPAIR|>": 5,
8
+ "<|state|>": 6,
9
+ "<|tactic|>": 7
10
+ }
tokenizer_v1.json ADDED
The diff for this file is too large to render. See raw diff