duotactic full package: checkpoints, tokenizer, config, code, docs (part 3)
Browse files- data/minif2f/numbertheory_2pownm1prime_nprime.lean +10 -0
- data/minif2f/numbertheory_3pow2pownm1mod2pownp3eq2pownp2.lean +9 -0
- data/minif2f/numbertheory_4x3m7y3neq2003.lean +8 -0
- data/minif2f/numbertheory_aneqprodakp4_anmsqrtanp1eq2.lean +10 -0
- data/minif2f/numbertheory_aoddbdiv4asqpbsqmod8eq1.lean +12 -0
- data/minif2f/numbertheory_exk2powkeqapb2mulbpa2_aeq1.lean +10 -0
- data/minif2f/numbertheory_fxeq4powxp6powxp9powx_f2powmdvdf2pown.lean +12 -0
- data/minif2f/numbertheory_nckeqnm1ckpnm1ckm1.lean +10 -0
- data/minif2f/numbertheory_notequiv2i2jasqbsqdiv8.lean +7 -0
- data/minif2f/numbertheory_prmdvsneqnsqmodpeq0.lean +10 -0
- data/minif2f/numbertheory_sqmod3in01d.lean +8 -0
- data/minif2f/numbertheory_sqmod4in01d.lean +8 -0
- data/minif2f/numbertheory_sumkmulnckeqnmul2pownm1.lean +9 -0
- data/minif2f/numbertheory_x5neqy2p4.lean +8 -0
- data/minif2f/numbertheory_xsqpysqintdenomeq.lean +9 -0
- data/minif2f/test.jsonl +0 -0
- data/minif2f/valid.jsonl +0 -0
- data/whitelist/README.md +12 -0
- data/whitelist/train_ids.npy +3 -0
- data/whitelist/train_mask_start.npy +3 -0
- docs/USAGE.md +197 -0
- docs/USAGE.zh.md +181 -0
- examples/dev_sample.jsonl +16 -0
- first_token_whitelist.json +322 -0
- metrics.json +101 -0
- special_tokens_v1.json +10 -0
- tokenizer_v1.json +0 -0
data/minif2f/numbertheory_2pownm1prime_nprime.lean
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_2pownm1prime_nprime
|
| 7 |
+
(n : ℕ)
|
| 8 |
+
(h₀ : 0 < n)
|
| 9 |
+
(h₁ : Nat.Prime (2^n - 1)) :
|
| 10 |
+
Nat.Prime n := sorry
|
data/minif2f/numbertheory_3pow2pownm1mod2pownp3eq2pownp2.lean
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_3pow2pownm1mod2pownp3eq2pownp2
|
| 7 |
+
(n : ℕ)
|
| 8 |
+
(h₀ : 0 < n) :
|
| 9 |
+
(3^(2^n) - 1) % (2^(n + 3)) = 2^(n + 2) := sorry
|
data/minif2f/numbertheory_4x3m7y3neq2003.lean
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_4x3m7y3neq2003
|
| 7 |
+
(x y : ℤ) :
|
| 8 |
+
4 * x^3 - 7 * y^3 ≠ 2003 := sorry
|
data/minif2f/numbertheory_aneqprodakp4_anmsqrtanp1eq2.lean
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_aneqprodakp4_anmsqrtanp1eq2
|
| 7 |
+
(a : ℕ → ℝ)
|
| 8 |
+
(h₀ : a 0 = 1)
|
| 9 |
+
(h₁ : ∀ n, a (n + 1) = (∏ k ∈ Finset.range (n + 1), (a k)) + 4) :
|
| 10 |
+
∀ n ≥ 1, a n - Real.sqrt (a (n + 1)) = 2 := sorry
|
data/minif2f/numbertheory_aoddbdiv4asqpbsqmod8eq1.lean
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_aoddbdiv4asqpbsqmod8eq1
|
| 7 |
+
(a : ℤ)
|
| 8 |
+
(b : ℤ)
|
| 9 |
+
(h₀ : Odd a)
|
| 10 |
+
(h₁ : 4 ∣ b)
|
| 11 |
+
(h₂ : b >= 0) :
|
| 12 |
+
(a^2 + b^2) % 8 = 1 := sorry
|
data/minif2f/numbertheory_exk2powkeqapb2mulbpa2_aeq1.lean
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_exk2powkeqapb2mulbpa2_aeq1
|
| 7 |
+
(a b : ℕ)
|
| 8 |
+
(h₀ : 0 < a ∧ 0 < b)
|
| 9 |
+
(h₁ : ∃ k > 0, 2^k = (a + b^2) * (b + a^2)) :
|
| 10 |
+
a = 1 := sorry
|
data/minif2f/numbertheory_fxeq4powxp6powxp9powx_f2powmdvdf2pown.lean
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_fxeq4powxp6powxp9powx_f2powmdvdf2pown
|
| 7 |
+
(m n : ℕ)
|
| 8 |
+
(f : ℕ → ℕ)
|
| 9 |
+
(h₀ : ∀ x, f x = 4^x + 6^x + 9^x)
|
| 10 |
+
(h₁ : 0 < m ∧ 0 < n)
|
| 11 |
+
(h₂ : m ≤ n) :
|
| 12 |
+
f (2^m)∣f (2^n) := sorry
|
data/minif2f/numbertheory_nckeqnm1ckpnm1ckm1.lean
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_nckeqnm1ckpnm1ckm1
|
| 7 |
+
(n k : ℕ)
|
| 8 |
+
(h₀ : 0 < n ∧ 0 < k)
|
| 9 |
+
(h₁ : k ≤ n) :
|
| 10 |
+
Nat.choose n k = Nat.choose (n - 1) k + Nat.choose (n - 1) (k - 1) := sorry
|
data/minif2f/numbertheory_notequiv2i2jasqbsqdiv8.lean
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_notEquiv2i2jasqbsqdiv8 :
|
| 7 |
+
¬ (∀ a b : ℤ, (∃ i j, a = 2*i ∧ b=2*j) ↔ (∃ k, a^2 + b^2 = 8*k)) := sorry
|
data/minif2f/numbertheory_prmdvsneqnsqmodpeq0.lean
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_prmdvsneqnsqmodpeq0
|
| 7 |
+
(n : ℤ)
|
| 8 |
+
(p : ℕ)
|
| 9 |
+
(h₀ : Nat.Prime p) :
|
| 10 |
+
↑p ∣ n ↔ (n^2) % p = 0 := sorry
|
data/minif2f/numbertheory_sqmod3in01d.lean
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_sqmod3in01d
|
| 7 |
+
(a : ℤ) :
|
| 8 |
+
(a^2) % 3 = 0 ∨ (a^2) % 3 = 1 := sorry
|
data/minif2f/numbertheory_sqmod4in01d.lean
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_sqmod4in01d
|
| 7 |
+
(a : ℤ) :
|
| 8 |
+
(a^2 % 4) = 0 ∨ (a^2 % 4) = 1 := sorry
|
data/minif2f/numbertheory_sumkmulnckeqnmul2pownm1.lean
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_sumkmulnckeqnmul2pownm1
|
| 7 |
+
(n : ℕ)
|
| 8 |
+
(h₀ : 0 < n) :
|
| 9 |
+
∑ k ∈ Finset.Icc 1 n, (k * Nat.choose n k) = n * 2^(n - 1) := sorry
|
data/minif2f/numbertheory_x5neqy2p4.lean
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_x5neqy2p4
|
| 7 |
+
(x y : ℤ) :
|
| 8 |
+
x^5 ≠ y^2 + 4 := sorry
|
data/minif2f/numbertheory_xsqpysqintdenomeq.lean
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Mathlib
|
| 2 |
+
open BigOperators
|
| 3 |
+
open Real
|
| 4 |
+
open Nat
|
| 5 |
+
open Topology
|
| 6 |
+
theorem numbertheory_xsqpysqintdenomeq
|
| 7 |
+
(x y : ℚ)
|
| 8 |
+
(h₀ : (x^2 + y^2).den = 1) :
|
| 9 |
+
x.den = y.den := sorry
|
data/minif2f/test.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/minif2f/valid.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/whitelist/README.md
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# whitelist seed
|
| 2 |
+
|
| 3 |
+
The packaged search harness (`code/eval_minif2f.py` -> `leanoar/search.py`) builds its
|
| 4 |
+
first-token whitelist from `<steps_root>/train_ids.npy` + `train_mask_start.npy`.
|
| 5 |
+
Shipping the 479 MB training arrays just for that would be silly, so these two small
|
| 6 |
+
files are a *seed*: `train_ids.npy` is `(N, 2)` int64 whose **column 1** holds the
|
| 7 |
+
first tactic token of a training sample (repeated in proportion to its frequency in
|
| 8 |
+
the real train split) and `train_mask_start.npy` is all ones, so the derived histogram
|
| 9 |
+
reproduces the real top-64 first tokens in the real order.
|
| 10 |
+
|
| 11 |
+
Top-64 entries cover 97.6% of first-token occurrences; see
|
| 12 |
+
`../first_token_whitelist.json` for the same list with shares and token text.
|
data/whitelist/train_ids.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d43e31c1c50c0b71f67f89e5073d56f97a3bcb9c5482a829e59a2e7499531b77
|
| 3 |
+
size 800128
|
data/whitelist/train_mask_start.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a0c4c298c32c224a8ea8dfe760e68fde773e416c8fc78c9823b3daae3bbc9faa
|
| 3 |
+
size 400128
|
docs/USAGE.md
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Usage guide (English)
|
| 2 |
+
|
| 3 |
+
This package is **DuoTactic** — a Lean 4 proof-search harness — together with the 39.58 M-parameter
|
| 4 |
+
tactic model (`leanoar-schemec-40m`) that it can use as an optional candidate generator.
|
| 5 |
+
|
| 6 |
+
* To **run the harness** (sections 3 onward) you need Lean `4.35.0-rc2` and a mathlib4 checkout with a
|
| 7 |
+
prebuilt `.lake` cache (~10 GB). This is what produces miniF2F `test` 32.8 % / `valid` 34.8 %, at
|
| 8 |
+
45 s per problem, with every accepted proof re-compiled by the kernel.
|
| 9 |
+
* To **load or sample the model** (sections 0–2) you need no Lean. Note that `SchemeC` is **not** a
|
| 10 |
+
standard `transformers` model: it is defined in `code/leanoar/model.py`, its weights are plain
|
| 11 |
+
`lit_model.pth` / `model.safetensors` files, and it emits a *single Lean tactic* per call. There is
|
| 12 |
+
no `AutoModelForCausalLM.from_pretrained`. Everything below is copy-paste runnable from the package
|
| 13 |
+
root.
|
| 14 |
+
|
| 15 |
+
```
|
| 16 |
+
duotactic/
|
| 17 |
+
├── code/eval_minif2f.py DuoTactic itself: the end-to-end search harness
|
| 18 |
+
├── code/leanoar/ search + batched verifier + the 40M model
|
| 19 |
+
├── checkpoints/e6/ model weights (highest dev top-1)
|
| 20 |
+
├── checkpoints/stage3-e3/ model weights, end-to-end best variant (same pass@1, 16% faster)
|
| 21 |
+
├── data/dev/ dev protocol arrays (1024 rows)
|
| 22 |
+
├── data/minif2f/ miniF2F valid/test records + .lean files
|
| 23 |
+
├── data/whitelist/ seed for the search's first-token whitelist
|
| 24 |
+
├── examples/dev_sample.jsonl 16 real dev states + the real mathlib tactic
|
| 25 |
+
├── config.json metrics.json tokenizer_v1.json special_tokens_v1.json SHA256SUMS
|
| 26 |
+
└── README.md README.zh.md docs/USAGE.md docs/USAGE.zh.md
|
| 27 |
+
```
|
| 28 |
+
|
| 29 |
+
## 0. Install and verify
|
| 30 |
+
|
| 31 |
+
```bash
|
| 32 |
+
pip install -r code/requirements.txt # torch>=2.4, tokenizers>=0.19, numpy>=1.26, safetensors>=0.4
|
| 33 |
+
python -c "import torch; print(torch.__version__, torch.cuda.is_available())"
|
| 34 |
+
|
| 35 |
+
python code/verify_package.py # full check, 1024 dev rows (≈1 min on GPU)
|
| 36 |
+
python code/verify_package.py --quick # 128 rows, CPU-friendly
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
`verify_package.py` prints four sections and ends with `PASS` / `FAIL`:
|
| 40 |
+
|
| 41 |
+
1. **parameter count** — rebuilds `SchemeC` from `config.json`, must be 39,582,595 for both variants;
|
| 42 |
+
2. **dev protocol** — recomputes top-1/top-5/CE on the packaged 1024 rows and compares with
|
| 43 |
+
`metrics.json` (tolerance 5e-4). This is the check that makes the published numbers auditable;
|
| 44 |
+
3. **inference wiring** — runs the packaged example state through the sampler (it also shows whether
|
| 45 |
+
the model's preferred first token matches mathlib's; a miss is normal and does not fail the check);
|
| 46 |
+
4. **checksums** — every file against `SHA256SUMS`.
|
| 47 |
+
|
| 48 |
+
Exit code is 0 only if 1, 2 and 4 pass. Use `--skip-sums` if you only have the weights.
|
| 49 |
+
|
| 50 |
+
## 1. Standalone inference (no Lean)
|
| 51 |
+
|
| 52 |
+
```bash
|
| 53 |
+
python code/generate.py # packaged dev example #0
|
| 54 |
+
python code/generate.py --example 3 --k 8 # another real dev state
|
| 55 |
+
python code/generate.py --state-file state.txt # your own state (format below)
|
| 56 |
+
python code/generate.py --ckpt checkpoints/stage3-e3 # the other variant
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
A **state** is the text Lean prints for a goal, i.e. hypotheses then `⊢ goal`:
|
| 60 |
+
|
| 61 |
+
```
|
| 62 |
+
n m : ℕ
|
| 63 |
+
h : n < m
|
| 64 |
+
⊢ n ≤ m
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
You can obtain one from Lean with `trace_state` inside a proof, or from an `unsolved goals` error
|
| 68 |
+
message. In your own code:
|
| 69 |
+
|
| 70 |
+
```python
|
| 71 |
+
import json, os, sys, torch
|
| 72 |
+
sys.path.insert(0, 'code')
|
| 73 |
+
from common import load_model, load_tokenizer, encode_state, specials, whitelist
|
| 74 |
+
from generate import propose
|
| 75 |
+
|
| 76 |
+
net, cfg, device = load_model('checkpoints/e6') # or 'checkpoints/stage3-e3'
|
| 77 |
+
tok, wl = load_tokenizer(), whitelist()
|
| 78 |
+
for tactic, avg_logprob in propose(net, tok, wl, state, k=16, device=device):
|
| 79 |
+
... # send to Lean for verification
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
Four rules that matter:
|
| 83 |
+
|
| 84 |
+
* **Restrict the first token** to `first_token_whitelist.json` (64 ids, 97.6 % of training first
|
| 85 |
+
tokens). `propose()` already does this. Without it, most candidates start with meaningless tokens
|
| 86 |
+
and the type-check rate collapses.
|
| 87 |
+
* **One candidate is one tactic.** The model separates two intended steps by a double space and may
|
| 88 |
+
end a truncated tactic with a dangling connective; `propose()` cuts both.
|
| 89 |
+
* **Truncation**: states longer than `ctx − 4 − max_new_tokens` tokens keep the head 60 % and the
|
| 90 |
+
tail 40 % (the `⊢ goal` line is last). Never feed more than `context_length` tokens total.
|
| 91 |
+
* **Only the Lean kernel decides correctness.** Ask for many candidates and verify them; do not trust
|
| 92 |
+
any single output. Whole-tactic exact match is 0/200, and a "valid" (type-checking) candidate is
|
| 93 |
+
still not a proof.
|
| 94 |
+
|
| 95 |
+
## 2. Reproduce the dev metric
|
| 96 |
+
|
| 97 |
+
```bash
|
| 98 |
+
python code/eval_dev.py --all --compare # both variants, 1024 rows, compare with metrics.json
|
| 99 |
+
python code/eval_dev.py --ckpt checkpoints/e6 --rows 256
|
| 100 |
+
```
|
| 101 |
+
|
| 102 |
+
Expected (tolerance 5e-4):
|
| 103 |
+
|
| 104 |
+
| variant | top-1 | top-5 | CE |
|
| 105 |
+
|---|---|---|---|
|
| 106 |
+
| `checkpoints/e6` | 0.3076 | 0.7090 | 2.5494 |
|
| 107 |
+
| `checkpoints/stage3-e3` | 0.2891 | 0.7246 | 2.4971 |
|
| 108 |
+
|
| 109 |
+
The protocol: take the first 1024 rows of `data/dev/dev_ids.npy` whose second token is `<|FORWARD|>`,
|
| 110 |
+
read the logits at `mask_start − 1` and score the token at `mask_start`. **Do not compare these with
|
| 111 |
+
the numbers a trainer prints** — those come from a length-bucketed reshuffle of the dev split.
|
| 112 |
+
|
| 113 |
+
## 3. End-to-end proof search on miniF2F (needs Lean + mathlib)
|
| 114 |
+
|
| 115 |
+
### 3.1 Prerequisites
|
| 116 |
+
|
| 117 |
+
* A **Lean 4.35.0-rc2** toolchain (e.g. `elan toolchain install leanprover/lean4:v4.35.0-rc2`).
|
| 118 |
+
Note: this is a *release candidate*; newer Lean versions will re-elaborate mathlib differently.
|
| 119 |
+
* A **mathlib4 checkout with a prebuilt `.lake` cache** (this project used the Sept 2026 mathlib
|
| 120 |
+
snapshot; the exact commit is not pinned in the release, so state text may drift slightly on other
|
| 121 |
+
revisions). Build the cache with `lake exe cache get` inside the checkout — a full local build takes
|
| 122 |
+
hours, the olean cache takes minutes and ~7 GB.
|
| 123 |
+
* ≥ 16 GB RAM if you use `--lean-parallel 6` (each `lean --json` process loads mathlib; ~0.4 s per
|
| 124 |
+
candidate, 6 in parallel measured at 0.034 s/candidate).
|
| 125 |
+
|
| 126 |
+
```bash
|
| 127 |
+
export LEANOAR_LEAN_BIN=/path/to/lean-4.35.0-rc2-linux/bin/lean
|
| 128 |
+
export LEANOAR_MATHLIB=/path/to/mathlib4
|
| 129 |
+
$LEANOAR_LEAN_BIN --version # sanity check
|
| 130 |
+
```
|
| 131 |
+
|
| 132 |
+
### 3.2 The run that produced the published number
|
| 133 |
+
|
| 134 |
+
```bash
|
| 135 |
+
python code/eval_minif2f.py --split valid --n 89 --start 0 \
|
| 136 |
+
--k 16 --beam 4 --depth 6 --budget 45 --round-budget 30 \
|
| 137 |
+
--force --force-set v2 --force-depth 99 --lean-parallel 6 \
|
| 138 |
+
--ckpt checkpoints/stage3-e3 --tag mine
|
| 139 |
+
```
|
| 140 |
+
|
| 141 |
+
Expected: **30/89 = 33.7 %**, ≈52 min on an RTX 4060 Laptop (the `e6` variant gives the same 30
|
| 142 |
+
problems in ≈60 min). The full `valid` split was run end-to-end with this exact configuration:
|
| 143 |
+
**85/244 = 34.8 %** in ≈2.3 h (segments 33.7 % / 40.4 % / 28.8 %; 78 of the 85 solutions are one
|
| 144 |
+
tactic). Longer runs are easier as `--lean-parallel` rises: the fixed cost of every `lean --json`
|
| 145 |
+
call is ≈6.6 s of `import Mathlib` (vs ≈0.03 s per candidate), so parallelism and import trimming
|
| 146 |
+
are where the wall clock goes.
|
| 147 |
+
|
| 148 |
+
What the flags do — this is where the score comes from:
|
| 149 |
+
|
| 150 |
+
| Flag | Meaning |
|
| 151 |
+
|---|---|
|
| 152 |
+
| `--k 16` | candidates per state (top-16 whitelisted first tokens, greedy continuation) |
|
| 153 |
+
| `--beam 4` | how many open states are expanded per round |
|
| 154 |
+
| `--depth 6` | max proof length in tactics (measured: 25 of 30 solutions are depth 1) |
|
| 155 |
+
| `--budget 45` | wall-clock seconds per problem; `--round-budget 30` caps a single round |
|
| 156 |
+
| `--force --force-set v2` | inject the exported automation table (`rfl`, `ring`, `simp_all`, `omega`, …, `simp_all ; ring`, `gcongr`) at every depth. **This is the single biggest lever**: with it the first-89 result is 30/89, without it the whole 244 gives 8/244. |
|
| 157 |
+
| `--lean-parallel 6` | verify 6 chunks of candidates concurrently (pure engineering: 2.07× wall clock) |
|
| 158 |
+
| `--narrow-imports` | per problem, replace the full Mathlib import by `#min_imports` + `Mathlib.Tactic` (measured 3.41 s → 1.74 s per `lean --json` call). A candidate that fails with "unknown identifier/tactic" is automatically re-checked with the full import, so narrowing cannot lose a proof (verified: 85/85 accepted valid proofs still compile) |
|
| 159 |
+
| `--no-model` | ablation switch: verify only the injected table, no model proposals. On valid problems 0-9 this reproduces the model's result exactly (2/10, same problems, same proof terms) in half the wall clock |
|
| 160 |
+
|
| 161 |
+
Before quoting any pass rate, read **“Where this score comes from”** in `README.md`: these are
|
| 162 |
+
*system* numbers (injected automation table + search + kernel verification), and the shipped model
|
| 163 |
+
supplies 0 of the accepted proofs.
|
| 164 |
+
|
| 165 |
+
### 3.3 Reading and resuming results
|
| 166 |
+
|
| 167 |
+
Results are appended to `reports/search_eval_valid_mine.jsonl` (one JSON object per problem:
|
| 168 |
+
`solved`, `depth`, `seconds`, `verified`, `proof`, `confirmed`, `reason`) and every run prints
|
| 169 |
+
cumulative progress with an ETA. Because the file is append-only, an interrupted run resumes with
|
| 170 |
+
`--start <rows already written>` — or simply re-run the same command.
|
| 171 |
+
|
| 172 |
+
### 3.4 Troubleshooting
|
| 173 |
+
|
| 174 |
+
* `initial-state-failed` for a problem: its statement does not even compile against your mathlib
|
| 175 |
+
revision. Two miniF2F problems fail on mathlib master for everyone
|
| 176 |
+
(`mathd_numbertheory_711`, `imosl_2007_algebra_p6`).
|
| 177 |
+
* `No such file or directory: .../bin/lean`: set `LEANOAR_LEAN_BIN`.
|
| 178 |
+
* Lean errors about missing oleans / `unknown module prefix 'Mathlib'`: set `LEANOAR_MATHLIB` to a
|
| 179 |
+
checkout with a **built** `.lake/build/lib/lean` (and `.lake/packages/*`); the verifier assembles
|
| 180 |
+
`LEAN_PATH` from those paths.
|
| 181 |
+
* Very low `verified` counts: check that the first-token whitelist is being applied and that the
|
| 182 |
+
tokenizer is `tokenizer_v1.json`.
|
| 183 |
+
* A state longer than the prompt budget used to overflow the context by one token in old builds; the
|
| 184 |
+
shipped `search.py` budgets `ctx − 4 − max_new_tokens` and hard-stops at `ctx`. If you edit the
|
| 185 |
+
sampler, keep that invariant.
|
| 186 |
+
|
| 187 |
+
## 4. Extending
|
| 188 |
+
|
| 189 |
+
* Swap in your own weights: put `lit_model.pth` (+ `model.safetensors`) in a directory and pass
|
| 190 |
+
`--ckpt <dir>` to `generate.py` / `eval_dev.py` / `eval_minif2f.py`. `config.json` must match the
|
| 191 |
+
architecture; the vocabulary and context length are fixed.
|
| 192 |
+
* Add automation tactics to `FORCE_SETS` in `code/eval_minif2f.py` — measured gains in this project
|
| 193 |
+
came almost exclusively from that table and from the search/verifier side, not from the model.
|
| 194 |
+
* The value head is shipped but unusable (AUC 0.5201, below the policy's own confidence). Do not
|
| 195 |
+
build a reranker on it.
|
| 196 |
+
* `code/leanoar/data.py` is included because the search code imports `load_specials` from it; the
|
| 197 |
+
training pipeline itself is not part of this package.
|
docs/USAGE.zh.md
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 使用说明(中文)
|
| 2 |
+
|
| 3 |
+
本包是 **DuoTactic** —— 一个 Lean 4 证明搜索 harness —— 连同它可选的候选生成器:39.58 M 参数的
|
| 4 |
+
tactic 模型(`leanoar-schemec-40m`)。
|
| 5 |
+
|
| 6 |
+
* **跑 harness**(第 3 节起)需要 Lean `4.35.0-rc2` 与带预编译 `.lake` 缓存的 mathlib4(约 10 GB)。
|
| 7 |
+
miniF2F `test` 32.8% / `valid` 34.8% 就是它跑出来的:每题 45 s,每条被接受的证明都经内核重编译。
|
| 8 |
+
* **加载/采样模型**(第 0–2 节)不需要 Lean。注意 `SchemeC` **不是**标准 `transformers` 模型:它定义在
|
| 9 |
+
`code/leanoar/model.py`,权重是普通的 `lit_model.pth` / `model.safetensors`,每次调用只产**一条
|
| 10 |
+
Lean tactic**,没有 `AutoModelForCausalLM.from_pretrained` 这种入口。下面所有命令都可以在包根目录
|
| 11 |
+
直接复制运行。
|
| 12 |
+
|
| 13 |
+
```
|
| 14 |
+
duotactic/
|
| 15 |
+
├── code/eval_minif2f.py DuoTactic 本体:端到端搜索 harness
|
| 16 |
+
├── code/leanoar/ 搜索 + 批量验证器 + 40M 模型
|
| 17 |
+
├── checkpoints/e6/ 模型权重(dev top-1 最高)
|
| 18 |
+
├── checkpoints/stage3-e3/ 模型权重,端到端最佳变体(同样通过率,快 16%)
|
| 19 |
+
├── data/dev/ dev 协议数组(1024 行)
|
| 20 |
+
├── data/minif2f/ miniF2F valid/test 题目 + .lean 文件
|
| 21 |
+
├── data/whitelist/ 搜索用首 token 白名单的种子文件
|
| 22 |
+
├── examples/dev_sample.jsonl 16 条真实 dev 状态 + mathlib 真实用过的 tactic
|
| 23 |
+
├── config.json metrics.json tokenizer_v1.json special_tokens_v1.json SHA256SUMS
|
| 24 |
+
└── README.md README.zh.md docs/USAGE.md docs/USAGE.zh.md
|
| 25 |
+
```
|
| 26 |
+
|
| 27 |
+
## 0. 安装与验证
|
| 28 |
+
|
| 29 |
+
```bash
|
| 30 |
+
pip install -r code/requirements.txt # torch>=2.4, tokenizers>=0.19, numpy>=1.26, safetensors>=0.4
|
| 31 |
+
python -c "import torch; print(torch.__version__, torch.cuda.is_available())"
|
| 32 |
+
|
| 33 |
+
python code/verify_package.py # 完整校验,1024 行(GPU 约 1 分钟)
|
| 34 |
+
python code/verify_package.py --quick # 128 行,CPU 也能跑
|
| 35 |
+
```
|
| 36 |
+
|
| 37 |
+
`verify_package.py` 打印四段,最后给出 `PASS` / `FAIL`:
|
| 38 |
+
|
| 39 |
+
1. **参数数量**——从 `config.json` 重建 `SchemeC`,两个变体都必须是 39,582,595;
|
| 40 |
+
2. **dev 协议**——在包内 1024 行上复算 top-1/top-5/CE 并与 `metrics.json` 对比(容差 5e-4)。
|
| 41 |
+
这一步让公开数字是可审计的;
|
| 42 |
+
3. **推理连通性**——拿包内示例状态跑一遍采样(同时显示模型首选首 token 是否与 mathlib 一致;
|
| 43 |
+
不一致是正常的,不影响判定);
|
| 44 |
+
4. **校验和**——逐文件对照 `SHA256SUMS`。
|
| 45 |
+
|
| 46 |
+
只有 1、2、4 全过才返回 0。只有权重时可用 `--skip-sums`。
|
| 47 |
+
|
| 48 |
+
## 1. 独立推理(不需要 Lean)
|
| 49 |
+
|
| 50 |
+
```bash
|
| 51 |
+
python code/generate.py # 包内真实 dev 样本 #0
|
| 52 |
+
python code/generate.py --example 3 --k 8 # 另一条真实 dev 状态
|
| 53 |
+
python code/generate.py --state-file state.txt # 你自己的状态(格式见下)
|
| 54 |
+
python code/generate.py --ckpt checkpoints/stage3-e3 # 换变体
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
**状态**就是 Lean 打印目标时的那段文本——先是假设,然后 `⊢ 目标`:
|
| 58 |
+
|
| 59 |
+
```
|
| 60 |
+
n m : ℕ
|
| 61 |
+
h : n < m
|
| 62 |
+
⊢ n ≤ m
|
| 63 |
+
```
|
| 64 |
+
|
| 65 |
+
可以在证明里用 `trace_state` 拿到,或从 `unsolved goals` 报错里抠出来。在自己的代码里调用:
|
| 66 |
+
|
| 67 |
+
```python
|
| 68 |
+
import json, os, sys, torch
|
| 69 |
+
sys.path.insert(0, 'code')
|
| 70 |
+
from common import load_model, load_tokenizer, encode_state, specials, whitelist
|
| 71 |
+
from generate import propose
|
| 72 |
+
|
| 73 |
+
net, cfg, device = load_model('checkpoints/e6') # 或 'checkpoints/stage3-e3'
|
| 74 |
+
tok, wl = load_tokenizer(), whitelist()
|
| 75 |
+
for tactic, avg_logprob in propose(net, tok, wl, state, k=16, device=device):
|
| 76 |
+
... # 交给 Lean 验证
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
**四条必须遵守的规则**:
|
| 80 |
+
|
| 81 |
+
* **首个 token 必须限制**在 `first_token_whitelist.json`(64 个 id,覆盖训练集 97.6% 的首 token)。
|
| 82 |
+
`propose()` 已经做了这件事;不做的话多数候选会以无意义 token 开头,编译通过率会崩。
|
| 83 |
+
* **一条候选 = 一条 tactic**。模型用双空格分隔它想表达的两步;被截断的 tactic 可能以悬挂连接词结尾,
|
| 84 |
+
`propose()` 两种都会切掉。
|
| 85 |
+
* **截断规则**:超过 `ctx − 4 − max_new_tokens` 个 token 的状态,会保留头部 60% + 尾部 40%
|
| 86 |
+
(`⊢ 目标` 在最后)。总输入绝不能超过 `context_length`。
|
| 87 |
+
* **只有 Lean 内核能判对错**。请多要候选 + 逐个验证,不要相信任何单条输出。整条 tactic 的精确匹配率是
|
| 88 |
+
0/200,而且「通过编译(valid)」也不等于「证明成立」。
|
| 89 |
+
|
| 90 |
+
## 2. 复算 dev 指标
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
python code/eval_dev.py --all --compare # 两个变体 × 1024 行,并与 metrics.json 对比
|
| 94 |
+
python code/eval_dev.py --ckpt checkpoints/e6 --rows 256
|
| 95 |
+
```
|
| 96 |
+
|
| 97 |
+
预期(容差 5e-4):
|
| 98 |
+
|
| 99 |
+
| 变体 | top-1 | top-5 | CE |
|
| 100 |
+
|---|---|---|---|
|
| 101 |
+
| `checkpoints/e6` | 0.3076 | 0.7090 | 2.5494 |
|
| 102 |
+
| `checkpoints/stage3-e3` | 0.2891 | 0.7246 | 2.4971 |
|
| 103 |
+
|
| 104 |
+
协议:取 `data/dev/dev_ids.npy` ��第二个 token 为 `<|FORWARD|>` 的前 1024 行,读 `mask_start − 1`
|
| 105 |
+
位置的 logits,对 `mask_start` 处的 token 打分。**不要拿它跟训练器打印的 dev 数字比**——后者按长度
|
| 106 |
+
分桶重排过,不是同一批样本。
|
| 107 |
+
|
| 108 |
+
## 3. miniF2F 端到端搜索(需要 Lean + mathlib)
|
| 109 |
+
|
| 110 |
+
### 3.1 前置条件
|
| 111 |
+
|
| 112 |
+
* **Lean 4.35.0-rc2** 工具链(例如 `elan toolchain install leanprover/lean4:v4.35.0-rc2`)。
|
| 113 |
+
注意这是 release candidate;换 Lean 版本后 mathlib 的展开方式与状态文本都会变。
|
| 114 |
+
* **带预编译 `.lake` 缓存的 mathlib4 检出**(本项目用 2026-09 的快照;发布包里没有钉死 commit,
|
| 115 |
+
所以换 revision 时状态文本可能有轻微漂移)。在检出目录里执行 `lake exe cache get` 拿 olean 缓存——
|
| 116 |
+
本地全量编译要数小时,缓存只要几分钟、约 7 GB。
|
| 117 |
+
* 用 `--lean-parallel 6` 时建议 ≥16 GB 内存(每个 `lean --json` 进程都要加载 mathlib;
|
| 118 |
+
实测单候选 0.4 s、6 路并行后 0.034 s/候选)。
|
| 119 |
+
|
| 120 |
+
```bash
|
| 121 |
+
export LEANOAR_LEAN_BIN=/path/to/lean-4.35.0-rc2-linux/bin/lean
|
| 122 |
+
export LEANOAR_MATHLIB=/path/to/mathlib4
|
| 123 |
+
$LEANOAR_LEAN_BIN --version # 自检
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
### 3.2 复现公开成绩的那条命令
|
| 127 |
+
|
| 128 |
+
```bash
|
| 129 |
+
python code/eval_minif2f.py --split valid --n 89 --start 0 \
|
| 130 |
+
--k 16 --beam 4 --depth 6 --budget 45 --round-budget 30 \
|
| 131 |
+
--force --force-set v2 --force-depth 99 --lean-parallel 6 \
|
| 132 |
+
--ckpt checkpoints/stage3-e3 --tag mine
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
预期:**30/89 = 33.7%**,RTX 4060 Laptop 上约 52 分钟(换成 `e6` 变体也是同样 30 题,约 60 分钟)。
|
| 136 |
+
全量 `valid` 已用这份完全相同的配置跑完:**85/244 = 34.8%**,约 2.3 小时(分段 33.7% / 40.4% / 28.8%;
|
| 137 |
+
85 条解出里 78 条只有一条 tactic)。想再快就动两处:每次 `lean --json` 调用的固定成本约 **6.6 s**
|
| 138 |
+
的 `import Mathlib`(而每条候选只要约 0.03 s),所以墙钟全花在并行度和导入裁剪上。
|
| 139 |
+
|
| 140 |
+
各参数的含义——成绩其实来自这里:
|
| 141 |
+
|
| 142 |
+
| 参数 | 含义 |
|
| 143 |
+
|---|---|
|
| 144 |
+
| `--k 16` | 每个状态提 16 条候选(白名单内 top-16 首 token + 贪心续写) |
|
| 145 |
+
| `--beam 4` | 每轮扩展 4 个开放状态 |
|
| 146 |
+
| `--depth 6` | 证明长度上限(以 tactic 计)。实测:30 条解出里有 25 条是 depth 1 |
|
| 147 |
+
| `--budget 45` | 每题墙钟预算 45 秒;`--round-budget 30` 限制单轮 |
|
| 148 |
+
| `--force --force-set v2` | 在每一层注入导出的自动化表(`rfl`、`ring`、`simp_all`、`omega`、…、`simp_all ; ring`、`gcongr`)。**这是最大的杠杆**:带它前 89 题是 30/89;不带它的全量 244 题只有 8/244 |
|
| 149 |
+
| `--lean-parallel 6` | 6 路并发验证候选(纯工程收益:墙钟 2.07×) |
|
| 150 |
+
| `--narrow-imports` | 每题把全量 Mathlib 导入换成 `#min_imports` + `Mathlib.Tactic`(实测每次 `lean --json` 从 3.41 s 降到 1.74 s)。命中「未知标识符/tactic」的候选会自动用全量导入重验,所以窄导入**不可能丢证明**(已验:85/85 条已验证证明仍可编译) |
|
| 151 |
+
| `--no-model` | 对照开关:只验证注入表、不提模型候选。在 valid 前 10 题上它与带模型的结果完全一致(2/10、同题、同证明项),而墙钟减半 |
|
| 152 |
+
|
| 153 |
+
引用任何通过率之前,请先读 `README.zh.md` 的「**这个成绩是怎么来的**」:这些是**系统**成绩
|
| 154 |
+
(注入的自动化表 + 搜索 + 内核验证),随包模型的贡献是 0 条。
|
| 155 |
+
|
| 156 |
+
### 3.3 看结果与续跑
|
| 157 |
+
|
| 158 |
+
结果按行追加到 `reports/search_eval_valid_mine.jsonl`(每个题目一个 JSON:`solved`、`depth`、
|
| 159 |
+
`seconds`、`verified`、`proof`、`confirmed`、`reason`),同时实时打印累计进度与 ETA。因为是追加写入,
|
| 160 |
+
中断后用 `--start <已写入行数>` 续跑即可,或者直接重跑同一条命令。
|
| 161 |
+
|
| 162 |
+
### 3.4 疑难排查
|
| 163 |
+
|
| 164 |
+
* 某题报 `initial-state-failed`:它的陈述在你的 mathlib revision 上根本编译不过。有 2 道 miniF2F
|
| 165 |
+
题对所有人都不通过(`mathd_numbertheory_711`、`imosl_2007_algebra_p6`)。
|
| 166 |
+
* 报 `No such file or directory: .../bin/lean`:设 `LEANOAR_LEAN_BIN`。
|
| 167 |
+
* Lean 报缺 olean / `unknown module prefix 'Mathlib'`:把 `LEANOAR_MATHLIB` 指向**已编译**
|
| 168 |
+
(`.lake/build/lib/lean` 与 `.lake/packages/*` 齐备)的检出目录,验证器会据此拼 `LEAN_PATH`。
|
| 169 |
+
* `verified` 数特别低:确认首 token 白名单生效、切词器用的是 `tokenizer_v1.json`。
|
| 170 |
+
* 旧版本在超长状态上会让上下文溢出 1 个 token;本包内的 `search.py` 已按
|
| 171 |
+
`ctx − 4 − max_new_tokens` 计算预算并在 `ctx` 处硬停。如果你改采样器,请保持这个不变式。
|
| 172 |
+
|
| 173 |
+
## 4. 扩展
|
| 174 |
+
|
| 175 |
+
* 换自己的权重:把 `lit_model.pth`(和 `model.safetensors`)放进一个目录,给
|
| 176 |
+
`generate.py` / `eval_dev.py` / `eval_minif2f.py` 传 `--ckpt <目录>`。`config.json` 必须与架构一致;
|
| 177 |
+
词表与 ctx 是固定的。
|
| 178 |
+
* 往 `code/eval_minif2f.py` 的 `FORCE_SETS` 里加自动化 tactic——本项目所有实测收益几乎都来自这张表
|
| 179 |
+
与搜索/验���器侧,而不是模型。
|
| 180 |
+
* Value 头虽然随包发出但不可用(AUC 0.5201,低于策略自身置信度)。不要拿它做重排。
|
| 181 |
+
* `code/leanoar/data.py` 只是因为搜索代码要用里面的 `load_specials` 才包含进来;训练流水线不在本包内。
|
examples/dev_sample.jsonl
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"dev_row": 0, "state": "L : Language\nL' : Language\nM : Type w\nN : Type u_1\nP : Type ?u\ninst : Structure L M\ninst : Structure L N\ninst : Structure L P\nα : Type u'\nβ : Type v'\nn : ℕ\nT : Theory L\nh : M ≅[L] N\n⊢ M ⊨ T ↔ N ⊨ T", "true_first_token": "rw", "true_tactic": "rw [Theory.model_iff_subset_completeTheory, Theory.model_iff_subset_completeTheory,\n h.completeTheory_eq]"}
|
| 2 |
+
{"dev_row": 68, "state": "case codisjoint\nR : Type u_1\ninst : Ring R\nE : Type u_2\ninst : AddCommGroup E\ninst : Module R E\nF : Type ?u\ninst : AddCommGroup F\ninst : Module R F\nG : Type ?u\ninst : AddCommGroup G\ninst : Module R G\np q : Submodule R E\nS : Type ?u\ninst : Semiring S\nM : Type ?u\ninst : AddCommMonoid M\ninst : Module S M\nm : Submodule S M\nf : E →ₗ[R] { x // x ∈ p }\nhf : ∀ (x : { x // x ∈ p }), ↑f ↑x = x\nx : E\na : x ∈ ⊤\n⊢ ∃ y z, ↑y + ↑z = x", "true_first_token": "refine", "true_tactic": "refine' ⟨f x, ⟨x - f x, _⟩, add_sub_cancel'_right _ _⟩"}
|
| 3 |
+
{"dev_row": 136, "state": "case inr.mp\nE : ℕ → Type u_1\nx y : (n : ℕ) → E n\nn : ℕ\nhne : y ≠ x\nhy : ∀ (i : ℕ), i < n → y i = x i\n⊢ n ≤ firstDiff y x", "true_first_token": "by", "true_tactic": "by_contra' H"}
|
| 4 |
+
{"dev_row": 214, "state": "E : Type u_1\nX : Type u_2\ninst : TopologicalSpace E\ninst : TopologicalSpace X\nf : E → X\ns : Set X\nhf : IsCoveringMapOn f s\n⊢ IsLocallyHomeomorphOn f (f ⁻¹' s)", "true_first_token": "refine", "true_tactic": "refine' IsLocallyHomeomorphOn.mk f (f ⁻¹' s) fun x hx => _"}
|
| 5 |
+
{"dev_row": 285, "state": "case neg.mp\nz : ℂ\nh₀ : ¬z = 0\nh : arg z = π\n⊢ (↑(↑abs z) * (cos ↑π + sin ↑π * I)).re < 0 ∧ (↑(↑abs z) * (cos ↑π + sin ↑π * I)).im = 0", "true_first_token": "simp", "true_tactic": "simp [h₀]"}
|
| 6 |
+
{"dev_row": 359, "state": "case this\nα : Type u\nβ : Type v\nX : Type ?u\ninst : PseudoEMetricSpace α\nx y z : α\nε ε₁ ε₂ : ℝ≥0∞\ns t : Set α\nhs : ∀ (ε : ℝ≥0∞), ε > 0 → ∃ t, Set.Countable t ∧ (⋃ (x : α) (_ : x ∈ t), closedBall x ε) = univ\n⊢ ∀ (ε : ℝ≥0∞), ε > 0 → ∃ t, Set.Countable t ∧ univ ⊆ ⋃ (x : α) (_ : x ∈ t), closedBall x ε", "true_first_token": "simpa", "true_tactic": "simpa only [univ_subset_iff] using hs"}
|
| 7 |
+
{"dev_row": 432, "state": "V : Type u_1\nP : Type u_2\ninst : NormedAddCommGroup V\ninst : InnerProductSpace ℝ V\ninst : MetricSpace P\ninst : NormedAddTorsor V P\np₁ p₂ p₃ : P\nh : ∠ p₁ p₂ p₃ = π / 2\nh0 : p₁ = p₂ ∨ p₃ ≠ p₂\n⊢ dist p₃ p₂ / Real.cos (∠ p₂ p₃ p₁) = dist p₁ p₃", "true_first_token": "rw", "true_tactic": "rw [angle, ← inner_eq_zero_iff_angle_eq_pi_div_two, real_inner_comm, ← neg_eq_zero, ←\n inner_neg_left, neg_vsub_eq_vsub_rev] at h"}
|
| 8 |
+
{"dev_row": 503, "state": "R : Type u\nS : Type v\nk : Type y\nA : Type z\na b : R\nn : ℕ\ninst : Field R\np q : R[X]\nhp0 : IsUnit (leadingCoeff p)\n⊢ Monic (↑normalize p)", "true_first_token": "rw", "true_tactic": "rw [Monic, leadingCoeff_normalize, normalize_eq_one]"}
|
| 9 |
+
{"dev_row": 578, "state": "V : Type u_1\nP : Type u_2\ninst : NormedAddCommGroup V\ninst : InnerProductSpace ℝ V\ninst : MetricSpace P\ninst : NormedAddTorsor V P\ns : AffineSubspace ℝ P\ninst : FiniteDimensional ℝ { x // x ∈ direction s }\nhd : finrank ℝ { x // x ∈ direction s } = 2\nc₁ c₂ p₁ p₂ p : P\nhc₁s : c₁ ∈ s\nhc₂s : c₂ ∈ s\nhp₁s : p₁ ∈ s\nhp₂s : p₂ ∈ s\nhps : p ∈ s\nr₁ r₂ : ℝ\nhc : c₁ ≠ c₂\nhp : p₁ ≠ p₂\nhp₁c₁ : dist p₁ c₁ = r₁\nhp₂c₁ : dist p₂ c₁ = r₁\nhpc₁ : dist p c₁ = r₁\nhp₁c₂ : dist p₁ c₂ = r₂\nhp₂c₂ : dist p₂ c₂ = r₂\nhpc₂ : dist p c₂ = r₂\n⊢ p = p₁ ∨ p = p₂", "true_first_token": "have", "true_tactic": "have ho : ⟪c₂ -ᵥ c₁, p₂ -ᵥ p₁⟫ = 0 :=\n inner_vsub_vsub_of_dist_eq_of_dist_eq (hp₁c₁.trans hp₂c₁.symm) (hp₁c₂.trans hp₂c₂.symm)"}
|
| 10 |
+
{"dev_row": 648, "state": "ι : Type u_1\nα : Type u_2\nβ : Type ?u\nr : α → α → Prop\ns : Finset ι\nf : ι → Set α\n⊢ Set.PartiallyWellOrderedOn (sup ∅ f) r ↔ ∀ (i : ι), i ∈ ∅ → Set.PartiallyWellOrderedOn (f i) r", "true_first_token": "simp", "true_tactic": "simp"}
|
| 11 |
+
{"dev_row": 721, "state": "α : Type ?u\nβ : Type ?u\nγ : Type ?u\nr : α → α → Prop\ns : β → β → Prop\nt : γ → γ → Prop\nα : Type u_1\nr : α → α → Prop\nwo : IsWellOrder α r\nh : ∀ (a : Ordinal), a < type r → succ a < type r\nx : α\n⊢ ∃ y, r x y", "true_first_token": "use", "true_tactic": "use enum r (succ (typein r x)) (h _ (typein_lt_type r x))"}
|
| 12 |
+
{"dev_row": 793, "state": "R : Type u_1\ninst : CommRing R\nn : ℕ\nx : MvPolynomial Bool R := MvPolynomial.X true\ny : MvPolynomial Bool R := MvPolynomial.X false\npderiv_true_x : ↑(pderiv true) x = 1\npderiv_true_y : ↑(pderiv true) y = 0\ne : Bool → R[X] := fun i => bif i then X else 1 - X\n⊢ ↑(MvPolynomial.aeval e) (↑(pderiv true) (↑(pderiv true) ((x + y) ^ n))) * X ^ 2 = (n * (n - 1)) • X ^ 2", "true_first_token": "simp", "true_tactic": "simp only [pderiv_one, pderiv_mul, (pderiv _).leibniz_pow, (pderiv _).map_coe_nat,\n (pderiv true).map_add, pderiv_true_x, pderiv_true_y, Algebra.id.smul_eq_mul, add_zero,\n mul_one, Derivation.map_smul_of_tower, map_nsmul, map_pow, map_add, Bool.cond_true,\n Bool.cond_false, MvPolynomial.aeval_X, add_sub_cancel'_right, one_pow, smul_smul,\n smul_one_mul]"}
|
| 13 |
+
{"dev_row": 865, "state": "case neg.intro\nα : Type u\nβ : Type v\nγ : Type w\ninst : TopologicalSpace α\ninst : TopologicalSpace β\ninst : LinearOrder α\ninst : LinearOrder β\ninst : OrderTopology α\ninst : OrderTopology β\nt : Set α\nx : α\ninst : IsCountablyGenerated (𝓝 x)\nhtx : IsLUB t x\nht : Set.Nonempty t\nh : ¬x ∈ t\nu : ℕ → α\nhu : StrictMono u ∧ (∀ (n : ℕ), u n < x) ∧ Tendsto u atTop (𝓝 x) ∧ ∀ (n : ℕ), u n ∈ t\n⊢ ∃ u, Monotone u ∧ (∀ (n : ℕ), u n ≤ x) ∧ Tendsto u atTop (𝓝 x) ∧ ∀ (n : ℕ), u n ∈ t", "true_first_token": "exact", "true_tactic": "exact ⟨u, hu.1.monotone, fun n => (hu.2.1 n).le, hu.2.2⟩"}
|
| 14 |
+
{"dev_row": 934, "state": "case neg\nR : Type u_1\ninst : AddMonoidWithOne R\nn k : ℕ\nhk : ∀ (m : ℕ), m < succ k → Nat.binCast m = ↑m\nh : ¬(k + 1) % 2 = 0\n⊢ ↑(succ k / 2 + succ k / 2) + 1 = ↑(succ k % 2 + 2 * (succ k / 2))", "true_first_token": "have", "true_tactic": "have h1 := Or.resolve_left (Nat.mod_two_eq_zero_or_one (succ k)) h"}
|
| 15 |
+
{"dev_row": 1010, "state": "case neg\nα : Type u_1\nE : Type u_2\nF : Type ?u\nG : Type ?u\nm m0 : MeasurableSpace α\np : ℝ≥0∞\nq : ℝ\nμ ν : Measure α\ninst : NormedAddCommGroup E\ninst : NormedAddCommGroup F\ninst : NormedAddCommGroup G\np q : ℝ≥0∞\nhpq : p ≤ q\nf : α → E\nhf : AEStronglyMeasurable f μ\nhp0 : p ≠ 0\n⊢ snorm f p μ ≤ snorm f q μ * ↑↑μ Set.univ ^ (1 / ENNReal.toReal p - 1 / ENNReal.toReal q)", "true_first_token": "have", "true_tactic": "have hp0_lt : 0 < p := lt_of_le_of_ne (zero_le _) hp0.symm"}
|
| 16 |
+
{"dev_row": 1080, "state": "case inl\nn : Type u_2\ninst : Fintype n\ninst : DecidableEq n\nR : Type u_1\ninst : Field R\nA : Matrix n n R\nhAps : Splits (RingHom.id R) (charpoly A)\nh : IsEmpty n\n⊢ trace A = Multiset.sum (roots (charpoly A))", "true_first_token": "rw", "true_tactic": "rw [Matrix.trace, Fintype.sum_empty, Matrix.charpoly,\n det_eq_one_of_card_eq_zero (Fintype.card_eq_zero_iff.2 h), Polynomial.roots_one,\n Multiset.empty_eq_zero, Multiset.sum_zero]"}
|
first_token_whitelist.json
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"id": 782,
|
| 4 |
+
"token": "rw",
|
| 5 |
+
"share": 0.1764
|
| 6 |
+
},
|
| 7 |
+
{
|
| 8 |
+
"id": 559,
|
| 9 |
+
"token": "simp",
|
| 10 |
+
"share": 0.11225
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"id": 1180,
|
| 14 |
+
"token": "exact",
|
| 15 |
+
"share": 0.08363
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"id": 1704,
|
| 19 |
+
"token": "have",
|
| 20 |
+
"share": 0.07999
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"id": 1240,
|
| 24 |
+
"token": "refine",
|
| 25 |
+
"share": 0.06312
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"id": 1924,
|
| 29 |
+
"token": "·",
|
| 30 |
+
"share": 0.05229
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"id": 686,
|
| 34 |
+
"token": "apply",
|
| 35 |
+
"share": 0.04007
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"id": 3231,
|
| 39 |
+
"token": "obtain",
|
| 40 |
+
"share": 0.03498
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"id": 450,
|
| 44 |
+
"token": "intro",
|
| 45 |
+
"share": 0.03398
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"id": 829,
|
| 49 |
+
"token": "ext",
|
| 50 |
+
"share": 0.02623
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"id": 3249,
|
| 54 |
+
"token": "rcases",
|
| 55 |
+
"share": 0.02152
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"id": 2970,
|
| 59 |
+
"token": "let",
|
| 60 |
+
"share": 0.01925
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"id": 3267,
|
| 64 |
+
"token": "rintro",
|
| 65 |
+
"share": 0.01759
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"id": 2965,
|
| 69 |
+
"token": "simpa",
|
| 70 |
+
"share": 0.01715
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"id": 965,
|
| 74 |
+
"token": "cases",
|
| 75 |
+
"share": 0.01655
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"id": 909,
|
| 79 |
+
"token": "by",
|
| 80 |
+
"share": 0.01543
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"id": 2048,
|
| 84 |
+
"token": "induction",
|
| 85 |
+
"share": 0.01204
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"id": 1088,
|
| 89 |
+
"token": "const",
|
| 90 |
+
"share": 0.01068
|
| 91 |
+
},
|
| 92 |
+
{
|
| 93 |
+
"id": 1438,
|
| 94 |
+
"token": "rfl",
|
| 95 |
+
"share": 0.01055
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"id": 580,
|
| 99 |
+
"token": "con",
|
| 100 |
+
"share": 0.00963
|
| 101 |
+
},
|
| 102 |
+
{
|
| 103 |
+
"id": 75,
|
| 104 |
+
"token": "d",
|
| 105 |
+
"share": 0.00959
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"id": 319,
|
| 109 |
+
"token": "su",
|
| 110 |
+
"share": 0.0072
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"id": 1121,
|
| 114 |
+
"token": "congr",
|
| 115 |
+
"share": 0.00628
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"id": 2156,
|
| 119 |
+
"token": "use",
|
| 120 |
+
"share": 0.0055
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"id": 15,
|
| 124 |
+
"token": "(",
|
| 125 |
+
"share": 0.00534
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"id": 296,
|
| 129 |
+
"token": "re",
|
| 130 |
+
"share": 0.00532
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"id": 306,
|
| 134 |
+
"token": "--",
|
| 135 |
+
"share": 0.00506
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"id": 74,
|
| 139 |
+
"token": "c",
|
| 140 |
+
"share": 0.00491
|
| 141 |
+
},
|
| 142 |
+
{
|
| 143 |
+
"id": 3037,
|
| 144 |
+
"token": "split",
|
| 145 |
+
"share": 0.00434
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"id": 982,
|
| 149 |
+
"token": "cal",
|
| 150 |
+
"share": 0.00417
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"id": 70,
|
| 154 |
+
"token": "_",
|
| 155 |
+
"share": 0.00409
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"id": 612,
|
| 159 |
+
"token": "ch",
|
| 160 |
+
"share": 0.00406
|
| 161 |
+
},
|
| 162 |
+
{
|
| 163 |
+
"id": 583,
|
| 164 |
+
"token": "sub",
|
| 165 |
+
"share": 0.00363
|
| 166 |
+
},
|
| 167 |
+
{
|
| 168 |
+
"id": 1331,
|
| 169 |
+
"token": "cont",
|
| 170 |
+
"share": 0.00355
|
| 171 |
+
},
|
| 172 |
+
{
|
| 173 |
+
"id": 1489,
|
| 174 |
+
"token": "filter",
|
| 175 |
+
"share": 0.00337
|
| 176 |
+
},
|
| 177 |
+
{
|
| 178 |
+
"id": 3701,
|
| 179 |
+
"token": "conv",
|
| 180 |
+
"share": 0.00334
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"id": 860,
|
| 184 |
+
"token": "norm",
|
| 185 |
+
"share": 0.00324
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"id": 301,
|
| 189 |
+
"token": "un",
|
| 190 |
+
"share": 0.00312
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"id": 416,
|
| 194 |
+
"token": "set",
|
| 195 |
+
"share": 0.00296
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"id": 276,
|
| 199 |
+
"token": "er",
|
| 200 |
+
"share": 0.00222
|
| 201 |
+
},
|
| 202 |
+
{
|
| 203 |
+
"id": 1880,
|
| 204 |
+
"token": "ring",
|
| 205 |
+
"share": 0.00216
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"id": 78,
|
| 209 |
+
"token": "g",
|
| 210 |
+
"share": 0.00213
|
| 211 |
+
},
|
| 212 |
+
{
|
| 213 |
+
"id": 1606,
|
| 214 |
+
"token": "choose",
|
| 215 |
+
"share": 0.00189
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"id": 443,
|
| 219 |
+
"token": "case",
|
| 220 |
+
"share": 0.00182
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"id": 449,
|
| 224 |
+
"token": "fun",
|
| 225 |
+
"share": 0.0018
|
| 226 |
+
},
|
| 227 |
+
{
|
| 228 |
+
"id": 2323,
|
| 229 |
+
"token": "lin",
|
| 230 |
+
"share": 0.00174
|
| 231 |
+
},
|
| 232 |
+
{
|
| 233 |
+
"id": 1141,
|
| 234 |
+
"token": "lift",
|
| 235 |
+
"share": 0.00163
|
| 236 |
+
},
|
| 237 |
+
{
|
| 238 |
+
"id": 87,
|
| 239 |
+
"token": "p",
|
| 240 |
+
"share": 0.00145
|
| 241 |
+
},
|
| 242 |
+
{
|
| 243 |
+
"id": 85,
|
| 244 |
+
"token": "n",
|
| 245 |
+
"share": 0.00132
|
| 246 |
+
},
|
| 247 |
+
{
|
| 248 |
+
"id": 21,
|
| 249 |
+
"token": ".",
|
| 250 |
+
"share": 0.00131
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"id": 2472,
|
| 254 |
+
"token": "spec",
|
| 255 |
+
"share": 0.00127
|
| 256 |
+
},
|
| 257 |
+
{
|
| 258 |
+
"id": 1173,
|
| 259 |
+
"token": "cle",
|
| 260 |
+
"share": 0.00115
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"id": 2816,
|
| 264 |
+
"token": "tfae",
|
| 265 |
+
"share": 0.00111
|
| 266 |
+
},
|
| 267 |
+
{
|
| 268 |
+
"id": 2607,
|
| 269 |
+
"token": "nth",
|
| 270 |
+
"share": 0.0011
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"id": 706,
|
| 274 |
+
"token": "←",
|
| 275 |
+
"share": 0.00107
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"id": 90,
|
| 279 |
+
"token": "s",
|
| 280 |
+
"share": 0.00104
|
| 281 |
+
},
|
| 282 |
+
{
|
| 283 |
+
"id": 2044,
|
| 284 |
+
"token": "swap",
|
| 285 |
+
"share": 0.001
|
| 286 |
+
},
|
| 287 |
+
{
|
| 288 |
+
"id": 376,
|
| 289 |
+
"token": "as",
|
| 290 |
+
"share": 0.00095
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"id": 662,
|
| 294 |
+
"token": "symm",
|
| 295 |
+
"share": 0.00092
|
| 296 |
+
},
|
| 297 |
+
{
|
| 298 |
+
"id": 3412,
|
| 299 |
+
"token": "show",
|
| 300 |
+
"share": 0.00092
|
| 301 |
+
},
|
| 302 |
+
{
|
| 303 |
+
"id": 2101,
|
| 304 |
+
"token": "dec",
|
| 305 |
+
"share": 0.00087
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"id": 99,
|
| 309 |
+
"token": "|",
|
| 310 |
+
"share": 0.00083
|
| 311 |
+
},
|
| 312 |
+
{
|
| 313 |
+
"id": 1372,
|
| 314 |
+
"token": "inf",
|
| 315 |
+
"share": 0.00081
|
| 316 |
+
},
|
| 317 |
+
{
|
| 318 |
+
"id": 2980,
|
| 319 |
+
"token": "field",
|
| 320 |
+
"share": 0.00079
|
| 321 |
+
}
|
| 322 |
+
]
|
metrics.json
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"system": "DuoTactic - a Lean 4 proof-search harness (the main artifact of this package: code/eval_minif2f.py + code/leanoar/{search,lean_batch}.py). The 39.58M model documented below is an optional candidate generator for it and is measured to contribute 0 of the 165 accepted proofs; see the where_the_score_comes_from block.",
|
| 3 |
+
"_warning": "dev numbers from the training log use a length-bucketed reshuffle and are NOT comparable across datasets; only the protocol below is.",
|
| 4 |
+
"dev_protocol": "first 1024 rows with dev_ids[k,1]==<|FORWARD|>, logits at position (mask_start-1) predicting the token at mask_start; ctx=768",
|
| 5 |
+
"dev": {
|
| 6 |
+
"stage4_e6_full_best": {
|
| 7 |
+
"top1": 0.3076,
|
| 8 |
+
"top5": 0.709,
|
| 9 |
+
"ce": 2.5494,
|
| 10 |
+
"step": 5000,
|
| 11 |
+
"train_data": "steps_v2 (317,492 samples)"
|
| 12 |
+
},
|
| 13 |
+
"stage3_e3_best": {
|
| 14 |
+
"top1": 0.2891,
|
| 15 |
+
"top5": 0.7246,
|
| 16 |
+
"ce": 2.4971,
|
| 17 |
+
"step": 2800,
|
| 18 |
+
"train_data": "steps (188,501 samples)"
|
| 19 |
+
},
|
| 20 |
+
"trivial_baseline": {
|
| 21 |
+
"top1": 0.2171,
|
| 22 |
+
"top5": 0.6079
|
| 23 |
+
}
|
| 24 |
+
},
|
| 25 |
+
"end_to_end_minif2f_valid_first89": {
|
| 26 |
+
"search_config": "--force --force-set v2 --force-depth 99 --lean-parallel 6 --k 16 --beam 4 --depth 6 --budget 45 --round-budget 30",
|
| 27 |
+
"stage3_e3_best": {
|
| 28 |
+
"passed": 30,
|
| 29 |
+
"of": 89,
|
| 30 |
+
"rate": 0.337,
|
| 31 |
+
"wall_min": 51.7
|
| 32 |
+
},
|
| 33 |
+
"stage4_e6_full_best": {
|
| 34 |
+
"passed": 30,
|
| 35 |
+
"of": 89,
|
| 36 |
+
"rate": 0.337,
|
| 37 |
+
"wall_min": 59.8,
|
| 38 |
+
"note": "same 30 problems at the same depths: zero new, zero lost"
|
| 39 |
+
},
|
| 40 |
+
"no_neural_proposals_244": {
|
| 41 |
+
"passed": 8,
|
| 42 |
+
"of": 244,
|
| 43 |
+
"rate": 0.033,
|
| 44 |
+
"note": "pure automation-tactic injection, no model output"
|
| 45 |
+
}
|
| 46 |
+
},
|
| 47 |
+
"end_to_end_minif2f_valid_full244": {
|
| 48 |
+
"search_config": "--force --force-set v2 --force-depth 99 --lean-parallel 6 --k 16 --beam 4 --depth 6 --budget 45 --round-budget 30",
|
| 49 |
+
"checkpoint": "checkpoints/stage3-e3 (lit_model.pth)",
|
| 50 |
+
"passed": 85,
|
| 51 |
+
"of": 244,
|
| 52 |
+
"rate": 0.348,
|
| 53 |
+
"wall_min": 137,
|
| 54 |
+
"wall_note": "sum of the recorded per-problem seconds: 63.2 min for problems 0-106 + 73.6 min for 107-243; solved problems average 13 s, unsolved ones 45 s (they run to the 45 s budget)",
|
| 55 |
+
"segments": {"0-89": 0.337, "89-178": 0.404, "178-244": 0.288},
|
| 56 |
+
"solution_depths": {"1": 78, "2": 2, "3": 2, "4": 3},
|
| 57 |
+
"all_confirmed": true,
|
| 58 |
+
"per_problem_seconds_median": 40,
|
| 59 |
+
"lean_calls_second_slice": 2125,
|
| 60 |
+
"lean_seconds_second_slice": 11975,
|
| 61 |
+
"note": "one run per problem, one attempt: pass@1 with the search harness (not pass@k). Problems 0-106 ran with a prompt budget of 741 tokens, 107-243 with 740 (off-by-one fix), which only affects states longer than 740 tokens."
|
| 62 |
+
},
|
| 63 |
+
"end_to_end_minif2f_test_full244": {
|
| 64 |
+
"search_config": "--force --force-set v2 --force-depth 99 --lean-parallel 10 --verify-chunk 64 --narrow-imports --k 16 --beam 4 --depth 6 --budget 45 --round-budget 30",
|
| 65 |
+
"checkpoint": "checkpoints/stage3-e3 (lit_model.pth)",
|
| 66 |
+
"passed": 80,
|
| 67 |
+
"of": 244,
|
| 68 |
+
"rate": 0.328,
|
| 69 |
+
"wall_min": 148.5,
|
| 70 |
+
"segments": {"0-89": 0.382, "89-178": 0.270, "178-244": 0.333},
|
| 71 |
+
"solution_depths": {"1": 76, "2": 2, "4": 2},
|
| 72 |
+
"all_confirmed": true,
|
| 73 |
+
"lean_calls": 2921,
|
| 74 |
+
"lean_seconds": 9078,
|
| 75 |
+
"note": "same search as the valid run; the run was additionally accelerated (narrow imports, chunk 64, 10 parallel), which changes the per-call cost (5.6 s/call -> 3.1 s/call), not the verdicts."
|
| 76 |
+
},
|
| 77 |
+
"where_the_score_comes_from": {
|
| 78 |
+
"final_tactic_of_each_accepted_proof": {
|
| 79 |
+
"valid_85_solved": {"injected_table_single_step": 78, "injected_table_multi_step_chain": 7, "model_proposed": 0},
|
| 80 |
+
"test_80_solved": {"injected_table_single_step": 76, "injected_table_multi_step_chain": 4, "model_proposed": 0}
|
| 81 |
+
},
|
| 82 |
+
"injected_table": ["rfl", "ring", "omega", "simp_all", "linarith", "trivial", "nlinarith", "aesop", "simp", "norm_num", "simp_all ; ring", "simp_all ; norm_num", "simp_all ; omega", "simp_all ; linarith", "gcongr", "field_simp ; ring"],
|
| 83 |
+
"ablation_no_model": {
|
| 84 |
+
"slice": "valid problems 0-9",
|
| 85 |
+
"with_model": {"passed": 2, "of": 10, "wall_min": 6.5},
|
| 86 |
+
"without_model": {"passed": 2, "of": 10, "wall_min": 3.3},
|
| 87 |
+
"note": "same problems, same depths, same proof terms; the model's candidates doubled the wall clock and added no solution"
|
| 88 |
+
},
|
| 89 |
+
"note": "The published rates are SYSTEM numbers: injected public automation tactics (16 entries, injected at every frontier node and every depth, so their compositions are searched too) + depth-batched best-first search + kernel-checked verification. The shipped 40M model supplies 0 of the accepted proofs. Do not cite these as neural tactic-generation results, and do not compare them with model-only pass@1 figures that have no automation table (this harness would score 0 without one)."
|
| 90 |
+
},
|
| 91 |
+
"candidate_level_stage3_e3": {
|
| 92 |
+
"valid_rate": 0.079,
|
| 93 |
+
"ge1_valid_per_problem": 0.566,
|
| 94 |
+
"ge1_direct_closer_per_problem": 0.063
|
| 95 |
+
},
|
| 96 |
+
"whole_tactic_greedy_exact_match": {
|
| 97 |
+
"matched": 0,
|
| 98 |
+
"of": 200,
|
| 99 |
+
"note": "a full tactic must be copied token-exactly"
|
| 100 |
+
}
|
| 101 |
+
}
|
special_tokens_v1.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<|pad|>": 0,
|
| 3 |
+
"<|bos|>": 1,
|
| 4 |
+
"<|eos|>": 2,
|
| 5 |
+
"<|endoftext|>": 3,
|
| 6 |
+
"<|FORWARD|>": 4,
|
| 7 |
+
"<|REPAIR|>": 5,
|
| 8 |
+
"<|state|>": 6,
|
| 9 |
+
"<|tactic|>": 7
|
| 10 |
+
}
|
tokenizer_v1.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|