robbiethompson2018 commited on
Commit
c28bd4e
·
verified ·
1 Parent(s): 0e98d45

Add 510-fable_more-shorter-rounds_s2

Browse files
runs/510-fable_more-shorter-rounds_s2/eval_dev.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/510-fable_more-shorter-rounds_s2/final.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa7ed279295cbf07bb96040f2bd6fab10c3bafb96c8cce46fe4a3ce779d2801c
3
+ size 25624925
runs/510-fable_more-shorter-rounds_s2/found_targets.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/510-fable_more-shorter-rounds_s2/lean_gate.disagree.jsonl ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"prompt": "THM Q , ( ( Q v ( P & S ) ) v Q ) SEQ ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) PRF", "lean_text": "have n29 : Q := h1 ; have n30 : ( ( Q ∨ ( P ∧ S ) ) ∨ Q ) := Or.inr n29 ; have n31 : ( ( ( Q → S ) ∧ ( R ∧ R ) ) → ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) ) := ( fun ( n32 : ( ( Q → S ) ∧ ( R ∧ R ) ) ) => by have n33 : ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) := ( fun ( n34 : ( ¬ ( ¬ S ) ) ) => by have n35 : ( Q ∧ ( ¬ ( ¬ S ) ) ) := ⟨ n29 , n34 ⟩ ; exact n35 ) ; exact n33 ) ; exact n31", "nd": "N1 Q : PR ; N2 ( ( Q v ( P & S ) ) v Q ) : ORI2 N1 ; N3 | ( ( Q > S ) & ( R & R ) ) : AS ; N4 | | ( ~ ( ~ S ) ) : AS ; N5 | | ( Q & ( ~ ( ~ S ) ) ) : ANDI N1 N4 ; N6 | ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) : IMPI N4 N5 ; N7 ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) : IMPI N3 N6 ; QED", "nd_ok": false, "lean_ok": true}
2
+ {"prompt": "THM Q , ( ( Q v ( P & S ) ) v Q ) SEQ ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) PRF", "lean_text": "have n34 : Q := h1 ; have n35 : ( ( Q ∨ ( P ∧ S ) ) ∨ Q ) := Or.inr n34 ; have n36 : ( ( ( Q → S ) ∧ ( R ∧ R ) ) → ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) ) := ( fun ( n37 : ( ( Q → S ) ∧ ( R ∧ R ) ) ) => by have n38 : ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) := ( fun ( n39 : ( ¬ ( ¬ S ) ) ) => by have n40 : ( Q ∧ ( ¬ ( ¬ S ) ) ) := ⟨ n34 , n39 ⟩ ; exact n40 ) ; exact n38 ) ; exact n36", "nd": "N1 Q : PR ; N2 ( ( Q v ( P & S ) ) v Q ) : ORI2 N1 ; N3 | ( ( Q > S ) & ( R & R ) ) : AS ; N4 | | ( ~ ( ~ S ) ) : AS ; N5 | | ( Q & ( ~ ( ~ S ) ) ) : ANDI N1 N4 ; N6 | ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) : IMPI N4 N5 ; N7 ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) : IMPI N3 N6 ; QED", "nd_ok": false, "lean_ok": true}
3
+ {"prompt": "THM Q , ( ( Q v ( P & S ) ) v Q ) SEQ ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) PRF", "lean_text": "have n26 : Q := h1 ; have n27 : ( ( Q ∨ ( P ∧ S ) ) ∨ Q ) := Or.inr n26 ; have n28 : ( ( ( Q → S ) ∧ ( R ∧ R ) ) → ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) ) := ( fun ( n29 : ( ( Q → S ) ∧ ( R ∧ R ) ) ) => by have n30 : ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) := ( fun ( n31 : ( ¬ ( ¬ S ) ) ) => by have n32 : ( Q ∧ ( ¬ ( ¬ S ) ) ) := ⟨ n26 , n31 ⟩ ; exact n32 ) ; exact n30 ) ; exact n28", "nd": "N1 Q : PR ; N2 ( ( Q v ( P & S ) ) v Q ) : ORI2 N1 ; N3 | ( ( Q > S ) & ( R & R ) ) : AS ; N4 | | ( ~ ( ~ S ) ) : AS ; N5 | | ( Q & ( ~ ( ~ S ) ) ) : ANDI N1 N4 ; N6 | ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) : IMPI N4 N5 ; N7 ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) : IMPI N3 N6 ; QED", "nd_ok": false, "lean_ok": true}
4
+ {"prompt": "THM Q , ( ( Q v ( P & S ) ) v Q ) SEQ ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) PRF", "lean_text": "have n37 : Q := h1 ; have n38 : ( ( Q ∨ ( P ∧ S ) ) ∨ Q ) := Or.inr n37 ; have n39 : ( ( ( Q → S ) ∧ ( R ∧ R ) ) → ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) ) := ( fun ( n40 : ( ( Q → S ) ∧ ( R ∧ R ) ) ) => by have n41 : ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) := ( fun ( n42 : ( ¬ ( ¬ S ) ) ) => by have n43 : ( Q ∧ ( ¬ ( ¬ S ) ) ) := ⟨ n37 , n42 ⟩ ; exact n43 ) ; exact n41 ) ; exact n39", "nd": "N1 Q : PR ; N2 ( ( Q v ( P & S ) ) v Q ) : ORI2 N1 ; N3 | ( ( Q > S ) & ( R & R ) ) : AS ; N4 | | ( ~ ( ~ S ) ) : AS ; N5 | | ( Q & ( ~ ( ~ S ) ) ) : ANDI N1 N4 ; N6 | ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) : IMPI N4 N5 ; N7 ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) : IMPI N3 N6 ; QED", "nd_ok": false, "lean_ok": true}
5
+ {"prompt": "THM Q , ( ( Q v ( P & S ) ) v Q ) SEQ ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) PRF", "lean_text": "have n6 : Q := h1 ; have n7 : ( ( Q ∨ ( P ∧ S ) ) ∨ Q ) := Or.inr n6 ; have n8 : ( ( ( Q → S ) ∧ ( R ∧ R ) ) → ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) ) := ( fun ( n9 : ( ( Q → S ) ∧ ( R ∧ R ) ) ) => by have n10 : ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) := ( fun ( n11 : ( ¬ ( ¬ S ) ) ) => by have n12 : ( Q ∧ ( ¬ ( ¬ S ) ) ) := ⟨ n6 , n11 ⟩ ; exact n12 ) ; exact n10 ) ; exact n8", "nd": "N1 Q : PR ; N2 ( ( Q v ( P & S ) ) v Q ) : ORI2 N1 ; N3 | ( ( Q > S ) & ( R & R ) ) : AS ; N4 | | ( ~ ( ~ S ) ) : AS ; N5 | | ( Q & ( ~ ( ~ S ) ) ) : ANDI N1 N4 ; N6 | ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) : IMPI N4 N5 ; N7 ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) : IMPI N3 N6 ; QED", "nd_ok": false, "lean_ok": true}
6
+ {"prompt": "THM Q , ( ( Q v ( P & S ) ) v Q ) SEQ ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) PRF", "lean_text": "have n51 : Q := h1 ; have n52 : ( ( Q ∨ ( P ∧ S ) ) ∨ Q ) := Or.inr n51 ; have n53 : ( ( ( Q → S ) ∧ ( R ∧ R ) ) → ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) ) := ( fun ( n54 : ( ( Q → S ) ∧ ( R ∧ R ) ) ) => by have n55 : ( ( ¬ ( ¬ S ) ) → ( Q ∧ ( ¬ ( ¬ S ) ) ) ) := ( fun ( n56 : ( ¬ ( ¬ S ) ) ) => by have n57 : ( Q ∧ ( ¬ ( ¬ S ) ) ) := ⟨ n51 , n56 ⟩ ; exact n57 ) ; exact n55 ) ; exact n53", "nd": "N1 Q : PR ; N2 ( ( Q v ( P & S ) ) v Q ) : ORI2 N1 ; N3 | ( ( Q > S ) & ( R & R ) ) : AS ; N4 | | ( ~ ( ~ S ) ) : AS ; N5 | | ( Q & ( ~ ( ~ S ) ) ) : ANDI N1 N4 ; N6 | ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) : IMPI N4 N5 ; N7 ( ( ( Q > S ) & ( R & R ) ) > ( ( ~ ( ~ S ) ) > ( Q & ( ~ ( ~ S ) ) ) ) ) : IMPI N3 N6 ; QED", "nd_ok": false, "lean_ok": true}
7
+ {"prompt": "THM ( R v ( R v R ) ) , ( ( S > Q ) > S ) , ( S > Q ) SEQ ( S > ( ( ~ S ) > ( ~ Q ) ) ) PRF", "lean_text": "have n49 : ( R ∨ ( R ∨ R ) ) := h1 ; have n50 : ( ( S → Q ) → S ) := h2 ; have n51 : ( S → ( ( ¬ S ) → ( ¬ Q ) ) ) := ( fun ( n52 : S ) => by have n53 : ( ( ¬ S ) → ( ¬ Q ) ) := ( fun ( n54 : ( ¬ S ) ) => by have n55 : False := n54 n52 ; have n56 : ( ¬ Q ) := False.elim n55 ; exact n56 ) ; exact n53 ) ; exact n51", "nd": "N1 ( R v ( R v R ) ) : PR ; N2 ( ( S > Q ) > S ) : PR ; N3 | S : AS ; N4 | | ( ~ S ) : AS ; N5 | | F : NEGE N3 N4 ; N6 | | ( ~ Q ) : BOTE N5 ; N7 | ( ( ~ S ) > ( ~ Q ) ) : IMPI N4 N6 ; N8 ( S > ( ( ~ S ) > ( ~ Q ) ) ) : IMPI N3 N7 ; QED", "nd_ok": false, "lean_ok": true}
8
+ {"prompt": "THM ( R v ( R v R ) ) , ( ( S > Q ) > S ) , ( S > Q ) SEQ ( S > ( ( ~ S ) > ( ~ Q ) ) ) PRF", "lean_text": "have n50 : ( R ∨ ( R ∨ R ) ) := h1 ; have n51 : ( ( S → Q ) → S ) := h2 ; have n52 : ( S → ( ( ¬ S ) → ( ¬ Q ) ) ) := ( fun ( n53 : S ) => by have n54 : S := n53 ; have n55 : ( ( ¬ S ) → ( ¬ Q ) ) := ( fun ( n56 : ( ¬ S ) ) => by have n57 : ( ¬ Q ) := ( fun ( n58 : Q ) => by have n59 : False := n56 n54 ; exact ( n59 : False ) ) ; exact n57 ) ; exact n55 ) ; exact n52", "nd": "N1 ( R v ( R v R ) ) : PR ; N2 ( ( S > Q ) > S ) : PR ; N3 | S : AS ; N4 | S : R N3 ; N5 | | ( ~ S ) : AS ; N6 | | | Q : AS ; N7 | | | F : NEGE N4 N5 ; N8 | | ( ~ Q ) : NEGI N6 N7 ; N9 | ( ( ~ S ) > ( ~ Q ) ) : IMPI N5 N8 ; N10 ( S > ( ( ~ S ) > ( ~ Q ) ) ) : IMPI N3 N9 ; QED", "nd_ok": false, "lean_ok": true}
9
+ {"prompt": "THM ( R v ( R v R ) ) , ( ( S > Q ) > S ) , ( S > Q ) SEQ ( S > ( ( ~ S ) > ( ~ Q ) ) ) PRF", "lean_text": "have n53 : ( R ∨ ( R ∨ R ) ) := h1 ; have n54 : ( ( S → Q ) → S ) := h2 ; have n55 : ( S → ( ( ¬ S ) → ( ¬ Q ) ) ) := ( fun ( n56 : S ) => by have n57 : ( ( ¬ S ) → ( ¬ Q ) ) := ( fun ( n58 : ( ¬ S ) ) => by have n59 : ( ¬ Q ) := ( fun ( n60 : Q ) => by have n61 : False := n58 n56 ; exact ( n61 : False ) ) ; exact n59 ) ; exact n57 ) ; exact n55", "nd": "N1 ( R v ( R v R ) ) : PR ; N2 ( ( S > Q ) > S ) : PR ; N3 | S : AS ; N4 | | ( ~ S ) : AS ; N5 | | | Q : AS ; N6 | | | F : NEGE N3 N4 ; N7 | | ( ~ Q ) : NEGI N5 N6 ; N8 | ( ( ~ S ) > ( ~ Q ) ) : IMPI N4 N7 ; N9 ( S > ( ( ~ S ) > ( ~ Q ) ) ) : IMPI N3 N8 ; QED", "nd_ok": false, "lean_ok": true}
10
+ {"prompt": "THM ( R v ( R v R ) ) , ( ( S > Q ) > S ) , ( S > Q ) SEQ ( S > ( ( ~ S ) > ( ~ Q ) ) ) PRF", "lean_text": "have n50 : ( R ∨ ( R ∨ R ) ) := h1 ; have n51 : ( ( S → Q ) → S ) := h2 ; have n52 : ( S → ( ( ¬ S ) → ( ¬ Q ) ) ) := ( fun ( n53 : S ) => by have n54 : ( ( ¬ S ) → ( ¬ Q ) ) := ( fun ( n55 : ( ¬ S ) ) => by have n56 : ( ¬ Q ) := ( fun ( n57 : Q ) => by have n58 : False := n55 n53 ; exact ( n58 : False ) ) ; exact n56 ) ; exact n54 ) ; exact n52", "nd": "N1 ( R v ( R v R ) ) : PR ; N2 ( ( S > Q ) > S ) : PR ; N3 | S : AS ; N4 | | ( ~ S ) : AS ; N5 | | | Q : AS ; N6 | | | F : NEGE N3 N4 ; N7 | | ( ~ Q ) : NEGI N5 N6 ; N8 | ( ( ~ S ) > ( ~ Q ) ) : IMPI N4 N7 ; N9 ( S > ( ( ~ S ) > ( ~ Q ) ) ) : IMPI N3 N8 ; QED", "nd_ok": false, "lean_ok": true}
11
+ {"prompt": "THM ( ~ ( ( ~ Q ) > R ) ) SEQ ( ~ ( R v R ) ) PRF", "lean_text": "have n16 : ( ¬ ( ( ¬ Q ) → R ) ) := h1 ; have n17 : ( ¬ ( R ∨ R ) ) := ( fun ( n18 : ( R ∨ R ) ) => by have n19 : ( ( ¬ Q ) → R ) := ( fun ( n20 : ( ¬ Q ) ) => by have n21 : R := Or.elim n18 ( fun ( n22 : R ) => by exact n22 ) ( fun ( n23 : R ) => by exact n23 ) ; exact n21 ) ; have n24 : False := n16 n19 ; exact n24 ) ; exact n17", "nd": "N1 ( ~ ( ( ~ Q ) > R ) ) : PR ; N2 | ( R v R ) : AS ; N3 | | ( ~ Q ) : AS ; N4 | | | R : AS ; N5 | | | R : AS ; N6 | | R : ORE N2 N4 N4 N5 N5 ; N7 | ( ( ~ Q ) > R ) : IMPI N3 N6 ; N8 | F : NEGE N7 N1 ; N9 ( ~ ( R v R ) ) : IMPI N2 N8 ; QED", "nd_ok": false, "lean_ok": true}
12
+ {"prompt": "THM P , ( ~ S ) SEQ ( ~ ( S v ( ~ P ) ) ) PRF", "lean_text": "have n13 : P := h1 ; have n14 : ( ¬ S ) := h2 ; have n15 : ( ¬ ( S ∨ ( ¬ P ) ) ) := ( fun ( n16 : ( S ∨ ( ¬ P ) ) ) => by have n17 : ( S ∨ ( ¬ P ) ) := n16 ; have n18 : S := Or.elim n17 ( fun ( n19 : S ) => by exact n19 ) ( fun ( n20 : ( ¬ P ) ) => by have n21 : False := n20 n13 ; have n22 : S := False.elim n21 ; exact n22 ) ; have n23 : False := n14 n18 ; exact n23 ) ; exact n15", "nd": "N1 P : PR ; N2 ( ~ S ) : PR ; N3 | ( S v ( ~ P ) ) : AS ; N4 | ( S v ( ~ P ) ) : R N3 ; N5 | | S : AS ; N6 | | ( ~ P ) : AS ; N7 | | F : NEGE N1 N6 ; N8 | | S : BOTE N7 ; N9 | S : ORE N4 N5 N5 N6 N8 ; N10 | F : NEGE N9 N2 ; N11 ( ~ ( S v ( ~ P ) ) ) : IMPI N3 N10 ; QED", "nd_ok": false, "lean_ok": true}
13
+ {"prompt": "THM P , ( ( ~ P ) v ( ~ R ) ) SEQ ( ~ ( ~ ( ~ R ) ) ) PRF", "lean_text": "have n3 : P := h1 ; have n4 : ( ( ¬ P ) ∨ ( ¬ R ) ) := h2 ; have n5 : ( ¬ ( ¬ ( ¬ R ) ) ) := ( fun ( n6 : ( ¬ ( ¬ R ) ) ) => by have n7 : False := Or.elim n4 ( fun ( n8 : ( ¬ P ) ) => by have n9 : ( ¬ P ) := n8 ; have n10 : False := n9 n3 ; exact n10 ) ( fun ( n11 : ( ¬ R ) ) => by have n12 : False := n6 n11 ; exact n12 ) ; exact n7 ) ; exact n5", "nd": "N1 P : PR ; N2 ( ( ~ P ) v ( ~ R ) ) : PR ; N3 | ( ~ ( ~ R ) ) : AS ; N4 | | ( ~ P ) : AS ; N5 | | ( ~ P ) : R N4 ; N6 | | F : NEGE N1 N5 ; N7 | | ( ~ R ) : AS ; N8 | | F : NEGE N7 N3 ; N9 | F : ORE N2 N4 N6 N7 N8 ; N10 ( ~ ( ~ ( ~ R ) ) ) : IMPI N3 N9 ; QED", "nd_ok": false, "lean_ok": true}
14
+ {"prompt": "THM P , ( ( ~ P ) v ( ~ R ) ) SEQ ( ~ ( ~ ( ~ R ) ) ) PRF", "lean_text": "have n51 : P := h1 ; have n52 : ( ( ¬ P ) ∨ ( ¬ R ) ) := h2 ; have n53 : ( ¬ ( ¬ ( ¬ R ) ) ) := ( fun ( n54 : ( ¬ ( ¬ R ) ) ) => by have n55 : False := Or.elim n52 ( fun ( n56 : ( ¬ P ) ) => by have n57 : False := n56 n51 ; exact n57 ) ( fun ( n58 : ( ¬ R ) ) => by have n59 : False := n54 n58 ; exact n59 ) ; exact n55 ) ; exact n53", "nd": "N1 P : PR ; N2 ( ( ~ P ) v ( ~ R ) ) : PR ; N3 | ( ~ ( ~ R ) ) : AS ; N4 | | ( ~ P ) : AS ; N5 | | F : NEGE N1 N4 ; N6 | | ( ~ R ) : AS ; N7 | | F : NEGE N6 N3 ; N8 | F : ORE N2 N4 N5 N6 N7 ; N9 ( ~ ( ~ ( ~ R ) ) ) : IMPI N3 N8 ; QED", "nd_ok": false, "lean_ok": true}
15
+ {"prompt": "THM ( ( ~ ( S & P ) ) v Q ) , ( ~ Q ) SEQ ( ( S & P ) > ( ~ ( S & P ) ) ) PRF", "lean_text": "have n28 : ( ( ¬ ( S ∧ P ) ) ∨ Q ) := h1 ; have n29 : ( ¬ Q ) := h2 ; have n30 : ( ( S ∧ P ) → ( ¬ ( S ∧ P ) ) ) := ( fun ( n31 : ( S ∧ P ) ) => by have n32 : ( ¬ ( S ∧ P ) ) := Or.elim n28 ( fun ( n33 : ( ¬ ( S ∧ P ) ) ) => by exact n33 ) ( fun ( n34 : Q ) => by have n35 : ( ( S ∧ P ) → False ) := ( fun ( n36 : ( S ∧ P ) ) => by have n37 : False := n29 n34 ; exact n37 ) ; exact n35 ) ; exact n32 ) ; exact n30", "nd": "N1 ( ( ~ ( S & P ) ) v Q ) : PR ; N2 ( ~ Q ) : PR ; N3 | ( S & P ) : AS ; N4 | | ( ~ ( S & P ) ) : AS ; N5 | | Q : AS ; N6 | | | ( S & P ) : AS ; N7 | | | F : NEGE N5 N2 ; N8 | | ( ( S & P ) > F ) : IMPI N6 N7 ; N9 | ( ~ ( S & P ) ) : ORE N1 N4 N4 N5 N8 ; N10 ( ( S & P ) > ( ~ ( S & P ) ) ) : IMPI N3 N9 ; QED", "nd_ok": false, "lean_ok": true}
16
+ {"prompt": "THM ( ( ~ P ) v ( ~ Q ) ) , Q , P SEQ ( ~ ( R > R ) ) PRF", "lean_text": "have n45 : ( ( ¬ P ) ∨ ( ¬ Q ) ) := h1 ; have n46 : Q := h2 ; have n47 : P := h3 ; have n48 : Q := n46 ; have n49 : Q := n46 ; have n50 : ( ¬ ( R → R ) ) := ( fun ( n51 : ( R → R ) ) => by have n52 : False := Or.elim n45 ( fun ( n53 : ( ¬ P ) ) => by have n54 : False := n53 n47 ; exact n54 ) ( fun ( n55 : ( ¬ Q ) ) => by have n56 : False := n55 n48 ; exact n56 ) ; exact n52 ) ; exact n50", "nd": "N1 ( ( ~ P ) v ( ~ Q ) ) : PR ; N2 Q : PR ; N3 P : PR ; N4 Q : R N2 ; N5 Q : R N2 ; N6 | ( R > R ) : AS ; N7 | | ( ~ P ) : AS ; N8 | | F : NEGE N3 N7 ; N9 | | ( ~ Q ) : AS ; N10 | | F : NEGE N4 N9 ; N11 | F : ORE N1 N7 N8 N9 N10 ; N12 ( ~ ( R > R ) ) : IMPI N6 N11 ; QED", "nd_ok": false, "lean_ok": true}
17
+ {"prompt": "THM ( ~ ( ( ~ Q ) > R ) ) SEQ ( ~ ( R v R ) ) PRF", "lean_text": "have n16 : ( ¬ ( ( ¬ Q ) → R ) ) := h1 ; have n17 : ( ¬ ( R ∨ R ) ) := ( fun ( n18 : ( R ∨ R ) ) => by have n19 : ( ( ¬ Q ) → R ) := ( fun ( n20 : ( ¬ Q ) ) => by have n21 : R := Or.elim n18 ( fun ( n22 : R ) => by exact n22 ) ( fun ( n23 : R ) => by exact n23 ) ; exact n21 ) ; have n24 : False := n16 n19 ; exact n24 ) ; exact n17", "nd": "N1 ( ~ ( ( ~ Q ) > R ) ) : PR ; N2 | ( R v R ) : AS ; N3 | | ( ~ Q ) : AS ; N4 | | | R : AS ; N5 | | | R : AS ; N6 | | R : ORE N2 N4 N4 N5 N5 ; N7 | ( ( ~ Q ) > R ) : IMPI N3 N6 ; N8 | F : NEGE N7 N1 ; N9 ( ~ ( R v R ) ) : IMPI N2 N8 ; QED", "nd_ok": false, "lean_ok": true}
18
+ {"prompt": "THM ( ( ~ ( Q > S ) ) v ( S v P ) ) , ( ~ ( ~ ( Q > S ) ) ) , ( ~ ( S v P ) ) SEQ ( ~ ( ~ ( ~ Q ) ) ) PRF", "lean_text": "have n12 : ( ( ¬ ( Q → S ) ) ∨ ( S ∨ P ) ) := h1 ; have n13 : ( ¬ ( ¬ ( Q → S ) ) ) := h2 ; have n14 : ( ¬ ( S ∨ P ) ) := h3 ; have n15 : ( ¬ ( ¬ ( ¬ Q ) ) ) := ( fun ( n16 : ( ¬ ( ¬ Q ) ) ) => by have n17 : ( S ∨ P ) := Or.elim n12 ( fun ( n18 : ( ¬ ( Q → S ) ) ) => by have n19 : False := n13 n18 ; have n20 : S := False.elim n19 ; have n21 : ( S ∨ P ) := Or.inl n20 ; exact n21 ) ( fun ( n22 : ( S ∨ P ) ) => by exact n22 ) ; have n23 : False := n14 n17 ; exact n23 ) ; exact n15", "nd": "N1 ( ( ~ ( Q > S ) ) v ( S v P ) ) : PR ; N2 ( ~ ( ~ ( Q > S ) ) ) : PR ; N3 ( ~ ( S v P ) ) : PR ; N4 | ( ~ ( ~ Q ) ) : AS ; N5 | | ( ~ ( Q > S ) ) : AS ; N6 | | F : NEGE N5 N2 ; N7 | | S : BOTE N6 ; N8 | | ( S v P ) : ORI1 N7 ; N9 | | ( S v P ) : AS ; N10 | ( S v P ) : ORE N1 N5 N8 N9 N9 ; N11 | F : NEGE N10 N3 ; N12 ( ~ ( ~ ( ~ Q ) ) ) : IMPI N4 N11 ; QED", "nd_ok": false, "lean_ok": true}
runs/510-fable_more-shorter-rounds_s2/lean_gate.jsonl ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"utc": "2026-10-07T23:26:55Z", "samples": 2000, "parse_fail": 12, "distinct_checked": 1988, "both_ok": 1929, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 59, "parse_reasons": {"unbound": 7, "expected ;": 1, "exact not last": 1, "expected exact": 1, "no-eos": 1, "name": 1}, "lean_wall_s": 3.948446273803711, "lean_proc_s": 18.37922978401184, "nd_verify_s": 0.11109113693237305, "workers": 96, "chunk": 400}
2
+ {"utc": "2026-10-07T23:27:52Z", "samples": 12000, "parse_fail": 5732, "distinct_checked": 6089, "both_ok": 703, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 6, "both_rej": 5380, "parse_reasons": {"unbound": 3909, "formula": 1057, "expected )": 206, "no-eos": 197, "exact not last": 106, "name": 80, "expected (": 47, "expected :=": 38, "PR position": 32, "expected ;": 31, "ORE box": 22, "expected False": 4, "expected fun": 2, "trailing tokens": 1}, "lean_wall_s": 22.06902265548706, "lean_proc_s": 333.3039882183075, "nd_verify_s": 0.31034159660339355, "workers": 96, "chunk": 400}
3
+ {"utc": "2026-10-07T23:28:49Z", "samples": 10536, "parse_fail": 5378, "distinct_checked": 4739, "both_ok": 142, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 4597, "parse_reasons": {"unbound": 4563, "formula": 366, "expected )": 98, "name": 74, "no-eos": 62, "exact not last": 61, "PR position": 53, "expected (": 27, "ORE box": 21, "expected :=": 20, "expected fun": 18, "expected ;": 10, "trailing tokens": 3, "expected hh": 1, "expected False": 1}, "lean_wall_s": 18.426530838012695, "lean_proc_s": 217.8754198551178, "nd_verify_s": 0.23628449440002441, "workers": 96, "chunk": 400}
4
+ {"utc": "2026-10-07T23:30:12Z", "samples": 10536, "parse_fail": 5899, "distinct_checked": 4268, "both_ok": 228, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 4040, "parse_reasons": {"unbound": 5064, "formula": 244, "name": 137, "expected )": 98, "expected :=": 84, "no-eos": 76, "expected (": 71, "exact not last": 60, "PR position": 34, "ORE box": 18, "expected .elim": 3, "expected hh": 3, "trailing tokens": 2, "expected ;": 2, "expected False": 1, "expected fun": 1, "expected exact": 1}, "lean_wall_s": 15.204795598983765, "lean_proc_s": 164.085027217865, "nd_verify_s": 0.19936060905456543, "workers": 96, "chunk": 400}
5
+ {"utc": "2026-10-07T23:32:14Z", "samples": 10536, "parse_fail": 6248, "distinct_checked": 3944, "both_ok": 357, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3587, "parse_reasons": {"unbound": 5425, "formula": 267, "expected )": 141, "name": 137, "PR position": 65, "exact not last": 57, "expected :=": 47, "no-eos": 41, "expected (": 31, "ORE box": 22, "expected False": 6, "trailing tokens": 4, "expected ;": 3, "expected None": 2}, "lean_wall_s": 16.115500450134277, "lean_proc_s": 158.12905097007751, "nd_verify_s": 0.20213985443115234, "workers": 96, "chunk": 400}
6
+ {"utc": "2026-10-07T23:33:37Z", "samples": 12000, "parse_fail": 3775, "distinct_checked": 7968, "both_ok": 3344, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 4624, "parse_reasons": {"expected )": 1721, "unbound": 1392, "formula": 348, "expected (": 122, "PR position": 50, "exact not last": 47, "name": 37, "no-eos": 23, "ORE box": 15, "expected :=": 11, "expected ;": 8, "expected exact": 1}, "lean_wall_s": 25.832230806350708, "lean_proc_s": 503.6751780509949, "nd_verify_s": 0.43486833572387695, "workers": 96, "chunk": 400}
7
+ {"utc": "2026-10-07T23:34:13Z", "samples": 6816, "parse_fail": 2514, "distinct_checked": 3924, "both_ok": 203, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3721, "parse_reasons": {"unbound": 1347, "expected )": 879, "formula": 130, "name": 45, "expected (": 37, "ORE box": 17, "exact not last": 16, "PR position": 15, "no-eos": 9, "expected ;": 6, "expected False": 5, "expected :=": 4, "expected None": 2, "trailing tokens": 2}, "lean_wall_s": 16.465848445892334, "lean_proc_s": 163.2021450996399, "nd_verify_s": 0.18684673309326172, "workers": 96, "chunk": 400}
8
+ {"utc": "2026-10-07T23:34:59Z", "samples": 6816, "parse_fail": 2813, "distinct_checked": 3696, "both_ok": 339, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3357, "parse_reasons": {"unbound": 1587, "expected )": 855, "formula": 170, "name": 80, "expected (": 61, "exact not last": 19, "ORE box": 12, "expected :=": 10, "PR position": 5, "no-eos": 5, "expected False": 4, "expected ;": 1, "expected by": 1, "expected fun": 1, "expected hh": 1, "expected None": 1}, "lean_wall_s": 15.287702083587646, "lean_proc_s": 142.16980743408203, "nd_verify_s": 0.19347071647644043, "workers": 96, "chunk": 400}
9
+ {"utc": "2026-10-07T23:36:19Z", "samples": 6792, "parse_fail": 2849, "distinct_checked": 3606, "both_ok": 396, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3210, "parse_reasons": {"unbound": 1744, "expected )": 790, "formula": 149, "name": 72, "expected (": 58, "expected :=": 10, "exact not last": 9, "no-eos": 6, "PR position": 5, "expected False": 3, "ORE box": 2, "expected hh": 1}, "lean_wall_s": 14.776204109191895, "lean_proc_s": 132.65187168121338, "nd_verify_s": 0.1841580867767334, "workers": 96, "chunk": 400}
10
+ {"utc": "2026-10-07T23:37:49Z", "samples": 12000, "parse_fail": 1993, "distinct_checked": 9662, "both_ok": 5285, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 4377, "parse_reasons": {"unbound": 1315, "formula": 249, "expected )": 134, "expected (": 105, "name": 50, "expected :=": 49, "exact not last": 37, "no-eos": 27, "ORE box": 18, "trailing tokens": 4, "expected fun": 2, "PR position": 2, "expected hh": 1}, "lean_wall_s": 28.8557231426239, "lean_proc_s": 691.6084740161896, "nd_verify_s": 0.5468356609344482, "workers": 96, "chunk": 400}
11
+ {"utc": "2026-10-07T23:38:18Z", "samples": 4752, "parse_fail": 1180, "distinct_checked": 3276, "both_ok": 293, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2983, "parse_reasons": {"unbound": 814, "formula": 114, "expected (": 65, "name": 47, "expected :=": 41, "ORE box": 41, "expected )": 35, "exact not last": 13, "no-eos": 4, "trailing tokens": 3, "PR position": 1, "expected =>": 1, "expected fun": 1}, "lean_wall_s": 14.039007186889648, "lean_proc_s": 113.98067045211792, "nd_verify_s": 0.17417025566101074, "workers": 96, "chunk": 400}
12
+ {"utc": "2026-10-07T23:38:55Z", "samples": 4752, "parse_fail": 1345, "distinct_checked": 3136, "both_ok": 367, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2769, "parse_reasons": {"unbound": 994, "expected (": 107, "formula": 72, "name": 66, "expected )": 50, "exact not last": 24, "expected :=": 17, "trailing tokens": 5, "ORE box": 3, "expected fun": 2, "no-eos": 2, "expected =>": 1, "expected ;": 1, "PR position": 1}, "lean_wall_s": 13.016057252883911, "lean_proc_s": 101.51363158226013, "nd_verify_s": 0.2018263339996338, "workers": 96, "chunk": 400}
13
+ {"utc": "2026-10-07T23:39:49Z", "samples": 4728, "parse_fail": 1176, "distinct_checked": 3261, "both_ok": 456, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2805, "parse_reasons": {"unbound": 881, "expected (": 87, "formula": 62, "name": 53, "expected )": 45, "expected :=": 21, "exact not last": 15, "expected =>": 4, "trailing tokens": 3, "no-eos": 2, "expected fun": 1, "expected None": 1, "ORE box": 1}, "lean_wall_s": 13.891843557357788, "lean_proc_s": 113.26257085800171, "nd_verify_s": 0.19654130935668945, "workers": 96, "chunk": 400}
14
+ {"utc": "2026-10-07T23:41:23Z", "samples": 12000, "parse_fail": 1423, "distinct_checked": 10219, "both_ok": 6114, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 4105, "parse_reasons": {"unbound": 882, "expected )": 141, "formula": 133, "expected (": 78, "name": 78, "no-eos": 27, "exact not last": 25, "expected :=": 25, "expected ;": 12, "ORE box": 9, "expected False": 7, "PR position": 5, "trailing tokens": 1}, "lean_wall_s": 30.037781476974487, "lean_proc_s": 758.3684666156769, "nd_verify_s": 0.6034116744995117, "workers": 96, "chunk": 400}
15
+ {"utc": "2026-10-07T23:41:47Z", "samples": 3936, "parse_fail": 1023, "distinct_checked": 2703, "both_ok": 189, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2514, "parse_reasons": {"unbound": 695, "expected (": 110, "name": 65, "expected )": 61, "formula": 47, "expected ;": 15, "expected :=": 15, "exact not last": 7, "no-eos": 3, "PR position": 2, "ORE box": 1, "expected None": 1, "expected False": 1}, "lean_wall_s": 11.548668622970581, "lean_proc_s": 79.33414006233215, "nd_verify_s": 0.15134739875793457, "workers": 96, "chunk": 400}
16
+ {"utc": "2026-10-07T23:42:21Z", "samples": 3936, "parse_fail": 1091, "distinct_checked": 2640, "both_ok": 350, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2290, "parse_reasons": {"unbound": 783, "name": 95, "expected (": 67, "expected )": 61, "formula": 33, "ORE box": 16, "no-eos": 15, "exact not last": 13, "expected ;": 3, "PR position": 3, "expected False": 1, "expected None": 1}, "lean_wall_s": 11.15588116645813, "lean_proc_s": 73.45760440826416, "nd_verify_s": 0.12489175796508789, "workers": 96, "chunk": 400}
17
+ {"utc": "2026-10-07T23:43:09Z", "samples": 3912, "parse_fail": 984, "distinct_checked": 2704, "both_ok": 336, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2368, "parse_reasons": {"unbound": 705, "expected (": 107, "name": 67, "expected )": 53, "formula": 36, "exact not last": 6, "ORE box": 4, "expected :=": 3, "no-eos": 2, "expected ;": 1}, "lean_wall_s": 11.52327013015747, "lean_proc_s": 79.08200335502625, "nd_verify_s": 0.13580679893493652, "workers": 96, "chunk": 400}
18
+ {"utc": "2026-10-07T23:44:41Z", "samples": 12000, "parse_fail": 1309, "distinct_checked": 10376, "both_ok": 6568, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3808, "parse_reasons": {"unbound": 876, "formula": 106, "expected (": 98, "expected )": 70, "name": 57, "exact not last": 32, "expected :=": 23, "ORE box": 21, "no-eos": 15, "expected ;": 5, "expected =>": 4, "trailing tokens": 2}, "lean_wall_s": 30.306573390960693, "lean_proc_s": 778.1396203041077, "nd_verify_s": 0.6033384799957275, "workers": 96, "chunk": 400}
19
+ {"utc": "2026-10-07T23:45:03Z", "samples": 3456, "parse_fail": 998, "distinct_checked": 2305, "both_ok": 314, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1991, "parse_reasons": {"unbound": 713, "expected (": 92, "expected )": 53, "name": 35, "expected :=": 26, "formula": 23, "ORE box": 21, "no-eos": 17, "exact not last": 11, "expected =>": 2, "expected None": 2, "expected ;": 1, "PR position": 1, "trailing tokens": 1}, "lean_wall_s": 9.749022483825684, "lean_proc_s": 57.38953971862793, "nd_verify_s": 0.12968182563781738, "workers": 96, "chunk": 400}
20
+ {"utc": "2026-10-07T23:45:32Z", "samples": 3432, "parse_fail": 1071, "distinct_checked": 2190, "both_ok": 312, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1878, "parse_reasons": {"unbound": 764, "expected (": 141, "expected )": 65, "name": 35, "formula": 35, "ORE box": 14, "exact not last": 13, "expected ;": 2, "expected =>": 1, "no-eos": 1}, "lean_wall_s": 9.391076564788818, "lean_proc_s": 52.419575929641724, "nd_verify_s": 0.1294710636138916, "workers": 96, "chunk": 400}
21
+ {"utc": "2026-10-07T23:46:19Z", "samples": 3432, "parse_fail": 1084, "distinct_checked": 2156, "both_ok": 332, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1824, "parse_reasons": {"unbound": 806, "expected (": 118, "name": 58, "expected )": 45, "formula": 26, "exact not last": 16, "no-eos": 6, "PR position": 3, "ORE box": 2, "expected ;": 2, "expected =>": 1, "trailing tokens": 1}, "lean_wall_s": 8.658765077590942, "lean_proc_s": 46.9758198261261, "nd_verify_s": 0.10631203651428223, "workers": 96, "chunk": 400}
22
+ {"utc": "2026-10-07T23:47:53Z", "samples": 12000, "parse_fail": 1212, "distinct_checked": 10495, "both_ok": 6926, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3569, "parse_reasons": {"unbound": 907, "formula": 104, "name": 49, "expected (": 40, "expected )": 32, "exact not last": 23, "expected :=": 20, "trailing tokens": 14, "no-eos": 12, "expected ;": 6, "ORE box": 3, "PR position": 2}, "lean_wall_s": 31.261536836624146, "lean_proc_s": 811.9277687072754, "nd_verify_s": 0.6436045169830322, "workers": 96, "chunk": 400}
23
+ {"utc": "2026-10-07T23:48:14Z", "samples": 3024, "parse_fail": 835, "distinct_checked": 2022, "both_ok": 196, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1826, "parse_reasons": {"unbound": 656, "expected (": 65, "name": 62, "expected )": 21, "formula": 10, "expected ;": 5, "exact not last": 4, "PR position": 3, "no-eos": 3, "expected :=": 3, "ORE box": 2, "trailing tokens": 1}, "lean_wall_s": 8.72870135307312, "lean_proc_s": 43.85058808326721, "nd_verify_s": 0.12318873405456543, "workers": 96, "chunk": 400}
24
+ {"utc": "2026-10-07T23:48:39Z", "samples": 3000, "parse_fail": 916, "distinct_checked": 1945, "both_ok": 256, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1689, "parse_reasons": {"unbound": 654, "name": 104, "expected (": 87, "expected )": 17, "formula": 16, "expected ;": 14, "PR position": 7, "trailing tokens": 6, "exact not last": 5, "no-eos": 4, "ORE box": 2}, "lean_wall_s": 8.754411458969116, "lean_proc_s": 43.21687984466553, "nd_verify_s": 0.0972144603729248, "workers": 96, "chunk": 400}
25
+ {"utc": "2026-10-07T23:49:22Z", "samples": 3000, "parse_fail": 888, "distinct_checked": 1956, "both_ok": 169, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1787, "parse_reasons": {"unbound": 660, "name": 101, "expected :=": 40, "expected (": 31, "expected )": 27, "formula": 9, "exact not last": 7, "PR position": 5, "no-eos": 4, "expected ;": 2, "trailing tokens": 1, "ORE box": 1}, "lean_wall_s": 8.68090558052063, "lean_proc_s": 42.41691493988037, "nd_verify_s": 0.09403324127197266, "workers": 96, "chunk": 400}
26
+ {"utc": "2026-10-07T23:51:00Z", "samples": 12000, "parse_fail": 1427, "distinct_checked": 10275, "both_ok": 6919, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 4, "both_rej": 3352, "parse_reasons": {"unbound": 902, "formula": 139, "expected (": 105, "expected )": 82, "name": 66, "no-eos": 57, "exact not last": 38, "expected ;": 9, "expected :=": 9, "expected False": 7, "ORE box": 4, "PR position": 4, "expected =>": 2, "trailing tokens": 2, "expected fun": 1}, "lean_wall_s": 31.383375644683838, "lean_proc_s": 797.2228610515594, "nd_verify_s": 0.6305513381958008, "workers": 96, "chunk": 400}
27
+ {"utc": "2026-10-07T23:51:19Z", "samples": 2856, "parse_fail": 894, "distinct_checked": 1841, "both_ok": 155, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1686, "parse_reasons": {"unbound": 595, "formula": 100, "expected (": 59, "name": 51, "expected )": 41, "expected :=": 16, "exact not last": 9, "expected ;": 8, "ORE box": 7, "no-eos": 6, "PR position": 1, "expected None": 1}, "lean_wall_s": 7.924750566482544, "lean_proc_s": 38.28320288658142, "nd_verify_s": 0.09891700744628906, "workers": 96, "chunk": 400}
28
+ {"utc": "2026-10-07T23:51:45Z", "samples": 2832, "parse_fail": 961, "distinct_checked": 1737, "both_ok": 212, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1525, "parse_reasons": {"unbound": 690, "expected (": 77, "expected )": 66, "name": 60, "formula": 23, "no-eos": 21, "exact not last": 12, "ORE box": 7, "PR position": 3, "expected :=": 1, "expected ;": 1}, "lean_wall_s": 7.375456809997559, "lean_proc_s": 32.14392399787903, "nd_verify_s": 0.08982515335083008, "workers": 96, "chunk": 400}
29
+ {"utc": "2026-10-07T23:52:25Z", "samples": 2832, "parse_fail": 1097, "distinct_checked": 1632, "both_ok": 229, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1403, "parse_reasons": {"unbound": 827, "expected (": 99, "expected )": 65, "name": 60, "exact not last": 14, "expected ;": 9, "formula": 7, "expected :=": 4, "no-eos": 4, "PR position": 3, "expected False": 3, "expected =>": 1, "ORE box": 1}, "lean_wall_s": 7.263158321380615, "lean_proc_s": 29.213247060775757, "nd_verify_s": 0.08062434196472168, "workers": 96, "chunk": 400}
30
+ {"utc": "2026-10-07T23:54:02Z", "samples": 12000, "parse_fail": 1259, "distinct_checked": 10455, "both_ok": 7256, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3199, "parse_reasons": {"unbound": 910, "formula": 102, "name": 54, "expected :=": 42, "expected )": 37, "exact not last": 26, "expected (": 22, "no-eos": 21, "expected ;": 19, "trailing tokens": 14, "ORE box": 3, "PR position": 3, "expected None": 2, "expected False": 2, "expected =>": 1, "expected fun": 1}, "lean_wall_s": 30.287680864334106, "lean_proc_s": 780.7023830413818, "nd_verify_s": 0.6440978050231934, "workers": 96, "chunk": 400}
31
+ {"utc": "2026-10-07T23:54:21Z", "samples": 2664, "parse_fail": 794, "distinct_checked": 1764, "both_ok": 132, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1632, "parse_reasons": {"unbound": 652, "name": 38, "expected )": 28, "formula": 21, "expected :=": 18, "expected (": 16, "expected ;": 10, "no-eos": 8, "exact not last": 2, "PR position": 1}, "lean_wall_s": 6.993078708648682, "lean_proc_s": 31.54142999649048, "nd_verify_s": 0.10533738136291504, "workers": 96, "chunk": 400}
32
+ {"utc": "2026-10-07T23:54:45Z", "samples": 2664, "parse_fail": 923, "distinct_checked": 1631, "both_ok": 151, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1480, "parse_reasons": {"unbound": 717, "name": 75, "formula": 34, "expected )": 31, "expected (": 30, "no-eos": 19, "expected ;": 9, "exact not last": 4, "eof": 1, "trailing tokens": 1, "expected :=": 1, "PR position": 1}, "lean_wall_s": 7.350590944290161, "lean_proc_s": 30.22095036506653, "nd_verify_s": 0.07902216911315918, "workers": 96, "chunk": 400}
33
+ {"utc": "2026-10-07T23:55:23Z", "samples": 2664, "parse_fail": 850, "distinct_checked": 1719, "both_ok": 199, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1520, "parse_reasons": {"unbound": 664, "name": 94, "no-eos": 27, "expected (": 20, "expected )": 20, "formula": 8, "exact not last": 7, "PR position": 4, "expected ;": 3, "expected =>": 2, "expected False": 1}, "lean_wall_s": 7.381716251373291, "lean_proc_s": 31.813352584838867, "nd_verify_s": 0.08797168731689453, "workers": 96, "chunk": 400}
34
+ {"utc": "2026-10-07T23:57:02Z", "samples": 12000, "parse_fail": 1417, "distinct_checked": 10287, "both_ok": 7494, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2793, "parse_reasons": {"unbound": 1001, "formula": 128, "expected )": 119, "name": 53, "no-eos": 33, "expected (": 29, "exact not last": 24, "expected ;": 18, "expected :=": 8, "PR position": 2, "expected None": 2}, "lean_wall_s": 29.160413026809692, "lean_proc_s": 746.7005352973938, "nd_verify_s": 0.639855146408081, "workers": 96, "chunk": 400}
35
+ {"utc": "2026-10-07T23:57:21Z", "samples": 2520, "parse_fail": 880, "distinct_checked": 1548, "both_ok": 118, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1430, "parse_reasons": {"unbound": 693, "formula": 59, "name": 33, "expected )": 30, "expected ;": 26, "expected (": 15, "no-eos": 14, "exact not last": 10}, "lean_wall_s": 6.621351718902588, "lean_proc_s": 26.154228448867798, "nd_verify_s": 0.08618378639221191, "workers": 96, "chunk": 400}
36
+ {"utc": "2026-10-07T23:57:44Z", "samples": 2520, "parse_fail": 889, "distinct_checked": 1527, "both_ok": 140, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1387, "parse_reasons": {"unbound": 675, "name": 68, "formula": 47, "expected (": 38, "expected )": 21, "expected ;": 18, "no-eos": 10, "exact not last": 6, "PR position": 3, "ORE box": 3}, "lean_wall_s": 6.366706371307373, "lean_proc_s": 25.03095817565918, "nd_verify_s": 0.08992552757263184, "workers": 96, "chunk": 400}
37
+ {"utc": "2026-10-07T23:58:20Z", "samples": 2520, "parse_fail": 1001, "distinct_checked": 1423, "both_ok": 124, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1299, "parse_reasons": {"unbound": 779, "name": 55, "formula": 50, "expected )": 40, "expected (": 34, "expected :=": 16, "exact not last": 8, "no-eos": 6, "expected ;": 6, "ORE box": 5, "PR position": 1, "expected exact": 1}, "lean_wall_s": 6.022954225540161, "lean_proc_s": 22.224732875823975, "nd_verify_s": 0.10215187072753906, "workers": 96, "chunk": 400}
38
+ {"utc": "2026-10-08T00:00:02Z", "samples": 12000, "parse_fail": 1173, "distinct_checked": 10564, "both_ok": 7587, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2977, "parse_reasons": {"unbound": 854, "formula": 105, "name": 84, "exact not last": 29, "expected :=": 26, "no-eos": 25, "expected ;": 17, "expected (": 15, "expected )": 8, "trailing tokens": 3, "expected =>": 2, "PR position": 2, "expected fun": 2, "ORE box": 1}, "lean_wall_s": 31.35661745071411, "lean_proc_s": 814.7558350563049, "nd_verify_s": 0.6190037727355957, "workers": 96, "chunk": 400}
39
+ {"utc": "2026-10-08T00:00:20Z", "samples": 2448, "parse_fail": 785, "distinct_checked": 1560, "both_ok": 110, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1450, "parse_reasons": {"unbound": 551, "name": 90, "formula": 33, "expected (": 32, "no-eos": 27, "expected :=": 15, "exact not last": 11, "ORE box": 11, "expected ;": 8, "expected )": 7}, "lean_wall_s": 6.554641008377075, "lean_proc_s": 25.875765800476074, "nd_verify_s": 0.07120609283447266, "workers": 96, "chunk": 400}
40
+ {"utc": "2026-10-08T00:00:40Z", "samples": 2424, "parse_fail": 812, "distinct_checked": 1508, "both_ok": 174, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1334, "parse_reasons": {"unbound": 620, "name": 118, "expected (": 24, "exact not last": 14, "expected )": 10, "ORE box": 8, "formula": 8, "trailing tokens": 4, "expected ;": 3, "no-eos": 2, "expected None": 1}, "lean_wall_s": 6.443192720413208, "lean_proc_s": 24.836196899414062, "nd_verify_s": 0.12735462188720703, "workers": 96, "chunk": 400}
41
+ {"utc": "2026-10-08T00:01:05Z", "samples": 2424, "parse_fail": 898, "distinct_checked": 1440, "both_ok": 77, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1363, "parse_reasons": {"unbound": 668, "name": 119, "expected (": 51, "exact not last": 15, "ORE box": 11, "expected )": 10, "formula": 9, "expected ;": 8, "PR position": 4, "trailing tokens": 1, "expected False": 1, "eof": 1}, "lean_wall_s": 5.709032297134399, "lean_proc_s": 21.266552686691284, "nd_verify_s": 0.058714866638183594, "workers": 96, "chunk": 400}
42
+ {"utc": "2026-10-08T00:02:46Z", "samples": 12000, "parse_fail": 1196, "distinct_checked": 10538, "both_ok": 7734, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2804, "parse_reasons": {"unbound": 866, "formula": 95, "name": 66, "expected (": 39, "expected ;": 32, "expected )": 31, "no-eos": 21, "exact not last": 20, "expected :=": 17, "PR position": 5, "eof": 1, "expected None": 1, "expected fun": 1, "trailing tokens": 1}, "lean_wall_s": 31.483027696609497, "lean_proc_s": 822.8103158473969, "nd_verify_s": 0.6750767230987549, "workers": 96, "chunk": 400}
43
+ {"utc": "2026-10-08T00:03:04Z", "samples": 2424, "parse_fail": 806, "distinct_checked": 1524, "both_ok": 149, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1375, "parse_reasons": {"unbound": 633, "name": 70, "expected (": 23, "formula": 18, "no-eos": 17, "expected ;": 10, "expected )": 10, "trailing tokens": 9, "exact not last": 7, "expected :=": 3, "eof": 2, "expected False": 2, "PR position": 1, "expected None": 1}, "lean_wall_s": 6.529600143432617, "lean_proc_s": 25.771021842956543, "nd_verify_s": 0.07306909561157227, "workers": 96, "chunk": 400}
44
+ {"utc": "2026-10-08T00:03:25Z", "samples": 2400, "parse_fail": 1043, "distinct_checked": 1291, "both_ok": 103, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1188, "parse_reasons": {"unbound": 797, "name": 86, "expected (": 75, "expected ;": 23, "expected )": 19, "no-eos": 14, "exact not last": 9, "formula": 7, "ORE box": 7, "PR position": 4, "eof": 2}, "lean_wall_s": 5.856619596481323, "lean_proc_s": 19.297532320022583, "nd_verify_s": 0.07703018188476562, "workers": 96, "chunk": 400}
45
+ {"utc": "2026-10-08T00:04:00Z", "samples": 2400, "parse_fail": 838, "distinct_checked": 1468, "both_ok": 120, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1348, "parse_reasons": {"unbound": 644, "expected (": 52, "name": 47, "no-eos": 30, "formula": 28, "expected )": 12, "expected ;": 11, "PR position": 4, "exact not last": 3, "eof": 3, "ORE box": 2, "expected False": 1, "expected =>": 1}, "lean_wall_s": 6.272042512893677, "lean_proc_s": 24.36971378326416, "nd_verify_s": 0.07132577896118164, "workers": 96, "chunk": 400}
46
+ {"utc": "2026-10-08T00:05:42Z", "samples": 12000, "parse_fail": 1312, "distinct_checked": 10475, "both_ok": 7425, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3050, "parse_reasons": {"unbound": 956, "formula": 114, "no-eos": 93, "name": 58, "exact not last": 41, "expected ;": 19, "expected (": 12, "PR position": 6, "expected False": 4, "expected )": 3, "ORE box": 2, "expected :=": 2, "trailing tokens": 1, "expected None": 1}, "lean_wall_s": 30.943066358566284, "lean_proc_s": 800.4569823741913, "nd_verify_s": 0.6612663269042969, "workers": 96, "chunk": 400}
47
+ {"utc": "2026-10-08T00:06:00Z", "samples": 2376, "parse_fail": 739, "distinct_checked": 1540, "both_ok": 161, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1379, "parse_reasons": {"unbound": 617, "name": 52, "expected ;": 17, "expected (": 15, "formula": 12, "no-eos": 7, "exact not last": 7, "expected )": 5, "PR position": 3, "ORE box": 2, "expected False": 1, "expected None": 1}, "lean_wall_s": 6.625076055526733, "lean_proc_s": 25.773365259170532, "nd_verify_s": 0.07830071449279785, "workers": 96, "chunk": 400}
48
+ {"utc": "2026-10-08T00:06:19Z", "samples": 2376, "parse_fail": 805, "distinct_checked": 1478, "both_ok": 154, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1324, "parse_reasons": {"unbound": 621, "name": 80, "expected )": 31, "formula": 24, "expected (": 14, "exact not last": 13, "expected ;": 9, "no-eos": 6, "expected :=": 3, "PR position": 2, "ORE box": 1, "expected False": 1}, "lean_wall_s": 6.203248500823975, "lean_proc_s": 24.01581883430481, "nd_verify_s": 0.07248854637145996, "workers": 96, "chunk": 400}
49
+ {"utc": "2026-10-08T00:06:54Z", "samples": 2376, "parse_fail": 847, "distinct_checked": 1449, "both_ok": 162, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1287, "parse_reasons": {"unbound": 632, "name": 72, "no-eos": 43, "expected (": 32, "expected ;": 19, "formula": 14, "expected )": 11, "exact not last": 8, "expected :=": 5, "PR position": 4, "ORE box": 3, "expected False": 3, "trailing tokens": 1}, "lean_wall_s": 5.67194128036499, "lean_proc_s": 20.813690662384033, "nd_verify_s": 0.07692527770996094, "workers": 96, "chunk": 400}
50
+ {"utc": "2026-10-08T00:20:44Z", "samples": 70912, "parse_fail": 15431, "distinct_checked": 48054, "both_ok": 26474, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 4, "both_rej": 21576, "parse_reasons": {"unbound": 11488, "name": 1583, "expected (": 837, "formula": 498, "exact not last": 211, "no-eos": 168, "expected ;": 164, "ORE box": 163, "expected )": 159, "expected =>": 64, "PR position": 56, "expected :=": 15, "trailing tokens": 9, "expected None": 8, "eof": 3, "expected exact": 2, "expected False": 1, "expected \u27e9": 1, "expected fun": 1}, "lean_wall_s": 128.50845575332642, "lean_proc_s": 10242.422759771347, "nd_verify_s": 3.0480639934539795, "workers": 96, "chunk": 400}
51
+ {"utc": "2026-10-08T00:20:59Z", "samples": 2000, "parse_fail": 5, "distinct_checked": 1995, "both_ok": 1970, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 25, "parse_reasons": {"unbound": 3, "name": 1, "expected (": 1}, "lean_wall_s": 3.776629686355591, "lean_proc_s": 17.11406660079956, "nd_verify_s": 0.08057880401611328, "workers": 96, "chunk": 400}
52
+ {"utc": "2026-10-08T00:27:10Z", "samples": 70912, "parse_fail": 14452, "distinct_checked": 49454, "both_ok": 25163, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 4, "both_rej": 24287, "parse_reasons": {"unbound": 11096, "name": 1166, "expected (": 529, "formula": 494, "no-eos": 258, "exact not last": 232, "expected ;": 203, "expected )": 160, "ORE box": 155, "PR position": 48, "expected =>": 46, "expected :=": 23, "trailing tokens": 20, "expected None": 8, "eof": 6, "expected fun": 6, "expected exact": 1, "expected by": 1}, "lean_wall_s": 131.05873155593872, "lean_proc_s": 10413.672190904617, "nd_verify_s": 3.102260112762451, "workers": 96, "chunk": 400}
runs/510-fable_more-shorter-rounds_s2/pretrain.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b403da2bd322eb7a0784e3ceaaeca8cb4938be1f7103621cb3eb102dd023ed22
3
+ size 25625153
runs/510-fable_more-shorter-rounds_s2/source/harness.py ADDED
@@ -0,0 +1,529 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # imports the fork (model, tokenizer, train, ...), which is not resolvable here
3
+ # pyright: reportMissingImports=false
4
+ """FROZEN harness for the test-time-loops autoresearch loop. The research agent does not edit it.
5
+
6
+ Forked from ../autoresearch/harness.py (sha 2aa52f23e7ec) on 2026-10-07. What changed: the RL
7
+ stage belongs to the agent, behind a wall-clock budget and a sampling oracle, and the metric is
8
+ measured with the model's test-time compute knob (KNOB) set to EVAL_LEVEL.
9
+
10
+ One timed run = harness owns the clocks, the oracle and the eval; `pretrain.py` (agent-editable)
11
+ owns pretraining and RL:
12
+
13
+ 1. load frozen data (Dan's cap-6 Stage-1 file, ladder RL-target pool, ladder transfer pool)
14
+ 2. call pretrain.pretrain(train_recs, PRETRAIN_SECS, ...) -> (model, tok, stats); charged for
15
+ everything including model construction and torch.compile. Overrun past the grace -> timeout.
16
+ 3. check n_params <= PARAM_CAP, tok.mode == TOK_MODE and that the model has the KNOB attribute
17
+ 4. call pretrain.rl(model, tok, targets, train_recs, oracle, RL_SECS, ...): expert iteration in
18
+ whatever form the agent likes. The targets are statements only, and `oracle.sample` is the
19
+ only way to turn them into training data: it samples the model's own attempts and returns
20
+ the ones Lean AND nd_verify accept. RL_SECS is wall-clock inside rl() minus the time the
21
+ oracle spent in the Lean gate (pod CPUs differ; GPU work is what is rationed). Overrun past
22
+ the grace, or an oracle call after the budget is spent -> timeout.
23
+ 5. eval on the dev half of the transfer pool with the harness setting the model's KNOB:
24
+ METRIC = dev theorems solved at L_true >= 7 with EVAL_K attempts at EVAL_LEVEL.
25
+ The same eval at DIAG_LEVEL is recorded as a diagnostic, not scored.
26
+ 6. SIGALRM at TOTAL_SECS from process start kills everything -> status "timeout"
27
+
28
+ The holdout half of the transfer pool is never scored here. Everything is written to --out.
29
+
30
+ python3 harness.py --out artifacts/x # on the pod (see pod.py / remote.sh)
31
+ ND_FORK=... python3 harness.py --out /tmp/smoke --smoke # CPU, ~1 min, plumbing check
32
+ """
33
+
34
+ import argparse
35
+ import collections
36
+ import hashlib
37
+ import importlib
38
+ import json
39
+ import os
40
+ import random
41
+ import shutil
42
+ import signal
43
+ import sys
44
+ import time
45
+ import traceback
46
+ from pathlib import Path
47
+
48
+ HERE = os.path.dirname(os.path.abspath(__file__))
49
+ FORK = os.environ.get("ND_FORK", "/workspace/nd-takehome")
50
+ sys.path.insert(0, HERE)
51
+ sys.path.insert(
52
+ 0, FORK
53
+ ) # fork first: its train.py provides batch/loss_on; the agent's file is pretrain.py
54
+ import torch # noqa: E402
55
+ from eval_set import judge, wilson # noqa: E402
56
+ from sample import generate # noqa: E402
57
+
58
+ # ---------------------------------------------------------------- the rules
59
+ # (hash-checked into every result)
60
+ TOTAL_SECS = int(
61
+ os.environ.get("ND_TOTAL_SECS", 5400)
62
+ ) # process start -> summary written; SIGALRM past this. env: benchmarks only
63
+ PRETRAIN_SECS = 300 # agent's pretraining budget, compile included
64
+ PRETRAIN_GRACE = 15 # seconds the agent may overrun before the run is a "timeout"
65
+ RL_SECS = 1800 # agent's RL budget: wall-clock inside rl() minus the oracle's Lean-gate time
66
+ RL_GRACE = 60
67
+ PARAM_CAP = 10_000_000 # unique trainable params, embeddings included; checked after RL too
68
+ N_TARGETS, TEMP, MAX_NEW, SAMPLE_BATCH = 1500, 0.8, 512, 1024
69
+ EVAL_K = 64
70
+ # The model's test-time compute knob: an integer attribute the harness sets before each eval and
71
+ # rl() sets before oracle calls. A sibling loop for another architecture changes these three
72
+ # constants and nothing else in this file.
73
+ KNOB = "n_loops" # loops of the shared core
74
+ EVAL_LEVEL = 8 # the metric: the knob's value during the scored eval
75
+ DIAG_LEVEL = 2 # the same eval again at this value, recorded but not scored
76
+ TOK_MODE = "lean_seq" # required tokenizer mode (fork lean_tok.py)
77
+ DEV = "cuda" if torch.cuda.is_available() else "cpu"
78
+
79
+
80
+ def read(fn):
81
+ return [json.loads(line) for line in open(fn) if line.strip()]
82
+
83
+
84
+ def log(msg):
85
+ print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
86
+
87
+
88
+ def sha(fn):
89
+ return hashlib.sha256(open(fn, "rb").read()).hexdigest()[:12]
90
+
91
+
92
+ class Timeout(Exception):
93
+ pass
94
+
95
+
96
+ class BudgetSpent(Timeout):
97
+ """rl() asked the oracle for samples after RL_SECS was used up."""
98
+
99
+
100
+ # ---------------------------------------------------------------- bookkeeping
101
+
102
+
103
+ def save_checkpoint(a, model, tok, stage):
104
+ """Publish complete weight snapshots atomically; these are not optimizer-resume files."""
105
+ path = Path(a.out) / f"{stage}.pt"
106
+ pending = path.with_suffix(".pt.tmp")
107
+ try:
108
+ torch.save(
109
+ {
110
+ "state": model.state_dict(),
111
+ "tok_mode": getattr(tok, "mode", None),
112
+ "stage": stage,
113
+ "seed": a.seed,
114
+ "harness_sha": sha(f"{HERE}/harness.py"),
115
+ "pretrain_sha": sha(f"{HERE}/pretrain.py"),
116
+ },
117
+ pending,
118
+ )
119
+ os.replace(pending, path)
120
+ finally:
121
+ pending.unlink(missing_ok=True)
122
+
123
+
124
+ def lstar(solved_names, recs, need=5):
125
+ ls_solved = [r["n_lines"] for r in recs if r["name"] in solved_names]
126
+ ge = {L: sum(1 for x in ls_solved if x >= L) for L in range(7, 15)}
127
+ return max([L for L, c in ge.items() if c >= need], default=0), ge
128
+
129
+
130
+ def by_bin(solved_names, recs):
131
+ by = collections.defaultdict(lambda: [0, 0])
132
+ for r in recs:
133
+ by[r["n_lines"]][0] += r["name"] in solved_names
134
+ by[r["n_lines"]][1] += 1
135
+ return {
136
+ str(L): {"solved": k, "n": n, "rate": k / n, "ci": list(wilson(k, n))}
137
+ for L, (k, n) in sorted(by.items())
138
+ }
139
+
140
+
141
+ def gate_seconds():
142
+ """Seconds the fork's lean_gate has logged so far (Lean wall-clock plus nd_verify)."""
143
+ fn = os.environ.get("LEAN_GATE_LOG")
144
+ if not fn or not os.path.exists(fn):
145
+ return 0.0
146
+ return sum(rec["lean_wall_s"] + rec["nd_verify_s"] for rec in read(fn))
147
+
148
+
149
+ def sample_and_judge(model, tok, recs, k, temperature, seed, batch, max_new):
150
+ prompts = [r["prompt"] for r in recs for _ in range(k)]
151
+ t0, g0 = time.time(), gate_seconds()
152
+ flat = generate(
153
+ model,
154
+ tok,
155
+ prompts,
156
+ greedy=(k == 1 and temperature == 0),
157
+ temperature=temperature,
158
+ batch=batch,
159
+ seed=seed,
160
+ max_new=max_new,
161
+ )
162
+ secs, gate = time.time() - t0, gate_seconds() - g0
163
+ rows = judge(recs, [flat[i * k : (i + 1) * k] for i in range(len(recs))], "n_lines")
164
+ return rows, {
165
+ "samples": len(prompts),
166
+ "secs": secs,
167
+ "gate_secs": gate,
168
+ "samples_per_sec": len(prompts) / secs,
169
+ }
170
+
171
+
172
+ def greedy_rate(model, tok, recs):
173
+ rows, _ = sample_and_judge(model, tok, recs, 1, 0, 0, SAMPLE_BATCH, MAX_NEW)
174
+ return sum(r["solved"] for r in rows) / max(len(rows), 1)
175
+
176
+
177
+ def dev_split(transfer):
178
+ return [r for r in transfer if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 0]
179
+
180
+
181
+ def holdout_split(transfer):
182
+ """Never scored by the loop. Score final.pt on this offline when checking in."""
183
+ return [r for r in transfer if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 1]
184
+
185
+
186
+ # ---------------------------------------------------------------- the RL oracle
187
+
188
+
189
+ class Oracle:
190
+ """What rl() gets instead of data: the model's own attempts at the RL targets, filtered to
191
+ the ones Lean AND nd_verify accept. It also keeps the RL clock and the record of every call.
192
+ """
193
+
194
+ def __init__(self, a, tok, targets):
195
+ self.a, self.tok = a, tok
196
+ self.targets = {t["name"]: t for t in targets}
197
+ self.found = collections.defaultdict(dict) # name -> {proof text: record}
198
+ self.calls = []
199
+ self.t0 = self.last = time.time()
200
+ self.uncharged = 0.0
201
+
202
+ def used(self):
203
+ """RL seconds charged so far: wall-clock since rl() started, minus Lean-gate time."""
204
+ return time.time() - self.t0 - self.uncharged
205
+
206
+ def left(self):
207
+ return self.a.rl_secs - self.used()
208
+
209
+ def sample(self, model, targets, k, temperature=TEMP, batch=SAMPLE_BATCH):
210
+ """k attempts at each of `targets` (records rl() was given, any subset, in any order) from
211
+ `model` as it is now, in eval mode, at whatever its KNOB is set to. Returns one
212
+ dict per target, in order: name, prompt, n_lines, proofs (the distinct accepted proof
213
+ texts), n_ok (accepted attempts) and n_tried. Rejected attempts are not returned.
214
+ Raises BudgetSpent, which ends the run as a timeout, once the budget is used up."""
215
+ if self.left() <= 0:
216
+ raise BudgetSpent(f"oracle called with {self.used():.0f}s of {self.a.rl_secs}s used")
217
+ recs = [self.targets[t["name"]] for t in targets]
218
+ call = len(self.calls) + 1
219
+ model.eval()
220
+ rows, samp = sample_and_judge(
221
+ model, self.tok, recs, k, temperature, self.a.seed * 1000 + call, batch, self.a.max_new
222
+ )
223
+ self.uncharged += samp["gate_secs"]
224
+ new = 0
225
+ for t, row in zip(recs, rows, strict=True):
226
+ for p, wl, pl in zip(
227
+ row["proofs"], row["written_lens"], row["pruned_lens"], strict=True
228
+ ):
229
+ if p not in self.found[t["name"]]:
230
+ self.found[t["name"]][p] = {
231
+ "proof": p,
232
+ "written": wl,
233
+ "pruned": pl,
234
+ "round": call,
235
+ }
236
+ new += 1
237
+ solved = {n for n, d in self.found.items() if d}
238
+ ls, ge = lstar(solved, list(self.targets.values()))
239
+ now = time.time()
240
+ self.calls.append(
241
+ {
242
+ "round": call,
243
+ "level": getattr(model, KNOB, None),
244
+ "k": k,
245
+ "n_targets": len(recs),
246
+ "sample_acc": sum(x["n_ok"] for x in rows) / max(samp["samples"], 1),
247
+ "new_proofs": new,
248
+ "targets_solved": len(solved),
249
+ "lstar": ls,
250
+ "ge": ge,
251
+ "mix_rl_records": sum(len(x["proofs"]) for x in rows), # proofs handed back
252
+ "sampling": samp,
253
+ "secs": now - self.last, # since the previous call returned: includes rl()'s work
254
+ "rl_secs_used": self.used(),
255
+ }
256
+ )
257
+ self.last = now
258
+ c = self.calls[-1]
259
+ log(
260
+ f"ei r{call}: acc {c['sample_acc']:.4f}, +{new} proofs, "
261
+ f"solved {len(solved)}/{len(self.targets)}, L*={ls}, {KNOB} {c['level']}, "
262
+ f"{len(recs)} targets x {k}, {samp['samples_per_sec']:.0f} samples/s, "
263
+ f"RL clock {c['rl_secs_used']:.0f}/{self.a.rl_secs}s"
264
+ )
265
+ return [
266
+ {
267
+ "name": t["name"],
268
+ "prompt": t["prompt"],
269
+ "n_lines": t["n_lines"],
270
+ "proofs": list(row["proofs"]),
271
+ "n_ok": row["n_ok"],
272
+ "n_tried": row["n_tried"],
273
+ }
274
+ for t, row in zip(recs, rows, strict=True)
275
+ ]
276
+
277
+
278
+ # ---------------------------------------------------------------- stages
279
+
280
+
281
+ def stage_rl(a, agent, tok, model, targets, train_recs):
282
+ oracle = Oracle(a, tok, targets)
283
+ # statements only: what the agent's RL may know about a target
284
+ visible = [{"name": t["name"], "prompt": t["prompt"], "n_lines": t["n_lines"]} for t in targets]
285
+ stats = agent.rl(model, tok, visible, train_recs, oracle, a.rl_secs, DEV, a.seed, log)
286
+ if DEV == "cuda":
287
+ torch.cuda.synchronize()
288
+ used = oracle.used()
289
+ model.eval()
290
+ with open(f"{a.out}/found_targets.jsonl", "w") as f:
291
+ for t in targets:
292
+ for x in oracle.found[t["name"]].values():
293
+ f.write(
294
+ json.dumps({"name": t["name"], "thm": t["thm"], "L_true": t["n_lines"], **x})
295
+ + "\n"
296
+ )
297
+ log(f"rl returned: {used:.0f}s of {a.rl_secs}s charged, {oracle.uncharged:.0f}s Lean gate")
298
+ return {
299
+ "rounds": oracle.calls,
300
+ "secs": time.time() - oracle.t0,
301
+ "rl_secs_used": used,
302
+ "gate_secs": oracle.uncharged,
303
+ "targets": len(targets),
304
+ "stats": dict(stats or {}),
305
+ }
306
+
307
+
308
+ def eval_at(a, tok, model, dev_pool, level):
309
+ setattr(model, KNOB, level)
310
+ rows, samp = sample_and_judge(
311
+ model, tok, dev_pool, a.eval_k, TEMP, a.seed * 1000 + 777, SAMPLE_BATCH, a.max_new
312
+ )
313
+ solved = {r["name"] for r in rows if r["solved"]}
314
+ ls, ge = lstar(solved, dev_pool)
315
+ out = {
316
+ "level": level,
317
+ "attempts_per_theorem": a.eval_k,
318
+ "solved": len(solved),
319
+ "n": len(dev_pool),
320
+ "solved_ge7": ge[7],
321
+ "lstar": ls,
322
+ "ge": ge,
323
+ "by_bin": by_bin(solved, dev_pool),
324
+ "sample_acc": sum(r["n_ok"] for r in rows) / max(samp["samples"], 1),
325
+ "max_pruned": max((max(r["pruned_lens"]) for r in rows if r["solved"]), default=0),
326
+ "sampling": samp,
327
+ }
328
+ log(
329
+ f"eval at {KNOB} {level}: solved {len(solved)}/{len(dev_pool)}, ge7 {ge[7]}, L*={ls}, "
330
+ f"per-sample {out['sample_acc']:.4f}, {samp['samples_per_sec']:.0f} samples/s"
331
+ )
332
+ return out, rows
333
+
334
+
335
+ def stage_eval(a, tok, model, dev_pool, heldout_recs):
336
+ out, rows = eval_at(a, tok, model, dev_pool, a.eval_level)
337
+ out["heldout_greedy"] = greedy_rate(model, tok, heldout_recs)
338
+ with open(f"{a.out}/eval_dev.jsonl", "w") as f:
339
+ for r in rows:
340
+ keys = ("name", "n_lines", "solved", "n_ok", "proofs", "written_lens", "pruned_lens")
341
+ f.write(json.dumps({k: r[k] for k in keys}) + "\n")
342
+ diag, _ = eval_at(a, tok, model, dev_pool, a.diag_level)
343
+ out["diag"] = diag
344
+ setattr(model, KNOB, a.eval_level)
345
+ log(
346
+ f"eval: solved {out['solved']}/{len(dev_pool)}, ge7 {out['solved_ge7']} at "
347
+ f"{KNOB} {a.eval_level} ({diag['solved_ge7']} at {a.diag_level}), L*={out['lstar']}, "
348
+ f"max pruned {out['max_pruned']}, held-out greedy {out['heldout_greedy']:.3f}"
349
+ )
350
+ return out
351
+
352
+
353
+ def lean_gate_totals():
354
+ """Sum the fork lean_gate's per-generate() log lines (remote.sh sets LEAN_GATE_LOG)."""
355
+ fn = os.environ.get("LEAN_GATE_LOG")
356
+ if not fn or not os.path.exists(fn):
357
+ return None
358
+ keys = ("samples", "parse_fail", "distinct_checked", "both_ok", "nd_ok_lean_rej")
359
+ tot = dict.fromkeys(keys + ("nd_rej_lean_ok", "lean_wall_s"), 0)
360
+ for rec in read(fn):
361
+ for k in tot:
362
+ tot[k] += rec[k]
363
+ return tot
364
+
365
+
366
+ RULES = (
367
+ "TOTAL_SECS",
368
+ "PRETRAIN_SECS",
369
+ "RL_SECS",
370
+ "PARAM_CAP",
371
+ "N_TARGETS",
372
+ "EVAL_K",
373
+ "KNOB",
374
+ "EVAL_LEVEL",
375
+ "DIAG_LEVEL",
376
+ "TOK_MODE",
377
+ )
378
+
379
+
380
+ def write_summary(a, status, **kw):
381
+ s = {
382
+ "status": status,
383
+ "metric": kw.get("eval", {}).get("solved_ge7"),
384
+ "metric_diag": kw.get("eval", {}).get("diag", {}).get("solved_ge7"),
385
+ "metric_rl": (kw.get("rl", {}).get("rounds") or [{}])[-1].get(
386
+ "targets_solved"
387
+ ), # secondary: cumulative RL targets solved
388
+ "harness_sha": sha(f"{HERE}/harness.py"),
389
+ "pretrain_sha": sha(f"{HERE}/pretrain.py"),
390
+ "device": DEV,
391
+ "gpu": torch.cuda.get_device_name(0) if DEV == "cuda" else "cpu",
392
+ "torch": torch.__version__,
393
+ "lean_gate": lean_gate_totals(),
394
+ "total_secs": time.time() - a.T0,
395
+ "rules": {k: globals()[k] for k in RULES},
396
+ **kw,
397
+ }
398
+ with open(f"{a.out}/summary.json", "w") as f:
399
+ json.dump(s, f, indent=1)
400
+ log(
401
+ f"SUMMARY status={status} metric={s['metric']} metric_diag={s['metric_diag']} "
402
+ f"metric_rl={s['metric_rl']} total {s['total_secs']:.0f}s"
403
+ )
404
+ return s
405
+
406
+
407
+ def main():
408
+ ap = argparse.ArgumentParser()
409
+ ap.add_argument("--out", required=True)
410
+ ap.add_argument("--seed", type=int, default=0)
411
+ ap.add_argument(
412
+ "--pretrain-only", action="store_true", help="save pretrain.pt and stop before eval/RL"
413
+ )
414
+ ap.add_argument(
415
+ "--smoke", action="store_true", help="tiny everything, for a CPU plumbing check"
416
+ )
417
+ a = ap.parse_args()
418
+ a.pretrain_secs, a.rl_secs, a.eval_k, a.max_new, a.n_targets = (
419
+ PRETRAIN_SECS,
420
+ RL_SECS,
421
+ EVAL_K,
422
+ MAX_NEW,
423
+ N_TARGETS,
424
+ )
425
+ a.eval_level, a.diag_level = EVAL_LEVEL, DIAG_LEVEL
426
+ if a.smoke:
427
+ a.pretrain_secs, a.rl_secs, a.eval_k, a.max_new, a.n_targets = 10, 30, 2, 120, 8
428
+ os.makedirs(a.out, exist_ok=True)
429
+ # Preserve discarded variants too: weights alone do not specify the model architecture.
430
+ source_dir = Path(a.out) / "source"
431
+ source_dir.mkdir(exist_ok=True)
432
+ for filename in ("pretrain.py", "harness.py", "remote.sh"):
433
+ shutil.copy2(Path(HERE) / filename, source_dir / filename)
434
+ a.T0 = time.time()
435
+ torch.manual_seed(a.seed)
436
+
437
+ def on_alarm(signum, frame):
438
+ raise Timeout()
439
+
440
+ signal.signal(signal.SIGALRM, on_alarm)
441
+ signal.alarm(TOTAL_SECS if not a.smoke else 900)
442
+
443
+ train_recs = read(f"{FORK}/data/train.jsonl")
444
+ assert all(r["n_lines"] <= 6 for r in train_recs), "pretrain file violates cap 6"
445
+ heldout_all = read(f"{FORK}/data/heldout.jsonl")
446
+ # Length-ordered file: take 2000 evenly spaced records, so every length 2-6 is covered.
447
+ heldout_all = [heldout_all[i * len(heldout_all) // 2000] for i in range(2000)]
448
+ heldout_recs = heldout_all if not a.smoke else heldout_all[:16]
449
+ targets_all = read(f"{FORK}/data/ladder/rl_targets.jsonl")
450
+ transfer = read(f"{FORK}/data/ladder/transfer.jsonl")
451
+ dev_pool = dev_split(transfer) if not a.smoke else dev_split(transfer)[:8]
452
+ targets = random.Random(a.seed).sample(targets_all, min(a.n_targets, len(targets_all)))
453
+ tk, dk, hk = (
454
+ {t["key"] for t in targets},
455
+ {r["key"] for r in dev_pool},
456
+ {r["key"] for r in train_recs},
457
+ )
458
+ assert not (tk & dk) and not (tk & hk) and not (dk & hk), "renaming-class leak between pools"
459
+ log(
460
+ f"pools: train {len(train_recs):,}, targets {len(targets)}, dev-transfer {len(dev_pool)}, "
461
+ f"held-out {len(heldout_recs)}; "
462
+ f"device {DEV}; harness {sha(f'{HERE}/harness.py')} pretrain {sha(f'{HERE}/pretrain.py')}"
463
+ )
464
+
465
+ pre = {}
466
+ try:
467
+ agent = importlib.import_module("pretrain")
468
+ t0 = time.time()
469
+ model, tok, pre = agent.pretrain(train_recs, a.pretrain_secs, DEV, a.seed, log)
470
+ if DEV == "cuda":
471
+ torch.cuda.synchronize()
472
+ pre = dict(pre or {})
473
+ pre["secs"] = time.time() - t0
474
+ model.eval()
475
+ n_params = sum(p.numel() for p in model.parameters())
476
+ pre["n_params"] = n_params
477
+ log(
478
+ f"pretrain returned after {pre['secs']:.0f}s (budget {a.pretrain_secs}s), "
479
+ f"{n_params:,} params"
480
+ )
481
+ if pre["secs"] > a.pretrain_secs + PRETRAIN_GRACE and not a.smoke:
482
+ over = pre["secs"] - a.pretrain_secs
483
+ write_summary(
484
+ a, "timeout", pretrain=pre, reason=f"pretrain overran budget by {over:.0f}s"
485
+ )
486
+ return
487
+ if getattr(tok, "mode", None) != TOK_MODE:
488
+ write_summary(a, "crash", pretrain=pre, reason=f"tok.mode {tok.mode!r} != {TOK_MODE!r}")
489
+ return
490
+ if n_params > PARAM_CAP:
491
+ write_summary(a, "crash", pretrain=pre, reason=f"{n_params} params > cap {PARAM_CAP}")
492
+ return
493
+ if not isinstance(getattr(model, KNOB, None), int):
494
+ reason = f"model has no integer {KNOB} for the harness to set at eval"
495
+ write_summary(a, "crash", pretrain=pre, reason=reason)
496
+ return
497
+ save_checkpoint(a, model, tok, "pretrain")
498
+ if a.pretrain_only:
499
+ write_summary(a, "ok", stage="pretrain_only", pretrain=pre)
500
+ return
501
+ pre["heldout_greedy"] = greedy_rate(model, tok, heldout_recs)
502
+ log(f"pretrain held-out greedy {pre['heldout_greedy']:.3f}")
503
+ rl = stage_rl(a, agent, tok, model, targets, train_recs)
504
+ if rl["rl_secs_used"] > a.rl_secs + RL_GRACE and not a.smoke:
505
+ over = rl["rl_secs_used"] - a.rl_secs
506
+ reason = f"rl overran budget by {over:.0f}s"
507
+ write_summary(a, "timeout", pretrain=pre, rl=rl, reason=reason)
508
+ return
509
+ n_after = sum(p.numel() for p in model.parameters())
510
+ if n_after > PARAM_CAP:
511
+ reason = f"{n_after} params > cap {PARAM_CAP} after rl"
512
+ write_summary(a, "crash", pretrain=pre, rl=rl, reason=reason)
513
+ return
514
+ save_checkpoint(a, model, tok, "final")
515
+ ev = stage_eval(a, tok, model, dev_pool, heldout_recs)
516
+ write_summary(a, "ok", pretrain=pre, rl=rl, eval=ev)
517
+ except BudgetSpent as e:
518
+ write_summary(a, "timeout", pretrain=pre, reason=f"rl: {e}")
519
+ except Timeout:
520
+ write_summary(a, "timeout", pretrain=pre, reason=f"exceeded TOTAL_SECS={TOTAL_SECS}")
521
+ except Exception:
522
+ traceback.print_exc()
523
+ write_summary(a, "crash", pretrain=pre, reason=traceback.format_exc()[-2000:])
524
+ finally:
525
+ signal.alarm(0)
526
+
527
+
528
+ if __name__ == "__main__":
529
+ main()
runs/510-fable_more-shorter-rounds_s2/source/pretrain.py ADDED
@@ -0,0 +1,611 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # imports the fork (model, tokenizer, train, ...), which is not resolvable here
2
+ # pyright: reportMissingImports=false
3
+ """AGENT-EDITABLE: pretraining and RL. This is the only file the autoresearch agent changes.
4
+
5
+ Contract with harness.py (frozen):
6
+
7
+ pretrain(train_recs, budget_secs, device, seed, log) -> (model, tok, stats)
8
+
9
+ train_recs list of dicts {prompt, proof, n_lines<=6, key, ...}; the frozen cap-6 Stage-1 data
10
+ budget_secs wall-clock you are charged for, from call to return, INCLUDING model construction
11
+ and torch.compile. Overrun by more than the harness grace -> the run is a timeout.
12
+ model nn.Module; <= PARAM_CAP unique params (checked again after rl). The harness samples
13
+ from it with the fork's sample.generate, so it must provide:
14
+ - forward(idx) -> logits (B,T,V) and forward(idx, pos=, mask=, caches=) with a
15
+ per-block KV cache dict, exactly like the fork's GPT (model.blocks must exist)
16
+ - an integer `n_loops` (the harness's KNOB): the loop count applied in eval
17
+ mode. The harness sets it to EVAL_LEVEL for the scored eval and to DIAG_LEVEL
18
+ for the diagnostic one.
19
+ tok must provide encode_prompt, encode_proof, decode, shift_abs, pad, eos, mode, and
20
+ mode must be "lean_seq" (harness TOK_MODE): Lean text in and out, Lean-gated samples.
21
+ Decoder rule: decode() is deterministic and single-pass; it may not call the
22
+ verifier or try alternatives.
23
+ stats dict recorded in summary.json and copied into results.json. MUST contain `curve`:
24
+ a list of {secs, step, train_loss, val_loss} taken every ~CURVE_EVERY seconds
25
+ (see log_point below), so a run's trajectory survives in git. Add anything else.
26
+ val_loss is on VAL_N records held out from train_recs by this file; keep that.
27
+
28
+ rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log) -> stats | None
29
+
30
+ targets list of dicts {name, prompt, n_lines}: the 1,500 RL target statements. No proofs.
31
+ oracle oracle.sample(model, targets_subset, k, temperature=0.8, batch=1024) samples k
32
+ attempts per target from the model in eval mode at its current n_loops, and returns
33
+ per target {name, prompt, n_lines, proofs, n_ok, n_tried}; `proofs` are the distinct
34
+ attempts Lean AND nd_verify accepted. This is the only source of new training data.
35
+ oracle.used() / oracle.left(): seconds of the RL budget charged / remaining.
36
+ budget_secs RL_SECS: wall-clock inside rl() minus the oracle's Lean-gate time. Returning more
37
+ than the grace late, or calling the oracle with nothing left, is a timeout.
38
+ The model is trained in place. Everything else about RL is yours: rounds, samples per target,
39
+ loop counts when sampling, which proofs to keep, weighting, replay, optimizer, loss.
40
+
41
+ Anything imported from the fork (ND_FORK) is fair game to copy in here and modify.
42
+ """
43
+
44
+ import math
45
+ import random
46
+ import time
47
+
48
+ import torch
49
+ import torch._inductor.config # noqa: F401 (explicit, so the attribute resolves for pyright)
50
+ from lean_tok import MAXN # fork
51
+ from model import GPT, Block # fork
52
+ from normalize import norm # fork
53
+ from tokenizer import make_tokenizer # fork
54
+ from train import loss_on # fork
55
+
56
+ # ---- knobs
57
+ # Unique blocks: one untied entry block, N_CORE shared core blocks, one untied exit block.
58
+ N_CORE = 2
59
+ N_LAYER, D, N_HEAD = 2 + N_CORE, 384, 8
60
+ # Core applications (loops). Training draws one count per batch from TRAIN_LOOPS, in pretraining
61
+ # and in rl()'s fine-tune; eval and sampling apply model.n_loops, which the harness sets for its
62
+ # evals and rl() sets before each oracle call. DEFAULT_LOOPS is used for the validation curve and
63
+ # the harness's post-pretraining greedy check.
64
+ TRAIN_LOOPS = (1, 2, 3, 4, 5, 6, 7, 8)
65
+ DEFAULT_LOOPS = 2
66
+ VAL_LOOPS = (1, 2, 4, 8) # val loss at each count, once, when pretraining ends
67
+ # ---- RL knobs: the old harness's expert iteration (600 AdamW steps per round on up to 4 proofs
68
+ # per solved target, weighted x4, mixed with 20,000 replayed pretraining records), except for
69
+ # where the samples go. Each round every target gets RL_K attempts at RL_FIRST_LOOPS; a target
70
+ # all of them fail on gets RL_RETRY_K more (32 in all, as before) at ONE deeper loop count from
71
+ # RL_RETRY_LOOPS, assigned at random, so proofs that only deeper sampling finds enter the
72
+ # training set and no deep sample is spent on a target one loop already solves. The random
73
+ # assignment makes the retry calls a controlled comparison of loop counts on resistant targets.
74
+ # Rounds repeat until another one would not fit in the budget.
75
+ RL_FIRST_LOOPS, RL_K = 1, 8
76
+ RL_RETRY_LOOPS, RL_RETRY_K = (2, 4, 8), 24
77
+ # The fine-tune sees FT_RECORDS records per round, half the old harness's 600 steps x 128: in
78
+ # 509 the fine-tune had grown to ~74 s of a ~137 s round, so halving it fits about 16 rounds of
79
+ # sampling in RL_SECS instead of 11, with 27% fewer fine-tune records per run; the mix (20-33k
80
+ # records) still gets at least one epoch per round. Records come in the same token-budget
81
+ # batches as pretraining (TOK_BUDGET below) instead of 128-record batches padded to their
82
+ # longest proof, which with 7-14 line RL proofs in nearly every batch was ~3x the content.
83
+ # It uses pretraining's optimizer split: Muon (muon_step, MUON_WD) on the block matrices and the
84
+ # head, AdamW at FT_LR on embeddings, norms and biases. FT_MUON_LR keeps pretraining's 10:1
85
+ # Muon:AdamW ratio and is where pretraining's cosine ends (MUON_LR * MIN_LR_FRAC).
86
+ FT_RECORDS, FT_LR, FT_MUON_LR, FT_WARMUP = 300 * 128, 3e-4, 3e-3, 50
87
+ RETAIN, MAX_PER_THM, RL_WEIGHT = 20000, 4, 4
88
+ # MLP width held at 151's 1280: the fork's Block hardcodes 4 * D (1536 at D 384 would pass the
89
+ # 10M cap), so PeriLNBlock swaps fc1/fc2. 1280 != 3 * D keeps muon_step's QKV-by-shape test valid.
90
+ D_FF = 1280
91
+ BS, LR, MIN_LR_FRAC, WD = 128, 1e-3, 0.3, 0.1
92
+ MUON_LR, MUON_MOMENTUM = 0.01, 0.95
93
+ # Polar Express (Amsel et al. 2025) per-iteration quintic coefficients, as in modded-nanogpt
94
+ # (num_iters=5, safety factor 2e-2): singular values land in ~[0.86, 1.14] instead of the
95
+ # fixed quintic's ~[0.68, 1.13], at the same five iterations.
96
+ NS_COEFFS = (
97
+ (8.156554524902461, -22.48329292557795, 15.878769915207462),
98
+ (4.042929935166739, -2.808917465908714, 0.5000178451051316),
99
+ (3.8916678022926607, -2.772484153217685, 0.5060648178503393),
100
+ (3.285753657755655, -2.3681294933425376, 0.46449024233003106),
101
+ (2.3465413258596377, -1.7097828382687081, 0.42323551169305323),
102
+ )
103
+ # Ten times the AdamW group's per-step multiplicative decay (lr * wd) on the Muon matrices.
104
+ MUON_WD = 10 * WD * LR / MUON_LR
105
+ TOK_MODE = "lean_seq" # required by the harness (Lean protocol)
106
+ # Hybrid heads: half the heads get ALiBi (Press et al. 2021) with the standard slope set for
107
+ # H/2 heads, 2^(-8 i / (H/2)) = 1/4 .. 1/256 with 8 heads; the other half get slope 0, i.e. no
108
+ # positional signal at all (NoPE), so they can look citations up by content at any distance.
109
+ N_ALIBI = N_HEAD // 2
110
+ ALIBI_SLOPES = tuple(2.0 ** (-8.0 * (i + 1) / N_ALIBI) for i in range(N_ALIBI)) + (0.0,) * (
111
+ N_HEAD - N_ALIBI
112
+ )
113
+ VAL_N, CURVE_EVERY = 1024, 15 # records held out for val_loss; seconds between curve points
114
+ # Token-budget batches (fairseq max-tokens): each step takes a run of length-sorted records with
115
+ # count * padded_length <= TOK_BUDGET, so a batch of short proofs has many records and a batch of
116
+ # long ones few, and the padded token count per step is ~constant. 128 x 144 matches the real
117
+ # (unpadded) tokens of the incumbent's 128-record batches (mean record ~135 tokens), whose
118
+ # padding to the batch max (~400) made every step ~3x larger than its content.
119
+ TOK_BUDGET = 128 * 144
120
+ # Multi-token prediction (DeepSeek-V3 2024, Gloeckle et al. 2024), pretraining only: one extra
121
+ # Peri-LN block reads [trunk state at t; embedding of token t+1] and predicts token t+2 through
122
+ # the model's own ln_f/head. Its loss is added at this weight (DeepSeek-V3's early-training value).
123
+ MTP_WEIGHT = 0.3
124
+
125
+
126
+ # Muon update adapted from KellerJordan/Muon, commit
127
+ # f98f1cacc0263b04290753e32be8d498c1efc806 (muon.py).
128
+ # MIT License
129
+ # Copyright (c) 2024 Keller Jordan
130
+ #
131
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
132
+ # of this software and associated documentation files (the "Software"), to deal
133
+ # in the Software without restriction, including without limitation the rights
134
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
135
+ # copies of the Software, and to permit persons to whom the Software is
136
+ # furnished to do so, subject to the following conditions:
137
+ # The above copyright notice and this permission notice shall be included in all
138
+ # copies or substantial portions of the Software.
139
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
140
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
141
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
142
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
143
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
144
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
145
+ # SOFTWARE.
146
+ @torch.no_grad()
147
+ def muon_step(params, buffers, lr):
148
+ """Nesterov momentum plus five bf16 Polar Express steps, on hidden matrices only."""
149
+ for p, momentum in zip(params, buffers, strict=True):
150
+ if p.grad is None:
151
+ continue
152
+ momentum.lerp_(p.grad, 1 - MUON_MOMENTUM)
153
+ update = p.grad.lerp(momentum, MUON_MOMENTUM).bfloat16()
154
+ # Only QKV has shape (3D, D) in this GPT; orthogonalize its roles separately.
155
+ if update.size(0) == 3 * update.size(1):
156
+ update = update.reshape(3, update.size(1), update.size(1))
157
+ transpose = update.size(-2) > update.size(-1)
158
+ if transpose:
159
+ update = update.mT
160
+ update = update / (update.norm(dim=(-2, -1), keepdim=True) * (1 + 2e-2) + 1e-6)
161
+ for a, b, c in NS_COEFFS:
162
+ gram = update @ update.mT
163
+ polynomial = b * gram + c * gram @ gram
164
+ update = a * update + polynomial @ update
165
+ if transpose:
166
+ update = update.mT
167
+ p.mul_(1 - lr * MUON_WD)
168
+ # lr may be a 0-d tensor (CUDA-graphed step); scale in p's dtype like add_(alpha=-lr)
169
+ p.sub_(lr * update.reshape_as(p).to(p.dtype))
170
+
171
+
172
+ class MuonStep:
173
+ """muon_step on `matrices` with fresh momentum buffers, called once per step after backward
174
+ (grads must keep fixed addresses: zero_grad(set_to_none=False)). On CUDA the first call
175
+ captures the step as one CUDA graph and later calls replay it instead of ~400 kernel
176
+ launches; CPU smoke runs call muon_step directly."""
177
+
178
+ def __init__(self, matrices, device):
179
+ self.matrices, self.device = matrices, device
180
+ self.buffers = [torch.zeros_like(p) for p in matrices]
181
+ self.lr, self.graph = torch.zeros((), device=device), None
182
+
183
+ def __call__(self, lr):
184
+ self.lr.fill_(lr)
185
+ if self.device != "cuda":
186
+ muon_step(self.matrices, self.buffers, self.lr)
187
+ return
188
+ if self.graph is None:
189
+ state = self.matrices + self.buffers
190
+ saved = [t.clone() for t in state]
191
+ side = torch.cuda.Stream()
192
+ side.wait_stream(torch.cuda.current_stream())
193
+ with torch.cuda.stream(side):
194
+ muon_step(self.matrices, self.buffers, self.lr) # warm-up (lazy cuBLAS init)
195
+ torch.cuda.current_stream().wait_stream(side)
196
+ self.graph = torch.cuda.CUDAGraph()
197
+ with torch.cuda.graph(self.graph):
198
+ muon_step(self.matrices, self.buffers, self.lr)
199
+ with torch.no_grad():
200
+ for t, v in zip(state, saved, strict=True):
201
+ t.copy_(v) # undo the warm-up and capture updates
202
+ self.graph.replay()
203
+
204
+
205
+ class PeriLNBlock(Block):
206
+ """P22 Peri-LN (Kim et al. 2025): the fork's pre-LN block plus a LayerNorm on each branch
207
+ output, so every residual write has unit scale times a learned gain. Attention has no RoPE:
208
+ `mask` is the additive ALiBi bias (B or 1, H, T, S) that LoopedGPT builds once per forward
209
+ and cos/sin are unused."""
210
+
211
+ def __init__(self, d, h):
212
+ super().__init__(d, h)
213
+ self.fc1, self.fc2 = torch.nn.Linear(d, D_FF), torch.nn.Linear(D_FF, d)
214
+ self.ln_attn_out, self.ln_mlp_out = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
215
+
216
+ def attn(self, x, cos, sin, mask, cache):
217
+ B, T, D = x.shape
218
+ q, k, v = self.qkv(x).view(B, T, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
219
+ if cache is not None: # the fork's preallocated KV cache, unchanged
220
+ if "k" not in cache:
221
+ cache["k"] = k.new_empty(B, self.h, cache["max"], D // self.h)
222
+ cache["v"] = v.new_empty(B, self.h, cache["max"], D // self.h)
223
+ cache["n"] = 0
224
+ n0, n1 = cache["n"], cache["n"] + T
225
+ cache["k"][:, :, n0:n1] = k
226
+ cache["v"][:, :, n0:n1] = v
227
+ cache["n"] = n1
228
+ k, v = cache["k"][:, :, :n1], cache["v"][:, :, :n1]
229
+ y = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask.to(q.dtype))
230
+ return self.proj(y.transpose(1, 2).reshape(B, T, D))
231
+
232
+ def forward(self, x, cos, sin, mask=None, cache=None):
233
+ x = x + self.ln_attn_out(self.attn(self.ln1(x), cos, sin, mask, cache))
234
+ return x + self.ln_mlp_out(self.fc2(torch.nn.functional.gelu(self.fc1(self.ln2(x)))))
235
+
236
+
237
+ class LoopedGPT(GPT):
238
+ """Looped trunk (Universal Transformer, Dehghani et al. 2018; sandwich layout and input
239
+ injection as in Geiping et al. 2025): blocks[0] is the entry block, blocks[1:-1] the shared
240
+ core applied n times, blocks[-1] the exit block. Every core pass starts from
241
+ ln_loop(state) + e, with e the entry block's output, so each pass sees an input of the same
242
+ scale and the prompt is re-injected instead of being buried under accumulated writes.
243
+
244
+ P34: the fork's GPT with ALiBi (Press et al. 2021) instead of RoPE. Head h's attention
245
+ logits get -slope_h * (query position - key position), slopes 2^-1 .. 2^-8 for 8 heads, so
246
+ no rotation of q/k anywhere. Positions come from `pos` (sample.generate's left-padded
247
+ batches) or arange; key positions are kept in caches[0] so cached decoding sees the same
248
+ bias as the full forward. Causality/padding come from `mask` when given, else from rel < 0."""
249
+
250
+ def __init__(self, vocab, n_layer, d, n_head, seed, max_len=1024):
251
+ super().__init__(vocab, n_layer, d, n_head, max_len)
252
+ slopes = torch.tensor(ALIBI_SLOPES).view(1, n_head, 1, 1)
253
+ self.register_buffer("slopes", slopes, persistent=False)
254
+ self.ln_loop = torch.nn.LayerNorm(d)
255
+ self.n_loops, self.loop_rng = DEFAULT_LOOPS, random.Random(seed)
256
+
257
+ def forward(self, idx, pos=None, mask=None, caches=None):
258
+ return self.head(self.ln_f(self.features(idx, pos, mask, caches)[0]))
259
+
260
+ def features(self, idx, pos=None, mask=None, caches=None):
261
+ """The trunk: (final hidden state before ln_f, the ALiBi bias used)."""
262
+ B, T = idx.shape
263
+ if pos is None: # training/val: one (1, H, T, T) bias broadcast over the batch
264
+ pos = torch.arange(T, device=idx.device)[None]
265
+ key_pos = pos
266
+ if caches is not None: # incremental decoding: keys are every cached token
267
+ c = caches[0]
268
+ if "pos" not in c:
269
+ c["pos"], c["n_pos"] = pos.new_empty(pos.size(0), c["max"]), 0
270
+ n0, n1 = c["n_pos"], c["n_pos"] + T
271
+ c["pos"][:, n0:n1] = pos
272
+ c["n_pos"] = n1
273
+ key_pos = c["pos"][:, :n1]
274
+ rel = pos[:, None, :, None] - key_pos[:, None, None, :] # (B or 1, 1, T, S)
275
+ bias = -self.slopes * rel.to(self.slopes.dtype)
276
+ bias = bias.masked_fill(rel < 0 if mask is None else ~mask, float("-inf"))
277
+ n = self.loop_rng.choice(TRAIN_LOOPS) if self.training else self.n_loops
278
+ entry, *core, exit_block = self.blocks
279
+ x = e = entry(self.emb(idx), None, None, bias, None if caches is None else caches[0])
280
+ for loop in range(n):
281
+ x = self.ln_loop(x) + e
282
+ for j, b in enumerate(core, 1):
283
+ # sample.generate makes one cache dict per unique block; a shared block's keys
284
+ # and values differ per pass, so each pass gets its own sub-cache inside it.
285
+ c = caches and caches[j].setdefault(loop, {"max": caches[j]["max"]})
286
+ x = b(x, None, None, bias, c)
287
+ x = exit_block(x, None, None, bias, None if caches is None else caches[-1])
288
+ return x, bias
289
+
290
+
291
+ class MTPHead(torch.nn.Module):
292
+ """The MTP module. A separate nn.Module, never registered on the model, so the harness's
293
+ parameter count, RL AdamW and sampler never see it; it is dropped when pretrain() returns."""
294
+
295
+ def __init__(self, d, h):
296
+ super().__init__()
297
+ self.ln_h, self.ln_e = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
298
+ self.proj = torch.nn.Linear(2 * d, d, bias=False)
299
+ self.block = PeriLNBlock(d, h)
300
+
301
+
302
+ def mtp_loss(model, mtp, x, m):
303
+ """The fork's loss_on (next-token CE on proof tokens) plus MTP_WEIGHT times the CE of the
304
+ MTP module's token-(t+2) prediction at the same proof positions. Returns (total, main)."""
305
+ h, bias = model.features(x[:, :-1])
306
+ logits = model.head(model.ln_f(h))
307
+ V = logits.size(-1)
308
+ ce = torch.nn.functional.cross_entropy(
309
+ logits.reshape(-1, V).float(), x[:, 1:].reshape(-1), reduction="none"
310
+ )
311
+ main = (ce * m[:, 1:].reshape(-1)).sum() / m[:, 1:].sum()
312
+ # Position t (0..T-3) gets h_t and emb(x_{t+1}); causal attention over these keeps every
313
+ # prediction of x_{t+2} a function of x_{<=t+1}.
314
+ z = mtp.proj(torch.cat((mtp.ln_h(h[:, :-1]), mtp.ln_e(model.emb(x[:, 1:-1]))), -1))
315
+ z = mtp.block(z, None, None, bias[..., :-1, :-1], None)
316
+ ce2 = torch.nn.functional.cross_entropy(
317
+ model.head(model.ln_f(z)).reshape(-1, V).float(), x[:, 2:].reshape(-1), reduction="none"
318
+ )
319
+ aux = (ce2 * m[:, 2:].reshape(-1)).sum() / m[:, 2:].sum().clamp(min=1)
320
+ return main + MTP_WEIGHT * aux, main, aux
321
+
322
+
323
+ def val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast):
324
+ """Mean loss over the first VAL_N records (never trained on), no shift augmentation."""
325
+ model.eval()
326
+ tot = 0.0
327
+ with torch.no_grad(), autocast:
328
+ for i in range(0, VAL_N, BS):
329
+ idxs = torch.arange(i, min(i + BS, VAL_N), device=ids.device)
330
+ x, m = ids[idxs], mask[idxs]
331
+ T = int(lens[idxs].max())
332
+ tot += loss_on(model, x[:, :T], m[:, :T]).item() * len(idxs)
333
+ model.train()
334
+ return tot / VAL_N
335
+
336
+
337
+ def to_gpu(data, tok, device):
338
+ """Pad all (prompt, proof) records once into (N, T) ids + proof mask on the device."""
339
+ N, T = len(data), max(len(p) + len(q) for p, q in data)
340
+ ids = torch.full((N, T), tok.pad, dtype=torch.long)
341
+ mask = torch.zeros((N, T), dtype=torch.bool)
342
+ ref_max = torch.zeros(
343
+ N, dtype=torch.long
344
+ ) # largest N<k> index used in the proof, for the shift augmentation
345
+ for i, (p, q) in enumerate(data):
346
+ ids[i, : len(p) + len(q)] = torch.tensor(p + q)
347
+ mask[i, len(p) : len(p) + len(q)] = True
348
+ ref_max[i] = max((x - tok.ref0 + 1 for x in q if x >= tok.ref0), default=0)
349
+ return ids.to(device), mask.to(device), ref_max.to(device), (ids != tok.pad).sum(1).to(device)
350
+
351
+
352
+ def token_batches(lens, n_train, gen, rng, first=VAL_N):
353
+ """One epoch of token-budget batches: a random permutation of the n_train records from
354
+ `first` on, stably sorted by length (equal lengths stay shuffled), cut greedily into runs of
355
+ consecutive records with count * max_len <= TOK_BUDGET, and the runs' order shuffled.
356
+ Returns (order, cuts)."""
357
+ perm = torch.randperm(n_train, device=lens.device, generator=gen) + first
358
+ order = perm[torch.argsort(lens[perm], stable=True)]
359
+ sorted_lens = lens[order].cpu()
360
+ cuts, s = [], 0
361
+ while s < len(sorted_lens):
362
+ window = sorted_lens[s : s + TOK_BUDGET // int(sorted_lens[s])]
363
+ # count * max_len is non-decreasing along the window, so the fits form a prefix.
364
+ e = s + int((torch.arange(1, len(window) + 1) * window <= TOK_BUDGET).sum())
365
+ cuts.append((s, e, int(sorted_lens[e - 1]))) # sorted: last is the batch's max length
366
+ s = e
367
+ rng.shuffle(cuts)
368
+ return order, cuts
369
+
370
+
371
+ def gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, max_len=None):
372
+ """The fork's batch() + tok.shift_abs, vectorised: a random per-row offset is added to
373
+ every ref token."""
374
+ x, m = ids[idxs], mask[idxs]
375
+ # Known on the host for training batches: passing it avoids a GPU sync every step.
376
+ T = int(lens[idxs].max()) if max_len is None else max_len
377
+ x, m = x[:, :T].clone(), m[:, :T]
378
+ if tok.mode in ("abs", "lean_seq"): # lean_seq: names n1.. get the same start offset
379
+ room = MAXN - ref_max[idxs]
380
+ shift = (
381
+ (torch.rand(len(idxs), device=x.device, generator=gen) * (room + 1))
382
+ .long()
383
+ .clamp(max=room)
384
+ )
385
+ is_ref = (x >= tok.ref0) & m
386
+ x = torch.where(is_ref, x + shift[:, None], x)
387
+ return x, m
388
+
389
+
390
+ def pretrain(train_recs, budget_secs, device, seed, log):
391
+ t0 = time.time()
392
+ tok = make_tokenizer(TOK_MODE)
393
+ data = [(tok.encode_prompt(r["prompt"]), tok.encode_proof(r["proof"])) for r in train_recs]
394
+ ids, mask, ref_max, lens = to_gpu(data, tok, device)
395
+ gen = torch.Generator(device=device)
396
+ gen.manual_seed(seed)
397
+ model = LoopedGPT(tok.vocab_size, N_LAYER, D, N_HEAD, seed)
398
+ model.blocks = torch.nn.ModuleList(PeriLNBlock(D, N_HEAD) for _ in range(N_LAYER))
399
+ model.blocks.apply(model._init) # fork init: zero biases
400
+ model = model.to(device)
401
+ mtp = MTPHead(D, N_HEAD)
402
+ mtp.apply(model._init)
403
+ mtp = mtp.to(device)
404
+ # Compile the repeated blocks once each (torch >= 2.9; dynamic shapes, ~14 s): ~14% faster
405
+ # steps. The coalescing tiling analysis crashes on dynamic shapes, so it is off. Undone at
406
+ # handoff so the harness's RL fine-tune and KV-cache sampling run eager.
407
+ torch._inductor.config.triton.coalesce_tiling_analysis = False
408
+ compiled = list(model.blocks) + [mtp.block]
409
+ for b in compiled if device == "cuda" else []: # CPU smoke runs stay eager
410
+ b.compile(dynamic=True)
411
+ for p in list(model.blocks.parameters()) + list(mtp.parameters()):
412
+ if p.ndim == 2:
413
+ # Orthogonal matrices have element second moment gain^2 / max(rows, cols).
414
+ torch.nn.init.orthogonal_(p, gain=0.02 * math.sqrt(max(p.shape)))
415
+ log(
416
+ f"pretrain: {model.n_params():,} params, {len(data):,} records on {device} "
417
+ f"after {time.time() - t0:.0f}s, budget {budget_secs}s"
418
+ )
419
+ # The output readout joins the hidden matrices under Muon; embeddings stay on AdamW.
420
+ trainable = list(model.parameters()) + list(mtp.parameters())
421
+ matrices = [p for p in model.blocks.parameters() if p.ndim == 2] + [model.head.weight]
422
+ matrices += [p for p in mtp.parameters() if p.ndim == 2]
423
+ matrix_ids = {id(p) for p in matrices}
424
+ auxiliary = [p for p in trainable if id(p) not in matrix_ids]
425
+ muon = MuonStep(matrices, device)
426
+ opt = torch.optim.AdamW(
427
+ auxiliary, lr=LR, weight_decay=WD, betas=(0.9, 0.95), fused=device == "cuda"
428
+ )
429
+ autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
430
+ model.train()
431
+ step, last_loss, records_seen = 0, float("nan"), 0
432
+ curve, recent, recent_aux, next_point = [], [], [], CURVE_EVERY
433
+ n_train = len(data) - VAL_N # records [0, VAL_N) are the val slice
434
+ rng, cuts, batches_per_epoch = random.Random(seed), [], None
435
+ order = torch.empty(0, dtype=torch.long, device=device)
436
+ while True:
437
+ frac = (
438
+ time.time() - t0
439
+ ) / budget_secs # cosine over wall-clock, so the cap is the schedule
440
+ if frac >= 1.0:
441
+ break
442
+ step += 1
443
+ # No warmup (as in modded-nanogpt): Muon's step is normalized and Adam's is ~lr per
444
+ # element from step 1, and Peri-LN normalizes every branch write.
445
+ lr = LR * MIN_LR_FRAC + 0.5 * LR * (1 - MIN_LR_FRAC) * (1 + math.cos(math.pi * frac))
446
+ for g in opt.param_groups:
447
+ g["lr"] = lr
448
+ if not cuts: # new epoch
449
+ order, cuts = token_batches(lens, n_train, gen, rng)
450
+ batches_per_epoch = len(cuts)
451
+ s, e, T = cuts.pop()
452
+ idxs = order[s:e]
453
+ records_seen += e - s
454
+ x, m = gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, T)
455
+ with autocast:
456
+ loss, main, aux = mtp_loss(model, mtp, x, m)
457
+ model.zero_grad(set_to_none=False) # grads keep fixed addresses for the Muon CUDA graph
458
+ mtp.zero_grad(set_to_none=False)
459
+ loss.backward()
460
+ torch.nn.utils.clip_grad_norm_(trainable, 1.0)
461
+ opt.step()
462
+ muon(MUON_LR * lr / LR)
463
+ if step % 20 == 0:
464
+ last_loss = main.item() # the next-token CE, comparable with earlier runs
465
+ recent.append(last_loss)
466
+ recent_aux.append(aux.item())
467
+ if (
468
+ time.time() - t0 >= next_point
469
+ ): # curve point: mean recent train loss + val loss on the held-out slice
470
+ curve.append(
471
+ {
472
+ "secs": round(time.time() - t0),
473
+ "step": step,
474
+ "train_loss": sum(recent) / len(recent) if recent else last_loss,
475
+ "val_loss": val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast),
476
+ "mtp_loss": sum(recent_aux) / len(recent_aux) if recent_aux else None,
477
+ }
478
+ )
479
+ log(
480
+ f"pretrain step {step} train {curve[-1]['train_loss']:.4f} "
481
+ f"val {curve[-1]['val_loss']:.4f} lr {lr:.2e} {curve[-1]['secs']}s"
482
+ )
483
+ recent, recent_aux, next_point = [], [], next_point + CURVE_EVERY
484
+ val_by_loops = {}
485
+ for n in VAL_LOOPS if step else (): # no steps: a checkpoint rebuild, not a training run
486
+ model.n_loops = n
487
+ val_by_loops[n] = val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast)
488
+ model.n_loops = DEFAULT_LOOPS
489
+ log(f"pretrain val loss by loops: {val_by_loops}")
490
+ for b in compiled:
491
+ b._compiled_call_impl = None
492
+ model.eval()
493
+ return (
494
+ model,
495
+ tok,
496
+ {
497
+ "steps": step,
498
+ "records_seen": records_seen,
499
+ "epochs": records_seen / n_train,
500
+ "batches_per_epoch": batches_per_epoch,
501
+ "tok_budget": TOK_BUDGET,
502
+ "last_loss": last_loss,
503
+ "train_loops": TRAIN_LOOPS,
504
+ "default_loops": DEFAULT_LOOPS,
505
+ "val_by_loops": val_by_loops,
506
+ "curve": curve,
507
+ },
508
+ )
509
+
510
+
511
+ def finetune(model, tok, data, rng, device):
512
+ """The RL fine-tune: pretraining's optimizers (Muon on the hidden matrices, AdamW on the
513
+ rest), linear warmup then cosine to lr/10, over FT_RECORDS records in token-budget batches
514
+ (to_gpu / token_batches / gpu_batch, as pretraining). Muon's momentum restarts every round,
515
+ as AdamW's moments did. The model is in train mode, so every batch draws its own loop count
516
+ from TRAIN_LOOPS. Returns (last loss, steps)."""
517
+ ids, mask, ref_max, lens = to_gpu(data, tok, device)
518
+ gen = torch.Generator(device=device)
519
+ gen.manual_seed(rng.getrandbits(31))
520
+ plan, seen = [], 0 # (order, start, end, padded length) per step, epoch after epoch
521
+ while seen < FT_RECORDS:
522
+ order, cuts = token_batches(lens, len(data), gen, rng, first=0)
523
+ for s, e, T in cuts:
524
+ if seen >= FT_RECORDS:
525
+ break
526
+ plan.append((order, s, e, T))
527
+ seen += e - s
528
+ matrices = [p for p in model.blocks.parameters() if p.ndim == 2] + [model.head.weight]
529
+ matrix_ids = {id(p) for p in matrices}
530
+ auxiliary = [p for p in model.parameters() if id(p) not in matrix_ids]
531
+ opt = torch.optim.AdamW(auxiliary, lr=FT_LR, weight_decay=0.1, betas=(0.9, 0.95))
532
+ muon = MuonStep(matrices, device)
533
+ autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
534
+ model.train()
535
+ for step, (order, s, e, T) in enumerate(plan, 1):
536
+ lr = (
537
+ FT_LR * step / FT_WARMUP
538
+ if step < FT_WARMUP
539
+ else FT_LR / 10 + 0.45 * FT_LR * (1 + math.cos(math.pi * step / len(plan)))
540
+ )
541
+ for g in opt.param_groups:
542
+ g["lr"] = lr
543
+ x, m = gpu_batch(ids, mask, ref_max, lens, order[s:e], tok, gen, T)
544
+ with autocast:
545
+ loss = loss_on(model, x, m)
546
+ model.zero_grad(set_to_none=False) # grads keep fixed addresses for the Muon CUDA graph
547
+ loss.backward()
548
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
549
+ opt.step()
550
+ muon(FT_MUON_LR * lr / FT_LR)
551
+ model.eval()
552
+ return loss.item(), len(plan)
553
+
554
+
555
+ def rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log):
556
+ rng = random.Random(seed)
557
+ found = {t["name"]: {} for t in targets} # name -> {normalized proof: proof text}
558
+ rounds, round_secs, retries, ft_steps = 0, 0.0, [], []
559
+ # Start a round only if one as long as the last (plus 15%) still fits in the budget.
560
+ while oracle.left() > 1.15 * round_secs:
561
+ t0 = oracle.used()
562
+ model.n_loops = RL_FIRST_LOOPS
563
+ rows = oracle.sample(model, targets, RL_K)
564
+ failed = [t for t, row in zip(targets, rows, strict=True) if not row["proofs"]]
565
+ rng.shuffle(failed)
566
+ for i, n in enumerate(RL_RETRY_LOOPS):
567
+ part = failed[i :: len(RL_RETRY_LOOPS)]
568
+ if not part:
569
+ continue
570
+ model.n_loops = n
571
+ part_rows = oracle.sample(model, part, RL_RETRY_K)
572
+ cracked = sum(bool(row["proofs"]) for row in part_rows)
573
+ retries.append(
574
+ {"round": rounds + 1, "loops": n, "targets": len(part), "cracked": cracked}
575
+ )
576
+ rows += part_rows
577
+ for row in rows:
578
+ for proof in row["proofs"]:
579
+ found[row["name"]].setdefault(norm(proof), proof)
580
+ mix = []
581
+ for t in targets:
582
+ proofs = list(found[t["name"]].values())
583
+ rng.shuffle(proofs)
584
+ mix += [(t["prompt"], proof) for proof in proofs[:MAX_PER_THM]] * RL_WEIGHT
585
+ n_rl = len(mix)
586
+ if not n_rl:
587
+ break
588
+ mix += [(r["prompt"], r["proof"]) for r in rng.sample(train_recs, RETAIN)]
589
+ data = [(tok.encode_prompt(p), tok.encode_proof(q)) for p, q in mix]
590
+ loss, steps = finetune(model, tok, data, rng, device)
591
+ ft_steps.append(steps)
592
+ rounds += 1
593
+ round_secs = oracle.used() - t0
594
+ log(
595
+ f"rl round {rounds}: {n_rl} RL records, fine-tune loss {loss:.4f} in {steps} steps, "
596
+ f"round {round_secs:.0f}s, {oracle.left():.0f}s left"
597
+ )
598
+ model.n_loops = DEFAULT_LOOPS
599
+ return {
600
+ "rounds": rounds,
601
+ "first_loops": RL_FIRST_LOOPS,
602
+ "k_first": RL_K,
603
+ "retry_loops": RL_RETRY_LOOPS,
604
+ "k_retry": RL_RETRY_K,
605
+ "retries": retries,
606
+ "ft_records": FT_RECORDS,
607
+ "ft_steps": ft_steps,
608
+ "ft_optimizer": "muon (polar express) on block matrices and head + adamw on the rest",
609
+ "ft_lr": FT_LR,
610
+ "ft_muon_lr": FT_MUON_LR,
611
+ }
runs/510-fable_more-shorter-rounds_s2/source/remote.sh ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ # Runs on the pod. Checks out Dan's fork at the pinned commit (frozen data + verifier + Lean
3
+ # tokenizer/gate), installs Lean, runs the harness.
4
+ # Usage: bash remote.sh <out-dir> [harness.py args...]
5
+ set -euo pipefail
6
+ FORK_COMMIT=51604b3feaa5e66821d1faba22238fe6d96d46d0 # dan-pandori/nd-takehome, branch dan_lean_seed2 (lean_seq with the BOTE fix)
7
+ export ND_FORK=/workspace/nd-takehome
8
+ OUT=$1; shift
9
+
10
+ if [ ! -d "$ND_FORK/.git" ]; then
11
+ git clone -q https://github.com/dan-pandori/nd-takehome.git "$ND_FORK"
12
+ fi
13
+ git -C "$ND_FORK" cat-file -e "$FORK_COMMIT" 2>/dev/null || git -C "$ND_FORK" fetch -q origin
14
+ git -C "$ND_FORK" checkout -q "$FORK_COMMIT"
15
+ [ -f "$ND_FORK/data/train.jsonl" ] || gunzip -k "$ND_FORK/data/train.jsonl.gz"
16
+ # Lean 4 core checks every sampled proof (fork lean_gate.py, called from sample.generate): the
17
+ # reward is Lean AND nd_verify. Installed before harness.py starts, so it is not charged to TOTAL_SECS.
18
+ [ -x ~/.elan/bin/lean ] || curl -sSf https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh \
19
+ | sh -s -- -y --default-toolchain leanprover/lean4:v4.34.0 > /dev/null
20
+ # One Lean process per CPU: on these pods (nproc 96) 12 workers took 146 s per RL-round gate
21
+ # pass (099) against 67-84 s with 96 workers (098).
22
+ export LEAN_GATE_WORKERS=$(nproc) LEAN_GATE_LOG="$OUT/lean_gate.jsonl"
23
+ nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
24
+ echo "cpus $(nproc), $(~/.elan/bin/lean --version)"
25
+ # Newer RunPod images put torch under `python` and leave `python3` as bare system Python.
26
+ PY=$(for p in python python3; do "$p" -c "import torch" 2>/dev/null && echo "$p" && break; done)
27
+ echo "python $($PY --version 2>&1), torch $($PY -c 'import torch; print(torch.__version__)')"
28
+ cd /workspace/autoresearch
29
+ $PY harness.py --out "$OUT" "$@" 2>&1 | tee "$OUT.log"
runs/510-fable_more-shorter-rounds_s2/summary.json ADDED
@@ -0,0 +1,2051 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "ok",
3
+ "metric": 734,
4
+ "metric_diag": 729,
5
+ "metric_rl": 1243,
6
+ "harness_sha": "2eec1b61666a",
7
+ "pretrain_sha": "6c50c444778d",
8
+ "device": "cuda",
9
+ "gpu": "NVIDIA A40",
10
+ "torch": "2.9.1+cu129",
11
+ "lean_gate": {
12
+ "samples": 432936,
13
+ "parse_fail": 107282,
14
+ "distinct_checked": 301445,
15
+ "both_ok": 136796,
16
+ "nd_ok_lean_rej": 0,
17
+ "nd_rej_lean_ok": 18,
18
+ "lean_wall_s": 964.8486626148224
19
+ },
20
+ "total_secs": 3930.6053726673126,
21
+ "rules": {
22
+ "TOTAL_SECS": 5400,
23
+ "PRETRAIN_SECS": 300,
24
+ "RL_SECS": 1800,
25
+ "PARAM_CAP": 10000000,
26
+ "N_TARGETS": 1500,
27
+ "EVAL_K": 64,
28
+ "KNOB": "n_loops",
29
+ "EVAL_LEVEL": 8,
30
+ "DIAG_LEVEL": 2,
31
+ "TOK_MODE": "lean_seq"
32
+ },
33
+ "pretrain": {
34
+ "steps": 3422,
35
+ "records_seen": 465120,
36
+ "epochs": 3.0209266981021785,
37
+ "batches_per_epoch": 1132,
38
+ "tok_budget": 18432,
39
+ "last_loss": 0.03159652650356293,
40
+ "train_loops": [
41
+ 1,
42
+ 2,
43
+ 3,
44
+ 4,
45
+ 5,
46
+ 6,
47
+ 7,
48
+ 8
49
+ ],
50
+ "default_loops": 2,
51
+ "val_by_loops": {
52
+ "1": 0.0941051784902811,
53
+ "2": 0.09122707694768906,
54
+ "4": 0.09095337800681591,
55
+ "8": 0.09084572643041611
56
+ },
57
+ "curve": [
58
+ {
59
+ "secs": 31,
60
+ "step": 1,
61
+ "train_loss": NaN,
62
+ "val_loss": 3.9639104306697845,
63
+ "mtp_loss": null
64
+ },
65
+ {
66
+ "secs": 34,
67
+ "step": 2,
68
+ "train_loss": NaN,
69
+ "val_loss": 3.456164538860321,
70
+ "mtp_loss": null
71
+ },
72
+ {
73
+ "secs": 45,
74
+ "step": 151,
75
+ "train_loss": 0.7055290639400482,
76
+ "val_loss": 0.13757031224668026,
77
+ "mtp_loss": 0.6738688903195518
78
+ },
79
+ {
80
+ "secs": 60,
81
+ "step": 344,
82
+ "train_loss": 0.12564483880996705,
83
+ "val_loss": 0.11503235530108213,
84
+ "mtp_loss": 0.12334308475255966
85
+ },
86
+ {
87
+ "secs": 75,
88
+ "step": 529,
89
+ "train_loss": 0.0807655511630906,
90
+ "val_loss": 0.0937309805303812,
91
+ "mtp_loss": 0.08277232986357477
92
+ },
93
+ {
94
+ "secs": 90,
95
+ "step": 724,
96
+ "train_loss": 0.0704643715173006,
97
+ "val_loss": 0.08781554270535707,
98
+ "mtp_loss": 0.07319770194590092
99
+ },
100
+ {
101
+ "secs": 105,
102
+ "step": 911,
103
+ "train_loss": 0.07867435697052214,
104
+ "val_loss": 0.07898464426398277,
105
+ "mtp_loss": 0.07781906674305598
106
+ },
107
+ {
108
+ "secs": 120,
109
+ "step": 1112,
110
+ "train_loss": 0.07457459047436714,
111
+ "val_loss": 0.08002733252942562,
112
+ "mtp_loss": 0.07482383698225022
113
+ },
114
+ {
115
+ "secs": 135,
116
+ "step": 1314,
117
+ "train_loss": 0.06505260448902846,
118
+ "val_loss": 0.07819176651537418,
119
+ "mtp_loss": 0.0670597344636917
120
+ },
121
+ {
122
+ "secs": 150,
123
+ "step": 1515,
124
+ "train_loss": 0.07578374780714511,
125
+ "val_loss": 0.08091318607330322,
126
+ "mtp_loss": 0.074917022138834
127
+ },
128
+ {
129
+ "secs": 165,
130
+ "step": 1705,
131
+ "train_loss": 0.07846935167908668,
132
+ "val_loss": 0.08359375037252903,
133
+ "mtp_loss": 0.07880706004798413
134
+ },
135
+ {
136
+ "secs": 180,
137
+ "step": 1899,
138
+ "train_loss": 0.076803477274047,
139
+ "val_loss": 0.08226866368204355,
140
+ "mtp_loss": 0.07677057592405213
141
+ },
142
+ {
143
+ "secs": 195,
144
+ "step": 2087,
145
+ "train_loss": 0.062361786141991615,
146
+ "val_loss": 0.08998655900359154,
147
+ "mtp_loss": 0.06061466410756111
148
+ },
149
+ {
150
+ "secs": 210,
151
+ "step": 2279,
152
+ "train_loss": 0.05334144292606248,
153
+ "val_loss": 0.09138383623212576,
154
+ "mtp_loss": 0.05310566474994024
155
+ },
156
+ {
157
+ "secs": 225,
158
+ "step": 2463,
159
+ "train_loss": 0.07295445762574673,
160
+ "val_loss": 0.08277123048901558,
161
+ "mtp_loss": 0.07242496199905872
162
+ },
163
+ {
164
+ "secs": 240,
165
+ "step": 2657,
166
+ "train_loss": 0.07529298547241423,
167
+ "val_loss": 0.07695397827774286,
168
+ "mtp_loss": 0.0754928630259302
169
+ },
170
+ {
171
+ "secs": 255,
172
+ "step": 2843,
173
+ "train_loss": 0.05310075879096985,
174
+ "val_loss": 0.08155703824013472,
175
+ "mtp_loss": 0.052278506383299825
176
+ },
177
+ {
178
+ "secs": 270,
179
+ "step": 3038,
180
+ "train_loss": 0.061006077047851354,
181
+ "val_loss": 0.08341224864125252,
182
+ "mtp_loss": 0.061530229532056384
183
+ },
184
+ {
185
+ "secs": 285,
186
+ "step": 3229,
187
+ "train_loss": 0.06030359528958797,
188
+ "val_loss": 0.07512469217181206,
189
+ "mtp_loss": 0.06038339361548424
190
+ },
191
+ {
192
+ "secs": 300,
193
+ "step": 3422,
194
+ "train_loss": 0.04641678910702467,
195
+ "val_loss": 0.09122707694768906,
196
+ "mtp_loss": 0.0466538218781352
197
+ }
198
+ ],
199
+ "secs": 301.030366897583,
200
+ "n_params": 6400256,
201
+ "heldout_greedy": 0.9645
202
+ },
203
+ "rl": {
204
+ "rounds": [
205
+ {
206
+ "round": 1,
207
+ "level": 1,
208
+ "k": 8,
209
+ "n_targets": 1500,
210
+ "sample_acc": 0.06116666666666667,
211
+ "new_proofs": 219,
212
+ "targets_solved": 183,
213
+ "lstar": 8,
214
+ "ge": {
215
+ "7": 183,
216
+ "8": 27,
217
+ "9": 2,
218
+ "10": 0,
219
+ "11": 0,
220
+ "12": 0,
221
+ "13": 0,
222
+ "14": 0
223
+ },
224
+ "mix_rl_records": 219,
225
+ "sampling": {
226
+ "samples": 12000,
227
+ "secs": 55.99422359466553,
228
+ "gate_secs": 22.379364252090454,
229
+ "samples_per_sec": 214.3078201577782
230
+ },
231
+ "secs": 56.33411383628845,
232
+ "rl_secs_used": 33.9550507068634
233
+ },
234
+ {
235
+ "round": 2,
236
+ "level": 2,
237
+ "k": 24,
238
+ "n_targets": 439,
239
+ "sample_acc": 0.014236902050113895,
240
+ "new_proofs": 58,
241
+ "targets_solved": 220,
242
+ "lstar": 9,
243
+ "ge": {
244
+ "7": 220,
245
+ "8": 46,
246
+ "9": 11,
247
+ "10": 0,
248
+ "11": 0,
249
+ "12": 0,
250
+ "13": 0,
251
+ "14": 0
252
+ },
253
+ "mix_rl_records": 58,
254
+ "sampling": {
255
+ "samples": 10536,
256
+ "secs": 56.80263304710388,
257
+ "gate_secs": 18.66281533241272,
258
+ "samples_per_sec": 185.4843593476198
259
+ },
260
+ "secs": 57.08036184310913,
261
+ "rl_secs_used": 72.37240147590637
262
+ },
263
+ {
264
+ "round": 3,
265
+ "level": 4,
266
+ "k": 24,
267
+ "n_targets": 439,
268
+ "sample_acc": 0.023253606681852695,
269
+ "new_proofs": 79,
270
+ "targets_solved": 277,
271
+ "lstar": 9,
272
+ "ge": {
273
+ "7": 277,
274
+ "8": 67,
275
+ "9": 20,
276
+ "10": 1,
277
+ "11": 0,
278
+ "12": 0,
279
+ "13": 0,
280
+ "14": 0
281
+ },
282
+ "mix_rl_records": 79,
283
+ "sampling": {
284
+ "samples": 10536,
285
+ "secs": 83.33086013793945,
286
+ "gate_secs": 15.40415620803833,
287
+ "samples_per_sec": 126.4357524038456
288
+ },
289
+ "secs": 83.58716893196106,
290
+ "rl_secs_used": 140.5553755760193
291
+ },
292
+ {
293
+ "round": 4,
294
+ "level": 8,
295
+ "k": 24,
296
+ "n_targets": 439,
297
+ "sample_acc": 0.03673120728929385,
298
+ "new_proofs": 97,
299
+ "targets_solved": 343,
300
+ "lstar": 9,
301
+ "ge": {
302
+ "7": 343,
303
+ "8": 101,
304
+ "9": 29,
305
+ "10": 2,
306
+ "11": 0,
307
+ "12": 0,
308
+ "13": 0,
309
+ "14": 0
310
+ },
311
+ "mix_rl_records": 97,
312
+ "sampling": {
313
+ "samples": 10536,
314
+ "secs": 121.83794355392456,
315
+ "gate_secs": 16.31764030456543,
316
+ "samples_per_sec": 86.47552390226322
317
+ },
318
+ "secs": 122.10132646560669,
319
+ "rl_secs_used": 246.33908772468567
320
+ },
321
+ {
322
+ "round": 5,
323
+ "level": 1,
324
+ "k": 8,
325
+ "n_targets": 1500,
326
+ "sample_acc": 0.29183333333333333,
327
+ "new_proofs": 490,
328
+ "targets_solved": 666,
329
+ "lstar": 10,
330
+ "ge": {
331
+ "7": 666,
332
+ "8": 265,
333
+ "9": 81,
334
+ "10": 10,
335
+ "11": 0,
336
+ "12": 0,
337
+ "13": 0,
338
+ "14": 0
339
+ },
340
+ "mix_rl_records": 855,
341
+ "sampling": {
342
+ "samples": 12000,
343
+ "secs": 53.70738339424133,
344
+ "gate_secs": 26.267099142074585,
345
+ "samples_per_sec": 223.4329665981582
346
+ },
347
+ "secs": 82.9831690788269,
348
+ "rl_secs_used": 303.05529141426086
349
+ },
350
+ {
351
+ "round": 6,
352
+ "level": 2,
353
+ "k": 24,
354
+ "n_targets": 284,
355
+ "sample_acc": 0.03198356807511737,
356
+ "new_proofs": 53,
357
+ "targets_solved": 712,
358
+ "lstar": 10,
359
+ "ge": {
360
+ "7": 712,
361
+ "8": 293,
362
+ "9": 99,
363
+ "10": 11,
364
+ "11": 0,
365
+ "12": 0,
366
+ "13": 0,
367
+ "14": 0
368
+ },
369
+ "mix_rl_records": 58,
370
+ "sampling": {
371
+ "samples": 6816,
372
+ "secs": 35.301345109939575,
373
+ "gate_secs": 16.652695178985596,
374
+ "samples_per_sec": 193.08046134709076
375
+ },
376
+ "secs": 35.53109169006348,
377
+ "rl_secs_used": 321.9335308074951
378
+ },
379
+ {
380
+ "round": 7,
381
+ "level": 4,
382
+ "k": 24,
383
+ "n_targets": 284,
384
+ "sample_acc": 0.05443075117370892,
385
+ "new_proofs": 74,
386
+ "targets_solved": 765,
387
+ "lstar": 10,
388
+ "ge": {
389
+ "7": 765,
390
+ "8": 332,
391
+ "9": 125,
392
+ "10": 18,
393
+ "11": 0,
394
+ "12": 0,
395
+ "13": 0,
396
+ "14": 0
397
+ },
398
+ "mix_rl_records": 83,
399
+ "sampling": {
400
+ "samples": 6816,
401
+ "secs": 45.6511390209198,
402
+ "gate_secs": 15.481172800064087,
403
+ "samples_per_sec": 149.3062417758414
404
+ },
405
+ "secs": 45.87594819068909,
406
+ "rl_secs_used": 352.32831025123596
407
+ },
408
+ {
409
+ "round": 8,
410
+ "level": 8,
411
+ "k": 24,
412
+ "n_targets": 283,
413
+ "sample_acc": 0.06448763250883392,
414
+ "new_proofs": 76,
415
+ "targets_solved": 819,
416
+ "lstar": 10,
417
+ "ge": {
418
+ "7": 819,
419
+ "8": 362,
420
+ "9": 145,
421
+ "10": 20,
422
+ "11": 0,
423
+ "12": 0,
424
+ "13": 0,
425
+ "14": 0
426
+ },
427
+ "mix_rl_records": 85,
428
+ "sampling": {
429
+ "samples": 6792,
430
+ "secs": 79.50895357131958,
431
+ "gate_secs": 14.960362195968628,
432
+ "samples_per_sec": 85.42434147253078
433
+ },
434
+ "secs": 79.72172689437866,
435
+ "rl_secs_used": 417.08966732025146
436
+ },
437
+ {
438
+ "round": 9,
439
+ "level": 1,
440
+ "k": 8,
441
+ "n_targets": 1500,
442
+ "sample_acc": 0.46066666666666667,
443
+ "new_proofs": 401,
444
+ "targets_solved": 951,
445
+ "lstar": 10,
446
+ "ge": {
447
+ "7": 951,
448
+ "8": 469,
449
+ "9": 221,
450
+ "10": 39,
451
+ "11": 2,
452
+ "12": 1,
453
+ "13": 0,
454
+ "14": 0
455
+ },
456
+ "mix_rl_records": 1348,
457
+ "sampling": {
458
+ "samples": 12000,
459
+ "secs": 58.47705101966858,
460
+ "gate_secs": 29.40255880355835,
461
+ "samples_per_sec": 205.20870650546036
462
+ },
463
+ "secs": 90.66570830345154,
464
+ "rl_secs_used": 478.3530042171478
465
+ },
466
+ {
467
+ "round": 10,
468
+ "level": 2,
469
+ "k": 24,
470
+ "n_targets": 198,
471
+ "sample_acc": 0.06776094276094276,
472
+ "new_proofs": 47,
473
+ "targets_solved": 983,
474
+ "lstar": 10,
475
+ "ge": {
476
+ "7": 983,
477
+ "8": 496,
478
+ "9": 242,
479
+ "10": 45,
480
+ "11": 4,
481
+ "12": 1,
482
+ "13": 0,
483
+ "14": 0
484
+ },
485
+ "mix_rl_records": 62,
486
+ "sampling": {
487
+ "samples": 4752,
488
+ "secs": 28.847393035888672,
489
+ "gate_secs": 14.21317744255066,
490
+ "samples_per_sec": 164.7289234797785
491
+ },
492
+ "secs": 29.032156705856323,
493
+ "rl_secs_used": 493.17177987098694
494
+ },
495
+ {
496
+ "round": 11,
497
+ "level": 4,
498
+ "k": 24,
499
+ "n_targets": 198,
500
+ "sample_acc": 0.08712121212121213,
501
+ "new_proofs": 52,
502
+ "targets_solved": 1014,
503
+ "lstar": 11,
504
+ "ge": {
505
+ "7": 1014,
506
+ "8": 521,
507
+ "9": 264,
508
+ "10": 53,
509
+ "11": 7,
510
+ "12": 1,
511
+ "13": 0,
512
+ "14": 0
513
+ },
514
+ "mix_rl_records": 64,
515
+ "sampling": {
516
+ "samples": 4752,
517
+ "secs": 36.4263436794281,
518
+ "gate_secs": 13.217883586883545,
519
+ "samples_per_sec": 130.45503665754157
520
+ },
521
+ "secs": 36.618452310562134,
522
+ "rl_secs_used": 516.5723531246185
523
+ },
524
+ {
525
+ "round": 12,
526
+ "level": 8,
527
+ "k": 24,
528
+ "n_targets": 197,
529
+ "sample_acc": 0.10596446700507614,
530
+ "new_proofs": 62,
531
+ "targets_solved": 1046,
532
+ "lstar": 11,
533
+ "ge": {
534
+ "7": 1046,
535
+ "8": 544,
536
+ "9": 285,
537
+ "10": 58,
538
+ "11": 8,
539
+ "12": 1,
540
+ "13": 0,
541
+ "14": 0
542
+ },
543
+ "mix_rl_records": 88,
544
+ "sampling": {
545
+ "samples": 4728,
546
+ "secs": 54.11488699913025,
547
+ "gate_secs": 14.088384866714478,
548
+ "samples_per_sec": 87.36967334101594
549
+ },
550
+ "secs": 54.30283308029175,
551
+ "rl_secs_used": 556.7867941856384
552
+ },
553
+ {
554
+ "round": 13,
555
+ "level": 1,
556
+ "k": 8,
557
+ "n_targets": 1500,
558
+ "sample_acc": 0.5320833333333334,
559
+ "new_proofs": 320,
560
+ "targets_solved": 1065,
561
+ "lstar": 11,
562
+ "ge": {
563
+ "7": 1065,
564
+ "8": 556,
565
+ "9": 293,
566
+ "10": 61,
567
+ "11": 8,
568
+ "12": 1,
569
+ "13": 0,
570
+ "14": 0
571
+ },
572
+ "mix_rl_records": 1606,
573
+ "sampling": {
574
+ "samples": 12000,
575
+ "secs": 59.93379306793213,
576
+ "gate_secs": 30.641193151474,
577
+ "samples_per_sec": 200.22093356244892
578
+ },
579
+ "secs": 93.84149837493896,
580
+ "rl_secs_used": 619.9873306751251
581
+ },
582
+ {
583
+ "round": 14,
584
+ "level": 2,
585
+ "k": 24,
586
+ "n_targets": 164,
587
+ "sample_acc": 0.04928861788617886,
588
+ "new_proofs": 33,
589
+ "targets_solved": 1084,
590
+ "lstar": 11,
591
+ "ge": {
592
+ "7": 1084,
593
+ "8": 568,
594
+ "9": 300,
595
+ "10": 65,
596
+ "11": 9,
597
+ "12": 1,
598
+ "13": 0,
599
+ "14": 0
600
+ },
601
+ "mix_rl_records": 51,
602
+ "sampling": {
603
+ "samples": 3936,
604
+ "secs": 23.673223733901978,
605
+ "gate_secs": 11.700016021728516,
606
+ "samples_per_sec": 166.26379424460592
607
+ },
608
+ "secs": 23.824730396270752,
609
+ "rl_secs_used": 632.1118004322052
610
+ },
611
+ {
612
+ "round": 15,
613
+ "level": 4,
614
+ "k": 24,
615
+ "n_targets": 164,
616
+ "sample_acc": 0.09730691056910569,
617
+ "new_proofs": 38,
618
+ "targets_solved": 1103,
619
+ "lstar": 11,
620
+ "ge": {
621
+ "7": 1103,
622
+ "8": 582,
623
+ "9": 310,
624
+ "10": 71,
625
+ "11": 12,
626
+ "12": 1,
627
+ "13": 0,
628
+ "14": 0
629
+ },
630
+ "mix_rl_records": 64,
631
+ "sampling": {
632
+ "samples": 3936,
633
+ "secs": 33.744935512542725,
634
+ "gate_secs": 11.280772924423218,
635
+ "samples_per_sec": 116.63972504961582
636
+ },
637
+ "secs": 33.89532399177551,
638
+ "rl_secs_used": 654.7263526916504
639
+ },
640
+ {
641
+ "round": 16,
642
+ "level": 8,
643
+ "k": 24,
644
+ "n_targets": 163,
645
+ "sample_acc": 0.09458077709611452,
646
+ "new_proofs": 47,
647
+ "targets_solved": 1125,
648
+ "lstar": 11,
649
+ "ge": {
650
+ "7": 1125,
651
+ "8": 601,
652
+ "9": 327,
653
+ "10": 75,
654
+ "11": 13,
655
+ "12": 1,
656
+ "13": 0,
657
+ "14": 0
658
+ },
659
+ "mix_rl_records": 68,
660
+ "sampling": {
661
+ "samples": 3912,
662
+ "secs": 48.00184774398804,
663
+ "gate_secs": 11.659076929092407,
664
+ "samples_per_sec": 81.49686280545224
665
+ },
666
+ "secs": 48.16022706031799,
667
+ "rl_secs_used": 691.2275061607361
668
+ },
669
+ {
670
+ "round": 17,
671
+ "level": 1,
672
+ "k": 8,
673
+ "n_targets": 1500,
674
+ "sample_acc": 0.5685,
675
+ "new_proofs": 394,
676
+ "targets_solved": 1135,
677
+ "lstar": 11,
678
+ "ge": {
679
+ "7": 1135,
680
+ "8": 610,
681
+ "9": 334,
682
+ "10": 77,
683
+ "11": 13,
684
+ "12": 1,
685
+ "13": 0,
686
+ "14": 0
687
+ },
688
+ "mix_rl_records": 1893,
689
+ "sampling": {
690
+ "samples": 12000,
691
+ "secs": 57.32696509361267,
692
+ "gate_secs": 30.90991187095642,
693
+ "samples_per_sec": 209.3255761996902
694
+ },
695
+ "secs": 92.28701949119568,
696
+ "rl_secs_used": 752.6048817634583
697
+ },
698
+ {
699
+ "round": 18,
700
+ "level": 2,
701
+ "k": 24,
702
+ "n_targets": 144,
703
+ "sample_acc": 0.09866898148148148,
704
+ "new_proofs": 36,
705
+ "targets_solved": 1145,
706
+ "lstar": 11,
707
+ "ge": {
708
+ "7": 1145,
709
+ "8": 617,
710
+ "9": 338,
711
+ "10": 79,
712
+ "11": 13,
713
+ "12": 1,
714
+ "13": 0,
715
+ "14": 0
716
+ },
717
+ "mix_rl_records": 68,
718
+ "sampling": {
719
+ "samples": 3456,
720
+ "secs": 21.91321849822998,
721
+ "gate_secs": 9.878704309463501,
722
+ "samples_per_sec": 157.71302605681385
723
+ },
724
+ "secs": 22.0466730594635,
725
+ "rl_secs_used": 764.7725718021393
726
+ },
727
+ {
728
+ "round": 19,
729
+ "level": 4,
730
+ "k": 24,
731
+ "n_targets": 143,
732
+ "sample_acc": 0.10110722610722611,
733
+ "new_proofs": 21,
734
+ "targets_solved": 1155,
735
+ "lstar": 11,
736
+ "ge": {
737
+ "7": 1155,
738
+ "8": 623,
739
+ "9": 342,
740
+ "10": 80,
741
+ "11": 13,
742
+ "12": 1,
743
+ "13": 0,
744
+ "14": 0
745
+ },
746
+ "mix_rl_records": 40,
747
+ "sampling": {
748
+ "samples": 3432,
749
+ "secs": 28.827359914779663,
750
+ "gate_secs": 9.52054762840271,
751
+ "samples_per_sec": 119.0535661311263
752
+ },
753
+ "secs": 28.953237295150757,
754
+ "rl_secs_used": 784.2052631378174
755
+ },
756
+ {
757
+ "round": 20,
758
+ "level": 8,
759
+ "k": 24,
760
+ "n_targets": 143,
761
+ "sample_acc": 0.11013986013986014,
762
+ "new_proofs": 26,
763
+ "targets_solved": 1162,
764
+ "lstar": 11,
765
+ "ge": {
766
+ "7": 1162,
767
+ "8": 628,
768
+ "9": 347,
769
+ "10": 80,
770
+ "11": 13,
771
+ "12": 1,
772
+ "13": 0,
773
+ "14": 0
774
+ },
775
+ "mix_rl_records": 59,
776
+ "sampling": {
777
+ "samples": 3432,
778
+ "secs": 46.35999059677124,
779
+ "gate_secs": 8.765077114105225,
780
+ "samples_per_sec": 74.02935064958842
781
+ },
782
+ "secs": 46.48157525062561,
783
+ "rl_secs_used": 821.9217636585236
784
+ },
785
+ {
786
+ "round": 21,
787
+ "level": 1,
788
+ "k": 8,
789
+ "n_targets": 1500,
790
+ "sample_acc": 0.5968333333333333,
791
+ "new_proofs": 413,
792
+ "targets_solved": 1173,
793
+ "lstar": 11,
794
+ "ge": {
795
+ "7": 1173,
796
+ "8": 636,
797
+ "9": 354,
798
+ "10": 81,
799
+ "11": 13,
800
+ "12": 1,
801
+ "13": 0,
802
+ "14": 0
803
+ },
804
+ "mix_rl_records": 2128,
805
+ "sampling": {
806
+ "samples": 12000,
807
+ "secs": 58.088133573532104,
808
+ "gate_secs": 31.905141353607178,
809
+ "samples_per_sec": 206.5826402359708
810
+ },
811
+ "secs": 95.09784436225891,
812
+ "rl_secs_used": 885.1146233081818
813
+ },
814
+ {
815
+ "round": 22,
816
+ "level": 2,
817
+ "k": 24,
818
+ "n_targets": 126,
819
+ "sample_acc": 0.06944444444444445,
820
+ "new_proofs": 33,
821
+ "targets_solved": 1177,
822
+ "lstar": 11,
823
+ "ge": {
824
+ "7": 1177,
825
+ "8": 637,
826
+ "9": 354,
827
+ "10": 81,
828
+ "11": 13,
829
+ "12": 1,
830
+ "13": 0,
831
+ "14": 0
832
+ },
833
+ "mix_rl_records": 56,
834
+ "sampling": {
835
+ "samples": 3024,
836
+ "secs": 20.099300622940063,
837
+ "gate_secs": 8.851890087127686,
838
+ "samples_per_sec": 150.45299618776778
839
+ },
840
+ "secs": 20.21461582183838,
841
+ "rl_secs_used": 896.4771890640259
842
+ },
843
+ {
844
+ "round": 23,
845
+ "level": 4,
846
+ "k": 24,
847
+ "n_targets": 125,
848
+ "sample_acc": 0.095,
849
+ "new_proofs": 24,
850
+ "targets_solved": 1181,
851
+ "lstar": 11,
852
+ "ge": {
853
+ "7": 1181,
854
+ "8": 639,
855
+ "9": 356,
856
+ "10": 81,
857
+ "11": 13,
858
+ "12": 1,
859
+ "13": 0,
860
+ "14": 0
861
+ },
862
+ "mix_rl_records": 51,
863
+ "sampling": {
864
+ "samples": 3000,
865
+ "secs": 25.22296643257141,
866
+ "gate_secs": 8.851625919342041,
867
+ "samples_per_sec": 118.93922184053584
868
+ },
869
+ "secs": 25.34110951423645,
870
+ "rl_secs_used": 912.9666712284088
871
+ },
872
+ {
873
+ "round": 24,
874
+ "level": 8,
875
+ "k": 24,
876
+ "n_targets": 125,
877
+ "sample_acc": 0.059,
878
+ "new_proofs": 22,
879
+ "targets_solved": 1186,
880
+ "lstar": 11,
881
+ "ge": {
882
+ "7": 1186,
883
+ "8": 643,
884
+ "9": 360,
885
+ "10": 82,
886
+ "11": 13,
887
+ "12": 1,
888
+ "13": 0,
889
+ "14": 0
890
+ },
891
+ "mix_rl_records": 44,
892
+ "sampling": {
893
+ "samples": 3000,
894
+ "secs": 42.12534427642822,
895
+ "gate_secs": 8.774938821792603,
896
+ "samples_per_sec": 71.2160351809561
897
+ },
898
+ "secs": 42.24040746688843,
899
+ "rl_secs_used": 946.4321458339691
900
+ },
901
+ {
902
+ "round": 25,
903
+ "level": 1,
904
+ "k": 8,
905
+ "n_targets": 1500,
906
+ "sample_acc": 0.5979166666666667,
907
+ "new_proofs": 523,
908
+ "targets_solved": 1191,
909
+ "lstar": 11,
910
+ "ge": {
911
+ "7": 1191,
912
+ "8": 647,
913
+ "9": 362,
914
+ "10": 83,
915
+ "11": 13,
916
+ "12": 1,
917
+ "13": 0,
918
+ "14": 0
919
+ },
920
+ "mix_rl_records": 2374,
921
+ "sampling": {
922
+ "samples": 12000,
923
+ "secs": 61.237125396728516,
924
+ "gate_secs": 32.01392698287964,
925
+ "samples_per_sec": 195.959557576507
926
+ },
927
+ "secs": 98.70070838928223,
928
+ "rl_secs_used": 1013.1191744804382
929
+ },
930
+ {
931
+ "round": 26,
932
+ "level": 2,
933
+ "k": 24,
934
+ "n_targets": 119,
935
+ "sample_acc": 0.0584733893557423,
936
+ "new_proofs": 7,
937
+ "targets_solved": 1193,
938
+ "lstar": 11,
939
+ "ge": {
940
+ "7": 1193,
941
+ "8": 648,
942
+ "9": 363,
943
+ "10": 84,
944
+ "11": 13,
945
+ "12": 1,
946
+ "13": 0,
947
+ "14": 0
948
+ },
949
+ "mix_rl_records": 30,
950
+ "sampling": {
951
+ "samples": 2856,
952
+ "secs": 18.311748027801514,
953
+ "gate_secs": 8.023667573928833,
954
+ "samples_per_sec": 155.96544882902083
955
+ },
956
+ "secs": 18.418205499649048,
957
+ "rl_secs_used": 1023.5134658813477
958
+ },
959
+ {
960
+ "round": 27,
961
+ "level": 4,
962
+ "k": 24,
963
+ "n_targets": 118,
964
+ "sample_acc": 0.08333333333333333,
965
+ "new_proofs": 25,
966
+ "targets_solved": 1200,
967
+ "lstar": 11,
968
+ "ge": {
969
+ "7": 1200,
970
+ "8": 655,
971
+ "9": 369,
972
+ "10": 85,
973
+ "11": 13,
974
+ "12": 1,
975
+ "13": 0,
976
+ "14": 0
977
+ },
978
+ "mix_rl_records": 49,
979
+ "sampling": {
980
+ "samples": 2832,
981
+ "secs": 26.172778844833374,
982
+ "gate_secs": 7.465281963348389,
983
+ "samples_per_sec": 108.20402437164404
984
+ },
985
+ "secs": 26.27826166152954,
986
+ "rl_secs_used": 1042.3264372348785
987
+ },
988
+ {
989
+ "round": 28,
990
+ "level": 8,
991
+ "k": 24,
992
+ "n_targets": 118,
993
+ "sample_acc": 0.08721751412429378,
994
+ "new_proofs": 31,
995
+ "targets_solved": 1204,
996
+ "lstar": 11,
997
+ "ge": {
998
+ "7": 1204,
999
+ "8": 659,
1000
+ "9": 373,
1001
+ "10": 87,
1002
+ "11": 13,
1003
+ "12": 1,
1004
+ "13": 0,
1005
+ "14": 0
1006
+ },
1007
+ "mix_rl_records": 65,
1008
+ "sampling": {
1009
+ "samples": 2832,
1010
+ "secs": 40.27161502838135,
1011
+ "gate_secs": 7.343782663345337,
1012
+ "samples_per_sec": 70.32248391340038
1013
+ },
1014
+ "secs": 40.37286067008972,
1015
+ "rl_secs_used": 1075.355520248413
1016
+ },
1017
+ {
1018
+ "round": 29,
1019
+ "level": 1,
1020
+ "k": 8,
1021
+ "n_targets": 1500,
1022
+ "sample_acc": 0.624,
1023
+ "new_proofs": 539,
1024
+ "targets_solved": 1206,
1025
+ "lstar": 11,
1026
+ "ge": {
1027
+ "7": 1206,
1028
+ "8": 661,
1029
+ "9": 375,
1030
+ "10": 87,
1031
+ "11": 13,
1032
+ "12": 1,
1033
+ "13": 0,
1034
+ "14": 0
1035
+ },
1036
+ "mix_rl_records": 2632,
1037
+ "sampling": {
1038
+ "samples": 12000,
1039
+ "secs": 58.79963517189026,
1040
+ "gate_secs": 30.9317786693573,
1041
+ "samples_per_sec": 204.08289889758905
1042
+ },
1043
+ "secs": 97.54042410850525,
1044
+ "rl_secs_used": 1141.964429616928
1045
+ },
1046
+ {
1047
+ "round": 30,
1048
+ "level": 2,
1049
+ "k": 24,
1050
+ "n_targets": 111,
1051
+ "sample_acc": 0.05292792792792793,
1052
+ "new_proofs": 15,
1053
+ "targets_solved": 1209,
1054
+ "lstar": 11,
1055
+ "ge": {
1056
+ "7": 1209,
1057
+ "8": 662,
1058
+ "9": 376,
1059
+ "10": 87,
1060
+ "11": 13,
1061
+ "12": 1,
1062
+ "13": 0,
1063
+ "14": 0
1064
+ },
1065
+ "mix_rl_records": 29,
1066
+ "sampling": {
1067
+ "samples": 2664,
1068
+ "secs": 17.711304903030396,
1069
+ "gate_secs": 7.098416090011597,
1070
+ "samples_per_sec": 150.41240691103403
1071
+ },
1072
+ "secs": 17.81486439704895,
1073
+ "rl_secs_used": 1152.680609703064
1074
+ },
1075
+ {
1076
+ "round": 31,
1077
+ "level": 4,
1078
+ "k": 24,
1079
+ "n_targets": 111,
1080
+ "sample_acc": 0.06043543543543543,
1081
+ "new_proofs": 18,
1082
+ "targets_solved": 1211,
1083
+ "lstar": 11,
1084
+ "ge": {
1085
+ "7": 1211,
1086
+ "8": 664,
1087
+ "9": 377,
1088
+ "10": 88,
1089
+ "11": 13,
1090
+ "12": 1,
1091
+ "13": 0,
1092
+ "14": 0
1093
+ },
1094
+ "mix_rl_records": 37,
1095
+ "sampling": {
1096
+ "samples": 2664,
1097
+ "secs": 24.180163860321045,
1098
+ "gate_secs": 7.42961311340332,
1099
+ "samples_per_sec": 110.17295066273506
1100
+ },
1101
+ "secs": 24.279335498809814,
1102
+ "rl_secs_used": 1169.5303373336792
1103
+ },
1104
+ {
1105
+ "round": 32,
1106
+ "level": 8,
1107
+ "k": 24,
1108
+ "n_targets": 111,
1109
+ "sample_acc": 0.08483483483483484,
1110
+ "new_proofs": 13,
1111
+ "targets_solved": 1216,
1112
+ "lstar": 11,
1113
+ "ge": {
1114
+ "7": 1216,
1115
+ "8": 669,
1116
+ "9": 381,
1117
+ "10": 91,
1118
+ "11": 15,
1119
+ "12": 1,
1120
+ "13": 0,
1121
+ "14": 0
1122
+ },
1123
+ "mix_rl_records": 33,
1124
+ "sampling": {
1125
+ "samples": 2664,
1126
+ "secs": 38.277426958084106,
1127
+ "gate_secs": 7.4696879386901855,
1128
+ "samples_per_sec": 69.59715455579673
1129
+ },
1130
+ "secs": 38.376280069351196,
1131
+ "rl_secs_used": 1200.436930656433
1132
+ },
1133
+ {
1134
+ "round": 33,
1135
+ "level": 1,
1136
+ "k": 8,
1137
+ "n_targets": 1500,
1138
+ "sample_acc": 0.6454166666666666,
1139
+ "new_proofs": 590,
1140
+ "targets_solved": 1220,
1141
+ "lstar": 11,
1142
+ "ge": {
1143
+ "7": 1220,
1144
+ "8": 671,
1145
+ "9": 383,
1146
+ "10": 92,
1147
+ "11": 16,
1148
+ "12": 2,
1149
+ "13": 0,
1150
+ "14": 0
1151
+ },
1152
+ "mix_rl_records": 2861,
1153
+ "sampling": {
1154
+ "samples": 12000,
1155
+ "secs": 59.769734144210815,
1156
+ "gate_secs": 29.800268173217773,
1157
+ "samples_per_sec": 200.77050988794298
1158
+ },
1159
+ "secs": 99.65524959564209,
1160
+ "rl_secs_used": 1270.292156457901
1161
+ },
1162
+ {
1163
+ "round": 34,
1164
+ "level": 2,
1165
+ "k": 24,
1166
+ "n_targets": 105,
1167
+ "sample_acc": 0.05198412698412699,
1168
+ "new_proofs": 17,
1169
+ "targets_solved": 1223,
1170
+ "lstar": 11,
1171
+ "ge": {
1172
+ "7": 1223,
1173
+ "8": 674,
1174
+ "9": 385,
1175
+ "10": 92,
1176
+ "11": 16,
1177
+ "12": 2,
1178
+ "13": 0,
1179
+ "14": 0
1180
+ },
1181
+ "mix_rl_records": 39,
1182
+ "sampling": {
1183
+ "samples": 2520,
1184
+ "secs": 17.360772371292114,
1185
+ "gate_secs": 6.7075355052948,
1186
+ "samples_per_sec": 145.1548321759629
1187
+ },
1188
+ "secs": 17.450846672058105,
1189
+ "rl_secs_used": 1281.0352182388306
1190
+ },
1191
+ {
1192
+ "round": 35,
1193
+ "level": 4,
1194
+ "k": 24,
1195
+ "n_targets": 105,
1196
+ "sample_acc": 0.06031746031746032,
1197
+ "new_proofs": 17,
1198
+ "targets_solved": 1225,
1199
+ "lstar": 11,
1200
+ "ge": {
1201
+ "7": 1225,
1202
+ "8": 676,
1203
+ "9": 387,
1204
+ "10": 93,
1205
+ "11": 17,
1206
+ "12": 2,
1207
+ "13": 0,
1208
+ "14": 0
1209
+ },
1210
+ "mix_rl_records": 39,
1211
+ "sampling": {
1212
+ "samples": 2520,
1213
+ "secs": 23.394718885421753,
1214
+ "gate_secs": 6.456631898880005,
1215
+ "samples_per_sec": 107.71661811120626
1216
+ },
1217
+ "secs": 23.48165273666382,
1218
+ "rl_secs_used": 1298.0602369308472
1219
+ },
1220
+ {
1221
+ "round": 36,
1222
+ "level": 8,
1223
+ "k": 24,
1224
+ "n_targets": 105,
1225
+ "sample_acc": 0.05396825396825397,
1226
+ "new_proofs": 13,
1227
+ "targets_solved": 1227,
1228
+ "lstar": 11,
1229
+ "ge": {
1230
+ "7": 1227,
1231
+ "8": 678,
1232
+ "9": 388,
1233
+ "10": 93,
1234
+ "11": 17,
1235
+ "12": 2,
1236
+ "13": 0,
1237
+ "14": 0
1238
+ },
1239
+ "mix_rl_records": 27,
1240
+ "sampling": {
1241
+ "samples": 2520,
1242
+ "secs": 35.543824911117554,
1243
+ "gate_secs": 6.1251060962677,
1244
+ "samples_per_sec": 70.89839110736176
1245
+ },
1246
+ "secs": 35.62865972518921,
1247
+ "rl_secs_used": 1327.5637955665588
1248
+ },
1249
+ {
1250
+ "round": 37,
1251
+ "level": 1,
1252
+ "k": 8,
1253
+ "n_targets": 1500,
1254
+ "sample_acc": 0.651,
1255
+ "new_proofs": 714,
1256
+ "targets_solved": 1228,
1257
+ "lstar": 11,
1258
+ "ge": {
1259
+ "7": 1228,
1260
+ "8": 679,
1261
+ "9": 389,
1262
+ "10": 94,
1263
+ "11": 17,
1264
+ "12": 2,
1265
+ "13": 0,
1266
+ "14": 0
1267
+ },
1268
+ "mix_rl_records": 3101,
1269
+ "sampling": {
1270
+ "samples": 12000,
1271
+ "secs": 63.128063678741455,
1272
+ "gate_secs": 31.975621223449707,
1273
+ "samples_per_sec": 190.08978417377045
1274
+ },
1275
+ "secs": 103.13500428199768,
1276
+ "rl_secs_used": 1398.7233264446259
1277
+ },
1278
+ {
1279
+ "round": 38,
1280
+ "level": 2,
1281
+ "k": 24,
1282
+ "n_targets": 102,
1283
+ "sample_acc": 0.04738562091503268,
1284
+ "new_proofs": 7,
1285
+ "targets_solved": 1228,
1286
+ "lstar": 11,
1287
+ "ge": {
1288
+ "7": 1228,
1289
+ "8": 679,
1290
+ "9": 389,
1291
+ "10": 94,
1292
+ "11": 17,
1293
+ "12": 2,
1294
+ "13": 0,
1295
+ "14": 0
1296
+ },
1297
+ "mix_rl_records": 22,
1298
+ "sampling": {
1299
+ "samples": 2448,
1300
+ "secs": 16.93323302268982,
1301
+ "gate_secs": 6.625847101211548,
1302
+ "samples_per_sec": 144.5677855327322
1303
+ },
1304
+ "secs": 17.02391743659973,
1305
+ "rl_secs_used": 1409.121250152588
1306
+ },
1307
+ {
1308
+ "round": 39,
1309
+ "level": 4,
1310
+ "k": 24,
1311
+ "n_targets": 101,
1312
+ "sample_acc": 0.07838283828382839,
1313
+ "new_proofs": 19,
1314
+ "targets_solved": 1230,
1315
+ "lstar": 11,
1316
+ "ge": {
1317
+ "7": 1230,
1318
+ "8": 681,
1319
+ "9": 391,
1320
+ "10": 95,
1321
+ "11": 17,
1322
+ "12": 2,
1323
+ "13": 0,
1324
+ "14": 0
1325
+ },
1326
+ "mix_rl_records": 43,
1327
+ "sampling": {
1328
+ "samples": 2424,
1329
+ "secs": 19.679768800735474,
1330
+ "gate_secs": 6.570547342300415,
1331
+ "samples_per_sec": 123.172178725464
1332
+ },
1333
+ "secs": 19.768434047698975,
1334
+ "rl_secs_used": 1422.3191435337067
1335
+ },
1336
+ {
1337
+ "round": 40,
1338
+ "level": 8,
1339
+ "k": 24,
1340
+ "n_targets": 101,
1341
+ "sample_acc": 0.03506600660066007,
1342
+ "new_proofs": 2,
1343
+ "targets_solved": 1230,
1344
+ "lstar": 11,
1345
+ "ge": {
1346
+ "7": 1230,
1347
+ "8": 681,
1348
+ "9": 391,
1349
+ "10": 95,
1350
+ "11": 17,
1351
+ "12": 2,
1352
+ "13": 0,
1353
+ "14": 0
1354
+ },
1355
+ "mix_rl_records": 11,
1356
+ "sampling": {
1357
+ "samples": 2424,
1358
+ "secs": 25.025669813156128,
1359
+ "gate_secs": 5.767747163772583,
1360
+ "samples_per_sec": 96.86054431700727
1361
+ },
1362
+ "secs": 25.099387407302856,
1363
+ "rl_secs_used": 1441.6507840156555
1364
+ },
1365
+ {
1366
+ "round": 41,
1367
+ "level": 1,
1368
+ "k": 8,
1369
+ "n_targets": 1500,
1370
+ "sample_acc": 0.6625833333333333,
1371
+ "new_proofs": 847,
1372
+ "targets_solved": 1231,
1373
+ "lstar": 11,
1374
+ "ge": {
1375
+ "7": 1231,
1376
+ "8": 682,
1377
+ "9": 392,
1378
+ "10": 96,
1379
+ "11": 18,
1380
+ "12": 2,
1381
+ "13": 0,
1382
+ "14": 0
1383
+ },
1384
+ "mix_rl_records": 3401,
1385
+ "sampling": {
1386
+ "samples": 12000,
1387
+ "secs": 60.50330400466919,
1388
+ "gate_secs": 32.15810441970825,
1389
+ "samples_per_sec": 198.3362759672419
1390
+ },
1391
+ "secs": 102.1084098815918,
1392
+ "rl_secs_used": 1511.6012694835663
1393
+ },
1394
+ {
1395
+ "round": 42,
1396
+ "level": 2,
1397
+ "k": 24,
1398
+ "n_targets": 101,
1399
+ "sample_acc": 0.06311881188118812,
1400
+ "new_proofs": 23,
1401
+ "targets_solved": 1232,
1402
+ "lstar": 11,
1403
+ "ge": {
1404
+ "7": 1232,
1405
+ "8": 683,
1406
+ "9": 393,
1407
+ "10": 96,
1408
+ "11": 18,
1409
+ "12": 2,
1410
+ "13": 0,
1411
+ "14": 0
1412
+ },
1413
+ "mix_rl_records": 54,
1414
+ "sampling": {
1415
+ "samples": 2424,
1416
+ "secs": 16.798370838165283,
1417
+ "gate_secs": 6.6026692390441895,
1418
+ "samples_per_sec": 144.2997075938317
1419
+ },
1420
+ "secs": 16.892906188964844,
1421
+ "rl_secs_used": 1521.8913114070892
1422
+ },
1423
+ {
1424
+ "round": 43,
1425
+ "level": 4,
1426
+ "k": 24,
1427
+ "n_targets": 100,
1428
+ "sample_acc": 0.04541666666666667,
1429
+ "new_proofs": 14,
1430
+ "targets_solved": 1235,
1431
+ "lstar": 11,
1432
+ "ge": {
1433
+ "7": 1235,
1434
+ "8": 685,
1435
+ "9": 394,
1436
+ "10": 96,
1437
+ "11": 18,
1438
+ "12": 2,
1439
+ "13": 0,
1440
+ "14": 0
1441
+ },
1442
+ "mix_rl_records": 24,
1443
+ "sampling": {
1444
+ "samples": 2400,
1445
+ "secs": 21.2550790309906,
1446
+ "gate_secs": 5.933649778366089,
1447
+ "samples_per_sec": 112.91418848646582
1448
+ },
1449
+ "secs": 21.336429357528687,
1450
+ "rl_secs_used": 1537.2941012382507
1451
+ },
1452
+ {
1453
+ "round": 44,
1454
+ "level": 8,
1455
+ "k": 24,
1456
+ "n_targets": 100,
1457
+ "sample_acc": 0.05291666666666667,
1458
+ "new_proofs": 19,
1459
+ "targets_solved": 1238,
1460
+ "lstar": 11,
1461
+ "ge": {
1462
+ "7": 1238,
1463
+ "8": 687,
1464
+ "9": 395,
1465
+ "10": 96,
1466
+ "11": 18,
1467
+ "12": 2,
1468
+ "13": 0,
1469
+ "14": 0
1470
+ },
1471
+ "mix_rl_records": 37,
1472
+ "sampling": {
1473
+ "samples": 2400,
1474
+ "secs": 34.69861912727356,
1475
+ "gate_secs": 6.343368291854858,
1476
+ "samples_per_sec": 69.16701760369389
1477
+ },
1478
+ "secs": 34.7859742641449,
1479
+ "rl_secs_used": 1565.736709356308
1480
+ },
1481
+ {
1482
+ "round": 45,
1483
+ "level": 1,
1484
+ "k": 8,
1485
+ "n_targets": 1500,
1486
+ "sample_acc": 0.6339166666666667,
1487
+ "new_proofs": 757,
1488
+ "targets_solved": 1240,
1489
+ "lstar": 11,
1490
+ "ge": {
1491
+ "7": 1240,
1492
+ "8": 688,
1493
+ "9": 396,
1494
+ "10": 96,
1495
+ "11": 18,
1496
+ "12": 2,
1497
+ "13": 0,
1498
+ "14": 0
1499
+ },
1500
+ "mix_rl_records": 3405,
1501
+ "sampling": {
1502
+ "samples": 12000,
1503
+ "secs": 62.353286266326904,
1504
+ "gate_secs": 31.60433268547058,
1505
+ "samples_per_sec": 192.45176507209126
1506
+ },
1507
+ "secs": 103.04225993156433,
1508
+ "rl_secs_used": 1637.174780845642
1509
+ },
1510
+ {
1511
+ "round": 46,
1512
+ "level": 2,
1513
+ "k": 24,
1514
+ "n_targets": 99,
1515
+ "sample_acc": 0.07365319865319865,
1516
+ "new_proofs": 20,
1517
+ "targets_solved": 1240,
1518
+ "lstar": 11,
1519
+ "ge": {
1520
+ "7": 1240,
1521
+ "8": 688,
1522
+ "9": 396,
1523
+ "10": 96,
1524
+ "11": 18,
1525
+ "12": 2,
1526
+ "13": 0,
1527
+ "14": 0
1528
+ },
1529
+ "mix_rl_records": 44,
1530
+ "sampling": {
1531
+ "samples": 2376,
1532
+ "secs": 17.075836658477783,
1533
+ "gate_secs": 6.703376770019531,
1534
+ "samples_per_sec": 139.14398735012304
1535
+ },
1536
+ "secs": 17.17088484764099,
1537
+ "rl_secs_used": 1647.6421418190002
1538
+ },
1539
+ {
1540
+ "round": 47,
1541
+ "level": 4,
1542
+ "k": 24,
1543
+ "n_targets": 99,
1544
+ "sample_acc": 0.06860269360269361,
1545
+ "new_proofs": 21,
1546
+ "targets_solved": 1243,
1547
+ "lstar": 11,
1548
+ "ge": {
1549
+ "7": 1243,
1550
+ "8": 690,
1551
+ "9": 398,
1552
+ "10": 98,
1553
+ "11": 18,
1554
+ "12": 2,
1555
+ "13": 0,
1556
+ "14": 0
1557
+ },
1558
+ "mix_rl_records": 53,
1559
+ "sampling": {
1560
+ "samples": 2376,
1561
+ "secs": 19.267151832580566,
1562
+ "gate_secs": 6.275737047195435,
1563
+ "samples_per_sec": 123.31869394324319
1564
+ },
1565
+ "secs": 19.357197046279907,
1566
+ "rl_secs_used": 1660.7235941886902
1567
+ },
1568
+ {
1569
+ "round": 48,
1570
+ "level": 8,
1571
+ "k": 24,
1572
+ "n_targets": 99,
1573
+ "sample_acc": 0.0723905723905724,
1574
+ "new_proofs": 35,
1575
+ "targets_solved": 1243,
1576
+ "lstar": 11,
1577
+ "ge": {
1578
+ "7": 1243,
1579
+ "8": 690,
1580
+ "9": 398,
1581
+ "10": 98,
1582
+ "11": 18,
1583
+ "12": 2,
1584
+ "13": 0,
1585
+ "14": 0
1586
+ },
1587
+ "mix_rl_records": 58,
1588
+ "sampling": {
1589
+ "samples": 2376,
1590
+ "secs": 34.03534483909607,
1591
+ "gate_secs": 5.748866558074951,
1592
+ "samples_per_sec": 69.80978189680958
1593
+ },
1594
+ "secs": 34.12078928947449,
1595
+ "rl_secs_used": 1689.095514535904
1596
+ }
1597
+ ],
1598
+ "secs": 2439.724471092224,
1599
+ "rl_secs_used": 1730.696391582489,
1600
+ "gate_secs": 708.9617705345154,
1601
+ "targets": 1500,
1602
+ "stats": {
1603
+ "rounds": 12,
1604
+ "first_loops": 1,
1605
+ "k_first": 8,
1606
+ "retry_loops": [
1607
+ 2,
1608
+ 4,
1609
+ 8
1610
+ ],
1611
+ "k_retry": 24,
1612
+ "retries": [
1613
+ {
1614
+ "round": 1,
1615
+ "loops": 2,
1616
+ "targets": 439,
1617
+ "cracked": 37
1618
+ },
1619
+ {
1620
+ "round": 1,
1621
+ "loops": 4,
1622
+ "targets": 439,
1623
+ "cracked": 57
1624
+ },
1625
+ {
1626
+ "round": 1,
1627
+ "loops": 8,
1628
+ "targets": 439,
1629
+ "cracked": 66
1630
+ },
1631
+ {
1632
+ "round": 2,
1633
+ "loops": 2,
1634
+ "targets": 284,
1635
+ "cracked": 49
1636
+ },
1637
+ {
1638
+ "round": 2,
1639
+ "loops": 4,
1640
+ "targets": 284,
1641
+ "cracked": 60
1642
+ },
1643
+ {
1644
+ "round": 2,
1645
+ "loops": 8,
1646
+ "targets": 283,
1647
+ "cracked": 60
1648
+ },
1649
+ {
1650
+ "round": 3,
1651
+ "loops": 2,
1652
+ "targets": 198,
1653
+ "cracked": 46
1654
+ },
1655
+ {
1656
+ "round": 3,
1657
+ "loops": 4,
1658
+ "targets": 198,
1659
+ "cracked": 43
1660
+ },
1661
+ {
1662
+ "round": 3,
1663
+ "loops": 8,
1664
+ "targets": 197,
1665
+ "cracked": 48
1666
+ },
1667
+ {
1668
+ "round": 4,
1669
+ "loops": 2,
1670
+ "targets": 164,
1671
+ "cracked": 35
1672
+ },
1673
+ {
1674
+ "round": 4,
1675
+ "loops": 4,
1676
+ "targets": 164,
1677
+ "cracked": 38
1678
+ },
1679
+ {
1680
+ "round": 4,
1681
+ "loops": 8,
1682
+ "targets": 163,
1683
+ "cracked": 40
1684
+ },
1685
+ {
1686
+ "round": 5,
1687
+ "loops": 2,
1688
+ "targets": 144,
1689
+ "cracked": 37
1690
+ },
1691
+ {
1692
+ "round": 5,
1693
+ "loops": 4,
1694
+ "targets": 143,
1695
+ "cracked": 28
1696
+ },
1697
+ {
1698
+ "round": 5,
1699
+ "loops": 8,
1700
+ "targets": 143,
1701
+ "cracked": 25
1702
+ },
1703
+ {
1704
+ "round": 6,
1705
+ "loops": 2,
1706
+ "targets": 126,
1707
+ "cracked": 20
1708
+ },
1709
+ {
1710
+ "round": 6,
1711
+ "loops": 4,
1712
+ "targets": 125,
1713
+ "cracked": 22
1714
+ },
1715
+ {
1716
+ "round": 6,
1717
+ "loops": 8,
1718
+ "targets": 125,
1719
+ "cracked": 19
1720
+ },
1721
+ {
1722
+ "round": 7,
1723
+ "loops": 2,
1724
+ "targets": 119,
1725
+ "cracked": 15
1726
+ },
1727
+ {
1728
+ "round": 7,
1729
+ "loops": 4,
1730
+ "targets": 118,
1731
+ "cracked": 21
1732
+ },
1733
+ {
1734
+ "round": 7,
1735
+ "loops": 8,
1736
+ "targets": 118,
1737
+ "cracked": 21
1738
+ },
1739
+ {
1740
+ "round": 8,
1741
+ "loops": 2,
1742
+ "targets": 111,
1743
+ "cracked": 15
1744
+ },
1745
+ {
1746
+ "round": 8,
1747
+ "loops": 4,
1748
+ "targets": 111,
1749
+ "cracked": 15
1750
+ },
1751
+ {
1752
+ "round": 8,
1753
+ "loops": 8,
1754
+ "targets": 111,
1755
+ "cracked": 17
1756
+ },
1757
+ {
1758
+ "round": 9,
1759
+ "loops": 2,
1760
+ "targets": 105,
1761
+ "cracked": 13
1762
+ },
1763
+ {
1764
+ "round": 9,
1765
+ "loops": 4,
1766
+ "targets": 105,
1767
+ "cracked": 13
1768
+ },
1769
+ {
1770
+ "round": 9,
1771
+ "loops": 8,
1772
+ "targets": 105,
1773
+ "cracked": 12
1774
+ },
1775
+ {
1776
+ "round": 10,
1777
+ "loops": 2,
1778
+ "targets": 102,
1779
+ "cracked": 12
1780
+ },
1781
+ {
1782
+ "round": 10,
1783
+ "loops": 4,
1784
+ "targets": 101,
1785
+ "cracked": 15
1786
+ },
1787
+ {
1788
+ "round": 10,
1789
+ "loops": 8,
1790
+ "targets": 101,
1791
+ "cracked": 6
1792
+ },
1793
+ {
1794
+ "round": 11,
1795
+ "loops": 2,
1796
+ "targets": 101,
1797
+ "cracked": 15
1798
+ },
1799
+ {
1800
+ "round": 11,
1801
+ "loops": 4,
1802
+ "targets": 100,
1803
+ "cracked": 10
1804
+ },
1805
+ {
1806
+ "round": 11,
1807
+ "loops": 8,
1808
+ "targets": 100,
1809
+ "cracked": 13
1810
+ },
1811
+ {
1812
+ "round": 12,
1813
+ "loops": 2,
1814
+ "targets": 99,
1815
+ "cracked": 11
1816
+ },
1817
+ {
1818
+ "round": 12,
1819
+ "loops": 4,
1820
+ "targets": 99,
1821
+ "cracked": 16
1822
+ },
1823
+ {
1824
+ "round": 12,
1825
+ "loops": 8,
1826
+ "targets": 99,
1827
+ "cracked": 13
1828
+ }
1829
+ ],
1830
+ "ft_records": 38400,
1831
+ "ft_steps": [
1832
+ 296,
1833
+ 314,
1834
+ 340,
1835
+ 343,
1836
+ 357,
1837
+ 356,
1838
+ 369,
1839
+ 378,
1840
+ 379,
1841
+ 383,
1842
+ 383,
1843
+ 392
1844
+ ],
1845
+ "ft_optimizer": "muon (polar express) on block matrices and head + adamw on the rest",
1846
+ "ft_lr": 0.0003,
1847
+ "ft_muon_lr": 0.003
1848
+ }
1849
+ },
1850
+ "eval": {
1851
+ "level": 8,
1852
+ "attempts_per_theorem": 64,
1853
+ "solved": 734,
1854
+ "n": 1108,
1855
+ "solved_ge7": 734,
1856
+ "lstar": 12,
1857
+ "ge": {
1858
+ "7": 734,
1859
+ "8": 625,
1860
+ "9": 513,
1861
+ "10": 126,
1862
+ "11": 30,
1863
+ "12": 7,
1864
+ "13": 2,
1865
+ "14": 0
1866
+ },
1867
+ "by_bin": {
1868
+ "7": {
1869
+ "solved": 109,
1870
+ "n": 150,
1871
+ "rate": 0.7266666666666667,
1872
+ "ci": [
1873
+ 0.6503534444650609,
1874
+ 0.7916596391222133
1875
+ ]
1876
+ },
1877
+ "8": {
1878
+ "solved": 112,
1879
+ "n": 152,
1880
+ "rate": 0.7368421052631579,
1881
+ "ci": [
1882
+ 0.6616208335298267,
1883
+ 0.8003867433944348
1884
+ ]
1885
+ },
1886
+ "9": {
1887
+ "solved": 387,
1888
+ "n": 477,
1889
+ "rate": 0.8113207547169812,
1890
+ "ci": [
1891
+ 0.7737737427372532,
1892
+ 0.8438932810809855
1893
+ ]
1894
+ },
1895
+ "10": {
1896
+ "solved": 96,
1897
+ "n": 219,
1898
+ "rate": 0.4383561643835616,
1899
+ "ci": [
1900
+ 0.3742620490839017,
1901
+ 0.5045756544687563
1902
+ ]
1903
+ },
1904
+ "11": {
1905
+ "solved": 23,
1906
+ "n": 49,
1907
+ "rate": 0.46938775510204084,
1908
+ "ci": [
1909
+ 0.33703281897894777,
1910
+ 0.6061937298008014
1911
+ ]
1912
+ },
1913
+ "12": {
1914
+ "solved": 5,
1915
+ "n": 54,
1916
+ "rate": 0.09259259259259259,
1917
+ "ci": [
1918
+ 0.040200542070567336,
1919
+ 0.1991012753065453
1920
+ ]
1921
+ },
1922
+ "13": {
1923
+ "solved": 2,
1924
+ "n": 5,
1925
+ "rate": 0.4,
1926
+ "ci": [
1927
+ 0.1176182311592533,
1928
+ 0.769280067791163
1929
+ ]
1930
+ },
1931
+ "14": {
1932
+ "solved": 0,
1933
+ "n": 2,
1934
+ "rate": 0.0,
1935
+ "ci": [
1936
+ 0.0,
1937
+ 0.6576280471103807
1938
+ ]
1939
+ }
1940
+ },
1941
+ "sample_acc": 0.4433805279783393,
1942
+ "max_pruned": 16,
1943
+ "sampling": {
1944
+ "samples": 70912,
1945
+ "secs": 788.7471218109131,
1946
+ "gate_secs": 131.5565197467804,
1947
+ "samples_per_sec": 89.90460698885414
1948
+ },
1949
+ "heldout_greedy": 0.985,
1950
+ "diag": {
1951
+ "level": 2,
1952
+ "attempts_per_theorem": 64,
1953
+ "solved": 729,
1954
+ "n": 1108,
1955
+ "solved_ge7": 729,
1956
+ "lstar": 12,
1957
+ "ge": {
1958
+ "7": 729,
1959
+ "8": 623,
1960
+ "9": 513,
1961
+ "10": 123,
1962
+ "11": 30,
1963
+ "12": 6,
1964
+ "13": 1,
1965
+ "14": 0
1966
+ },
1967
+ "by_bin": {
1968
+ "7": {
1969
+ "solved": 106,
1970
+ "n": 150,
1971
+ "rate": 0.7066666666666667,
1972
+ "ci": [
1973
+ 0.6293750243784035,
1974
+ 0.7736369047746994
1975
+ ]
1976
+ },
1977
+ "8": {
1978
+ "solved": 110,
1979
+ "n": 152,
1980
+ "rate": 0.7236842105263158,
1981
+ "ci": [
1982
+ 0.6477452174876825,
1983
+ 0.7885952718296756
1984
+ ]
1985
+ },
1986
+ "9": {
1987
+ "solved": 390,
1988
+ "n": 477,
1989
+ "rate": 0.8176100628930818,
1990
+ "ci": [
1991
+ 0.780462779346945,
1992
+ 0.8496823661645916
1993
+ ]
1994
+ },
1995
+ "10": {
1996
+ "solved": 93,
1997
+ "n": 219,
1998
+ "rate": 0.4246575342465753,
1999
+ "ci": [
2000
+ 0.3610439955583555,
2001
+ 0.4908687532282265
2002
+ ]
2003
+ },
2004
+ "11": {
2005
+ "solved": 24,
2006
+ "n": 49,
2007
+ "rate": 0.4897959183673469,
2008
+ "ci": [
2009
+ 0.3557488315940086,
2010
+ 0.6253266846659078
2011
+ ]
2012
+ },
2013
+ "12": {
2014
+ "solved": 5,
2015
+ "n": 54,
2016
+ "rate": 0.09259259259259259,
2017
+ "ci": [
2018
+ 0.040200542070567336,
2019
+ 0.1991012753065453
2020
+ ]
2021
+ },
2022
+ "13": {
2023
+ "solved": 1,
2024
+ "n": 5,
2025
+ "rate": 0.2,
2026
+ "ci": [
2027
+ 0.03622316096978745,
2028
+ 0.6244717358814613
2029
+ ]
2030
+ },
2031
+ "14": {
2032
+ "solved": 0,
2033
+ "n": 2,
2034
+ "rate": 0.0,
2035
+ "ci": [
2036
+ 0.0,
2037
+ 0.6576280471103807
2038
+ ]
2039
+ }
2040
+ },
2041
+ "sample_acc": 0.417122630866426,
2042
+ "max_pruned": 15,
2043
+ "sampling": {
2044
+ "samples": 70912,
2045
+ "secs": 370.52518916130066,
2046
+ "gate_secs": 134.16099166870117,
2047
+ "samples_per_sec": 191.38240010216927
2048
+ }
2049
+ }
2050
+ }
2051
+ }