Add FinCalc-NLI eval (financial arithmetic) + results for all checkpoints

README.md

@@ -160,6 +160,7 @@ a small MLP head on the last-token latent (`mlp_heads_35b/`, one head per task,
 * `code/openjev_decide.py` — `OpenJev.decide(state, questions)`: typed decisions over a closed option set,
   and the JevBench adapter. `code/eval_jevbench.py`, `code/eval_extra.py`, `code/eval_security.py`,
   `code/order_test.py` reproduce every table above.
+* `code/eval_fincalc.py` — financial arithmetic: [FinCalc-NLI](https://huggingface.co/datasets/anespo28/fincalc-nli) (ratios, covenant tests, net figures; synthetic, labels computed in code). 4B v5 63.2%, 4B v2 58.2%, 0.8B v2s-long 50.5%, 4B original 31.9% (4,000-item test split).
 * `code/` — everything used here: the trainer and data mixture builder, the evaluation harness, Flappy Bird, Doom
   (text and pixels), the Minecraft scaffold and bot, the radar, the SGLang package / launcher / client / benchmark.
 * `videos/` — Flappy Bird, Doom and Minecraft replays; `results/` — raw JSON for every run and the full report.

code/eval_fincalc.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""Financial arithmetic for jev cross-encoders: FinCalc-NLI test split. Nothing here is trained on.
+
+    python eval_fincalc.py --models ckpt/qwen3.5-4b-nli-v5 ckpt/qwen3.5-4b-nli --out results/fincalc_4b.json
+
+FinCalc-NLI (anespo28/fincalc-nli, MIT, synthetic): premise = evidence with the inputs and distractor figures
+(another company, the prior year, policy lines), hypothesis = a derived figure (LTV, DSCR, interest cover, net
+leverage, EBITDA margin, CET1, NPL ratio, NPL coverage, annual interest, net debt), a covenant pass/breach, or a
+plain lookup. Labels are computed in code. The test split uses names and templates disjoint from train, and one
+formula (NPL coverage) that train never shows.
+
+Reported: 3-way accuracy (argmax), per mistake kind (slip, inverted ratio, wrong entity, unit error, flipped
+covenant, missing input, ...) and per metric.
+"""
+import argparse
+import json
+from collections import defaultdict
+
+import numpy as np
+
+from eval import NLIScorer
+
+REPO = "anespo28/fincalc-nli"
+
+
+def load(n):
+    from datasets import load_dataset
+    ds = load_dataset(REPO, data_files={"test": "test.jsonl"}, split="test")
+    return ds.shuffle(seed=0).select(range(min(n, len(ds)))) if n else ds
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--models", nargs="+", required=True)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--n", type=int, default=0, help="subsample the 4,000-item test split (0 = all)")
+    ap.add_argument("--bs", type=int, default=32)
+    a = ap.parse_args()
+    ds = load(a.n)
+    pairs = list(zip(ds["premise"], ds["hypothesis"]))
+    gold = np.array(ds["label"])  # already in jev order: 0 contradiction, 1 entailment, 2 neutral
+    res = {}
+    for path in a.models:
+        pred = NLIScorer(path, bs=a.bs).predict(pairs).argmax(1)
+        by = defaultdict(list)
+        for k, m, ok in zip(ds["kind"], ds["metric"], pred == gold):
+            by["kind:" + k].append(ok)
+            by["metric:" + m].append(ok)
+        res[path] = {"n": len(pairs), "accuracy": round(float((pred == gold).mean()), 4),
+                     **{k: round(float(np.mean(v)), 4) for k, v in sorted(by.items())}}
+        print(path, json.dumps(res[path]), flush=True)
+    with open(a.out, "w") as f:
+        json.dump(res, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()