deVision v0.3 (github model-v0.3, 43cf900e42ba7a8557be1506679d79de8c45786a)

.gitattributes

@@ -1,2 +1,3 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+assets/logo.png filter=lfs diff=lfs merge=lfs -text

MANIFEST.json

@@ -8,32 +8,56 @@
   "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
  },
  "README.md": {
-  "size": 9508,
-  "sha256": "86835a042c9036323960f5388401ba2257f1d9bced24825aeb3c902e315abc94"
+  "size": 12369,
+  "sha256": "9e5087ee27134943c32a5a3a3bd3bf90bbceffc942304dc00d7a5638a743a2e8"
+ },
+ "assets/logo.png": {
+  "size": 450887,
+  "sha256": "4c1123544e9b08d79eb84ef399b244907249e5353bcbd5167667400e3a62a0f7"
  },
  "config.json": {
-  "size": 353,
-  "sha256": "26fdd9541256d09d8017bd467962bbf74a7cfc3986ca340e89e63faae3c8d11e"
+  "size": 420,
+  "sha256": "f51a34393cc41ea56012bfd29eb35fbe9197934d897f24085eefb5e9a626452a"
  },
  "encoder/config.json": {
   "size": 2084,
   "sha256": "5268d24ad3b77c8151de5dcb0762ba4391619aad9ab0bda33e36fb083cfeae6d"
  },
+ "evaluation/accept_vs_v0.2.json": {
+  "size": 13821,
+  "sha256": "8e63d73d13f985160e787b741c41895c029ef00eefff8895ad94b2a9d026184c"
+ },
+ "evaluation/adoption.md": {
+  "size": 2269,
+  "sha256": "0b044443cf371dbe8e017cb2fc705b1b841f58e46059c1cb272c161bfecfe988"
+ },
+ "evaluation/cpu_latency.json": {
+  "size": 8940,
+  "sha256": "1740cad45a9547c74c21e678f5ac51c906bb46890348472a1be9bebfa83328d8"
+ },
  "evaluation/results.json": {
-  "size": 2819,
-  "sha256": "adb3ec9aeff1dc30974a9f9e2ed54c589672bb65b1d9b66e70792158172a9988"
+  "size": 3567,
+  "sha256": "b9733f4f573a2652b079f7b899c71ca0290c0b199a85f34c0607bb0e577d6b22"
  },
  "evaluation/results.md": {
-  "size": 1623,
-  "sha256": "cdfcdfe16977c52129a0fd7e9001fe7143bdbc494c9246ef92321aef2b9d645c"
+  "size": 2077,
+  "sha256": "a133c35428d625adde778a9e0ff2898aa227ea854156a980eef7b5561aa6edad"
+ },
+ "evaluation/seeded_from.json": {
+  "size": 143,
+  "sha256": "c167f7139c72fb0c70c200e60448af1a5302ed4cc218e609c53e10a0c3d993e4"
+ },
+ "evaluation/training.yaml": {
+  "size": 3979,
+  "sha256": "a7733d9bc41db37b133868839c3080683167374eaf919c3cdd29d0575a894401"
  },
  "model.safetensors": {
   "size": 2073754784,
-  "sha256": "07a3e8aacb986396d3975d12d672526c6d3b8ec240b9d9d503ef235c5c87c3ae"
+  "sha256": "110fea95889947c373258c88a396ad3eec574971815ee437b9c7cc3df8a5d2d5"
  },
  "provenance.json": {
-  "size": 1324,
-  "sha256": "bd3d37bdd771f2e4c859ab868ff9b9f5f8207652fb04ef6c2bc08d1c18ef34f6"
+  "size": 4546,
+  "sha256": "d135fc16bd5e92fb5556972cd47ae0d783ed07d4715a8dde5b255c7f5e1d2f55"
  },
  "tokenizer/tokenizer.json": {
   "size": 3583228,

README.md

@@ -29,7 +29,7 @@ tags:
 - rlcd
 - cpu
 model-index:
-- name: deVision v0.2
+- name: deVision v0.3
   results:
   - task:
       type: visual-question-answering
@@ -38,9 +38,9 @@ model-index:
       type: test_exist
     metrics:
     - type: accuracy
-      value: 0.9384
+      value: 0.9473
     - type: ece
-      value: 0.0204
+      value: 0.0341
   - task:
       type: visual-question-answering
     dataset:
@@ -48,9 +48,9 @@ model-index:
       type: test_vqa_choice
     metrics:
     - type: accuracy
-      value: 0.8979
+      value: 0.9092
     - type: ece
-      value: 0.0262
+      value: 0.0384
   - task:
       type: visual-question-answering
     dataset:
@@ -58,9 +58,9 @@ model-index:
       type: test_size
     metrics:
     - type: accuracy
-      value: 0.86
+      value: 0.8718
     - type: ece
-      value: 0.036
+      value: 0.0295
   - task:
       type: visual-question-answering
     dataset:
@@ -68,9 +68,9 @@ model-index:
       type: bench_pope
     metrics:
     - type: accuracy
-      value: 0.8511
+      value: 0.8496
     - type: ece
-      value: 0.0578
+      value: 0.0393
   - task:
       type: visual-question-answering
     dataset:
@@ -78,9 +78,9 @@ model-index:
       type: test_gqa
     metrics:
     - type: accuracy
-      value: 0.7843
+      value: 0.7903
     - type: ece
-      value: 0.0419
+      value: 0.0396
   - task:
       type: visual-question-answering
     dataset:
@@ -88,9 +88,9 @@ model-index:
       type: test_position
     metrics:
     - type: accuracy
-      value: 0.8721
+      value: 0.8846
     - type: ece
-      value: 0.0251
+      value: 0.029
   - task:
       type: visual-question-answering
     dataset:
@@ -98,9 +98,9 @@ model-index:
       type: test_vqa_yesno
     metrics:
     - type: accuracy
-      value: 0.71
+      value: 0.704
     - type: ece
-      value: 0.0223
+      value: 0.0265
   - task:
       type: visual-question-answering
     dataset:
@@ -108,9 +108,9 @@ model-index:
       type: test_relation
     metrics:
     - type: accuracy
-      value: 0.7108
+      value: 0.7549
     - type: ece
-      value: 0.0471
+      value: 0.0384
   - task:
       type: visual-question-answering
     dataset:
@@ -118,9 +118,9 @@ model-index:
       type: test_vsr
     metrics:
     - type: accuracy
-      value: 0.6792
+      value: 0.6615
     - type: ece
-      value: 0.0478
+      value: 0.0465
   - task:
       type: visual-question-answering
     dataset:
@@ -130,7 +130,7 @@ model-index:
     - type: accuracy
       value: 0.721
     - type: ece
-      value: 0.0302
+      value: 0.0411
   - task:
       type: visual-question-answering
     dataset:
@@ -138,12 +138,64 @@ model-index:
       type: test_count_fresh
     metrics:
     - type: accuracy
-      value: 0.74
+      value: 0.7633
     - type: ece
-      value: 0.0648
+      value: 0.054
+  - task:
+      type: visual-question-answering
+    dataset:
+      name: CountBench, 0–10 choices
+      type: bench_countbench
+    metrics:
+    - type: accuracy
+      value: 0.3564
+    - type: ece
+      value: 0.0523
+  - task:
+      type: visual-question-answering
+    dataset:
+      name: COCO mirrored position
+      type: test_position_flip
+    metrics:
+    - type: accuracy
+      value: 0.8469
+    - type: ece
+      value: 0.0444
+  - task:
+      type: visual-question-answering
+    dataset:
+      name: COCO mirrored relative position
+      type: test_relation_flip
+    metrics:
+    - type: accuracy
+      value: 0.6977
+    - type: ece
+      value: 0.03
+  - task:
+      type: visual-question-answering
+    dataset:
+      name: GQA binding test
+      type: test_bind_gqa
+    metrics:
+    - type: accuracy
+      value: 0.6765
+    - type: ece
+      value: 0.0277
+  - task:
+      type: visual-question-answering
+    dataset:
+      name: What'sUp
+      type: bench_whatsup
+    metrics:
+    - type: accuracy
+      value: 0.2866
+    - type: ece
+      value: 0.2447
 ---
 
-# deVision v0.2
+# deVision v0.3
+
+<p align="center"><img src="assets/logo.png" alt="deVision" width="200"></p>
 
 **Image + English questions → structured answers with calibrated probabilities.** deVision pairs a SigLIP2 vision encoder with Laya's ModernBERT decision model. It scores the options of yes/no (`noul`) and multiple-choice (`choice`) questions instead of generating text, follows the Jev answer format, and runs without a GPU. Several questions about one image share a single image encoding.
 
@@ -156,7 +208,7 @@ pip install devision
 ```python
 import devision
 
-model = devision.load("lukbit/devision")   # latest release; revision="v0.2" pins this one; device defaults to auto
+model = devision.load("lukbit/devision")   # latest release; device defaults to auto
 result = model.predict(
     image="photo.jpg",   # a path, an http(s) URL, a data URI, base64, bytes or a PIL image
     state="",            # text context; "" when there is none
@@ -184,52 +236,70 @@ Over HTTP, `image` is an http(s) URL, a data URI or base64; the server never rea
 
 ## Evaluation
 
-Accuracy through `decide` with the fitted temperatures. *Mismatched*: the same questions with every picture swapped for an unrelated one. Test sets use held-out pictures; they are project subsets, not official leaderboard scores.
-
-| Set | Questions | Accuracy | Mismatched | ECE |
-|---|---:|---:|---:|---:|
-| COCO object presence | 1,120 | 0.938 | 0.493 | 0.020 |
-| VQAv2 multiple choice | 1,420 | 0.898 | 0.399 | 0.026 |
-| COCO size | 1,100 | 0.860 | 0.504 | 0.036 |
-| POPE, project filtered set | 8,676 | 0.851 | 0.527 | 0.058 |
-| GQA val subset | 992 | 0.784 | 0.532 | 0.042 |
-| COCO position | 1,274 | 0.872 | 0.493 | 0.025 |
-| VQAv2 yes/no subset | 1,000 | 0.710 | 0.518 | 0.022 |
-| COCO relative position | 1,950 | 0.711 | 0.484 | 0.047 |
-| VSR, project held-out split | 904 | 0.679 | 0.481 | 0.048 |
-| Visual7W, project held-out split | 1,000 | 0.721 | 0.422 | 0.030 |
-| Fresh counting test (unseen pictures) | 600 | 0.740 | 0.507 | 0.065 |
+Accuracy through `decide` with the fitted temperatures. *Mismatched*: the same questions with every picture swapped for an unrelated one. These are project evaluation subsets, not official leaderboard scores. Although test pictures are excluded from training, these sets have informed successive experiments and release decisions; they are not a fresh independent test. See the release limitations below.
+
+| Set | Questions | Accuracy | Mismatched | NLL | ECE |
+|---|---:|---:|---:|---:|---:|
+| COCO object presence | 1,120 | 0.947 | 0.504 | 0.163 | 0.034 |
+| VQAv2 multiple choice | 1,420 | 0.909 | 0.380 | 0.271 | 0.038 |
+| COCO size | 1,100 | 0.872 | 0.511 | 0.336 | 0.029 |
+| POPE, project filtered set | 8,676 | 0.850 | 0.527 | 0.368 | 0.039 |
+| GQA val subset | 992 | 0.790 | 0.550 | 0.410 | 0.040 |
+| COCO position | 1,274 | 0.885 | 0.493 | 0.288 | 0.029 |
+| VQAv2 yes/no subset | 1,000 | 0.704 | 0.512 | 0.574 | 0.026 |
+| COCO relative position | 1,950 | 0.755 | 0.496 | 0.490 | 0.038 |
+| VSR, project held-out split | 904 | 0.661 | 0.487 | 0.620 | 0.046 |
+| Visual7W, project held-out split | 1,000 | 0.721 | 0.419 | 0.673 | 0.041 |
+| Fresh counting test (unseen pictures) | 600 | 0.763 | 0.405 | 0.574 | 0.054 |
+| CountBench, 0–10 choices | 491 | 0.356 | 0.104 | 1.602 | 0.052 |
+| COCO mirrored position | 640 | 0.847 | 0.477 | 0.359 | 0.044 |
+| COCO mirrored relative position | 1,492 | 0.698 | 0.507 | 0.545 | 0.030 |
+| GQA binding test | 1,484 | 0.676 | 0.498 | 0.559 | 0.028 |
+| What'sUp | 820 | 0.287 | 0.245 | 1.491 | 0.245 |
 
 Against Laya Vision 201M on the same questions (its published per-question predictions; difference in points, 95% interval from paired resampling by picture):
 
 | Set | Questions | Laya Vision | deVision | Difference |
 |---|---:|---:|---:|---|
-| VQAv2 yes/no | 4,887 | 0.717 | 0.725 | +0.8 [-0.9, +2.3] |
-| A-OKVQA | 1,138 | 0.598 | 0.626 | +2.7 [-0.6, +6.0] |
-| ScienceQA with images | 2,097 | 0.824 | 0.766 | -5.8 [-7.9, -3.6] |
-| ScienceQA natural science, needs the picture | 323 | 0.700 | 0.455 | -24.5 [-31.6, -17.6] |
+| VQAv2 yes/no | 4,887 | 0.717 | 0.725 | +0.8 [-0.8, +2.4] |
+| A-OKVQA | 1,138 | 0.598 | 0.613 | +1.4 [-1.8, +4.7] |
+| ScienceQA with images | 2,097 | 0.824 | 0.789 | -3.5 [-5.6, -1.4] |
+| ScienceQA natural science, needs the picture | 323 | 0.700 | 0.495 | -20.4 [-27.2, -13.3] |
 
-On the full POPE random / popular / adversarial sets (3,000 questions each) deVision scores **0.891 / 0.868 / 0.791**, against Laya Vision's published **0.836 / 0.819 / 0.777** (aggregate scores only, not paired).
+On the full POPE random / popular / adversarial sets (3,000 questions each) deVision scores **0.880 / 0.859 / 0.805**, against Laya Vision's published **0.836 / 0.819 / 0.777** (aggregate scores only, not paired).
 
-CPU latency: P50 148 ms, P95 161 ms (Darwin arm64, 4 threads, FP32, one question per request, warm-up excluded).
+CPU latency: P50 175 ms, P95 190 ms (Darwin arm64, 4 threads, FP32, one question per request, warm-up excluded).
 
 ## Model
 
 | | |
 |---|---|
-| Vision | Frozen SigLIP2-B/16, 256 × 256 letterboxed input, 64 visual tokens after a 2 × 2 merge and an MLP projector |
+| Vision | SigLIP2-B/16, 256 × 256 letterboxed input, 64 visual tokens after a 2 × 2 merge and an MLP projector; training policy described below |
 | Decision | Laya-initialised ModernBERT-large and Laya's decision head; each option is scored at its `[MASK]`, then a temperature-scaled softmax |
 | Size | About 518M parameters, FP32, 2.07 GB (LoRA merged) |
 
-Training: the projector is first aligned on COCO captions, then the decision is trained with Laya's RLCD objective on yes/no and multiple-choice questions (projector, decision head and a LoRA on ModernBERT), round after round. This release continued for one pass over 353,830 questions, 315,342 of them from thirteen public datasets (Objects365, TallyQA, CLEVR, CLEVR-Math, Super-CLEVR, FigureQA, MapQA, IconQA, SNLI-VE, Vision-Flan, VisOnlyQA, SpatialSense, PixMo-Count) and the rest replaying earlier data (VQAv2, GQA, A-OKVQA, ScienceQA, VSR, Visual7W, COCO-derived questions). Temperatures are fitted per question type and option count. Details: the [training log](https://github.com/byebyebruce/devision/blob/master/docs/training-log.md).
+This release continues v0.2 with 81,875 questions of counting, object presence, mirrored spatial questions, VSR and broad replay, for one 10,235-step schedule (seed 1). The first 2,000 steps were run as exp19; r15 resumes that exact training state. SigLIP2's top four encoder layers and final LayerNorm were fine-tuned at 5e-6 alongside the projector, decision head and ModernBERT LoRA. LoRA is merged into the released weights. The input remains 256×256 with 64 visual tokens; S2 question conditioning is **not** enabled.
+
+Temperatures were fitted on the designated calibration set, by question type and option-count bucket. Missing buckets fall back to the question-type temperature:
+
+| Bucket | Temperature |
+|---|---:|
+| Choice, 11+ options | 1.2012 |
+| Choice, 2 options | 1.9488 |
+| Choice, 3–5 options | 1.7575 |
+| Noul (yes / no) | 1.8258 |
+| Choice, any other count | 1.5922 |
+
+The owner adopted v0.3 as a trade-off: counting, relations and ScienceQA improved, while four old-capability protection conditions were not met. The original automated verdict remains `candidate=false`. See the [adoption decision and v0.2 comparison](evaluation/adoption.md), [original acceptance report](evaluation/accept_vs_v0.2.json) and [training configuration](evaluation/training.yaml).
 
 ## Limitations
 
-- **Comparing two places named in the question** (two magnet poles, two chart series, two map regions) stays near chance; on the ScienceQA questions that need the picture it is about 24 points behind Laya Vision.
-- **Relative-position yes/no questions** are weak: right on both a picture and its mirror for only 19% of pairs (65% for relative-position choice questions).
-- **Presence leans towards "yes"**: it says an absent object is there more often than the previous release (POPE adversarial 0.791).
-- English only, one image, `noul` and `choice` only; 256 × 256 input, so no small text.
-- Calibration was fitted on this project's data and may not hold on yours.
+- English, one image, yes/no and choice questions only. `score` is not implemented. At 256×256, small text and fine diagram details are hard to read.
+- CountBench improved by 16.7 points over v0.2, but absolute accuracy is only 35.6%. Answer-order sensitivity increased (39.5% change their answer when options are reversed, versus 16.9%); this is not a general-purpose accurate counter.
+- VSR accuracy is 1.8 points lower and A-OKVQA 1.3 points lower than v0.2; their overall intervals contain zero, but non-inferiority was not established. VSR positive recall fell by 5.3 points. A-OKVQA and What'sUp have worse probability quality than v0.2. No prewritten NLL warning was triggered; that does not mean all probabilities improved.
+- Diagram, chart and map comparisons and question-to-region binding remain weak. On natural-science ScienceQA questions that require the picture, accuracy is 49.5% versus Laya Vision's 70.0%.
+- Calibration reflects the calibration distribution, not every deployment domain. Validate thresholds on your own examples.
+- Test sets have repeatedly informed experiment and release decisions. This is one training seed, not a fresh independent confirmation. Gains cannot be attributed solely to unfreezing the vision encoder.
 
 ## Licence
 
@@ -237,4 +307,4 @@ Weights and code: Apache-2.0, like the three base models ([Laya](https://hugging
 
 ## Links
 
-[Code and training records](https://github.com/byebyebruce/devision) (this release: tag `model-v0.2`) · [evaluation details](evaluation/results.md) · [provenance](provenance.json) · [Laya Vision](https://github.com/r33drichards/laya-vision)
+[Code and training records](https://github.com/byebyebruce/devision) (this release: tag `model-v0.3`) · [evaluation details](evaluation/results.md) · [provenance](provenance.json) · [Laya Vision](https://github.com/r33drichards/laya-vision)

config.json

@@ -1,5 +1,5 @@
 {
- "model_name": "devision-v0.2",
+ "model_name": "devision-v0.3",
  "image_size": 256,
  "visual_shuffle": 2,
  "max_len": 512,
@@ -7,13 +7,15 @@
  "head_layers": 2,
  "n_act": 2,
  "temperature": [
-  1.9567745923995972,
+  1.5922062397003174,
   1.0,
-  1.6382114887237549
+  1.825764536857605
  ],
  "temperature_by_options": {
-  "choice:2": 1.8836495876312256,
-  "choice:3-5": 1.979034662246704,
-  "noul:2": 1.6382114887237549
- }
+  "choice:11+": 1.201171636581421,
+  "choice:2": 1.9487814903259277,
+  "choice:3-5": 1.7575252056121826,
+  "noul:2": 1.825764536857605
+ },
+ "question_conditioning": false
 }

evaluation/accept_vs_v0.2.json

@@ -0,0 +1,660 @@
+{
+ "sets": {
+  "test_count_fresh": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 600,
+    "pictures": 600,
+    "a": 0.74,
+    "b": 0.7633333333333333,
+    "difference": 0.023333333333333317,
+    "ci95": [
+     -0.0050000000000000044,
+     0.05166666666666664
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 600,
+    "pictures": 600,
+    "a": 0.5763476151482702,
+    "b": 0.5738475317419256,
+    "difference": -0.0025000834063445554,
+    "ci95": [
+     -0.029590039246708333,
+     0.02324226594506207
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "bench_countbench": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 491,
+    "pictures": 491,
+    "a": 0.1894093686354379,
+    "b": 0.3564154786150713,
+    "difference": 0.1670061099796334,
+    "ci95": [
+     0.12830957230142565,
+     0.20977596741344195
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 491,
+    "pictures": 491,
+    "a": 2.249204785673313,
+    "b": 1.6021778431421219,
+    "difference": -0.647026942531191,
+    "ci95": [
+     -0.7236271559411285,
+     -0.5697184849200367
+    ],
+    "resamples": 2000
+   },
+   "nll_warning": false
+  },
+  "test_vsr": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 904,
+    "pictures": 552,
+    "a": 0.6792035398230089,
+    "b": 0.661504424778761,
+    "difference": -0.017699115044247815,
+    "ci95": [
+     -0.03871681415929196,
+     0.003293084522502765
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 904,
+    "pictures": 552,
+    "a": 0.6155330847335275,
+    "b": 0.6197663897233163,
+    "difference": 0.0042333049897888,
+    "ci95": [
+     -0.008103858831641997,
+     0.016373944210066993
+    ],
+    "resamples": 2000
+   },
+   "held": false,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_v7w": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1000,
+    "pictures": 1000,
+    "a": 0.721,
+    "b": 0.721,
+    "difference": 0.0,
+    "ci95": [
+     -0.017000000000000015,
+     0.018000000000000016
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1000,
+    "pictures": 1000,
+    "a": 0.6838468317414087,
+    "b": 0.6725771682747884,
+    "difference": -0.011269663466620283,
+    "ci95": [
+     -0.03041221689888618,
+     0.007412588911638762
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "bench_pope": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 8676,
+    "pictures": 482,
+    "a": 0.851083448593822,
+    "b": 0.8495850622406639,
+    "difference": -0.001498386353158132,
+    "ci95": [
+     -0.008529276164130906,
+     0.005301982480405698
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 8676,
+    "pictures": 482,
+    "a": 0.3702472832864825,
+    "b": 0.36794475743031013,
+    "difference": -0.002302525856172355,
+    "ci95": [
+     -0.01004565387165457,
+     0.005636645091598225
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_vqa_yesno": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1000,
+    "pictures": 984,
+    "a": 0.71,
+    "b": 0.704,
+    "difference": -0.006000000000000005,
+    "ci95": [
+     -0.021956087824351322,
+     0.01106639839034207
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1000,
+    "pictures": 984,
+    "a": 0.5768279689372598,
+    "b": 0.5741047062190436,
+    "difference": -0.002723262718216213,
+    "ci95": [
+     -0.012621285357050205,
+     0.0073198649324632825
+    ],
+    "resamples": 2000
+   },
+   "held": false,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_vqa_choice": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1420,
+    "pictures": 1346,
+    "a": 0.897887323943662,
+    "b": 0.9091549295774648,
+    "difference": 0.011267605633802802,
+    "ci95": [
+     0.0020862308762169546,
+     0.022487702037948032
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1420,
+    "pictures": 1346,
+    "a": 0.2706275980383607,
+    "b": 0.2705435796140065,
+    "difference": -8.401842435418194e-05,
+    "ci95": [
+     -0.011447355800969727,
+     0.011158424151348323
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_gqa": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 992,
+    "pictures": 882,
+    "a": 0.7842741935483871,
+    "b": 0.7903225806451613,
+    "difference": 0.006048387096774133,
+    "ci95": [
+     -0.011033099297893645,
+     0.02300000000000002
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 992,
+    "pictures": 882,
+    "a": 0.4089840430444238,
+    "b": 0.4104783294623267,
+    "difference": 0.0014942864179028903,
+    "ci95": [
+     -0.013564724731577937,
+     0.016149505199867686
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_exist": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1120,
+    "pictures": 1099,
+    "a": 0.9383928571428571,
+    "b": 0.9473214285714285,
+    "difference": 0.008928571428571397,
+    "ci95": [
+     -0.000889679715302516,
+     0.018766756032171594
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1120,
+    "pictures": 1099,
+    "a": 0.16199041568107028,
+    "b": 0.16308653751641133,
+    "difference": 0.0010961218353410518,
+    "ci95": [
+     -0.012942116093576261,
+     0.01502907012673671
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_position": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1274,
+    "pictures": 1231,
+    "a": 0.8720565149136578,
+    "b": 0.8846153846153846,
+    "difference": 0.012558869701726816,
+    "ci95": [
+     -0.0007886435331231345,
+     0.026025236593059997
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1274,
+    "pictures": 1231,
+    "a": 0.3045267949584121,
+    "b": 0.2882323744865599,
+    "difference": -0.0162944204718522,
+    "ci95": [
+     -0.03009423881355039,
+     -0.003500244757008719
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_relation": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1950,
+    "pictures": 1453,
+    "a": 0.7107692307692308,
+    "b": 0.7548717948717949,
+    "difference": 0.04410256410256408,
+    "ci95": [
+     0.028018339276617454,
+     0.059034907597535935
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1950,
+    "pictures": 1453,
+    "a": 0.5401162626494179,
+    "b": 0.4896254970901639,
+    "difference": -0.050490765559254025,
+    "ci95": [
+     -0.061521469864747935,
+     -0.038556349244042165
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_size": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1100,
+    "pictures": 975,
+    "a": 0.86,
+    "b": 0.8718181818181818,
+    "difference": 0.011818181818181839,
+    "ci95": [
+     -0.0009165902841430373,
+     0.025711662075298514
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1100,
+    "pictures": 975,
+    "a": 0.35032188023362587,
+    "b": 0.3361044083603172,
+    "difference": -0.014217471873308685,
+    "ci95": [
+     -0.028957271795469974,
+     0.0003334966656227678
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_position_flip": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 640,
+    "pictures": 636,
+    "a": 0.85,
+    "b": 0.846875,
+    "difference": -0.0031249999999999334,
+    "ci95": [
+     -0.021943573667711602,
+     0.014128728414442682
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 640,
+    "pictures": 636,
+    "a": 0.37288566299069764,
+    "b": 0.35918046542178717,
+    "difference": -0.013705197568910477,
+    "ci95": [
+     -0.029400569310068958,
+     0.002968877802506875
+    ],
+    "resamples": 2000
+   },
+   "held": false,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_relation_flip": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1492,
+    "pictures": 1192,
+    "a": 0.6689008042895442,
+    "b": 0.6977211796246648,
+    "difference": 0.028820375335120607,
+    "ci95": [
+     0.009510869565217406,
+     0.04778156996587024
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1492,
+    "pictures": 1192,
+    "a": 0.585918810016869,
+    "b": 0.5450049355513791,
+    "difference": -0.040913874465489886,
+    "ci95": [
+     -0.052748318876262856,
+     -0.029602205956100636
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "test_bind_gqa": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1484,
+    "pictures": 967,
+    "a": 0.6785714285714286,
+    "b": 0.6765498652291105,
+    "difference": -0.0020215633423180668,
+    "ci95": [
+     -0.01720578114246385,
+     0.013468013468013518
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1484,
+    "pictures": 967,
+    "a": 0.56543531676673,
+    "b": 0.5592760342365289,
+    "difference": -0.006159282530201038,
+    "ci95": [
+     -0.015578151742373092,
+     0.0028278953758936742
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "bench_lv_vqav2_yesno": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 4887,
+    "pictures": 2069,
+    "a": 0.724780028647432,
+    "b": 0.7249846531614488,
+    "difference": 0.0002046245140168157,
+    "ci95": [
+     -0.0075005067909993395,
+     0.008213984835720334
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 4887,
+    "pictures": 2069,
+    "a": 0.551391599768774,
+    "b": 0.5517217482239184,
+    "difference": 0.0003301484551443412,
+    "ci95": [
+     -0.004196828299816491,
+     0.005027468860727358
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "bench_lv_aokvqa": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 1138,
+    "pictures": 1138,
+    "a": 0.6256590509666081,
+    "b": 0.6124780316344464,
+    "difference": -0.013181019332161759,
+    "ci95": [
+     -0.02899824253075567,
+     0.002636203866432374
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1138,
+    "pictures": 1138,
+    "a": 0.9295007201330518,
+    "b": 0.9357389801118688,
+    "difference": 0.006238259978816996,
+    "ci95": [
+     -0.011132842065626924,
+     0.023444144517427712
+    ],
+    "resamples": 2000
+   },
+   "held": false,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "bench_lv_scienceqa": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 2097,
+    "pictures": 2097,
+    "a": 0.7663328564616119,
+    "b": 0.7892226990939437,
+    "difference": 0.022889842632331847,
+    "ci95": [
+     0.010968049594659024,
+     0.03528850739151168
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 2097,
+    "pictures": 2097,
+    "a": 0.505658483494164,
+    "b": 0.4709893216879221,
+    "difference": -0.034669161806241855,
+    "ci95": [
+     -0.04485873743892388,
+     -0.024565908726882524
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "bench_lv_pope": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 9000,
+    "pictures": 500,
+    "a": 0.8502222222222222,
+    "b": 0.8481111111111111,
+    "difference": -0.0021111111111110636,
+    "ci95": [
+     -0.008888888888888946,
+     0.004555555555555535
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 9000,
+    "pictures": 500,
+    "a": 0.37292244276041664,
+    "b": 0.3721321698679672,
+    "difference": -0.0007902728924494284,
+    "ci95": [
+     -0.008944474453509432,
+     0.007254876618201789
+    ],
+    "resamples": 2000
+   },
+   "held": true,
+   "proven_worse": false,
+   "nll_warning": false
+  },
+  "bench_whatsup": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 820,
+    "pictures": 820,
+    "a": 0.2841463414634146,
+    "b": 0.2865853658536585,
+    "difference": 0.0024390243902439046,
+    "ci95": [
+     -0.01341463414634142,
+     0.018292682926829285
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 820,
+    "pictures": 820,
+    "a": 1.4723660060388828,
+    "b": 1.4909160791754408,
+    "difference": 0.018550073136557987,
+    "ci95": [
+     0.005068862787158679,
+     0.031356396984290935
+    ],
+    "resamples": 2000
+   },
+   "nll_warning": false
+  },
+  "dev_mix": {
+   "accuracy": {
+    "metric": "accuracy",
+    "questions": 5178,
+    "pictures": 3087,
+    "a": 0.753765932792584,
+    "b": 0.7553109308613364,
+    "difference": 0.0015449980687524123,
+    "ci95": [
+     -0.0056006179992273974,
+     0.009378663540445475
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 5178,
+    "pictures": 3087,
+    "a": 0.5427544545511686,
+    "b": 0.5446964962906559,
+    "difference": 0.0019420417394873102,
+    "ci95": [
+     -0.0049800214856609015,
+     0.00870632376742897
+    ],
+    "resamples": 2000
+   },
+   "nll_warning": false
+  }
+ },
+ "primary": true,
+ "protection": false,
+ "not_held": [
+  "test_vsr",
+  "test_vqa_yesno",
+  "test_position_flip",
+  "bench_lv_aokvqa"
+ ],
+ "proven_worse": [],
+ "warnings": [],
+ "candidate": false
+}

evaluation/adoption.md

@@ -0,0 +1,27 @@
+# deVision v0.3 adoption decision
+
+Recorded: 2026-10-14 22:49:39 +0800
+
+The project owner explicitly chose r15-vision to replace v12-ext and authorized publication as model v0.3. This is an owner-approved trade-off. The prewritten acceptance rule was not passed: the original `candidate=false` is preserved in [accept_vs_v0.2.json](accept_vs_v0.2.json). The counting condition passed; four old-capability protection conditions did not pass; no prewritten NLL warning was triggered.
+
+| Set | v0.2 | v0.3 | Difference, points [95% CI] |
+|---|---:|---:|---:|
+| bench_countbench | 18.9% | 35.6% | +16.7 [+12.8, +21.0] |
+| test_relation | 71.1% | 75.5% | +4.4 [+2.8, +5.9] |
+| test_relation_flip | 66.9% | 69.8% | +2.9 [+1.0, +4.8] |
+| bench_lv_scienceqa | 76.6% | 78.9% | +2.3 [+1.1, +3.5] |
+| test_vsr | 67.9% | 66.2% | -1.8 [-3.9, +0.3] |
+| bench_lv_aokvqa | 62.6% | 61.2% | -1.3 [-2.9, +0.3] |
+| test_vqa_yesno | 71.0% | 70.4% | -0.6 [-2.2, +1.1] |
+| test_position_flip | 85.0% | 84.7% | -0.3 [-2.2, +1.4] |
+
+Intervals use paired resampling by picture. These evaluation sets have informed multiple experiments; the intervals do not correct for adaptive selection or multiple comparisons. A confidence interval containing zero does not establish equivalence.
+
+Accepted limitations:
+
+- VSR positive recall fell from 72.6% to 67.3% (post-hoc diagnostic); A-OKVQA lost 15 net correct answers, with worse NLL and ECE.
+- CountBench accuracy is still only 35.6%. Reversing answer order changes the selected answer on 39.5% of questions, versus 16.9% for v0.2; reversed-order accuracy also improved, so the gain is not confined to the original ordering.
+- What'sUp NLL increased by 0.0186 [0.0051, 0.0314], below the prewritten 0.02 warning threshold but still a deterioration in probability quality.
+- Diagram reasoning, small text and general spatial binding remain limited. There is no completed frozen-vision control isolating the effect of unfreezing.
+
+The final checkpoint is `runs/r15-vision/stage2`, step 10,235, seed 1. It continues exp19 at step 2,000 with the same optimizer/schedule state; see [seeded_from.json](seeded_from.json) and [training.yaml](training.yaml). Calibration is unchanged from the completed run; publication does not train or refit temperatures.

evaluation/cpu_latency.json

@@ -0,0 +1,433 @@
+{
+  "n": 200,
+  "images": 73,
+  "accuracy_all": 0.81,
+  "accuracy": {
+    "noul": 0.8202247191011236,
+    "choice": 0.8018018018018018
+  },
+  "nll": 0.4400090990936913,
+  "brier": 0.25543111423553433,
+  "ece": 0.06582706868648526,
+  "ece_bins": 15,
+  "reliability": [
+    {
+      "bin": [
+        0.2,
+        0.3
+      ],
+      "n": 1,
+      "confidence": 0.28821372985839844,
+      "accuracy": 1.0
+    },
+    {
+      "bin": [
+        0.3,
+        0.4
+      ],
+      "n": 5,
+      "confidence": 0.3615827262401581,
+      "accuracy": 0.4
+    },
+    {
+      "bin": [
+        0.4,
+        0.5
+      ],
+      "n": 7,
+      "confidence": 0.4423674430165972,
+      "accuracy": 0.42857142857142855
+    },
+    {
+      "bin": [
+        0.5,
+        0.6
+      ],
+      "n": 33,
+      "confidence": 0.5360189441478613,
+      "accuracy": 0.6363636363636364
+    },
+    {
+      "bin": [
+        0.6,
+        0.7
+      ],
+      "n": 26,
+      "confidence": 0.6526116694395359,
+      "accuracy": 0.6923076923076923
+    },
+    {
+      "bin": [
+        0.7,
+        0.8
+      ],
+      "n": 15,
+      "confidence": 0.7433490335941315,
+      "accuracy": 0.8
+    },
+    {
+      "bin": [
+        0.8,
+        0.9
+      ],
+      "n": 38,
+      "confidence": 0.8569445978654059,
+      "accuracy": 0.8421052631578947
+    },
+    {
+      "bin": [
+        0.9,
+        1.0
+      ],
+      "n": 75,
+      "confidence": 0.953959968884786,
+      "accuracy": 0.9733333333333334
+    }
+  ],
+  "thresholds": {
+    "0.6": {
+      "coverage": 0.77,
+      "accuracy": 0.8766233766233766
+    },
+    "0.7": {
+      "coverage": 0.64,
+      "accuracy": 0.9140625
+    },
+    "0.8": {
+      "coverage": 0.565,
+      "accuracy": 0.9292035398230089
+    },
+    "0.9": {
+      "coverage": 0.375,
+      "accuracy": 0.9733333333333334
+    }
+  },
+  "by_source": {
+    "ai2d": {
+      "n": 2,
+      "images": 1,
+      "accuracy": 0.5,
+      "nll": 1.2898404388867466,
+      "ece": 0.3732716739177704
+    },
+    "aokvqa": {
+      "n": 4,
+      "images": 2,
+      "accuracy": 0.75,
+      "nll": 0.611274152606737,
+      "ece": 0.2411895990371704
+    },
+    "coco-exist": {
+      "n": 39,
+      "images": 25,
+      "accuracy": 0.9230769230769231,
+      "nll": 0.23563569869662415,
+      "ece": 0.0540550612868407
+    },
+    "coco-position": {
+      "n": 26,
+      "images": 18,
+      "accuracy": 0.9230769230769231,
+      "nll": 0.20050254680859508,
+      "ece": 0.12922574006594145
+    },
+    "coco-relation": {
+      "n": 10,
+      "images": 5,
+      "accuracy": 0.8,
+      "nll": 0.4535431995357432,
+      "ece": 0.29919506013393404
+    },
+    "coco-size": {
+      "n": 2,
+      "images": 2,
+      "accuracy": 1.0,
+      "nll": 0.3333616534201184,
+      "ece": 0.2508668899536133
+    },
+    "gqa-v2": {
+      "n": 25,
+      "images": 7,
+      "accuracy": 0.68,
+      "nll": 0.5026796189257893,
+      "ece": 0.21928748846054075
+    },
+    "tqa": {
+      "n": 10,
+      "images": 2,
+      "accuracy": 0.5,
+      "nll": 1.2075195747510135,
+      "ece": 0.35332188010215765
+    },
+    "vqav2-choice": {
+      "n": 35,
+      "images": 21,
+      "accuracy": 0.8857142857142857,
+      "nll": 0.3503283097869317,
+      "ece": 0.15751930900982447
+    },
+    "vqav2-yesno": {
+      "n": 38,
+      "images": 24,
+      "accuracy": 0.7368421052631579,
+      "nll": 0.5690740238980908,
+      "ece": 0.11526736793549436
+    },
+    "vsr": {
+      "n": 9,
+      "images": 4,
+      "accuracy": 0.7777777777777778,
+      "nll": 0.5381709545359337,
+      "ece": 0.2373169246647093
+    }
+  },
+  "by_kind": {
+    "ai2d": {
+      "n": 2,
+      "images": 1,
+      "accuracy": 0.5,
+      "nll": 1.2898404388867466,
+      "ece": 0.3732716739177704
+    },
+    "animal": {
+      "n": 2,
+      "images": 2,
+      "accuracy": 1.0,
+      "nll": 0.08400760063551806,
+      "ece": 0.07780307531356812
+    },
+    "aokvqa": {
+      "n": 4,
+      "images": 2,
+      "accuracy": 0.75,
+      "nll": 0.611274152606737,
+      "ece": 0.2411895990371704
+    },
+    "color": {
+      "n": 14,
+      "images": 11,
+      "accuracy": 0.8571428571428571,
+      "nll": 0.3618174527077554,
+      "ece": 0.09759957662650516
+    },
+    "count": {
+      "n": 10,
+      "images": 7,
+      "accuracy": 0.8,
+      "nll": 0.6097179990869517,
+      "ece": 0.3152927696704865
+    },
+    "exist": {
+      "n": 39,
+      "images": 25,
+      "accuracy": 0.9230769230769231,
+      "nll": 0.23563569869662415,
+      "ece": 0.0540550612868407
+    },
+    "food": {
+      "n": 1,
+      "images": 1,
+      "accuracy": 1.0,
+      "nll": 0.0042057737329474765,
+      "ece": 0.00419694185256958
+    },
+    "gqa": {
+      "n": 11,
+      "images": 6,
+      "accuracy": 0.7272727272727273,
+      "nll": 0.5031112251493984,
+      "ece": 0.2824250134554776
+    },
+    "material": {
+      "n": 2,
+      "images": 2,
+      "accuracy": 1.0,
+      "nll": 0.22872908175288753,
+      "ece": 0.19662755727767944
+    },
+    "meat": {
+      "n": 1,
+      "images": 1,
+      "accuracy": 1.0,
+      "nll": 0.1798803908020786,
+      "ece": 0.16462987661361694
+    },
+    "position": {
+      "n": 26,
+      "images": 18,
+      "accuracy": 0.9230769230769231,
+      "nll": 0.20050254680859508,
+      "ece": 0.12922574006594145
+    },
+    "relation": {
+      "n": 10,
+      "images": 5,
+      "accuracy": 0.8,
+      "nll": 0.4535431995357432,
+      "ece": 0.29919506013393404
+    },
+    "room": {
+      "n": 1,
+      "images": 1,
+      "accuracy": 1.0,
+      "nll": 0.005263986423792628,
+      "ece": 0.005250155925750732
+    },
+    "size": {
+      "n": 2,
+      "images": 2,
+      "accuracy": 1.0,
+      "nll": 0.3333616534201184,
+      "ece": 0.2508668899536133
+    },
+    "spatial": {
+      "n": 14,
+      "images": 5,
+      "accuracy": 0.6428571428571429,
+      "nll": 0.5023404997500963,
+      "ece": 0.1696794331073761
+    },
+    "tqa": {
+      "n": 10,
+      "images": 2,
+      "accuracy": 0.5,
+      "nll": 1.2075195747510135,
+      "ece": 0.35332188010215765
+    },
+    "vegetable": {
+      "n": 4,
+      "images": 4,
+      "accuracy": 1.0,
+      "nll": 0.07101074950722219,
+      "ece": 0.06772895157337189
+    },
+    "vsr": {
+      "n": 9,
+      "images": 4,
+      "accuracy": 0.7777777777777778,
+      "nll": 0.5381709545359337,
+      "ece": 0.2373169246647093
+    },
+    "yesno": {
+      "n": 38,
+      "images": 24,
+      "accuracy": 0.7368421052631579,
+      "nll": 0.5690740238980908,
+      "ece": 0.11526736793549436
+    }
+  },
+  "by_axis": {
+    "-": {
+      "n": 162,
+      "images": 67,
+      "accuracy": 0.7901234567901234,
+      "nll": 0.47554891340501837,
+      "ece": 0.05595421993438109
+    },
+    "lr": {
+      "n": 27,
+      "images": 18,
+      "accuracy": 0.8888888888888888,
+      "nll": 0.3277449296076739,
+      "ece": 0.19794687407988087
+    },
+    "tb": {
+      "n": 11,
+      "images": 11,
+      "accuracy": 0.9090909090909091,
+      "nll": 0.1921620679743721,
+      "ece": 0.09886410019614479
+    }
+  },
+  "by_options": {
+    "2": {
+      "n": 157,
+      "images": 64,
+      "accuracy": 0.8280254777070064,
+      "nll": 0.38058168910086837,
+      "ece": 0.06867314277181197
+    },
+    "3": {
+      "n": 10,
+      "images": 9,
+      "accuracy": 0.9,
+      "nll": 0.4393064845098579,
+      "ece": 0.22365574836730956
+    },
+    "4": {
+      "n": 31,
+      "images": 18,
+      "accuracy": 0.6774193548387096,
+      "nll": 0.7616504939115978,
+      "ece": 0.10811415699220474
+    },
+    "5": {
+      "n": 2,
+      "images": 2,
+      "accuracy": 1.0,
+      "nll": 0.12313223677190567,
+      "ece": 0.1151973307132721
+    }
+  },
+  "noul_confusion": {
+    "tp": 42,
+    "fp": 10,
+    "fn": 6,
+    "tn": 31,
+    "yes_ratio": 0.5842696629213483
+  },
+  "request_latency_ms": {
+    "p50": 174.91745902225375,
+    "p95": 189.9587146937847,
+    "requests": 200,
+    "warmup_excluded": 3
+  },
+  "questions_per_request": {
+    "mean": 1.0,
+    "p50": 1.0,
+    "max": 1
+  },
+  "ms_per_question": {
+    "p50": 174.91745902225375,
+    "p95": 189.9587146937847,
+    "overall": 172.57170106749982
+  },
+  "latency_ms": {
+    "p50": 174.91745902225375,
+    "p95": 189.9587146937847,
+    "requests": 200,
+    "warmup_excluded": 3
+  },
+  "control": "none",
+  "model": "devision-0.1",
+  "temperature": {
+    "choice": 1.5922062397003174,
+    "noul": 1.825764536857605,
+    "by_options": {
+      "choice:11+": 1.201171636581421,
+      "choice:2": 1.9487814903259277,
+      "choice:3-5": 1.7575252056121826,
+      "noul:2": 1.825764536857605
+    }
+  },
+  "grouping": {
+    "max_questions_per_request": 1,
+    "by": "none (one question per request)"
+  },
+  "environment": {
+    "threads": 4,
+    "machine": "arm64",
+    "system": "Darwin"
+  },
+  "run": {
+    "checkpoint": "runs/r15-vision/stage2",
+    "data": "data/r12/cpu_latency.jsonl",
+    "data_sha256": "352f0b4676aeb7389b45a93086169096d6b8e3962568543cfa373877c3e84020",
+    "role": "unspecified",
+    "device": "cpu",
+    "temperature": "fitted",
+    "control": "none",
+    "seed": 0,
+    "code": "6feb0b3+dirty"
+  }
+}

evaluation/results.json

@@ -1,153 +1,189 @@
 {
  "temperature": {
-  "choice": 1.9567745923995972,
-  "noul": 1.6382114887237549,
+  "choice": 1.5922062397003174,
+  "noul": 1.825764536857605,
   "by_options": {
-   "choice:2": 1.8836495876312256,
-   "choice:3-5": 1.979034662246704,
-   "noul:2": 1.6382114887237549
+   "choice:11+": 1.201171636581421,
+   "choice:2": 1.9487814903259277,
+   "choice:3-5": 1.7575252056121826,
+   "noul:2": 1.825764536857605
   }
  },
  "test_sets": {
   "test_exist": {
    "questions": 1120,
-   "accuracy": 0.9384,
-   "nll": 0.162,
-   "ece": 0.0204,
-   "accuracy_mismatched": 0.4929
+   "accuracy": 0.9473,
+   "nll": 0.1631,
+   "ece": 0.0341,
+   "accuracy_mismatched": 0.5036
   },
   "test_vqa_choice": {
    "questions": 1420,
-   "accuracy": 0.8979,
-   "nll": 0.2706,
-   "ece": 0.0262,
-   "accuracy_mismatched": 0.3993
+   "accuracy": 0.9092,
+   "nll": 0.2705,
+   "ece": 0.0384,
+   "accuracy_mismatched": 0.3803
   },
   "test_size": {
    "questions": 1100,
-   "accuracy": 0.86,
-   "nll": 0.3503,
-   "ece": 0.036,
-   "accuracy_mismatched": 0.5045
+   "accuracy": 0.8718,
+   "nll": 0.3361,
+   "ece": 0.0295,
+   "accuracy_mismatched": 0.5109
   },
   "bench_pope": {
    "questions": 8676,
-   "accuracy": 0.8511,
-   "nll": 0.3702,
-   "ece": 0.0578,
-   "accuracy_mismatched": 0.527
+   "accuracy": 0.8496,
+   "nll": 0.3679,
+   "ece": 0.0393,
+   "accuracy_mismatched": 0.5269
   },
   "test_gqa": {
    "questions": 992,
-   "accuracy": 0.7843,
-   "nll": 0.409,
-   "ece": 0.0419,
-   "accuracy_mismatched": 0.5323
+   "accuracy": 0.7903,
+   "nll": 0.4105,
+   "ece": 0.0396,
+   "accuracy_mismatched": 0.5504
   },
   "test_position": {
    "questions": 1274,
-   "accuracy": 0.8721,
-   "nll": 0.3045,
-   "ece": 0.0251,
+   "accuracy": 0.8846,
+   "nll": 0.2882,
+   "ece": 0.029,
    "accuracy_mismatched": 0.4929
   },
   "test_vqa_yesno": {
    "questions": 1000,
-   "accuracy": 0.71,
-   "nll": 0.5768,
-   "ece": 0.0223,
-   "accuracy_mismatched": 0.518
+   "accuracy": 0.704,
+   "nll": 0.5741,
+   "ece": 0.0265,
+   "accuracy_mismatched": 0.512
   },
   "test_relation": {
    "questions": 1950,
-   "accuracy": 0.7108,
-   "nll": 0.5401,
-   "ece": 0.0471,
-   "accuracy_mismatched": 0.4841
+   "accuracy": 0.7549,
+   "nll": 0.4896,
+   "ece": 0.0384,
+   "accuracy_mismatched": 0.4959
   },
   "test_vsr": {
    "questions": 904,
-   "accuracy": 0.6792,
-   "nll": 0.6155,
-   "ece": 0.0478,
-   "accuracy_mismatched": 0.4812
+   "accuracy": 0.6615,
+   "nll": 0.6198,
+   "ece": 0.0465,
+   "accuracy_mismatched": 0.4867
   },
   "test_v7w": {
    "questions": 1000,
    "accuracy": 0.721,
-   "nll": 0.6838,
-   "ece": 0.0302,
-   "accuracy_mismatched": 0.422
+   "nll": 0.6726,
+   "ece": 0.0411,
+   "accuracy_mismatched": 0.419
   },
   "test_count_fresh": {
    "questions": 600,
-   "accuracy": 0.74,
-   "nll": 0.5763,
-   "ece": 0.0648,
-   "accuracy_mismatched": 0.5067
+   "accuracy": 0.7633,
+   "nll": 0.5738,
+   "ece": 0.054,
+   "accuracy_mismatched": 0.405
+  },
+  "bench_countbench": {
+   "questions": 491,
+   "accuracy": 0.3564,
+   "nll": 1.6022,
+   "ece": 0.0523,
+   "accuracy_mismatched": 0.1039
+  },
+  "test_position_flip": {
+   "questions": 640,
+   "accuracy": 0.8469,
+   "nll": 0.3592,
+   "ece": 0.0444,
+   "accuracy_mismatched": 0.4766
+  },
+  "test_relation_flip": {
+   "questions": 1492,
+   "accuracy": 0.6977,
+   "nll": 0.545,
+   "ece": 0.03,
+   "accuracy_mismatched": 0.5074
+  },
+  "test_bind_gqa": {
+   "questions": 1484,
+   "accuracy": 0.6765,
+   "nll": 0.5593,
+   "ece": 0.0277,
+   "accuracy_mismatched": 0.498
+  },
+  "bench_whatsup": {
+   "questions": 820,
+   "accuracy": 0.2866,
+   "nll": 1.4909,
+   "ece": 0.2447,
+   "accuracy_mismatched": 0.2451
   }
  },
  "laya_vision": {
   "vqav2_yesno": {
    "questions": 4887,
    "laya_vision": 0.7172,
-   "devision": 0.7248,
-   "difference": 0.0076,
+   "devision": 0.725,
+   "difference": 0.0078,
    "ci95": [
-    -0.0087,
-    0.0233
+    -0.0075,
+    0.0241
    ]
   },
   "aokvqa": {
    "questions": 1138,
    "laya_vision": 0.5984,
-   "devision": 0.6257,
-   "difference": 0.0272,
+   "devision": 0.6125,
+   "difference": 0.0141,
    "ci95": [
-    -0.0062,
-    0.0598
+    -0.0185,
+    0.0466
    ]
   },
   "scienceqa": {
    "questions": 2097,
    "laya_vision": 0.824,
-   "devision": 0.7663,
-   "difference": -0.0577,
+   "devision": 0.7892,
+   "difference": -0.0348,
    "ci95": [
-    -0.0792,
-    -0.0362
+    -0.0558,
+    -0.0143
    ]
   },
   "scienceqa_natural_needs_picture": {
    "questions": 323,
    "laya_vision": 0.6997,
-   "devision": 0.4551,
-   "difference": -0.2446,
+   "devision": 0.4954,
+   "difference": -0.2043,
    "ci95": [
-    -0.3158,
-    -0.1765
+    -0.2724,
+    -0.1331
    ]
   },
   "pope": {
    "random": {
     "laya_vision_published": 0.836,
-    "devision": 0.8913
+    "devision": 0.8797
    },
    "popular": {
     "laya_vision_published": 0.819,
-    "devision": 0.868
+    "devision": 0.8593
    },
    "adversarial": {
     "laya_vision_published": 0.777,
-    "devision": 0.7913
+    "devision": 0.8053
    }
   }
  },
  "cpu_latency": {
-  "p50": 148,
-  "p95": 161,
+  "p50": 175,
+  "p95": 190,
   "threads": 4,
   "machine": "Darwin arm64",
-  "source": "runs/v12-ext/cpu/single.json"
+  "source": "release/v0.3/cpu_latency.json"
  }
 }

evaluation/results.md

@@ -1,37 +1,43 @@
-# deVision v0.2 evaluation
+# deVision v0.3 evaluation
 
 Generated by `scripts/release/hf.py` from the evaluation of this release.
 
-| Set | Questions | Accuracy | Mismatched | ECE |
-|---|---:|---:|---:|---:|
-| COCO object presence | 1,120 | 0.938 | 0.493 | 0.020 |
-| VQAv2 multiple choice | 1,420 | 0.898 | 0.399 | 0.026 |
-| COCO size | 1,100 | 0.860 | 0.504 | 0.036 |
-| POPE, project filtered set | 8,676 | 0.851 | 0.527 | 0.058 |
-| GQA val subset | 992 | 0.784 | 0.532 | 0.042 |
-| COCO position | 1,274 | 0.872 | 0.493 | 0.025 |
-| VQAv2 yes/no subset | 1,000 | 0.710 | 0.518 | 0.022 |
-| COCO relative position | 1,950 | 0.711 | 0.484 | 0.047 |
-| VSR, project held-out split | 904 | 0.679 | 0.481 | 0.048 |
-| Visual7W, project held-out split | 1,000 | 0.721 | 0.422 | 0.030 |
-| Fresh counting test (unseen pictures) | 600 | 0.740 | 0.507 | 0.065 |
+| Set | Questions | Accuracy | Mismatched | NLL | ECE |
+|---|---:|---:|---:|---:|---:|
+| COCO object presence | 1,120 | 0.947 | 0.504 | 0.163 | 0.034 |
+| VQAv2 multiple choice | 1,420 | 0.909 | 0.380 | 0.271 | 0.038 |
+| COCO size | 1,100 | 0.872 | 0.511 | 0.336 | 0.029 |
+| POPE, project filtered set | 8,676 | 0.850 | 0.527 | 0.368 | 0.039 |
+| GQA val subset | 992 | 0.790 | 0.550 | 0.410 | 0.040 |
+| COCO position | 1,274 | 0.885 | 0.493 | 0.288 | 0.029 |
+| VQAv2 yes/no subset | 1,000 | 0.704 | 0.512 | 0.574 | 0.026 |
+| COCO relative position | 1,950 | 0.755 | 0.496 | 0.490 | 0.038 |
+| VSR, project held-out split | 904 | 0.661 | 0.487 | 0.620 | 0.046 |
+| Visual7W, project held-out split | 1,000 | 0.721 | 0.419 | 0.673 | 0.041 |
+| Fresh counting test (unseen pictures) | 600 | 0.763 | 0.405 | 0.574 | 0.054 |
+| CountBench, 0–10 choices | 491 | 0.356 | 0.104 | 1.602 | 0.052 |
+| COCO mirrored position | 640 | 0.847 | 0.477 | 0.359 | 0.044 |
+| COCO mirrored relative position | 1,492 | 0.698 | 0.507 | 0.545 | 0.030 |
+| GQA binding test | 1,484 | 0.676 | 0.498 | 0.559 | 0.028 |
+| What'sUp | 820 | 0.287 | 0.245 | 1.491 | 0.245 |
 
 ## Laya Vision 201M, same questions
 
 | Set | Questions | Laya Vision | deVision | Difference |
 |---|---:|---:|---:|---|
-| VQAv2 yes/no | 4,887 | 0.717 | 0.725 | +0.8 [-0.9, +2.3] |
-| A-OKVQA | 1,138 | 0.598 | 0.626 | +2.7 [-0.6, +6.0] |
-| ScienceQA with images | 2,097 | 0.824 | 0.766 | -5.8 [-7.9, -3.6] |
-| ScienceQA natural science, needs the picture | 323 | 0.700 | 0.455 | -24.5 [-31.6, -17.6] |
+| VQAv2 yes/no | 4,887 | 0.717 | 0.725 | +0.8 [-0.8, +2.4] |
+| A-OKVQA | 1,138 | 0.598 | 0.613 | +1.4 [-1.8, +4.7] |
+| ScienceQA with images | 2,097 | 0.824 | 0.789 | -3.5 [-5.6, -1.4] |
+| ScienceQA natural science, needs the picture | 323 | 0.700 | 0.495 | -20.4 [-27.2, -13.3] |
 
-On the full POPE random / popular / adversarial sets (3,000 questions each) deVision scores **0.891 / 0.868 / 0.791**, against Laya Vision's published **0.836 / 0.819 / 0.777** (aggregate scores only, not paired).
+On the full POPE random / popular / adversarial sets (3,000 questions each) deVision scores **0.880 / 0.859 / 0.805**, against Laya Vision's published **0.836 / 0.819 / 0.777** (aggregate scores only, not paired).
 
 ## Temperatures
 
 | Bucket | Temperature |
 |---|---:|
-| Choice, 2 options | 1.8836 |
-| Choice, 3–5 options | 1.9790 |
-| Noul (yes / no) | 1.6382 |
-| Choice, any other count | 1.9568 |
+| Choice, 11+ options | 1.2012 |
+| Choice, 2 options | 1.9488 |
+| Choice, 3–5 options | 1.7575 |
+| Noul (yes / no) | 1.8258 |
+| Choice, any other count | 1.5922 |

evaluation/seeded_from.json

@@ -0,0 +1 @@
+{"seeded_from": "runs/exp19-vision/stage2", "step": 2000, "resume_sha256": "78e5e435fde346c9fe910359abf741afe5fd18464381f3e089006a4e3774a0a5"}

evaluation/training.yaml

@@ -0,0 +1,68 @@
+# Round 15 (r15-vision; owner 2026-10-14: option 1 after exp19, "1"): r14b's mix (r14 + v6 mirror pairs + VSR) with
+# the SigLIP tower's top 4 layers and final norm trained at 5e-6, the whole 10,235-step schedule, seed 1 -- exp19's
+# configuration exactly, continued from its step-2000 training state (scripts/r15/chain.sh seeds the stage directory;
+# the resume check refuses any setting that differs). r14b's frozen stop rules every 2,000 steps; temperatures fitted
+# on data/r13b/val.jsonl; full evaluation = r14b's sets; acceptance = r14's rules against v12, written before training
+# in docs/experiments/2026-10-14-r15-vision.md.
+name: r15-vision
+device: mps
+common:
+  log_every: 100
+
+stages:
+  - name: stage2
+    kind: train
+    init: runs/v12-ext/stage2
+    data: data/r14b/train.jsonl
+    val: data/r13b/val.jsonl
+    eval: {exist: data/r13b/panel_exist.jsonl, science: data/r13b/panel_science.jsonl, ext: data/ext/dev_ext.jsonl,
+           bind_gqa: data/r13/gqa/dev_bind_gqa.jsonl, count_narrow: data/v11/dev_count_transfer.jsonl,
+           count_wide: data/r13b/count_wide.jsonl, px_wide: data/exp14/pixmo_count_wide.jsonl,
+           px_narrow: data/exp14/pixmo_count_narrow.jsonl, px_exist: data/exp14/pixmo_exist.jsonl,
+           flip: data/v6/dev_lr_pairs.jsonl, vsr: data/v5/dev_vsr.jsonl}
+    params: {epochs: 1, lr_new: 2.0e-5, lr_head: 1.0e-5, lr_lora: 5.0e-5, warmup: 300, eval_every: 2000,
+             save_every: 500, amp: "off", seed: 1, monitor_details: true,
+             vision_unfreeze_top: 4, lr_vision: 5.0e-6,
+             start_check: runs/exp16-r14b-s1/stage2/monitor/step0, start_tolerance: 1.0e-5}
+
+evaluate:
+  calibration_fit: data/r13b/val.jsonl
+  history: [data/ext/dev_ext.jsonl, data/r13/gqa/dev_bind_gqa.jsonl, data/r13b/panel_exist.jsonl, data/r13b/panel_science.jsonl,
+            data/r13b/count_wide.jsonl, data/v11/dev_count_transfer.jsonl, data/exp14/pixmo_count_wide.jsonl,
+            data/exp14/pixmo_count_narrow.jsonl, data/exp14/pixmo_exist.jsonl,
+            data/v6/dev_lr_pairs.jsonl, data/v5/dev_vsr.jsonl]
+  image_identity: data/v5/image_identity.json
+  controls: true
+  sets:
+    exist: {path: data/r13b/panel_exist.jsonl, role: monitoring}
+    science: {path: data/r13b/panel_science.jsonl, role: monitoring}
+    ext: {path: data/ext/dev_ext.jsonl, role: monitoring}
+    bind_gqa: {path: data/r13/gqa/dev_bind_gqa.jsonl, role: monitoring}
+    count_narrow: {path: data/v11/dev_count_transfer.jsonl, role: monitoring}
+    count_wide: {path: data/r13b/count_wide.jsonl, role: monitoring}
+    flip: {path: data/v6/dev_lr_pairs.jsonl, role: monitoring}
+    vsr: {path: data/v5/dev_vsr.jsonl, role: monitoring}
+    dev_mix: data/v5/dev_mix.jsonl
+    test_vsr: data/v5/test_vsr.jsonl
+    test_v7w: data/v5/test_v7w.jsonl
+    bench_pope: data/v2/bench_pope.jsonl
+    test_vqa_yesno: data/v2/test_vqa_yesno.jsonl
+    test_vqa_choice: data/v2/test_vqa_choice.jsonl
+    test_gqa: data/v2/test_gqa.jsonl
+    test_exist: data/v2/test_exist.jsonl
+    test_position: data/v2/test_position.jsonl
+    test_relation: data/v2/test_relation.jsonl
+    test_size: data/v2/test_size.jsonl
+    test_position_flip: data/v6/test_position_flip.jsonl
+    test_relation_flip: data/v6/test_relation_flip.jsonl
+    bench_lv_vqav2_yesno: data/lv_bench/vqav2_yesno.jsonl
+    bench_lv_aokvqa: data/lv_bench/aokvqa.jsonl
+    bench_lv_scienceqa: data/lv_bench/scienceqa.jsonl
+    bench_lv_pope: data/lv_bench/pope.jsonl
+    test_count_fresh: {path: data/v11/test_count_fresh.jsonl, role: heldout}
+    test_bind_gqa: {path: data/r13/gqa/test_bind_gqa.jsonl, role: heldout}
+    bench_countbench: {path: data/r13/bench/countbench.jsonl, role: heldout}
+    bench_whatsup: {path: data/r13/bench/whatsup.jsonl, role: heldout}
+  commands:
+    - "uv run python scripts/r13b/judge.py --rules scripts/r14b/rules.json final {eval_dir} runs/r13b-v12-base/stage2-recal || true"
+    - "uv run python scripts/lv_report.py {eval_dir} --prev runs/v12-ext/eval || true"

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07a3e8aacb986396d3975d12d672526c6d3b8ec240b9d9d503ef235c5c87c3ae
+oid sha256:110fea95889947c373258c88a396ad3eec574971815ee437b9c7cc3df8a5d2d5
 size 2073754784

provenance.json

@@ -1,20 +1,20 @@
 {
- "version": "v0.2",
- "git_tag": "model-v0.2",
- "built_from_commit": "bc1dedb9530935a7ed0eaebe61a9789dedb263bf",
- "round": "v12-ext",
+ "version": "v0.3",
+ "git_tag": "model-v0.3",
+ "built_from_commit": "85956234d5e6638618b1d5c8c0907709575df3d5",
+ "round": "r15-vision",
  "stage": "stage2",
- "training_commit": "43c21d692aa03594deb988934f8e88c39ffdb626+dirty",
- "training_started": "2026-10-06 22:54:15",
+ "training_commit": "aea46fc3c4bb753c13cde0a81e8261815399d9e9+dirty",
+ "training_started": "2026-10-14 12:57:49",
  "stages": [
   {
    "name": "stage2",
-   "out": "runs/v12-ext/stage2",
-   "init": "runs/v11-v9b-recovery/stage2"
+   "out": "runs/r15-vision/stage2",
+   "init": "runs/v12-ext/stage2"
   }
  ],
  "training_config": {
-  "name": "v12-ext",
+  "name": "r15-vision",
   "device": "mps",
   "common": {
    "log_every": 100
@@ -23,36 +23,129 @@
    {
     "name": "stage2",
     "kind": "train",
-    "init": "runs/v11-v9b-recovery/stage2",
-    "data": "data/ext/train_mix.jsonl",
-    "val": "data/v5/dev_mix.jsonl",
+    "init": "runs/v12-ext/stage2",
+    "data": "data/r14b/train.jsonl",
+    "val": "data/r13b/val.jsonl",
     "eval": {
-     "ext": "data/ext/dev_ext.jsonl"
+     "exist": "data/r13b/panel_exist.jsonl",
+     "science": "data/r13b/panel_science.jsonl",
+     "ext": "data/ext/dev_ext.jsonl",
+     "bind_gqa": "data/r13/gqa/dev_bind_gqa.jsonl",
+     "count_narrow": "data/v11/dev_count_transfer.jsonl",
+     "count_wide": "data/r13b/count_wide.jsonl",
+     "px_wide": "data/exp14/pixmo_count_wide.jsonl",
+     "px_narrow": "data/exp14/pixmo_count_narrow.jsonl",
+     "px_exist": "data/exp14/pixmo_exist.jsonl",
+     "flip": "data/v6/dev_lr_pairs.jsonl",
+     "vsr": "data/v5/dev_vsr.jsonl"
     },
     "params": {
      "epochs": 1,
-     "lr_new": 5e-05,
-     "lr_head": 5e-05,
-     "lr_lora": 0.0001,
-     "warmup": 500,
-     "eval_every": 5000,
-     "save_every": 1000,
+     "lr_new": 2e-05,
+     "lr_head": 1e-05,
+     "lr_lora": 5e-05,
+     "warmup": 300,
+     "eval_every": 2000,
+     "save_every": 500,
      "amp": "off",
-     "seed": 0
+     "seed": 1,
+     "monitor_details": true,
+     "vision_unfreeze_top": 4,
+     "lr_vision": 5e-06,
+     "start_check": "runs/exp16-r14b-s1/stage2/monitor/step0",
+     "start_tolerance": 1e-05
     }
    }
   ],
   "evaluate": {
+   "calibration_fit": "data/r13b/val.jsonl",
+   "history": [
+    "data/ext/dev_ext.jsonl",
+    "data/r13/gqa/dev_bind_gqa.jsonl",
+    "data/r13b/panel_exist.jsonl",
+    "data/r13b/panel_science.jsonl",
+    "data/r13b/count_wide.jsonl",
+    "data/v11/dev_count_transfer.jsonl",
+    "data/exp14/pixmo_count_wide.jsonl",
+    "data/exp14/pixmo_count_narrow.jsonl",
+    "data/exp14/pixmo_exist.jsonl",
+    "data/v6/dev_lr_pairs.jsonl",
+    "data/v5/dev_vsr.jsonl"
+   ],
    "image_identity": "data/v5/image_identity.json",
+   "controls": true,
    "sets": {
-    "dev_mix": "data/v5/dev_mix.jsonl",
-    "dev_lr_pairs": "data/v6/dev_lr_pairs.jsonl",
-    "dev_ext": {
+    "exist": {
+     "path": "data/r13b/panel_exist.jsonl",
+     "role": "monitoring"
+    },
+    "science": {
+     "path": "data/r13b/panel_science.jsonl",
+     "role": "monitoring"
+    },
+    "ext": {
      "path": "data/ext/dev_ext.jsonl",
      "role": "monitoring"
+    },
+    "bind_gqa": {
+     "path": "data/r13/gqa/dev_bind_gqa.jsonl",
+     "role": "monitoring"
+    },
+    "count_narrow": {
+     "path": "data/v11/dev_count_transfer.jsonl",
+     "role": "monitoring"
+    },
+    "count_wide": {
+     "path": "data/r13b/count_wide.jsonl",
+     "role": "monitoring"
+    },
+    "flip": {
+     "path": "data/v6/dev_lr_pairs.jsonl",
+     "role": "monitoring"
+    },
+    "vsr": {
+     "path": "data/v5/dev_vsr.jsonl",
+     "role": "monitoring"
+    },
+    "dev_mix": "data/v5/dev_mix.jsonl",
+    "test_vsr": "data/v5/test_vsr.jsonl",
+    "test_v7w": "data/v5/test_v7w.jsonl",
+    "bench_pope": "data/v2/bench_pope.jsonl",
+    "test_vqa_yesno": "data/v2/test_vqa_yesno.jsonl",
+    "test_vqa_choice": "data/v2/test_vqa_choice.jsonl",
+    "test_gqa": "data/v2/test_gqa.jsonl",
+    "test_exist": "data/v2/test_exist.jsonl",
+    "test_position": "data/v2/test_position.jsonl",
+    "test_relation": "data/v2/test_relation.jsonl",
+    "test_size": "data/v2/test_size.jsonl",
+    "test_position_flip": "data/v6/test_position_flip.jsonl",
+    "test_relation_flip": "data/v6/test_relation_flip.jsonl",
+    "bench_lv_vqav2_yesno": "data/lv_bench/vqav2_yesno.jsonl",
+    "bench_lv_aokvqa": "data/lv_bench/aokvqa.jsonl",
+    "bench_lv_scienceqa": "data/lv_bench/scienceqa.jsonl",
+    "bench_lv_pope": "data/lv_bench/pope.jsonl",
+    "test_count_fresh": {
+     "path": "data/v11/test_count_fresh.jsonl",
+     "role": "heldout"
+    },
+    "test_bind_gqa": {
+     "path": "data/r13/gqa/test_bind_gqa.jsonl",
+     "role": "heldout"
+    },
+    "bench_countbench": {
+     "path": "data/r13/bench/countbench.jsonl",
+     "role": "heldout"
+    },
+    "bench_whatsup": {
+     "path": "data/r13/bench/whatsup.jsonl",
+     "role": "heldout"
     }
-   }
+   },
+   "commands": [
+    "uv run python scripts/r13b/judge.py --rules scripts/r14b/rules.json final {eval_dir} runs/r13b-v12-base/stage2-recal || true",
+    "uv run python scripts/lv_report.py {eval_dir} --prev runs/v12-ext/eval || true"
+   ]
   }
  },
- "source_checkpoint_sha256": "07a3e8aacb986396d3975d12d672526c6d3b8ec240b9d9d503ef235c5c87c3ae"
+ "source_checkpoint_sha256": "110fea95889947c373258c88a396ad3eec574971815ee437b9c7cc3df8a5d2d5"
 }